From 1a784ade85f826935901e250de57ef7a71610b1e Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 20 Jul 2020 16:31:48 +0200 Subject: [PATCH] added GET /sample/number/{number} route --- api/parameters.yaml | 8 + api/sample.yaml | 29 +- data_import/import.js | 495 ++++++++++++++++++++++++---------- src/helpers/csv.ts | 15 +- src/routes/sample.spec.ts | 61 +++++ src/routes/sample.ts | 60 +++-- src/routes/validate/sample.ts | 4 +- 7 files changed, 508 insertions(+), 164 deletions(-) diff --git a/api/parameters.yaml b/api/parameters.yaml index 066fc3a..67ac778 100644 --- a/api/parameters.yaml +++ b/api/parameters.yaml @@ -6,6 +6,14 @@ Id: type: string example: 5ea0450ed851c30a90e70894 +Number: + name: number + in: path + required: true + schema: + type: string + example: Rng740 + Name: name: name description: has to be URL encoded diff --git a/api/sample.yaml b/api/sample.yaml index 2b0ce31..82d6c7c 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -140,10 +140,10 @@ application/json: schema: $ref: 'api.yaml#/components/schemas/SampleDetail' - 400: - $ref: 'api.yaml#/components/responses/400' 401: $ref: 'api.yaml#/components/responses/401' + 403: + $ref: 'api.yaml#/components/responses/403' 404: $ref: 'api.yaml#/components/responses/404' 500: @@ -201,6 +201,31 @@ 500: $ref: 'api.yaml#/components/responses/500' +/sample/number/{number}: + parameters: + - $ref: 'api.yaml#/components/parameters/Number' + get: + summary: sample details + description: 'Auth: all, levels: read, write, maintain, dev, admin
Returns validated as well as new measurements' + x-doc: deleted samples are available only for maintain/admin + tags: + - /sample + responses: + 200: + description: samples details + content: + application/json: + schema: + $ref: 'api.yaml#/components/schemas/SampleDetail' + 401: + $ref: 'api.yaml#/components/responses/401' + 403: + $ref: 'api.yaml#/components/responses/403' + 404: + $ref: 'api.yaml#/components/responses/404' + 500: + $ref: 'api.yaml#/components/responses/500' + /sample/restore/{id}: parameters: - $ref: 'api.yaml#/components/parameters/Id' diff --git a/data_import/import.js b/data_import/import.js index 06c31be..8c4f924 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -7,75 +7,148 @@ const pdfReader = require('pdfreader'); const iconv = require('iconv-lite'); const _ = require('lodash'); -const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv'; // metadata files -const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv'; -const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv'; -const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs'; // NormMaster Documents -const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT'; // Spectrum files +const stages = { + materials: false, + samples: true, + measurements: false, + dpt: false +} + +const docs = [ + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2_A.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2_B.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Ap.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Bj.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh_B.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh_Duroplasten.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell_A.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell_B.csv", + "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_WaP.csv", +]; +const errors = []; +const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\nmDocs'; // NormMaster Documents +const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\DPT'; // Spectrum files const host = 'http://localhost:3000'; // const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; +const requiredProperties = ['samplenumber','materialnumber','materialname','supplier','reinforcementmaterial','material','granulate/part','color','charge/batch','comments']; +dict = { // dictionary + 'Granulat': 'granulate', + 'Zugstab': 'tension rod', + 'Stecker': 'plug' +}; let data = []; // metadata contents let materials = {}; -const numberToColor = {}; +let numberToColor = {}; let samples = []; let normMaster = {}; let sampleDevices = {}; +const sampleReferences = []; // references to other samples in format {sample, referencedSample, relation} +let comments = []; -// TODO: BASF twice, BASF as color -// TODO: duplicate kf values // TODO: conditions // TODO: comment and reference handling +// TODO: measurement device to spectrum + // TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC +// TODO: samples, conditions + main(); async function main() { - if (1) { // materials + if (stages.materials) { // materials await getNormMaster(); - await importCsv(metaDoc); - await allMaterials(); - await saveMaterials(); - await importCsv(kfDoc); - await allMaterials(); - await saveMaterials(); - await importCsv(vzDoc); - await allMaterials(); - await saveMaterials(); + for (let i in docs) { + await importCsv(docs[i]); + await allMaterials(); + await saveMaterials(); + } + fs.writeFileSync('./data_import/numberToColor.json', JSON.stringify(numberToColor)); } - if (1) { // samples + if (stages.samples) { // samples sampleDeviceMap(); - if (1) { - console.log('-------- META ----------'); - await importCsv(metaDoc); + numberToColor = JSON.parse(fs.readFileSync('./data_import/numberToColor.json'), 'utf-8'); + for (let i in docs) { + await importCsv(docs[i]); await allSamples(); await saveSamples(); - } - if (1) { - console.log('-------- KF ----------'); - await importCsv(kfDoc); - await allSamples(); - await saveSamples(); - await allKfVz(); - } - if (1) { - console.log('-------- VZ ----------'); - await importCsv(vzDoc); - await allSamples(); - await saveSamples(); - await allKfVz(); - } + console.log(samples); + } // TODO: get sample by number, implement smple references + fs.writeFileSync('./data_import/comments.txt', comments.join('\r\n')); + // if (1) { // TODO: KfVz + // console.log('-------- META ----------'); + // await importCsv(metaDoc); + // await allSamples(); + // await saveSamples(); + // } + // if (1) { + // console.log('-------- KF ----------'); + // await importCsv(kfDoc); + // await allSamples(); + // await saveSamples(); + // await allKfVz(); + // } + // if (1) { + // console.log('-------- VZ ----------'); + // await importCsv(vzDoc); + // await allSamples(); + // await saveSamples(); + // await allKfVz(); + // } } - if (1) { // DPT + if (stages.dpt) { // DPT await allDpts(); } if (0) { // pdf test - console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf')); + console.log(await readPdf('N28_BN05-OX023_2019-07-16.pdf')); + } + if (errors.length) { + // console.log(errors); + fs.writeFileSync('./data_import/errors/errors_' + new Date().getTime() + '.txt', errors.join('\r\n')); } } async function importCsv(doc) { + // Uniform name samplenumber materialnumber materialname supplier material plastic reinforcingmaterial granulate/part color charge/batch comments vz(ml/g) kfingew% degradation(%) glassfibrecontent(%) stabwn + // Metadata__AnP2.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material, granulate/Part,Color,Charge/ Batch, Comments + // Metadata__AnP2_A.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material, Granulate/Part, Comments, Humidity [ppm] + // Metadata__AnP2_B.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material, Granulate/Part, VZ [ml/g], glass fibre content + // Metadata_Ap.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material, Granulate/Part,Color,Charge/Batch, Comments + // Metadata_Bj.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material, Granulate/Part,Color,Charge/batch granulate/part,Comments + // Metadata_Eh.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material, Granulate/Part,Color,Charge/Batch granulate/part,Comments, VZ [cm³/g], Spalte1 + // Metadata_Eh_B.csv Sample number, Material name,Supplier, Plastic,Reinforcing material, Granulate/Part,Color, Comments, VZ [cm³/g] + // Metadata_Eh_Duroplasten.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material, Granulate/Part,Color,Charge/Batch granulate/part,Comments + // Metadata_Rng_aktuell.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material, Granulate/Part,Color,Charge/batch granulate/part,Comments, VZ (ml/g), Degradation(%),Glas fibre content (%) + // Metadata_Rng_aktuell_A.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material, Granulate/Part,Farbe,Charge/batch granulate/part,Comments, KF in Gew%, Stabwn + // Metadata_Rng_aktuell_B.csv Sample number, Material name,Supplier, Plastic,Reinforcing material (content in %),Granulate/Part, Comments, VZ (ml/g), Degradation (%), Alterungszeit in h + // Metadata_WaP.csv Probennummer, Name, Firma, Material, Teil/Rohstoff, Charge, Anmerkung,VZ (ml/g), Abbau (%), Verstärkungsstoffgehalt (%), Versuchsnummer + const nameCorrection = { // map to right column names + 'probennummer': 'samplenumber', + 'name': 'materialname', + 'firma': 'supplier', + 'reinforcingmaterial(contentin%)': 'reinforcingmaterial', + 'teil/rohstoff': 'granulate/part', + 'charge/batchgranulate/part': 'charge/batch', + 'charge': 'charge/batch', + 'anmerkung': 'comments', + 'vz[ml/g]': 'vz(ml/g)', + 'vz[cm³/g]': 'vz(ml/g)', + 'abbau(%)': 'degradation(%)', + 'verstärkungsstoffgehalt(%)': 'glassfibrecontent(%)' + }; + const missingFieldsFill = [ // column names to fill if they do not exist + 'color', + 'charge/batch', + 'comments', + 'materialnumber', + 'reinforcementmaterial' + ] + console.log('importing ' + doc); data = []; await new Promise(resolve => { fs.createReadStream(doc) @@ -85,10 +158,47 @@ async function importCsv(doc) { data.push(row); }) .on('end', () => { + data = data.map(e => { + const newE = {}; + Object.keys(e).forEach(key => { + newE[key.toLowerCase().replace(/ /g, '')] = e[key]; + }); + // replace wrong column names + Object.keys(newE).forEach(key => { + if (nameCorrection.hasOwnProperty(key)) { + newE[nameCorrection[key]] = newE[key]; + delete newE[key]; + } + }); + + // add missing fields with empty values + missingFieldsFill.forEach(field => { + if (!newE.hasOwnProperty(field)) { + newE[field] = ''; + } + }); + if (newE['supplier'] === '') { // empty supplier fields + newE['supplier'] = 'unknown'; + } + if (!newE.hasOwnProperty('material')) { + newE['material'] = newE['plastic'].indexOf(' GF') >= 0 ? newE['plastic'].split(' ')[0] : newE['plastic']; + } + return newE; + }).filter(e => { + const missingProperties = requiredProperties.filter(el => !e.hasOwnProperty(el)); + if (e['materialname'] === '') { + missingProperties.push('materialname'); + } + if (e['samplenumber'] === '') { // empty row + return false; + } + else if (missingProperties.length > 0) { // incomplete sample + errors.push(`${doc}: ${JSON.stringify(e)}is missing the required properties ${missingProperties}`); + return false; + } + return true; + }); console.info('CSV file successfully processed'); - if (data[0]['Farbe']) { // fix German column names - data.map(e => {e['Color'] = e['Farbe']; return e; }); - } resolve(); }); }); @@ -158,6 +268,7 @@ async function allDpts() { } } +// TODO: VZ from comments async function allKfVz() { let res = await axios({ method: 'get', @@ -266,53 +377,56 @@ async function allSamples() { for (let index in data) { console.info(`${index}/${data.length}`); let sample = data[index]; - if (sample['Sample number'] !== '') { // TODO: what about samples without color - if (sample['Supplier'] === '') { // empty supplier fields - sample['Supplier'] = 'unknown'; + if (sample['granulate/Part'] === '') { // empty supplier fields + sample['granulate/Part'] = 'unknown'; + } + const material = dbMaterials[trim(sample['materialname'])]; + if (!material) { // could not find material, skipping sample + errors.push(`Could not find a material for ${JSON.stringify(sample)}`); + continue; + } + samples.push({ + number: sample['samplenumber'], + type: sampleType(sample['granulate/part']), + batch: sample['charge/batch'], + material_id: material._id, + notes: { + custom_fields: customFields(sample['comments'], sample['samplenumber']) } - if (sample['Granulate/Part'] === '') { // empty supplier fields - sample['Granulate/Part'] = 'unknown'; - } - const material = dbMaterials[trim(sample['Material name'])]; - if (!material) { // could not find material, skipping sample - continue; - } - samples.push({ - number: sample['Sample number'], - type: sample['Granulate/Part'], - batch: sample['Charge/batch granulate/part'] || '', - material_id: material._id, - notes: { - comment: sample['Comments'] - } - }); - const si = samples.length - 1; - if (sample['Material number'] !== '' && material.numbers.find(e => e.number === sample['Material number'])) { // TODO: fix because of false material/material number - samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color; - } - else if (sample['Color'] && sample['Color'] !== '') { + }); + // if (sample['comments']) { + // comments.push(sample['samplenumber'] + ' ' + sample['comments']); + // } + const si = samples.length - 1; // sample index + if (samples[si].notes.custom_fields.hasOwnProperty('xRest')) { // reroute xRest property to comment + samples[si].notes.comment = samples[si].notes.custom_fields.xRest; + delete samples[si].notes.custom_fields.xRest; + } + if (Object.keys(samples[si].notes.custom_fields). length === 0) { + delete samples[si].notes.custom_fields; + } + if (sample['materialnumber'] !== '' && material.numbers.find(e => e.number === sample['materialnumber'])) { + samples[si].color = material.numbers.find(e => e.number === sample['materialnumber']).color; + } + else if (sample['color'] !== '') { // find color with all edge cases + let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['color'])) >= 0); + if (!number && /black/.test(sample['color'])) { // special case bk for black console.log(material); - let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['Color'])) >= 0); - if (!number && /black/.test(sample['Color'])) { // special case bk for black - number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0); - if (!number) { // try German word - number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0); - } - } - if (number) { - samples[si].color = number.color; - } - else { - samples[si].color = ''; + number = material.numbers.find(e => e.color && e.color.toLowerCase().indexOf('bk') >= 0); + if (!number) { // try German word + number = material.numbers.find(e => e.color && e.color.toLowerCase().indexOf('schwarz') >= 0); } } - else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz - samples[si].color = sampleColors[sample['Sample number'].split('_')[0]]; - } - else { - samples[si].color = ''; + if (number) { + samples[si].color = number.color; } } + else if (sampleColors[sample['samplenumber'].split('_')[0]]) { // derive color from main sample for kf/vz + samples[si].color = sampleColors[sample['samplenumber'].split('_')[0]]; + } + if (!samples[si].color) { + samples[si].color = sample['color']; + } } } @@ -321,6 +435,7 @@ async function saveSamples() { console.info(`${i}/${samples.length}`); let credentials = ['admin', 'Abc123!#']; if (sampleDevices[samples[i].number]) { + console.log(sampleDevices[samples[i].number]); credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!'] } await axios({ @@ -335,6 +450,7 @@ async function saveSamples() { if (err.response.data.status && err.response.data.status !== 'Sample number already taken') { console.log(samples[i]); console.error(err.response.data); + errors.push(`Upload for ${JSON.stringify(samples[i])} failed: ${JSON.stringify(err.response.data)}`) } }); } @@ -342,56 +458,65 @@ async function saveSamples() { } async function allMaterials() { - materials = {}; + // materials = {}; + let res = await axios({ + method: 'get', + url: host + '/template/materials', + auth: { + username: 'admin', + password: 'Abc123!#' + } + }); + const materialTemplate = res.data.find(e => e.name === 'plastic')._id; + + // process all samples for (let index in data) { let sample = data[index]; - if (sample['Sample number'] && sample['Sample number'] !== '') { - if (sample['Supplier'] === '') { // empty supplier fields - sample['Supplier'] = 'unknown'; - } - if (sample['Material name'] === '') { // empty name fields - sample['Material name'] = sample['Material']; - } - if (!sample['Material']) { // column Material is named Plastic in VZ metadata - sample['Material'] = sample['Plastic']; - } - sample['Material name'] = trim(sample['Material name']); - if (materials.hasOwnProperty(sample['Material name'])) { // material already found at least once - if (sample['Material number'] && sample['Material number'] !== '') { - if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) { // new material number - if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) { // color already in list, only number missing - materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']); - } - else { - materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])}); - } + if (sample['supplier'] === '') { // empty supplier fields + sample['supplier'] = 'unknown'; + } + if (sample['materialname'] === '') { // empty name fields + sample['materialname'] = sample['material']; + } + sample['materialname'] = trim(sample['materialname']); + if (materials.hasOwnProperty(sample['materialname'])) { // material already found at least once + if (sample['materialnumber'] !== '') { // material number given + if (materials[sample['materialname']].numbers.length === 0 || !materials[sample['materialname']].numbers.find(e => e.number === stripSpaces(sample['materialnumber']))) { // new material number + if (materials[sample['materialname']].numbers.find(e => e.color === sample['color'] && e.number === '')) { // color already in list, only number missing + materials[sample['materialname']].numbers.find(e => e.color === sample['color'] && e.number === '').number = stripSpaces(sample['materialnumber']); } - } - else if (sample['Color'] && sample['Color'] !== '') { - if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) { // new material color - materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''}); + else { // completely new number entry + materials[sample['materialname']].numbers.push({color: trim(sample['color']), number: stripSpaces(sample['materialnumber'])}); } } } - else { // new material - console.info(`${index}/${data.length} ${sample['Material name']}`); - materials[sample['Material name']] = { - name: sample['Material name'], - supplier: trim(sample['Supplier']), - group: trim(sample['Material']) - }; - materials[sample['Material name']].properties = {material_template: '5f0efe6fce7fd20ce4e99013'}; - let tmp = /M(\d+)/.exec(sample['Reinforcing material']); - materials[sample['Material name']].properties.mineral = tmp ? tmp[1] : 0; - tmp = /GF(\d+)/.exec(sample['Reinforcing material']); - materials[sample['Material name']].properties.glass_fiber = tmp ? tmp[1] : 0; - tmp = /CF(\d+)/.exec(sample['Reinforcing material']); - materials[sample['Material name']].properties.carbon_fiber = tmp ? tmp[1] : 0; - materials[sample['Material name']].numbers = await numbersFetch(sample); - console.log(materials[sample['Material name']]); + else if (sample['color'] !== '') { // color given + if (!materials[sample['materialname']].numbers.find(e => e.color === stripSpaces(sample['color']))) { // new material color + materials[sample['materialname']].numbers.push({color: trim(sample['color']), number: ''}); + } } } + else { // new material + console.info(`${index}/${data.length} ${sample['materialname']}`); + materials[sample['materialname']] = { + name: trim(sample['materialname']), + supplier: trim(sample['supplier']), + group: trim(sample['material']) + }; + materials[sample['materialname']].numbers = await numbersFetch(sample); + + // material properties + materials[sample['materialname']].properties = {material_template: materialTemplate}; + let tmp = /M(\d+)/.exec(sample['reinforcingmaterial']); + materials[sample['materialname']].properties.mineral = tmp ? tmp[1] : 0; + tmp = /GF(\d+)/.exec(sample['reinforcingmaterial']); + materials[sample['materialname']].properties.glass_fiber = tmp ? tmp[1] : 0; + tmp = /CF(\d+)/.exec(sample['reinforcingmaterial']); + materials[sample['materialname']].properties.carbon_fiber = tmp ? tmp[1] : 0; + } } + + // Fill numberToColor array Object.keys(materials).forEach(mKey => { materials[mKey].numbers.forEach(number => { if (number.number && number.color) { @@ -419,6 +544,7 @@ async function saveMaterials() { if (err.response.data.status && err.response.data.status !== 'Material name already taken') { console.info(material); console.error(err.response.data); + errors.push(`Upload for ${JSON.stringify(material)} failed: ${JSON.stringify(err.response.data)}`) } }); } @@ -428,17 +554,17 @@ async function saveMaterials() { async function numbersFetch(sample) { let nm = []; let res = []; - if (sample['Material number']) { // sample has a material number - nm = normMaster[stripSpaces(sample['Material number'])]? [normMaster[stripSpaces(sample['Material number'])]] : []; + if (sample['materialnumber']) { // sample has a material number + nm = normMaster[stripSpaces(sample['materialnumber'])]? [normMaster[stripSpaces(sample['materialnumber'])]] : []; } else { // try finding via material name - nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['Material name'])).map(e => normMaster[e]); + nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['materialnumber'])).map(e => normMaster[e]); } if (nm.length > 0) { for (let i in nm) { - // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded - // await getNormMasterDoc(nm[i].url.replace(/ /g, '%20')); - // } + if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded + await getNormMasterDoc(nm[i].url.replace(/ /g, '%20')); + } // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded // console.info('Retrying download...'); // await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 2.2); @@ -454,21 +580,22 @@ async function numbersFetch(sample) { break; } else if (i + 1 >= nm.length) { - console.error('Download failed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'); + errors.push(`Download of ${nm[i].url.replace(/ /g, '%20')} for material number ${sample['materialnumber']} failed`); + errors.push(nm[i].doc.replace(/ /g, '_')); } } } if (res.length === 0) { // no results - if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) { - return [{color: trim(sample['Color']), number: sample['Material number']}]; + if (sample['color'] !== '' || sample['materialnumber'] !== '') { // information in data available + return [{color: trim(sample['color']), number: sample['materialnumber']}]; } else { return []; } } else { - if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed - res.push({color: trim(sample['Color']), number: sample['Material number']}); + if (!res.find(e => e.number === sample['materialnumber'])) { // sometimes norm master does not include sample number even if listed + res.push({color: trim(sample['color']), number: sample['materialnumber']}); } return res; } @@ -538,7 +665,7 @@ function readPdf(file) { let lastLastText = ''; // text of last last item await new pdfReader.PdfReader().parseFileItems(nmDocs + '\\' + file, (err, item) => { if (item && item.text) { - if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsupplier') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsupplier') >= 0)) { // table area starts + if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsuppl') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsupplier') >= 0)) { // table area starts table = countdown; } if (table > 0) { @@ -587,6 +714,102 @@ function sampleDeviceMap() { } } +function customFields (comment, sampleNumber) { + const customFields = [ + {docKey: 'Versuchsreihe', dbKey: 'test series', regex: /Versuchsreihe (\d+),/, reference: false}, + {docKey: 'Stillstand', dbKey: 'idle', regex: /(\d+ min)/, reference: false}, + {docKey: 'Serienzyklus', dbKey: 'cycle', regex: /(\d+.) Serienzyklus (\(.*?\))/, reference: false}, + {docKey: 'Berstdruck', dbKey: 'bursting pressure', regex: /Berstdruck: (.*?bar);/, reference: false}, + {docKey: 'gemessen am', dbKey: 'measured at', regex: /gemessen am (.*20\d\d)/, reference: false}, + {docKey: 'used for', dbKey: 'used for', regex: /used for (.*)/, reference: false}, + {docKey: 'Stabilized', dbKey: 'stabilized', regex: null, reference: false}, + {docKey: 'parts from field', dbKey: 'parts from field', regex: null, reference: false}, + {docKey: 'side', dbKey: 'side', regex: /(\S*?) side/, reference: false}, + {docKey: 'Creep test', dbKey: 'creep test', regex: null, reference: false}, + {docKey: 'Variante', dbKey: 'variant', regex: /(.*)/, reference: false}, + {docKey: 'Parameter', dbKey: 'parameter', regex: /Parameter (\d)/, reference: false}, + {docKey: 'days without cooling', dbKey: 'days without cooling', regex: /(\d+) days without cooling/, reference: false}, + {docKey: 'Zyklus', dbKey: 'cycle', regex: /Zyklus (\d+ s)/, reference: false}, + {docKey: 'fast cure', dbKey: 'fast cure', regex: null, reference: false}, + {docKey: 'Stoff gesperrt', dbKey: 'material blocked', regex: null, reference: false}, + {docKey: 'anwendungsbeschränkt', dbKey: 'limited application', regex: null, reference: false}, + {docKey: 'für Neuanwendungen gesperrt', dbKey: 'blocked for new applications', regex: null, reference: false}, + {docKey: 'V', dbKey: 'test', regex: /V(\d+-\d+);/, reference: false}, + {docKey: 'Twz', dbKey: 'twz', regex: /Twz \(°C\): (\d+);/, reference: false}, + {docKey: 'Pnach', dbKey: 'pressure after', regex: /Pnach \(bar\): (\d+);/, reference: false}, + {docKey: 'Vein', dbKey: 'volume in', regex: /Vein \(ccm\/s\): (\d+)/, reference: false}, + {docKey: 'low emission', dbKey: 'low emission', regex: /low emission (.*?);/, reference: false}, + {docKey: 'aus', dbKey: 'from', regex: /aus (.*)/, reference: false}, + {docKey: 'Erprobung', dbKey: 'trial', regex: /Erprobung (.*?);/, reference: false}, + {docKey: 'Auftragsnummer', dbKey: 'job number', regex: /Auftragsnummer: (.*?);/, reference: false}, + {docKey: 'Wärmealterung', dbKey: 'heat aging', regex: /Wärmealterung: (.*)/, reference: false}, + {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall', regex: /Steg.*?A: (\d+)/, reference: false}, + {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall', regex: /Steg.*?I: (\d+)/, reference: false}, + {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall', regex: /Steg.*?S: (\d+)/, reference: false}, + {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall degraded', regex: /Degradation:.*?A: (\d+)/, reference: false}, + {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall degraded', regex: /Degradation:.*?I: (\d+)/, reference: false}, + {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall degraded', regex: /Degradation:.*?S: (\d+)/, reference: false}, + {docKey: 'Reines Polymer', dbKey: 'pure polymer', regex: null, reference: false}, + {docKey: 'Rücksendung erforderlich', dbKey: 'return needed', regex: /(.*?,) Rücksendung erforderlich, (.*)/, reference: false}, + {docKey: 'Prio', dbKey: 'priority', regex: /Prio (\d+)/, reference: false}, + {docKey: 'beanstandet', dbKey: 'faulty', regex: null, reference: false}, + {docKey: 'aged', dbKey: 'aged', regex: /aged: (.*)/, reference: false}, + {docKey: 'DOPPELT!!', dbKey: 'double', regex: null, reference: false}, + {docKey: 'Bauteil', dbKey: 'construction part', regex: /Bauteil (\S+)/, reference: false}, + {docKey: 'T =', dbKey: 'temperature', regex: /T = (\S+)/, reference: false}, + {docKey: 'nicht vorgealtert', dbKey: 'not preaged', regex: /nicht vorgealtert (.*)/, reference: false}, + {docKey: 'TS119', dbKey: 'TS119', regex: /TS119 (W\S+);/, reference: false}, + {docKey: 'GF vom Datenblatt', dbKey: 'glass fibre from data sheet', regex: null, reference: false}, + {docKey: 'nach Datensatz', dbKey: 'according to dataset', regex: null, reference: false}, + {docKey: 'Dosiergeschw', dbKey: 'metering speed', regex: /Dosiergeschw.*? (.*?min)/, reference: false}, + {docKey: 'Einspritzgeschw', dbKey: 'injection speed', regex: /Einspritzgeschw.*? (.*\/s)/, reference: false}, + {docKey: 'Heizbänder', dbKey: 'heating lines', regex: /Heizbänder (.*)/, reference: false}, + {docKey: 'Verweilzeit', dbKey: 'dwell time', regex: /Verweilzeit (.*?min)/, reference: false}, + {docKey: 'Probe', dbKey: 'belongs to', regex: /Probe (\S*\d+)/, reference: true}, + {docKey: 'zu', dbKey: 'belongs to', regex: /zu (\S*\d+)/, reference: true}, + {docKey: 'granulate zu', dbKey: 'granulate to', regex: /granulate zu.* (\S*\d+)/, reference: true}, + {docKey: 'construction part', dbKey: 'construction part', regex: /(? { + if (comment.indexOf(cField.docKey) >= 0) { // comment contains docKey + if (cField.regex !== null) { + const regexRes = cField.regex.exec(comment); + if (regexRes) { + usedParts.push(regexRes[0]); + if (cField.reference) { + sampleReferences.push({sample: sampleNumber, referencedSample: regexRes[1], relation: cField.dbKey}); + } + else { + res[cField.dbKey] = regexRes.filter((e, i) => i > 0).join(' '); + } + } + } + else { + usedParts.push(cField.docKey); + res[cField.dbKey] = true; + } + } + }); + usedParts.forEach(part => { + const index = comment.indexOf(part); + if (index >= 0) { + comment = comment.slice(0, index) + comment.slice(index + part.length); + } + }); + if (/\w+/.test(comment)) { + res.xRest = comment; + } + return res; +} + +function sampleType (type) { + const allowedTypes = ['tension rod', 'part', 'granulate']; + return allowedTypes.indexOf(type) >= 0 ? type : (type === '' ? 'unknown' : 'other'); +} + function stripSpaces(s) { return s ? s.replace(/ /g,'') : ''; } diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts index 38c487a..e6f07b2 100644 --- a/src/helpers/csv.ts +++ b/src/helpers/csv.ts @@ -13,11 +13,16 @@ function flatten (data) { // flatten object: {a: {b: true}} -> {a.b: true} result[prop] = cur; } else if (Array.isArray(cur)) { - let l = 0; - for(let i = 0, l = cur.length; i < l; i++) - recurse(cur[i], prop + "[" + i + "]"); - if (l == 0) - result[prop] = []; + if (cur.length && (Object(cur[0]) !== cur || Object.keys(cur[0]).length === 0)) { // array of non-objects + result[prop] = '[' + cur.join(', ') + ']'; + } + else { + let l = 0; + for(let i = 0, l = cur.length; i < l; i++) + recurse(cur[i], prop + "[" + i + "]"); + if (l == 0) + result[prop] = []; + } } else { let isEmpty = true; diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 8d6c515..1989746 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -1343,6 +1343,67 @@ describe('/sample', () => { }); }); + describe('GET /sample/number/{number}', () => { + it('returns the right sample', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/sample/number/33', + auth: {basic: 'janedoe'}, + httpStatus: 200, + res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 33, carbon_fiber: 0}, numbers: ['5514262406']}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'} + }); + }); + it('works with an API key', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/sample/number/33', + auth: {key: 'janedoe'}, + httpStatus: 200, + res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 33, carbon_fiber: 0}, numbers: ['5514262406']}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'} + }); + }); + it('returns a deleted sample for a maintain/admin user', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/sample/number/Rng33', + auth: {basic: 'admin'}, + httpStatus: 200, + res: {_id: '400000000000000000000005', number: 'Rng33', type: 'granulate', color: 'black', batch: '1653000308', condition: {condition_template: '200000000000000000000003'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 33, carbon_fiber: 0}, numbers: ['5514262406']}, notes: {}, measurements: [], user: 'admin'} + }); + }); + it('returns 403 for a write user when requesting a deleted sample', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/sample/number/Rng33', + auth: {basic: 'janedoe'}, + httpStatus: 403 + }); + }); + it('returns 404 for an unknown sample', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/sample/number/Rng883', + auth: {basic: 'janedoe'}, + httpStatus: 404 + }); + }); + it('rejects an invalid id', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/sample/number/xx-xx', + auth: {basic: 'janedoe'}, + httpStatus: 404 + }); + }); + it('rejects unauthorized requests', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/sample/number/33', + httpStatus: 401 + }); + }); + }); + describe('PUT /sample/restore/{id}', () => { it('sets the status', done => { TestHelper.request(server, done, { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 67eb7b5..2d23d95 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -242,6 +242,13 @@ router.get('/samples', async (req, res, next) => { && e !== filters.sort[0] // field was not in sort ); + if (fieldsToAdd.find(e => e === 'notes')) { // add notes + queryPtr.push( + {$lookup: {from: 'notes', localField: 'note_id', foreignField: '_id', as: 'notes'}}, + {$addFields: {notes: { $arrayElemAt: ['$notes', 0]}}} + ); + } + if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already queryPtr.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, @@ -387,26 +394,7 @@ router.get('/sample/' + IdValidate.parameter(), (req, res, next) => { SampleModel.findById(req.params.id).populate('material_id').populate('user_id', 'name').populate('note_id').exec(async (err, sampleData: any) => { if (err) return next(err); - - if (sampleData) { - await sampleData.populate('material_id.group_id').populate('material_id.supplier_id').execPopulate().catch(err => next(err)); - if (sampleData instanceof Error) return; - sampleData = sampleData.toObject(); - - if (sampleData.status === globals.status.deleted && !req.auth(res, ['maintain', 'admin'], 'all')) return; // deleted samples only available for maintain/admin - sampleData.material = sampleData.material_id; // map data to right keys - sampleData.material.group = sampleData.material.group_id.name; - sampleData.material.supplier = sampleData.material.supplier_id.name; - sampleData.user = sampleData.user_id.name; - sampleData.notes = sampleData.note_id ? sampleData.note_id : {}; - MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id), status: {$ne: globals.status.deleted}}).lean().exec((err, data) => { - sampleData.measurements = data; - res.json(SampleValidate.output(sampleData, 'details')); - }); - } - else { - res.status(404).json({status: 'Not found'}); - } + await sampleReturn(sampleData, req, res, next); }); }); @@ -514,6 +502,15 @@ router.delete('/sample/' + IdValidate.parameter(), (req, res, next) => { }); }); +router.get('/sample/number/:number', (req, res, next) => { + if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; + + SampleModel.findOne({number: req.params.number}).populate('material_id').populate('user_id', 'name').populate('note_id').exec(async (err, sampleData: any) => { + if (err) return next(err); + await sampleReturn(sampleData, req, res, next); + }); +}); + router.put('/sample/restore/' + IdValidate.parameter(), (req, res, next) => { if (!req.auth(res, ['maintain', 'admin'], 'basic')) return; @@ -769,4 +766,27 @@ function filterQueries (filters) { function dateToOId (date) { // convert date to ObjectId return mongoose.Types.ObjectId(Math.floor(date / 1000).toString(16) + '0000000000000000'); +} + +async function sampleReturn (sampleData, req, res, next) { + if (sampleData) { + console.log(sampleData); + await sampleData.populate('material_id.group_id').populate('material_id.supplier_id').execPopulate().catch(err => next(err)); + if (sampleData instanceof Error) return; + sampleData = sampleData.toObject(); + + if (sampleData.status === globals.status.deleted && !req.auth(res, ['maintain', 'admin'], 'all')) return; // deleted samples only available for maintain/admin + sampleData.material = sampleData.material_id; // map data to right keys + sampleData.material.group = sampleData.material.group_id.name; + sampleData.material.supplier = sampleData.material.supplier_id.name; + sampleData.user = sampleData.user_id.name; + sampleData.notes = sampleData.note_id ? sampleData.note_id : {}; + MeasurementModel.find({sample_id: sampleData._id, status: {$ne: globals.status.deleted}}).lean().exec((err, data) => { + sampleData.measurements = data; + res.json(SampleValidate.output(sampleData, 'details')); + }); + } + else { + res.status(404).json({status: 'Not found'}); + } } \ No newline at end of file diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 6663986..e5f8ffe 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -70,13 +70,14 @@ export default class SampleValidate { private static fieldKeys = [ ...SampleValidate.sortKeys, 'condition', + 'notes', 'material_id', 'material', 'note_id', 'user_id', 'material._id', 'material.numbers', - 'measurements.spectrum.dpt' + 'measurements.spectrum.dpt', ]; static input (data, param) { // validate input, set param to 'new' to make all attributes required @@ -134,6 +135,7 @@ export default class SampleValidate { material_id: IdValidate.get(), material: MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}), note_id: IdValidate.get().allow(null), + notes: this.sample.notes, user_id: IdValidate.get(), added: this.sample.added };