added GET /sample/number/{number} route

api/parameters.yaml

@@ -6,6 +6,14 @@ Id:
     type: string
   example: 5ea0450ed851c30a90e70894
 
+Number:
+  name: number
+  in: path
+  required: true
+  schema:
+    type: string
+  example: Rng740
+
 Name:
   name: name
   description: has to be URL encoded

api/sample.yaml

@@ -140,10 +140,10 @@
           application/json:
             schema:
               $ref: 'api.yaml#/components/schemas/SampleDetail'
-      400:
-        $ref: 'api.yaml#/components/responses/400'
       401:
         $ref: 'api.yaml#/components/responses/401'
+      403:
+        $ref: 'api.yaml#/components/responses/403'
       404:
         $ref: 'api.yaml#/components/responses/404'
       500:
@@ -201,6 +201,31 @@
       500:
         $ref: 'api.yaml#/components/responses/500'
 
+/sample/number/{number}:
+  parameters:
+    - $ref: 'api.yaml#/components/parameters/Number'
+  get:
+    summary: sample details
+    description: 'Auth: all, levels: read, write, maintain, dev, admin<br>Returns validated as well as new measurements'
+    x-doc: deleted samples are available only for maintain/admin
+    tags:
+      - /sample
+    responses:
+      200:
+        description: samples details
+        content:
+          application/json:
+            schema:
+              $ref: 'api.yaml#/components/schemas/SampleDetail'
+      401:
+        $ref: 'api.yaml#/components/responses/401'
+      403:
+        $ref: 'api.yaml#/components/responses/403'
+      404:
+        $ref: 'api.yaml#/components/responses/404'
+      500:
+        $ref: 'api.yaml#/components/responses/500'
+
 /sample/restore/{id}:
   parameters:
     - $ref: 'api.yaml#/components/parameters/Id'

data_import/import.js

@@ -7,75 +7,148 @@ const pdfReader = require('pdfreader');
 const iconv = require('iconv-lite');
 const _ = require('lodash');
 
-const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv';  // metadata files
+const stages = {
-const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
+  materials: false,
-const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv';
+  samples: true,
-const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs';  // NormMaster Documents
+  measurements: false,
-const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT';  // Spectrum files
+  dpt: false
+}
+
+const docs = [
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2_A.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2_B.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Ap.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Bj.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh_B.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh_Duroplasten.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell_A.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell_B.csv",
+  "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_WaP.csv",
+];
+const errors = [];
+const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\nmDocs';  // NormMaster Documents
+const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\DPT';  // Spectrum files
 const host = 'http://localhost:3000';
 // const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
+const requiredProperties = ['samplenumber','materialnumber','materialname','supplier','reinforcementmaterial','material','granulate/part','color','charge/batch','comments'];
+dict = {  // dictionary
+  'Granulat': 'granulate',
+  'Zugstab': 'tension rod',
+  'Stecker': 'plug'
+};
 let data = [];  // metadata contents
 let materials = {};
-const numberToColor = {};
+let numberToColor = {};
 let samples = [];
 let normMaster = {};
 let sampleDevices = {};
+const sampleReferences = [];  // references to other samples in format {sample, referencedSample, relation}
+let comments = [];
 
-// TODO: BASF twice, BASF as color
-// TODO: duplicate kf values
 // TODO: conditions
 // TODO: comment and reference handling
 
+// TODO: measurement device to spectrum
+
 
 // TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC
 
+// TODO: samples, conditions
+
 main();
 
 async function main() {
-  if (1) {  // materials
+  if (stages.materials) {  // materials
     await getNormMaster();
-    await importCsv(metaDoc);
+    for (let i in docs) {
-    await allMaterials();
+      await importCsv(docs[i]);
-    await saveMaterials();
+      await allMaterials();
-    await importCsv(kfDoc);
+      await saveMaterials();
-    await allMaterials();
+    }
-    await saveMaterials();
+    fs.writeFileSync('./data_import/numberToColor.json', JSON.stringify(numberToColor));
-    await importCsv(vzDoc);
-    await allMaterials();
-    await saveMaterials();
   }
-  if (1) {  // samples
+  if (stages.samples) {  // samples
     sampleDeviceMap();
-    if (1) {
+    numberToColor = JSON.parse(fs.readFileSync('./data_import/numberToColor.json'), 'utf-8');
-      console.log('-------- META ----------');
+    for (let i in docs) {
-      await importCsv(metaDoc);
+      await importCsv(docs[i]);
       await allSamples();
       await saveSamples();
-    }
+      console.log(samples);
-    if (1) {
+    }  // TODO: get sample by number, implement smple references
-      console.log('-------- KF ----------');
+    fs.writeFileSync('./data_import/comments.txt', comments.join('\r\n'));
-      await importCsv(kfDoc);
+    // if (1) {  // TODO: KfVz
-      await allSamples();
+    //   console.log('-------- META ----------');
-      await saveSamples();
+    //   await importCsv(metaDoc);
-      await allKfVz();
+    //   await allSamples();
-    }
+    //   await saveSamples();
-    if (1) {
+    // }
-      console.log('-------- VZ ----------');
+    // if (1) {
-      await importCsv(vzDoc);
+    //   console.log('-------- KF ----------');
-      await allSamples();
+    //   await importCsv(kfDoc);
-      await saveSamples();
+    //   await allSamples();
-      await allKfVz();
+    //   await saveSamples();
-    }
+    //   await allKfVz();
+    // }
+    // if (1) {
+    //   console.log('-------- VZ ----------');
+    //   await importCsv(vzDoc);
+    //   await allSamples();
+    //   await saveSamples();
+    //   await allKfVz();
+    // }
   }
-  if (1) {  // DPT
+  if (stages.dpt) {  // DPT
     await allDpts();
   }
   if (0) {  // pdf test
-    console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf'));
+    console.log(await readPdf('N28_BN05-OX023_2019-07-16.pdf'));
+  }
+  if (errors.length) {
+    // console.log(errors);
+    fs.writeFileSync('./data_import/errors/errors_' + new Date().getTime() + '.txt', errors.join('\r\n'));
   }
 }
 
 async function importCsv(doc) {
+  // Uniform name                 samplenumber  materialnumber  materialname  supplier material plastic reinforcingmaterial                 granulate/part color charge/batch                comments  vz(ml/g)  kfingew%       degradation(%) glassfibrecontent(%)        stabwn
+  // Metadata__AnP2.csv           Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,               granulate/Part,Color,Charge/ Batch,              Comments
+  // Metadata__AnP2_A.csv         Sample number,Material number,Material name,Supplier,         Plastic,Reinforcing material,               Granulate/Part,                                  Comments,           Humidity [ppm]
+  // Metadata__AnP2_B.csv         Sample number,Material number,Material name,Supplier,         Plastic,Reinforcing material,               Granulate/Part,                                            VZ [ml/g],                              glass fibre content
+  // Metadata_Ap.csv              Sample number,Material number,Material name,Supplier,         Plastic,Reinforcing material,               Granulate/Part,Color,Charge/Batch,               Comments
+  // Metadata_Bj.csv              Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,               Granulate/Part,Color,Charge/batch granulate/part,Comments
+  // Metadata_Eh.csv              Sample number,Material number,Material name,Supplier,Material,        Reinforcing material,               Granulate/Part,Color,Charge/Batch granulate/part,Comments, VZ [cm³/g],                                                                Spalte1
+  // Metadata_Eh_B.csv            Sample number,                Material name,Supplier,         Plastic,Reinforcing material,               Granulate/Part,Color,                            Comments, VZ [cm³/g]
+  // Metadata_Eh_Duroplasten.csv  Sample number,Material number,Material name,Supplier,Material,        Reinforcing material,               Granulate/Part,Color,Charge/Batch granulate/part,Comments
+  // Metadata_Rng_aktuell.csv     Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,               Granulate/Part,Color,Charge/batch granulate/part,Comments, VZ (ml/g),               Degradation(%),Glas fibre content (%)
+  // Metadata_Rng_aktuell_A.csv   Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,               Granulate/Part,Farbe,Charge/batch granulate/part,Comments,           KF in Gew%,                                               Stabwn
+  // Metadata_Rng_aktuell_B.csv   Sample number,                Material name,Supplier,         Plastic,Reinforcing material (content in %),Granulate/Part,                                  Comments, VZ (ml/g),               Degradation (%),                                  Alterungszeit in h
+  // Metadata_WaP.csv             Probennummer,                 Name,         Firma,   Material,                                            Teil/Rohstoff,       Charge,                     Anmerkung,VZ (ml/g),               Abbau (%),     Verstärkungsstoffgehalt (%),       Versuchsnummer
+  const nameCorrection = {  // map to right column names
+    'probennummer': 'samplenumber',
+    'name': 'materialname',
+    'firma': 'supplier',
+    'reinforcingmaterial(contentin%)': 'reinforcingmaterial',
+    'teil/rohstoff': 'granulate/part',
+    'charge/batchgranulate/part': 'charge/batch',
+    'charge': 'charge/batch',
+    'anmerkung': 'comments',
+    'vz[ml/g]': 'vz(ml/g)',
+    'vz[cm³/g]': 'vz(ml/g)',
+    'abbau(%)': 'degradation(%)',
+    'verstärkungsstoffgehalt(%)': 'glassfibrecontent(%)'
+  };
+  const missingFieldsFill = [  // column names to fill if they do not exist
+    'color',
+    'charge/batch',
+    'comments',
+    'materialnumber',
+    'reinforcementmaterial'
+  ]
+  console.log('importing ' + doc);
   data = [];
   await new Promise(resolve => {
     fs.createReadStream(doc)
@@ -85,10 +158,47 @@ async function importCsv(doc) {
         data.push(row);
       })
       .on('end', () => {
+        data = data.map(e => {
+          const newE = {};
+          Object.keys(e).forEach(key => {
+            newE[key.toLowerCase().replace(/ /g, '')] = e[key];
+          });
+          // replace wrong column names
+          Object.keys(newE).forEach(key => {
+            if (nameCorrection.hasOwnProperty(key)) {
+              newE[nameCorrection[key]] = newE[key];
+              delete newE[key];
+            }
+          });
+
+          // add missing fields with empty values
+          missingFieldsFill.forEach(field => {
+            if (!newE.hasOwnProperty(field)) {
+              newE[field] = '';
+            }
+          });
+          if (newE['supplier'] === '') {  // empty supplier fields
+            newE['supplier'] = 'unknown';
+          }
+          if (!newE.hasOwnProperty('material')) {
+            newE['material'] = newE['plastic'].indexOf(' GF') >= 0 ? newE['plastic'].split(' ')[0] : newE['plastic'];
+          }
+          return newE;
+        }).filter(e => {
+          const missingProperties = requiredProperties.filter(el => !e.hasOwnProperty(el));
+          if (e['materialname'] === '') {
+            missingProperties.push('materialname');
+          }
+          if (e['samplenumber'] === '') {  // empty row
+            return false;
+          }
+          else if (missingProperties.length > 0) {  // incomplete sample
+            errors.push(`${doc}: ${JSON.stringify(e)}is missing the required properties ${missingProperties}`);
+            return false;
+          }
+          return true;
+        });
         console.info('CSV file successfully processed');
-        if (data[0]['Farbe']) {  // fix German column names
-          data.map(e => {e['Color'] = e['Farbe']; return e; });
-        }
         resolve();
       });
   });
@@ -158,6 +268,7 @@ async function allDpts() {
   }
 }
 
+// TODO: VZ from comments
 async function allKfVz() {
   let res = await axios({
     method: 'get',
@@ -266,53 +377,56 @@ async function allSamples() {
   for (let index in data) {
     console.info(`${index}/${data.length}`);
     let sample = data[index];
-    if (sample['Sample number'] !== '') {  // TODO: what about samples without color
+    if (sample['granulate/Part'] === '') {  // empty supplier fields
-      if (sample['Supplier'] === '') {  // empty supplier fields
+      sample['granulate/Part'] = 'unknown';
-        sample['Supplier'] = 'unknown';
+    }
+    const material = dbMaterials[trim(sample['materialname'])];
+    if (!material) { // could not find material, skipping sample
+      errors.push(`Could not find a material for ${JSON.stringify(sample)}`);
+      continue;
+    }
+    samples.push({
+      number: sample['samplenumber'],
+      type: sampleType(sample['granulate/part']),
+      batch: sample['charge/batch'],
+      material_id: material._id,
+      notes: {
+        custom_fields: customFields(sample['comments'], sample['samplenumber'])
       }
-      if (sample['Granulate/Part'] === '') {  // empty supplier fields
+    });
-        sample['Granulate/Part'] = 'unknown';
+    // if (sample['comments']) {
-      }
+    //   comments.push(sample['samplenumber'] + '   ' + sample['comments']);
-      const material = dbMaterials[trim(sample['Material name'])];
+    // }
-      if (!material) { // could not find material, skipping sample
+    const si = samples.length - 1;  // sample index
-        continue;
+    if (samples[si].notes.custom_fields.hasOwnProperty('xRest')) {  // reroute xRest property to comment
-      }
+      samples[si].notes.comment = samples[si].notes.custom_fields.xRest;
-      samples.push({
+      delete samples[si].notes.custom_fields.xRest;
-        number: sample['Sample number'],
+    }
-        type: sample['Granulate/Part'],
+    if (Object.keys(samples[si].notes.custom_fields). length === 0) {
-        batch: sample['Charge/batch granulate/part'] || '',
+      delete samples[si].notes.custom_fields;
-        material_id: material._id,
+    }
-        notes: {
+    if (sample['materialnumber'] !== '' && material.numbers.find(e => e.number === sample['materialnumber'])) {
-          comment: sample['Comments']
+      samples[si].color = material.numbers.find(e => e.number === sample['materialnumber']).color;
-        }
+    }
-      });
+    else if (sample['color'] !== '') {  // find color with all edge cases
-      const si = samples.length - 1;
+      let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['color'])) >= 0);
-      if (sample['Material number'] !== '' && material.numbers.find(e => e.number === sample['Material number'])) {  // TODO: fix because of false material/material number
+      if (!number && /black/.test(sample['color'])) {  // special case bk for black
-        samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
-      }
-      else if (sample['Color'] && sample['Color'] !== '') {
         console.log(material);
-        let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['Color'])) >= 0);
+        number = material.numbers.find(e => e.color && e.color.toLowerCase().indexOf('bk') >= 0);
-        if (!number && /black/.test(sample['Color'])) {  // special case bk for black
+        if (!number) {  // try German word
-          number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
+          number = material.numbers.find(e => e.color && e.color.toLowerCase().indexOf('schwarz') >= 0);
-          if (!number) {  // try German word
-            number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
-          }
-        }
-        if (number) {
-          samples[si].color = number.color;
-        }
-        else {
-          samples[si].color = '';
         }
       }
-      else if (sampleColors[sample['Sample number'].split('_')[0]]) {  // derive color from main sample for kf/vz
+      if (number) {
-        samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
+        samples[si].color = number.color;
-      }
-      else {
-        samples[si].color = '';
       }
     }
+    else if (sampleColors[sample['samplenumber'].split('_')[0]]) {  // derive color from main sample for kf/vz
+      samples[si].color = sampleColors[sample['samplenumber'].split('_')[0]];
+    }
+    if (!samples[si].color) {
+      samples[si].color = sample['color'];
+    }
   }
 }
 
@@ -321,6 +435,7 @@ async function saveSamples() {
     console.info(`${i}/${samples.length}`);
     let credentials = ['admin', 'Abc123!#'];
     if (sampleDevices[samples[i].number]) {
+      console.log(sampleDevices[samples[i].number]);
       credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!']
     }
     await axios({
@@ -335,6 +450,7 @@ async function saveSamples() {
       if (err.response.data.status && err.response.data.status !== 'Sample number already taken') {
         console.log(samples[i]);
         console.error(err.response.data);
+        errors.push(`Upload for ${JSON.stringify(samples[i])} failed: ${JSON.stringify(err.response.data)}`)
       }
     });
   }
@@ -342,56 +458,65 @@ async function saveSamples() {
 }
 
 async function allMaterials() {
-  materials = {};
+  // materials = {};
+  let res = await axios({
+    method: 'get',
+    url: host + '/template/materials',
+    auth: {
+      username: 'admin',
+      password: 'Abc123!#'
+    }
+  });
+  const materialTemplate = res.data.find(e => e.name === 'plastic')._id;
+
+  // process all samples
   for (let index in data) {
     let sample = data[index];
-    if (sample['Sample number'] && sample['Sample number'] !== '') {
+    if (sample['supplier'] === '') {  // empty supplier fields
-      if (sample['Supplier'] === '') {  // empty supplier fields
+      sample['supplier'] = 'unknown';
-        sample['Supplier'] = 'unknown';
+    }
-      }
+    if (sample['materialname'] === '') {  // empty name fields
-      if (sample['Material name'] === '') {  // empty name fields
+      sample['materialname'] = sample['material'];
-        sample['Material name'] = sample['Material'];
+    }
-      }
+    sample['materialname'] = trim(sample['materialname']);
-      if (!sample['Material']) {  // column Material is named Plastic in VZ metadata
+    if (materials.hasOwnProperty(sample['materialname'])) {  // material already found at least once
-        sample['Material'] = sample['Plastic'];
+      if (sample['materialnumber'] !== '') {  // material number given
-      }
+        if (materials[sample['materialname']].numbers.length === 0 || !materials[sample['materialname']].numbers.find(e => e.number === stripSpaces(sample['materialnumber']))) {  // new material number
-      sample['Material name'] = trim(sample['Material name']);
+          if (materials[sample['materialname']].numbers.find(e => e.color === sample['color'] && e.number === '')) {  // color already in list, only number missing
-      if (materials.hasOwnProperty(sample['Material name'])) {  // material already found at least once
+            materials[sample['materialname']].numbers.find(e => e.color === sample['color'] && e.number === '').number = stripSpaces(sample['materialnumber']);
-        if (sample['Material number'] && sample['Material number'] !== '') {
-          if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) {  // new material number
-            if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) {  // color already in list, only number missing
-              materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']);
-            }
-            else {
-              materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])});
-            }
           }
-        }
+          else {  // completely new number entry
-        else if (sample['Color'] && sample['Color'] !== '') {
+            materials[sample['materialname']].numbers.push({color: trim(sample['color']), number: stripSpaces(sample['materialnumber'])});
-          if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) {  // new material color
-            materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''});
           }
         }
       }
-      else {  // new material
+      else if (sample['color'] !== '') {  // color given
-        console.info(`${index}/${data.length}  ${sample['Material name']}`);
+        if (!materials[sample['materialname']].numbers.find(e => e.color === stripSpaces(sample['color']))) {  // new material color
-        materials[sample['Material name']] = {
+          materials[sample['materialname']].numbers.push({color: trim(sample['color']), number: ''});
-          name: sample['Material name'],
+        }
-          supplier: trim(sample['Supplier']),
-          group: trim(sample['Material'])
-        };
-        materials[sample['Material name']].properties = {material_template: '5f0efe6fce7fd20ce4e99013'};
-        let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].properties.mineral = tmp ? tmp[1] : 0;
-        tmp = /GF(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].properties.glass_fiber = tmp ? tmp[1] : 0;
-        tmp = /CF(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].properties.carbon_fiber = tmp ? tmp[1] : 0;
-        materials[sample['Material name']].numbers = await numbersFetch(sample);
-        console.log(materials[sample['Material name']]);
       }
     }
+    else {  // new material
+      console.info(`${index}/${data.length}  ${sample['materialname']}`);
+      materials[sample['materialname']] = {
+        name: trim(sample['materialname']),
+        supplier: trim(sample['supplier']),
+        group: trim(sample['material'])
+      };
+      materials[sample['materialname']].numbers = await numbersFetch(sample);
+
+      // material properties
+      materials[sample['materialname']].properties = {material_template: materialTemplate};
+      let tmp = /M(\d+)/.exec(sample['reinforcingmaterial']);
+      materials[sample['materialname']].properties.mineral = tmp ? tmp[1] : 0;
+      tmp = /GF(\d+)/.exec(sample['reinforcingmaterial']);
+      materials[sample['materialname']].properties.glass_fiber = tmp ? tmp[1] : 0;
+      tmp = /CF(\d+)/.exec(sample['reinforcingmaterial']);
+      materials[sample['materialname']].properties.carbon_fiber = tmp ? tmp[1] : 0;
+    }
   }
+
+  // Fill numberToColor array
   Object.keys(materials).forEach(mKey => {
     materials[mKey].numbers.forEach(number => {
       if (number.number && number.color) {
@@ -419,6 +544,7 @@ async function saveMaterials() {
       if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
         console.info(material);
         console.error(err.response.data);
+        errors.push(`Upload for ${JSON.stringify(material)} failed: ${JSON.stringify(err.response.data)}`)
       }
     });
   }
@@ -428,17 +554,17 @@ async function saveMaterials() {
 async function numbersFetch(sample) {
   let nm = [];
   let res = [];
-  if (sample['Material number']) {  // sample has a material number
+  if (sample['materialnumber']) {  // sample has a material number
-    nm = normMaster[stripSpaces(sample['Material number'])]? [normMaster[stripSpaces(sample['Material number'])]] : [];
+    nm = normMaster[stripSpaces(sample['materialnumber'])]? [normMaster[stripSpaces(sample['materialnumber'])]] : [];
   }
   else {  // try finding via material name
-    nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['Material name'])).map(e => normMaster[e]);
+    nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['materialnumber'])).map(e => normMaster[e]);
   }
   if (nm.length > 0) {
     for (let i in nm) {
-      // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) {  // document not loaded
+      if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) {  // document not loaded
-      //   await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'));
+        await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'));
-      // }
+      }
       // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) {  // document not loaded
       //   console.info('Retrying download...');
       //   await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 2.2);
@@ -454,21 +580,22 @@ async function numbersFetch(sample) {
         break;
       }
       else if (i + 1 >= nm.length) {
-        console.error('Download failed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
+        errors.push(`Download of ${nm[i].url.replace(/ /g, '%20')} for material number ${sample['materialnumber']} failed`);
+        errors.push(nm[i].doc.replace(/ /g, '_'));
       }
     }
   }
   if (res.length === 0) {  // no results
-    if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) {
+    if (sample['color'] !== '' || sample['materialnumber'] !== '') {  // information in data available
-      return [{color: trim(sample['Color']), number: sample['Material number']}];
+      return [{color: trim(sample['color']), number: sample['materialnumber']}];
     }
     else {
       return [];
     }
   }
   else {
-    if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) {  // sometimes norm master does not include sample number even if listed
+    if (!res.find(e => e.number === sample['materialnumber'])) {  // sometimes norm master does not include sample number even if listed
-      res.push({color: trim(sample['Color']), number: sample['Material number']});
+      res.push({color: trim(sample['color']), number: sample['materialnumber']});
     }
     return res;
   }
@@ -538,7 +665,7 @@ function readPdf(file) {
     let lastLastText = '';  // text of last last item
     await new pdfReader.PdfReader().parseFileItems(nmDocs + '\\' + file, (err, item) => {
       if (item && item.text) {
-        if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsupplier') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsupplier') >= 0)) {  // table area starts
+        if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsuppl') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsupplier') >= 0)) {  // table area starts
           table = countdown;
         }
         if (table > 0) {
@@ -587,6 +714,102 @@ function sampleDeviceMap() {
   }
 }
 
+function customFields (comment, sampleNumber) {
+  const customFields = [
+    {docKey: 'Versuchsreihe', dbKey: 'test series', regex: /Versuchsreihe (\d+),/, reference: false},
+    {docKey: 'Stillstand', dbKey: 'idle', regex: /(\d+ min)/, reference: false},
+    {docKey: 'Serienzyklus', dbKey: 'cycle', regex: /(\d+.) Serienzyklus (\(.*?\))/, reference: false},
+    {docKey: 'Berstdruck', dbKey: 'bursting pressure', regex: /Berstdruck: (.*?bar);/, reference: false},
+    {docKey: 'gemessen am', dbKey: 'measured at', regex: /gemessen am (.*20\d\d)/, reference: false},
+    {docKey: 'used for', dbKey: 'used for', regex: /used for (.*)/, reference: false},
+    {docKey: 'Stabilized', dbKey: 'stabilized', regex: null, reference: false},
+    {docKey: 'parts from field', dbKey: 'parts from field', regex: null, reference: false},
+    {docKey: 'side', dbKey: 'side', regex: /(\S*?) side/, reference: false},
+    {docKey: 'Creep test', dbKey: 'creep test', regex: null, reference: false},
+    {docKey: 'Variante', dbKey: 'variant', regex: /(.*)/, reference: false},
+    {docKey: 'Parameter', dbKey: 'parameter', regex: /Parameter (\d)/, reference: false},
+    {docKey: 'days without cooling', dbKey: 'days without cooling', regex: /(\d+) days without cooling/, reference: false},
+    {docKey: 'Zyklus', dbKey: 'cycle', regex: /Zyklus (\d+ s)/, reference: false},
+    {docKey: 'fast cure', dbKey: 'fast cure', regex: null, reference: false},
+    {docKey: 'Stoff gesperrt', dbKey: 'material blocked', regex: null, reference: false},
+    {docKey: 'anwendungsbeschränkt', dbKey: 'limited application', regex: null, reference: false},
+    {docKey: 'für Neuanwendungen gesperrt', dbKey: 'blocked for new applications', regex: null, reference: false},
+    {docKey: 'V', dbKey: 'test', regex: /V(\d+-\d+);/, reference: false},
+    {docKey: 'Twz', dbKey: 'twz', regex: /Twz \(°C\): (\d+);/, reference: false},
+    {docKey: 'Pnach', dbKey: 'pressure after', regex: /Pnach \(bar\): (\d+);/, reference: false},
+    {docKey: 'Vein', dbKey: 'volume in', regex: /Vein \(ccm\/s\): (\d+)/, reference: false},
+    {docKey: 'low emission', dbKey: 'low emission', regex: /low emission (.*?);/, reference: false},
+    {docKey: 'aus', dbKey: 'from', regex: /aus (.*)/, reference: false},
+    {docKey: 'Erprobung', dbKey: 'trial', regex: /Erprobung (.*?);/, reference: false},
+    {docKey: 'Auftragsnummer', dbKey: 'job number', regex: /Auftragsnummer: (.*?);/, reference: false},
+    {docKey: 'Wärmealterung', dbKey: 'heat aging', regex: /Wärmealterung: (.*)/, reference: false},
+    {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall', regex: /Steg.*?A: (\d+)/, reference: false},
+    {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall', regex: /Steg.*?I: (\d+)/, reference: false},
+    {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall', regex: /Steg.*?S: (\d+)/, reference: false},
+    {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall degraded', regex: /Degradation:.*?A: (\d+)/, reference: false},
+    {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall degraded', regex: /Degradation:.*?I: (\d+)/, reference: false},
+    {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall degraded', regex: /Degradation:.*?S: (\d+)/, reference: false},
+    {docKey: 'Reines Polymer', dbKey: 'pure polymer', regex: null, reference: false},
+    {docKey: 'Rücksendung erforderlich', dbKey: 'return needed', regex: /(.*?,) Rücksendung erforderlich, (.*)/, reference: false},
+    {docKey: 'Prio', dbKey: 'priority', regex: /Prio (\d+)/, reference: false},
+    {docKey: 'beanstandet', dbKey: 'faulty', regex: null, reference: false},
+    {docKey: 'aged', dbKey: 'aged', regex: /aged: (.*)/, reference: false},
+    {docKey: 'DOPPELT!!', dbKey: 'double', regex: null, reference: false},
+    {docKey: 'Bauteil', dbKey: 'construction part', regex: /Bauteil (\S+)/, reference: false},
+    {docKey: 'T =', dbKey: 'temperature', regex: /T = (\S+)/, reference: false},
+    {docKey: 'nicht vorgealtert', dbKey: 'not preaged', regex: /nicht vorgealtert (.*)/, reference: false},
+    {docKey: 'TS119', dbKey: 'TS119', regex: /TS119 (W\S+);/, reference: false},
+    {docKey: 'GF vom Datenblatt', dbKey: 'glass fibre from data sheet', regex: null, reference: false},
+    {docKey: 'nach Datensatz', dbKey: 'according to dataset', regex: null, reference: false},
+    {docKey: 'Dosiergeschw', dbKey: 'metering speed', regex: /Dosiergeschw.*? (.*?min)/, reference: false},
+    {docKey: 'Einspritzgeschw', dbKey: 'injection speed', regex: /Einspritzgeschw.*? (.*\/s)/, reference: false},
+    {docKey: 'Heizbänder', dbKey: 'heating lines', regex: /Heizbänder (.*)/, reference: false},
+    {docKey: 'Verweilzeit', dbKey: 'dwell time', regex: /Verweilzeit (.*?min)/, reference: false},
+    {docKey: 'Probe', dbKey: 'belongs to', regex: /Probe (\S*\d+)/, reference: true},
+    {docKey: 'zu', dbKey: 'belongs to', regex: /zu (\S*\d+)/, reference: true},
+    {docKey: 'granulate zu', dbKey: 'granulate to', regex: /granulate zu.* (\S*\d+)/, reference: true},
+    {docKey: 'construction part', dbKey: 'construction part', regex: /(?<!granulate)construction part.* (\S*\d+)/, reference: true},
+
+  ];
+  const res = {};  // returned result
+  const usedParts = [];  // all substrings used for custom fields, subtract at the end, as some parts are used multiple times
+  customFields.forEach(cField => {
+    if (comment.indexOf(cField.docKey) >= 0) {  // comment contains docKey
+      if (cField.regex !== null) {
+        const regexRes = cField.regex.exec(comment);
+        if (regexRes) {
+          usedParts.push(regexRes[0]);
+          if (cField.reference) {
+            sampleReferences.push({sample: sampleNumber, referencedSample: regexRes[1], relation: cField.dbKey});
+          }
+          else {
+            res[cField.dbKey] = regexRes.filter((e, i) => i > 0).join(' ');
+          }
+        }
+      }
+      else {
+        usedParts.push(cField.docKey);
+        res[cField.dbKey] = true;
+      }
+    }
+  });
+  usedParts.forEach(part => {
+    const index = comment.indexOf(part);
+    if (index >= 0) {
+      comment = comment.slice(0, index) + comment.slice(index + part.length);
+    }
+  });
+  if (/\w+/.test(comment)) {
+    res.xRest = comment;
+  }
+  return res;
+}
+
+function sampleType (type) {
+  const allowedTypes = ['tension rod', 'part', 'granulate'];
+  return allowedTypes.indexOf(type) >= 0 ? type : (type === '' ? 'unknown' : 'other');
+}
+
 function stripSpaces(s) {
   return s ? s.replace(/ /g,'') : '';
 }

src/helpers/csv.ts

@@ -13,11 +13,16 @@ function flatten (data) {  // flatten object: {a: {b: true}} -> {a.b: true}
       result[prop] = cur;
     }
     else if (Array.isArray(cur)) {
-      let l = 0;
+      if (cur.length && (Object(cur[0]) !== cur || Object.keys(cur[0]).length === 0)) {  // array of non-objects
-      for(let i = 0, l = cur.length; i < l; i++)
+        result[prop] = '[' + cur.join(', ') + ']';
-        recurse(cur[i], prop + "[" + i + "]");
+      }
-      if (l == 0)
+      else {
-        result[prop] = [];
+        let l = 0;
+        for(let i = 0, l = cur.length; i < l; i++)
+          recurse(cur[i], prop + "[" + i + "]");
+        if (l == 0)
+          result[prop] = [];
+      }
     }
     else {
       let isEmpty = true;

src/routes/sample.spec.ts

@@ -1343,6 +1343,67 @@ describe('/sample', () => {
     });
   });
 
+  describe('GET /sample/number/{number}', () => {
+    it('returns the right sample', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/sample/number/33',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200,
+        res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 33, carbon_fiber: 0}, numbers: ['5514262406']}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'}
+      });
+    });
+    it('works with an API key', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/sample/number/33',
+        auth: {key: 'janedoe'},
+        httpStatus: 200,
+        res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 33, carbon_fiber: 0}, numbers: ['5514262406']}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'}
+      });
+    });
+    it('returns a deleted sample for a maintain/admin user', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/sample/number/Rng33',
+        auth: {basic: 'admin'},
+        httpStatus: 200,
+        res: {_id: '400000000000000000000005', number: 'Rng33', type: 'granulate', color: 'black', batch: '1653000308', condition: {condition_template: '200000000000000000000003'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 33, carbon_fiber: 0}, numbers: ['5514262406']}, notes: {}, measurements: [], user: 'admin'}
+      });
+    });
+    it('returns 403 for a write user when requesting a deleted sample', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/sample/number/Rng33',
+        auth: {basic: 'janedoe'},
+        httpStatus: 403
+      });
+    });
+    it('returns 404 for an unknown sample', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/sample/number/Rng883',
+        auth: {basic: 'janedoe'},
+        httpStatus: 404
+      });
+    });
+    it('rejects an invalid id', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/sample/number/xx-xx',
+        auth: {basic: 'janedoe'},
+        httpStatus: 404
+      });
+    });
+    it('rejects unauthorized requests', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/sample/number/33',
+        httpStatus: 401
+      });
+    });
+  });
+
   describe('PUT /sample/restore/{id}', () => {
     it('sets the status', done => {
       TestHelper.request(server, done, {

src/routes/sample.ts

@@ -242,6 +242,13 @@ router.get('/samples', async (req, res, next) => {
     && e !== filters.sort[0]                           // field was not in sort
   );
 
+  if (fieldsToAdd.find(e => e === 'notes')) {  // add notes
+    queryPtr.push(
+      {$lookup: {from: 'notes', localField: 'note_id', foreignField: '_id', as: 'notes'}},
+      {$addFields: {notes: { $arrayElemAt: ['$notes', 0]}}}
+    );
+  }
+
   if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) {  // add material, was not added already
     queryPtr.push(
       {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
@@ -387,26 +394,7 @@ router.get('/sample/' + IdValidate.parameter(), (req, res, next) => {
 
   SampleModel.findById(req.params.id).populate('material_id').populate('user_id', 'name').populate('note_id').exec(async (err, sampleData: any) => {
     if (err) return next(err);
-
+    await sampleReturn(sampleData, req, res, next);
-    if (sampleData) {
-      await sampleData.populate('material_id.group_id').populate('material_id.supplier_id').execPopulate().catch(err => next(err));
-      if (sampleData instanceof Error) return;
-      sampleData = sampleData.toObject();
-
-      if (sampleData.status === globals.status.deleted && !req.auth(res, ['maintain', 'admin'], 'all')) return;  // deleted samples only available for maintain/admin
-      sampleData.material = sampleData.material_id;  // map data to right keys
-      sampleData.material.group = sampleData.material.group_id.name;
-      sampleData.material.supplier = sampleData.material.supplier_id.name;
-      sampleData.user = sampleData.user_id.name;
-      sampleData.notes = sampleData.note_id ? sampleData.note_id : {};
-      MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id), status: {$ne: globals.status.deleted}}).lean().exec((err, data) => {
-        sampleData.measurements = data;
-        res.json(SampleValidate.output(sampleData, 'details'));
-      });
-    }
-    else {
-      res.status(404).json({status: 'Not found'});
-    }
   });
 });
 
@@ -514,6 +502,15 @@ router.delete('/sample/' + IdValidate.parameter(), (req, res, next) => {
   });
 });
 
+router.get('/sample/number/:number', (req, res, next) => {
+  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
+
+  SampleModel.findOne({number: req.params.number}).populate('material_id').populate('user_id', 'name').populate('note_id').exec(async (err, sampleData: any) => {
+    if (err) return next(err);
+    await sampleReturn(sampleData, req, res, next);
+  });
+});
+
 router.put('/sample/restore/' + IdValidate.parameter(), (req, res, next) => {
   if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;
 
@@ -770,3 +767,26 @@ function filterQueries (filters) {
 function dateToOId (date) {  // convert date to ObjectId
   return mongoose.Types.ObjectId(Math.floor(date / 1000).toString(16) + '0000000000000000');
 }
+
+async function sampleReturn (sampleData, req, res, next) {
+  if (sampleData) {
+    console.log(sampleData);
+    await sampleData.populate('material_id.group_id').populate('material_id.supplier_id').execPopulate().catch(err => next(err));
+    if (sampleData instanceof Error) return;
+    sampleData = sampleData.toObject();
+
+    if (sampleData.status === globals.status.deleted && !req.auth(res, ['maintain', 'admin'], 'all')) return;  // deleted samples only available for maintain/admin
+    sampleData.material = sampleData.material_id;  // map data to right keys
+    sampleData.material.group = sampleData.material.group_id.name;
+    sampleData.material.supplier = sampleData.material.supplier_id.name;
+    sampleData.user = sampleData.user_id.name;
+    sampleData.notes = sampleData.note_id ? sampleData.note_id : {};
+    MeasurementModel.find({sample_id: sampleData._id, status: {$ne: globals.status.deleted}}).lean().exec((err, data) => {
+      sampleData.measurements = data;
+      res.json(SampleValidate.output(sampleData, 'details'));
+    });
+  }
+  else {
+    res.status(404).json({status: 'Not found'});
+  }
+}

src/routes/validate/sample.ts

@@ -70,13 +70,14 @@ export default class SampleValidate {
   private static fieldKeys = [
     ...SampleValidate.sortKeys,
     'condition',
+    'notes',
     'material_id',
     'material',
     'note_id',
     'user_id',
     'material._id',
     'material.numbers',
-    'measurements.spectrum.dpt'
+    'measurements.spectrum.dpt',
   ];
 
   static input (data, param) {  // validate input, set param to 'new' to make all attributes required
@@ -134,6 +135,7 @@ export default class SampleValidate {
         material_id: IdValidate.get(),
         material: MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}),
         note_id: IdValidate.get().allow(null),
+        notes: this.sample.notes,
         user_id: IdValidate.get(),
         added: this.sample.added
       };