spectrum field working again

2020-07-09 13:48:27 +02:00
parent 6a02f09e7f
commit 1ddc2b617a
13 changed files with 269 additions and 115 deletions
--- a/api/api.yaml
+++ b/api/api.yaml
@@ -39,10 +39,10 @@ info:


 servers:
+  - url: https://definma-api.apps.de1.bosch-iot-cloud.com
+    description: server on the BIC
  - url: http://localhost:3000
    description: local server
-  - url: https://digital-fingerprint-of-plastics-api.apps.de1.bosch-iot-cloud.com/
-    description: server on the BIC


 security:
--- a/api/sample.yaml
+++ b/api/sample.yaml
@@ -2,7 +2,7 @@
  get:
    summary: all samples in overview
    description: 'Auth: all, levels: read, write, maintain, dev, admin'
-    x-doc: returns only samples with status 10
+    x-doc: 'Limitations: paging and csv output does not work when including the spectrum measurement fields as well as the returned number of total samples'
    tags:
      - /sample
    parameters:
@@ -61,6 +61,12 @@
    responses:
      200:
        description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)
+        headers:
+          X-Total-Items:
+            description: Total number of available items when page is specified
+            schema:
+              type: integer
+              example: 243
        content:
          application/json:
            schema:
--- a/data_import/import.js
+++ b/data_import/import.js
@@ -1,55 +1,82 @@
 const csv = require('csv-parser');
 const fs = require('fs');
 const axios = require('axios');
-const {Builder} = require('selenium-webdriver');
+const {Builder} = require('selenium-webdriver');    // selenium and the chrome driver must be installed and configured separately
 const chrome = require('selenium-webdriver/chrome');
 const pdfReader = require('pdfreader');
 const iconv = require('iconv-lite');

-const metadata = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\VZ.csv';  // metadata file
-const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\nmDocs';  // NormMaster Documents
-const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\DPT';  // Spectrum files
+const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv';  // metadata files
+const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
+const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv';
+const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs';  // NormMaster Documents
+const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT';  // Spectrum files
+// const host = 'http://localhost:3000';
+const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
 let data = [];  // metadata contents
 let materials = {};
 let samples = [];
 let normMaster = {};
+let sampleDevices = {};

-// TODO: integrate measurement device information from DPT names using different users
-// TODO: supplier: other for supplierless samples
 // TODO: BASF twice, BASF as color
-// TODO: trim color names
 // TODO: duplicate kf values
+// TODO: conditions
+// TODO: comment and reference handling
+
+
+// TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC

 main();

 async function main() {
  if (0) {  // materials
    await getNormMaster();
-    await importCsv();
+    await importCsv(metaDoc);
+    await allMaterials();
+    await saveMaterials();
+    await importCsv(kfDoc);
+    await allMaterials();
+    await saveMaterials();
+    await importCsv(vzDoc);
    await allMaterials();
-    fs.writeFileSync('./data_import/materials.json', JSON.stringify(materials));
    await saveMaterials();
  }
-  else if (0) {  // samples
-    await importCsv();
+  if (0) {  // samples
+    sampleDeviceMap();
+    if (1) {
+      console.log('-------- META ----------');
+      await importCsv(metaDoc);
      await allSamples();
      await saveSamples();
    }
-  else if (1) {  // DPT
-    await allDpts();
-  }
-  else if (0) {  // KF/VZ
-    await importCsv();
+    if (1) {
+      console.log('-------- KF ----------');
+      await importCsv(kfDoc);
+      await allSamples();
+      await saveSamples();
      await allKfVz();
    }
-  else if (0) {  // pdf test
-    console.log(await readPdf('N28_BN22-O010_2018-03-08.pdf'));
+    if (1) {
+      console.log('-------- VZ ----------');
+      await importCsv(vzDoc);
+      await allSamples();
+      await saveSamples();
+      await allKfVz();
+    }
+  }
+  if (1) {  // DPT
+    await allDpts();
+  }
+  if (0) {  // pdf test
+    console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf'));
  }
 }

-async function importCsv() {
+async function importCsv(doc) {
+  data = [];
  await new Promise(resolve => {
-    fs.createReadStream(metadata)
+    fs.createReadStream(doc)
      .pipe(iconv.decodeStream('win1252'))
      .pipe(csv())
      .on('data', (row) => {
@@ -57,6 +84,9 @@ async function importCsv() {
      })
      .on('end', () => {
        console.info('CSV file successfully processed');
+        if (data[0]['Farbe']) {  // fix German column names
+          data.map(e => {e['Color'] = e['Farbe']; return e; });
+        }
        resolve();
      });
  });
@@ -65,7 +95,7 @@ async function importCsv() {
 async function allDpts() {
  let res = await axios({
    method: 'get',
-    url: 'http://localhost:3000/template/measurements',
+    url: host + '/template/measurements',
    auth: {
      username: 'admin',
      password: 'Abc123!#'
@@ -74,7 +104,7 @@ async function allDpts() {
  const measurement_template = res.data.find(e => e.name === 'spectrum')._id;
  res = await axios({
    method: 'get',
-    url: 'http://localhost:3000/samples?status=all',
+    url: host + '/samples?status=all',
    auth: {
      username: 'admin',
      password: 'Abc123!#'
@@ -84,10 +114,10 @@ async function allDpts() {
  res.data.forEach(sample => {
    sampleIds[sample.number] = sample._id;
  });
-  const regex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/;
+  const dptRegex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/;
  const dpts = fs.readdirSync(dptFiles);
  for (let i in dpts) {
-    const regexRes = regex.exec(dpts[i])
+    const regexRes = dptRegex.exec(dpts[i])
    if (regexRes && sampleIds[regexRes[1]]) {  // found matching sample
      console.log(dpts[i]);
      const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8');
@@ -99,7 +129,7 @@ async function allDpts() {
      data.values.dpt = f.split('\r\n').map(e => e.split(','));
      await axios({
        method: 'post',
-        url: 'http://localhost:3000/measurement/new',
+        url: host + '/measurement/new',
        auth: {
          username: 'admin',
          password: 'Abc123!#'
@@ -110,13 +140,16 @@ async function allDpts() {
        console.error(err.response.data);
      });
    }
+    else {
+      console.log(`Could not find sample for ${dpts[i]} !!!!!!`);
+    }
  }
 }

 async function allKfVz() {
  let res = await axios({
    method: 'get',
-    url: 'http://localhost:3000/template/measurements',
+    url: host + '/template/measurements',
    auth: {
      username: 'admin',
      password: 'Abc123!#'
@@ -126,7 +159,7 @@ async function allKfVz() {
  const vz_template = res.data.find(e => e.name === 'vz')._id;
  res = await axios({
    method: 'get',
-    url: 'http://localhost:3000/samples?status=all',
+    url: host + '/samples?status=all',
    auth: {
      username: 'admin',
      password: 'Abc123!#'
@@ -140,13 +173,17 @@ async function allKfVz() {
    console.info(`${index}/${data.length}`);
    let sample = data[index];
    if (sample['Sample number'] !== '') {
+      let credentials = ['admin', 'Abc123!#'];
+      if (sampleDevices[sample['Sample number']]) {
+        credentials = [sampleDevices[sample['Sample number']], '2020DeFinMachen!']
+      }
      if (sample['KF in Gew%']) {
        await axios({
          method: 'post',
-          url: 'http://localhost:3000/measurement/new',
+          url: host + '/measurement/new',
          auth: {
-            username: 'admin',
-            password: 'Abc123!#'
+            username: credentials[0],
+            password: credentials[1]
          },
          data: {
            sample_id: sampleIds[sample['Sample number']],
@@ -164,10 +201,10 @@ async function allKfVz() {
      if (sample['VZ (ml/g)']) {
        await axios({
          method: 'post',
-          url: 'http://localhost:3000/measurement/new',
+          url: host + '/measurement/new',
          auth: {
-            username: 'admin',
-            password: 'Abc123!#'
+            username: credentials[0],
+            password: credentials[1]
          },
          data: {
            sample_id: sampleIds[sample['Sample number']],
@@ -186,9 +223,10 @@ async function allKfVz() {
 }

 async function allSamples() {
+  samples = [];
  let res = await axios({
    method: 'get',
-    url: 'http://localhost:3000/materials?status=all',
+    url: host + '/materials?status=all',
    auth: {
      username: 'admin',
      password: 'Abc123!#'
@@ -200,7 +238,7 @@ async function allSamples() {
  })
  res = await axios({
    method: 'get',
-    url: 'http://localhost:3000/samples?status=all',
+    url: host + '/samples?status=all',
    auth: {
      username: 'admin',
      password: 'Abc123!#'
@@ -215,7 +253,13 @@ async function allSamples() {
  for (let index in data) {
    console.info(`${index}/${data.length}`);
    let sample = data[index];
-    if (sample['Sample number'] !== '' && sample['Supplier'] !== '' && sample['Granulate/Part'] !== '') {  // TODO: wait for decision about samples without suppliers/color/type
+    if (sample['Sample number'] !== '') {  // TODO: what about samples without color
+      if (sample['Supplier'] === '') {  // empty supplier fields
+        sample['Supplier'] = 'unknown';
+      }
+      if (sample['Granulate/Part'] === '') {  // empty supplier fields
+        sample['Granulate/Part'] = 'unknown';
+      }
      const material = dbMaterials[trim(sample['Material name'])];
      if (!material) { // could not find material, skipping sample
        continue;
@@ -236,13 +280,20 @@ async function allSamples() {
        samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
      }
      else if (sample['Color'] && sample['Color'] !== '') {
-        samples[si].color = material.numbers.find(e => e.color.indexOf(sample['Color']) >= 0).color;
+        let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0);
+        if (!number && /black/.test(sample['Color'])) {  // special case bk for black
+          number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
+          if (!number) {  // try German word
+            number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
+          }
+        }
+        samples[si].color = number.color;
      }
      else if (sampleColors[sample['Sample number'].split('_')[0]]) {  // derive color from main sample for kf/vz
        samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
      }
-      else {  // TODO: no color information at all
-        samples.pop();
+      else {
+        samples[si].color = '';
      }
    }
  }
@@ -251,41 +302,57 @@ async function allSamples() {
 async function saveSamples() {
  for (let i in samples) {
    console.info(`${i}/${samples.length}`);
+    let credentials = ['admin', 'Abc123!#'];
+    if (sampleDevices[samples[i].number]) {
+      credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!']
+    }
    await axios({
      method: 'post',
-      url: 'http://localhost:3000/sample/new',
+      url: host + '/sample/new',
      auth: {
-        username: 'admin',
-        password: 'Abc123!#'
+        username: credentials[0],
+        password: credentials[1]
      },
      data: samples[i]
    }).catch(err => {
+      if (err.response.data.status && err.response.data.status !== 'Sample number already taken') {
        console.log(samples[i]);
        console.error(err.response.data);
+      }
    });
  }
  console.info('saved all samples');
 }

 async function allMaterials() {
+  materials = {};
  for (let index in data) {
    let sample = data[index];
-    if (sample['Sample number'] !== '' && sample['Supplier'] !== '') {  // TODO: wait for decision about supplierless samples
+    if (sample['Sample number'] && sample['Sample number'] !== '') {
+      if (sample['Supplier'] === '') {  // empty supplier fields
+        sample['Supplier'] = 'unknown';
+      }
+      if (sample['Material name'] === '') {  // empty name fields
+        sample['Material name'] = sample['Material'];
+      }
+      if (!sample['Material']) {  // column Material is named Plastic in VZ metadata
+        sample['Material'] = sample['Plastic'];
+      }
      sample['Material name'] = trim(sample['Material name']);
      if (materials.hasOwnProperty(sample['Material name'])) {  // material already found at least once
-        if (sample['Material number'] !== '') {
+        if (sample['Material number'] && sample['Material number'] !== '') {
          if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) {  // new material number
            if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) {  // color already in list, only number missing
              materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']);
            }
            else {
-              materials[sample['Material name']].numbers.push({color: sample['Color'], number: stripSpaces(sample['Material number'])});
+              materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])});
            }
          }
        }
-        else if (sample['Color'] !== '') {
+        else if (sample['Color'] && sample['Color'] !== '') {
          if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) {  // new material color
-            materials[sample['Material name']].numbers.push({color: sample['Color'], number: ''});
+            materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''});
          }
        }
      }
@@ -293,8 +360,8 @@ async function allMaterials() {
        console.info(`${index}/${data.length}  ${sample['Material name']}`);
        materials[sample['Material name']] = {
          name: sample['Material name'],
-          supplier: sample['Supplier'],
-          group: sample['Material']
+          supplier: trim(sample['Supplier']),
+          group: trim(sample['Material'])
        };
        let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
        materials[sample['Material name']].mineral = tmp ? tmp[1] : 0;
@@ -312,17 +379,20 @@ async function allMaterials() {
 async function saveMaterials() {
  const mKeys = Object.keys(materials)
  for (let i in mKeys) {
+    console.info(`${i}/${mKeys.length}`);
    await axios({
      method: 'post',
-      url: 'http://localhost:3000/material/new',
+      url: host + '/material/new',
      auth: {
        username: 'admin',
        password: 'Abc123!#'
      },
      data: materials[mKeys[i]]
    }).catch(err => {
-      console.log(materials[mKeys[i]]);
+      if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
+        console.info(materials[mKeys[i]]);
        console.error(err.response.data);
+      }
    });
  }
  console.info('saved all materials');
@@ -362,16 +432,16 @@ async function numbersFetch(sample) {
    }
  }
  if (res.length === 0) {  // no results
-    if (sample['Color'] !== '' || sample['Material number'] !== '') {
-      return [{color: sample['Color'], number: sample['Material number']}];
+    if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) {
+      return [{color: trim(sample['Color']), number: sample['Material number']}];
    }
    else {
      return [];
    }
  }
  else {
-    if (!res.find(e => e.number === sample['Material number'])) {  // sometimes norm master does not include sample number even if listed
-      res.push({color: sample['Color'], number: sample['Material number']});
+    if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) {  // sometimes norm master does not include sample number even if listed
+      res.push({color: trim(sample['Color']), number: sample['Material number']});
    }
    return res;
  }
@@ -403,7 +473,7 @@ async function getNormMaster(fetchAgain = false) {
 }

 function getNormMasterDoc(url, timing = 1) {
-  console.log(url);
+  console.info(url);
  return new Promise(async resolve => {
    const options = new chrome.Options();
    options.setUserPreferences({
@@ -453,7 +523,7 @@ function readPdf(file) {
            rows.push(item.text);
          }
          else {  // still the same row row
-            rows[rows.length - 1] += (item.x - lastX > 1.1 ? '$' : '') + item.text;  // push to row, detect if still same cell
+            rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text;  // push to row, detect if still same cell
          }
          lastX = (item.w * 0.055) + item.x;

@@ -465,7 +535,7 @@ function readPdf(file) {
            table = -1;
            // console.log(rows);
            rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e)));  // filter non-table rows
-            resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; }));
+            resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
          }
        }
        lastLastText = lastText;
@@ -473,12 +543,23 @@ function readPdf(file) {
      }
      if (!item && table !== -1) {  // document ended
        rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e)));  // filter non-table rows
-        resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; }));
+        resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
      }
    });
  });
 }

+function sampleDeviceMap() {
+  const dpts = fs.readdirSync(dptFiles);
+  const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/;
+  for (let i in dpts) {
+    const regexRes = regex.exec(dpts[i])
+    if (regexRes) {  // found matching sample
+      sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase();
+    }
+  }
+}
+
 function stripSpaces(s) {
  return s ? s.replace(/ /g,'') : '';
 }
--- a/manifest.yml
+++ b/manifest.yml
@@ -1,6 +1,7 @@
 ---
 applications:
-  - name: digital-fingerprint-of-plastics-api
+  - name: definma-api
+    path: dist/
    instances: 1
    memory: 256M
    stack: cflinuxfs3
@@ -10,4 +11,4 @@ applications:
      NODE_ENV: production
      OPTIMIZE_MEMORY: true
    services:
-      - dfopdb
+      - definmadb
--- a/package-lock.json
+++ b/package-lock.json
@@ -1174,6 +1174,15 @@
      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
      "dev": true
    },
+    "cors": {
+      "version": "2.8.5",
+      "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
+      "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
+      "requires": {
+        "object-assign": "^4",
+        "vary": "^1"
+      }
+    },
    "cross-spawn": {
      "version": "7.0.3",
      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -2866,6 +2875,11 @@
        }
      }
    },
+    "object-assign": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
+    },
    "object-inspect": {
      "version": "1.7.0",
      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz",
--- a/package.json
+++ b/package.json
@@ -7,6 +7,7 @@
    "tsc": "tsc",
    "tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc",
    "build": "build.bat",
+    "build-push": "build.bat && cf push",
    "test": "mocha dist/**/**.spec.js",
    "start": "node index.js",
    "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"",
@@ -28,6 +29,7 @@
    "cfenv": "^1.2.2",
    "compression": "^1.7.4",
    "content-filter": "^1.1.2",
+    "cors": "^2.8.5",
    "express": "^4.17.1",
    "helmet": "^3.22.0",
    "json-schema": "^0.2.5",
@@ -35,7 +37,7 @@
    "lodash": "^4.17.15",
    "mongo-sanitize": "^1.1.0",
    "mongoose": "^5.8.7",
-    "swagger-ui-express": "^4.1.2"
+    "swagger-ui-express": "4.1.2"
  },
  "devDependencies": {
    "@types/bcrypt": "^3.0.0",
--- a/src/api.ts
+++ b/src/api.ts
@@ -18,6 +18,7 @@ export default class api {
    jsonRefParser.bundle('api/api.yaml', (err, doc) => {  // parse yaml
      if (err) throw err;
      apiDoc = doc;
+      apiDoc.servers.splice(process.env.NODE_ENV === 'production', 1);
      apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e));  // bundle routes
      apiDoc = this.resolveXDoc(apiDoc);
      oasParser.validate(apiDoc, (err, api) => {  // validate oas schema
--- a/src/helpers/csv.ts
+++ b/src/helpers/csv.ts
@@ -1,12 +1,6 @@
 import {parseAsync} from 'json2csv';

 export default function csv(input: any[], f: (err, data) => void) {
-  console.log(input[1000]);
-  console.log(flatten(input[1000]));
-  parseAsync([flatten(input[1000])]).then(csv => console.log(csv));
-  console.log(input[1]);
-  console.log(flatten(input[1]));
-  parseAsync([flatten(input[1])]).then(csv => console.log(csv));
  parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
    .then(csv => f(null, csv))
    .catch(err => f(err, null));
--- a/src/helpers/mail.ts
+++ b/src/helpers/mail.ts
@@ -17,7 +17,7 @@ export default (mailAddress, subject, content, f) => {  // callback, executed em
          contentType: "text/html"
        },
        from: {
-          eMail: "dfop@bosch-iot.com",
+          eMail: "definma@bosch-iot.com",
          password: "PlasticsOfFingerprintDigital"
        }
      }
--- a/src/index.ts
+++ b/src/index.ts
@@ -4,6 +4,7 @@ import compression from 'compression';
 import contentFilter from 'content-filter';
 import mongoSanitize from 'mongo-sanitize';
 import helmet from 'helmet';
+import cors from 'cors';
 import api from './api';
 import db from './db';

@@ -42,9 +43,11 @@ app.use((req, res, next) => {  // no database connection error
    next();
  }
  else {
+    console.error('No database connection');
    res.status(500).send({status: 'Internal server error'});
  }
 });
+app.use(cors());  // CORS headers
 app.use(require('./helpers/authorize'));  // handle authentication

 // redirect /api routes for Angular proxy in development
--- a/src/routes/sample.spec.ts
+++ b/src/routes/sample.spec.ts
@@ -21,6 +21,7 @@ describe('/sample', () => {

  // TODO: sort, added date filter, has measurements/condition filter
  // TODO: check if conditions work in sort/fields/filters
+  // TODO: test for numbers as strings in glass_fiber
  describe('GET /samples', () => {
    it('returns all samples', done => {
      TestHelper.request(server, done, {
--- a/src/routes/sample.ts
+++ b/src/routes/sample.ts
@@ -21,6 +21,12 @@ import csv from '../helpers/csv';

 const router = express.Router();

+// TODO: check added filter
+// TODO: return total number of pages -> use facet
+// TODO: use query pointer
+// TODO: convert filter value to number according to table model
+// TODO: validation for filter parameters
+// TODO: location/device sort/filter
 router.get('/samples', async (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

@@ -37,6 +43,7 @@ router.get('/samples', async (req, res, next) => {
  if (!filters['to-page']) {  // set to-page default
    filters['to-page'] = 0;
  }
+  console.log(filters);

  const sortFilterKeys = filters.filters.map(e => e.field);

@@ -70,7 +77,7 @@ router.get('/samples', async (req, res, next) => {
      {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}},                                  // fetch samples and restructure them to fit sample structure
      {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
      {$match: statusQuery(filters, 'sample.status')},  // filter out wrong status once samples were added
-      {$set: {['sample.' + measurementName]: '$measurement.values'}},  // more restructuring
+      {$addFields: {['sample.' + measurementName]: '$measurement.values'}},  // more restructuring
      {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
    );
    addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0));  // sample filters
@@ -106,25 +113,25 @@ router.get('/samples', async (req, res, next) => {
    materialAdded = true;
    materialQuery.push(  // add material properties
      {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},  // TODO: project out unnecessary fields
-      {$set: {material: {$arrayElemAt: ['$material', 0]}}}
+      {$addFields: {material: {$arrayElemAt: ['$material', 0]}}}
    );
    const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
    addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0));  // base material filters
    if (sortFilterKeys.find(e => e === 'material.supplier')) {  // add supplier if needed
      materialQuery.push(
        {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
-        {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
+        {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
      );
    }
    if (sortFilterKeys.find(e => e === 'material.group')) {  // add group if needed
      materialQuery.push(
        {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
-        {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
+        {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
      );
    }
    if (sortFilterKeys.find(e => e === 'material.number')) {  // add material number if needed
      materialQuery.push(
-        {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
+        {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
      );
    }
    const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
@@ -157,10 +164,10 @@ router.get('/samples', async (req, res, next) => {
        as: 'measurements'
    }});
    measurementTemplates.forEach(template => {
-      query.push({$set: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
+      query.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
        vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
        in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
-      }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
+      }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
    });
    addFilterQueries(query, filters.filters
      .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
@@ -173,39 +180,40 @@ router.get('/samples', async (req, res, next) => {
    sortFilterKeys.indexOf(e) < 0                      // field was not in filter
    && e !== filters.sort[0]                           // field was not in sort
  );
+  console.log(fieldsToAdd);

  if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) {  // add material, was not added already
    query.push(
      {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
-      {$set: {material: { $arrayElemAt: ['$material', 0]}}}
+      {$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
    );
  }
  if (fieldsToAdd.indexOf('material.supplier') >= 0) {  // add supplier if needed
    query.push(
      {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
-      {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
+      {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
    );
  }
  if (fieldsToAdd.indexOf('material.group') >= 0) {  // add group if needed
    query.push(
      {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
-      {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
+      {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
    );
  }
  if (fieldsToAdd.indexOf('material.number') >= 0) {  // add material number if needed
    query.push(
-      {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
+      {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
    );
  }

-  let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]));  // filter measurement names and remove duplicates from parameters
+  let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]));  // filter measurement names and remove duplicates from parameters
  if (fieldsToAdd.find(e => /measurements\./.test(e))) {  // add measurement fields
    const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
    if (measurementTemplates instanceof Error) return;
    if (measurementTemplates.length < measurementFieldsFields.length) {
      return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
    }
-    if (fieldsToAdd.find(e => e === 'measurements.spectrum')) {  // use different lookup methods with and without spectrum for the best performance
+    if (fieldsToAdd.find(e => /spectrum\./.test(e))) {  // use different lookup methods with and without spectrum for the best performance
      query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
    }
    else {
@@ -216,15 +224,15 @@ router.get('/samples', async (req, res, next) => {
        }});
    }
    measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => {  // TODO: hard coded dpt for special treatment, change later
-      query.push({$set: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
+      query.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
              vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
              in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
-            }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
+            }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
    });
    if (measurementFieldsFields.find(e => e === 'spectrum')) {  // TODO: remove hardcoded as well
      query.push(
-        {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
-        {$set: {spectrum: '$spectrum.values.dpt'}},
+        {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
+        {$addFields: {spectrum: '$spectrum.values'}},
        {$unwind: '$spectrum'}
      );
    }
@@ -233,19 +241,22 @@ router.get('/samples', async (req, res, next) => {

  const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
  if (filters.fields.indexOf('added') >= 0) {  // add added date
-    projection.added = {$toDate: '$_id'};
+    // projection.added = {$toDate: '$_id'};
+    // projection.added = { $convert: { input: '$_id', to: "date" } }  // TODO
  }
  if (!(filters.fields.indexOf('_id') >= 0)) {  // disable _id explicitly
    projection._id = false;
  }
  query.push({$project: projection});

+  if (!fieldsToAdd.find(e => /spectrum\./.test(e))) {  // use streaming when including spectrum files
    collection.aggregate(query).exec((err, data) => {
      if (err) return next(err);
+      console.log(data.length);
      if (filters['to-page'] < 0) {
        data.reverse();
      }
-    const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]);
+      const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]);
      if (filters.csv) {  // output as csv
        csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
          if (err) return next(err);
@@ -256,7 +267,19 @@ router.get('/samples', async (req, res, next) => {
      else {
        res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));  // validate all and filter null values from validation errors
      }
-  })
+    });
+  }
+  else {
+    res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
+    res.write('[');
+    let count = 0;
+    const stream = collection.aggregate(query).cursor().exec();
+    stream.on('data', data => { res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; });
+    stream.on('close', () => {
+      res.write(']');
+      res.end();
+    });
+  }
 });

 router.get('/samples/:state(new|deleted)', (req, res, next) => {
@@ -537,7 +560,7 @@ async function materialCheck (sample, res, next, id = sample.material_id) {  //
    res.status(400).json({status: 'Material not available'});
    return false;
  }
-  if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) {  // color for material not specified
+  if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) {  // color for material not specified
    res.status(400).json({status: 'Color not available for material'});
    return false;
  }
--- a/src/routes/validate/sample.ts
+++ b/src/routes/validate/sample.ts
@@ -11,7 +11,8 @@ export default class SampleValidate {
      .max(128),

    color: Joi.string()
-      .max(128),
+      .max(128)
+      .allow(''),

    type: Joi.string()
      .max(128),
@@ -77,7 +78,7 @@ export default class SampleValidate {
    'user_id',
    'material._id',
    'material.numbers',
-    'measurements.spectrum'
+    'measurements.spectrum.dpt'
  ];

  static input (data, param) {  // validate input, set param to 'new' to make all attributes required
@@ -170,6 +171,33 @@ export default class SampleValidate {
      try {
        for (let i in data.filters) {
          data.filters[i] = JSON.parse(data.filters[i]);
+          data.filters[i].values = data.filters[i].values.map(e => {  // validate filter values
+            let validator;
+            let field = data.filters[i].field
+            if (/material\./.test(field)) {  // select right validation model
+              validator = MaterialValidate.outputV();
+              field = field.replace('material.', '');
+            }
+            else if (/measurements\./.test(field)) {
+              validator = Joi.object({
+                value: Joi.alternatives()
+                  .try(
+                    Joi.string().max(128),
+                    Joi.number(),
+                    Joi.boolean(),
+                    Joi.array()
+                  )
+                  .allow(null)
+              });
+              field = 'value';
+            }
+            else {
+              validator = Joi.object(this.sample);
+            }
+            const {value, error} = validator.validate({[field]: e});
+            if (error) throw error;  // reject invalid values
+            return value[field];
+          });
        }
      }
      catch {