Merge pull request #14 in ~VLE2FE/dfop-api from sample-filters to develop

* commit '3dda3d77a13f84a40f8132f49f4ad704f2810a75':
  minor fixes
  implemented added filters
  added workaround for 'added' field compatible to MongoDB 3.6
  implemented x-total-items header
  spectrum field working again
  reworked filters
  added filters
  restructured aggregation
  implementation of measurement fields
  first implementation of fields
  base for csv export
  switched to aggregation, included material sort keys
  sorting for direct sample properties added
  added /samples/count
  changed last-id behaviour to from-id
  implemented paging
  fixed validation to return measurements in /sample/{id}
  added status filter for materials
  added status filter

api/api.yaml

@@ -39,10 +39,10 @@ info:
 
 
 servers:
+  - url: https://definma-api.apps.de1.bosch-iot-cloud.com
+    description: server on the BIC
   - url: http://localhost:3000
     description: local server
-  - url: https://digital-fingerprint-of-plastics-api.apps.de1.bosch-iot-cloud.com/
-    description: server on the BIC
 
 
 security:

api/material.yaml

@@ -5,6 +5,13 @@
     x-doc: returns only materials with status 10
     tags:
       - /material
+    parameters:
+      - name: status
+        description: 'values: validated|new|all, defaults to validated'
+        in: query
+        schema:
+          type: string
+        example: all
     responses:
       200:
         description: all material details

api/sample.yaml

@@ -2,20 +2,83 @@
   get:
     summary: all samples in overview
     description: 'Auth: all, levels: read, write, maintain, dev, admin'
-    x-doc: returns only samples with status 10
+    x-doc: 'Limitations: paging and csv output does not work when including the spectrum measurement fields as well as the returned number of total samples'
     tags:
       - /sample
+    parameters:
+      - name: status
+        description: 'values: validated|new|all, defaults to validated'
+        in: query
+        schema:
+          type: string
+        example: all
+      - name: from-id
+        description: first id of the requested page, if not given the results are displayed from start
+        in: query
+        schema:
+          type: string
+        example: 5ea0450ed851c30a90e70894
+      - name: to-page
+        description: relative change of pages, use negative values to get back, defaults to 0, works only together with page-size
+        in: query
+        schema:
+          type: string
+        example: 1
+      - name: page-size
+        description: number of items per page
+        in: query
+        schema:
+          type: string
+        example: 30
+      - name: sort
+        description: sorting of results, in format 'key-asc/desc'
+        in: query
+        schema:
+          type: string
+        example: color-asc
+      - name: csv
+        description: output as csv
+        in: query
+        schema:
+          type: boolean
+        example: false
+      - name: fields[]
+        description: the fields to include in the output as array, defaults to ['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']
+        in: query
+        schema:
+         type: array
+         items:
+           type: string
+        example: ['number', 'batch']
+      - name: filters[]
+        description: "the filters to apply as an array of URIComponent encoded objects in the form {mode: 'eq/ne/lt/lte/gt/gte/in/nin', field: 'material.m', values: ['15']} using encodeURIComponent(JSON.stringify({}))"
+        in: query
+        schema:
+         type: array
+         items:
+           type: string
+        example: ["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D", "%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22%5D%7D"]
     responses:
       200:
-        description: samples overview
+        description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)
+        headers:
+          x-total-items:
+            description: Total number of available items when from-id is not specified and spectrum field is not included
+            schema:
+              type: integer
+              example: 243
         content:
           application/json:
             schema:
               type: array
               items:
                 $ref: 'api.yaml#/components/schemas/SampleRefs'
+      400:
+        $ref: 'api.yaml#/components/responses/400'
       401:
         $ref: 'api.yaml#/components/responses/401'
+      404:
+        $ref: 'api.yaml#/components/responses/404'
       500:
         $ref: 'api.yaml#/components/responses/500'
 
@@ -42,12 +105,31 @@
       500:
         $ref: 'api.yaml#/components/responses/500'
 
+/samples/count:
+  get:
+    summary: total number of samples
+    description: 'Auth: all, levels: read, write, maintain, dev, admin'
+    tags:
+      - /sample
+    responses:
+      200:
+        description: sample count
+        content:
+          application/json:
+            schema:
+              properties:
+                count:
+                  type: number
+                  example: 864
+      500:
+        $ref: 'api.yaml#/components/responses/500'
+
 /sample/{id}:
   parameters:
     - $ref: 'api.yaml#/components/parameters/Id'
   get:
-    summary: TODO sample details
+    summary: sample details
-    description: 'Auth: all, levels: read, write, maintain, dev, admin'
+    description: 'Auth: all, levels: read, write, maintain, dev, admin<br>Returns validated as well as new measurements'
     x-doc: deleted samples are available only for maintain/admin
     tags:
       - /sample
@@ -216,12 +298,14 @@
         content:
           application/json:
             schema:
-              properties:
+              type: array
-                name:
+              items:
-                  type: string
+                properties:
-                qty:
+                  name:
-                  type: number
+                    type: string
-                  example: 20
+                  qty:
+                    type: number
+                    example: 20
       401:
         $ref: 'api.yaml#/components/responses/401'
       500:

api/schemas.yaml

@@ -46,6 +46,9 @@ SampleRefs:
       $ref: 'api.yaml#/components/schemas/Id'
     user_id:
       $ref: 'api.yaml#/components/schemas/Id'
+    added:
+      type: string
+      example: 1970-01-01T00:00:00.000Z
 Sample:
   allOf:
     - $ref: 'api.yaml#/components/schemas/_Id'
@@ -69,6 +72,8 @@ Sample:
               relation:
                 type: string
                 example: part to this sample
+        custom_fields:
+          type: object
 
 SampleDetail:
   allOf:

data_import/import.js

@@ -0,0 +1,579 @@
+const csv = require('csv-parser');
+const fs = require('fs');
+const axios = require('axios');
+const {Builder} = require('selenium-webdriver');    // selenium and the chrome driver must be installed and configured separately
+const chrome = require('selenium-webdriver/chrome');
+const pdfReader = require('pdfreader');
+const iconv = require('iconv-lite');
+
+const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv';  // metadata files
+const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
+const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv';
+const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs';  // NormMaster Documents
+const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT';  // Spectrum files
+const host = 'http://localhost:3000';
+// const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
+let data = [];  // metadata contents
+let materials = {};
+let samples = [];
+let normMaster = {};
+let sampleDevices = {};
+
+// TODO: BASF twice, BASF as color
+// TODO: duplicate kf values
+// TODO: conditions
+// TODO: comment and reference handling
+
+
+// TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC
+
+main();
+
+async function main() {
+  if (0) {  // materials
+    await getNormMaster();
+    await importCsv(metaDoc);
+    await allMaterials();
+    await saveMaterials();
+    await importCsv(kfDoc);
+    await allMaterials();
+    await saveMaterials();
+    await importCsv(vzDoc);
+    await allMaterials();
+    await saveMaterials();
+  }
+  if (0) {  // samples
+    sampleDeviceMap();
+    if (1) {
+      console.log('-------- META ----------');
+      await importCsv(metaDoc);
+      await allSamples();
+      await saveSamples();
+    }
+    if (1) {
+      console.log('-------- KF ----------');
+      await importCsv(kfDoc);
+      await allSamples();
+      await saveSamples();
+      await allKfVz();
+    }
+    if (1) {
+      console.log('-------- VZ ----------');
+      await importCsv(vzDoc);
+      await allSamples();
+      await saveSamples();
+      await allKfVz();
+    }
+  }
+  if (1) {  // DPT
+    await allDpts();
+  }
+  if (0) {  // pdf test
+    console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf'));
+  }
+}
+
+async function importCsv(doc) {
+  data = [];
+  await new Promise(resolve => {
+    fs.createReadStream(doc)
+      .pipe(iconv.decodeStream('win1252'))
+      .pipe(csv())
+      .on('data', (row) => {
+        data.push(row);
+      })
+      .on('end', () => {
+        console.info('CSV file successfully processed');
+        if (data[0]['Farbe']) {  // fix German column names
+          data.map(e => {e['Color'] = e['Farbe']; return e; });
+        }
+        resolve();
+      });
+  });
+}
+
+async function allDpts() {
+  let res = await axios({
+    method: 'get',
+    url: host + '/template/measurements',
+    auth: {
+      username: 'admin',
+      password: 'Abc123!#'
+    }
+  });
+  const measurement_template = res.data.find(e => e.name === 'spectrum')._id;
+  res = await axios({
+    method: 'get',
+    url: host + '/samples?status=all',
+    auth: {
+      username: 'admin',
+      password: 'Abc123!#'
+    }
+  });
+  const sampleIds = {};
+  res.data.forEach(sample => {
+    sampleIds[sample.number] = sample._id;
+  });
+  const dptRegex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/;
+  const dpts = fs.readdirSync(dptFiles);
+  for (let i in dpts) {
+    const regexRes = dptRegex.exec(dpts[i])
+    if (regexRes && sampleIds[regexRes[1]]) {  // found matching sample
+      console.log(dpts[i]);
+      const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8');
+      const data = {
+        sample_id: sampleIds[regexRes[1]],
+        values: {},
+        measurement_template
+      };
+      data.values.dpt = f.split('\r\n').map(e => e.split(','));
+      let rescale = false;
+      for (let i in data.values.dpt) {
+        if (data.values.dpt[i][1] > 2) {
+          rescale = true;
+          break;
+        }
+      }
+      if (rescale) {
+        data.values.dpt = data.values.dpt.map(e => [e[0], e[1] / 100]);
+      }
+      await axios({
+        method: 'post',
+        url: host + '/measurement/new',
+        auth: {
+          username: 'admin',
+          password: 'Abc123!#'
+        },
+        data
+      }).catch(err => {
+        console.log(dpts[i]);
+        console.error(err.response.data);
+      });
+    }
+    else {
+      console.log(`Could not find sample for ${dpts[i]} !!!!!!`);
+    }
+  }
+}
+
+async function allKfVz() {
+  let res = await axios({
+    method: 'get',
+    url: host + '/template/measurements',
+    auth: {
+      username: 'admin',
+      password: 'Abc123!#'
+    }
+  });
+  const kf_template = res.data.find(e => e.name === 'kf')._id;
+  const vz_template = res.data.find(e => e.name === 'vz')._id;
+  res = await axios({
+    method: 'get',
+    url: host + '/samples?status=all',
+    auth: {
+      username: 'admin',
+      password: 'Abc123!#'
+    }
+  });
+  const sampleIds = {};
+  res.data.forEach(sample => {
+    sampleIds[sample.number] = sample._id;
+  });
+  for (let index in data) {
+    console.info(`${index}/${data.length}`);
+    let sample = data[index];
+    if (sample['Sample number'] !== '') {
+      let credentials = ['admin', 'Abc123!#'];
+      if (sampleDevices[sample['Sample number']]) {
+        credentials = [sampleDevices[sample['Sample number']], '2020DeFinMachen!']
+      }
+      if (sample['KF in Gew%']) {
+        await axios({
+          method: 'post',
+          url: host + '/measurement/new',
+          auth: {
+            username: credentials[0],
+            password: credentials[1]
+          },
+          data: {
+            sample_id: sampleIds[sample['Sample number']],
+            measurement_template: kf_template,
+            values: {
+              'weight %': sample['KF in Gew%'],
+              'standard deviation': sample['Stabwn']
+            }
+          }
+        }).catch(err => {
+          console.log(sample['Sample number']);
+          console.error(err.response.data);
+        });
+      }
+      if (sample['VZ (ml/g)']) {
+        await axios({
+          method: 'post',
+          url: host + '/measurement/new',
+          auth: {
+            username: credentials[0],
+            password: credentials[1]
+          },
+          data: {
+            sample_id: sampleIds[sample['Sample number']],
+            measurement_template: vz_template,
+            values: {
+              vz: sample['VZ (ml/g)']
+            }
+          }
+        }).catch(err => {
+          console.log(sample['Sample number']);
+          console.error(err.response.data);
+        });
+      }
+    }
+  }
+}
+
+async function allSamples() {
+  samples = [];
+  let res = await axios({
+    method: 'get',
+    url: host + '/materials?status=all',
+    auth: {
+      username: 'admin',
+      password: 'Abc123!#'
+    }
+  });
+  const dbMaterials = {}
+  res.data.forEach(m => {
+    dbMaterials[m.name] = m;
+  })
+  res = await axios({
+    method: 'get',
+    url: host + '/samples?status=all',
+    auth: {
+      username: 'admin',
+      password: 'Abc123!#'
+    }
+  });
+  const sampleColors = {};
+  res.data.forEach(sample => {
+    sampleColors[sample.number] = sample.color;
+  });
+
+
+  for (let index in data) {
+    console.info(`${index}/${data.length}`);
+    let sample = data[index];
+    if (sample['Sample number'] !== '') {  // TODO: what about samples without color
+      if (sample['Supplier'] === '') {  // empty supplier fields
+        sample['Supplier'] = 'unknown';
+      }
+      if (sample['Granulate/Part'] === '') {  // empty supplier fields
+        sample['Granulate/Part'] = 'unknown';
+      }
+      const material = dbMaterials[trim(sample['Material name'])];
+      if (!material) { // could not find material, skipping sample
+        continue;
+      }
+      console.log(sample['Material name']);
+      console.log(material._id);
+      samples.push({
+        number: sample['Sample number'],
+        type: sample['Granulate/Part'],
+        batch: sample['Charge/batch granulate/part'] || '',
+        material_id: material._id,
+        notes: {
+          comment: sample['Comments']
+        }
+      });
+      const si = samples.length - 1;
+      if (sample['Material number'] !== '' && material.numbers.find(e => e.number === sample['Material number'])) {  // TODO: fix because of false material/material number
+        samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
+      }
+      else if (sample['Color'] && sample['Color'] !== '') {
+        let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0);
+        if (!number && /black/.test(sample['Color'])) {  // special case bk for black
+          number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
+          if (!number) {  // try German word
+            number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
+          }
+        }
+        samples[si].color = number.color;
+      }
+      else if (sampleColors[sample['Sample number'].split('_')[0]]) {  // derive color from main sample for kf/vz
+        samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
+      }
+      else {
+        samples[si].color = '';
+      }
+    }
+  }
+}
+
+async function saveSamples() {
+  for (let i in samples) {
+    console.info(`${i}/${samples.length}`);
+    let credentials = ['admin', 'Abc123!#'];
+    if (sampleDevices[samples[i].number]) {
+      credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!']
+    }
+    await axios({
+      method: 'post',
+      url: host + '/sample/new',
+      auth: {
+        username: credentials[0],
+        password: credentials[1]
+      },
+      data: samples[i]
+    }).catch(err => {
+      if (err.response.data.status && err.response.data.status !== 'Sample number already taken') {
+        console.log(samples[i]);
+        console.error(err.response.data);
+      }
+    });
+  }
+  console.info('saved all samples');
+}
+
+async function allMaterials() {
+  materials = {};
+  for (let index in data) {
+    let sample = data[index];
+    if (sample['Sample number'] && sample['Sample number'] !== '') {
+      if (sample['Supplier'] === '') {  // empty supplier fields
+        sample['Supplier'] = 'unknown';
+      }
+      if (sample['Material name'] === '') {  // empty name fields
+        sample['Material name'] = sample['Material'];
+      }
+      if (!sample['Material']) {  // column Material is named Plastic in VZ metadata
+        sample['Material'] = sample['Plastic'];
+      }
+      sample['Material name'] = trim(sample['Material name']);
+      if (materials.hasOwnProperty(sample['Material name'])) {  // material already found at least once
+        if (sample['Material number'] && sample['Material number'] !== '') {
+          if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) {  // new material number
+            if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) {  // color already in list, only number missing
+              materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']);
+            }
+            else {
+              materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])});
+            }
+          }
+        }
+        else if (sample['Color'] && sample['Color'] !== '') {
+          if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) {  // new material color
+            materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''});
+          }
+        }
+      }
+      else {  // new material
+        console.info(`${index}/${data.length}  ${sample['Material name']}`);
+        materials[sample['Material name']] = {
+          name: sample['Material name'],
+          supplier: trim(sample['Supplier']),
+          group: trim(sample['Material'])
+        };
+        let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
+        materials[sample['Material name']].mineral = tmp ? tmp[1] : 0;
+        tmp = /GF(\d+)/.exec(sample['Reinforcing material']);
+        materials[sample['Material name']].glass_fiber = tmp ? tmp[1] : 0;
+        tmp = /CF(\d+)/.exec(sample['Reinforcing material']);
+        materials[sample['Material name']].carbon_fiber = tmp ? tmp[1] : 0;
+        materials[sample['Material name']].numbers = await numbersFetch(sample);
+        console.log(materials[sample['Material name']]);
+      }
+    }
+  }
+}
+
+async function saveMaterials() {
+  const mKeys = Object.keys(materials)
+  for (let i in mKeys) {
+    console.info(`${i}/${mKeys.length}`);
+    await axios({
+      method: 'post',
+      url: host + '/material/new',
+      auth: {
+        username: 'admin',
+        password: 'Abc123!#'
+      },
+      data: materials[mKeys[i]]
+    }).catch(err => {
+      if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
+        console.info(materials[mKeys[i]]);
+        console.error(err.response.data);
+      }
+    });
+  }
+  console.info('saved all materials');
+}
+
+async function numbersFetch(sample) {
+  let nm = [];
+  let res = [];
+  if (sample['Material number']) {  // sample has a material number
+    nm = normMaster[stripSpaces(sample['Material number'])]? [normMaster[stripSpaces(sample['Material number'])]] : [];
+  }
+  else {  // try finding via material name
+    nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['Material name'])).map(e => normMaster[e]);
+  }
+  if (nm.length > 0) {
+    for (let i in nm) {
+      // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) {  // document not loaded
+      //   await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'));
+      // }
+      // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) {  // document not loaded
+      //   console.info('Retrying download...');
+      //   await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 2.2);
+      // }
+      // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) {  // document not loaded
+      //   console.info('Retrying download again...');
+      //   await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 5);
+      // }
+      if (fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) {  // document loaded
+        res = await readPdf(fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0));
+      }
+      if (res.length > 0) {  // no results
+        break;
+      }
+      else if (i + 1 >= nm.length) {
+        console.error('Download failed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
+      }
+    }
+  }
+  if (res.length === 0) {  // no results
+    if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) {
+      return [{color: trim(sample['Color']), number: sample['Material number']}];
+    }
+    else {
+      return [];
+    }
+  }
+  else {
+    if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) {  // sometimes norm master does not include sample number even if listed
+      res.push({color: trim(sample['Color']), number: sample['Material number']});
+    }
+    return res;
+  }
+}
+
+async function getNormMaster(fetchAgain = false) {
+  if (fetchAgain) {
+    console.info('fetching norm master...');
+    const res = await axios({
+      method: 'get',
+      url: 'http://rb-normen.bosch.com/cgi-bin/searchRBNorm4TradeName'
+    });
+
+    console.info('finding documents...');
+    let match;
+    // const regex = /<tr>.*?<td>.*?<\/span>(.*?)<\/td><td>(\d+)<\/td>.*?<a href="(.*?)"/gm;
+    const regex = /<tr>.*?<td>.*?<\/span>(.*?)<\/td><td>(\d+)<\/td><td>40.*?<a href="(.*?)".*?<\/a>(.*?)<\/td>/gm;  // only valid materials
+    do {
+      match = regex.exec(res.data);
+      if (match) {
+        normMaster[match[2]] = {name: match[1], nameSpaceless: stripSpaces(match[1]), number: match[2], url: match[3], doc: match[4]};
+      }
+    } while (match);
+    fs.writeFileSync('./data_import/normMaster.json', JSON.stringify(normMaster));
+  }
+  else {
+    normMaster = JSON.parse(fs.readFileSync('./data_import/normMaster.json'), 'utf-8');
+  }
+}
+
+function getNormMasterDoc(url, timing = 1) {
+  console.info(url);
+  return new Promise(async resolve => {
+    const options = new chrome.Options();
+    options.setUserPreferences({
+      "download.default_directory": nmDocs,
+      "download.prompt_for_download": false,
+      "download.directory_upgrade": true,
+      "plugins.always_open_pdf_externally": true
+    });
+    let driver = await new Builder().forBrowser('chrome').setChromeOptions(options).build();
+    let timeout = 7000 * timing;
+    try {
+      await driver.get(url);
+      if (await driver.getCurrentUrl() !== 'https://rb-wam-saml.bosch.com/tfim/sps/normmaster/saml20/login') {  // got document selection page
+        timeout = 11000 * timing;
+        await driver.executeScript('Array.prototype.slice.call(document.querySelectorAll(\'.functionlink\')).filter(e => e.innerText === \'English\')[0].click()').catch(() => {timeout = 0; });
+      }
+    }
+    finally {
+      setTimeout(async () => {  // wait until download is finished
+        await driver.quit();
+        resolve();
+      }, timeout);
+    }
+  });
+}
+
+function readPdf(file) {
+  return new Promise(async resolve => {
+    const countdown = 100;  // value for text timeout
+    let table = 0;      // > 0 when in correct table area
+    let rows = [];      // found table rows
+    let lastY = 0;      // y of last row
+    let lastX = 0;      // right x of last item
+    let lastText = '';  // text of last item
+    let lastLastText = '';  // text of last last item
+    await new pdfReader.PdfReader().parseFileItems(nmDocs + '\\' + file, (err, item) => {
+      if (item && item.text) {
+        if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsupplier') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsupplier') >= 0)) {  // table area starts
+          table = countdown;
+        }
+        if (table > 0) {
+          // console.log(item);
+          // console.log(item.y - lastY);
+          // console.log(item.text);
+          if (item.y - lastY > 0.8 && Math.abs(item.x - lastX) > 5) {  // new row
+            lastY = item.y;
+            rows.push(item.text);
+          }
+          else {  // still the same row row
+            rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text;  // push to row, detect if still same cell
+          }
+          lastX = (item.w * 0.055) + item.x;
+
+          if (/\d \d\d\d \d\d\d \d\d\d/.test(item.text)) {
+            table = countdown;
+          }
+          table --;
+          if (table <= 0 || item.text.toLowerCase().indexOf('release document') >= 0 || item.text.toLowerCase().indexOf('normative references') >= 0) {  // table area ended
+            table = -1;
+            // console.log(rows);
+            rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e)));  // filter non-table rows
+            resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
+          }
+        }
+        lastLastText = lastText;
+        lastText = item.text;
+      }
+      if (!item && table !== -1) {  // document ended
+        rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e)));  // filter non-table rows
+        resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
+      }
+    });
+  });
+}
+
+function sampleDeviceMap() {
+  const dpts = fs.readdirSync(dptFiles);
+  const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/;
+  for (let i in dpts) {
+    const regexRes = regex.exec(dpts[i])
+    if (regexRes) {  // found matching sample
+      sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase();
+    }
+  }
+}
+
+function stripSpaces(s) {
+  return s ? s.replace(/ /g,'') : '';
+}
+
+function trim(s) {
+  return s.replace(/(^\s+|\s+$)/gm, '');
+}

manifest.yml

@@ -1,8 +1,9 @@
 ---
 applications:
-  - name: digital-fingerprint-of-plastics-api
+  - name: definma-api
+    path: dist/
     instances: 1
-    memory: 256M
+    memory: 1024M
     stack: cflinuxfs3
     buildpacks:
       - nodejs_buildpack
@@ -10,4 +11,4 @@ applications:
       NODE_ENV: production
       OPTIMIZE_MEMORY: true
     services:
-      - dfopdb
+      - definmadb

package.json

@@ -7,11 +7,13 @@
     "tsc": "tsc",
     "tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc",
     "build": "build.bat",
+    "build-push": "build.bat && cf push",
     "test": "mocha dist/**/**.spec.js",
     "start": "node index.js",
     "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"",
     "loadDev": "node dist/test/loadDev.js",
-    "coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000"
+    "coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000",
+    "import": "node data_import/import.js"
   },
   "keywords": [],
   "author": "",
@@ -27,13 +29,15 @@
     "cfenv": "^1.2.2",
     "compression": "^1.7.4",
     "content-filter": "^1.1.2",
+    "cors": "^2.8.5",
     "express": "^4.17.1",
     "helmet": "^3.22.0",
     "json-schema": "^0.2.5",
+    "json2csv": "^5.0.1",
     "lodash": "^4.17.15",
     "mongo-sanitize": "^1.1.0",
     "mongoose": "^5.8.7",
-    "swagger-ui-express": "^4.1.2"
+    "swagger-ui-express": "4.1.2"
   },
   "devDependencies": {
     "@types/bcrypt": "^3.0.0",
@@ -45,9 +49,13 @@
     "@types/node": "^13.1.6",
     "@types/qs": "^6.9.1",
     "@types/serve-static": "^1.13.3",
+    "csv-parser": "^2.3.3",
+    "iconv-lite": "^0.6.0",
     "mocha": "^7.1.2",
     "nodemon": "^2.0.3",
     "nyc": "^15.0.1",
+    "pdfreader": "^1.0.7",
+    "selenium-webdriver": "^4.0.0-alpha.7",
     "should": "^13.2.3",
     "supertest": "^4.0.2",
     "tslint": "^5.20.1",

src/api.ts

@@ -18,6 +18,7 @@ export default class api {
     jsonRefParser.bundle('api/api.yaml', (err, doc) => {  // parse yaml
       if (err) throw err;
       apiDoc = doc;
+      apiDoc.servers.splice(process.env.NODE_ENV === 'production', 1);
       apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e));  // bundle routes
       apiDoc = this.resolveXDoc(apiDoc);
       oasParser.validate(apiDoc, (err, api) => {  // validate oas schema

src/db.ts

@@ -3,11 +3,15 @@ import cfenv from 'cfenv';
 import _ from 'lodash';
 import ChangelogModel from './models/changelog';
 
-// mongoose.set('debug', true);  // enable mongoose debug
 
 // database urls, prod db url is retrieved automatically
 const TESTING_URL = 'mongodb://localhost/dfopdb_test';
 const DEV_URL = 'mongodb://localhost/dfopdb';
+const debugging = true;
+
+if (process.env.NODE_ENV !== 'production' && debugging) {
+  mongoose.set('debug', true);  // enable mongoose debug
+}
 
 export default class db {
   private static state = {  // db object and current mode (test, dev, prod)
@@ -43,10 +47,15 @@ export default class db {
       if (err) done(err);
     });
     mongoose.connection.on('error', console.error.bind(console, 'connection error:'));
+    mongoose.connection.on('connected', () => {  // evaluation connection behaviour on prod
+      if (process.env.NODE_ENV !== 'test') {  // Do not interfere with testing
+        console.info('Database connected');
+      }
+    });
     mongoose.connection.on('disconnected', () => {  // reset state on disconnect
       if (process.env.NODE_ENV !== 'test') {  // Do not interfere with testing
         console.info('Database disconnected');
-        this.state.db = 0;
+        // this.state.db = 0;   // prod database connects and disconnects automatically
       }
     });
     process.on('SIGINT', () => {  // close connection when app is terminated

src/helpers/authorize.ts

@@ -89,6 +89,9 @@ function key (req, next): any {  // checks API key and returns changed user obje
         if (err) return next(err);
         if (data.length === 1) {  // one user found
           resolve({level: data[0].level, name: data[0].name, id: data[0]._id.toString(), location: data[0].location});
+          if (!/^\/api/m.test(req.url)){
+            delete req.query.key;  // delete query parameter to avoid interference with later validation
+          }
         }
         else {
           resolve(null);

src/helpers/csv.ts

@@ -0,0 +1,34 @@
+import {parseAsync} from 'json2csv';
+
+export default function csv(input: any[], f: (err, data) => void) {
+  parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
+    .then(csv => f(null, csv))
+    .catch(err => f(err, null));
+}
+
+function flatten (data) {  // flatten object: {a: {b: true}} -> {a.b: true}
+  const result = {};
+  function recurse (cur, prop) {
+    if (Object(cur) !== cur || Object.keys(cur).length === 0) {
+      result[prop] = cur;
+    }
+    else if (Array.isArray(cur)) {
+      let l = 0;
+      for(let i = 0, l = cur.length; i < l; i++)
+        recurse(cur[i], prop + "[" + i + "]");
+      if (l == 0)
+        result[prop] = [];
+    }
+    else {
+      let isEmpty = true;
+      for (let p in cur) {
+        isEmpty = false;
+        recurse(cur[p], prop ? prop+"."+p : p);
+      }
+      if (isEmpty && prop)
+        result[prop] = {};
+    }
+  }
+  recurse(data, '');
+  return result;
+}

src/helpers/mail.ts

@@ -17,7 +17,7 @@ export default (mailAddress, subject, content, f) => {  // callback, executed em
           contentType: "text/html"
         },
         from: {
-          eMail: "dfop@bosch-iot.com",
+          eMail: "definma@bosch-iot.com",
           password: "PlasticsOfFingerprintDigital"
         }
       }

src/index.ts

@@ -4,9 +4,11 @@ import compression from 'compression';
 import contentFilter from 'content-filter';
 import mongoSanitize from 'mongo-sanitize';
 import helmet from 'helmet';
+import cors from 'cors';
 import api from './api';
 import db from './db';
 
+// TODO: working demo branch
 
 // tell if server is running in debug or production environment
 console.info(process.env.NODE_ENV === 'production' ? '===== PRODUCTION =====' : process.env.NODE_ENV === 'test' ? '' :'===== DEVELOPMENT =====');
@@ -24,11 +26,11 @@ const port = process.env.PORT || 3000;
 
 //middleware
 app.use(helmet());
+app.use(contentFilter());  // filter URL query attacks
 app.use(express.json({ limit: '5mb'}));
 app.use(express.urlencoded({ extended: false, limit: '5mb' }));
 app.use(compression());  // compress responses
 app.use(bodyParser.json());
-app.use(contentFilter());  // filter URL query attacks
 app.use((req, res, next) => {  // filter body query attacks
   req.body = mongoSanitize(req.body);
   next();
@@ -41,14 +43,16 @@ app.use((req, res, next) => {  // no database connection error
     next();
   }
   else {
+    console.error('No database connection');
     res.status(500).send({status: 'Internal server error'});
   }
 });
+app.use(cors());  // CORS headers
 app.use(require('./helpers/authorize'));  // handle authentication
 
 // redirect /api routes for Angular proxy in development
 if (process.env.NODE_ENV !== 'production') {
-  app.use('/api/:url', (req, res) => {
+  app.use('/api/:url([^]+)', (req, res) => {
     req.url = '/' + req.params.url;
     app.handle(req, res);
   });

src/models/material.ts

@@ -22,5 +22,7 @@ MaterialSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>>
   db.log(req, this);
   return this;
 }
+MaterialSchema.index({supplier_id: 1});
+MaterialSchema.index({group_id: 1});
 
 export default mongoose.model<any, mongoose.Model<any, any>>('material', MaterialSchema);

src/models/measurement.ts

@@ -17,5 +17,7 @@ MeasurementSchema.query.log = function <Q extends mongoose.DocumentQuery<any, an
   db.log(req, this);
   return this;
 }
+MeasurementSchema.index({sample_id: 1});
+MeasurementSchema.index({measurement_template: 1});
 
 export default mongoose.model<any, mongoose.Model<any, any>>('measurement', MeasurementSchema);

src/models/sample.ts

@@ -22,5 +22,8 @@ SampleSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>> (
   db.log(req, this);
   return this;
 }
+SampleSchema.index({material_id: 1});
+SampleSchema.index({note_id: 1});
+SampleSchema.index({user_id: 1});
 
 export default mongoose.model<any, mongoose.Model<any, any>>('sample', SampleSchema);

src/routes/material.spec.ts

@@ -72,6 +72,43 @@ describe('/material', () => {
         done();
       });
     });
+    it('allows filtering by state', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/materials?status=new',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).have.lengthOf(json.collections.materials.filter(e => e.status === globals.status.new).length);
+        should(res.body).matchEach(material => {
+          should(material).have.only.keys('_id', 'name', 'supplier', 'group', 'mineral', 'glass_fiber', 'carbon_fiber', 'numbers');
+          should(material).have.property('_id').be.type('string');
+          should(material).have.property('name').be.type('string');
+          should(material).have.property('supplier').be.type('string');
+          should(material).have.property('group').be.type('string');
+          should(material).have.property('mineral').be.type('number');
+          should(material).have.property('glass_fiber').be.type('number');
+          should(material).have.property('carbon_fiber').be.type('number');
+          should(material.numbers).matchEach(number => {
+            should(number).have.only.keys('color', 'number');
+            should(number).have.property('color').be.type('string');
+            should(number).have.property('number').be.type('string');
+          });
+        });
+        done();
+      });
+    });
+    it('rejects an invalid state name', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/materials?status=xxx',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"status" must be one of [validated, new, all]'}
+      });
+    });
     it('rejects unauthorized requests', done => {
       TestHelper.request(server, done, {
         method: 'get',

src/routes/material.ts

@@ -19,7 +19,24 @@ const router = express.Router();
 router.get('/materials', (req, res, next) => {
   if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
 
-  MaterialModel.find({status:globals.status.validated}).populate('group_id').populate('supplier_id').lean().exec((err, data) => {
+  const {error, value: filters} = MaterialValidate.query(req.query);
+  if (error) return res400(error, res);
+
+  let conditions;
+
+  if (filters.hasOwnProperty('status')) {
+    if(filters.status === 'all') {
+      conditions = {$or: [{status: globals.status.validated}, {status: globals.status.new}]}
+    }
+    else {
+      conditions = {status: globals.status[filters.status]};
+    }
+  }
+  else {  // default
+    conditions = {status: globals.status.validated};
+  }
+
+  MaterialModel.find(conditions).populate('group_id').populate('supplier_id').lean().exec((err, data) => {
     if (err) return next(err);
 
     res.json(_.compact(data.map(e => MaterialValidate.output(e))));  // validate all and filter null values from validation errors

src/routes/root.ts

@@ -17,6 +17,7 @@ router.get('/authorized', (req, res) => {
   res.json({status: 'Authorization successful', method: req.authDetails.method});
 });
 
+// TODO: evaluate exact changelog functionality (restoring, delting after time, etc.)
 router.get('/changelog/:timestamp/:page?/:pagesize?', (req, res, next) => {
   if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;
 

src/routes/sample.spec.ts

@@ -10,6 +10,7 @@ import mongoose from 'mongoose';
 // TODO: generate output for ML in format DPT -> data, implement filtering, field selection
 // TODO: generate csv
 // TODO: write script for data import
+// TODO: allowed types: tension rod, part, granulate, other
 
 describe('/sample', () => {
   let server;
@@ -18,6 +19,9 @@ describe('/sample', () => {
   afterEach(done => TestHelper.afterEach(server, done));
   after(done => TestHelper.after(done));
 
+  // TODO: sort, added date filter, has measurements/condition filter
+  // TODO: check if conditions work in sort/fields/filters
+  // TODO: test for numbers as strings in glass_fiber
   describe('GET /samples', () => {
     it('returns all samples', done => {
       TestHelper.request(server, done, {
@@ -30,7 +34,7 @@ describe('/sample', () => {
         const json = require('../test/db.json');
         should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.validated).length);
         should(res.body).matchEach(sample => {
-          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
+          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
           should(sample).have.property('_id').be.type('string');
           should(sample).have.property('number').be.type('string');
           should(sample).have.property('type').be.type('string');
@@ -41,6 +45,7 @@ describe('/sample', () => {
           should(sample).have.property('material_id').be.type('string');
           should(sample).have.property('note_id');
           should(sample).have.property('user_id').be.type('string');
+          should(sample).have.property('added').be.type('string');
         });
         done();
       });
@@ -56,7 +61,7 @@ describe('/sample', () => {
         const json = require('../test/db.json');
         should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.validated).length);
         should(res.body).matchEach(sample => {
-          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
+          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
           should(sample).have.property('_id').be.type('string');
           should(sample).have.property('number').be.type('string');
           should(sample).have.property('type').be.type('string');
@@ -67,10 +72,405 @@ describe('/sample', () => {
           should(sample).have.property('material_id').be.type('string');
           should(sample).have.property('note_id');
           should(sample).have.property('user_id').be.type('string');
+          should(sample).have.property('added').be.type('string');
         });
         done();
       });
     });
+    it('allows filtering by state', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=new',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.new).length);
+        should(res.body).matchEach(sample => {
+          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
+          should(sample).have.property('_id').be.type('string');
+          should(sample).have.property('number').be.type('string');
+          should(sample).have.property('type').be.type('string');
+          should(sample).have.property('color').be.type('string');
+          should(sample).have.property('batch').be.type('string');
+          should(sample).have.property('condition').be.type('object');
+          should(sample).have.property('material_id').be.type('string');
+          should(sample).have.property('note_id');
+          should(sample).have.property('user_id').be.type('string');
+          should(sample).have.property('added').be.type('string');
+        });
+        done();
+      });
+    });
+    it('uses the given page size', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&page-size=3',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(3);
+        done();
+      });
+    });
+    it('returns results starting from first-id', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&from-id=400000000000000000000002',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body[0]).have.property('_id', '400000000000000000000002');
+        should(res.body[1]).have.property('_id', '400000000000000000000003');
+        done();
+      });
+    });
+    it('returns the right page number', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&to-page=2&page-size=2',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body[0]).have.property('_id', '400000000000000000000006');
+        done();
+      });
+    });
+    it('works with negative page numbers', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&to-page=-1&page-size=2&from-id=400000000000000000000004',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body[0]).have.property('_id', '400000000000000000000002');
+        should(res.body[1]).have.property('_id', '400000000000000000000003');
+        done();
+      });
+    });
+    it('returns an empty array for a page number out of range', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&to-page=100&page-size=2',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(0);
+        should(res.body).be.eql([]);
+        done();
+      });
+    });
+    it('returns an empty array for a page number out of negative range', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&to-page=-100&page-size=3&from-id=400000000000000000000004',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(0);
+        should(res.body).be.eql([]);
+        done();
+      });
+    });
+    it('sorts the samples ascending', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&sort=color-asc',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body[0]).have.property('color', 'black');
+        should(res.body[res.body.length - 1]).have.property('color', 'natural');
+        done();
+      });
+    });
+    it('sorts the samples descending', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&sort=number-desc',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body[0]).have.property('number', 'Rng36');
+        should(res.body[1]).have.property('number', '33');
+        should(res.body[res.body.length - 1]).have.property('number', '1');
+        done();
+      });
+    });
+    it('sorts the samples correctly in combination with paging', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&sort=color-asc&page-size=2&from-id=400000000000000000000006',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body[0]).have.property('_id', '400000000000000000000006');
+        should(res.body[1]).have.property('_id', '400000000000000000000002');
+        done();
+      });
+    });
+    it('sorts the samples correctly in combination with going pages backward', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&sort=color-desc&page-size=2&from-id=400000000000000000000004&to-page=-1',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body[0]).have.property('_id', '400000000000000000000002');
+        should(res.body[1]).have.property('_id', '400000000000000000000006');
+        done();
+      });
+    });
+    it('sorts the samples correctly for material keys', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&sort=material.name-desc',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body[0]).have.property('_id', '400000000000000000000002');
+        should(res.body[1]).have.property('_id', '400000000000000000000006');
+        should(res.body[2]).have.property('_id', '400000000000000000000001');
+        done();
+      });
+    });
+    it('adds the specified measurements', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=measurements.kf',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null});
+        should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.6, 'standard deviation': null});
+        done();
+      });
+    });
+    it('multiplies the sample information for each spectrum', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=measurements.spectrum',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(2);
+        should(res.body[0]).have.property('spectrum', [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
+        should(res.body[1]).have.property('spectrum', [[3996.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
+        done();
+      });
+    });
+    it('filters a sample property', done => {  // TODO: implement filters
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=type&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22type%22%2C%22values%22%3A%5B%22part%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).have.lengthOf(json.collections.samples.filter(e => e.type === 'part').length);
+        should(res.body).matchEach(sample => {
+          should(sample).have.property('type', 'part');
+        });
+        done();
+      });
+    });
+    it('filters a material property', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22in%22%2C%22field%22%3A%22material.name%22%2C%22values%22%3A%5B%22Schulamid%2066%20GF%2025%20H%22%2C%22Stanyl%20TW%20200%20F8%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).have.lengthOf(json.collections.samples.filter(e => e.material_id == '100000000000000000000004' || e.material_id == '100000000000000000000001').length);
+        should(res.body).matchEach(sample => {
+          should(sample.material.name).be.equalOneOf('Schulamid 66 GF 25 H', 'Stanyl TW 200 F8');
+        });
+        done();
+      });
+    });
+    it('filters by measurement value', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=measurements.kf.weight%20%25&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length);
+        should(res.body).matchEach(sample => {
+          should(sample.kf['weight %']).be.above(0.5);
+        });
+        done();
+      });
+    });
+    it('filters by measurement value not in the fields', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length);
+        should(res.body[0]).have.property('number', 'Rng36');
+        done();
+      });
+    });
+    it('filters multiple properties', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.glass_fiber%22%2C%22values%22%3A%5B33%5D%7D&filters[]=%7B%22mode%22%3A%22lte%22%2C%22field%22%3A%22number%22%2C%22values%22%3A%5B%22Rng33%22%5D%7D&filters[]=%7B%22mode%22%3A%22nin%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(1);
+        should(res.body[0]).be.eql({number: '32', material: {glass_fiber: 33}, batch: '1653000308'});
+        done();
+      });
+    });  // TODO: do measurement pipeline, check if it works with UI
+    it('rejects an invalid JSON string as a filters parameter', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=xx',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: 'Invalid JSON string for filter parameter'}
+      });
+    });
+    it('rejects an invalid filter mode', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'}
+      });
+    });
+    it('rejects an filter field not existing', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22xx%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"filters[0].field" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'}
+      });
+    });
+    it('rejects unknown measurement names', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=measurements.xx',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: 'Measurement key not found'}
+      });
+    });
+    it('returns a correct csv file if specified', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&page-size=2&csv=true',
+        contentType: /text\/csv/,
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.text).be.eql('"_id","number","type","color","batch","condition.material","condition.weeks","condition.condition_template","material_id","note_id","user_id","added"\r\n' +
+          '"400000000000000000000001","1","granulate","black","","copper",3,"200000000000000000000001","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' +
+          '"400000000000000000000002","21","granulate","natural","1560237365","copper",3,"200000000000000000000001","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"');
+        done();
+      });
+    });
+    it('returns only the fields specified', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&page-size=1&fields[]=number&fields[]=condition&fields[]=color&fields[]=material.name&fields[]=material.mineral',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200,
+        res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', mineral: 0}}]
+      });
+    });
+    it('rejects a from-id not in the database', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?from-id=5ea0450ed851c30a90e70894&sort=color-asc',
+        auth: {basic: 'admin'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: 'from-id not found'}
+      });
+    });
+    it('rejects an invalid fields parameter', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&page-size=1&fields=number',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"fields" must be an array'}
+      });
+    });
+    it('rejects an unknown field name', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&page-size=1&fields[]=xx',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'}
+      });
+    });
+    it('rejects a negative page size', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?page-size=-3',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"page-size" must be larger than or equal to 1'}
+      });
+    });
+    it('rejects an invalid from-id', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?from-id=40000000000h000000000002',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"from-id" with value "40000000000h000000000002" fails to match the required pattern: /[0-9a-f]{24}/'}
+      });
+    });
+    it('rejects a to-page without page-size', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?to-page=3',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"to-page" missing required peer "page-size"'}
+      });
+    });
+    it('rejects an invalid state name', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=xxx',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: '"status" must be one of [validated, new, all]'}
+      });
+    });
     it('rejects unauthorized requests', done => {
       TestHelper.request(server, done, {
         method: 'get',
@@ -93,7 +493,7 @@ describe('/sample', () => {
         let asyncCounter = res.body.length;
         should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.new).length);
         should(res.body).matchEach(sample => {
-          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
+          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
           should(sample).have.property('_id').be.type('string');
           should(sample).have.property('number').be.type('string');
           should(sample).have.property('type').be.type('string');
@@ -106,6 +506,7 @@ describe('/sample', () => {
           should(sample).have.property('material_id').be.type('string');
           should(sample).have.property('note_id');
           should(sample).have.property('user_id').be.type('string');
+          should(sample).have.property('added').be.type('string');
           SampleModel.findById(sample._id).lean().exec((err, data) => {
             should(data).have.property('status',globals.status.new);
             if (--asyncCounter === 0) {
@@ -127,7 +528,7 @@ describe('/sample', () => {
         let asyncCounter = res.body.length;
         should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status === -1).length);
         should(res.body).matchEach(sample => {
-          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
+          should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
           should(sample).have.property('_id').be.type('string');
           should(sample).have.property('number').be.type('string');
           should(sample).have.property('type').be.type('string');
@@ -140,6 +541,7 @@ describe('/sample', () => {
           should(sample).have.property('material_id').be.type('string');
           should(sample).have.property('note_id');
           should(sample).have.property('user_id').be.type('string');
+          should(sample).have.property('added').be.type('string');
           SampleModel.findById(sample._id).lean().exec((err, data) => {
             should(data).have.property('status',globals.status.deleted);
             if (--asyncCounter === 0) {
@@ -174,6 +576,42 @@ describe('/sample', () => {
     });
   });
 
+  describe('GET /samples/count', () => {
+    it('returns the correct number of samples', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples/count',
+        auth: {basic: 'admin'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body.count).be.eql(json.collections.samples.length);
+        done();
+      });
+    });
+    it('works with an API key', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples/count',
+        auth: {key: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body.count).be.eql(json.collections.samples.length);
+        done();
+      });
+    });
+    it('rejects unauthorized requests', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples/count',
+        httpStatus: 401
+      });
+    });
+  });
+
   describe('GET /sample/{id}', () => {
     it('returns the right sample', done => {
       TestHelper.request(server, done, {
@@ -181,7 +619,7 @@ describe('/sample', () => {
         url: '/sample/400000000000000000000003',
         auth: {basic: 'janedoe'},
         httpStatus: 200,
-        res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, user: 'admin'}
+        res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'}
       });
     });
     it('works with an API key', done => {
@@ -190,7 +628,7 @@ describe('/sample', () => {
         url: '/sample/400000000000000000000003',
         auth: {key: 'janedoe'},
         httpStatus: 200,
-        res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, user: 'admin'}
+        res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'}
       });
     });
     it('returns a deleted sample for a maintain/admin user', done => {
@@ -199,7 +637,7 @@ describe('/sample', () => {
         url: '/sample/400000000000000000000005',
         auth: {basic: 'admin'},
         httpStatus: 200,
-        res: {_id: '400000000000000000000005', number: 'Rng33', type: 'granulate', color: 'black', batch: '1653000308', condition: {condition_template: '200000000000000000000003'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {}, user: 'admin'}
+        res: {_id: '400000000000000000000005', number: 'Rng33', type: 'granulate', color: 'black', batch: '1653000308', condition: {condition_template: '200000000000000000000003'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {}, measurements: [], user: 'admin'}
       });
     });
     it('returns 403 for a write user when requesting a deleted sample', done => {
@@ -235,7 +673,7 @@ describe('/sample', () => {
     });
   });
 
-  describe('PUT /sample/{id}', () => {
+  describe('PUT /sample/{id}', () => {  // TODO: fix tests, work on /samples
     it('returns the right sample', done => {
       TestHelper.request(server, done, {
         method: 'put',
@@ -243,7 +681,7 @@ describe('/sample', () => {
         auth: {basic: 'janedoe'},
         httpStatus: 200,
         req: {},
-        res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}
+        res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}
       });
     });
     it('keeps unchanged properties', done => {
@@ -255,7 +693,7 @@ describe('/sample', () => {
         req: {type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', notes: {}}
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'});
+        should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'});
         SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => {
           if (err) return done (err);
           should(data).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'status', '__v');
@@ -282,7 +720,7 @@ describe('/sample', () => {
         req: {type: 'granulate'}
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'});
+        should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'});
         SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => {
           if (err) return done (err);
           should(data).have.property('status',globals.status.validated);
@@ -299,7 +737,7 @@ describe('/sample', () => {
         req: {condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}}
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'});
+        should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'});
         SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => {
           if (err) return done (err);
           should(data).have.property('status',globals.status.validated);
@@ -316,7 +754,7 @@ describe('/sample', () => {
         req: {notes: {comment: 'Stoff gesperrt', sample_references: []}}
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.body).be.eql({_id: '400000000000000000000002', number: '21', type: 'granulate', color: 'natural', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', note_id: '500000000000000000000001', user_id: '000000000000000000000002'});
+        should(res.body).be.eql({_id: '400000000000000000000002', number: '21', type: 'granulate', color: 'natural', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', note_id: '500000000000000000000001', user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'});
         SampleModel.findById('400000000000000000000002').lean().exec((err, data: any) => {
           if (err) return done (err);
           should(data).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'status', '__v');
@@ -593,7 +1031,7 @@ describe('/sample', () => {
         auth: {basic: 'janedoe'},
         httpStatus: 200,
         req: {condition: {}},
-        res: {_id: '400000000000000000000006', number: 'Rng36', type: 'granulate', color: 'black', batch: '', condition: {}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}
+        res: {_id: '400000000000000000000006', number: 'Rng36', type: 'granulate', color: 'black', batch: '', condition: {}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}
       });
     });
     it('rejects an old version of a condition template', done => {
@@ -613,7 +1051,7 @@ describe('/sample', () => {
         auth: {basic: 'admin'},
         httpStatus: 200,
         req: {condition: {p1: 36, condition_template: '200000000000000000000004'}},
-        res: {_id: '400000000000000000000004', number: '32', type: 'granulate', color: 'black', batch: '1653000308', condition: {p1: 36, condition_template: '200000000000000000000004'}, material_id: '100000000000000000000005', note_id: '500000000000000000000003', user_id: '000000000000000000000003'}
+        res: {_id: '400000000000000000000004', number: '32', type: 'granulate', color: 'black', batch: '1653000308', condition: {p1: 36, condition_template: '200000000000000000000004'}, material_id: '100000000000000000000005', note_id: '500000000000000000000003', user_id: '000000000000000000000003', added: '2004-01-10T13:37:04.000Z'}
       });
     });
     it('rejects an changing back to an empty condition', done => {
@@ -660,7 +1098,7 @@ describe('/sample', () => {
         auth: {basic: 'admin'},
         httpStatus: 200,
         req: {},
-        res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {condition_template: '200000000000000000000001', material: 'copper', weeks: 3}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}
+        res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {condition_template: '200000000000000000000001', material: 'copper', weeks: 3}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}
       });
     });
     it('rejects requests from a read user', done => {
@@ -1051,7 +1489,7 @@ describe('/sample', () => {
         req: {color: 'black', type: 'granulate', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}}
       }).end((err, res) => {
         if (err) return done (err);
-        should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
+        should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
         should(res.body).have.property('_id').be.type('string');
         should(res.body).have.property('number', 'Rng37');
         should(res.body).have.property('color', 'black');
@@ -1061,6 +1499,8 @@ describe('/sample', () => {
         should(res.body).have.property('material_id', '100000000000000000000001');
         should(res.body).have.property('note_id').be.type('string');
         should(res.body).have.property('user_id', '000000000000000000000002');
+        should(res.body).have.property('added').be.type('string');
+        should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000);
         done();
       });
     });
@@ -1164,7 +1604,7 @@ describe('/sample', () => {
         req: {color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}}
       }).end((err, res) => {
         if (err) return done (err);
-        should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
+        should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
         should(res.body).have.property('_id').be.type('string');
         should(res.body).have.property('number', 'Fe1');
         should(res.body).have.property('color', 'black');
@@ -1173,6 +1613,8 @@ describe('/sample', () => {
         should(res.body).have.property('material_id', '100000000000000000000001');
         should(res.body).have.property('note_id').be.type('string');
         should(res.body).have.property('user_id', '000000000000000000000004');
+        should(res.body).have.property('added').be.type('string');
+        should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1500);
         done();
       });
     });
@@ -1185,7 +1627,7 @@ describe('/sample', () => {
         req: {color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}}
       }).end((err, res) => {
         if (err) return done (err);
-        should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
+        should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
         should(res.body).have.property('_id').be.type('string');
         should(res.body).have.property('number', 'Rng37');
         should(res.body).have.property('color', 'black');
@@ -1195,6 +1637,8 @@ describe('/sample', () => {
         should(res.body).have.property('material_id', '100000000000000000000001');
         should(res.body).have.property('note_id').be.type('string');
         should(res.body).have.property('user_id', '000000000000000000000002');
+        should(res.body).have.property('added').be.type('string');
+        should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000);
         done();
       });
     });
@@ -1237,7 +1681,7 @@ describe('/sample', () => {
         req: {number: 'Rng34', color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}},
       }).end((err, res) => {
         if (err) return done (err);
-        should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
+        should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
         should(res.body).have.property('_id').be.type('string');
         should(res.body).have.property('number', 'Rng34');
         should(res.body).have.property('color', 'black');
@@ -1247,6 +1691,8 @@ describe('/sample', () => {
         should(res.body).have.property('material_id', '100000000000000000000001');
         should(res.body).have.property('note_id').be.type('string');
         should(res.body).have.property('user_id', '000000000000000000000003');
+        should(res.body).have.property('added').be.type('string');
+        should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000);
         done();
       });
     });

src/routes/sample.ts

@@ -6,6 +6,7 @@ import NoteFieldValidate from './validate/note_field';
 import res400 from './validate/res400';
 import SampleModel from '../models/sample'
 import MeasurementModel from '../models/measurement';
+import MeasurementTemplateModel from '../models/measurement_template';
 import MaterialModel from '../models/material';
 import NoteModel from '../models/note';
 import NoteFieldModel from '../models/note_field';
@@ -15,17 +16,350 @@ import ConditionTemplateModel from '../models/condition_template';
 import ParametersValidate from './validate/parameters';
 import globals from '../globals';
 import db from '../db';
+import csv from '../helpers/csv';
 
 
 const router = express.Router();
 
-router.get('/samples', (req, res, next) => {
+// TODO: check added filter
+// TODO: return total number of pages -> use facet
+// TODO: use query pointer
+// TODO: convert filter value to number according to table model
+// TODO: validation for filter parameters
+// TODO: location/device sort/filter
+router.get('/samples', async (req, res, next) => {
   if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
 
-  SampleModel.find({status: globals.status.validated}).lean().exec((err, data) => {
+  const {error, value: filters} = SampleValidate.query(req.query);
-    if (err) return next(err);
+  if (error) return res400(error, res);
-    res.json(_.compact(data.map(e => SampleValidate.output(e))));  // validate all and filter null values from validation errors
+
-  })
+  // TODO: find a better place for these
+  const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', 'user_id'];
+
+  // evaluate sort parameter from 'color-asc' to ['color', 1]
+  filters.sort = filters.sort.split('-');
+  filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0];  // route added sorting criteria to _id
+  filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1;
+  if (!filters['to-page']) {  // set to-page default
+    filters['to-page'] = 0;
+  }
+  const addedFilter = filters.filters.find(e => e.field === 'added');
+  if (addedFilter) {  // convert added filter to object id
+    filters.filters.splice(filters.filters.findIndex(e => e.field === 'added'), 1);
+    if (addedFilter.mode === 'in') {
+      const v = [];  // query value
+      addedFilter.values.forEach(value => {
+        const date = [new Date(value).setHours(0,0,0,0), new Date(value).setHours(23,59,59,999)];
+        v.push({$and: [{ _id: { '$gte': dateToOId(date[0])}}, { _id: { '$lte': dateToOId(date[1])}}]});
+      });
+      filters.filters.push({mode: 'or', field: '_id', values: v});
+    }
+    else if (addedFilter.mode === 'nin') {
+      addedFilter.values = addedFilter.values.sort();
+      const v = [];  // query value
+
+      for (let i = 0; i <= addedFilter.values.length; i ++) {
+        v[i] = {$and: []};
+        if (i > 0) {
+          const date = new Date(addedFilter.values[i - 1]).setHours(23,59,59,999);
+          v[i].$and.push({ _id: { '$gt': dateToOId(date)}}) ;
+        }
+        if (i < addedFilter.values.length) {
+          const date = new Date(addedFilter.values[i]).setHours(0,0,0,0);
+          v[i].$and.push({ _id: { '$lt': dateToOId(date)}}) ;
+        }
+      }
+      filters.filters.push({mode: 'or', field: '_id', values: v});
+    }
+    else {
+      // start and end of day
+      const date = [new Date(addedFilter.values[0]).setHours(0,0,0,0), new Date(addedFilter.values[0]).setHours(23,59,59,999)];
+      if (addedFilter.mode === 'lt') {  // lt start
+        filters.filters.push({mode: 'lt', field: '_id', values: [dateToOId(date[0])]});
+      }
+      if (addedFilter.mode === 'eq' || addedFilter.mode === 'lte') {  // lte end
+        filters.filters.push({mode: 'lte', field: '_id', values: [dateToOId(date[1])]});
+      }
+      if (addedFilter.mode === 'gt') {  // gt end
+        filters.filters.push({mode: 'gt', field: '_id', values: [dateToOId(date[1])]});
+      }
+      if (addedFilter.mode === 'eq' || addedFilter.mode === 'gte') {  // gte start
+        filters.filters.push({mode: 'gte', field: '_id', values: [dateToOId(date[0])]});
+      }
+      if (addedFilter.mode === 'ne') {
+        filters.filters.push({mode: 'or', field: '_id', values: [{ _id: { '$lt': dateToOId(date[0])}}, { _id: { '$gt': dateToOId(date[1])}}]});
+      }
+    }
+  }
+
+  const sortFilterKeys = filters.filters.map(e => e.field);
+
+  let collection;
+  const query = [];
+  let queryPtr = query;
+  queryPtr.push({$match: {$and: []}});
+
+  if (filters.sort[0].indexOf('measurements.') >= 0) {  // sorting with measurements as starting collection
+    collection = MeasurementModel;
+    const [,measurementName, measurementParam] = filters.sort[0].split('.');
+    const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);});
+    if (measurementTemplate instanceof Error) return;
+    if (!measurementTemplate) {
+      return res.status(400).json({status: 'Invalid body format', details: filters.sort[0] + ' not found'});
+    }
+    let sortStartValue = null;
+    if (filters['from-id']) {  // from-id specified, fetch values for sorting
+      const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}).lean().exec().catch(err => {next(err);});  // TODO: what if more than one measurement for sample?
+      if (fromSample instanceof Error) return;
+      if (!fromSample) {
+        return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
+      }
+      sortStartValue = fromSample.values[measurementParam];
+    }
+    queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)});  // find measurements to sort
+    if (filters.filters.find(e => e.field === filters.sort[0])) {  // sorted measurement should also be filtered
+      queryPtr[0].$match.$and.push(...filterQueries(filters.filters.filter(e => e.field === filters.sort[0]).map(e => {e.field = 'values.' + e.field.split('.')[2]; return e; })));
+    }
+    queryPtr.push(
+      ...sortQuery(filters, ['values.' + measurementParam, 'sample_id'], sortStartValue),  // sort measurements
+      {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}},                                  // fetch samples and restructure them to fit sample structure
+      {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
+      {$match: statusQuery(filters, 'sample.status')},  // filter out wrong status once samples were added
+      {$addFields: {['sample.' + measurementName]: '$measurement.values'}},  // more restructuring
+      {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
+    );
+  }
+  else {  // sorting with samples as starting collection
+    collection = SampleModel;
+    queryPtr[0].$match.$and.push(statusQuery(filters, 'status'));
+
+    if (sampleKeys.indexOf(filters.sort[0]) >= 0) {  // sorting for sample keys
+      let sortStartValue = null;
+      if (filters['from-id']) {  // from-id specified
+        const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {
+          next(err);
+        });
+        if (fromSample instanceof Error) return;
+        if (!fromSample) {
+          return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
+        }
+        sortStartValue = fromSample[filters.sort[0]];
+      }
+      queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
+    }
+    else {  // add sort key to list to add field later
+      sortFilterKeys.push(filters.sort[0]);
+    }
+  }
+
+  addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0));  // sample filters
+
+  let materialQuery = [];  // put material query together separate first to reuse for first-id
+  let materialAdded = false;
+  if (sortFilterKeys.find(e => /material\./.test(e))) {  //  add material fields
+    materialAdded = true;
+    materialQuery.push(  // add material properties
+      {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},  // TODO: project out unnecessary fields
+      {$addFields: {material: {$arrayElemAt: ['$material', 0]}}}
+    );
+    const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
+    addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0));  // base material filters
+    if (sortFilterKeys.find(e => e === 'material.supplier')) {  // add supplier if needed
+      materialQuery.push(
+        {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
+        {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
+      );
+    }
+    if (sortFilterKeys.find(e => e === 'material.group')) {  // add group if needed
+      materialQuery.push(
+        {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
+        {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
+      );
+    }
+    if (sortFilterKeys.find(e => e === 'material.number')) {  // add material number if needed
+      materialQuery.push(
+        {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
+      );
+    }
+    const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
+    addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0));  // base material filters
+    queryPtr.push(...materialQuery);
+    if (/material\./.test(filters.sort[0])) {  // sort by material key
+      let sortStartValue = null;
+      if (filters['from-id']) {  // from-id specified
+        const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);});
+        if (fromSample instanceof Error) return;
+        if (!fromSample) {
+          return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
+        }
+        sortStartValue = fromSample[filters.sort[0]];
+      }
+      queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
+    }
+  }
+
+  const measurementFilterFields = _.uniq(sortFilterKeys.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]));  // filter measurement names and remove duplicates from parameters
+  if (sortFilterKeys.find(e => /measurements\./.test(e))) {  //  add measurement fields
+    const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFilterFields}}).lean().exec().catch(err => {next(err);});
+    if (measurementTemplates instanceof Error) return;
+    if (measurementTemplates.length < measurementFilterFields.length) {
+      return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
+    }
+    queryPtr.push({$lookup: {
+        from: 'measurements', let: {sId: '$_id'},
+        pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
+        as: 'measurements'
+    }});
+    measurementTemplates.forEach(template => {
+      queryPtr.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
+        vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
+        in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
+      }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
+    });
+    addFilterQueries(queryPtr, filters.filters
+      .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
+      .map(e => {e.field = e.field.replace('measurements.', ''); return e; })
+    );  // measurement filters
+  }
+
+  if (!filters.fields.find(e => /spectrum\./.test(e)) && !filters['from-id']) {  // count total number of items before $skip and $limit, only works when from-id is not specified and spectra are not included
+    queryPtr.push({$facet: {count: [{$count: 'count'}], samples: []}});
+    queryPtr = queryPtr[queryPtr.length - 1].$facet.samples;  // add rest of aggregation pipeline into $facet
+  }
+
+  // paging
+  if (filters['to-page']) {
+    queryPtr.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)})  // number to skip, if going back pages, one page has to be skipped less but on sample more
+  }
+  if (filters['page-size']) {
+    queryPtr.push({$limit: filters['page-size']});
+  }
+
+  const fieldsToAdd = filters.fields.filter(e =>   // fields to add
+    sortFilterKeys.indexOf(e) < 0                      // field was not in filter
+    && e !== filters.sort[0]                           // field was not in sort
+  );
+
+  if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) {  // add material, was not added already
+    queryPtr.push(
+      {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
+      {$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
+    );
+  }
+  if (fieldsToAdd.indexOf('material.supplier') >= 0) {  // add supplier if needed
+    queryPtr.push(
+      {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
+      {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
+    );
+  }
+  if (fieldsToAdd.indexOf('material.group') >= 0) {  // add group if needed
+    queryPtr.push(
+      {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
+      {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
+    );
+  }
+  if (fieldsToAdd.indexOf('material.number') >= 0) {  // add material number if needed
+    queryPtr.push(
+      {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
+    );
+  }
+
+  let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]));  // filter measurement names and remove duplicates from parameters
+  if (fieldsToAdd.find(e => /measurements\./.test(e))) {  // add measurement fields
+    const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
+    if (measurementTemplates instanceof Error) return;
+    if (measurementTemplates.length < measurementFieldsFields.length) {
+      return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
+    }
+    if (fieldsToAdd.find(e => /spectrum\./.test(e))) {  // use different lookup methods with and without spectrum for the best performance
+      queryPtr.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
+    }
+    else {
+      queryPtr.push({$lookup: {
+          from: 'measurements', let: {sId: '$_id'},
+          pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
+          as: 'measurements'
+        }});
+    }
+    measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => {  // TODO: hard coded dpt for special treatment, change later
+      queryPtr.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
+              vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
+              in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
+            }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
+    });
+    if (measurementFieldsFields.find(e => e === 'spectrum')) {  // TODO: remove hardcoded as well
+      queryPtr.push(
+        {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
+        {$addFields: {spectrum: '$spectrum.values'}},
+        {$unwind: '$spectrum'}
+      );
+    }
+    // queryPtr.push({$unset: 'measurements'});
+    queryPtr.push({$project: {measurements: 0}});
+  }
+
+  const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
+  if (filters.fields.indexOf('added') >= 0) {  // add added date
+    // projection.added = {$toDate: '$_id'};
+    // projection.added = { $convert: { input: '$_id', to: "date" } }  // TODO: upgrade MongoDB version or find alternative
+  }
+  if (filters.fields.indexOf('_id') < 0 && filters.fields.indexOf('added') < 0) {  // disable _id explicitly
+    projection._id = false;
+  }
+  queryPtr.push({$project: projection});
+
+  if (!fieldsToAdd.find(e => /spectrum\./.test(e))) {  // use streaming when including spectrum files
+    collection.aggregate(query).exec((err, data) => {
+      if (err) return next(err);
+      if (data[0].count) {
+        res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0);
+        res.header('Access-Control-Expose-Headers', 'x-total-items');
+        data = data[0].samples;
+      }
+      if (filters.fields.indexOf('added') >= 0) {  // add added date
+        data.map(e => {
+          e.added = e._id.getTimestamp();
+          if (filters.fields.indexOf('_id') < 0) {
+            delete e._id;
+          }
+          return e
+        });
+      }
+      if (filters['to-page'] < 0) {
+        data.reverse();
+      }
+      const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]);
+      if (filters.csv) {  // output as csv
+        csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
+          if (err) return next(err);
+          res.set('Content-Type', 'text/csv');
+          res.send(data);
+        });
+      }
+      else {
+        res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));  // validate all and filter null values from validation errors
+      }
+    });
+  }
+  else {
+    res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
+    res.write('[');
+    let count = 0;
+    const stream = collection.aggregate(query).cursor().exec();
+    stream.on('data', data => {
+      if (filters.fields.indexOf('added') >= 0) {  // add added date
+        data.added = data._id.getTimestamp();
+        if (filters.fields.indexOf('_id') < 0) {
+          delete data._id;
+        }
+      }
+      res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++;
+    });
+    stream.on('close', () => {
+      res.write(']');
+      res.end();
+    });
+  }
 });
 
 router.get('/samples/:state(new|deleted)', (req, res, next) => {
@@ -37,6 +371,15 @@ router.get('/samples/:state(new|deleted)', (req, res, next) => {
   });
 });
 
+router.get('/samples/count', (req, res, next) => {
+  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
+
+  SampleModel.estimatedDocumentCount((err, data) => {
+    if (err) return next(err);
+    res.json({count: data});
+  });
+});
+
 router.get('/sample/' + IdValidate.parameter(), (req, res, next) => {
   if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
 
@@ -54,7 +397,7 @@ router.get('/sample/' + IdValidate.parameter(), (req, res, next) => {
       sampleData.material.supplier = sampleData.material.supplier_id.name;
       sampleData.user = sampleData.user_id.name;
       sampleData.notes = sampleData.note_id ? sampleData.note_id : {};
-      MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id)}).lean().exec((err, data) => {
+      MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id), status: {$ne: globals.status.deleted}}).lean().exec((err, data) => {
         sampleData.measurements = data;
         res.json(SampleValidate.output(sampleData, 'details'));
       });
@@ -272,13 +615,23 @@ module.exports = router;
 
 async function numberGenerate (sample, req, res, next) {  // generate number in format Location32, returns false on error
   const sampleData = await SampleModel
-    .findOne({number: new RegExp('^' + req.authDetails.location + '[0-9]+$', 'm')})
+    // .findOne({number: new RegExp('^' + req.authDetails.location + '[0-9]+$', 'm')})
-    .sort({number: -1})
+    // .sort({number: -1})
-    .lean()
+    // .lean()
+    .aggregate([
+      {$match: {number: new RegExp('^' + 'Rng' + '[0-9]+$', 'm')}},
+      // {$addFields: {number2: {$toDecimal: {$arrayElemAt: [{$split: [{$arrayElemAt: [{$split: ['$number', 'Rng']}, 1]}, '_']}, 0]}}}},  // not working with MongoDb 3.6
+      {$addFields: {sortNumber: {$let: {
+        vars: {tmp: {$concat: ['000000000000000000000000000000', {$arrayElemAt: [{$split: [{$arrayElemAt: [{$split: ['$number', 'Rng']}, 1]}, '_']}, 0]}]}},
+        in: {$substrCP: ['$$tmp', {$subtract: [{$strLenCP: '$$tmp'}, 30]}, {$strLenCP: '$$tmp'}]}
+      }}}},
+      {$sort: {sortNumber: -1}},
+      {$limit: 1}
+    ])
     .exec()
     .catch(err => next(err));
   if (sampleData instanceof Error) return false;
-  return req.authDetails.location + (sampleData ? Number(sampleData.number.replace(/[^0-9]+/g, '')) + 1 : 1);
+  return req.authDetails.location + (sampleData[0] ? Number(sampleData[0].number.replace(/[^0-9]+/g, '')) + 1 : 1);
 }
 
 async function numberCheck(sample, res, next) {
@@ -297,7 +650,7 @@ async function materialCheck (sample, res, next, id = sample.material_id) {  //
     res.status(400).json({status: 'Material not available'});
     return false;
   }
-  if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) {  // color for material not specified
+  if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) {  // color for material not specified
     res.status(400).json({status: 'Color not available for material'});
     return false;
   }
@@ -374,4 +727,54 @@ function customFieldsChange (fields, amount, req) {  // update custom_fields and
       }
     });
   });
+}
+
+function sortQuery(filters, sortKeys, sortStartValue) {  // sortKeys = ['primary key', 'secondary key']
+  if (filters['from-id']) {  // from-id specified
+    if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) {  // asc
+      return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
+        {$sort: {[sortKeys[0]]: 1, _id: 1}}];
+    } else {
+      return [{$match: {$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
+        {$sort: {[sortKeys[0]]: -1, _id: -1}}];
+    }
+  } else {  // sort from beginning
+    return [{$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}];  // set _id as secondary sort
+  }
+}
+
+function statusQuery(filters, field) {
+  if (filters.hasOwnProperty('status')) {
+    if(filters.status === 'all') {
+      return {$or: [{[field]: globals.status.validated}, {[field]: globals.status.new}]};
+    }
+    else {
+      return {[field]: globals.status[filters.status]};
+    }
+  }
+  else {  // default
+    return {[field]: globals.status.validated};
+  }
+}
+
+function addFilterQueries (queryPtr, filters) {  // returns array of match queries from given filters
+  if (filters.length) {
+    queryPtr.push({$match: {$and: filterQueries(filters)}});
+  }
+}
+
+function filterQueries (filters) {
+  console.log(filters);
+  return filters.map(e => {
+    if (e.mode === 'or') {  // allow or queries (needed for $ne added)
+      return {['$' + e.mode]: e.values};
+    }
+    else {
+      return {[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}};  // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin
+    }
+  });
+}
+
+function dateToOId (date) {  // convert date to ObjectId
+  return mongoose.Types.ObjectId(Math.floor(date / 1000).toString(16) + '0000000000000000');
 }

src/routes/validate/material.ts

@@ -107,4 +107,10 @@ export default class MaterialValidate {  // validate input for material
       numbers: this.material.numbers
     });
   }
+
+  static query (data) {
+    return Joi.object({
+      status: Joi.string().valid('validated', 'new', 'all')
+    }).validate(data);
+  }
 }

src/routes/validate/measurement.ts

@@ -44,4 +44,13 @@ export default class MeasurementValidate {
     }).validate(data, {stripUnknown: true});
     return error !== undefined? null : value;
   }
+
+  static outputV() {  // return output validator
+    return Joi.object({
+      _id: IdValidate.get(),
+      sample_id: IdValidate.get(),
+      values: this.measurement.values,
+      measurement_template: IdValidate.get()
+    });
+  }
 }

src/routes/validate/sample.ts

@@ -3,6 +3,7 @@ import Joi from '@hapi/joi';
 import IdValidate from './id';
 import UserValidate from './user';
 import MaterialValidate from './material';
+import MeasurementValidate from './measurement';
 
 export default class SampleValidate {
   private static sample = {
@@ -10,7 +11,8 @@ export default class SampleValidate {
       .max(128),
 
     color: Joi.string()
-      .max(128),
+      .max(128)
+      .allow(''),
 
     type: Joi.string()
       .max(128),
@@ -43,9 +45,42 @@ export default class SampleValidate {
             Joi.date()
           )
         )
-    })
+    }),
+
+    added: Joi.date()
+      .iso()
+      .min('1970-01-01T00:00:00.000Z')
   };
 
+  private static sortKeys = [
+    '_id',
+    'color',
+    'number',
+    'type',
+    'batch',
+    'added',
+    'material.name',
+    'material.supplier',
+    'material.group',
+    'material.mineral',
+    'material.glass_fiber',
+    'material.carbon_fiber',
+    'material.number',
+    'measurements.(?!spectrum)*'
+  ];
+
+  private static fieldKeys = [
+    ...SampleValidate.sortKeys,
+    'condition',
+    'material_id',
+    'material',
+    'note_id',
+    'user_id',
+    'material._id',
+    'material.numbers',
+    'measurements.spectrum.dpt'
+  ];
+
   static input (data, param) {  // validate input, set param to 'new' to make all attributes required
     if (param === 'new') {
       return Joi.object({
@@ -83,7 +118,11 @@ export default class SampleValidate {
     }
   }
 
-  static output (data, param = 'refs') {  // validate output and strip unwanted properties, returns null if not valid
+  static output (data, param = 'refs+added', additionalParams = []) {  // validate output and strip unwanted properties, returns null if not valid
+    if (param === 'refs+added') {
+      param = 'refs';
+      data.added = data._id.getTimestamp();
+    }
     data = IdValidate.stringify(data);
     let joiObject;
     if (param === 'refs') {
@@ -95,8 +134,10 @@ export default class SampleValidate {
         batch: this.sample.batch,
         condition: this.sample.condition,
         material_id: IdValidate.get(),
+        material: MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}),
         note_id: IdValidate.get().allow(null),
-        user_id: IdValidate.get()
+        user_id: IdValidate.get(),
+        added: this.sample.added
       };
     }
     else if(param === 'details') {
@@ -108,6 +149,7 @@ export default class SampleValidate {
         batch: this.sample.batch,
         condition: this.sample.condition,
         material: MaterialValidate.outputV(),
+        measurements: Joi.array().items(MeasurementValidate.outputV()),
         notes: this.sample.notes,
         user: UserValidate.username()
       }
@@ -115,7 +157,67 @@ export default class SampleValidate {
     else {
       return null;
     }
+    additionalParams.forEach(param => {
+      joiObject[param] = Joi.any();
+    });
     const {value, error} = Joi.object(joiObject).validate(data, {stripUnknown: true});
     return error !== undefined? null : value;
   }
+
+  static query (data) {
+    if (data.filters && data.filters.length) {
+      const filterValidation = Joi.array().items(Joi.string()).validate(data.filters);
+      if (filterValidation.error) return filterValidation;
+      try {
+        for (let i in data.filters) {
+          data.filters[i] = JSON.parse(data.filters[i]);
+          data.filters[i].values = data.filters[i].values.map(e => {  // validate filter values
+            let validator;
+            let field = data.filters[i].field
+            if (/material\./.test(field)) {  // select right validation model
+              validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')});
+              field = field.replace('material.', '');
+            }
+            else if (/measurements\./.test(field)) {
+              validator = Joi.object({
+                value: Joi.alternatives()
+                  .try(
+                    Joi.number(),
+                    Joi.string().max(128),
+                    Joi.boolean(),
+                    Joi.array()
+                  )
+                  .allow(null)
+              });
+              field = 'value';
+            }
+            else {
+              validator = Joi.object(this.sample);
+            }
+            const {value, error} = validator.validate({[field]: e});
+            console.log(value);
+            if (error) throw error;  // reject invalid values  // TODO: return exact error description, handle in frontend filters
+            return value[field];
+          });
+        }
+      }
+      catch {
+        return {error: {details: [{message: 'Invalid JSON string for filter parameter'}]}, value: null}
+      }
+    }
+    return Joi.object({
+      status: Joi.string().valid('validated', 'new', 'all'),
+      'from-id': IdValidate.get(),
+      'to-page': Joi.number().integer(),
+      'page-size': Joi.number().integer().min(1),
+      sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'),
+      csv: Joi.boolean().default(false),
+      fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']),
+      filters: Joi.array().items(Joi.object({
+        mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'),
+        field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')),
+        values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso())).min(1)
+      })).default([])
+    }).with('to-page', 'page-size').validate(data);
+  }
 }

src/routes/validate/template.ts

@@ -1,6 +1,7 @@
 import Joi from '@hapi/joi';
 import IdValidate from './id';
 
+// TODO: do not allow a . in the name
 export default class TemplateValidate {
   private static template = {
     name: Joi.string()

src/test/db.json

@@ -411,12 +411,26 @@
         "_id": {"$oid":"800000000000000000000006"},
         "sample_id": {"$oid":"400000000000000000000006"},
         "values": {
-          "weight %": 0.5,
+          "weight %": 0.6,
           "standard deviation":null
         },
         "status": 0,
         "measurement_template": {"$oid":"300000000000000000000002"},
         "__v": 0
+      },
+      {
+        "_id": {"$oid":"800000000000000000000007"},
+        "sample_id": {"$oid":"400000000000000000000001"},
+        "values": {
+          "dpt": [
+            [3996.12558,98.00555],
+            [3995.08519,98.03253],
+            [3993.04480,98.02657]
+          ]
+        },
+        "status": 10,
+        "measurement_template": {"$oid":"300000000000000000000001"},
+        "__v": 0
       }
     ],
     "condition_templates": [

src/test/helper.ts

@@ -38,15 +38,7 @@ export default class TestHelper {
     return server
   }
 
-  static afterEach (server, done) {
+  static request (server, done, options) {  // options in form: {method, url, contentType, auth: {key/basic: 'name' or 'key'/{name, pass}}, httpStatus, req, res, default (set to false if you want to dismiss default .end handling)}
-    server.close(done);
-  }
-
-  static after(done) {
-    db.disconnect(done);
-  }
-
-  static request (server, done, options) {  // options in form: {method, url, auth: {key/basic: 'name' or 'key'/{name, pass}}, httpStatus, req, res, default (set to false if you want to dismiss default .end handling)}
     let st = supertest(server);
     if (options.hasOwnProperty('auth') && options.auth.hasOwnProperty('key')) {  // resolve API key
       options.url += '?key=' + (this.auth.hasOwnProperty(options.auth.key)? this.auth[options.auth.key].key : options.auth.key);
@@ -79,8 +71,12 @@ export default class TestHelper {
         st = st.auth(options.auth.basic.name, options.auth.basic.pass)
       }
     }
-    st = st.expect('Content-type', /json/)
+    if (options.hasOwnProperty('contentType')) {
-      .expect(options.httpStatus);
+      st = st.expect('Content-type', options.contentType).expect(options.httpStatus);
+    }
+    else {
+      st = st.expect('Content-type', /json/).expect(options.httpStatus);
+    }
     if (options.hasOwnProperty('res')) {  // evaluate result
       return st.end((err, res) => {
         if (err) return done (err);
@@ -128,4 +124,12 @@ export default class TestHelper {
       return st;
     }
   }
+
+  static afterEach (server, done) {
+    server.close(done);
+  }
+
+  static after(done) {
+    db.disconnect(done);
+  }
 }