implementation of measurement fields

api/sample.yaml

@@ -42,12 +42,14 @@
         schema:
           type: boolean
         example: false
-      - name: fields
+      - name: fields[]
         description: the fields to include in the output as array, defaults to ['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']
         in: query
         schema:
-         type: string
+         type: array
-        example: '&fields[]=number&fields[]=batch'
+         items:
+           type: string
+        example: ['number', 'batch']
     responses:
       200:
         description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)

data_import/import.js

@@ -32,10 +32,10 @@ async function main() {
     await allSamples();
     await saveSamples();
   }
-  else if (0) {  // DPT
+  else if (1) {  // DPT
     await allDpts();
   }
-  else if (1) {  // KF/VZ
+  else if (0) {  // KF/VZ
     await importCsv();
     await allKfVz();
   }

src/helpers/csv.ts

@@ -1,7 +1,34 @@
 import {parseAsync} from 'json2csv';
 
-export default function csv(input: object, fields: string[], f: (err, data) => void) {
+export default function csv(input: any[], f: (err, data) => void) {
-  parseAsync(input)
+  parseAsync(input.map(e => flatten(e)))
     .then(csv => f(null, csv))
     .catch(err => f(err, null));
+}
+
+function flatten (data) {  // flatten object: {a: {b: true}} -> {a.b: true}
+  const result = {};
+  function recurse (cur, prop) {
+    if (Object(cur) !== cur || Object.keys(cur).length === 0) {
+      result[prop] = cur;
+    }
+    else if (Array.isArray(cur)) {
+      let l = 0;
+      for(let i = 0, l = cur.length; i < l; i++)
+        recurse(cur[i], prop + "[" + i + "]");
+      if (l == 0)
+        result[prop] = [];
+    }
+    else {
+      let isEmpty = true;
+      for (let p in cur) {
+        isEmpty = false;
+        recurse(cur[p], prop ? prop+"."+p : p);
+      }
+      if (isEmpty && prop)
+        result[prop] = {};
+    }
+  }
+  recurse(data, '');
+  return result;
 }

src/routes/sample.spec.ts

@@ -244,6 +244,42 @@ describe('/sample', () => {
         done();
       });
     });
+    it('adds the specified measurements', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=measurements.kf',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body.find(e => e.number === '1')).have.property('kf', {});
+        should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null});
+        done();
+      });
+    });
+    it('multiplies the sample information for each spectrum', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=measurements.spectrum',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(2);
+        should(res.body[0]).have.property('spectrum', [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
+        should(res.body[1]).have.property('spectrum', [[3996.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
+        done();
+      });
+    });
+    it('rejects unknown measurement names', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=measurements.xx',
+        auth: {basic: 'janedoe'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: 'Measurement key not found'}
+      });
+    });
     it('returns a correct csv file if specified', done => {
       TestHelper.request(server, done, {
         method: 'get',
@@ -253,9 +289,9 @@ describe('/sample', () => {
         httpStatus: 200
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.text).be.eql('"_id","number","type","color","batch","condition","material_id","note_id","user_id","added"\r\n' +
+        should(res.text).be.eql('"_id","number","type","color","batch","condition.material","condition.weeks","condition.condition_template","material_id","note_id","user_id","added"\r\n' +
-          '"400000000000000000000001","1","granulate","black","","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' +
+          '"400000000000000000000001","1","granulate","black","","copper",3,"200000000000000000000001","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' +
-          '"400000000000000000000002","21","granulate","natural","1560237365","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"');
+          '"400000000000000000000002","21","granulate","natural","1560237365","copper",3,"200000000000000000000001","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"');
         done();
       });
     });
@@ -268,6 +304,15 @@ describe('/sample', () => {
         res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', mineral: 0}}]
       });
     });
+    it('rejects a from-id not in the database', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?from-id=5ea0450ed851c30a90e70894&sort=color-asc',
+        auth: {basic: 'admin'},
+        httpStatus: 400,
+        res: {status: 'Invalid body format', details: 'from-id not found'}
+      });
+    });
     it('rejects an invalid fields parameter', done => {
       TestHelper.request(server, done, {
         method: 'get',
@@ -283,7 +328,7 @@ describe('/sample', () => {
         url: '/samples?status=all&page-size=1&fields[]=xx',
         auth: {basic: 'janedoe'},
         httpStatus: 400,
-        res: {status: 'Invalid body format', details: '"fields[0]" must be one of [_id, color, number, type, batch, added, material.name, material.supplier, material.group, material.mineral, material.glass_fiber, material.carbon_fiber, material.number, condition, material_id, note_id, user_id, material._id, material.numbers]'}
+        res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\..+)$/m'}
       });
     });
     it('rejects a negative page size', done => {
@@ -329,7 +374,7 @@ describe('/sample', () => {
         httpStatus: 401
       });
     });
-  });  // TODO: measurement fields
+  });
 
   describe('GET /samples/{state}', () => {
     it('returns all new samples', done => {

src/routes/sample.ts

@@ -6,6 +6,7 @@ import NoteFieldValidate from './validate/note_field';
 import res400 from './validate/res400';
 import SampleModel from '../models/sample'
 import MeasurementModel from '../models/measurement';
+import MeasurementTemplateModel from '../models/measurement_template';
 import MaterialModel from '../models/material';
 import NoteModel from '../models/note';
 import NoteFieldModel from '../models/note_field';
@@ -63,6 +64,9 @@ router.get('/samples', async (req, res, next) => {
   if (filters['from-id']) {  // from-id specified
     const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);});
     if (fromSample instanceof Error) return;
+    if (!fromSample) {
+      return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
+    }
 
     if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) {  // asc
       query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
@@ -77,10 +81,6 @@ router.get('/samples', async (req, res, next) => {
     query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}});  // set _id as secondary sort
   }
 
-  // if (filters.sort[0].indexOf('material.') >= 0) {  // unpopulate materials again
-  //   query.push({$unset: 'material'});
-  // }
-
   if (filters['to-page']) {
     query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)})  // number to skip, if going back pages, one page has to be skipped less but on sample more
   }
@@ -88,13 +88,38 @@ router.get('/samples', async (req, res, next) => {
   if (filters['page-size']) {
     query.push({$limit: filters['page-size']});
   }
-  console.log(filters.fields);
+
-  const projection = filters.fields.reduce((s, e) => {s[e] = true; return s; }, {});
+  let measurementFields = [];
+  if (filters.fields.find(e => /measurements\./.test(e))) {  // joining measurements is required
+    query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
+    measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', ''));
+    const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
+    if (measurementTemplates instanceof Error) return;
+    if (measurementTemplates.length < measurementFields.length) {
+      return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
+    }
+    measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => {  // TODO: hard coded dpt for special treatment, change later
+      query.push({$set: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
+        vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
+        in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
+      }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}});
+    });
+    console.log(measurementFields);
+    if (measurementFields.find(e => e === 'spectrum')) {  // TODO: remove hardcoded as well
+      query.push(
+        {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
+        {$set: {spectrum: '$spectrum.values.dpt'}},
+        {$unwind: '$spectrum'}
+      );
+    }
+    query.push({$unset: 'measurements'});
+  }
+
+  const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
   if (filters.fields.indexOf('added') >= 0) {  // add added date
     projection.added = {$toDate: '$_id'};
   }
   if (!(filters.fields.indexOf('_id') >= 0)) {  // disable _id explicitly
-    console.log('disable id');
     projection._id = false;
   }
   query.push({$project: projection});
@@ -104,15 +129,16 @@ router.get('/samples', async (req, res, next) => {
     if (filters['to-page'] < 0) {
       data.reverse();
     }
-    if (filters.csv) {  // output as csv  // TODO: csv example in OAS
+    console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));
-      csv(_.compact(data.map(e => SampleValidate.output(e, 'refs'))), ['_id', 'number'], (err, data) => {
+    if (filters.csv) {  // output as csv
+      csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
         if (err) return next(err);
         res.set('Content-Type', 'text/csv');
         res.send(data);
       });
     }
     else {
-      res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs'))));  // validate all and filter null values from validation errors
+      res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));  // validate all and filter null values from validation errors
     }
   })
 });

src/routes/validate/sample.ts

@@ -71,10 +71,12 @@ export default class SampleValidate {
     ...SampleValidate.sortKeys,
     'condition',
     'material_id',
+    'material',
     'note_id',
     'user_id',
     'material._id',
-    'material.numbers'
+    'material.numbers',
+    'measurements.*'
   ];
 
   static input (data, param) {  // validate input, set param to 'new' to make all attributes required
@@ -114,7 +116,7 @@ export default class SampleValidate {
     }
   }
 
-  static output (data, param = 'refs+added') {  // validate output and strip unwanted properties, returns null if not valid
+  static output (data, param = 'refs+added', additionalParams = []) {  // validate output and strip unwanted properties, returns null if not valid
     if (param === 'refs+added') {
       param = 'refs';
       data.added = data._id.getTimestamp();
@@ -130,7 +132,7 @@ export default class SampleValidate {
         batch: this.sample.batch,
         condition: this.sample.condition,
         material_id: IdValidate.get(),
-        material: MaterialValidate.outputV(),
+        material: MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}),
         note_id: IdValidate.get().allow(null),
         user_id: IdValidate.get(),
         added: this.sample.added
@@ -153,6 +155,9 @@ export default class SampleValidate {
     else {
       return null;
     }
+    additionalParams.forEach(param => {
+      joiObject[param] = Joi.any();
+    });
     const {value, error} = Joi.object(joiObject).validate(data, {stripUnknown: true});
     return error !== undefined? null : value;
   }
@@ -165,7 +170,7 @@ export default class SampleValidate {
       'page-size': Joi.number().integer().min(1),
       sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'),
       csv: Joi.boolean().default(false),
-      fields: Joi.array().items(Joi.string().valid(...this.fieldKeys)).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])
+      fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])
     }).with('to-page', 'page-size').validate(data);
   }
 }

src/test/db.json

@@ -417,6 +417,20 @@
         "status": 0,
         "measurement_template": {"$oid":"300000000000000000000002"},
         "__v": 0
+      },
+      {
+        "_id": {"$oid":"800000000000000000000007"},
+        "sample_id": {"$oid":"400000000000000000000001"},
+        "values": {
+          "dpt": [
+            [3996.12558,98.00555],
+            [3995.08519,98.03253],
+            [3993.04480,98.02657]
+          ]
+        },
+        "status": 10,
+        "measurement_template": {"$oid":"300000000000000000000001"},
+        "__v": 0
       }
     ],
     "condition_templates": [