restructured aggregation

data_import/import.js

@@ -16,6 +16,7 @@ let normMaster = {};
 
 // TODO: integrate measurement device information from DPT names using different users
 // TODO: supplier: other for supplierless samples
+// TODO: BASF twice, BASF as color
 
 main();
 

src/helpers/csv.ts

@@ -1,7 +1,13 @@
 import {parseAsync} from 'json2csv';
 
 export default function csv(input: any[], f: (err, data) => void) {
-  parseAsync(input.map(e => flatten(e)))
+  console.log(input[1000]);
+  console.log(flatten(input[1000]));
+  parseAsync([flatten(input[1000])]).then(csv => console.log(csv));
+  console.log(input[1]);
+  console.log(flatten(input[1]));
+  parseAsync([flatten(input[1])]).then(csv => console.log(csv));
+  parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
     .then(csv => f(null, csv))
     .catch(err => f(err, null));
 }

src/index.ts

@@ -7,6 +7,7 @@ import helmet from 'helmet';
 import api from './api';
 import db from './db';
 
+// TODO: working demo branch
 
 // tell if server is running in debug or production environment
 console.info(process.env.NODE_ENV === 'production' ? '===== PRODUCTION =====' : process.env.NODE_ENV === 'test' ? '' :'===== DEVELOPMENT =====');

src/models/material.ts

@@ -22,5 +22,7 @@ MaterialSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>>
   db.log(req, this);
   return this;
 }
+MaterialSchema.index({supplier_id: 1});
+MaterialSchema.index({group_id: 1});
 
 export default mongoose.model<any, mongoose.Model<any, any>>('material', MaterialSchema);

src/models/measurement.ts

@@ -17,5 +17,7 @@ MeasurementSchema.query.log = function <Q extends mongoose.DocumentQuery<any, an
   db.log(req, this);
   return this;
 }
+MeasurementSchema.index({sample_id: 1});
+MeasurementSchema.index({measurement_template: 1});
 
 export default mongoose.model<any, mongoose.Model<any, any>>('measurement', MeasurementSchema);

src/models/sample.ts

@@ -22,5 +22,8 @@ SampleSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>> (
   db.log(req, this);
   return this;
 }
+SampleSchema.index({material_id: 1});
+SampleSchema.index({note_id: 1});
+SampleSchema.index({user_id: 1});
 
 export default mongoose.model<any, mongoose.Model<any, any>>('sample', SampleSchema);

src/routes/sample.spec.ts

@@ -252,7 +252,7 @@ describe('/sample', () => {
         httpStatus: 200
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.body.find(e => e.number === '1')).have.property('kf', {});
+        should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null});
         should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null});
         done();
       });
@@ -328,7 +328,7 @@ describe('/sample', () => {
         url: '/samples?status=all&page-size=1&fields[]=xx',
         auth: {basic: 'janedoe'},
         httpStatus: 400,
-        res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\..+)$/m'}
+        res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'}
       });
     });
     it('rejects a negative page size', done => {

src/routes/sample.ts

@@ -27,21 +27,10 @@ router.get('/samples', async (req, res, next) => {
   const {error, value: filters} = SampleValidate.query(req.query);
   if (error) return res400(error, res);
 
-  const query = [];
+  // TODO: find a better place for these
-  query.push({$match: {$and: []}});
+  const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', 'user_id'];
-  if (filters.hasOwnProperty('status')) {
-    if(filters.status === 'all') {
-      query[0].$match.$and.push({$or: [{status: globals.status.validated}, {status: globals.status.new}]});
-    }
-    else {
-      query[0].$match.$and.push({status: globals.status[filters.status]});
-    }
-  }
-  else {  // default
-    query[0].$match.$and.push({status: globals.status.validated});
-  }
 
-  // sorting
+  // evaluate sort parameter from 'color-asc' to ['color', 1]
   filters.sort = filters.sort.split('-');
   filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0];  // route added sorting criteria to _id
   filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1;
@@ -49,62 +38,139 @@ router.get('/samples', async (req, res, next) => {
     filters['to-page'] = 0;
   }
 
-  query.push(
+  let collection;
-    {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
+  const query = [];
-    {$set: {material: { $arrayElemAt: ['$material', 0]}}},
+  query.push({$match: {$and: []}});
-    {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
+
-    {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}},
+  if (filters.sort[0].indexOf('measurements.') >= 0) {  // sorting with measurements as starting collection
-    {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
+    collection = MeasurementModel;
-    {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}},
+    const measurementName = filters.sort[0].replace('measurements.', '');
-    {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}
+    const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);});
+    if (measurementTemplate instanceof Error) return;
+    if (!measurementTemplate) {
+      return res.status(400).json({status: 'Invalid body format', details: filters.sort[0] + ' not found'});
+    }
+    let sortStartValue = null;
+    if (filters['from-id']) {  // from-id specified, fetch values for sorting
+      const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}).lean().exec().catch(err => {next(err);});  // TODO: what if more than one measurement for sample?
+      if (fromSample instanceof Error) return;
+      if (!fromSample) {
+        return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
       }
+      sortStartValue = fromSample.values[measurementTemplate.parameters[0].name];
     }
-  );
+    query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)});  // find measurements to sort
+    query.push(
+      sortQuery(query, filters, ['values.' + measurementTemplate.parameters[0].name, 'sample_id'], sortStartValue),  // sort measurements
+      {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}},                                  // fetch samples and restructure them to fit sample structure
+      {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
+      {$match: statusQuery(filters, 'sample.status')}  // filter out wrong status once samples were added
+    );
+    addSkipLimit(query, filters);  // skip and limit to select right page
+    query.push(
+      {$set: {['sample.' + measurementName]: '$measurement.values'}},  // more restructuring
+      {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
+    );
 
-  if (filters['from-id']) {  // from-id specified
+  }
-    const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);});
+  else {  // sorting with samples as starting collection
-    if (fromSample instanceof Error) return;
+    collection = SampleModel;
-    if (!fromSample) {
+    // filter for status
-      return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
+    query[0].$match.$and.push(statusQuery(filters, 'status'));
-    }
 
-    if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) {  // asc
+    // differentiate by sort key to do sorting, skip and limit as early as possible
-      query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
+    if (sampleKeys.indexOf(filters.sort[0]) >= 0) {  // sorting for sample keys
-      query.push({$sort: {[filters.sort[0]]: 1, _id: 1}});
+      let sortStartValue = null;
+      if (filters['from-id']) {  // from-id specified
+        const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);});
+        if (fromSample instanceof Error) return;
+        if (!fromSample) {
+          return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
+        }
+        sortStartValue = fromSample[filters.sort[0]];
+      }
+      query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
+      addSkipLimit(query, filters);
     }
-    else {
+    else {  // sorting for material keys
-      query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$lt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
+      let materialQuery = []
-      query.push({$sort: {[filters.sort[0]]: -1, _id: -1}});
+      materialQuery.push(  // add material properties
+        {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
+        {$set: {material: { $arrayElemAt: ['$material', 0]}}}
+      );
+      if (filters.sort[0] === 'material.supplier') {  // add supplier if needed
+        materialQuery.push(
+          {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
+          {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
+        );
+      }
+      if (filters.sort[0] === 'material.group') {  // add group if needed
+        materialQuery.push(
+          {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
+          {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
+        );
+      }
+      if (filters.sort[0] === 'material.number') {  // add material number if needed
+        materialQuery.push(
+          {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
+        );
+      }
+      query.push(...materialQuery);
+      let sortStartValue = null;
+      if (filters['from-id']) {  // from-id specified
+        const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);});
+        if (fromSample instanceof Error) return;
+        if (!fromSample) {
+          return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
+        }
+        sortStartValue = fromSample[filters.sort[0]];
+      }
+      query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
+      addSkipLimit(query, filters);
     }
   }
-  else {  // sort from beginning
+
-    query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}});  // set _id as secondary sort
+  const fieldsNoSort = filters.fields.filter(e => e !== filters.sort[0]);  // sort field was definitely added already, exclude from further field operations
+  if (fieldsNoSort.find(e => /material\./.test(e))) {  //  add material fields
+    query.push(
+      {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
+      {$set: {material: { $arrayElemAt: ['$material', 0]}}}
+    );
+  }
+  if (fieldsNoSort.indexOf('material.supplier') >= 0) {  // add supplier if needed
+    query.push(
+      {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
+      {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
+    );
+  }
+  if (fieldsNoSort.indexOf('material.group') >= 0) {  // add group if needed
+    query.push(
+      {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
+      {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
+    );
+  }
+  if (fieldsNoSort.indexOf('material.number') >= 0) {  // add material number if needed
+    query.push(
+      {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
+    );
   }
 
-  if (filters['to-page']) {
+  let measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', ''));
-    query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)})  // number to skip, if going back pages, one page has to be skipped less but on sample more
+  console.log(fieldsNoSort);
-  }
+  console.log(measurementFields);
-
+  if (fieldsNoSort.find(e => /measurements\./.test(e))) {  //  add measurement fields
-  if (filters['page-size']) {
-    query.push({$limit: filters['page-size']});
-  }
-
-  let measurementFields = [];
-  if (filters.fields.find(e => /measurements\./.test(e))) {  // joining measurements is required
     query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
-    measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', ''));
+    const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
-    const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
     if (measurementTemplates instanceof Error) return;
     if (measurementTemplates.length < measurementFields.length) {
       return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
     }
     measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => {  // TODO: hard coded dpt for special treatment, change later
       query.push({$set: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
-        vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
+              vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
-        in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
+              in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
-      }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}});
+            }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
     });
-    console.log(measurementFields);
     if (measurementFields.find(e => e === 'spectrum')) {  // TODO: remove hardcoded as well
       query.push(
         {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
@@ -124,12 +190,11 @@ router.get('/samples', async (req, res, next) => {
   }
   query.push({$project: projection});
 
-  SampleModel.aggregate(query).exec((err, data) => {
+  collection.aggregate(query).exec((err, data) => {
     if (err) return next(err);
     if (filters['to-page'] < 0) {
       data.reverse();
     }
-    console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));
     if (filters.csv) {  // output as csv
       csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
         if (err) return next(err);
@@ -499,3 +564,41 @@ function customFieldsChange (fields, amount, req) {  // update custom_fields and
     });
   });
 }
+
+function sortQuery(query, filters, sortKeys, sortStartValue) {  // sortKeys = ['primary key', 'secondary key']
+  if (filters['from-id']) {  // from-id specified
+    if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) {  // asc
+      query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
+      return {$sort: {[sortKeys[0]]: 1, _id: 1}};
+    } else {
+      query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
+      return {$sort: {[sortKeys[0]]: -1, _id: -1}};
+    }
+  } else {  // sort from beginning
+    return {$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}};  // set _id as secondary sort
+  }
+}
+
+function addSkipLimit(query, filters) {
+  if (filters['to-page']) {
+    query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)})  // number to skip, if going back pages, one page has to be skipped less but on sample more
+  }
+
+  if (filters['page-size']) {
+    query.push({$limit: filters['page-size']});
+  }
+}
+
+function statusQuery(filters, field) {
+  if (filters.hasOwnProperty('status')) {
+    if(filters.status === 'all') {
+      return {$or: [{[field]: globals.status.validated}, {[field]: globals.status.new}]};
+    }
+    else {
+      return {[field]: globals.status[filters.status]};
+    }
+  }
+  else {  // default
+    return {[field]: globals.status.validated};
+  }
+}

src/routes/validate/sample.ts

@@ -64,7 +64,8 @@ export default class SampleValidate {
     'material.mineral',
     'material.glass_fiber',
     'material.carbon_fiber',
-    'material.number'
+    'material.number',
+    'measurements.(?!spectrum)*'
   ];
 
   private static fieldKeys = [
@@ -76,7 +77,7 @@ export default class SampleValidate {
     'user_id',
     'material._id',
     'material.numbers',
-    'measurements.*'
+    'measurements.spectrum'
   ];
 
   static input (data, param) {  // validate input, set param to 'new' to make all attributes required
@@ -168,7 +169,7 @@ export default class SampleValidate {
       'from-id': IdValidate.get(),
       'to-page': Joi.number().integer(),
       'page-size': Joi.number().integer().min(1),
-      sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'),
+      sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'),
       csv: Joi.boolean().default(false),
       fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])
     }).with('to-page', 'page-size').validate(data);