fixed double unwinding with spectrum filter

src/routes/sample.spec.ts

@@ -7,7 +7,6 @@ import TestHelper from "../test/helper";
 import mongoose from 'mongoose';
 
 
-
 describe('/sample', () => {
   let server;
   before(done => TestHelper.before(done));
@@ -255,6 +254,24 @@ describe('/sample', () => {
         done();
       });
     });
+    it('unwinds only once when filtering for spectrum measurements and including the dpt field', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status[]=new&status[]=validated&fields[]=number&fields[]=measurements.spectrum.device&fields=measurements.spectrum.dpt&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22measurements.spectrum.device%22%2C%22values%22%3A%5B%22Alpha%20II%22%5D%7D',
+        auth: {basic: 'admin'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.values.device === 'Alpha II' && e.status !== 'deleted').length);
+        should(res.body).matchEach(sample => {
+          should(sample).have.only.keys('number', 'spectrum');
+          should(sample.spectrum).have.only.keys('device', 'dpt');
+          should(sample.spectrum).have.property('device', 'Alpha II');
+        });
+        done();
+      });
+    })
     it('adds the status if specified', done => {
       TestHelper.request(server, done, {
         method: 'get',
@@ -289,7 +306,7 @@ describe('/sample', () => {
         httpStatus: 200
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.body.filter(e => e.spectrum.dpt)).have.lengthOf(2);
+        should(res.body.filter(e => e.spectrum.dpt)).have.lengthOf(3);
         should(res.body[0].spectrum).have.property('dpt', [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
         should(res.body[1].spectrum).have.property('dpt', [[3996.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
         done();
@@ -783,7 +800,7 @@ describe('/sample', () => {
         url: '/sample/400000000000000000000001',
         auth: {basic: 'janedoe'},
         httpStatus: 200,
-        res: {_id: '400000000000000000000001', number: '1', type: 'as-delivered/raw', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {numbers: ['5513933405'], _id: '100000000000000000000004', name: 'Schulamid 66 GF 25 H', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 25, carbon_fiber: 0}, group: 'PA66', supplier: 'Schulmann'}, user: 'janedoe', notes: {}, measurements: [{_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {device: 'Alpha I'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000007', sample_id: '400000000000000000000001', values: {device: 'Alpha II'}, measurement_template: '300000000000000000000001'}], status: 'validated'}
+        res: {_id: '400000000000000000000001', number: '1', type: 'as-delivered/raw', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {numbers: ['5513933405'], _id: '100000000000000000000004', name: 'Schulamid 66 GF 25 H', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 25, carbon_fiber: 0}, group: 'PA66', supplier: 'Schulmann'}, user: 'janedoe', notes: {}, measurements: [{_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {device: 'Alpha I'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000007', sample_id: '400000000000000000000001', values: {device: 'Alpha II'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000009', sample_id: '400000000000000000000001', values: {device: 'Alpha II'}, measurement_template: '300000000000000000000001'}], status: 'validated'}
       });
     });
     it ('returns spectral data for an admin user', done => {
@@ -792,7 +809,7 @@ describe('/sample', () => {
         url: '/sample/400000000000000000000001',
         auth: {basic: 'admin'},
         httpStatus: 200,
-        res: {_id: '400000000000000000000001', number: '1', type: 'as-delivered/raw', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {numbers: ['5513933405'], _id: '100000000000000000000004', name: 'Schulamid 66 GF 25 H', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 25, carbon_fiber: 0}, group: 'PA66', supplier: 'Schulmann'}, user: 'janedoe', notes: {}, measurements: [{_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[ 3997.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]],device: 'Alpha I'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000007', sample_id: '400000000000000000000001', values: {dpt: [[ 3996.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]], device: 'Alpha II'}, measurement_template: '300000000000000000000001'}], status: 'validated'}
+        res: {_id: '400000000000000000000001', number: '1', type: 'as-delivered/raw', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {numbers: ['5513933405'], _id: '100000000000000000000004', name: 'Schulamid 66 GF 25 H', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 25, carbon_fiber: 0}, group: 'PA66', supplier: 'Schulmann'}, user: 'janedoe', notes: {}, measurements: [{_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[ 3997.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]],device: 'Alpha I'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000007', sample_id: '400000000000000000000001', values: {dpt: [[ 3996.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]], device: 'Alpha II'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000009', sample_id: '400000000000000000000001', values: {dpt: [[ 3996.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]], device: 'Alpha II'}, measurement_template: '300000000000000000000001'}], status: 'validated'}
       });
     });
     it('returns a deleted sample for a dev/admin user', done => {
@@ -1523,7 +1540,7 @@ describe('/sample', () => {
         url: '/sample/number/1',
         auth: {basic: 'janedoe'},
         httpStatus: 200,
-        res: {_id: '400000000000000000000001', number: '1', type: 'as-delivered/raw', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {numbers: ['5513933405'], _id: '100000000000000000000004', name: 'Schulamid 66 GF 25 H', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 25, carbon_fiber: 0}, group: 'PA66', supplier: 'Schulmann'}, user: 'janedoe', notes: {}, measurements: [{_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {device: 'Alpha I'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000007', sample_id: '400000000000000000000001', values: {device: 'Alpha II'}, measurement_template: '300000000000000000000001'}], status: 'validated'}
+        res: {_id: '400000000000000000000001', number: '1', type: 'as-delivered/raw', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {numbers: ['5513933405'], _id: '100000000000000000000004', name: 'Schulamid 66 GF 25 H', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 25, carbon_fiber: 0}, group: 'PA66', supplier: 'Schulmann'}, user: 'janedoe', notes: {}, measurements: [{_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {device: 'Alpha I'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000007', sample_id: '400000000000000000000001', values: {device: 'Alpha II'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000009', sample_id: '400000000000000000000001', values: {device: 'Alpha II'}, measurement_template: '300000000000000000000001'}], status: 'validated'}
       });
     });
     it ('returns spectral data for an admin user', done => {
@@ -1532,7 +1549,7 @@ describe('/sample', () => {
         url: '/sample/number/1',
         auth: {basic: 'admin'},
         httpStatus: 200,
-        res: {_id: '400000000000000000000001', number: '1', type: 'as-delivered/raw', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {numbers: ['5513933405'], _id: '100000000000000000000004', name: 'Schulamid 66 GF 25 H', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 25, carbon_fiber: 0}, group: 'PA66', supplier: 'Schulmann'}, user: 'janedoe', notes: {}, measurements: [{_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[ 3997.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]],device: 'Alpha I'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000007', sample_id: '400000000000000000000001', values: {dpt: [[ 3996.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]], device: 'Alpha II'}, measurement_template: '300000000000000000000001'}], status: 'validated'}
+        res: {_id: '400000000000000000000001', number: '1', type: 'as-delivered/raw', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {numbers: ['5513933405'], _id: '100000000000000000000004', name: 'Schulamid 66 GF 25 H', properties: {material_template: '130000000000000000000003', mineral: 0, glass_fiber: 25, carbon_fiber: 0}, group: 'PA66', supplier: 'Schulmann'}, user: 'janedoe', notes: {}, measurements: [{_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[ 3997.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]],device: 'Alpha I'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000007', sample_id: '400000000000000000000001', values: {dpt: [[ 3996.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]], device: 'Alpha II'}, measurement_template: '300000000000000000000001'}, {_id: '800000000000000000000009', sample_id: '400000000000000000000001', values: {dpt: [[ 3996.12558, 98.00555 ], [ 3995.08519, 98.03253 ], [ 3993.0448, 98.02657 ]], device: 'Alpha II'}, measurement_template: '300000000000000000000001'}], status: 'validated'}
       });
     });
     it('returns 403 for a write user when requesting a deleted sample', done => {

src/routes/sample.ts

@@ -228,13 +228,20 @@ router.get('/samples', async (req, res, next) => {
     if (measurementTemplates.length < measurementFilterFields.length) {
       return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
     }
-    queryPtr.push({$lookup: {
+    const pipeline: any[] = [{$match: {$expr: {$and: [
-        from: 'measurements', let: {sId: '$_id'},
-        pipeline: [{$match: {$expr: {$and: [
             {$eq: ['$sample_id', '$$sId']},
             {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}
           ]}}}
-        ],
+    ];
+    if (measurementFilterFields.indexOf(globals.spectrum.spectrum) >= 0) {  // filter out dpts
+      pipeline.push(
+        {$project: {'values.device': true, measurement_template: true}},
+        {$addFields: {'values._id': '$_id'}}
+      );
+    }
+    queryPtr.push({$lookup: {
+        from: 'measurements', let: {sId: '$_id'},
+        pipeline: pipeline,
         as: 'measurements'
     }});
     const groupedMeasurementTemplates = measurementTemplates.reduce((s, e) => {
@@ -323,10 +330,18 @@ router.get('/samples', async (req, res, next) => {
     }
     // use different lookup methods with and without dpt for the best performance
     if (fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.' + globals.spectrum.dpt) >= 0)) {  // with dpt
+      // spectrum was already used for filters
+      if (sortFilterKeys.find(e => new RegExp('measurements\\.' + globals.spectrum.spectrum).test(e))) {
+        queryPtr.push(
+          {$lookup: {from: 'measurements', localField: 'spectrum._id', foreignField: '_id', as: 'measurements'}}
+        );
+      }
+      else {
         queryPtr.push(
           {$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}
         );
       }
+    }
     else {
       queryPtr.push({$lookup: {
           from: 'measurements', let: {sId: '$_id'},
@@ -705,11 +720,7 @@ async function materialCheck (sample, res, next) {
 
 // validate treatment template, returns false if invalid, otherwise template data
 async function conditionCheck (condition, param, res, next, checkVersion = true) {
-  console.log(condition);
-  console.log(condition.condition_template);
-  console.log(IdValidate.valid(condition.condition_template));
   if (!condition.condition_template || !IdValidate.valid(condition.condition_template)) {  // template id not found
-    console.log('A');
     res.status(400).json({status: 'Condition template not available'});
     return false;
   }
@@ -717,7 +728,6 @@ async function conditionCheck (condition, param, res, next, checkVersion = true)
     .lean().exec().catch(err => next(err)) as any;
   if (conditionData instanceof Error) return false;
   if (!conditionData) {  // template not found
-    console.log('B');
     res.status(400).json({status: 'Condition template not available'});
     return false;
   }

src/test/db.json

@@ -492,6 +492,21 @@
         "status": "deleted",
         "measurement_template": {"$oid":"300000000000000000000001"},
         "__v": 0
+      },
+      {
+        "_id": {"$oid":"800000000000000000000009"},
+        "sample_id": {"$oid":"400000000000000000000001"},
+        "values": {
+          "dpt": [
+            [3996.12558,98.00555],
+            [3995.08519,98.03253],
+            [3993.04480,98.02657]
+          ],
+          "device": "Alpha II"
+        },
+        "status": "validated",
+        "measurement_template": {"$oid":"300000000000000000000001"},
+        "__v": 0
       }
     ],
     "condition_templates": [