fixed sample requests for the restructured material

api/schemas.yaml

@@ -121,7 +121,7 @@ Material:
         material_template:
           $ref: 'api.yaml#/components/schemas/Id'
       example:
-        condition_template: 5ea0450ed851c30a90e70894
+        material_template: 5ea0450ed851c30a90e70894
         mineral: 0
         glass_fiber: 40
         carbon_fiber: 0

data_import/import.js

@@ -5,6 +5,7 @@ const {Builder} = require('selenium-webdriver');    // selenium and the chrome d
 const chrome = require('selenium-webdriver/chrome');
 const pdfReader = require('pdfreader');
 const iconv = require('iconv-lite');
+const _ = require('lodash');
 
 const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv';  // metadata files
 const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
@@ -15,6 +16,7 @@ const host = 'http://localhost:3000';
 // const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
 let data = [];  // metadata contents
 let materials = {};
+const numberToColor = {};
 let samples = [];
 let normMaster = {};
 let sampleDevices = {};
@@ -30,7 +32,7 @@ let sampleDevices = {};
 main();
 
 async function main() {
-  if (0) {  // materials
+  if (1) {  // materials
     await getNormMaster();
     await importCsv(metaDoc);
     await allMaterials();
@@ -42,7 +44,7 @@ async function main() {
     await allMaterials();
     await saveMaterials();
   }
-  if (0) {  // samples
+  if (1) {  // samples
     sampleDeviceMap();
     if (1) {
       console.log('-------- META ----------');
@@ -244,6 +246,7 @@ async function allSamples() {
   });
   const dbMaterials = {}
   res.data.forEach(m => {
+    m.numbers = m.numbers.map(e => ({number: e, color: numberToColor[e]}));
     dbMaterials[m.name] = m;
   })
   res = await axios({
@@ -274,8 +277,6 @@ async function allSamples() {
       if (!material) { // could not find material, skipping sample
         continue;
       }
-      console.log(sample['Material name']);
-      console.log(material._id);
       samples.push({
         number: sample['Sample number'],
         type: sample['Granulate/Part'],
@@ -290,14 +291,20 @@ async function allSamples() {
         samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
       }
       else if (sample['Color'] && sample['Color'] !== '') {
-        let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0);
+        console.log(material);
+        let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['Color'])) >= 0);
         if (!number && /black/.test(sample['Color'])) {  // special case bk for black
           number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
           if (!number) {  // try German word
             number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
           }
         }
-        samples[si].color = number.color;
+        if (number) {
+          samples[si].color = number.color;
+        }
+        else {
+          samples[si].color = '';
+        }
       }
       else if (sampleColors[sample['Sample number'].split('_')[0]]) {  // derive color from main sample for kf/vz
         samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
@@ -373,23 +380,33 @@ async function allMaterials() {
           supplier: trim(sample['Supplier']),
           group: trim(sample['Material'])
         };
+        materials[sample['Material name']].properties = {material_template: '5f0efe6fce7fd20ce4e99013'};
         let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].mineral = tmp ? tmp[1] : 0;
+        materials[sample['Material name']].properties.mineral = tmp ? tmp[1] : 0;
         tmp = /GF(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].glass_fiber = tmp ? tmp[1] : 0;
+        materials[sample['Material name']].properties.glass_fiber = tmp ? tmp[1] : 0;
         tmp = /CF(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].carbon_fiber = tmp ? tmp[1] : 0;
+        materials[sample['Material name']].properties.carbon_fiber = tmp ? tmp[1] : 0;
         materials[sample['Material name']].numbers = await numbersFetch(sample);
         console.log(materials[sample['Material name']]);
       }
     }
   }
+  Object.keys(materials).forEach(mKey => {
+    materials[mKey].numbers.forEach(number => {
+      if (number.number && number.color) {
+        numberToColor[number.number] = number.color;
+      }
+    })
+  });
 }
 
 async function saveMaterials() {
   const mKeys = Object.keys(materials)
   for (let i in mKeys) {
     console.info(`${i}/${mKeys.length}`);
+    const material = _.cloneDeep(materials[mKeys[i]]);
+    material.numbers = material.numbers.map(e => e.number).filter(e => e !== '').map(e => e.replace(/ /g, ''));
     await axios({
       method: 'post',
       url: host + '/material/new',
@@ -397,10 +414,10 @@ async function saveMaterials() {
         username: 'admin',
         password: 'Abc123!#'
       },
-      data: materials[mKeys[i]]
+      data: material
     }).catch(err => {
       if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
-        console.info(materials[mKeys[i]]);
+        console.info(material);
         console.error(err.response.data);
       }
     });

src/routes/material.ts

@@ -239,8 +239,11 @@ async function propertiesCheck (properties, param, res, next, checkVersion = tru
   }
 
   // validate parameters
-  const {error, value: ignore} = ParametersValidate.input(_.omit(properties, 'material_template'), materialData.parameters, param);
+  const {error, value} = ParametersValidate.input(_.omit(properties, 'material_template'), materialData.parameters, param);
   if (error) {res400(error, res); return false;}
+  Object.keys(value).forEach(key => {
+    properties[key] = value[key];
+  });
   return materialData;
 }
 

src/routes/sample.spec.ts

@@ -200,7 +200,7 @@ describe('/sample', () => {
       }).end((err, res) => {
         if (err) return done(err);
         should(res.body[0]).have.property('number', 'Rng36');
-        should(res.body[1]).have.property('number', '33');
+        should(res.body[1]).have.property('number', '34');
         should(res.body[res.body.length - 1]).have.property('number', '1');
         done();
       });
@@ -214,7 +214,7 @@ describe('/sample', () => {
       }).end((err, res) => {
         if (err) return done(err);
         should(res.body[0]).have.property('_id', '400000000000000000000006');
-        should(res.body[1]).have.property('_id', '400000000000000000000002');
+        should(res.body[1]).have.property('_id', '400000000000000000000007');
         done();
       });
     });
@@ -226,7 +226,7 @@ describe('/sample', () => {
         httpStatus: 200
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.body[0]).have.property('_id', '400000000000000000000002');
+        should(res.body[0]).have.property('_id', '400000000000000000000007');
         should(res.body[1]).have.property('_id', '400000000000000000000006');
         done();
       });
@@ -334,6 +334,38 @@ describe('/sample', () => {
         done();
       });
     });
+    it('filters by a measurement properties property', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=material.properties.glass_fiber&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.properties.glass_fiber%22%2C%22values%22%3A%5B%2225%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(2);
+        should(res.body).matchEach(sample => {
+          should(sample.material.properties.glass_fiber).be.eql(25);
+        });
+        done();
+      });
+    });
+    it('filters and sorts by a measurement properties property', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&sort=material.properties.glass_fiber-desc&fields[]=number&fields[]=material.name&fields[]=material.properties.glass_fiber&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.properties.glass_fiber%22%2C%22values%22%3A%5B%2225%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(2);
+        should(res.body[0].number).be.eql('Rng36');
+        should(res.body[1].number).be.eql('1');
+        should(res.body).matchEach(sample => {
+          should(sample.material.properties.glass_fiber).be.eql(25);
+        });
+        done();
+      });
+    });
     it('filters multiple properties', done => {
       TestHelper.request(server, done, {
         method: 'get',
@@ -342,7 +374,7 @@ describe('/sample', () => {
         httpStatus: 200
       }).end((err, res) => {
         if (err) return done(err);
-        should(res.body).have.lengthOf(3);
+        should(res.body).have.lengthOf(4);
         should(res.body[0]).be.eql({number: '1', batch: ''});
         done();
       });
@@ -359,7 +391,7 @@ describe('/sample', () => {
     it('rejects an invalid filter mode', done => {
       TestHelper.request(server, done, {
         method: 'get',
-        url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
+        url: '/samples?status=all&fields[]=number&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
         auth: {basic: 'janedoe'},
         httpStatus: 400,
         res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'}
@@ -407,6 +439,25 @@ describe('/sample', () => {
         res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', supplier: 'Schulmann'}}]
       });
     });
+    it('returns specified material properties fields', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.properties.glass_fiber&fields[]=material.name',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).matchEach(sample => {
+          const materialId = json.collections.samples.find(e => e.number === sample.number).material_id;
+          const material = json.collections.materials.find(e => e._id.toString() == materialId);
+          should(sample).have.only.keys('number', 'material');
+          should(sample.material.name).be.eql(material.name);
+          should(sample.material.properties.glass_fiber).be.eql(material.properties.glass_fiber);
+        });
+        done()
+      });
+    });
     it('rejects a from-id not in the database', done => {
       TestHelper.request(server, done, {
         method: 'get',

src/routes/sample.ts

@@ -22,7 +22,6 @@ import csv from '../helpers/csv';
 const router = express.Router();
 
 // TODO: check added filter
-// TODO: return total number of pages -> use facet
 // TODO: use query pointer
 // TODO: convert filter value to number according to table model
 // TODO: validation for filter parameters

src/routes/template.spec.ts

@@ -907,7 +907,7 @@ describe('/template', () => {
         }).end((err, res) => {
           if (err) return done(err);
           const json = require('../test/db.json');
-          should(res.body).have.lengthOf(json.collections.measurement_templates.length);
+          should(res.body).have.lengthOf(json.collections.material_templates.length);
           should(res.body).matchEach(measurement => {
             should(measurement).have.only.keys('_id', 'name', 'version', 'parameters');
             should(measurement).have.property('_id').be.type('string');

src/routes/template.ts

@@ -4,6 +4,7 @@ import _ from 'lodash';
 import TemplateValidate from './validate/template';
 import ConditionTemplateModel from '../models/condition_template';
 import MeasurementTemplateModel from '../models/measurement_template';
+import MaterialTemplateModel from '../models/material_template';
 import res400 from './validate/res400';
 import IdValidate from './validate/id';
 import mongoose from "mongoose";
@@ -82,5 +83,9 @@ router.post('/template/:collection(measurement|condition|material)/new', async (
 module.exports = router;
 
 function model (req) {  // return right template model
-  return req.params.collection === 'condition' ? ConditionTemplateModel : MeasurementTemplateModel;
+  switch (req.params.collection) {
+    case 'condition': return ConditionTemplateModel
+    case 'measurement': return MeasurementTemplateModel
+    case 'material': return MaterialTemplateModel
+  }
 }

src/routes/validate/material.ts

@@ -18,7 +18,7 @@ export default class MaterialValidate {  // validate input for material
     numbers: Joi.array()
       .items(
         Joi.string()
-          .length(10)
+          .max(64)
       )
   };
 

src/routes/validate/parameters.ts

@@ -10,6 +10,7 @@ export default class ParametersValidate {
           .valid(...parameter.range.values);
       }
       else if (parameter.range.hasOwnProperty('min') && parameter.range.hasOwnProperty('max')) {
+
         joiObject[parameter.name] = Joi.number()
           .min(parameter.range.min)
           .max(parameter.range.max);

src/routes/validate/sample.ts

@@ -62,10 +62,8 @@ export default class SampleValidate {
     'material.name',
     'material.supplier',
     'material.group',
-    'material.mineral',
-    'material.glass_fiber',
-    'material.carbon_fiber',
     'material.number',
+    'material.properties.*',
     'measurements.(?!spectrum)*'
   ];
 
@@ -175,8 +173,8 @@ export default class SampleValidate {
             let validator;
             let field = data.filters[i].field
             if (/material\./.test(field)) {  // select right validation model
-              validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')});
+              validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow(''), properties: Joi.alternatives().try(Joi.number(), Joi.string().max(128))});
-              field = field.replace('material.', '');
+              field = field.replace('material.', '').split('.')[0];
             }
             else if (/measurements\./.test(field)) {
               validator = Joi.object({
@@ -215,7 +213,7 @@ export default class SampleValidate {
       filters: Joi.array().items(Joi.object({
         mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'),
         field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')).messages({'string.pattern.base': 'Invalid filter field name'}),
-        values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso())).min(1)
+        values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso(), Joi.object())).min(1)
       })).default([])
     }).with('to-page', 'page-size').validate(data);
   }

src/test/db.json

@@ -95,6 +95,19 @@
         "user_id": {"$oid":"000000000000000000000002"},
         "status": 0,
         "__v": 0
+      },
+      {
+        "_id": {"$oid":"400000000000000000000007"},
+        "number": "34",
+        "type": "liquid",
+        "color": "black",
+        "batch": "",
+        "condition": {},
+        "material_id": {"$oid":"100000000000000000000009"},
+        "note_id": null,
+        "user_id": {"$oid":"000000000000000000000002"},
+        "status": 0,
+        "__v": 0
       }
     ],
     "notes": [