fixed sample requests for the restructured material

2020-07-17 10:41:19 +02:00
parent 78d35c520e
commit cd81cbf4bd
11 changed files with 115 additions and 28 deletions
--- a/api/schemas.yaml
+++ b/api/schemas.yaml
@@ -121,7 +121,7 @@ Material:
        material_template:
          $ref: 'api.yaml#/components/schemas/Id'
      example:
-        condition_template: 5ea0450ed851c30a90e70894
+        material_template: 5ea0450ed851c30a90e70894
        mineral: 0
        glass_fiber: 40
        carbon_fiber: 0
--- a/data_import/import.js
+++ b/data_import/import.js
@@ -5,6 +5,7 @@ const {Builder} = require('selenium-webdriver');    // selenium and the chrome d
 const chrome = require('selenium-webdriver/chrome');
 const pdfReader = require('pdfreader');
 const iconv = require('iconv-lite');
+const _ = require('lodash');

 const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv';  // metadata files
 const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
@@ -15,6 +16,7 @@ const host = 'http://localhost:3000';
 // const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
 let data = [];  // metadata contents
 let materials = {};
+const numberToColor = {};
 let samples = [];
 let normMaster = {};
 let sampleDevices = {};
@@ -30,7 +32,7 @@ let sampleDevices = {};
 main();

 async function main() {
-  if (0) {  // materials
+  if (1) {  // materials
    await getNormMaster();
    await importCsv(metaDoc);
    await allMaterials();
@@ -42,7 +44,7 @@ async function main() {
    await allMaterials();
    await saveMaterials();
  }
-  if (0) {  // samples
+  if (1) {  // samples
    sampleDeviceMap();
    if (1) {
      console.log('-------- META ----------');
@@ -244,6 +246,7 @@ async function allSamples() {
  });
  const dbMaterials = {}
  res.data.forEach(m => {
+    m.numbers = m.numbers.map(e => ({number: e, color: numberToColor[e]}));
    dbMaterials[m.name] = m;
  })
  res = await axios({
@@ -274,8 +277,6 @@ async function allSamples() {
      if (!material) { // could not find material, skipping sample
        continue;
      }
-      console.log(sample['Material name']);
-      console.log(material._id);
      samples.push({
        number: sample['Sample number'],
        type: sample['Granulate/Part'],
@@ -290,15 +291,21 @@ async function allSamples() {
        samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
      }
      else if (sample['Color'] && sample['Color'] !== '') {
-        let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0);
+        console.log(material);
+        let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['Color'])) >= 0);
        if (!number && /black/.test(sample['Color'])) {  // special case bk for black
          number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
          if (!number) {  // try German word
            number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
          }
        }
+        if (number) {
          samples[si].color = number.color;
        }
+        else {
+          samples[si].color = '';
+        }
+      }
      else if (sampleColors[sample['Sample number'].split('_')[0]]) {  // derive color from main sample for kf/vz
        samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
      }
@@ -373,23 +380,33 @@ async function allMaterials() {
          supplier: trim(sample['Supplier']),
          group: trim(sample['Material'])
        };
+        materials[sample['Material name']].properties = {material_template: '5f0efe6fce7fd20ce4e99013'};
        let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].mineral = tmp ? tmp[1] : 0;
+        materials[sample['Material name']].properties.mineral = tmp ? tmp[1] : 0;
        tmp = /GF(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].glass_fiber = tmp ? tmp[1] : 0;
+        materials[sample['Material name']].properties.glass_fiber = tmp ? tmp[1] : 0;
        tmp = /CF(\d+)/.exec(sample['Reinforcing material']);
-        materials[sample['Material name']].carbon_fiber = tmp ? tmp[1] : 0;
+        materials[sample['Material name']].properties.carbon_fiber = tmp ? tmp[1] : 0;
        materials[sample['Material name']].numbers = await numbersFetch(sample);
        console.log(materials[sample['Material name']]);
      }
    }
  }
+  Object.keys(materials).forEach(mKey => {
+    materials[mKey].numbers.forEach(number => {
+      if (number.number && number.color) {
+        numberToColor[number.number] = number.color;
+      }
+    })
+  });
 }

 async function saveMaterials() {
  const mKeys = Object.keys(materials)
  for (let i in mKeys) {
    console.info(`${i}/${mKeys.length}`);
+    const material = _.cloneDeep(materials[mKeys[i]]);
+    material.numbers = material.numbers.map(e => e.number).filter(e => e !== '').map(e => e.replace(/ /g, ''));
    await axios({
      method: 'post',
      url: host + '/material/new',
@@ -397,10 +414,10 @@ async function saveMaterials() {
        username: 'admin',
        password: 'Abc123!#'
      },
-      data: materials[mKeys[i]]
+      data: material
    }).catch(err => {
      if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
-        console.info(materials[mKeys[i]]);
+        console.info(material);
        console.error(err.response.data);
      }
    });
--- a/src/routes/material.ts
+++ b/src/routes/material.ts
@@ -239,8 +239,11 @@ async function propertiesCheck (properties, param, res, next, checkVersion = tru
  }

  // validate parameters
-  const {error, value: ignore} = ParametersValidate.input(_.omit(properties, 'material_template'), materialData.parameters, param);
+  const {error, value} = ParametersValidate.input(_.omit(properties, 'material_template'), materialData.parameters, param);
  if (error) {res400(error, res); return false;}
+  Object.keys(value).forEach(key => {
+    properties[key] = value[key];
+  });
  return materialData;
 }

--- a/src/routes/sample.spec.ts
+++ b/src/routes/sample.spec.ts
@@ -200,7 +200,7 @@ describe('/sample', () => {
      }).end((err, res) => {
        if (err) return done(err);
        should(res.body[0]).have.property('number', 'Rng36');
-        should(res.body[1]).have.property('number', '33');
+        should(res.body[1]).have.property('number', '34');
        should(res.body[res.body.length - 1]).have.property('number', '1');
        done();
      });
@@ -214,7 +214,7 @@ describe('/sample', () => {
      }).end((err, res) => {
        if (err) return done(err);
        should(res.body[0]).have.property('_id', '400000000000000000000006');
-        should(res.body[1]).have.property('_id', '400000000000000000000002');
+        should(res.body[1]).have.property('_id', '400000000000000000000007');
        done();
      });
    });
@@ -226,7 +226,7 @@ describe('/sample', () => {
        httpStatus: 200
      }).end((err, res) => {
        if (err) return done(err);
-        should(res.body[0]).have.property('_id', '400000000000000000000002');
+        should(res.body[0]).have.property('_id', '400000000000000000000007');
        should(res.body[1]).have.property('_id', '400000000000000000000006');
        done();
      });
@@ -334,6 +334,38 @@ describe('/sample', () => {
        done();
      });
    });
+    it('filters by a measurement properties property', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=material.properties.glass_fiber&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.properties.glass_fiber%22%2C%22values%22%3A%5B%2225%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(2);
+        should(res.body).matchEach(sample => {
+          should(sample.material.properties.glass_fiber).be.eql(25);
+        });
+        done();
+      });
+    });
+    it('filters and sorts by a measurement properties property', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&sort=material.properties.glass_fiber-desc&fields[]=number&fields[]=material.name&fields[]=material.properties.glass_fiber&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.properties.glass_fiber%22%2C%22values%22%3A%5B%2225%22%5D%7D',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).have.lengthOf(2);
+        should(res.body[0].number).be.eql('Rng36');
+        should(res.body[1].number).be.eql('1');
+        should(res.body).matchEach(sample => {
+          should(sample.material.properties.glass_fiber).be.eql(25);
+        });
+        done();
+      });
+    });
    it('filters multiple properties', done => {
      TestHelper.request(server, done, {
        method: 'get',
@@ -342,7 +374,7 @@ describe('/sample', () => {
        httpStatus: 200
      }).end((err, res) => {
        if (err) return done(err);
-        should(res.body).have.lengthOf(3);
+        should(res.body).have.lengthOf(4);
        should(res.body[0]).be.eql({number: '1', batch: ''});
        done();
      });
@@ -359,7 +391,7 @@ describe('/sample', () => {
    it('rejects an invalid filter mode', done => {
      TestHelper.request(server, done, {
        method: 'get',
-        url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
+        url: '/samples?status=all&fields[]=number&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
        auth: {basic: 'janedoe'},
        httpStatus: 400,
        res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'}
@@ -407,6 +439,25 @@ describe('/sample', () => {
        res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', supplier: 'Schulmann'}}]
      });
    });
+    it('returns specified material properties fields', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/samples?status=all&fields[]=number&fields[]=material.properties.glass_fiber&fields[]=material.name',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        const json = require('../test/db.json');
+        should(res.body).matchEach(sample => {
+          const materialId = json.collections.samples.find(e => e.number === sample.number).material_id;
+          const material = json.collections.materials.find(e => e._id.toString() == materialId);
+          should(sample).have.only.keys('number', 'material');
+          should(sample.material.name).be.eql(material.name);
+          should(sample.material.properties.glass_fiber).be.eql(material.properties.glass_fiber);
+        });
+        done()
+      });
+    });
    it('rejects a from-id not in the database', done => {
      TestHelper.request(server, done, {
        method: 'get',
--- a/src/routes/sample.ts
+++ b/src/routes/sample.ts
@@ -22,7 +22,6 @@ import csv from '../helpers/csv';
 const router = express.Router();

 // TODO: check added filter
-// TODO: return total number of pages -> use facet
 // TODO: use query pointer
 // TODO: convert filter value to number according to table model
 // TODO: validation for filter parameters
--- a/src/routes/template.spec.ts
+++ b/src/routes/template.spec.ts
@@ -907,7 +907,7 @@ describe('/template', () => {
        }).end((err, res) => {
          if (err) return done(err);
          const json = require('../test/db.json');
-          should(res.body).have.lengthOf(json.collections.measurement_templates.length);
+          should(res.body).have.lengthOf(json.collections.material_templates.length);
          should(res.body).matchEach(measurement => {
            should(measurement).have.only.keys('_id', 'name', 'version', 'parameters');
            should(measurement).have.property('_id').be.type('string');
--- a/src/routes/template.ts
+++ b/src/routes/template.ts
@@ -4,6 +4,7 @@ import _ from 'lodash';
 import TemplateValidate from './validate/template';
 import ConditionTemplateModel from '../models/condition_template';
 import MeasurementTemplateModel from '../models/measurement_template';
+import MaterialTemplateModel from '../models/material_template';
 import res400 from './validate/res400';
 import IdValidate from './validate/id';
 import mongoose from "mongoose";
@@ -82,5 +83,9 @@ router.post('/template/:collection(measurement|condition|material)/new', async (
 module.exports = router;

 function model (req) {  // return right template model
-  return req.params.collection === 'condition' ? ConditionTemplateModel : MeasurementTemplateModel;
+  switch (req.params.collection) {
+    case 'condition': return ConditionTemplateModel
+    case 'measurement': return MeasurementTemplateModel
+    case 'material': return MaterialTemplateModel
+  }
 }
--- a/src/routes/validate/material.ts
+++ b/src/routes/validate/material.ts
@@ -18,7 +18,7 @@ export default class MaterialValidate {  // validate input for material
    numbers: Joi.array()
      .items(
        Joi.string()
-          .length(10)
+          .max(64)
      )
  };

--- a/src/routes/validate/parameters.ts
+++ b/src/routes/validate/parameters.ts
@@ -10,6 +10,7 @@ export default class ParametersValidate {
          .valid(...parameter.range.values);
      }
      else if (parameter.range.hasOwnProperty('min') && parameter.range.hasOwnProperty('max')) {
+
        joiObject[parameter.name] = Joi.number()
          .min(parameter.range.min)
          .max(parameter.range.max);
--- a/src/routes/validate/sample.ts
+++ b/src/routes/validate/sample.ts
@@ -62,10 +62,8 @@ export default class SampleValidate {
    'material.name',
    'material.supplier',
    'material.group',
-    'material.mineral',
-    'material.glass_fiber',
-    'material.carbon_fiber',
    'material.number',
+    'material.properties.*',
    'measurements.(?!spectrum)*'
  ];

@@ -175,8 +173,8 @@ export default class SampleValidate {
            let validator;
            let field = data.filters[i].field
            if (/material\./.test(field)) {  // select right validation model
-              validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')});
-              field = field.replace('material.', '');
+              validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow(''), properties: Joi.alternatives().try(Joi.number(), Joi.string().max(128))});
+              field = field.replace('material.', '').split('.')[0];
            }
            else if (/measurements\./.test(field)) {
              validator = Joi.object({
@@ -215,7 +213,7 @@ export default class SampleValidate {
      filters: Joi.array().items(Joi.object({
        mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'),
        field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')).messages({'string.pattern.base': 'Invalid filter field name'}),
-        values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso())).min(1)
+        values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso(), Joi.object())).min(1)
      })).default([])
    }).with('to-page', 'page-size').validate(data);
  }
--- a/src/test/db.json
+++ b/src/test/db.json
@@ -95,6 +95,19 @@
        "user_id": {"$oid":"000000000000000000000002"},
        "status": 0,
        "__v": 0
+      },
+      {
+        "_id": {"$oid":"400000000000000000000007"},
+        "number": "34",
+        "type": "liquid",
+        "color": "black",
+        "batch": "",
+        "condition": {},
+        "material_id": {"$oid":"100000000000000000000009"},
+        "note_id": null,
+        "user_id": {"$oid":"000000000000000000000002"},
+        "status": 0,
+        "__v": 0
      }
    ],
    "notes": [