From cd81cbf4bdbc13df8e227d20cdf32fc9d99356d0 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Fri, 17 Jul 2020 10:41:19 +0200 Subject: [PATCH] fixed sample requests for the restructured material --- api/schemas.yaml | 2 +- data_import/import.js | 39 ++++++++++++++------ src/routes/material.ts | 5 ++- src/routes/sample.spec.ts | 61 ++++++++++++++++++++++++++++--- src/routes/sample.ts | 1 - src/routes/template.spec.ts | 2 +- src/routes/template.ts | 7 +++- src/routes/validate/material.ts | 2 +- src/routes/validate/parameters.ts | 1 + src/routes/validate/sample.ts | 10 ++--- src/test/db.json | 13 +++++++ 11 files changed, 115 insertions(+), 28 deletions(-) diff --git a/api/schemas.yaml b/api/schemas.yaml index 1f41d46..3a332e0 100644 --- a/api/schemas.yaml +++ b/api/schemas.yaml @@ -121,7 +121,7 @@ Material: material_template: $ref: 'api.yaml#/components/schemas/Id' example: - condition_template: 5ea0450ed851c30a90e70894 + material_template: 5ea0450ed851c30a90e70894 mineral: 0 glass_fiber: 40 carbon_fiber: 0 diff --git a/data_import/import.js b/data_import/import.js index dc8c8d8..06c31be 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -5,6 +5,7 @@ const {Builder} = require('selenium-webdriver'); // selenium and the chrome d const chrome = require('selenium-webdriver/chrome'); const pdfReader = require('pdfreader'); const iconv = require('iconv-lite'); +const _ = require('lodash'); const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv'; // metadata files const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv'; @@ -15,6 +16,7 @@ const host = 'http://localhost:3000'; // const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; let data = []; // metadata contents let materials = {}; +const numberToColor = {}; let samples = []; let normMaster = {}; let sampleDevices = {}; @@ -30,7 +32,7 @@ let sampleDevices = {}; main(); async function main() { - if (0) { // materials + if (1) { // materials await getNormMaster(); await importCsv(metaDoc); await allMaterials(); @@ -42,7 +44,7 @@ async function main() { await allMaterials(); await saveMaterials(); } - if (0) { // samples + if (1) { // samples sampleDeviceMap(); if (1) { console.log('-------- META ----------'); @@ -244,6 +246,7 @@ async function allSamples() { }); const dbMaterials = {} res.data.forEach(m => { + m.numbers = m.numbers.map(e => ({number: e, color: numberToColor[e]})); dbMaterials[m.name] = m; }) res = await axios({ @@ -274,8 +277,6 @@ async function allSamples() { if (!material) { // could not find material, skipping sample continue; } - console.log(sample['Material name']); - console.log(material._id); samples.push({ number: sample['Sample number'], type: sample['Granulate/Part'], @@ -290,14 +291,20 @@ async function allSamples() { samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color; } else if (sample['Color'] && sample['Color'] !== '') { - let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0); + console.log(material); + let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['Color'])) >= 0); if (!number && /black/.test(sample['Color'])) { // special case bk for black number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0); if (!number) { // try German word number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0); } } - samples[si].color = number.color; + if (number) { + samples[si].color = number.color; + } + else { + samples[si].color = ''; + } } else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz samples[si].color = sampleColors[sample['Sample number'].split('_')[0]]; @@ -373,23 +380,33 @@ async function allMaterials() { supplier: trim(sample['Supplier']), group: trim(sample['Material']) }; + materials[sample['Material name']].properties = {material_template: '5f0efe6fce7fd20ce4e99013'}; let tmp = /M(\d+)/.exec(sample['Reinforcing material']); - materials[sample['Material name']].mineral = tmp ? tmp[1] : 0; + materials[sample['Material name']].properties.mineral = tmp ? tmp[1] : 0; tmp = /GF(\d+)/.exec(sample['Reinforcing material']); - materials[sample['Material name']].glass_fiber = tmp ? tmp[1] : 0; + materials[sample['Material name']].properties.glass_fiber = tmp ? tmp[1] : 0; tmp = /CF(\d+)/.exec(sample['Reinforcing material']); - materials[sample['Material name']].carbon_fiber = tmp ? tmp[1] : 0; + materials[sample['Material name']].properties.carbon_fiber = tmp ? tmp[1] : 0; materials[sample['Material name']].numbers = await numbersFetch(sample); console.log(materials[sample['Material name']]); } } } + Object.keys(materials).forEach(mKey => { + materials[mKey].numbers.forEach(number => { + if (number.number && number.color) { + numberToColor[number.number] = number.color; + } + }) + }); } async function saveMaterials() { const mKeys = Object.keys(materials) for (let i in mKeys) { console.info(`${i}/${mKeys.length}`); + const material = _.cloneDeep(materials[mKeys[i]]); + material.numbers = material.numbers.map(e => e.number).filter(e => e !== '').map(e => e.replace(/ /g, '')); await axios({ method: 'post', url: host + '/material/new', @@ -397,10 +414,10 @@ async function saveMaterials() { username: 'admin', password: 'Abc123!#' }, - data: materials[mKeys[i]] + data: material }).catch(err => { if (err.response.data.status && err.response.data.status !== 'Material name already taken') { - console.info(materials[mKeys[i]]); + console.info(material); console.error(err.response.data); } }); diff --git a/src/routes/material.ts b/src/routes/material.ts index 43c818b..54a49ab 100644 --- a/src/routes/material.ts +++ b/src/routes/material.ts @@ -239,8 +239,11 @@ async function propertiesCheck (properties, param, res, next, checkVersion = tru } // validate parameters - const {error, value: ignore} = ParametersValidate.input(_.omit(properties, 'material_template'), materialData.parameters, param); + const {error, value} = ParametersValidate.input(_.omit(properties, 'material_template'), materialData.parameters, param); if (error) {res400(error, res); return false;} + Object.keys(value).forEach(key => { + properties[key] = value[key]; + }); return materialData; } diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 25289c0..8d6c515 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -200,7 +200,7 @@ describe('/sample', () => { }).end((err, res) => { if (err) return done(err); should(res.body[0]).have.property('number', 'Rng36'); - should(res.body[1]).have.property('number', '33'); + should(res.body[1]).have.property('number', '34'); should(res.body[res.body.length - 1]).have.property('number', '1'); done(); }); @@ -214,7 +214,7 @@ describe('/sample', () => { }).end((err, res) => { if (err) return done(err); should(res.body[0]).have.property('_id', '400000000000000000000006'); - should(res.body[1]).have.property('_id', '400000000000000000000002'); + should(res.body[1]).have.property('_id', '400000000000000000000007'); done(); }); }); @@ -226,7 +226,7 @@ describe('/sample', () => { httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.body[0]).have.property('_id', '400000000000000000000002'); + should(res.body[0]).have.property('_id', '400000000000000000000007'); should(res.body[1]).have.property('_id', '400000000000000000000006'); done(); }); @@ -334,6 +334,38 @@ describe('/sample', () => { done(); }); }); + it('filters by a measurement properties property', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=material.properties.glass_fiber&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.properties.glass_fiber%22%2C%22values%22%3A%5B%2225%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(2); + should(res.body).matchEach(sample => { + should(sample.material.properties.glass_fiber).be.eql(25); + }); + done(); + }); + }); + it('filters and sorts by a measurement properties property', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&sort=material.properties.glass_fiber-desc&fields[]=number&fields[]=material.name&fields[]=material.properties.glass_fiber&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.properties.glass_fiber%22%2C%22values%22%3A%5B%2225%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(2); + should(res.body[0].number).be.eql('Rng36'); + should(res.body[1].number).be.eql('1'); + should(res.body).matchEach(sample => { + should(sample.material.properties.glass_fiber).be.eql(25); + }); + done(); + }); + }); it('filters multiple properties', done => { TestHelper.request(server, done, { method: 'get', @@ -342,7 +374,7 @@ describe('/sample', () => { httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.body).have.lengthOf(3); + should(res.body).have.lengthOf(4); should(res.body[0]).be.eql({number: '1', batch: ''}); done(); }); @@ -359,7 +391,7 @@ describe('/sample', () => { it('rejects an invalid filter mode', done => { TestHelper.request(server, done, { method: 'get', - url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', + url: '/samples?status=all&fields[]=number&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', auth: {basic: 'janedoe'}, httpStatus: 400, res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'} @@ -407,6 +439,25 @@ describe('/sample', () => { res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', supplier: 'Schulmann'}}] }); }); + it('returns specified material properties fields', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.properties.glass_fiber&fields[]=material.name', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).matchEach(sample => { + const materialId = json.collections.samples.find(e => e.number === sample.number).material_id; + const material = json.collections.materials.find(e => e._id.toString() == materialId); + should(sample).have.only.keys('number', 'material'); + should(sample.material.name).be.eql(material.name); + should(sample.material.properties.glass_fiber).be.eql(material.properties.glass_fiber); + }); + done() + }); + }); it('rejects a from-id not in the database', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 474a04e..67eb7b5 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -22,7 +22,6 @@ import csv from '../helpers/csv'; const router = express.Router(); // TODO: check added filter -// TODO: return total number of pages -> use facet // TODO: use query pointer // TODO: convert filter value to number according to table model // TODO: validation for filter parameters diff --git a/src/routes/template.spec.ts b/src/routes/template.spec.ts index ffb0ff4..b07014b 100644 --- a/src/routes/template.spec.ts +++ b/src/routes/template.spec.ts @@ -907,7 +907,7 @@ describe('/template', () => { }).end((err, res) => { if (err) return done(err); const json = require('../test/db.json'); - should(res.body).have.lengthOf(json.collections.measurement_templates.length); + should(res.body).have.lengthOf(json.collections.material_templates.length); should(res.body).matchEach(measurement => { should(measurement).have.only.keys('_id', 'name', 'version', 'parameters'); should(measurement).have.property('_id').be.type('string'); diff --git a/src/routes/template.ts b/src/routes/template.ts index f19587f..20f1b3b 100644 --- a/src/routes/template.ts +++ b/src/routes/template.ts @@ -4,6 +4,7 @@ import _ from 'lodash'; import TemplateValidate from './validate/template'; import ConditionTemplateModel from '../models/condition_template'; import MeasurementTemplateModel from '../models/measurement_template'; +import MaterialTemplateModel from '../models/material_template'; import res400 from './validate/res400'; import IdValidate from './validate/id'; import mongoose from "mongoose"; @@ -82,5 +83,9 @@ router.post('/template/:collection(measurement|condition|material)/new', async ( module.exports = router; function model (req) { // return right template model - return req.params.collection === 'condition' ? ConditionTemplateModel : MeasurementTemplateModel; + switch (req.params.collection) { + case 'condition': return ConditionTemplateModel + case 'measurement': return MeasurementTemplateModel + case 'material': return MaterialTemplateModel + } } \ No newline at end of file diff --git a/src/routes/validate/material.ts b/src/routes/validate/material.ts index 215ca90..74214d5 100644 --- a/src/routes/validate/material.ts +++ b/src/routes/validate/material.ts @@ -18,7 +18,7 @@ export default class MaterialValidate { // validate input for material numbers: Joi.array() .items( Joi.string() - .length(10) + .max(64) ) }; diff --git a/src/routes/validate/parameters.ts b/src/routes/validate/parameters.ts index e6070b0..61b48d3 100644 --- a/src/routes/validate/parameters.ts +++ b/src/routes/validate/parameters.ts @@ -10,6 +10,7 @@ export default class ParametersValidate { .valid(...parameter.range.values); } else if (parameter.range.hasOwnProperty('min') && parameter.range.hasOwnProperty('max')) { + joiObject[parameter.name] = Joi.number() .min(parameter.range.min) .max(parameter.range.max); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index b420e08..6663986 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -62,10 +62,8 @@ export default class SampleValidate { 'material.name', 'material.supplier', 'material.group', - 'material.mineral', - 'material.glass_fiber', - 'material.carbon_fiber', 'material.number', + 'material.properties.*', 'measurements.(?!spectrum)*' ]; @@ -175,8 +173,8 @@ export default class SampleValidate { let validator; let field = data.filters[i].field if (/material\./.test(field)) { // select right validation model - validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}); - field = field.replace('material.', ''); + validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow(''), properties: Joi.alternatives().try(Joi.number(), Joi.string().max(128))}); + field = field.replace('material.', '').split('.')[0]; } else if (/measurements\./.test(field)) { validator = Joi.object({ @@ -215,7 +213,7 @@ export default class SampleValidate { filters: Joi.array().items(Joi.object({ mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'), field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')).messages({'string.pattern.base': 'Invalid filter field name'}), - values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso())).min(1) + values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso(), Joi.object())).min(1) })).default([]) }).with('to-page', 'page-size').validate(data); } diff --git a/src/test/db.json b/src/test/db.json index 5c8a626..7b0fab9 100644 --- a/src/test/db.json +++ b/src/test/db.json @@ -95,6 +95,19 @@ "user_id": {"$oid":"000000000000000000000002"}, "status": 0, "__v": 0 + }, + { + "_id": {"$oid":"400000000000000000000007"}, + "number": "34", + "type": "liquid", + "color": "black", + "batch": "", + "condition": {}, + "material_id": {"$oid":"100000000000000000000009"}, + "note_id": null, + "user_id": {"$oid":"000000000000000000000002"}, + "status": 0, + "__v": 0 } ], "notes": [