banana/definma-api
Archived
2
Fork 0
Code Releases Activity

fixed sample requests for the restructured material

Browse Source
This commit is contained in:
VLE2FE 2020-07-17 10:41:19 +02:00
parent 78d35c520e
commit cd81cbf4bd
11 changed files with 115 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/schemas.yaml
View File

@ -121,7 +121,7 @@ Material:
material_template:
$ref: 'api.yaml#/components/schemas/Id'
example:
condition_template: 5ea0450ed851c30a90e70894
material_template: 5ea0450ed851c30a90e70894
mineral: 0
glass_fiber: 40
carbon_fiber: 0

37
data_import/import.js
View File

@ -5,6 +5,7 @@ const {Builder} = require('selenium-webdriver'); // selenium and the chrome d
const chrome = require('selenium-webdriver/chrome');
const pdfReader = require('pdfreader');
const iconv = require('iconv-lite');
const _ = require('lodash');
const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv'; // metadata files
const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
@ -15,6 +16,7 @@ const host = 'http://localhost:3000';
// const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
let data = []; // metadata contents
let materials = {};
const numberToColor = {};
let samples = [];
let normMaster = {};
let sampleDevices = {};
@ -30,7 +32,7 @@ let sampleDevices = {};
main();
async function main() {
if (0) { // materials
if (1) { // materials
await getNormMaster();
await importCsv(metaDoc);
await allMaterials();
@ -42,7 +44,7 @@ async function main() {
await allMaterials();
await saveMaterials();
}
if (0) { // samples
if (1) { // samples
sampleDeviceMap();
if (1) {
console.log('-------- META ----------');
@ -244,6 +246,7 @@ async function allSamples() {
});
const dbMaterials = {}
res.data.forEach(m => {
m.numbers = m.numbers.map(e => ({number: e, color: numberToColor[e]}));
dbMaterials[m.name] = m;
})
res = await axios({
@ -274,8 +277,6 @@ async function allSamples() {
if (!material) { // could not find material, skipping sample
continue;
}
console.log(sample['Material name']);
console.log(material._id);
samples.push({
number: sample['Sample number'],
type: sample['Granulate/Part'],
@ -290,15 +291,21 @@ async function allSamples() {
samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
}
else if (sample['Color'] && sample['Color'] !== '') {
let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0);
console.log(material);
let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['Color'])) >= 0);
if (!number && /black/.test(sample['Color'])) { // special case bk for black
number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
if (!number) { // try German word
number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
}
}
if (number) {
samples[si].color = number.color;
}
else {
samples[si].color = '';
}
}
else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz
samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
}
@ -373,23 +380,33 @@ async function allMaterials() {
supplier: trim(sample['Supplier']),
group: trim(sample['Material'])
};
materials[sample['Material name']].properties = {material_template: '5f0efe6fce7fd20ce4e99013'};
let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
materials[sample['Material name']].mineral = tmp ? tmp[1] : 0;
materials[sample['Material name']].properties.mineral = tmp ? tmp[1] : 0;
tmp = /GF(\d+)/.exec(sample['Reinforcing material']);
materials[sample['Material name']].glass_fiber = tmp ? tmp[1] : 0;
materials[sample['Material name']].properties.glass_fiber = tmp ? tmp[1] : 0;
tmp = /CF(\d+)/.exec(sample['Reinforcing material']);
materials[sample['Material name']].carbon_fiber = tmp ? tmp[1] : 0;
materials[sample['Material name']].properties.carbon_fiber = tmp ? tmp[1] : 0;
materials[sample['Material name']].numbers = await numbersFetch(sample);
console.log(materials[sample['Material name']]);
}
}
}
Object.keys(materials).forEach(mKey => {
materials[mKey].numbers.forEach(number => {
if (number.number && number.color) {
numberToColor[number.number] = number.color;
}
})
});
}
async function saveMaterials() {
const mKeys = Object.keys(materials)
for (let i in mKeys) {
console.info(`${i}/${mKeys.length}`);
const material = _.cloneDeep(materials[mKeys[i]]);
material.numbers = material.numbers.map(e => e.number).filter(e => e !== '').map(e => e.replace(/ /g, ''));
await axios({
method: 'post',
url: host + '/material/new',
@ -397,10 +414,10 @@ async function saveMaterials() {
username: 'admin',
password: 'Abc123!#'
},
data: materials[mKeys[i]]
data: material
}).catch(err => {
if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
console.info(materials[mKeys[i]]);
console.info(material);
console.error(err.response.data);
}
});

5
src/routes/material.ts
View File

@ -239,8 +239,11 @@ async function propertiesCheck (properties, param, res, next, checkVersion = tru
}
// validate parameters
const {error, value: ignore} = ParametersValidate.input(_.omit(properties, 'material_template'), materialData.parameters, param);
const {error, value} = ParametersValidate.input(_.omit(properties, 'material_template'), materialData.parameters, param);
if (error) {res400(error, res); return false;}
Object.keys(value).forEach(key => {
properties[key] = value[key];
});
return materialData;
}

61
src/routes/sample.spec.ts
View File

@ -200,7 +200,7 @@ describe('/sample', () => {
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('number', 'Rng36');
should(res.body[1]).have.property('number', '33');
should(res.body[1]).have.property('number', '34');
should(res.body[res.body.length - 1]).have.property('number', '1');
done();
});
@ -214,7 +214,7 @@ describe('/sample', () => {
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('_id', '400000000000000000000006');
should(res.body[1]).have.property('_id', '400000000000000000000002');
should(res.body[1]).have.property('_id', '400000000000000000000007');
done();
});
});
@ -226,7 +226,7 @@ describe('/sample', () => {
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('_id', '400000000000000000000002');
should(res.body[0]).have.property('_id', '400000000000000000000007');
should(res.body[1]).have.property('_id', '400000000000000000000006');
done();
});
@ -334,6 +334,38 @@ describe('/sample', () => {
done();
});
});
it('filters by a measurement properties property', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=material.properties.glass_fiber&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.properties.glass_fiber%22%2C%22values%22%3A%5B%2225%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(2);
should(res.body).matchEach(sample => {
should(sample.material.properties.glass_fiber).be.eql(25);
});
done();
});
});
it('filters and sorts by a measurement properties property', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&sort=material.properties.glass_fiber-desc&fields[]=number&fields[]=material.name&fields[]=material.properties.glass_fiber&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.properties.glass_fiber%22%2C%22values%22%3A%5B%2225%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(2);
should(res.body[0].number).be.eql('Rng36');
should(res.body[1].number).be.eql('1');
should(res.body).matchEach(sample => {
should(sample.material.properties.glass_fiber).be.eql(25);
});
done();
});
});
it('filters multiple properties', done => {
TestHelper.request(server, done, {
method: 'get',
@ -342,7 +374,7 @@ describe('/sample', () => {
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(3);
should(res.body).have.lengthOf(4);
should(res.body[0]).be.eql({number: '1', batch: ''});
done();
});
@ -359,7 +391,7 @@ describe('/sample', () => {
it('rejects an invalid filter mode', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
url: '/samples?status=all&fields[]=number&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'}
@ -407,6 +439,25 @@ describe('/sample', () => {
res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', supplier: 'Schulmann'}}]
});
});
it('returns specified material properties fields', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.properties.glass_fiber&fields[]=material.name',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).matchEach(sample => {
const materialId = json.collections.samples.find(e => e.number === sample.number).material_id;
const material = json.collections.materials.find(e => e._id.toString() == materialId);
should(sample).have.only.keys('number', 'material');
should(sample.material.name).be.eql(material.name);
should(sample.material.properties.glass_fiber).be.eql(material.properties.glass_fiber);
});
done()
});
});
it('rejects a from-id not in the database', done => {
TestHelper.request(server, done, {
method: 'get',

1
src/routes/sample.ts
View File

@ -22,7 +22,6 @@ import csv from '../helpers/csv';
const router = express.Router();
// TODO: check added filter
// TODO: return total number of pages -> use facet
// TODO: use query pointer
// TODO: convert filter value to number according to table model
// TODO: validation for filter parameters

2
src/routes/template.spec.ts
View File

@ -907,7 +907,7 @@ describe('/template', () => {
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.measurement_templates.length);
should(res.body).have.lengthOf(json.collections.material_templates.length);
should(res.body).matchEach(measurement => {
should(measurement).have.only.keys('_id', 'name', 'version', 'parameters');
should(measurement).have.property('_id').be.type('string');

7
src/routes/template.ts
View File

@ -4,6 +4,7 @@ import _ from 'lodash';
import TemplateValidate from './validate/template';
import ConditionTemplateModel from '../models/condition_template';
import MeasurementTemplateModel from '../models/measurement_template';
import MaterialTemplateModel from '../models/material_template';
import res400 from './validate/res400';
import IdValidate from './validate/id';
import mongoose from "mongoose";
@ -82,5 +83,9 @@ router.post('/template/:collection(measurement|condition|material)/new', async (
module.exports = router;
function model (req) { // return right template model
return req.params.collection === 'condition' ? ConditionTemplateModel : MeasurementTemplateModel;
switch (req.params.collection) {
case 'condition': return ConditionTemplateModel
case 'measurement': return MeasurementTemplateModel
case 'material': return MaterialTemplateModel
}
}

2
src/routes/validate/material.ts
View File

@ -18,7 +18,7 @@ export default class MaterialValidate { // validate input for material
numbers: Joi.array()
.items(
Joi.string()
.length(10)
.max(64)
)
};

1
src/routes/validate/parameters.ts
View File

@ -10,6 +10,7 @@ export default class ParametersValidate {
.valid(...parameter.range.values);
}
else if (parameter.range.hasOwnProperty('min') && parameter.range.hasOwnProperty('max')) {
joiObject[parameter.name] = Joi.number()
.min(parameter.range.min)
.max(parameter.range.max);

10
src/routes/validate/sample.ts
View File

@ -62,10 +62,8 @@ export default class SampleValidate {
'material.name',
'material.supplier',
'material.group',
'material.mineral',
'material.glass_fiber',
'material.carbon_fiber',
'material.number',
'material.properties.*',
'measurements.(?!spectrum)*'
];
@ -175,8 +173,8 @@ export default class SampleValidate {
let validator;
let field = data.filters[i].field
if (/material\./.test(field)) { // select right validation model
validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')});
field = field.replace('material.', '');
validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow(''), properties: Joi.alternatives().try(Joi.number(), Joi.string().max(128))});
field = field.replace('material.', '').split('.')[0];
}
else if (/measurements\./.test(field)) {
validator = Joi.object({
@ -215,7 +213,7 @@ export default class SampleValidate {
filters: Joi.array().items(Joi.object({
mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'),
field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')).messages({'string.pattern.base': 'Invalid filter field name'}),
values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso())).min(1)
values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso(), Joi.object())).min(1)
})).default([])
}).with('to-page', 'page-size').validate(data);
}

13
src/test/db.json
View File

@ -95,6 +95,19 @@
"user_id": {"$oid":"000000000000000000000002"},
"status": 0,
"__v": 0
},
{
"_id": {"$oid":"400000000000000000000007"},
"number": "34",
"type": "liquid",
"color": "black",
"batch": "",
"condition": {},
"material_id": {"$oid":"100000000000000000000009"},
"note_id": null,
"user_id": {"$oid":"000000000000000000000002"},
"status": 0,
"__v": 0
}
],
"notes": [