From 8cf1c14d887811e48247f08239e7e6b20817fd93 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Tue, 30 Jun 2020 14:16:37 +0200 Subject: [PATCH] implementation of measurement fields --- api/sample.yaml | 8 +++-- data_import/import.js | 4 +-- src/helpers/csv.ts | 31 ++++++++++++++++++-- src/routes/sample.spec.ts | 55 +++++++++++++++++++++++++++++++---- src/routes/sample.ts | 46 ++++++++++++++++++++++------- src/routes/validate/sample.ts | 13 ++++++--- src/test/db.json | 14 +++++++++ 7 files changed, 145 insertions(+), 26 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index f527513..91c57e0 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -42,12 +42,14 @@ schema: type: boolean example: false - - name: fields + - name: fields[] description: the fields to include in the output as array, defaults to ['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'] in: query schema: - type: string - example: '&fields[]=number&fields[]=batch' + type: array + items: + type: string + example: ['number', 'batch'] responses: 200: description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) diff --git a/data_import/import.js b/data_import/import.js index 28614c8..a72ee0c 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -32,10 +32,10 @@ async function main() { await allSamples(); await saveSamples(); } - else if (0) { // DPT + else if (1) { // DPT await allDpts(); } - else if (1) { // KF/VZ + else if (0) { // KF/VZ await importCsv(); await allKfVz(); } diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts index dbeb213..a307ca5 100644 --- a/src/helpers/csv.ts +++ b/src/helpers/csv.ts @@ -1,7 +1,34 @@ import {parseAsync} from 'json2csv'; -export default function csv(input: object, fields: string[], f: (err, data) => void) { - parseAsync(input) +export default function csv(input: any[], f: (err, data) => void) { + parseAsync(input.map(e => flatten(e))) .then(csv => f(null, csv)) .catch(err => f(err, null)); +} + +function flatten (data) { // flatten object: {a: {b: true}} -> {a.b: true} + const result = {}; + function recurse (cur, prop) { + if (Object(cur) !== cur || Object.keys(cur).length === 0) { + result[prop] = cur; + } + else if (Array.isArray(cur)) { + let l = 0; + for(let i = 0, l = cur.length; i < l; i++) + recurse(cur[i], prop + "[" + i + "]"); + if (l == 0) + result[prop] = []; + } + else { + let isEmpty = true; + for (let p in cur) { + isEmpty = false; + recurse(cur[p], prop ? prop+"."+p : p); + } + if (isEmpty && prop) + result[prop] = {}; + } + } + recurse(data, ''); + return result; } \ No newline at end of file diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 29e35b7..df7c242 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -244,6 +244,42 @@ describe('/sample', () => { done(); }); }); + it('adds the specified measurements', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=measurements.kf', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body.find(e => e.number === '1')).have.property('kf', {}); + should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null}); + done(); + }); + }); + it('multiplies the sample information for each spectrum', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=measurements.spectrum', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(2); + should(res.body[0]).have.property('spectrum', [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]); + should(res.body[1]).have.property('spectrum', [[3996.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]); + done(); + }); + }); + it('rejects unknown measurement names', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=measurements.xx', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: 'Measurement key not found'} + }); + }); it('returns a correct csv file if specified', done => { TestHelper.request(server, done, { method: 'get', @@ -253,9 +289,9 @@ describe('/sample', () => { httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.text).be.eql('"_id","number","type","color","batch","condition","material_id","note_id","user_id","added"\r\n' + - '"400000000000000000000001","1","granulate","black","","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' + - '"400000000000000000000002","21","granulate","natural","1560237365","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"'); + should(res.text).be.eql('"_id","number","type","color","batch","condition.material","condition.weeks","condition.condition_template","material_id","note_id","user_id","added"\r\n' + + '"400000000000000000000001","1","granulate","black","","copper",3,"200000000000000000000001","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' + + '"400000000000000000000002","21","granulate","natural","1560237365","copper",3,"200000000000000000000001","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"'); done(); }); }); @@ -268,6 +304,15 @@ describe('/sample', () => { res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', mineral: 0}}] }); }); + it('rejects a from-id not in the database', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?from-id=5ea0450ed851c30a90e70894&sort=color-asc', + auth: {basic: 'admin'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: 'from-id not found'} + }); + }); it('rejects an invalid fields parameter', done => { TestHelper.request(server, done, { method: 'get', @@ -283,7 +328,7 @@ describe('/sample', () => { url: '/samples?status=all&page-size=1&fields[]=xx', auth: {basic: 'janedoe'}, httpStatus: 400, - res: {status: 'Invalid body format', details: '"fields[0]" must be one of [_id, color, number, type, batch, added, material.name, material.supplier, material.group, material.mineral, material.glass_fiber, material.carbon_fiber, material.number, condition, material_id, note_id, user_id, material._id, material.numbers]'} + res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\..+)$/m'} }); }); it('rejects a negative page size', done => { @@ -329,7 +374,7 @@ describe('/sample', () => { httpStatus: 401 }); }); - }); // TODO: measurement fields + }); describe('GET /samples/{state}', () => { it('returns all new samples', done => { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index f356211..eef30f9 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -6,6 +6,7 @@ import NoteFieldValidate from './validate/note_field'; import res400 from './validate/res400'; import SampleModel from '../models/sample' import MeasurementModel from '../models/measurement'; +import MeasurementTemplateModel from '../models/measurement_template'; import MaterialModel from '../models/material'; import NoteModel from '../models/note'; import NoteFieldModel from '../models/note_field'; @@ -63,6 +64,9 @@ router.get('/samples', async (req, res, next) => { if (filters['from-id']) { // from-id specified const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); if (fromSample instanceof Error) return; + if (!fromSample) { + return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); + } if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); @@ -77,10 +81,6 @@ router.get('/samples', async (req, res, next) => { query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}}); // set _id as secondary sort } - // if (filters.sort[0].indexOf('material.') >= 0) { // unpopulate materials again - // query.push({$unset: 'material'}); - // } - if (filters['to-page']) { query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more } @@ -88,13 +88,38 @@ router.get('/samples', async (req, res, next) => { if (filters['page-size']) { query.push({$limit: filters['page-size']}); } - console.log(filters.fields); - const projection = filters.fields.reduce((s, e) => {s[e] = true; return s; }, {}); + + let measurementFields = []; + if (filters.fields.find(e => /measurements\./.test(e))) { // joining measurements is required + query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); + measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', '')); + const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); + if (measurementTemplates instanceof Error) return; + if (measurementTemplates.length < measurementFields.length) { + return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); + } + measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later + query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, + in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} + }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}}); + }); + console.log(measurementFields); + if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well + query.push( + {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, + {$set: {spectrum: '$spectrum.values.dpt'}}, + {$unwind: '$spectrum'} + ); + } + query.push({$unset: 'measurements'}); + } + + const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date projection.added = {$toDate: '$_id'}; } if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly - console.log('disable id'); projection._id = false; } query.push({$project: projection}); @@ -104,15 +129,16 @@ router.get('/samples', async (req, res, next) => { if (filters['to-page'] < 0) { data.reverse(); } - if (filters.csv) { // output as csv // TODO: csv example in OAS - csv(_.compact(data.map(e => SampleValidate.output(e, 'refs'))), ['_id', 'number'], (err, data) => { + console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); + if (filters.csv) { // output as csv + csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { if (err) return next(err); res.set('Content-Type', 'text/csv'); res.send(data); }); } else { - res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs')))); // validate all and filter null values from validation errors + res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors } }) }); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 520f91b..41435dd 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -71,10 +71,12 @@ export default class SampleValidate { ...SampleValidate.sortKeys, 'condition', 'material_id', + 'material', 'note_id', 'user_id', 'material._id', - 'material.numbers' + 'material.numbers', + 'measurements.*' ]; static input (data, param) { // validate input, set param to 'new' to make all attributes required @@ -114,7 +116,7 @@ export default class SampleValidate { } } - static output (data, param = 'refs+added') { // validate output and strip unwanted properties, returns null if not valid + static output (data, param = 'refs+added', additionalParams = []) { // validate output and strip unwanted properties, returns null if not valid if (param === 'refs+added') { param = 'refs'; data.added = data._id.getTimestamp(); @@ -130,7 +132,7 @@ export default class SampleValidate { batch: this.sample.batch, condition: this.sample.condition, material_id: IdValidate.get(), - material: MaterialValidate.outputV(), + material: MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}), note_id: IdValidate.get().allow(null), user_id: IdValidate.get(), added: this.sample.added @@ -153,6 +155,9 @@ export default class SampleValidate { else { return null; } + additionalParams.forEach(param => { + joiObject[param] = Joi.any(); + }); const {value, error} = Joi.object(joiObject).validate(data, {stripUnknown: true}); return error !== undefined? null : value; } @@ -165,7 +170,7 @@ export default class SampleValidate { 'page-size': Joi.number().integer().min(1), sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'), csv: Joi.boolean().default(false), - fields: Joi.array().items(Joi.string().valid(...this.fieldKeys)).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) + fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file diff --git a/src/test/db.json b/src/test/db.json index ef26a63..aa68283 100644 --- a/src/test/db.json +++ b/src/test/db.json @@ -417,6 +417,20 @@ "status": 0, "measurement_template": {"$oid":"300000000000000000000002"}, "__v": 0 + }, + { + "_id": {"$oid":"800000000000000000000007"}, + "sample_id": {"$oid":"400000000000000000000001"}, + "values": { + "dpt": [ + [3996.12558,98.00555], + [3995.08519,98.03253], + [3993.04480,98.02657] + ] + }, + "status": 10, + "measurement_template": {"$oid":"300000000000000000000001"}, + "__v": 0 } ], "condition_templates": [