From 29eefce0c9b6a641690a1089fe43f20a80a8f0ee Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 6 Jul 2020 09:43:04 +0200 Subject: [PATCH] added filters --- api/sample.yaml | 8 +++ data_import/import.js | 2 + src/routes/sample.spec.ts | 105 +++++++++++++++++++++++++++++++++- src/routes/sample.ts | 57 +++++++++++++----- src/routes/validate/sample.ts | 19 +++++- src/test/db.json | 2 +- 6 files changed, 176 insertions(+), 17 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index 91c57e0..acdd33c 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -50,6 +50,14 @@ items: type: string example: ['number', 'batch'] + - name: filters[] + description: "the filters to apply as an array of URIComponent encoded objects in the form {mode: 'eq/ne/lt/lte/gt/gte/in/nin', field: 'material.m', values: ['15']} using encodeURIComponent(JSON.stringify({}))" + in: query + schema: + type: array + items: + type: string + example: ["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D", "%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22%5D%7D"] responses: 200: description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) diff --git a/data_import/import.js b/data_import/import.js index 3d84160..4f31f8b 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -17,6 +17,8 @@ let normMaster = {}; // TODO: integrate measurement device information from DPT names using different users // TODO: supplier: other for supplierless samples // TODO: BASF twice, BASF as color +// TODO: trim color names +// TODO: duplicate kf values main(); diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 2f82bc9..d15200b 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -20,6 +20,7 @@ describe('/sample', () => { after(done => TestHelper.after(done)); // TODO: sort, added date filter, has measurements/condition filter + // TODO: check if conditions work in sort/fields/filters describe('GET /samples', () => { it('returns all samples', done => { TestHelper.request(server, done, { @@ -253,7 +254,7 @@ describe('/sample', () => { }).end((err, res) => { if (err) return done(err); should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null}); - should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null}); + should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.6, 'standard deviation': null}); done(); }); }); @@ -271,6 +272,108 @@ describe('/sample', () => { done(); }); }); + it('filters a sample property', done => { // TODO: implement filters + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=type&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22type%22%2C%22values%22%3A%5B%22part%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.samples.filter(e => e.type === 'part').length); + should(res.body).matchEach(sample => { + should(sample).have.property('type', 'part'); + }); + done(); + }); + }); + it('filters a material property', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22in%22%2C%22field%22%3A%22material.name%22%2C%22values%22%3A%5B%22Schulamid%2066%20GF%2025%20H%22%2C%22Stanyl%20TW%20200%20F8%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.samples.filter(e => e.material_id == '100000000000000000000004' || e.material_id == '100000000000000000000001').length); + should(res.body).matchEach(sample => { + should(sample.material.name).be.equalOneOf('Schulamid 66 GF 25 H', 'Stanyl TW 200 F8'); + }); + done(); + }); + }); + it('filters by measurement value', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=measurements.kf.weight%20%25&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length); + should(res.body).matchEach(sample => { + should(sample.kf['weight %']).be.above(0.5); + }); + done(); + }); + }); + it('filters by measurement value not in the fields', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length); + should(res.body[0]).have.property('number', 'Rng36'); + done(); + }); + }); + it('filters multiple properties', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.glass_fiber%22%2C%22values%22%3A%5B33%5D%7D&filters[]=%7B%22mode%22%3A%22lte%22%2C%22field%22%3A%22number%22%2C%22values%22%3A%5B%22Rng33%22%5D%7D&filters[]=%7B%22mode%22%3A%22nin%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(1); + should(res.body[0]).be.eql({number: '32', material: {glass_fiber: 33}, batch: '1653000308'}); + done(); + }); + }); // TODO: do measurement pipeline, check if it works with UI + it('rejects an invalid JSON string as a filters parameter', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=xx', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: 'Invalid JSON string for filter parameter'} + }); + }); + it('rejects an invalid filter mode', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'} + }); + }); + it('rejects an filter field not existing', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22xx%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"filters[0].field" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'} + }); + }); it('rejects unknown measurement names', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 7b2af04..240c156 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -44,7 +44,7 @@ router.get('/samples', async (req, res, next) => { if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection collection = MeasurementModel; - const measurementName = filters.sort[0].replace('measurements.', ''); + const [,measurementName, measurementParam] = filters.sort[0].split('.'); const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);}); if (measurementTemplate instanceof Error) return; if (!measurementTemplate) { @@ -57,11 +57,14 @@ router.get('/samples', async (req, res, next) => { if (!fromSample) { return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); } - sortStartValue = fromSample.values[measurementTemplate.parameters[0].name]; + sortStartValue = fromSample.values[measurementParam]; } query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort + if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered + query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); + } query.push( - sortQuery(query, filters, ['values.' + measurementTemplate.parameters[0].name, 'sample_id'], sortStartValue), // sort measurements + sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, {$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added @@ -71,12 +74,13 @@ router.get('/samples', async (req, res, next) => { {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); - + addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters } else { // sorting with samples as starting collection collection = SampleModel; // filter for status query[0].$match.$and.push(statusQuery(filters, 'status')); + addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters // differentiate by sort key to do sorting, skip and limit as early as possible if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys @@ -90,6 +94,7 @@ router.get('/samples', async (req, res, next) => { sortStartValue = fromSample[filters.sort[0]]; } query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + // material filters addSkipLimit(query, filters); } else { // sorting for material keys @@ -130,41 +135,54 @@ router.get('/samples', async (req, res, next) => { } } - const fieldsNoSort = filters.fields.filter(e => e !== filters.sort[0]); // sort field was definitely added already, exclude from further field operations - if (fieldsNoSort.find(e => /material\./.test(e))) { // add material fields + const fieldsToAdd = [ + ...filters.fields, + ...filters.filters.map(e => e.field) // add filter fields in case they were not specified to display + ].filter(e => e !== filters.sort[0]) // sort field was definitely added already, exclude from further field operations + .filter((e, i, self) => self.indexOf(e) === i); // remove duplicates + if (fieldsToAdd.find(e => /material\./.test(e))) { // add material fields query.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$set: {material: { $arrayElemAt: ['$material', 0]}}} ); } - if (fieldsNoSort.indexOf('material.supplier') >= 0) { // add supplier if needed + if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed query.push( {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); } - if (fieldsNoSort.indexOf('material.group') >= 0) { // add group if needed + if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed query.push( {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } - if (fieldsNoSort.indexOf('material.number') >= 0) { // add material number if needed + if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed query.push( {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} ); } - let measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', '')); - console.log(fieldsNoSort); - console.log(measurementFields); - if (fieldsNoSort.find(e => /measurements\./.test(e))) { // add measurement fields - query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); + addFilterQueries(query, filters.filters.filter(e => /material\./.test(e.field))); // material filters + + let measurementFields = fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]).filter((e, i, self) => self.indexOf(e) === i); // filter measurement names and remove duplicates from parameters + if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); if (measurementTemplates instanceof Error) return; if (measurementTemplates.length < measurementFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } + if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance + query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); + } + else { + query.push({$lookup: { + from: 'measurements', let: {sId: '$_id'}, + pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], + as: 'measurements' + }}); + } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, @@ -180,6 +198,7 @@ router.get('/samples', async (req, res, next) => { } query.push({$unset: 'measurements'}); } + addFilterQueries(query, filters.filters.filter(e => /measurements\./.test(e.field)).map(e => {e.field = e.field.replace('measurements.', ''); return e; })); // measurement filters const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date @@ -601,4 +620,14 @@ function statusQuery(filters, field) { else { // default return {[field]: globals.status.validated}; } +} + +function addFilterQueries (query, filters) { // returns array of match queries from given filters + if (filters.length) { + query.push({$match: {$and: filterQueries(filters)}}); + } +} + +function filterQueries (filters) { + return filters.map(e => ({[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}})) // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin } \ No newline at end of file diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 16bc8a1..b0cae01 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -164,6 +164,18 @@ export default class SampleValidate { } static query (data) { + if (data.filters && data.filters.length) { + const filterValidation = Joi.array().items(Joi.string()).validate(data.filters); + if (filterValidation.error) return filterValidation; + try { + for (let i in data.filters) { + data.filters[i] = JSON.parse(data.filters[i]); + } + } + catch { + return {error: {details: [{message: 'Invalid JSON string for filter parameter'}]}, value: null} + } + } return Joi.object({ status: Joi.string().valid('validated', 'new', 'all'), 'from-id': IdValidate.get(), @@ -171,7 +183,12 @@ export default class SampleValidate { 'page-size': Joi.number().integer().min(1), sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'), csv: Joi.boolean().default(false), - fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) + fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']), + filters: Joi.array().items(Joi.object({ + mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'), + field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')), + values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean())).min(1) + })).default([]) }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file diff --git a/src/test/db.json b/src/test/db.json index aa68283..99ae417 100644 --- a/src/test/db.json +++ b/src/test/db.json @@ -411,7 +411,7 @@ "_id": {"$oid":"800000000000000000000006"}, "sample_id": {"$oid":"400000000000000000000006"}, "values": { - "weight %": 0.5, + "weight %": 0.6, "standard deviation":null }, "status": 0,