banana/definma-api
Archived
2
Fork 0
Code Releases Activity

added filters

Browse Source
This commit is contained in:
VLE2FE 2020-07-06 09:43:04 +02:00
parent e4bc5a77f1
commit 29eefce0c9
6 changed files with 176 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api/sample.yaml
View File

@ -50,6 +50,14 @@
items:
type: string
example: ['number', 'batch']
- name: filters[]
description: "the filters to apply as an array of URIComponent encoded objects in the form {mode: 'eq/ne/lt/lte/gt/gte/in/nin', field: 'material.m', values: ['15']} using encodeURIComponent(JSON.stringify({}))"
in: query
schema:
type: array
items:
type: string
example: ["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D", "%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22%5D%7D"]
responses:
200:
description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)

2
data_import/import.js
View File

@ -17,6 +17,8 @@ let normMaster = {};
// TODO: integrate measurement device information from DPT names using different users
// TODO: supplier: other for supplierless samples
// TODO: BASF twice, BASF as color
// TODO: trim color names
// TODO: duplicate kf values
main();

105
src/routes/sample.spec.ts
View File

@ -20,6 +20,7 @@ describe('/sample', () => {
after(done => TestHelper.after(done));
// TODO: sort, added date filter, has measurements/condition filter
// TODO: check if conditions work in sort/fields/filters
describe('GET /samples', () => {
it('returns all samples', done => {
TestHelper.request(server, done, {
@ -253,7 +254,7 @@ describe('/sample', () => {
}).end((err, res) => {
if (err) return done(err);
should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null});
should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null});
should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.6, 'standard deviation': null});
done();
});
});
@ -271,6 +272,108 @@ describe('/sample', () => {
done();
});
});
it('filters a sample property', done => { // TODO: implement filters
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=type&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22type%22%2C%22values%22%3A%5B%22part%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.type === 'part').length);
should(res.body).matchEach(sample => {
should(sample).have.property('type', 'part');
});
done();
});
});
it('filters a material property', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22in%22%2C%22field%22%3A%22material.name%22%2C%22values%22%3A%5B%22Schulamid%2066%20GF%2025%20H%22%2C%22Stanyl%20TW%20200%20F8%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.material_id == '100000000000000000000004' || e.material_id == '100000000000000000000001').length);
should(res.body).matchEach(sample => {
should(sample.material.name).be.equalOneOf('Schulamid 66 GF 25 H', 'Stanyl TW 200 F8');
});
done();
});
});
it('filters by measurement value', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=measurements.kf.weight%20%25&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length);
should(res.body).matchEach(sample => {
should(sample.kf['weight %']).be.above(0.5);
});
done();
});
});
it('filters by measurement value not in the fields', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length);
should(res.body[0]).have.property('number', 'Rng36');
done();
});
});
it('filters multiple properties', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.glass_fiber%22%2C%22values%22%3A%5B33%5D%7D&filters[]=%7B%22mode%22%3A%22lte%22%2C%22field%22%3A%22number%22%2C%22values%22%3A%5B%22Rng33%22%5D%7D&filters[]=%7B%22mode%22%3A%22nin%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(1);
should(res.body[0]).be.eql({number: '32', material: {glass_fiber: 33}, batch: '1653000308'});
done();
});
}); // TODO: do measurement pipeline, check if it works with UI
it('rejects an invalid JSON string as a filters parameter', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=xx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: 'Invalid JSON string for filter parameter'}
});
});
it('rejects an invalid filter mode', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'}
});
});
it('rejects an filter field not existing', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22xx%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"filters[0].field" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'}
});
});
it('rejects unknown measurement names', done => {
TestHelper.request(server, done, {
method: 'get',

57
src/routes/sample.ts
View File

@ -44,7 +44,7 @@ router.get('/samples', async (req, res, next) => {
if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection
collection = MeasurementModel;
const measurementName = filters.sort[0].replace('measurements.', '');
const [,measurementName, measurementParam] = filters.sort[0].split('.');
const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);});
if (measurementTemplate instanceof Error) return;
if (!measurementTemplate) {
@ -57,11 +57,14 @@ router.get('/samples', async (req, res, next) => {
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
sortStartValue = fromSample.values[measurementTemplate.parameters[0].name];
sortStartValue = fromSample.values[measurementParam];
}
query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort
if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered
query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0])));
}
query.push(
sortQuery(query, filters, ['values.' + measurementTemplate.parameters[0].name, 'sample_id'], sortStartValue), // sort measurements
sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
{$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added
@ -71,12 +74,13 @@ router.get('/samples', async (req, res, next) => {
{$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
);
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
}
else { // sorting with samples as starting collection
collection = SampleModel;
// filter for status
query[0].$match.$and.push(statusQuery(filters, 'status'));
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
// differentiate by sort key to do sorting, skip and limit as early as possible
if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys
@ -90,6 +94,7 @@ router.get('/samples', async (req, res, next) => {
sortStartValue = fromSample[filters.sort[0]];
}
query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
// material filters
addSkipLimit(query, filters);
}
else { // sorting for material keys
@ -130,41 +135,54 @@ router.get('/samples', async (req, res, next) => {
}
}
const fieldsNoSort = filters.fields.filter(e => e !== filters.sort[0]); // sort field was definitely added already, exclude from further field operations
if (fieldsNoSort.find(e => /material\./.test(e))) { // add material fields
const fieldsToAdd = [
...filters.fields,
...filters.filters.map(e => e.field) // add filter fields in case they were not specified to display
].filter(e => e !== filters.sort[0]) // sort field was definitely added already, exclude from further field operations
.filter((e, i, self) => self.indexOf(e) === i); // remove duplicates
if (fieldsToAdd.find(e => /material\./.test(e))) { // add material fields
query.push(
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$set: {material: { $arrayElemAt: ['$material', 0]}}}
);
}
if (fieldsNoSort.indexOf('material.supplier') >= 0) { // add supplier if needed
if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed
query.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
);
}
if (fieldsNoSort.indexOf('material.group') >= 0) { // add group if needed
if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed
query.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
if (fieldsNoSort.indexOf('material.number') >= 0) { // add material number if needed
if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed
query.push(
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
);
}
let measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', ''));
console.log(fieldsNoSort);
console.log(measurementFields);
if (fieldsNoSort.find(e => /measurements\./.test(e))) { // add measurement fields
query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
addFilterQueries(query, filters.filters.filter(e => /material\./.test(e.field))); // material filters
let measurementFields = fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]).filter((e, i, self) => self.indexOf(e) === i); // filter measurement names and remove duplicates from parameters
if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields
const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
if (measurementTemplates instanceof Error) return;
if (measurementTemplates.length < measurementFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance
query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
}
else {
query.push({$lookup: {
from: 'measurements', let: {sId: '$_id'},
pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
as: 'measurements'
}});
}
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
@ -180,6 +198,7 @@ router.get('/samples', async (req, res, next) => {
}
query.push({$unset: 'measurements'});
}
addFilterQueries(query, filters.filters.filter(e => /measurements\./.test(e.field)).map(e => {e.field = e.field.replace('measurements.', ''); return e; })); // measurement filters
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
if (filters.fields.indexOf('added') >= 0) { // add added date
@ -602,3 +621,13 @@ function statusQuery(filters, field) {
return {[field]: globals.status.validated};
}
}
function addFilterQueries (query, filters) { // returns array of match queries from given filters
if (filters.length) {
query.push({$match: {$and: filterQueries(filters)}});
}
}
function filterQueries (filters) {
return filters.map(e => ({[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}})) // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin
}

19
src/routes/validate/sample.ts
View File

@ -164,6 +164,18 @@ export default class SampleValidate {
}
static query (data) {
if (data.filters && data.filters.length) {
const filterValidation = Joi.array().items(Joi.string()).validate(data.filters);
if (filterValidation.error) return filterValidation;
try {
for (let i in data.filters) {
data.filters[i] = JSON.parse(data.filters[i]);
}
}
catch {
return {error: {details: [{message: 'Invalid JSON string for filter parameter'}]}, value: null}
}
}
return Joi.object({
status: Joi.string().valid('validated', 'new', 'all'),
'from-id': IdValidate.get(),
@ -171,7 +183,12 @@ export default class SampleValidate {
'page-size': Joi.number().integer().min(1),
sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'),
csv: Joi.boolean().default(false),
fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])
fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']),
filters: Joi.array().items(Joi.object({
mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'),
field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')),
values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean())).min(1)
})).default([])
}).with('to-page', 'page-size').validate(data);
}
}

2
src/test/db.json
View File

@ -411,7 +411,7 @@
"_id": {"$oid":"800000000000000000000006"},
"sample_id": {"$oid":"400000000000000000000006"},
"values": {
"weight %": 0.5,
"weight %": 0.6,
"standard deviation":null
},
"status": 0,