banana/definma-api
Archived
2
Fork 0
Code Releases Activity

reworked filters

Browse Source
This commit is contained in:
VLE2FE 2020-07-06 16:57:09 +02:00
parent 29eefce0c9
commit 6a02f09e7f
1 changed files with 88 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
src/routes/sample.ts
View File

@ -38,6 +38,8 @@ router.get('/samples', async (req, res, next) => {
filters['to-page'] = 0; filters['to-page'] = 0;
} }
const sortFilterKeys = filters.filters.map(e => e.field);
let collection; let collection;
const query = []; const query = [];
query.push({$match: {$and: []}}); query.push({$match: {$and: []}});
@ -64,13 +66,10 @@ router.get('/samples', async (req, res, next) => {
query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0])));
} }
query.push( query.push(
sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements ...sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
{$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added {$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added
);
addSkipLimit(query, filters); // skip and limit to select right page
query.push(
{$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
); );
@ -78,49 +77,60 @@ router.get('/samples', async (req, res, next) => {
} }
else { // sorting with samples as starting collection else { // sorting with samples as starting collection
collection = SampleModel; collection = SampleModel;
// filter for status
query[0].$match.$and.push(statusQuery(filters, 'status')); query[0].$match.$and.push(statusQuery(filters, 'status'));
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
// differentiate by sort key to do sorting, skip and limit as early as possible
if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys
let sortStartValue = null; let sortStartValue = null;
if (filters['from-id']) { // from-id specified if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {
next(err);
});
if (fromSample instanceof Error) return; if (fromSample instanceof Error) return;
if (!fromSample) { if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
} }
sortStartValue = fromSample[filters.sort[0]]; sortStartValue = fromSample[filters.sort[0]];
} }
query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
// material filters
addSkipLimit(query, filters);
} }
else { // sorting for material keys else { // add sort key to list to add field later
let materialQuery = [] sortFilterKeys.push(filters.sort[0]);
}
}
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
let materialQuery = []; // put material query together separate first to reuse for first-id
let materialAdded = false;
if (sortFilterKeys.find(e => /material\./.test(e))) { // add material fields
materialAdded = true;
materialQuery.push( // add material properties materialQuery.push( // add material properties
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields
{$set: {material: {$arrayElemAt: ['$material', 0]}}} {$set: {material: {$arrayElemAt: ['$material', 0]}}}
); );
if (filters.sort[0] === 'material.supplier') { // add supplier if needed const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters
if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed
materialQuery.push( materialQuery.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
); );
} }
if (filters.sort[0] === 'material.group') { // add group if needed if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed
materialQuery.push( materialQuery.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
); );
} }
if (filters.sort[0] === 'material.number') { // add material number if needed if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed
materialQuery.push( materialQuery.push(
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
); );
} }
const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); // base material filters
query.push(...materialQuery); query.push(...materialQuery);
if (/material\./.test(filters.sort[0])) { // sort by material key
let sortStartValue = null; let sortStartValue = null;
if (filters['from-id']) { // from-id specified if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);}); const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);});
@ -130,17 +140,41 @@ router.get('/samples', async (req, res, next) => {
} }
sortStartValue = fromSample[filters.sort[0]]; sortStartValue = fromSample[filters.sort[0]];
} }
query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
addSkipLimit(query, filters);
} }
} }
const fieldsToAdd = [ const measurementFilterFields = _.uniq(sortFilterKeys.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
...filters.fields, if (sortFilterKeys.find(e => /measurements\./.test(e))) { // add measurement fields
...filters.filters.map(e => e.field) // add filter fields in case they were not specified to display const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFilterFields}}).lean().exec().catch(err => {next(err);});
].filter(e => e !== filters.sort[0]) // sort field was definitely added already, exclude from further field operations if (measurementTemplates instanceof Error) return;
.filter((e, i, self) => self.indexOf(e) === i); // remove duplicates if (measurementTemplates.length < measurementFilterFields.length) {
if (fieldsToAdd.find(e => /material\./.test(e))) { // add material fields return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
query.push({$lookup: {
from: 'measurements', let: {sId: '$_id'},
pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
as: 'measurements'
}});
measurementTemplates.forEach(template => {
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
});
addFilterQueries(query, filters.filters
.filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
.map(e => {e.field = e.field.replace('measurements.', ''); return e; })
); // measurement filters
}
addSkipLimit(query, filters);
const fieldsToAdd = filters.fields.filter(e => // fields to add
sortFilterKeys.indexOf(e) < 0 // field was not in filter
&& e !== filters.sort[0] // field was not in sort
);
if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already
query.push( query.push(
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$set: {material: { $arrayElemAt: ['$material', 0]}}} {$set: {material: { $arrayElemAt: ['$material', 0]}}}
@ -164,13 +198,11 @@ router.get('/samples', async (req, res, next) => {
); );
} }
addFilterQueries(query, filters.filters.filter(e => /material\./.test(e.field))); // material filters let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
let measurementFields = fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]).filter((e, i, self) => self.indexOf(e) === i); // filter measurement names and remove duplicates from parameters
if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields
const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
if (measurementTemplates instanceof Error) return; if (measurementTemplates instanceof Error) return;
if (measurementTemplates.length < measurementFields.length) { if (measurementTemplates.length < measurementFieldsFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
} }
if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance
@ -189,7 +221,7 @@ router.get('/samples', async (req, res, next) => {
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
}); });
if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
query.push( query.push(
{$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
{$set: {spectrum: '$spectrum.values.dpt'}}, {$set: {spectrum: '$spectrum.values.dpt'}},
@ -198,7 +230,6 @@ router.get('/samples', async (req, res, next) => {
} }
query.push({$unset: 'measurements'}); query.push({$unset: 'measurements'});
} }
addFilterQueries(query, filters.filters.filter(e => /measurements\./.test(e.field)).map(e => {e.field = e.field.replace('measurements.', ''); return e; })); // measurement filters
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
if (filters.fields.indexOf('added') >= 0) { // add added date if (filters.fields.indexOf('added') >= 0) { // add added date
@ -214,6 +245,7 @@ router.get('/samples', async (req, res, next) => {
if (filters['to-page'] < 0) { if (filters['to-page'] < 0) {
data.reverse(); data.reverse();
} }
const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]);
if (filters.csv) { // output as csv if (filters.csv) { // output as csv
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
if (err) return next(err); if (err) return next(err);
@ -587,14 +619,14 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and
function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key']
if (filters['from-id']) { // from-id specified if (filters['from-id']) { // from-id specified
if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc
query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
return {$sort: {[sortKeys[0]]: 1, _id: 1}}; {$sort: {[sortKeys[0]]: 1, _id: 1}}];
} else { } else {
query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); return [{$match: {$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
return {$sort: {[sortKeys[0]]: -1, _id: -1}}; {$sort: {[sortKeys[0]]: -1, _id: -1}}];
} }
} else { // sort from beginning } else { // sort from beginning
return {$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}; // set _id as secondary sort return [{$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}]; // set _id as secondary sort
} }
} }