reworked filters
This commit is contained in:
VLE2FE
parent
29eefce0c9
commit
6a02f09e7f
@ -38,6 +38,8 @@ router.get('/samples', async (req, res, next) => {
|
||||
filters['to-page'] = 0;
|
||||
}
|
||||
|
||||
const sortFilterKeys = filters.filters.map(e => e.field);
|
||||
|
||||
let collection;
|
||||
const query = [];
|
||||
query.push({$match: {$and: []}});
|
||||
@ -64,13 +66,10 @@ router.get('/samples', async (req, res, next) => {
|
||||
query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0])));
|
||||
}
|
||||
query.push(
|
||||
sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements
|
||||
...sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements
|
||||
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
|
||||
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
|
||||
{$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added
|
||||
);
|
||||
addSkipLimit(query, filters); // skip and limit to select right page
|
||||
query.push(
|
||||
{$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added
|
||||
{$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
|
||||
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
|
||||
);
|
||||
@ -78,49 +77,60 @@ router.get('/samples', async (req, res, next) => {
|
||||
}
|
||||
else { // sorting with samples as starting collection
|
||||
collection = SampleModel;
|
||||
// filter for status
|
||||
query[0].$match.$and.push(statusQuery(filters, 'status'));
|
||||
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
|
||||
|
||||
// differentiate by sort key to do sorting, skip and limit as early as possible
|
||||
if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys
|
||||
let sortStartValue = null;
|
||||
if (filters['from-id']) { // from-id specified
|
||||
const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);});
|
||||
const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {
|
||||
next(err);
|
||||
});
|
||||
if (fromSample instanceof Error) return;
|
||||
if (!fromSample) {
|
||||
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
|
||||
}
|
||||
sortStartValue = fromSample[filters.sort[0]];
|
||||
}
|
||||
query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
|
||||
// material filters
|
||||
addSkipLimit(query, filters);
|
||||
query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
|
||||
}
|
||||
else { // sorting for material keys
|
||||
let materialQuery = []
|
||||
materialQuery.push( // add material properties
|
||||
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
|
||||
{$set: {material: { $arrayElemAt: ['$material', 0]}}}
|
||||
else { // add sort key to list to add field later
|
||||
sortFilterKeys.push(filters.sort[0]);
|
||||
}
|
||||
}
|
||||
|
||||
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
|
||||
|
||||
let materialQuery = []; // put material query together separate first to reuse for first-id
|
||||
let materialAdded = false;
|
||||
if (sortFilterKeys.find(e => /material\./.test(e))) { // add material fields
|
||||
materialAdded = true;
|
||||
materialQuery.push( // add material properties
|
||||
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields
|
||||
{$set: {material: {$arrayElemAt: ['$material', 0]}}}
|
||||
);
|
||||
const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
|
||||
addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters
|
||||
if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed
|
||||
materialQuery.push(
|
||||
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
|
||||
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
|
||||
);
|
||||
if (filters.sort[0] === 'material.supplier') { // add supplier if needed
|
||||
materialQuery.push(
|
||||
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
|
||||
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
|
||||
);
|
||||
}
|
||||
if (filters.sort[0] === 'material.group') { // add group if needed
|
||||
materialQuery.push(
|
||||
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
|
||||
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
|
||||
);
|
||||
}
|
||||
if (filters.sort[0] === 'material.number') { // add material number if needed
|
||||
materialQuery.push(
|
||||
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
|
||||
);
|
||||
}
|
||||
query.push(...materialQuery);
|
||||
}
|
||||
if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed
|
||||
materialQuery.push(
|
||||
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
|
||||
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
|
||||
);
|
||||
}
|
||||
if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed
|
||||
materialQuery.push(
|
||||
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
|
||||
);
|
||||
}
|
||||
const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
|
||||
addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); // base material filters
|
||||
query.push(...materialQuery);
|
||||
if (/material\./.test(filters.sort[0])) { // sort by material key
|
||||
let sortStartValue = null;
|
||||
if (filters['from-id']) { // from-id specified
|
||||
const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);});
|
||||
@ -130,17 +140,41 @@ router.get('/samples', async (req, res, next) => {
|
||||
}
|
||||
sortStartValue = fromSample[filters.sort[0]];
|
||||
}
|
||||
query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
|
||||
addSkipLimit(query, filters);
|
||||
query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
|
||||
}
|
||||
}
|
||||
|
||||
const fieldsToAdd = [
|
||||
...filters.fields,
|
||||
...filters.filters.map(e => e.field) // add filter fields in case they were not specified to display
|
||||
].filter(e => e !== filters.sort[0]) // sort field was definitely added already, exclude from further field operations
|
||||
.filter((e, i, self) => self.indexOf(e) === i); // remove duplicates
|
||||
if (fieldsToAdd.find(e => /material\./.test(e))) { // add material fields
|
||||
const measurementFilterFields = _.uniq(sortFilterKeys.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
|
||||
if (sortFilterKeys.find(e => /measurements\./.test(e))) { // add measurement fields
|
||||
const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFilterFields}}).lean().exec().catch(err => {next(err);});
|
||||
if (measurementTemplates instanceof Error) return;
|
||||
if (measurementTemplates.length < measurementFilterFields.length) {
|
||||
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
|
||||
}
|
||||
query.push({$lookup: {
|
||||
from: 'measurements', let: {sId: '$_id'},
|
||||
pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
|
||||
as: 'measurements'
|
||||
}});
|
||||
measurementTemplates.forEach(template => {
|
||||
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
|
||||
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
|
||||
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
|
||||
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
|
||||
});
|
||||
addFilterQueries(query, filters.filters
|
||||
.filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
|
||||
.map(e => {e.field = e.field.replace('measurements.', ''); return e; })
|
||||
); // measurement filters
|
||||
}
|
||||
addSkipLimit(query, filters);
|
||||
|
||||
const fieldsToAdd = filters.fields.filter(e => // fields to add
|
||||
sortFilterKeys.indexOf(e) < 0 // field was not in filter
|
||||
&& e !== filters.sort[0] // field was not in sort
|
||||
);
|
||||
|
||||
if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already
|
||||
query.push(
|
||||
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
|
||||
{$set: {material: { $arrayElemAt: ['$material', 0]}}}
|
||||
@ -164,13 +198,11 @@ router.get('/samples', async (req, res, next) => {
|
||||
);
|
||||
}
|
||||
|
||||
addFilterQueries(query, filters.filters.filter(e => /material\./.test(e.field))); // material filters
|
||||
|
||||
let measurementFields = fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]).filter((e, i, self) => self.indexOf(e) === i); // filter measurement names and remove duplicates from parameters
|
||||
let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
|
||||
if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields
|
||||
const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
|
||||
const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
|
||||
if (measurementTemplates instanceof Error) return;
|
||||
if (measurementTemplates.length < measurementFields.length) {
|
||||
if (measurementTemplates.length < measurementFieldsFields.length) {
|
||||
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
|
||||
}
|
||||
if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance
|
||||
@ -181,7 +213,7 @@ router.get('/samples', async (req, res, next) => {
|
||||
from: 'measurements', let: {sId: '$_id'},
|
||||
pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
|
||||
as: 'measurements'
|
||||
}});
|
||||
}});
|
||||
}
|
||||
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
|
||||
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
|
||||
@ -189,7 +221,7 @@ router.get('/samples', async (req, res, next) => {
|
||||
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
|
||||
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
|
||||
});
|
||||
if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
|
||||
if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
|
||||
query.push(
|
||||
{$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
|
||||
{$set: {spectrum: '$spectrum.values.dpt'}},
|
||||
@ -198,7 +230,6 @@ router.get('/samples', async (req, res, next) => {
|
||||
}
|
||||
query.push({$unset: 'measurements'});
|
||||
}
|
||||
addFilterQueries(query, filters.filters.filter(e => /measurements\./.test(e.field)).map(e => {e.field = e.field.replace('measurements.', ''); return e; })); // measurement filters
|
||||
|
||||
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
|
||||
if (filters.fields.indexOf('added') >= 0) { // add added date
|
||||
@ -214,6 +245,7 @@ router.get('/samples', async (req, res, next) => {
|
||||
if (filters['to-page'] < 0) {
|
||||
data.reverse();
|
||||
}
|
||||
const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]);
|
||||
if (filters.csv) { // output as csv
|
||||
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
|
||||
if (err) return next(err);
|
||||
@ -587,14 +619,14 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and
|
||||
function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key']
|
||||
if (filters['from-id']) { // from-id specified
|
||||
if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc
|
||||
query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
|
||||
return {$sort: {[sortKeys[0]]: 1, _id: 1}};
|
||||
return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
|
||||
{$sort: {[sortKeys[0]]: 1, _id: 1}}];
|
||||
} else {
|
||||
query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
|
||||
return {$sort: {[sortKeys[0]]: -1, _id: -1}};
|
||||
return [{$match: {$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
|
||||
{$sort: {[sortKeys[0]]: -1, _id: -1}}];
|
||||
}
|
||||
} else { // sort from beginning
|
||||
return {$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}; // set _id as secondary sort
|
||||
return [{$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}]; // set _id as secondary sort
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user