diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 240c156..11080f7 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -38,6 +38,8 @@ router.get('/samples', async (req, res, next) => { filters['to-page'] = 0; } + const sortFilterKeys = filters.filters.map(e => e.field); + let collection; const query = []; query.push({$match: {$and: []}}); @@ -64,13 +66,10 @@ router.get('/samples', async (req, res, next) => { query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); } query.push( - sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements + ...sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, - {$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added - ); - addSkipLimit(query, filters); // skip and limit to select right page - query.push( + {$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); @@ -78,49 +77,60 @@ router.get('/samples', async (req, res, next) => { } else { // sorting with samples as starting collection collection = SampleModel; - // filter for status query[0].$match.$and.push(statusQuery(filters, 'status')); - addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters - // differentiate by sort key to do sorting, skip and limit as early as possible if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys let sortStartValue = null; if (filters['from-id']) { // from-id specified - const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); + const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => { + next(err); + }); if (fromSample instanceof Error) return; if (!fromSample) { return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); } sortStartValue = fromSample[filters.sort[0]]; } - query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); - // material filters - addSkipLimit(query, filters); + query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); } - else { // sorting for material keys - let materialQuery = [] - materialQuery.push( // add material properties - {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, - {$set: {material: { $arrayElemAt: ['$material', 0]}}} + else { // add sort key to list to add field later + sortFilterKeys.push(filters.sort[0]); + } + } + + addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters + + let materialQuery = []; // put material query together separate first to reuse for first-id + let materialAdded = false; + if (sortFilterKeys.find(e => /material\./.test(e))) { // add material fields + materialAdded = true; + materialQuery.push( // add material properties + {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields + {$set: {material: {$arrayElemAt: ['$material', 0]}}} + ); + const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0); + addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters + if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed + materialQuery.push( + {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); - if (filters.sort[0] === 'material.supplier') { // add supplier if needed - materialQuery.push( - {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} - ); - } - if (filters.sort[0] === 'material.group') { // add group if needed - materialQuery.push( - {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} - ); - } - if (filters.sort[0] === 'material.number') { // add material number if needed - materialQuery.push( - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} - ); - } - query.push(...materialQuery); + } + if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed + materialQuery.push( + {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, + {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + ); + } + if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed + materialQuery.push( + {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + ); + } + const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); + addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); // base material filters + query.push(...materialQuery); + if (/material\./.test(filters.sort[0])) { // sort by material key let sortStartValue = null; if (filters['from-id']) { // from-id specified const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);}); @@ -130,17 +140,41 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample[filters.sort[0]]; } - query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); - addSkipLimit(query, filters); + query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); } } - const fieldsToAdd = [ - ...filters.fields, - ...filters.filters.map(e => e.field) // add filter fields in case they were not specified to display - ].filter(e => e !== filters.sort[0]) // sort field was definitely added already, exclude from further field operations - .filter((e, i, self) => self.indexOf(e) === i); // remove duplicates - if (fieldsToAdd.find(e => /material\./.test(e))) { // add material fields + const measurementFilterFields = _.uniq(sortFilterKeys.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters + if (sortFilterKeys.find(e => /measurements\./.test(e))) { // add measurement fields + const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFilterFields}}).lean().exec().catch(err => {next(err);}); + if (measurementTemplates instanceof Error) return; + if (measurementTemplates.length < measurementFilterFields.length) { + return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); + } + query.push({$lookup: { + from: 'measurements', let: {sId: '$_id'}, + pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], + as: 'measurements' + }}); + measurementTemplates.forEach(template => { + query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, + in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} + }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); + }); + addFilterQueries(query, filters.filters + .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) + .map(e => {e.field = e.field.replace('measurements.', ''); return e; }) + ); // measurement filters + } + addSkipLimit(query, filters); + + const fieldsToAdd = filters.fields.filter(e => // fields to add + sortFilterKeys.indexOf(e) < 0 // field was not in filter + && e !== filters.sort[0] // field was not in sort + ); + + if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already query.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$set: {material: { $arrayElemAt: ['$material', 0]}}} @@ -164,13 +198,11 @@ router.get('/samples', async (req, res, next) => { ); } - addFilterQueries(query, filters.filters.filter(e => /material\./.test(e.field))); // material filters - - let measurementFields = fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]).filter((e, i, self) => self.indexOf(e) === i); // filter measurement names and remove duplicates from parameters + let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields - const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); + const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);}); if (measurementTemplates instanceof Error) return; - if (measurementTemplates.length < measurementFields.length) { + if (measurementTemplates.length < measurementFieldsFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance @@ -181,7 +213,7 @@ router.get('/samples', async (req, res, next) => { from: 'measurements', let: {sId: '$_id'}, pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], as: 'measurements' - }}); + }}); } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values @@ -189,7 +221,7 @@ router.get('/samples', async (req, res, next) => { in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); - if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well + if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well query.push( {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, {$set: {spectrum: '$spectrum.values.dpt'}}, @@ -198,7 +230,6 @@ router.get('/samples', async (req, res, next) => { } query.push({$unset: 'measurements'}); } - addFilterQueries(query, filters.filters.filter(e => /measurements\./.test(e.field)).map(e => {e.field = e.field.replace('measurements.', ''); return e; })); // measurement filters const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date @@ -214,6 +245,7 @@ router.get('/samples', async (req, res, next) => { if (filters['to-page'] < 0) { data.reverse(); } + const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]); if (filters.csv) { // output as csv csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { if (err) return next(err); @@ -587,14 +619,14 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] if (filters['from-id']) { // from-id specified if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc - query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); - return {$sort: {[sortKeys[0]]: 1, _id: 1}}; + return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}}, + {$sort: {[sortKeys[0]]: 1, _id: 1}}]; } else { - query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); - return {$sort: {[sortKeys[0]]: -1, _id: -1}}; + return [{$match: {$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}}, + {$sort: {[sortKeys[0]]: -1, _id: -1}}]; } } else { // sort from beginning - return {$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}; // set _id as secondary sort + return [{$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}]; // set _id as secondary sort } }