diff --git a/data_import/import.js b/data_import/import.js index a72ee0c..3d84160 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -16,6 +16,7 @@ let normMaster = {}; // TODO: integrate measurement device information from DPT names using different users // TODO: supplier: other for supplierless samples +// TODO: BASF twice, BASF as color main(); diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts index a307ca5..18e633c 100644 --- a/src/helpers/csv.ts +++ b/src/helpers/csv.ts @@ -1,7 +1,13 @@ import {parseAsync} from 'json2csv'; export default function csv(input: any[], f: (err, data) => void) { - parseAsync(input.map(e => flatten(e))) + console.log(input[1000]); + console.log(flatten(input[1000])); + parseAsync([flatten(input[1000])]).then(csv => console.log(csv)); + console.log(input[1]); + console.log(flatten(input[1])); + parseAsync([flatten(input[1])]).then(csv => console.log(csv)); + parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true}) .then(csv => f(null, csv)) .catch(err => f(err, null)); } diff --git a/src/index.ts b/src/index.ts index 9af77cf..8116de7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import helmet from 'helmet'; import api from './api'; import db from './db'; +// TODO: working demo branch // tell if server is running in debug or production environment console.info(process.env.NODE_ENV === 'production' ? '===== PRODUCTION =====' : process.env.NODE_ENV === 'test' ? '' :'===== DEVELOPMENT ====='); diff --git a/src/models/material.ts b/src/models/material.ts index bcebb83..d7d5eb9 100644 --- a/src/models/material.ts +++ b/src/models/material.ts @@ -22,5 +22,7 @@ MaterialSchema.query.log = function > db.log(req, this); return this; } +MaterialSchema.index({supplier_id: 1}); +MaterialSchema.index({group_id: 1}); export default mongoose.model>('material', MaterialSchema); \ No newline at end of file diff --git a/src/models/measurement.ts b/src/models/measurement.ts index 1136e6b..55267ec 100644 --- a/src/models/measurement.ts +++ b/src/models/measurement.ts @@ -17,5 +17,7 @@ MeasurementSchema.query.log = function >('measurement', MeasurementSchema); \ No newline at end of file diff --git a/src/models/sample.ts b/src/models/sample.ts index 0e457d8..8eec7bd 100644 --- a/src/models/sample.ts +++ b/src/models/sample.ts @@ -22,5 +22,8 @@ SampleSchema.query.log = function > ( db.log(req, this); return this; } +SampleSchema.index({material_id: 1}); +SampleSchema.index({note_id: 1}); +SampleSchema.index({user_id: 1}); export default mongoose.model>('sample', SampleSchema); \ No newline at end of file diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index df7c242..2f82bc9 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -252,7 +252,7 @@ describe('/sample', () => { httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.body.find(e => e.number === '1')).have.property('kf', {}); + should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null}); should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null}); done(); }); @@ -328,7 +328,7 @@ describe('/sample', () => { url: '/samples?status=all&page-size=1&fields[]=xx', auth: {basic: 'janedoe'}, httpStatus: 400, - res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\..+)$/m'} + res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'} }); }); it('rejects a negative page size', done => { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index eef30f9..7b2af04 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -27,21 +27,10 @@ router.get('/samples', async (req, res, next) => { const {error, value: filters} = SampleValidate.query(req.query); if (error) return res400(error, res); - const query = []; - query.push({$match: {$and: []}}); - if (filters.hasOwnProperty('status')) { - if(filters.status === 'all') { - query[0].$match.$and.push({$or: [{status: globals.status.validated}, {status: globals.status.new}]}); - } - else { - query[0].$match.$and.push({status: globals.status[filters.status]}); - } - } - else { // default - query[0].$match.$and.push({status: globals.status.validated}); - } + // TODO: find a better place for these + const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', 'user_id']; - // sorting + // evaluate sort parameter from 'color-asc' to ['color', 1] filters.sort = filters.sort.split('-'); filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1; @@ -49,62 +38,139 @@ router.get('/samples', async (req, res, next) => { filters['to-page'] = 0; } - query.push( - {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, - {$set: {material: { $arrayElemAt: ['$material', 0]}}}, - {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}, - {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}, - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]} + let collection; + const query = []; + query.push({$match: {$and: []}}); + + if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection + collection = MeasurementModel; + const measurementName = filters.sort[0].replace('measurements.', ''); + const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);}); + if (measurementTemplate instanceof Error) return; + if (!measurementTemplate) { + return res.status(400).json({status: 'Invalid body format', details: filters.sort[0] + ' not found'}); + } + let sortStartValue = null; + if (filters['from-id']) { // from-id specified, fetch values for sorting + const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}).lean().exec().catch(err => {next(err);}); // TODO: what if more than one measurement for sample? + if (fromSample instanceof Error) return; + if (!fromSample) { + return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); } + sortStartValue = fromSample.values[measurementTemplate.parameters[0].name]; } - ); + query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort + query.push( + sortQuery(query, filters, ['values.' + measurementTemplate.parameters[0].name, 'sample_id'], sortStartValue), // sort measurements + {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure + {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, + {$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added + ); + addSkipLimit(query, filters); // skip and limit to select right page + query.push( + {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring + {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} + ); - if (filters['from-id']) { // from-id specified - const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); - if (fromSample instanceof Error) return; - if (!fromSample) { - return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); - } + } + else { // sorting with samples as starting collection + collection = SampleModel; + // filter for status + query[0].$match.$and.push(statusQuery(filters, 'status')); - if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc - query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); - query.push({$sort: {[filters.sort[0]]: 1, _id: 1}}); + // differentiate by sort key to do sorting, skip and limit as early as possible + if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys + let sortStartValue = null; + if (filters['from-id']) { // from-id specified + const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); + if (fromSample instanceof Error) return; + if (!fromSample) { + return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); + } + sortStartValue = fromSample[filters.sort[0]]; + } + query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + addSkipLimit(query, filters); } - else { - query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$lt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); - query.push({$sort: {[filters.sort[0]]: -1, _id: -1}}); + else { // sorting for material keys + let materialQuery = [] + materialQuery.push( // add material properties + {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, + {$set: {material: { $arrayElemAt: ['$material', 0]}}} + ); + if (filters.sort[0] === 'material.supplier') { // add supplier if needed + materialQuery.push( + {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + ); + } + if (filters.sort[0] === 'material.group') { // add group if needed + materialQuery.push( + {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, + {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + ); + } + if (filters.sort[0] === 'material.number') { // add material number if needed + materialQuery.push( + {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + ); + } + query.push(...materialQuery); + let sortStartValue = null; + if (filters['from-id']) { // from-id specified + const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);}); + if (fromSample instanceof Error) return; + if (!fromSample) { + return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); + } + sortStartValue = fromSample[filters.sort[0]]; + } + query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + addSkipLimit(query, filters); } } - else { // sort from beginning - query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}}); // set _id as secondary sort + + const fieldsNoSort = filters.fields.filter(e => e !== filters.sort[0]); // sort field was definitely added already, exclude from further field operations + if (fieldsNoSort.find(e => /material\./.test(e))) { // add material fields + query.push( + {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, + {$set: {material: { $arrayElemAt: ['$material', 0]}}} + ); + } + if (fieldsNoSort.indexOf('material.supplier') >= 0) { // add supplier if needed + query.push( + {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + ); + } + if (fieldsNoSort.indexOf('material.group') >= 0) { // add group if needed + query.push( + {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, + {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + ); + } + if (fieldsNoSort.indexOf('material.number') >= 0) { // add material number if needed + query.push( + {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + ); } - if (filters['to-page']) { - query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more - } - - if (filters['page-size']) { - query.push({$limit: filters['page-size']}); - } - - let measurementFields = []; - if (filters.fields.find(e => /measurements\./.test(e))) { // joining measurements is required + let measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', '')); + console.log(fieldsNoSort); + console.log(measurementFields); + if (fieldsNoSort.find(e => /measurements\./.test(e))) { // add measurement fields query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); - measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', '')); - const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); + const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); if (measurementTemplates instanceof Error) return; if (measurementTemplates.length < measurementFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values - vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, - in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} - }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}}); + vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, + in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} + }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); - console.log(measurementFields); if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well query.push( {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, @@ -124,12 +190,11 @@ router.get('/samples', async (req, res, next) => { } query.push({$project: projection}); - SampleModel.aggregate(query).exec((err, data) => { + collection.aggregate(query).exec((err, data) => { if (err) return next(err); if (filters['to-page'] < 0) { data.reverse(); } - console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); if (filters.csv) { // output as csv csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { if (err) return next(err); @@ -498,4 +563,42 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and } }); }); +} + +function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] + if (filters['from-id']) { // from-id specified + if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc + query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); + return {$sort: {[sortKeys[0]]: 1, _id: 1}}; + } else { + query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); + return {$sort: {[sortKeys[0]]: -1, _id: -1}}; + } + } else { // sort from beginning + return {$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}; // set _id as secondary sort + } +} + +function addSkipLimit(query, filters) { + if (filters['to-page']) { + query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more + } + + if (filters['page-size']) { + query.push({$limit: filters['page-size']}); + } +} + +function statusQuery(filters, field) { + if (filters.hasOwnProperty('status')) { + if(filters.status === 'all') { + return {$or: [{[field]: globals.status.validated}, {[field]: globals.status.new}]}; + } + else { + return {[field]: globals.status[filters.status]}; + } + } + else { // default + return {[field]: globals.status.validated}; + } } \ No newline at end of file diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 41435dd..16bc8a1 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -64,7 +64,8 @@ export default class SampleValidate { 'material.mineral', 'material.glass_fiber', 'material.carbon_fiber', - 'material.number' + 'material.number', + 'measurements.(?!spectrum)*' ]; private static fieldKeys = [ @@ -76,7 +77,7 @@ export default class SampleValidate { 'user_id', 'material._id', 'material.numbers', - 'measurements.*' + 'measurements.spectrum' ]; static input (data, param) { // validate input, set param to 'new' to make all attributes required @@ -168,7 +169,7 @@ export default class SampleValidate { 'from-id': IdValidate.get(), 'to-page': Joi.number().integer(), 'page-size': Joi.number().integer().min(1), - sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'), + sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'), csv: Joi.boolean().default(false), fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) }).with('to-page', 'page-size').validate(data);