banana/definma-api
Archived
2
Fork 0
Code Releases Activity

restructured aggregation

Browse Source
This commit is contained in:
VLE2FE 2020-07-02 12:18:01 +02:00
parent 8cf1c14d88
commit e4bc5a77f1
9 changed files with 180 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
data_import/import.js
View File

@ -16,6 +16,7 @@ let normMaster = {};
// TODO: integrate measurement device information from DPT names using different users
// TODO: supplier: other for supplierless samples
// TODO: BASF twice, BASF as color
main();

8
src/helpers/csv.ts
View File

@ -1,7 +1,13 @@
import {parseAsync} from 'json2csv';
export default function csv(input: any[], f: (err, data) => void) {
parseAsync(input.map(e => flatten(e)))
console.log(input[1000]);
console.log(flatten(input[1000]));
parseAsync([flatten(input[1000])]).then(csv => console.log(csv));
console.log(input[1]);
console.log(flatten(input[1]));
parseAsync([flatten(input[1])]).then(csv => console.log(csv));
parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
.then(csv => f(null, csv))
.catch(err => f(err, null));
}

1
src/index.ts
View File

@ -7,6 +7,7 @@ import helmet from 'helmet';
import api from './api';
import db from './db';
// TODO: working demo branch
// tell if server is running in debug or production environment
console.info(process.env.NODE_ENV === 'production' ? '===== PRODUCTION =====' : process.env.NODE_ENV === 'test' ? '' :'===== DEVELOPMENT =====');

2
src/models/material.ts
View File

@ -22,5 +22,7 @@ MaterialSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>>
db.log(req, this);
return this;
}
MaterialSchema.index({supplier_id: 1});
MaterialSchema.index({group_id: 1});
export default mongoose.model<any, mongoose.Model<any, any>>('material', MaterialSchema);

2
src/models/measurement.ts
View File

@ -17,5 +17,7 @@ MeasurementSchema.query.log = function <Q extends mongoose.DocumentQuery<any, an
db.log(req, this);
return this;
}
MeasurementSchema.index({sample_id: 1});
MeasurementSchema.index({measurement_template: 1});
export default mongoose.model<any, mongoose.Model<any, any>>('measurement', MeasurementSchema);

3
src/models/sample.ts
View File

@ -22,5 +22,8 @@ SampleSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>> (
db.log(req, this);
return this;
}
SampleSchema.index({material_id: 1});
SampleSchema.index({note_id: 1});
SampleSchema.index({user_id: 1});
export default mongoose.model<any, mongoose.Model<any, any>>('sample', SampleSchema);

4
src/routes/sample.spec.ts
View File

@ -252,7 +252,7 @@ describe('/sample', () => {
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body.find(e => e.number === '1')).have.property('kf', {});
should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null});
should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null});
done();
});
@ -328,7 +328,7 @@ describe('/sample', () => {
url: '/samples?status=all&page-size=1&fields[]=xx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\..+)$/m'}
res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'}
});
});
it('rejects a negative page size', done => {

213
src/routes/sample.ts
View File

@ -27,21 +27,10 @@ router.get('/samples', async (req, res, next) => {
const {error, value: filters} = SampleValidate.query(req.query);
if (error) return res400(error, res);
const query = [];
query.push({$match: {$and: []}});
if (filters.hasOwnProperty('status')) {
if(filters.status === 'all') {
query[0].$match.$and.push({$or: [{status: globals.status.validated}, {status: globals.status.new}]});
}
else {
query[0].$match.$and.push({status: globals.status[filters.status]});
}
}
else { // default
query[0].$match.$and.push({status: globals.status.validated});
}
// TODO: find a better place for these
const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', 'user_id'];
// sorting
// evaluate sort parameter from 'color-asc' to ['color', 1]
filters.sort = filters.sort.split('-');
filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id
filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1;
@ -49,62 +38,139 @@ router.get('/samples', async (req, res, next) => {
filters['to-page'] = 0;
}
query.push(
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$set: {material: { $arrayElemAt: ['$material', 0]}}},
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}},
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}},
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}
let collection;
const query = [];
query.push({$match: {$and: []}});
if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection
collection = MeasurementModel;
const measurementName = filters.sort[0].replace('measurements.', '');
const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);});
if (measurementTemplate instanceof Error) return;
if (!measurementTemplate) {
return res.status(400).json({status: 'Invalid body format', details: filters.sort[0] + ' not found'});
}
let sortStartValue = null;
if (filters['from-id']) { // from-id specified, fetch values for sorting
const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}).lean().exec().catch(err => {next(err);}); // TODO: what if more than one measurement for sample?
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
sortStartValue = fromSample.values[measurementTemplate.parameters[0].name];
}
);
query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort
query.push(
sortQuery(query, filters, ['values.' + measurementTemplate.parameters[0].name, 'sample_id'], sortStartValue), // sort measurements
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
{$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added
);
addSkipLimit(query, filters); // skip and limit to select right page
query.push(
{$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
);
if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);});
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
}
else { // sorting with samples as starting collection
collection = SampleModel;
// filter for status
query[0].$match.$and.push(statusQuery(filters, 'status'));
if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc
query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
query.push({$sort: {[filters.sort[0]]: 1, _id: 1}});
// differentiate by sort key to do sorting, skip and limit as early as possible
if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys
let sortStartValue = null;
if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);});
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
sortStartValue = fromSample[filters.sort[0]];
}
query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
addSkipLimit(query, filters);
}
else {
query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$lt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
query.push({$sort: {[filters.sort[0]]: -1, _id: -1}});
else { // sorting for material keys
let materialQuery = []
materialQuery.push( // add material properties
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$set: {material: { $arrayElemAt: ['$material', 0]}}}
);
if (filters.sort[0] === 'material.supplier') { // add supplier if needed
materialQuery.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
);
}
if (filters.sort[0] === 'material.group') { // add group if needed
materialQuery.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
if (filters.sort[0] === 'material.number') { // add material number if needed
materialQuery.push(
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
);
}
query.push(...materialQuery);
let sortStartValue = null;
if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);});
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
sortStartValue = fromSample[filters.sort[0]];
}
query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
addSkipLimit(query, filters);
}
}
else { // sort from beginning
query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}}); // set _id as secondary sort
const fieldsNoSort = filters.fields.filter(e => e !== filters.sort[0]); // sort field was definitely added already, exclude from further field operations
if (fieldsNoSort.find(e => /material\./.test(e))) { // add material fields
query.push(
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$set: {material: { $arrayElemAt: ['$material', 0]}}}
);
}
if (fieldsNoSort.indexOf('material.supplier') >= 0) { // add supplier if needed
query.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
);
}
if (fieldsNoSort.indexOf('material.group') >= 0) { // add group if needed
query.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
if (fieldsNoSort.indexOf('material.number') >= 0) { // add material number if needed
query.push(
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
);
}
if (filters['to-page']) {
query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more
}
if (filters['page-size']) {
query.push({$limit: filters['page-size']});
}
let measurementFields = [];
if (filters.fields.find(e => /measurements\./.test(e))) { // joining measurements is required
let measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', ''));
console.log(fieldsNoSort);
console.log(measurementFields);
if (fieldsNoSort.find(e => /measurements\./.test(e))) { // add measurement fields
query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', ''));
const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
if (measurementTemplates instanceof Error) return;
if (measurementTemplates.length < measurementFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}});
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
});
console.log(measurementFields);
if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
query.push(
{$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
@ -124,12 +190,11 @@ router.get('/samples', async (req, res, next) => {
}
query.push({$project: projection});
SampleModel.aggregate(query).exec((err, data) => {
collection.aggregate(query).exec((err, data) => {
if (err) return next(err);
if (filters['to-page'] < 0) {
data.reverse();
}
console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));
if (filters.csv) { // output as csv
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
if (err) return next(err);
@ -498,4 +563,42 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and
}
});
});
}
function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key']
if (filters['from-id']) { // from-id specified
if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc
query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
return {$sort: {[sortKeys[0]]: 1, _id: 1}};
} else {
query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
return {$sort: {[sortKeys[0]]: -1, _id: -1}};
}
} else { // sort from beginning
return {$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}; // set _id as secondary sort
}
}
function addSkipLimit(query, filters) {
if (filters['to-page']) {
query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more
}
if (filters['page-size']) {
query.push({$limit: filters['page-size']});
}
}
function statusQuery(filters, field) {
if (filters.hasOwnProperty('status')) {
if(filters.status === 'all') {
return {$or: [{[field]: globals.status.validated}, {[field]: globals.status.new}]};
}
else {
return {[field]: globals.status[filters.status]};
}
}
else { // default
return {[field]: globals.status.validated};
}
}

7
src/routes/validate/sample.ts
View File

@ -64,7 +64,8 @@ export default class SampleValidate {
'material.mineral',
'material.glass_fiber',
'material.carbon_fiber',
'material.number'
'material.number',
'measurements.(?!spectrum)*'
];
private static fieldKeys = [
@ -76,7 +77,7 @@ export default class SampleValidate {
'user_id',
'material._id',
'material.numbers',
'measurements.*'
'measurements.spectrum'
];
static input (data, param) { // validate input, set param to 'new' to make all attributes required
@ -168,7 +169,7 @@ export default class SampleValidate {
'from-id': IdValidate.get(),
'to-page': Joi.number().integer(),
'page-size': Joi.number().integer().min(1),
sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'),
sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'),
csv: Joi.boolean().default(false),
fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])
}).with('to-page', 'page-size').validate(data);