banana/definma-api
Archived
2
Fork 0
Code Releases Activity

fixed multiple template measurements

Browse Source
This commit is contained in:
VLE2FE
2020-08-14 11:34:15 +02:00
parent 08a9a12372
commit 11be1f6f5b
9 changed files with 69 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/db.ts
View File

@@ -7,7 +7,7 @@ import ChangelogModel from './models/changelog';
// database urls, prod db url is retrieved automatically
const TESTING_URL = 'mongodb://localhost/dfopdb_test';
const DEV_URL = 'mongodb://localhost/dfopdb';
const debugging = false;
const debugging = true;
if (process.env.NODE_ENV !== 'production' && debugging) {
mongoose.set('debug', true); // enable mongoose debug

1
src/index.ts
View File

@@ -8,7 +8,6 @@ import api from './api';
import db from './db';
import Mail from './helpers/mail';
// TODO: check header, also in UI
// tell if server is running in debug or production environment
console.info(process.env.NODE_ENV === 'production' ?

1
src/routes/root.ts
View File

@@ -22,7 +22,6 @@ router.get('/authorized', (req, res) => {
});
});
// TODO: evaluate exact changelog functionality (restoring, deleting after time, etc.)
router.get('/changelog/:timestamp/:page?/:pagesize?', (req, res, next) => {
if (!req.auth(res, ['dev', 'admin'], 'basic')) return;

7
src/routes/sample.spec.ts
View File

@@ -15,9 +15,6 @@ describe('/sample', () => {
afterEach(done => TestHelper.afterEach(server, done));
after(done => TestHelper.after(done));
// TODO: sort, added date filter, has measurements/condition filter
// TODO: check if conditions work in sort/fields/filters
// TODO: test for numbers as strings in glass_fiber
describe('GET /samples', () => {
it('returns all samples', done => {
TestHelper.request(server, done, {
@@ -298,7 +295,7 @@ describe('/sample', () => {
done();
});
});
it('filters a sample property', done => { // TODO: implement filters
it('filters a sample property', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status[]=new&status[]=validated&fields[]=number&fields[]=type&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22type%22%2C%22values%22%3A%5B%22processed%22%5D%7D',
@@ -801,7 +798,7 @@ describe('/sample', () => {
});
});
describe('PUT /sample/{id}', () => { // TODO: fix tests, work on /samples
describe('PUT /sample/{id}', () => {
it('returns the right sample', done => {
TestHelper.request(server, done, {
method: 'put',

108
src/routes/sample.ts
View File

@@ -22,13 +22,6 @@ import globals from '../globals';
const router = express.Router();
// TODO: check added filter
// TODO: convert filter value to number according to table model
// TODO: validation for filter parameters
// TODO: location/device sort/filter
// TODO: think about filter keys with measurement template versions
router.get('/samples', async (req, res, next) => {
if (!req.auth(res, ['read', 'write', 'dev', 'admin'], 'all')) return;
@@ -40,12 +33,6 @@ router.get('/samples', async (req, res, next) => {
if ((filters.fields.find(e => e.indexOf('.' + globals.spectrum.dpt) >= 0) || filters.output !== 'json') &&
!req.auth(res, ['dev', 'admin'], 'all')) return;
// TODO: find a better place for these
const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id',
'user_id'];
// TODO find further optimizations from bachelor thesis
// evaluate sort parameter from 'color-asc' to ['color', 1]
filters.sort = filters.sort.split('-');
filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id
@@ -123,7 +110,7 @@ router.get('/samples', async (req, res, next) => {
let sortStartValue = null;
if (filters['from-id']) { // from-id specified, fetch values for sorting
const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])})
.lean().exec().catch(err => {next(err);}); // TODO: what if more than one measurement for sample?
.lean().exec().catch(err => {next(err);});
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
@@ -149,7 +136,8 @@ router.get('/samples', async (req, res, next) => {
collection = SampleModel;
queryPtr[0].$match.$and.push(statusQuery(filters, 'status'));
if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys
// sorting for sample keys
if (SampleValidate.sampleKeys.indexOf(filters.sort[0]) >= 0 || /condition\./.test(filters.sort[0])) {
let sortStartValue = null;
if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {
@@ -168,13 +156,15 @@ router.get('/samples', async (req, res, next) => {
}
}
addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
addFilterQueries(queryPtr, filters.filters.filter(
e => (SampleValidate.sampleKeys.indexOf(e.field) >= 0) || /condition\./.test(e.field))
); // sample filters
let materialQuery = []; // put material query together separate first to reuse for first-id
let materialAdded = false;
if (sortFilterKeys.find(e => /material\./.test(e))) { // add material fields
materialAdded = true;
materialQuery.push( // add material properties // TODO: project out unnecessary fields
materialQuery.push( // add material properties
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$addFields: {material: {$arrayElemAt: ['$material', 0]}}}
);
@@ -198,16 +188,8 @@ router.get('/samples', async (req, res, next) => {
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
// TODO: adapt code to new numbers format
// if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed
// materialQuery.push(
// {$addFields: {'material.number': { $arrayElemAt: [
// '$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}
// ]}}}
// );
// }
const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e))
.filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); // TODO
.filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
// base material filters
addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0));
queryPtr.push(...materialQuery);
@@ -250,8 +232,17 @@ router.get('/samples', async (req, res, next) => {
]}}}],
as: 'measurements'
}});
measurementTemplates.forEach(template => {
addMeasurements(queryPtr, template);
const groupedMeasurementTemplates = measurementTemplates.reduce((s, e) => {
if (s.hasOwnProperty(e.name)) {
s[e.name].push(e);
}
else {
s[e.name] = [e];
}
return s;
}, {});
Object.values(groupedMeasurementTemplates).forEach(templates => {
addMeasurements(queryPtr, templates);
});
addFilterQueries(queryPtr, filters.filters
.filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
@@ -310,13 +301,6 @@ router.get('/samples', async (req, res, next) => {
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
// if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed // TODO
// queryPtr.push(
// {$addFields: {'material.number': {
// $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]
// }}}
// );
// }
let measurementFieldsFields: string[] = _.uniq(
fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])
@@ -328,8 +312,8 @@ router.get('/samples', async (req, res, next) => {
if (measurementTemplates.length < measurementFieldsFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
// use different lookup methods with and without spectrum for the best performance
if (fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.') >= 0)) {
// use different lookup methods with and without dpt for the best performance
if (fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.' + globals.spectrum.dpt) >= 0)) { // with dpt
queryPtr.push(
{$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}
);
@@ -344,21 +328,23 @@ router.get('/samples', async (req, res, next) => {
as: 'measurements'
}});
}
measurementTemplates.forEach(template => {
addMeasurements(queryPtr, template);
if (measurementFieldsFields.find(e => e === globals.spectrum.spectrum)) {
queryPtr.push({$unwind: '$' + globals.spectrum.spectrum});
const groupedMeasurementTemplates = measurementTemplates.reduce((s, e) => {
if (s.hasOwnProperty(e.name)) {
s[e.name].push(e);
}
else {
s[e.name] = [e];
}
return s;
}, {});
Object.values(groupedMeasurementTemplates).forEach(templates => {
addMeasurements(queryPtr, templates);
});
queryPtr.push({$project: {measurements: 0}});
}
const projection = filters.fields.map(e => e.replace('measurements.', ''))
.reduce((s, e) => {s[e] = true; return s; }, {});
if (filters.fields.indexOf('added') >= 0) { // add added date // TODO: upgrade MongoDB version or find alternative
// projection.added = {$toDate: '$_id'};
// projection.added = { $convert: { input: '$_id', to: "date" } }
}
if (filters.fields.indexOf('_id') < 0 && filters.fields.indexOf('added') < 0) { // disable _id explicitly
projection._id = false;
}
@@ -781,12 +767,13 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and
function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key']
if (filters['from-id']) { // from-id specified
const ssv = sortStartValue !== undefined; // if value is not given, match for existence
if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc
return [
{$match: {$or: [
{[sortKeys[0]]: {$gt: sortStartValue}},
{[sortKeys[0]]: ssv ? {$gt: sortStartValue} : {$exists: true}},
{$and: [
{[sortKeys[0]]: sortStartValue},
{[sortKeys[0]]: ssv ? sortStartValue : {$exists: false}},
{[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}
]}
]}},
@@ -795,9 +782,9 @@ function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary
} else {
return [
{$match: {$or: [
{[sortKeys[0]]: {$lt: sortStartValue}},
{[sortKeys[0]]: ssv ? {$lt: sortStartValue} : {$exists: false}},
{$and: [
{[sortKeys[0]]: sortStartValue},
{[sortKeys[0]]: ssv ? sortStartValue : {$exists: true}},
{[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}
]}
]}},
@@ -834,20 +821,25 @@ function filterQueries (filters) {
});
}
// add measurements as property [template.name], if one result, array is reduced to direct values
function addMeasurements(queryPtr, template) {
// add measurements as property [template.name], if one result, array is reduced to direct values. All given templates
// must have the same name
function addMeasurements(queryPtr, templates) {
queryPtr.push(
{$addFields: {[template.name]: {$let: {vars: {
{$addFields: {[templates[0].name]: {$let: {vars: {
arr: {$filter: {
input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}
input: '$measurements', cond: {$in: [
'$$this.measurement_template',
templates.map(e => mongoose.Types.ObjectId(e._id))
]}
}}},
in: {$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}},
{$addFields: {[template.name]: {$cond: [
'$' + template.name + '.values',
'$' + template.name + '.values',
template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})
]}}}
{$addFields: {[templates[0].name]: {$cond: [
'$' + templates[0].name + '.values',
'$' + templates[0].name + '.values',
templates[0].parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})
]}}},
{$unwind: '$' + templates[0].name}
);
}

1
src/routes/template.spec.ts
View File

@@ -4,7 +4,6 @@ import TemplateConditionModel from '../models/condition_template';
import TemplateMeasurementModel from '../models/measurement_template';
import TestHelper from "../test/helper";
// TODO: method to return only latest template versions -> rework frontend accordingly
describe('/template', () => {
let server;

1
src/routes/user.ts
View File

@@ -81,7 +81,6 @@ router.put('/user:username([/](?!key|new).?*|/?)', async (req, res, next) => {
});
});
// TODO: only possible if no data is linked to user, otherwise change status, etc.
// this path matches /user, /user/ and /user/xxx, but not /user/key or user/new.
// See https://forbeslindesay.github.io/express-route-tester/ for the generated regex
router.delete('/user:username([/](?!key|new).?*|/?)', (req, res, next) => {

17
src/routes/validate/sample.ts
View File

@@ -6,6 +6,7 @@ import MaterialValidate from './material';
import MeasurementValidate from './measurement';
import globals from '../../globals';
export default class SampleValidate {
private static sample = {
number: Joi.string()
@@ -56,6 +57,19 @@ export default class SampleValidate {
.valid(...Object.values(globals.status))
};
static readonly sampleKeys = [ // keys which can be found in the sample directly
'_id',
'color',
'number',
'type',
'batch',
'added',
'condition',
'material_id',
'note_id',
'user_id'
];
private static sortKeys = [
'_id',
'color',
@@ -68,6 +82,7 @@ export default class SampleValidate {
'material.supplier',
'material.group',
'material.properties.*',
'condition.*',
`measurements.(?!${globals.spectrum.spectrum}.${globals.spectrum.dpt})*`
];
@@ -189,7 +204,7 @@ export default class SampleValidate {
});
field = field.replace('material.', '').split('.')[0];
}
else if (/measurements\./.test(field)) {
else if (/measurements\./.test(field) || /condition\./.test(field)) {
validator = Joi.object({
value: Joi.alternatives()
.try(

1
src/routes/validate/template.ts
View File

@@ -1,7 +1,6 @@
import Joi from 'joi';
import IdValidate from './id';
// TODO: do not allow a . in the name !!!
export default class TemplateValidate {
private static template = {
name: Joi.string()