From 11be1f6f5be911e882c7b0c664a600971c0dac7f Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Fri, 14 Aug 2020 11:34:15 +0200 Subject: [PATCH] fixed multiple template measurements --- src/db.ts | 2 +- src/index.ts | 1 - src/routes/root.ts | 1 - src/routes/sample.spec.ts | 7 +-- src/routes/sample.ts | 108 +++++++++++++++----------------- src/routes/template.spec.ts | 1 - src/routes/user.ts | 1 - src/routes/validate/sample.ts | 17 ++++- src/routes/validate/template.ts | 1 - 9 files changed, 69 insertions(+), 70 deletions(-) diff --git a/src/db.ts b/src/db.ts index 2beb95a..cfebbbe 100644 --- a/src/db.ts +++ b/src/db.ts @@ -7,7 +7,7 @@ import ChangelogModel from './models/changelog'; // database urls, prod db url is retrieved automatically const TESTING_URL = 'mongodb://localhost/dfopdb_test'; const DEV_URL = 'mongodb://localhost/dfopdb'; -const debugging = false; +const debugging = true; if (process.env.NODE_ENV !== 'production' && debugging) { mongoose.set('debug', true); // enable mongoose debug diff --git a/src/index.ts b/src/index.ts index 5b4ed8f..a1c7417 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,6 @@ import api from './api'; import db from './db'; import Mail from './helpers/mail'; -// TODO: check header, also in UI // tell if server is running in debug or production environment console.info(process.env.NODE_ENV === 'production' ? diff --git a/src/routes/root.ts b/src/routes/root.ts index 23f3b8f..cee54fe 100644 --- a/src/routes/root.ts +++ b/src/routes/root.ts @@ -22,7 +22,6 @@ router.get('/authorized', (req, res) => { }); }); -// TODO: evaluate exact changelog functionality (restoring, deleting after time, etc.) router.get('/changelog/:timestamp/:page?/:pagesize?', (req, res, next) => { if (!req.auth(res, ['dev', 'admin'], 'basic')) return; diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 5a6405c..e10d2c2 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -15,9 +15,6 @@ describe('/sample', () => { afterEach(done => TestHelper.afterEach(server, done)); after(done => TestHelper.after(done)); - // TODO: sort, added date filter, has measurements/condition filter - // TODO: check if conditions work in sort/fields/filters - // TODO: test for numbers as strings in glass_fiber describe('GET /samples', () => { it('returns all samples', done => { TestHelper.request(server, done, { @@ -298,7 +295,7 @@ describe('/sample', () => { done(); }); }); - it('filters a sample property', done => { // TODO: implement filters + it('filters a sample property', done => { TestHelper.request(server, done, { method: 'get', url: '/samples?status[]=new&status[]=validated&fields[]=number&fields[]=type&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22type%22%2C%22values%22%3A%5B%22processed%22%5D%7D', @@ -801,7 +798,7 @@ describe('/sample', () => { }); }); - describe('PUT /sample/{id}', () => { // TODO: fix tests, work on /samples + describe('PUT /sample/{id}', () => { it('returns the right sample', done => { TestHelper.request(server, done, { method: 'put', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index ab8ff1c..55665e5 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -22,13 +22,6 @@ import globals from '../globals'; const router = express.Router(); -// TODO: check added filter -// TODO: convert filter value to number according to table model -// TODO: validation for filter parameters -// TODO: location/device sort/filter - -// TODO: think about filter keys with measurement template versions - router.get('/samples', async (req, res, next) => { if (!req.auth(res, ['read', 'write', 'dev', 'admin'], 'all')) return; @@ -40,12 +33,6 @@ router.get('/samples', async (req, res, next) => { if ((filters.fields.find(e => e.indexOf('.' + globals.spectrum.dpt) >= 0) || filters.output !== 'json') && !req.auth(res, ['dev', 'admin'], 'all')) return; - // TODO: find a better place for these - const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', - 'user_id']; - - // TODO find further optimizations from bachelor thesis - // evaluate sort parameter from 'color-asc' to ['color', 1] filters.sort = filters.sort.split('-'); filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id @@ -123,7 +110,7 @@ router.get('/samples', async (req, res, next) => { let sortStartValue = null; if (filters['from-id']) { // from-id specified, fetch values for sorting const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}) - .lean().exec().catch(err => {next(err);}); // TODO: what if more than one measurement for sample? + .lean().exec().catch(err => {next(err);}); if (fromSample instanceof Error) return; if (!fromSample) { return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); @@ -149,7 +136,8 @@ router.get('/samples', async (req, res, next) => { collection = SampleModel; queryPtr[0].$match.$and.push(statusQuery(filters, 'status')); - if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys + // sorting for sample keys + if (SampleValidate.sampleKeys.indexOf(filters.sort[0]) >= 0 || /condition\./.test(filters.sort[0])) { let sortStartValue = null; if (filters['from-id']) { // from-id specified const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => { @@ -168,13 +156,15 @@ router.get('/samples', async (req, res, next) => { } } - addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters + addFilterQueries(queryPtr, filters.filters.filter( + e => (SampleValidate.sampleKeys.indexOf(e.field) >= 0) || /condition\./.test(e.field)) + ); // sample filters let materialQuery = []; // put material query together separate first to reuse for first-id let materialAdded = false; if (sortFilterKeys.find(e => /material\./.test(e))) { // add material fields materialAdded = true; - materialQuery.push( // add material properties // TODO: project out unnecessary fields + materialQuery.push( // add material properties {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$addFields: {material: {$arrayElemAt: ['$material', 0]}}} ); @@ -198,16 +188,8 @@ router.get('/samples', async (req, res, next) => { {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } - // TODO: adapt code to new numbers format - // if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed - // materialQuery.push( - // {$addFields: {'material.number': { $arrayElemAt: [ - // '$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']} - // ]}}} - // ); - // } const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)) - .filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); // TODO + .filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); // base material filters addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); queryPtr.push(...materialQuery); @@ -250,8 +232,17 @@ router.get('/samples', async (req, res, next) => { ]}}}], as: 'measurements' }}); - measurementTemplates.forEach(template => { - addMeasurements(queryPtr, template); + const groupedMeasurementTemplates = measurementTemplates.reduce((s, e) => { + if (s.hasOwnProperty(e.name)) { + s[e.name].push(e); + } + else { + s[e.name] = [e]; + } + return s; + }, {}); + Object.values(groupedMeasurementTemplates).forEach(templates => { + addMeasurements(queryPtr, templates); }); addFilterQueries(queryPtr, filters.filters .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) @@ -310,13 +301,6 @@ router.get('/samples', async (req, res, next) => { {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } - // if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed // TODO - // queryPtr.push( - // {$addFields: {'material.number': { - // $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}] - // }}} - // ); - // } let measurementFieldsFields: string[] = _.uniq( fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]) @@ -328,8 +312,8 @@ router.get('/samples', async (req, res, next) => { if (measurementTemplates.length < measurementFieldsFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } - // use different lookup methods with and without spectrum for the best performance - if (fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.') >= 0)) { + // use different lookup methods with and without dpt for the best performance + if (fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.' + globals.spectrum.dpt) >= 0)) { // with dpt queryPtr.push( {$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}} ); @@ -344,21 +328,23 @@ router.get('/samples', async (req, res, next) => { as: 'measurements' }}); } - measurementTemplates.forEach(template => { - addMeasurements(queryPtr, template); - if (measurementFieldsFields.find(e => e === globals.spectrum.spectrum)) { - queryPtr.push({$unwind: '$' + globals.spectrum.spectrum}); + const groupedMeasurementTemplates = measurementTemplates.reduce((s, e) => { + if (s.hasOwnProperty(e.name)) { + s[e.name].push(e); } + else { + s[e.name] = [e]; + } + return s; + }, {}); + Object.values(groupedMeasurementTemplates).forEach(templates => { + addMeasurements(queryPtr, templates); }); queryPtr.push({$project: {measurements: 0}}); } const projection = filters.fields.map(e => e.replace('measurements.', '')) .reduce((s, e) => {s[e] = true; return s; }, {}); - if (filters.fields.indexOf('added') >= 0) { // add added date // TODO: upgrade MongoDB version or find alternative - // projection.added = {$toDate: '$_id'}; - // projection.added = { $convert: { input: '$_id', to: "date" } } - } if (filters.fields.indexOf('_id') < 0 && filters.fields.indexOf('added') < 0) { // disable _id explicitly projection._id = false; } @@ -781,12 +767,13 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] if (filters['from-id']) { // from-id specified + const ssv = sortStartValue !== undefined; // if value is not given, match for existence if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc return [ {$match: {$or: [ - {[sortKeys[0]]: {$gt: sortStartValue}}, + {[sortKeys[0]]: ssv ? {$gt: sortStartValue} : {$exists: true}}, {$and: [ - {[sortKeys[0]]: sortStartValue}, + {[sortKeys[0]]: ssv ? sortStartValue : {$exists: false}}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}} ]} ]}}, @@ -795,9 +782,9 @@ function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary } else { return [ {$match: {$or: [ - {[sortKeys[0]]: {$lt: sortStartValue}}, + {[sortKeys[0]]: ssv ? {$lt: sortStartValue} : {$exists: false}}, {$and: [ - {[sortKeys[0]]: sortStartValue}, + {[sortKeys[0]]: ssv ? sortStartValue : {$exists: true}}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}} ]} ]}}, @@ -834,20 +821,25 @@ function filterQueries (filters) { }); } -// add measurements as property [template.name], if one result, array is reduced to direct values -function addMeasurements(queryPtr, template) { +// add measurements as property [template.name], if one result, array is reduced to direct values. All given templates +// must have the same name +function addMeasurements(queryPtr, templates) { queryPtr.push( - {$addFields: {[template.name]: {$let: {vars: { + {$addFields: {[templates[0].name]: {$let: {vars: { arr: {$filter: { - input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]} + input: '$measurements', cond: {$in: [ + '$$this.measurement_template', + templates.map(e => mongoose.Types.ObjectId(e._id)) + ]} }}}, in: {$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, - {$addFields: {[template.name]: {$cond: [ - '$' + template.name + '.values', - '$' + template.name + '.values', - template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {}) - ]}}} + {$addFields: {[templates[0].name]: {$cond: [ + '$' + templates[0].name + '.values', + '$' + templates[0].name + '.values', + templates[0].parameters.reduce((s, e) => {s[e.name] = null; return s;}, {}) + ]}}}, + {$unwind: '$' + templates[0].name} ); } diff --git a/src/routes/template.spec.ts b/src/routes/template.spec.ts index f936c46..db924b3 100644 --- a/src/routes/template.spec.ts +++ b/src/routes/template.spec.ts @@ -4,7 +4,6 @@ import TemplateConditionModel from '../models/condition_template'; import TemplateMeasurementModel from '../models/measurement_template'; import TestHelper from "../test/helper"; -// TODO: method to return only latest template versions -> rework frontend accordingly describe('/template', () => { let server; diff --git a/src/routes/user.ts b/src/routes/user.ts index 963af27..976c1a7 100644 --- a/src/routes/user.ts +++ b/src/routes/user.ts @@ -81,7 +81,6 @@ router.put('/user:username([/](?!key|new).?*|/?)', async (req, res, next) => { }); }); -// TODO: only possible if no data is linked to user, otherwise change status, etc. // this path matches /user, /user/ and /user/xxx, but not /user/key or user/new. // See https://forbeslindesay.github.io/express-route-tester/ for the generated regex router.delete('/user:username([/](?!key|new).?*|/?)', (req, res, next) => { diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index db6ef19..665a01b 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -6,6 +6,7 @@ import MaterialValidate from './material'; import MeasurementValidate from './measurement'; import globals from '../../globals'; + export default class SampleValidate { private static sample = { number: Joi.string() @@ -56,6 +57,19 @@ export default class SampleValidate { .valid(...Object.values(globals.status)) }; + static readonly sampleKeys = [ // keys which can be found in the sample directly + '_id', + 'color', + 'number', + 'type', + 'batch', + 'added', + 'condition', + 'material_id', + 'note_id', + 'user_id' + ]; + private static sortKeys = [ '_id', 'color', @@ -68,6 +82,7 @@ export default class SampleValidate { 'material.supplier', 'material.group', 'material.properties.*', + 'condition.*', `measurements.(?!${globals.spectrum.spectrum}.${globals.spectrum.dpt})*` ]; @@ -189,7 +204,7 @@ export default class SampleValidate { }); field = field.replace('material.', '').split('.')[0]; } - else if (/measurements\./.test(field)) { + else if (/measurements\./.test(field) || /condition\./.test(field)) { validator = Joi.object({ value: Joi.alternatives() .try( diff --git a/src/routes/validate/template.ts b/src/routes/validate/template.ts index 8378d92..bcc515f 100644 --- a/src/routes/validate/template.ts +++ b/src/routes/validate/template.ts @@ -1,7 +1,6 @@ import Joi from 'joi'; import IdValidate from './id'; -// TODO: do not allow a . in the name !!! export default class TemplateValidate { private static template = { name: Joi.string()