From 08a9a12372e40a9b0217fcbba9e5236c8a106264 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 13 Aug 2020 12:07:40 +0200 Subject: [PATCH 1/3] model routes --- .idea/dictionaries/VLE2FE.xml | 3 + api/model.yaml | 14 +-- src/index.ts | 5 +- src/models/model.ts | 8 ++ src/routes/model.spec.ts | 197 ++++++++++++++++++++++++++++++++++ src/routes/model.ts | 47 ++++++++ src/test/db.json | 7 ++ src/test/helper.ts | 3 + 8 files changed, 273 insertions(+), 11 deletions(-) create mode 100644 src/models/model.ts create mode 100644 src/routes/model.spec.ts create mode 100644 src/routes/model.ts diff --git a/.idea/dictionaries/VLE2FE.xml b/.idea/dictionaries/VLE2FE.xml index 5e9cd2d..62d7fb4 100644 --- a/.idea/dictionaries/VLE2FE.xml +++ b/.idea/dictionaries/VLE2FE.xml @@ -37,6 +37,9 @@ lati lyucy materialnumber + modela + modelb + modelx nvmrc oldpass opblock diff --git a/api/model.yaml b/api/model.yaml index f9c3d72..b701df4 100644 --- a/api/model.yaml +++ b/api/model.yaml @@ -2,7 +2,7 @@ parameters: - $ref: 'api.yaml#/components/parameters/Name' get: - summary: TODO get model data by name + summary: get model data by name description: 'Auth: all, levels: dev, admin' tags: - /model @@ -22,14 +22,14 @@ $ref: 'api.yaml#/components/responses/404' 500: $ref: 'api.yaml#/components/responses/500' - put: - summary: TODO add/replace model data by name + post: + summary: add/replace model data by name description: 'Auth: all, levels: dev, admin' tags: - /model requestBody: required: true - description: binary model data + description: binary model data, Content-Type header must be set to application/octet-stream content: application/json: schema: @@ -38,18 +38,14 @@ responses: 200: $ref: 'api.yaml#/components/responses/Ok' - 400: - $ref: 'api.yaml#/components/responses/400' 401: $ref: 'api.yaml#/components/responses/401' 403: $ref: 'api.yaml#/components/responses/403' - 404: - $ref: 'api.yaml#/components/responses/404' 500: $ref: 'api.yaml#/components/responses/500' delete: - summary: TODO delete model data + summary: delete model data description: 'Auth: basic, levels: dev, admin' tags: - /model diff --git a/src/index.ts b/src/index.ts index 1e763b7..5b4ed8f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -111,9 +111,10 @@ if (process.env.NODE_ENV !== 'production') { app.use('/', require('./routes/root')); app.use('/', require('./routes/sample')); app.use('/', require('./routes/material')); -app.use('/', require('./routes/template')); -app.use('/', require('./routes/user')); app.use('/', require('./routes/measurement')); +app.use('/', require('./routes/template')); +app.use('/', require('./routes/model')); +app.use('/', require('./routes/user')); // static files app.use('/static', express.static('static')); diff --git a/src/models/model.ts b/src/models/model.ts new file mode 100644 index 0000000..925601f --- /dev/null +++ b/src/models/model.ts @@ -0,0 +1,8 @@ +import mongoose from 'mongoose'; + +const ModelSchema = new mongoose.Schema({ + name: {type: String, index: {unique: true}}, + data: Buffer +}); + +export default mongoose.model>('model', ModelSchema); \ No newline at end of file diff --git a/src/routes/model.spec.ts b/src/routes/model.spec.ts new file mode 100644 index 0000000..382efd4 --- /dev/null +++ b/src/routes/model.spec.ts @@ -0,0 +1,197 @@ +import should from 'should/as-function'; +import ModelModel from '../models/model'; +import TestHelper from "../test/helper"; + + +describe('/model', () => { + let server; + before(done => TestHelper.before(done)); + beforeEach(done => server = TestHelper.beforeEach(server, done)); + afterEach(done => TestHelper.afterEach(server, done)); + after(done => TestHelper.after(done)); + + describe('GET /model/{name}', (() => { + it('returns the binary data', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/model/modela', + auth: {basic: 'admin'}, + httpStatus: 200, + contentType: 'application/octet-stream; charset=utf-8', + }).end((err, res) => { + if (err) return done (err); + should(res.body.toString()).be.eql('binary data'); + done(); + }); + }); + it('returns the binary data for an API key', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/model/modela', + auth: {key: 'admin'}, + httpStatus: 200, + contentType: 'application/octet-stream; charset=utf-8', + }).end((err, res) => { + if (err) return done (err); + should(res.body.toString()).be.eql('binary data'); + done(); + }); + }); + it('returns 404 for an unknown name', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/model/modelx', + auth: {basic: 'admin'}, + httpStatus: 404 + }) + }); + it('rejects requests from a write user', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/model/modela', + auth: {basic: 'janedoe'}, + httpStatus: 403 + }) + }); + it('rejects unauthorized requests', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/model/modela', + httpStatus: 401 + }) + }); + })); + + describe('POST /model/{name}', () => { + it('stores the data', done => { + TestHelper.request(server, done, { + method: 'post', + url: '/model/modelb', + auth: {basic: 'admin'}, + httpStatus: 200, + reqContentType: 'application/octet-stream', + req: 'another binary data' + }).end((err, res) => { + if (err) return done (err); + should(res.body).be.eql({status: 'OK'}); + ModelModel.find({name: 'modelb'}).lean().exec((err, data) => { + if (err) return done (err); + should(data).have.lengthOf(1); + should(data[0]).have.only.keys('_id', 'name', 'data', '__v'); + should(data[0]).have.property('name', 'modelb'); + should(data[0].data.buffer.toString()).be.eql('another binary data'); + done(); + }); + }); + }); + it('stores the data with an API key', done => { + TestHelper.request(server, done, { + method: 'post', + url: '/model/modelb', + auth: {key: 'admin'}, + httpStatus: 200, + reqContentType: 'application/octet-stream', + req: 'another binary data' + }).end((err, res) => { + if (err) return done (err); + should(res.body).be.eql({status: 'OK'}); + ModelModel.find({name: 'modelb'}).lean().exec((err, data) => { + if (err) return done (err); + should(data).have.lengthOf(1); + should(data[0]).have.only.keys('_id', 'name', 'data', '__v'); + should(data[0]).have.property('name', 'modelb'); + should(data[0].data.buffer.toString()).be.eql('another binary data'); + done(); + }); + }); + }); + it('overwrites existing data', done => { + TestHelper.request(server, done, { + method: 'post', + url: '/model/modela', + auth: {basic: 'admin'}, + httpStatus: 200, + reqContentType: 'application/octet-stream', + req: 'another binary data' + }).end((err, res) => { + if (err) return done (err); + should(res.body).be.eql({status: 'OK'}); + ModelModel.find({name: 'modela'}).lean().exec((err, data) => { + if (err) return done (err); + should(data).have.lengthOf(1); + should(data[0]).have.only.keys('_id', 'name', 'data', '__v'); + should(data[0]).have.property('name', 'modela'); + should(data[0].data.buffer.toString()).be.eql('another binary data'); + done(); + }); + }); + }); + it('rejects requests from a write user', done => { + TestHelper.request(server, done, { + method: 'post', + url: '/model/modelb', + auth: {basic: 'janedoe'}, + httpStatus: 403, + req: 'another binary data' + }); + }); + it('rejects unauthorized requests', done => { + TestHelper.request(server, done, { + method: 'post', + url: '/model/modelb', + httpStatus: 401, + req: 'another binary data' + }); + }); + }); + + describe('DELETE /model/{name}', () => { + it('deletes the data', done => { + TestHelper.request(server, done, { + method: 'delete', + url: '/model/modela', + auth: {basic: 'admin'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).be.eql({status: 'OK'}); + ModelModel.find({name: 'modela'}).lean().exec((err, data) => { + if (err) return done(err); + should(data).have.lengthOf(0); + done(); + }); + }); + }); + it('returns 404 for an unknown name', done => { + TestHelper.request(server, done, { + method: 'delete', + url: '/model/modelx', + auth: {basic: 'admin'}, + httpStatus: 404 + }); + }); + it('rejects an API key', done => { + TestHelper.request(server, done, { + method: 'delete', + url: '/model/modela', + auth: {key: 'admin'}, + httpStatus: 401 + }); + }); + it('rejects a write user', done => { + TestHelper.request(server, done, { + method: 'delete', + url: '/model/modela', + auth: {basic: 'janedoe'}, + httpStatus: 403 + }); + }); + it('rejects an unauthorized request', done => { + TestHelper.request(server, done, { + method: 'delete', + url: '/model/modela', + httpStatus: 401 + }); + }); + }); +}); \ No newline at end of file diff --git a/src/routes/model.ts b/src/routes/model.ts new file mode 100644 index 0000000..882de58 --- /dev/null +++ b/src/routes/model.ts @@ -0,0 +1,47 @@ +import express from 'express'; +import bodyParser from 'body-parser'; + +import ModelModel from '../models/model'; + +const router = express.Router(); + +router.get('/model/:name', (req, res, next) => { + if (!req.auth(res, ['dev', 'admin'], 'all')) return; + + ModelModel.findOne({name: req.params.name}).lean().exec((err, data) => { + if (err) return next(err); + if (data) { + res.set('Content-Type', 'application/octet-stream'); + res.send(data.data.buffer); + } + else { + res.status(404).json({status: 'Not found'}); + } + }); +}); + +router.post('/model/:name', bodyParser.raw({limit: '500kb'}), (req, res, next) => { + if (!req.auth(res, ['dev', 'admin'], 'all')) return; + + ModelModel.replaceOne({name: req.params.name}, {name: req.params.name, data: req.body}).setOptions({upsert: true}) + .lean().exec(err => { + if (err) return next(err); + res.json({status: 'OK'}); + }); +}); + +router.delete('/model/:name', (req, res, next) => { + if (!req.auth(res, ['dev', 'admin'], 'basic')) return; + + ModelModel.findOneAndDelete({name: req.params.name}).lean().exec((err, data) => { + if (err) return next(err); + if (data) { + res.json({status: 'OK'}); + } + else { + res.status(404).json({status: 'Not found'}); + } + }); +}); + +module.exports = router; \ No newline at end of file diff --git a/src/test/db.json b/src/test/db.json index 83fba42..468c43d 100644 --- a/src/test/db.json +++ b/src/test/db.json @@ -678,6 +678,13 @@ "__v": 0 } ], + "models": [ + { + "_id": {"$oid":"140000000000000000000001"}, + "name": "modela", + "data": {"buffer": "binary data"} + } + ], "users": [ { "_id": {"$oid":"000000000000000000000001"}, diff --git a/src/test/helper.ts b/src/test/helper.ts index d4b11e4..ff337a3 100644 --- a/src/test/helper.ts +++ b/src/test/helper.ts @@ -69,6 +69,9 @@ export default class TestHelper { if (options.hasOwnProperty('req')) { // request body st = st.send(options.req); } + if (options.hasOwnProperty('reqContentType')) { // request body + st = st.set('Content-Type', options.reqContentType); + } if (options.hasOwnProperty('auth') && options.auth.hasOwnProperty('basic')) { // resolve basic auth if (this.auth.hasOwnProperty(options.auth.basic)) { st = st.auth(options.auth.basic, this.auth[options.auth.basic].pass) From 11be1f6f5be911e882c7b0c664a600971c0dac7f Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Fri, 14 Aug 2020 11:34:15 +0200 Subject: [PATCH 2/3] fixed multiple template measurements --- src/db.ts | 2 +- src/index.ts | 1 - src/routes/root.ts | 1 - src/routes/sample.spec.ts | 7 +-- src/routes/sample.ts | 108 +++++++++++++++----------------- src/routes/template.spec.ts | 1 - src/routes/user.ts | 1 - src/routes/validate/sample.ts | 17 ++++- src/routes/validate/template.ts | 1 - 9 files changed, 69 insertions(+), 70 deletions(-) diff --git a/src/db.ts b/src/db.ts index 2beb95a..cfebbbe 100644 --- a/src/db.ts +++ b/src/db.ts @@ -7,7 +7,7 @@ import ChangelogModel from './models/changelog'; // database urls, prod db url is retrieved automatically const TESTING_URL = 'mongodb://localhost/dfopdb_test'; const DEV_URL = 'mongodb://localhost/dfopdb'; -const debugging = false; +const debugging = true; if (process.env.NODE_ENV !== 'production' && debugging) { mongoose.set('debug', true); // enable mongoose debug diff --git a/src/index.ts b/src/index.ts index 5b4ed8f..a1c7417 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,6 @@ import api from './api'; import db from './db'; import Mail from './helpers/mail'; -// TODO: check header, also in UI // tell if server is running in debug or production environment console.info(process.env.NODE_ENV === 'production' ? diff --git a/src/routes/root.ts b/src/routes/root.ts index 23f3b8f..cee54fe 100644 --- a/src/routes/root.ts +++ b/src/routes/root.ts @@ -22,7 +22,6 @@ router.get('/authorized', (req, res) => { }); }); -// TODO: evaluate exact changelog functionality (restoring, deleting after time, etc.) router.get('/changelog/:timestamp/:page?/:pagesize?', (req, res, next) => { if (!req.auth(res, ['dev', 'admin'], 'basic')) return; diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 5a6405c..e10d2c2 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -15,9 +15,6 @@ describe('/sample', () => { afterEach(done => TestHelper.afterEach(server, done)); after(done => TestHelper.after(done)); - // TODO: sort, added date filter, has measurements/condition filter - // TODO: check if conditions work in sort/fields/filters - // TODO: test for numbers as strings in glass_fiber describe('GET /samples', () => { it('returns all samples', done => { TestHelper.request(server, done, { @@ -298,7 +295,7 @@ describe('/sample', () => { done(); }); }); - it('filters a sample property', done => { // TODO: implement filters + it('filters a sample property', done => { TestHelper.request(server, done, { method: 'get', url: '/samples?status[]=new&status[]=validated&fields[]=number&fields[]=type&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22type%22%2C%22values%22%3A%5B%22processed%22%5D%7D', @@ -801,7 +798,7 @@ describe('/sample', () => { }); }); - describe('PUT /sample/{id}', () => { // TODO: fix tests, work on /samples + describe('PUT /sample/{id}', () => { it('returns the right sample', done => { TestHelper.request(server, done, { method: 'put', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index ab8ff1c..55665e5 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -22,13 +22,6 @@ import globals from '../globals'; const router = express.Router(); -// TODO: check added filter -// TODO: convert filter value to number according to table model -// TODO: validation for filter parameters -// TODO: location/device sort/filter - -// TODO: think about filter keys with measurement template versions - router.get('/samples', async (req, res, next) => { if (!req.auth(res, ['read', 'write', 'dev', 'admin'], 'all')) return; @@ -40,12 +33,6 @@ router.get('/samples', async (req, res, next) => { if ((filters.fields.find(e => e.indexOf('.' + globals.spectrum.dpt) >= 0) || filters.output !== 'json') && !req.auth(res, ['dev', 'admin'], 'all')) return; - // TODO: find a better place for these - const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', - 'user_id']; - - // TODO find further optimizations from bachelor thesis - // evaluate sort parameter from 'color-asc' to ['color', 1] filters.sort = filters.sort.split('-'); filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id @@ -123,7 +110,7 @@ router.get('/samples', async (req, res, next) => { let sortStartValue = null; if (filters['from-id']) { // from-id specified, fetch values for sorting const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}) - .lean().exec().catch(err => {next(err);}); // TODO: what if more than one measurement for sample? + .lean().exec().catch(err => {next(err);}); if (fromSample instanceof Error) return; if (!fromSample) { return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); @@ -149,7 +136,8 @@ router.get('/samples', async (req, res, next) => { collection = SampleModel; queryPtr[0].$match.$and.push(statusQuery(filters, 'status')); - if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys + // sorting for sample keys + if (SampleValidate.sampleKeys.indexOf(filters.sort[0]) >= 0 || /condition\./.test(filters.sort[0])) { let sortStartValue = null; if (filters['from-id']) { // from-id specified const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => { @@ -168,13 +156,15 @@ router.get('/samples', async (req, res, next) => { } } - addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters + addFilterQueries(queryPtr, filters.filters.filter( + e => (SampleValidate.sampleKeys.indexOf(e.field) >= 0) || /condition\./.test(e.field)) + ); // sample filters let materialQuery = []; // put material query together separate first to reuse for first-id let materialAdded = false; if (sortFilterKeys.find(e => /material\./.test(e))) { // add material fields materialAdded = true; - materialQuery.push( // add material properties // TODO: project out unnecessary fields + materialQuery.push( // add material properties {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$addFields: {material: {$arrayElemAt: ['$material', 0]}}} ); @@ -198,16 +188,8 @@ router.get('/samples', async (req, res, next) => { {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } - // TODO: adapt code to new numbers format - // if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed - // materialQuery.push( - // {$addFields: {'material.number': { $arrayElemAt: [ - // '$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']} - // ]}}} - // ); - // } const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)) - .filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); // TODO + .filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); // base material filters addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); queryPtr.push(...materialQuery); @@ -250,8 +232,17 @@ router.get('/samples', async (req, res, next) => { ]}}}], as: 'measurements' }}); - measurementTemplates.forEach(template => { - addMeasurements(queryPtr, template); + const groupedMeasurementTemplates = measurementTemplates.reduce((s, e) => { + if (s.hasOwnProperty(e.name)) { + s[e.name].push(e); + } + else { + s[e.name] = [e]; + } + return s; + }, {}); + Object.values(groupedMeasurementTemplates).forEach(templates => { + addMeasurements(queryPtr, templates); }); addFilterQueries(queryPtr, filters.filters .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) @@ -310,13 +301,6 @@ router.get('/samples', async (req, res, next) => { {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } - // if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed // TODO - // queryPtr.push( - // {$addFields: {'material.number': { - // $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}] - // }}} - // ); - // } let measurementFieldsFields: string[] = _.uniq( fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]) @@ -328,8 +312,8 @@ router.get('/samples', async (req, res, next) => { if (measurementTemplates.length < measurementFieldsFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } - // use different lookup methods with and without spectrum for the best performance - if (fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.') >= 0)) { + // use different lookup methods with and without dpt for the best performance + if (fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.' + globals.spectrum.dpt) >= 0)) { // with dpt queryPtr.push( {$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}} ); @@ -344,21 +328,23 @@ router.get('/samples', async (req, res, next) => { as: 'measurements' }}); } - measurementTemplates.forEach(template => { - addMeasurements(queryPtr, template); - if (measurementFieldsFields.find(e => e === globals.spectrum.spectrum)) { - queryPtr.push({$unwind: '$' + globals.spectrum.spectrum}); + const groupedMeasurementTemplates = measurementTemplates.reduce((s, e) => { + if (s.hasOwnProperty(e.name)) { + s[e.name].push(e); } + else { + s[e.name] = [e]; + } + return s; + }, {}); + Object.values(groupedMeasurementTemplates).forEach(templates => { + addMeasurements(queryPtr, templates); }); queryPtr.push({$project: {measurements: 0}}); } const projection = filters.fields.map(e => e.replace('measurements.', '')) .reduce((s, e) => {s[e] = true; return s; }, {}); - if (filters.fields.indexOf('added') >= 0) { // add added date // TODO: upgrade MongoDB version or find alternative - // projection.added = {$toDate: '$_id'}; - // projection.added = { $convert: { input: '$_id', to: "date" } } - } if (filters.fields.indexOf('_id') < 0 && filters.fields.indexOf('added') < 0) { // disable _id explicitly projection._id = false; } @@ -781,12 +767,13 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] if (filters['from-id']) { // from-id specified + const ssv = sortStartValue !== undefined; // if value is not given, match for existence if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc return [ {$match: {$or: [ - {[sortKeys[0]]: {$gt: sortStartValue}}, + {[sortKeys[0]]: ssv ? {$gt: sortStartValue} : {$exists: true}}, {$and: [ - {[sortKeys[0]]: sortStartValue}, + {[sortKeys[0]]: ssv ? sortStartValue : {$exists: false}}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}} ]} ]}}, @@ -795,9 +782,9 @@ function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary } else { return [ {$match: {$or: [ - {[sortKeys[0]]: {$lt: sortStartValue}}, + {[sortKeys[0]]: ssv ? {$lt: sortStartValue} : {$exists: false}}, {$and: [ - {[sortKeys[0]]: sortStartValue}, + {[sortKeys[0]]: ssv ? sortStartValue : {$exists: true}}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}} ]} ]}}, @@ -834,20 +821,25 @@ function filterQueries (filters) { }); } -// add measurements as property [template.name], if one result, array is reduced to direct values -function addMeasurements(queryPtr, template) { +// add measurements as property [template.name], if one result, array is reduced to direct values. All given templates +// must have the same name +function addMeasurements(queryPtr, templates) { queryPtr.push( - {$addFields: {[template.name]: {$let: {vars: { + {$addFields: {[templates[0].name]: {$let: {vars: { arr: {$filter: { - input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]} + input: '$measurements', cond: {$in: [ + '$$this.measurement_template', + templates.map(e => mongoose.Types.ObjectId(e._id)) + ]} }}}, in: {$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, - {$addFields: {[template.name]: {$cond: [ - '$' + template.name + '.values', - '$' + template.name + '.values', - template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {}) - ]}}} + {$addFields: {[templates[0].name]: {$cond: [ + '$' + templates[0].name + '.values', + '$' + templates[0].name + '.values', + templates[0].parameters.reduce((s, e) => {s[e.name] = null; return s;}, {}) + ]}}}, + {$unwind: '$' + templates[0].name} ); } diff --git a/src/routes/template.spec.ts b/src/routes/template.spec.ts index f936c46..db924b3 100644 --- a/src/routes/template.spec.ts +++ b/src/routes/template.spec.ts @@ -4,7 +4,6 @@ import TemplateConditionModel from '../models/condition_template'; import TemplateMeasurementModel from '../models/measurement_template'; import TestHelper from "../test/helper"; -// TODO: method to return only latest template versions -> rework frontend accordingly describe('/template', () => { let server; diff --git a/src/routes/user.ts b/src/routes/user.ts index 963af27..976c1a7 100644 --- a/src/routes/user.ts +++ b/src/routes/user.ts @@ -81,7 +81,6 @@ router.put('/user:username([/](?!key|new).?*|/?)', async (req, res, next) => { }); }); -// TODO: only possible if no data is linked to user, otherwise change status, etc. // this path matches /user, /user/ and /user/xxx, but not /user/key or user/new. // See https://forbeslindesay.github.io/express-route-tester/ for the generated regex router.delete('/user:username([/](?!key|new).?*|/?)', (req, res, next) => { diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index db6ef19..665a01b 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -6,6 +6,7 @@ import MaterialValidate from './material'; import MeasurementValidate from './measurement'; import globals from '../../globals'; + export default class SampleValidate { private static sample = { number: Joi.string() @@ -56,6 +57,19 @@ export default class SampleValidate { .valid(...Object.values(globals.status)) }; + static readonly sampleKeys = [ // keys which can be found in the sample directly + '_id', + 'color', + 'number', + 'type', + 'batch', + 'added', + 'condition', + 'material_id', + 'note_id', + 'user_id' + ]; + private static sortKeys = [ '_id', 'color', @@ -68,6 +82,7 @@ export default class SampleValidate { 'material.supplier', 'material.group', 'material.properties.*', + 'condition.*', `measurements.(?!${globals.spectrum.spectrum}.${globals.spectrum.dpt})*` ]; @@ -189,7 +204,7 @@ export default class SampleValidate { }); field = field.replace('material.', '').split('.')[0]; } - else if (/measurements\./.test(field)) { + else if (/measurements\./.test(field) || /condition\./.test(field)) { validator = Joi.object({ value: Joi.alternatives() .try( diff --git a/src/routes/validate/template.ts b/src/routes/validate/template.ts index 8378d92..bcc515f 100644 --- a/src/routes/validate/template.ts +++ b/src/routes/validate/template.ts @@ -1,7 +1,6 @@ import Joi from 'joi'; import IdValidate from './id'; -// TODO: do not allow a . in the name !!! export default class TemplateValidate { private static template = { name: Joi.string() From c6f5186f7ea8bc933a613420fff59b4a83ff2f19 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Sun, 16 Aug 2020 20:01:18 +0200 Subject: [PATCH 3/3] debug --- data_import/import.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/data_import/import.js b/data_import/import.js index e79364f..6e8f870 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -9,8 +9,8 @@ const _ = require('lodash'); const stages = { materials: true, - samples: true, - dpt: true + samples: false, + dpt: false } const docs = [ @@ -30,8 +30,8 @@ const docs = [ const errors = []; const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\nmDocs'; // NormMaster Documents const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\DPT'; // Spectrum files -// const host = 'http://localhost:3000'; -const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; +const host = 'http://localhost:3000'; +// const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; const requiredProperties = ['samplenumber','materialnumber','materialname','supplier','reinforcementmaterial','material','granulate/part','color','charge/batch','comments']; dict = { // dictionary 'Granulat': 'granulate', @@ -62,7 +62,7 @@ async function main() { for (let i in docs) { await importCsv(docs[i]); await allMaterials(); - await saveMaterials(); + // await saveMaterials(); } fs.writeFileSync('./data_import/numberToColor.json', JSON.stringify(numberToColor)); } @@ -651,6 +651,9 @@ async function allMaterials() { // process all samples for (let index in data) { let sample = data[index]; + if (sample['materialname'].replace(/\s+/g, '') === 'Latamid66H2G30') { + console.log(sample); + } if (sample['supplier'] === '') { // empty supplier fields sample['supplier'] = 'unknown'; } @@ -676,7 +679,7 @@ async function allMaterials() { } } else { // new material - console.info(`MATERIAL LOAD ${index}/${data.length} ${sample['materialname']}`); + // console.info(`MATERIAL LOAD ${index}/${data.length} ${sample['materialname']}`); materials[sample['materialname']] = { name: trim(sample['materialname']), supplier: trim(sample['supplier']),