From 3b08fb63c7578748ac4331ee10a99478b849e9e3 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 23 Jul 2020 14:55:41 +0200 Subject: [PATCH] /authorized returns level, added device to spectrum measurement --- .nvmrc | 1 + api/root.yaml | 3 +++ build.bat | 2 ++ data_import/import.js | 11 ++++---- package.json | 2 +- src/routes/measurement.spec.ts | 14 +++++----- src/routes/root.spec.ts | 6 ++--- src/routes/root.ts | 2 +- src/routes/sample.spec.ts | 2 +- src/routes/sample.ts | 49 +++++++++++++++++++++------------- src/routes/template.spec.ts | 16 ++++++----- src/routes/validate/sample.ts | 3 +-- src/test/db.json | 10 +++++-- 13 files changed, 74 insertions(+), 47 deletions(-) create mode 100644 .nvmrc diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..339dacb --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +12.17 \ No newline at end of file diff --git a/api/root.yaml b/api/root.yaml index 3070412..af618a7 100644 --- a/api/root.yaml +++ b/api/root.yaml @@ -37,6 +37,9 @@ method: type: string example: 'basic' + level: + type: string + example: read 401: $ref: 'api.yaml#/components/responses/401' 500: diff --git a/build.bat b/build.bat index d632b14..3112adb 100644 --- a/build.bat +++ b/build.bat @@ -1,4 +1,6 @@ call npm run tsc-full copy package.json dist\package.json +copy package-lock.json dist\package-lock.json +copy .nvmrc dist\.nvmrc Xcopy /E /I api dist\api Xcopy /E /I static dist\static \ No newline at end of file diff --git a/data_import/import.js b/data_import/import.js index 8fac949..24a0292 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -30,8 +30,8 @@ const docs = [ const errors = []; const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\nmDocs'; // NormMaster Documents const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\DPT'; // Spectrum files -const host = 'http://localhost:3000'; -// const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; +// const host = 'http://localhost:3000'; +const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; const requiredProperties = ['samplenumber','materialnumber','materialname','supplier','reinforcementmaterial','material','granulate/part','color','charge/batch','comments']; dict = { // dictionary 'Granulat': 'granulate', @@ -194,7 +194,8 @@ async function allDpts() { password: 'Abc123!#' } }); - const measurement_template = res.data.find(e => e.name === 'spectrum')._id; + const measurement_templates = res.data.filter(e => e.name === 'spectrum'); + const measurement_template = measurement_templates[measurement_templates.length - 1]._id; res = await axios({ method: 'get', url: host + '/samples?status=all', @@ -207,7 +208,7 @@ async function allDpts() { res.data.forEach(sample => { sampleIds[sample.number] = sample._id; }); - const dptRegex = /(.*?)_(.*?)_(\d+|\d+_\d+).DPT/; + const dptRegex = /(.*?)_(.*?)_(\d+|\w+_\d+).DPT/; const dpts = fs.readdirSync(dptFiles); for (let i in dpts) { const regexRes = dptRegex.exec(dpts[i]) @@ -314,7 +315,7 @@ async function allKfVz() { errors.push(`KF/VZ upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`); }); } - if (sample['VZ (ml/g)']) { + if (sample['vz(ml/g)']) { await axios({ method: 'post', url: host + '/measurement/new', diff --git a/package.json b/package.json index f9494d3..ae69e4f 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "tsc": "tsc", "tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc", "build": "build.bat", - "build-push": "build.bat && cf push", + "build-push": "build.bat && timeout 3 && cf push", "test": "mocha dist/**/**.spec.js", "start": "node index.js", "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"", diff --git a/src/routes/measurement.spec.ts b/src/routes/measurement.spec.ts index d33bfdc..180f3ce 100644 --- a/src/routes/measurement.spec.ts +++ b/src/routes/measurement.spec.ts @@ -18,7 +18,7 @@ describe('/measurement', () => { url: '/measurement/800000000000000000000001', auth: {basic: 'janedoe'}, httpStatus: 200, - res: {_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]}, measurement_template: '300000000000000000000001'} + res: {_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]], device: 'Alpha I'}, measurement_template: '300000000000000000000001'} }); }); it('returns the measurement for an API key', done => { @@ -27,7 +27,7 @@ describe('/measurement', () => { url: '/measurement/800000000000000000000001', auth: {key: 'janedoe'}, httpStatus: 200, - res: {_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]}, measurement_template: '300000000000000000000001'} + res: {_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]], device: 'Alpha I'}, measurement_template: '300000000000000000000001'} }); }); it('returns deleted measurements for a maintain/admin user', done => { @@ -80,7 +80,7 @@ describe('/measurement', () => { auth: {basic: 'janedoe'}, httpStatus: 200, req: {}, - res: {_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]}, measurement_template: '300000000000000000000001'} + res: {_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]], device: 'Alpha I'}, measurement_template: '300000000000000000000001'} }); }); it('keeps unchanged values', done => { @@ -89,10 +89,10 @@ describe('/measurement', () => { url: '/measurement/800000000000000000000001', auth: {basic: 'janedoe'}, httpStatus: 200, - req: {values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]}} + req: {values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]], device: 'Alpha I'}} }).end((err, res) => { if (err) return done(err); - should(res.body).be.eql({_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]}, measurement_template: '300000000000000000000001'}); + should(res.body).be.eql({_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]], device: 'Alpha I'}, measurement_template: '300000000000000000000001'}); MeasurementModel.findById('800000000000000000000001').lean().exec((err, data: any) => { if (err) return done(err); should(data).have.property('status',globals.status.validated); @@ -126,7 +126,7 @@ describe('/measurement', () => { req: {values: {dpt: [[1,2],[3,4],[5,6]]}} }).end((err, res) => { if (err) return done(err); - should(res.body).be.eql({_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[1,2],[3,4],[5,6]]}, measurement_template: '300000000000000000000001'}); + should(res.body).be.eql({_id: '800000000000000000000001', sample_id: '400000000000000000000001', values: {dpt: [[1,2],[3,4],[5,6]], device: 'Alpha I'}, measurement_template: '300000000000000000000001'}); MeasurementModel.findById('800000000000000000000001').lean().exec((err, data: any) => { should(data).have.only.keys('_id', 'sample_id', 'values', 'measurement_template', 'status', '__v'); should(data.sample_id.toString()).be.eql('400000000000000000000001'); @@ -144,7 +144,7 @@ describe('/measurement', () => { url: '/measurement/800000000000000000000001', auth: {basic: 'janedoe'}, httpStatus: 200, - req: {values: {dpt: [[1,2],[3,4],[5,6]]}}, + req: {values: {dpt: [[1,2],[3,4],[5,6]], device: 'Alpha I'}}, log: { collection: 'measurements', dataAdd: { diff --git a/src/routes/root.spec.ts b/src/routes/root.spec.ts index 68531a5..b84d0c2 100644 --- a/src/routes/root.spec.ts +++ b/src/routes/root.spec.ts @@ -179,7 +179,7 @@ describe('/', () => { url: '/authorized', auth: {key: 'admin'}, httpStatus: 200, - res: {status: 'Authorization successful', method: 'key'} + res: {status: 'Authorization successful', method: 'key', level: 'admin'} }); }); it('works with basic auth', done => { @@ -188,7 +188,7 @@ describe('/', () => { url: '/authorized', auth: {basic: 'admin'}, httpStatus: 200, - res: {status: 'Authorization successful', method: 'basic'} + res: {status: 'Authorization successful', method: 'basic', level: 'admin'} }); }); }); @@ -242,7 +242,7 @@ describe('The /api/{url} redirect', () => { url: '/api/authorized', auth: {basic: 'admin'}, httpStatus: 200, - res: {status: 'Authorization successful', method: 'basic'} + res: {status: 'Authorization successful', method: 'basic', level: 'admin'} }); }); it('is disabled in production', done => { diff --git a/src/routes/root.ts b/src/routes/root.ts index 1547844..20f10b9 100644 --- a/src/routes/root.ts +++ b/src/routes/root.ts @@ -14,7 +14,7 @@ router.get('/', (req, res) => { router.get('/authorized', (req, res) => { if (!req.auth(res, globals.levels)) return; - res.json({status: 'Authorization successful', method: req.authDetails.method}); + res.json({status: 'Authorization successful', method: req.authDetails.method, level: req.authDetails.level}); }); // TODO: evaluate exact changelog functionality (restoring, delting after time, etc.) diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index ca62d16..d9db97a 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -266,7 +266,7 @@ describe('/sample', () => { httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.body).have.lengthOf(2); + should(res.body.filter(e => e.spectrum.dpt)).have.lengthOf(2); should(res.body[0].spectrum).have.property('dpt', [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]); should(res.body[1].spectrum).have.property('dpt', [[3996.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]); done(); diff --git a/src/routes/sample.ts b/src/routes/sample.ts index e468a41..10694ac 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -22,12 +22,11 @@ import csv from '../helpers/csv'; const router = express.Router(); // TODO: check added filter -// TODO: use query pointer // TODO: convert filter value to number according to table model // TODO: validation for filter parameters // TODO: location/device sort/filter -// TODO: think about material numbers +// TODO: think about filter keys with measurement template versions router.get('/samples', async (req, res, next) => { if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; @@ -104,9 +103,9 @@ router.get('/samples', async (req, res, next) => { if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection collection = MeasurementModel; const [,measurementName, measurementParam] = filters.sort[0].split('.'); - const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);}); - if (measurementTemplate instanceof Error) return; - if (!measurementTemplate) { + const measurementTemplates = await MeasurementTemplateModel.find({name: measurementName}).lean().exec().catch(err => {next(err);}); + if (measurementTemplates instanceof Error) return; + if (!measurementTemplates) { return res.status(400).json({status: 'Invalid body format', details: filters.sort[0] + ' not found'}); } let sortStartValue = null; @@ -118,7 +117,7 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample.values[measurementParam]; } - queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort + queryPtr[0].$match.$and.push({measurement_template: {$in: measurementTemplates.map(e => e._id)}}); // find measurements to sort if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered queryPtr[0].$match.$and.push(...filterQueries(filters.filters.filter(e => e.field === filters.sort[0]).map(e => {e.field = 'values.' + e.field.split('.')[2]; return e; }))); } @@ -194,7 +193,16 @@ router.get('/samples', async (req, res, next) => { if (!fromSample) { return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); } - sortStartValue = fromSample[filters.sort[0]]; + console.log(fromSample); + console.log(filters.sort[0]); + console.log(fromSample[filters.sort[0]]); + const filterKey = filters.sort[0].split('.'); + if (filterKey.length === 2) { + sortStartValue = fromSample[0][filterKey[0]][filterKey[1]]; + } + else { + sortStartValue = fromSample[0][filterKey[0]]; + } } queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue)); } @@ -290,19 +298,22 @@ router.get('/samples', async (req, res, next) => { as: 'measurements' }}); } - measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later + measurementTemplates.forEach(template => { // TODO: hard coded dpt for special treatment, change later queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); + if (measurementFieldsFields.find(e => e === 'spectrum')) { + queryPtr.push({$unwind: '$spectrum'}); + } }); - if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well - queryPtr.push( - {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, - {$addFields: {spectrum: '$spectrum.values'}}, - {$unwind: '$spectrum'} - ); - } + // if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well + // queryPtr.push( + // {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, + // {$addFields: {spectrum: '$spectrum.values'}}, + // {$unwind: '$spectrum'} + // ); + // } // queryPtr.push({$unset: 'measurements'}); queryPtr.push({$project: {measurements: 0}}); } @@ -316,9 +327,8 @@ router.get('/samples', async (req, res, next) => { projection._id = false; } queryPtr.push({$project: projection}); - if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files - collection.aggregate(query).exec((err, data) => { + collection.aggregate(query).allowDiskUse(true).exec((err, data) => { if (err) return next(err); if (data[0] && data[0].count) { res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0); @@ -354,7 +364,7 @@ router.get('/samples', async (req, res, next) => { res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'}); res.write('['); let count = 0; - const stream = collection.aggregate(query).cursor().exec(); + const stream = collection.aggregate(query).allowDiskUse(true).cursor().exec(); stream.on('data', data => { if (filters.fields.indexOf('added') >= 0) { // add added date data.added = data._id.getTimestamp(); @@ -364,6 +374,9 @@ router.get('/samples', async (req, res, next) => { } res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; }); + stream.on('error', err => { + console.error(err); + }); stream.on('close', () => { res.write(']'); res.end(); diff --git a/src/routes/template.spec.ts b/src/routes/template.spec.ts index b07014b..bf932cf 100644 --- a/src/routes/template.spec.ts +++ b/src/routes/template.spec.ts @@ -517,7 +517,7 @@ describe('/template', () => { url: '/template/measurement/300000000000000000000001', auth: {basic: 'janedoe'}, httpStatus: 200, - res: {_id: '300000000000000000000001', name: 'spectrum', version: 1, parameters: [{name: 'dpt', range: { type: 'array'}}]} + res: {_id: '300000000000000000000001', name: 'spectrum', version: 1, parameters: [{name: 'dpt', range: { type: 'array'}}, {name: 'device', range: {}}]} }); }); it('rejects an API key', done => { @@ -553,7 +553,7 @@ describe('/template', () => { auth: {basic: 'admin'}, httpStatus: 200, req: {}, - res: {_id: '300000000000000000000001', name: 'spectrum', version: 1, parameters: [{name: 'dpt', range: { type: 'array'}}]} + res: {_id: '300000000000000000000001', name: 'spectrum', version: 1, parameters: [{name: 'dpt', range: { type: 'array'}}, {name: 'device', range: {}}]} }); }); it('keeps unchanged properties', done => { @@ -562,8 +562,8 @@ describe('/template', () => { url: '/template/measurement/300000000000000000000001', auth: {basic: 'admin'}, httpStatus: 200, - req: {name: 'spectrum', parameters: [{name: 'dpt', range: { type: 'array'}}]}, - res: {_id: '300000000000000000000001', name: 'spectrum', version: 1, parameters: [{name: 'dpt', range: {type: 'array'}}]} + req: {name: 'spectrum', parameters: [{name: 'dpt', range: { type: 'array'}}, {name: 'device', range: {}}]}, + res: {_id: '300000000000000000000001', name: 'spectrum', version: 1, parameters: [{name: 'dpt', range: {type: 'array'}}, {name: 'device', range: {}}]} }); }); it('keeps only one unchanged property', done => { @@ -573,7 +573,7 @@ describe('/template', () => { auth: {basic: 'admin'}, httpStatus: 200, req: {name: 'spectrum'}, - res: {_id: '300000000000000000000001', name: 'spectrum', version: 1, parameters: [{name: 'dpt', range: {type: 'array'}}]} + res: {_id: '300000000000000000000001', name: 'spectrum', version: 1, parameters: [{name: 'dpt', range: {type: 'array'}}, {name: 'device', range: {}}]} }); }); it('changes the given properties', done => { @@ -626,17 +626,19 @@ describe('/template', () => { req: {name: 'IR spectrum'}, }).end((err, res) => { if (err) return done(err); - should(_.omit(res.body, '_id')).be.eql({name: 'IR spectrum', version: 2, parameters: [{name: 'dpt', range: {type: 'array'}}]}); + should(_.omit(res.body, '_id')).be.eql({name: 'IR spectrum', version: 2, parameters: [{name: 'dpt', range: {type: 'array'}}, {name: 'device', range: {}}]}); TemplateMeasurementModel.findById(res.body._id).lean().exec((err, data:any) => { if (err) return done(err); should(data).have.only.keys('_id', 'first_id', 'name', 'version', 'parameters', '__v'); should(data.first_id.toString()).be.eql('300000000000000000000001'); should(data).have.property('name', 'IR spectrum'); should(data).have.property('version', 2); - should(data).have.property('parameters').have.lengthOf(1); + should(data).have.property('parameters').have.lengthOf(2); should(data.parameters[0]).have.property('name', 'dpt'); should(data.parameters[0]).have.property('range'); should(data.parameters[0].range).have.property('type', 'array'); + should(data.parameters[1]).have.property('name', 'device'); + should(data.parameters[1]).have.property('range'); done(); }); }); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 3e9aed3..19f6b50 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -62,9 +62,8 @@ export default class SampleValidate { 'material.name', 'material.supplier', 'material.group', - 'material.number', 'material.properties.*', - 'measurements.(?!spectrum)*' + 'measurements.(?!spectrum\.dpt)*' ]; private static fieldKeys = [ diff --git a/src/test/db.json b/src/test/db.json index 7b0fab9..7930a94 100644 --- a/src/test/db.json +++ b/src/test/db.json @@ -403,7 +403,8 @@ [3997.12558,98.00555], [3995.08519,98.03253], [3993.04480,98.02657] - ] + ], + "device": "Alpha I" }, "status": 10, "measurement_template": {"$oid":"300000000000000000000001"}, @@ -470,7 +471,8 @@ [3996.12558,98.00555], [3995.08519,98.03253], [3993.04480,98.02657] - ] + ], + "device": "Alpha II" }, "status": 10, "measurement_template": {"$oid":"300000000000000000000001"}, @@ -551,6 +553,10 @@ "range": { "type": "array" } + }, + { + "name": "device", + "range": {} } ], "__v": 0