From c95af7bc0b58684034a02a98de39ba60880b328d Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 15 Jun 2020 12:49:32 +0200 Subject: [PATCH 01/19] added status filter --- api/sample.yaml | 9 +++++++++ src/helpers/authorize.ts | 1 + src/index.ts | 2 +- src/routes/sample.spec.ts | 34 ++++++++++++++++++++++++++++++++++ src/routes/sample.ts | 19 ++++++++++++++++++- src/routes/validate/sample.ts | 6 ++++++ 6 files changed, 69 insertions(+), 2 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index eae0ddc..1810a65 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -5,6 +5,13 @@ x-doc: returns only samples with status 10 tags: - /sample + parameters: + - name: status + description: 'values: validated|new|all, defaults to validated' + in: query + schema: + type: string + example: all responses: 200: description: samples overview @@ -14,6 +21,8 @@ type: array items: $ref: 'api.yaml#/components/schemas/SampleRefs' + 400: + $ref: 'api.yaml#/components/responses/400' 401: $ref: 'api.yaml#/components/responses/401' 500: diff --git a/src/helpers/authorize.ts b/src/helpers/authorize.ts index 21d43d5..71a42c2 100644 --- a/src/helpers/authorize.ts +++ b/src/helpers/authorize.ts @@ -89,6 +89,7 @@ function key (req, next): any { // checks API key and returns changed user obje if (err) return next(err); if (data.length === 1) { // one user found resolve({level: data[0].level, name: data[0].name, id: data[0]._id.toString(), location: data[0].location}); + delete req.query.key; // delete query parameter to avoid interference with later validation } else { resolve(null); diff --git a/src/index.ts b/src/index.ts index d274b89..9af77cf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -48,7 +48,7 @@ app.use(require('./helpers/authorize')); // handle authentication // redirect /api routes for Angular proxy in development if (process.env.NODE_ENV !== 'production') { - app.use('/api/:url', (req, res) => { + app.use('/api/:url([^]+)', (req, res) => { req.url = '/' + req.params.url; app.handle(req, res); }); diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 97b9eb3..ee4b07e 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -71,6 +71,40 @@ describe('/sample', () => { done(); }); }); + it('allows filtering by state', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=new', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.new).length); + should(res.body).matchEach(sample => { + should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(sample).have.property('_id').be.type('string'); + should(sample).have.property('number').be.type('string'); + should(sample).have.property('type').be.type('string'); + should(sample).have.property('color').be.type('string'); + should(sample).have.property('batch').be.type('string'); + should(sample).have.property('condition').be.type('object'); + should(sample).have.property('material_id').be.type('string'); + should(sample).have.property('note_id'); + should(sample).have.property('user_id').be.type('string'); + }); + done(); + }); + }); + it('rejects an invalid state name', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=xxx', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"status" must be one of [validated, new, all]'} + }); + }); it('rejects unauthorized requests', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 3966c9b..487711b 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -22,7 +22,24 @@ const router = express.Router(); router.get('/samples', (req, res, next) => { if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; - SampleModel.find({status: globals.status.validated}).lean().exec((err, data) => { + const {error, value: filters} = SampleValidate.query(req.query); + if (error) return res400(error, res); + + let conditions; + + if (filters.hasOwnProperty('status')) { + if(filters.status === 'all') { + conditions = {$or: [{status: globals.status.validated}, {status: globals.status.new}]} + } + else { + conditions = {status: globals.status[filters.status]}; + } + } + else { // default + conditions = {status: globals.status.validated}; + } + + SampleModel.find(conditions).lean().exec((err, data) => { if (err) return next(err); res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors }) diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 58c33ba..616e060 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -118,4 +118,10 @@ export default class SampleValidate { const {value, error} = Joi.object(joiObject).validate(data, {stripUnknown: true}); return error !== undefined? null : value; } + + static query (data) { + return Joi.object({ + status: Joi.string().valid('validated', 'new', 'all') + }).validate(data); + } } \ No newline at end of file From 869a675840fe54f0c1e169111c645b4e9db453ea Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Wed, 17 Jun 2020 13:42:14 +0200 Subject: [PATCH 02/19] added status filter for materials --- api/material.yaml | 7 +++++++ src/routes/material.spec.ts | 37 +++++++++++++++++++++++++++++++++ src/routes/material.ts | 19 ++++++++++++++++- src/routes/validate/material.ts | 6 ++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/api/material.yaml b/api/material.yaml index 378628d..593afb1 100644 --- a/api/material.yaml +++ b/api/material.yaml @@ -5,6 +5,13 @@ x-doc: returns only materials with status 10 tags: - /material + parameters: + - name: status + description: 'values: validated|new|all, defaults to validated' + in: query + schema: + type: string + example: all responses: 200: description: all material details diff --git a/src/routes/material.spec.ts b/src/routes/material.spec.ts index e91e87e..e412615 100644 --- a/src/routes/material.spec.ts +++ b/src/routes/material.spec.ts @@ -72,6 +72,43 @@ describe('/material', () => { done(); }); }); + it('allows filtering by state', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/materials?status=new', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.materials.filter(e => e.status === globals.status.new).length); + should(res.body).matchEach(material => { + should(material).have.only.keys('_id', 'name', 'supplier', 'group', 'mineral', 'glass_fiber', 'carbon_fiber', 'numbers'); + should(material).have.property('_id').be.type('string'); + should(material).have.property('name').be.type('string'); + should(material).have.property('supplier').be.type('string'); + should(material).have.property('group').be.type('string'); + should(material).have.property('mineral').be.type('number'); + should(material).have.property('glass_fiber').be.type('number'); + should(material).have.property('carbon_fiber').be.type('number'); + should(material.numbers).matchEach(number => { + should(number).have.only.keys('color', 'number'); + should(number).have.property('color').be.type('string'); + should(number).have.property('number').be.type('string'); + }); + }); + done(); + }); + }); + it('rejects an invalid state name', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/materials?status=xxx', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"status" must be one of [validated, new, all]'} + }); + }); it('rejects unauthorized requests', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/material.ts b/src/routes/material.ts index 8373c9d..3f34e3a 100644 --- a/src/routes/material.ts +++ b/src/routes/material.ts @@ -19,7 +19,24 @@ const router = express.Router(); router.get('/materials', (req, res, next) => { if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; - MaterialModel.find({status:globals.status.validated}).populate('group_id').populate('supplier_id').lean().exec((err, data) => { + const {error, value: filters} = MaterialValidate.query(req.query); + if (error) return res400(error, res); + + let conditions; + + if (filters.hasOwnProperty('status')) { + if(filters.status === 'all') { + conditions = {$or: [{status: globals.status.validated}, {status: globals.status.new}]} + } + else { + conditions = {status: globals.status[filters.status]}; + } + } + else { // default + conditions = {status: globals.status.validated}; + } + + MaterialModel.find(conditions).populate('group_id').populate('supplier_id').lean().exec((err, data) => { if (err) return next(err); res.json(_.compact(data.map(e => MaterialValidate.output(e)))); // validate all and filter null values from validation errors diff --git a/src/routes/validate/material.ts b/src/routes/validate/material.ts index 7a2c3fb..969ac43 100644 --- a/src/routes/validate/material.ts +++ b/src/routes/validate/material.ts @@ -107,4 +107,10 @@ export default class MaterialValidate { // validate input for material numbers: this.material.numbers }); } + + static query (data) { + return Joi.object({ + status: Joi.string().valid('validated', 'new', 'all') + }).validate(data); + } } \ No newline at end of file From ac72d8a9752dfa7c00bf08d1153925742408a948 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 18 Jun 2020 08:57:50 +0200 Subject: [PATCH 03/19] fixed validation to return measurements in /sample/{id} --- api/sample.yaml | 18 ++++++++++-------- api/schemas.yaml | 2 ++ src/routes/sample.spec.ts | 6 +++--- src/routes/sample.ts | 2 +- src/routes/validate/measurement.ts | 9 +++++++++ src/routes/validate/sample.ts | 2 ++ 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index 1810a65..9331806 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -55,8 +55,8 @@ parameters: - $ref: 'api.yaml#/components/parameters/Id' get: - summary: TODO sample details - description: 'Auth: all, levels: read, write, maintain, dev, admin' + summary: sample details + description: 'Auth: all, levels: read, write, maintain, dev, admin
Returns validated as well as new measurements' x-doc: deleted samples are available only for maintain/admin tags: - /sample @@ -225,12 +225,14 @@ content: application/json: schema: - properties: - name: - type: string - qty: - type: number - example: 20 + type: array + items: + properties: + name: + type: string + qty: + type: number + example: 20 401: $ref: 'api.yaml#/components/responses/401' 500: diff --git a/api/schemas.yaml b/api/schemas.yaml index 21ceddf..9704e08 100644 --- a/api/schemas.yaml +++ b/api/schemas.yaml @@ -69,6 +69,8 @@ Sample: relation: type: string example: part to this sample + custom_fields: + type: object SampleDetail: allOf: diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index ee4b07e..9a483a4 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -215,7 +215,7 @@ describe('/sample', () => { url: '/sample/400000000000000000000003', auth: {basic: 'janedoe'}, httpStatus: 200, - res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, user: 'admin'} + res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'} }); }); it('works with an API key', done => { @@ -224,7 +224,7 @@ describe('/sample', () => { url: '/sample/400000000000000000000003', auth: {key: 'janedoe'}, httpStatus: 200, - res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, user: 'admin'} + res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'} }); }); it('returns a deleted sample for a maintain/admin user', done => { @@ -233,7 +233,7 @@ describe('/sample', () => { url: '/sample/400000000000000000000005', auth: {basic: 'admin'}, httpStatus: 200, - res: {_id: '400000000000000000000005', number: 'Rng33', type: 'granulate', color: 'black', batch: '1653000308', condition: {condition_template: '200000000000000000000003'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {}, user: 'admin'} + res: {_id: '400000000000000000000005', number: 'Rng33', type: 'granulate', color: 'black', batch: '1653000308', condition: {condition_template: '200000000000000000000003'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {}, measurements: [], user: 'admin'} }); }); it('returns 403 for a write user when requesting a deleted sample', done => { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 487711b..d28e725 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -71,7 +71,7 @@ router.get('/sample/' + IdValidate.parameter(), (req, res, next) => { sampleData.material.supplier = sampleData.material.supplier_id.name; sampleData.user = sampleData.user_id.name; sampleData.notes = sampleData.note_id ? sampleData.note_id : {}; - MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id)}).lean().exec((err, data) => { + MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id), status: {$ne: globals.status.deleted}}).lean().exec((err, data) => { sampleData.measurements = data; res.json(SampleValidate.output(sampleData, 'details')); }); diff --git a/src/routes/validate/measurement.ts b/src/routes/validate/measurement.ts index 74c2409..0af8fbd 100644 --- a/src/routes/validate/measurement.ts +++ b/src/routes/validate/measurement.ts @@ -44,4 +44,13 @@ export default class MeasurementValidate { }).validate(data, {stripUnknown: true}); return error !== undefined? null : value; } + + static outputV() { // return output validator + return Joi.object({ + _id: IdValidate.get(), + sample_id: IdValidate.get(), + values: this.measurement.values, + measurement_template: IdValidate.get() + }); + } } \ No newline at end of file diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 616e060..1da56da 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -3,6 +3,7 @@ import Joi from '@hapi/joi'; import IdValidate from './id'; import UserValidate from './user'; import MaterialValidate from './material'; +import MeasurementValidate from './measurement'; export default class SampleValidate { private static sample = { @@ -108,6 +109,7 @@ export default class SampleValidate { batch: this.sample.batch, condition: this.sample.condition, material: MaterialValidate.outputV(), + measurements: Joi.array().items(MeasurementValidate.outputV()), notes: this.sample.notes, user: UserValidate.username() } From cd2962e186fc68d42aa75815ab73d451bd1fab44 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 25 Jun 2020 10:44:55 +0200 Subject: [PATCH 04/19] implemented paging --- api/sample.yaml | 20 ++ api/schemas.yaml | 3 + data_import/import.js | 485 ++++++++++++++++++++++++++++++++ package-lock.json | 505 ++++++++++++++++++++++++++++------ package.json | 7 +- src/routes/sample.spec.ts | 154 +++++++++-- src/routes/sample.ts | 33 ++- src/routes/validate/sample.ts | 17 +- 8 files changed, 1111 insertions(+), 113 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index 9331806..4ebb61f 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -12,6 +12,24 @@ schema: type: string example: all + - name: last-id + description: last id of current page, if not given the results are displayed from start + in: query + schema: + type: string + example: 5ea0450ed851c30a90e70894 + - name: to-page + description: relative change of pages, use negative values to get back, defaults to 0 (if last-id is given, the sample after is the first of the result, so the next page is selected automatically), works only together with page-size + in: query + schema: + type: string + example: 1 + - name: page-size + description: number of items per page + in: query + schema: + type: string + example: 30 responses: 200: description: samples overview @@ -25,6 +43,8 @@ $ref: 'api.yaml#/components/responses/400' 401: $ref: 'api.yaml#/components/responses/401' + 404: + $ref: 'api.yaml#/components/responses/404' 500: $ref: 'api.yaml#/components/responses/500' diff --git a/api/schemas.yaml b/api/schemas.yaml index 9704e08..99f7998 100644 --- a/api/schemas.yaml +++ b/api/schemas.yaml @@ -46,6 +46,9 @@ SampleRefs: $ref: 'api.yaml#/components/schemas/Id' user_id: $ref: 'api.yaml#/components/schemas/Id' + added: + type: string + example: 1970-01-01T00:00:00.000Z Sample: allOf: - $ref: 'api.yaml#/components/schemas/_Id' diff --git a/data_import/import.js b/data_import/import.js index e69de29..28614c8 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -0,0 +1,485 @@ +const csv = require('csv-parser'); +const fs = require('fs'); +const axios = require('axios'); +const {Builder} = require('selenium-webdriver'); +const chrome = require('selenium-webdriver/chrome'); +const pdfReader = require('pdfreader'); +const iconv = require('iconv-lite'); + +const metadata = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\VZ.csv'; // metadata file +const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\nmDocs'; // NormMaster Documents +const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\DPT'; // Spectrum files +let data = []; // metadata contents +let materials = {}; +let samples = []; +let normMaster = {}; + +// TODO: integrate measurement device information from DPT names using different users +// TODO: supplier: other for supplierless samples + +main(); + +async function main() { + if (0) { // materials + await getNormMaster(); + await importCsv(); + await allMaterials(); + fs.writeFileSync('./data_import/materials.json', JSON.stringify(materials)); + await saveMaterials(); + } + else if (0) { // samples + await importCsv(); + await allSamples(); + await saveSamples(); + } + else if (0) { // DPT + await allDpts(); + } + else if (1) { // KF/VZ + await importCsv(); + await allKfVz(); + } + else if (0) { // pdf test + console.log(await readPdf('N28_BN22-O010_2018-03-08.pdf')); + } +} + +async function importCsv() { + await new Promise(resolve => { + fs.createReadStream(metadata) + .pipe(iconv.decodeStream('win1252')) + .pipe(csv()) + .on('data', (row) => { + data.push(row); + }) + .on('end', () => { + console.info('CSV file successfully processed'); + resolve(); + }); + }); +} + +async function allDpts() { + let res = await axios({ + method: 'get', + url: 'http://localhost:3000/template/measurements', + auth: { + username: 'admin', + password: 'Abc123!#' + } + }); + const measurement_template = res.data.find(e => e.name === 'spectrum')._id; + res = await axios({ + method: 'get', + url: 'http://localhost:3000/samples?status=all', + auth: { + username: 'admin', + password: 'Abc123!#' + } + }); + const sampleIds = {}; + res.data.forEach(sample => { + sampleIds[sample.number] = sample._id; + }); + const regex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/; + const dpts = fs.readdirSync(dptFiles); + for (let i in dpts) { + const regexRes = regex.exec(dpts[i]) + if (regexRes && sampleIds[regexRes[1]]) { // found matching sample + console.log(dpts[i]); + const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8'); + const data = { + sample_id: sampleIds[regexRes[1]], + values: {}, + measurement_template + }; + data.values.dpt = f.split('\r\n').map(e => e.split(',')); + await axios({ + method: 'post', + url: 'http://localhost:3000/measurement/new', + auth: { + username: 'admin', + password: 'Abc123!#' + }, + data + }).catch(err => { + console.log(dpts[i]); + console.error(err.response.data); + }); + } + } +} + +async function allKfVz() { + let res = await axios({ + method: 'get', + url: 'http://localhost:3000/template/measurements', + auth: { + username: 'admin', + password: 'Abc123!#' + } + }); + const kf_template = res.data.find(e => e.name === 'kf')._id; + const vz_template = res.data.find(e => e.name === 'vz')._id; + res = await axios({ + method: 'get', + url: 'http://localhost:3000/samples?status=all', + auth: { + username: 'admin', + password: 'Abc123!#' + } + }); + const sampleIds = {}; + res.data.forEach(sample => { + sampleIds[sample.number] = sample._id; + }); + for (let index in data) { + console.info(`${index}/${data.length}`); + let sample = data[index]; + if (sample['Sample number'] !== '') { + if (sample['KF in Gew%']) { + await axios({ + method: 'post', + url: 'http://localhost:3000/measurement/new', + auth: { + username: 'admin', + password: 'Abc123!#' + }, + data: { + sample_id: sampleIds[sample['Sample number']], + measurement_template: kf_template, + values: { + 'weight %': sample['KF in Gew%'], + 'standard deviation': sample['Stabwn'] + } + } + }).catch(err => { + console.log(sample['Sample number']); + console.error(err.response.data); + }); + } + if (sample['VZ (ml/g)']) { + await axios({ + method: 'post', + url: 'http://localhost:3000/measurement/new', + auth: { + username: 'admin', + password: 'Abc123!#' + }, + data: { + sample_id: sampleIds[sample['Sample number']], + measurement_template: vz_template, + values: { + vz: sample['VZ (ml/g)'] + } + } + }).catch(err => { + console.log(sample['Sample number']); + console.error(err.response.data); + }); + } + } + } +} + +async function allSamples() { + let res = await axios({ + method: 'get', + url: 'http://localhost:3000/materials?status=all', + auth: { + username: 'admin', + password: 'Abc123!#' + } + }); + const dbMaterials = {} + res.data.forEach(m => { + dbMaterials[m.name] = m; + }) + res = await axios({ + method: 'get', + url: 'http://localhost:3000/samples?status=all', + auth: { + username: 'admin', + password: 'Abc123!#' + } + }); + const sampleColors = {}; + res.data.forEach(sample => { + sampleColors[sample.number] = sample.color; + }); + + + for (let index in data) { + console.info(`${index}/${data.length}`); + let sample = data[index]; + if (sample['Sample number'] !== '' && sample['Supplier'] !== '' && sample['Granulate/Part'] !== '') { // TODO: wait for decision about samples without suppliers/color/type + const material = dbMaterials[trim(sample['Material name'])]; + if (!material) { // could not find material, skipping sample + continue; + } + console.log(sample['Material name']); + console.log(material._id); + samples.push({ + number: sample['Sample number'], + type: sample['Granulate/Part'], + batch: sample['Charge/batch granulate/part'] || '', + material_id: material._id, + notes: { + comment: sample['Comments'] + } + }); + const si = samples.length - 1; + if (sample['Material number'] !== '' && material.numbers.find(e => e.number === sample['Material number'])) { // TODO: fix because of false material/material number + samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color; + } + else if (sample['Color'] && sample['Color'] !== '') { + samples[si].color = material.numbers.find(e => e.color.indexOf(sample['Color']) >= 0).color; + } + else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz + samples[si].color = sampleColors[sample['Sample number'].split('_')[0]]; + } + else { // TODO: no color information at all + samples.pop(); + } + } + } +} + +async function saveSamples() { + for (let i in samples) { + console.info(`${i}/${samples.length}`); + await axios({ + method: 'post', + url: 'http://localhost:3000/sample/new', + auth: { + username: 'admin', + password: 'Abc123!#' + }, + data: samples[i] + }).catch(err => { + console.log(samples[i]); + console.error(err.response.data); + }); + } + console.info('saved all samples'); +} + +async function allMaterials() { + for (let index in data) { + let sample = data[index]; + if (sample['Sample number'] !== '' && sample['Supplier'] !== '') { // TODO: wait for decision about supplierless samples + sample['Material name'] = trim(sample['Material name']); + if (materials.hasOwnProperty(sample['Material name'])) { // material already found at least once + if (sample['Material number'] !== '') { + if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) { // new material number + if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) { // color already in list, only number missing + materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']); + } + else { + materials[sample['Material name']].numbers.push({color: sample['Color'], number: stripSpaces(sample['Material number'])}); + } + } + } + else if (sample['Color'] !== '') { + if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) { // new material color + materials[sample['Material name']].numbers.push({color: sample['Color'], number: ''}); + } + } + } + else { // new material + console.info(`${index}/${data.length} ${sample['Material name']}`); + materials[sample['Material name']] = { + name: sample['Material name'], + supplier: sample['Supplier'], + group: sample['Material'] + }; + let tmp = /M(\d+)/.exec(sample['Reinforcing material']); + materials[sample['Material name']].mineral = tmp ? tmp[1] : 0; + tmp = /GF(\d+)/.exec(sample['Reinforcing material']); + materials[sample['Material name']].glass_fiber = tmp ? tmp[1] : 0; + tmp = /CF(\d+)/.exec(sample['Reinforcing material']); + materials[sample['Material name']].carbon_fiber = tmp ? tmp[1] : 0; + materials[sample['Material name']].numbers = await numbersFetch(sample); + console.log(materials[sample['Material name']]); + } + } + } +} + +async function saveMaterials() { + const mKeys = Object.keys(materials) + for (let i in mKeys) { + await axios({ + method: 'post', + url: 'http://localhost:3000/material/new', + auth: { + username: 'admin', + password: 'Abc123!#' + }, + data: materials[mKeys[i]] + }).catch(err => { + console.log(materials[mKeys[i]]); + console.error(err.response.data); + }); + } + console.info('saved all materials'); +} + +async function numbersFetch(sample) { + let nm = []; + let res = []; + if (sample['Material number']) { // sample has a material number + nm = normMaster[stripSpaces(sample['Material number'])]? [normMaster[stripSpaces(sample['Material number'])]] : []; + } + else { // try finding via material name + nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['Material name'])).map(e => normMaster[e]); + } + if (nm.length > 0) { + for (let i in nm) { + // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded + // await getNormMasterDoc(nm[i].url.replace(/ /g, '%20')); + // } + // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded + // console.info('Retrying download...'); + // await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 2.2); + // } + // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded + // console.info('Retrying download again...'); + // await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 5); + // } + if (fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document loaded + res = await readPdf(fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)); + } + if (res.length > 0) { // no results + break; + } + else if (i + 1 >= nm.length) { + console.error('Download failed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'); + } + } + } + if (res.length === 0) { // no results + if (sample['Color'] !== '' || sample['Material number'] !== '') { + return [{color: sample['Color'], number: sample['Material number']}]; + } + else { + return []; + } + } + else { + if (!res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed + res.push({color: sample['Color'], number: sample['Material number']}); + } + return res; + } +} + +async function getNormMaster(fetchAgain = false) { + if (fetchAgain) { + console.info('fetching norm master...'); + const res = await axios({ + method: 'get', + url: 'http://rb-normen.bosch.com/cgi-bin/searchRBNorm4TradeName' + }); + + console.info('finding documents...'); + let match; + // const regex = /.*?.*?<\/span>(.*?)<\/td>(\d+)<\/td>.*?.*?.*?<\/span>(.*?)<\/td>(\d+)<\/td>40.*?(.*?)<\/td>/gm; // only valid materials + do { + match = regex.exec(res.data); + if (match) { + normMaster[match[2]] = {name: match[1], nameSpaceless: stripSpaces(match[1]), number: match[2], url: match[3], doc: match[4]}; + } + } while (match); + fs.writeFileSync('./data_import/normMaster.json', JSON.stringify(normMaster)); + } + else { + normMaster = JSON.parse(fs.readFileSync('./data_import/normMaster.json'), 'utf-8'); + } +} + +function getNormMasterDoc(url, timing = 1) { + console.log(url); + return new Promise(async resolve => { + const options = new chrome.Options(); + options.setUserPreferences({ + "download.default_directory": nmDocs, + "download.prompt_for_download": false, + "download.directory_upgrade": true, + "plugins.always_open_pdf_externally": true + }); + let driver = await new Builder().forBrowser('chrome').setChromeOptions(options).build(); + let timeout = 7000 * timing; + try { + await driver.get(url); + if (await driver.getCurrentUrl() !== 'https://rb-wam-saml.bosch.com/tfim/sps/normmaster/saml20/login') { // got document selection page + timeout = 11000 * timing; + await driver.executeScript('Array.prototype.slice.call(document.querySelectorAll(\'.functionlink\')).filter(e => e.innerText === \'English\')[0].click()').catch(() => {timeout = 0; }); + } + } + finally { + setTimeout(async () => { // wait until download is finished + await driver.quit(); + resolve(); + }, timeout); + } + }); +} + +function readPdf(file) { + return new Promise(async resolve => { + const countdown = 100; // value for text timeout + let table = 0; // > 0 when in correct table area + let rows = []; // found table rows + let lastY = 0; // y of last row + let lastX = 0; // right x of last item + let lastText = ''; // text of last item + let lastLastText = ''; // text of last last item + await new pdfReader.PdfReader().parseFileItems(nmDocs + '\\' + file, (err, item) => { + if (item && item.text) { + if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsupplier') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsupplier') >= 0)) { // table area starts + table = countdown; + } + if (table > 0) { + // console.log(item); + // console.log(item.y - lastY); + // console.log(item.text); + if (item.y - lastY > 0.8 && Math.abs(item.x - lastX) > 5) { // new row + lastY = item.y; + rows.push(item.text); + } + else { // still the same row row + rows[rows.length - 1] += (item.x - lastX > 1.1 ? '$' : '') + item.text; // push to row, detect if still same cell + } + lastX = (item.w * 0.055) + item.x; + + if (/\d \d\d\d \d\d\d \d\d\d/.test(item.text)) { + table = countdown; + } + table --; + if (table <= 0 || item.text.toLowerCase().indexOf('release document') >= 0 || item.text.toLowerCase().indexOf('normative references') >= 0) { // table area ended + table = -1; + // console.log(rows); + rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows + resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; })); + } + } + lastLastText = lastText; + lastText = item.text; + } + if (!item && table !== -1) { // document ended + rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows + resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; })); + } + }); + }); +} + +function stripSpaces(s) { + return s ? s.replace(/ /g,'') : ''; +} + +function trim(s) { + return s.replace(/(^\s+|\s+$)/gm, ''); +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index d3b646e..93fdea0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.8.3.tgz", "integrity": "sha512-a9gxpmdXtZEInkCSHUJDLHZVBgb1QS0jhss4cPP93EW7s+uC5bikET2twEF3KV+7rDblJcmNvTR7VJejqd2C2g==", + "dev": true, "requires": { "@babel/highlight": "^7.8.3" } @@ -213,6 +214,7 @@ "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.8.3.tgz", "integrity": "sha512-PX4y5xQUvy0fnEVHrYOarRPXVWafSjTW9T0Hab8gVIawpl2Sj0ORyrygANq+KjcNlSSTw0YCLSNA8OyZ1I4yEg==", + "dev": true, "requires": { "chalk": "^2.0.0", "esutils": "^2.0.2", @@ -393,12 +395,14 @@ "@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", - "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==" + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==", + "dev": true }, "@szmarczak/http-timer": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "dev": true, "requires": { "defer-to-connect": "^1.0.1" } @@ -406,12 +410,14 @@ "@types/bcrypt": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/bcrypt/-/bcrypt-3.0.0.tgz", - "integrity": "sha512-nohgNyv+1ViVcubKBh0+XiNJ3dO8nYu///9aJ4cgSqv70gBL+94SNy/iC2NLzKPT2Zt/QavrOkBVbZRLZmw6NQ==" + "integrity": "sha512-nohgNyv+1ViVcubKBh0+XiNJ3dO8nYu///9aJ4cgSqv70gBL+94SNy/iC2NLzKPT2Zt/QavrOkBVbZRLZmw6NQ==", + "dev": true }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", "integrity": "sha512-W98JrE0j2K78swW4ukqMleo8R7h/pFETjM2DQ90MF6XK2i4LO4W3gQ71Lt4w3bfm2EvVSyWHplECvB5sK22yFQ==", + "dev": true, "requires": { "@types/connect": "*", "@types/node": "*" @@ -421,6 +427,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/bson/-/bson-4.0.2.tgz", "integrity": "sha512-+uWmsejEHfmSjyyM/LkrP0orfE2m5Mx9Xel4tXNeqi1ldK5XMQcDsFkBmLDtuyKUbxj2jGDo0H240fbCRJZo7Q==", + "dev": true, "requires": { "@types/node": "*" } @@ -428,12 +435,14 @@ "@types/color-name": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", - "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==" + "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", + "dev": true }, "@types/connect": { "version": "3.4.33", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.33.tgz", "integrity": "sha512-2+FrkXY4zllzTNfJth7jOqEHC+enpLeGslEhpnTAkg21GkRrWV4SsAtqchtT4YS9/nODBU2/ZfsBY2X4J/dX7A==", + "dev": true, "requires": { "@types/node": "*" } @@ -442,6 +451,7 @@ "version": "4.17.5", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.5.tgz", "integrity": "sha512-578YH5Lt88AKoADy0b2jQGwJtrBxezXtVe/MBqWXKZpqx91SnC0pVkVCcxcytz3lWW+cHBYDi3Ysh0WXc+rAYw==", + "dev": true, "requires": { "@types/node": "*", "@types/range-parser": "*" @@ -456,17 +466,20 @@ "@types/mime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", - "integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==" + "integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==", + "dev": true }, "@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", - "integrity": "sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==" + "integrity": "sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==", + "dev": true }, "@types/mongodb": { "version": "3.5.10", "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-3.5.10.tgz", "integrity": "sha512-6NkJNfFdFa/njBvN/9eAfq78bWUnapkdR3JbWGGpd7U71PjgKweA4Tlag8psi2mqm973vBYVTD1oc1u0lzRcig==", + "dev": true, "requires": { "@types/bson": "*", "@types/node": "*" @@ -476,6 +489,7 @@ "version": "5.7.12", "resolved": "https://registry.npmjs.org/@types/mongoose/-/mongoose-5.7.12.tgz", "integrity": "sha512-yzLJk3cdSwuMXaIacUCWUb8m960YcgnID7S4ZPOOgzT39aSC46670TuunN+ajDio7OUcGG4mGg8eOGs2Z6VmrA==", + "dev": true, "requires": { "@types/mongodb": "*", "@types/node": "*" @@ -484,22 +498,26 @@ "@types/node": { "version": "13.1.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.6.tgz", - "integrity": "sha512-Jg1F+bmxcpENHP23sVKkNuU3uaxPnsBMW0cLjleiikFKomJQbsn0Cqk2yDvQArqzZN6ABfBkZ0To7pQ8sLdWDg==" + "integrity": "sha512-Jg1F+bmxcpENHP23sVKkNuU3uaxPnsBMW0cLjleiikFKomJQbsn0Cqk2yDvQArqzZN6ABfBkZ0To7pQ8sLdWDg==", + "dev": true }, "@types/qs": { "version": "6.9.1", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.1.tgz", - "integrity": "sha512-lhbQXx9HKZAPgBkISrBcmAcMpZsmpe/Cd/hY7LGZS5OfkySUBItnPZHgQPssWYUET8elF+yCFBbP1Q0RZPTdaw==" + "integrity": "sha512-lhbQXx9HKZAPgBkISrBcmAcMpZsmpe/Cd/hY7LGZS5OfkySUBItnPZHgQPssWYUET8elF+yCFBbP1Q0RZPTdaw==", + "dev": true }, "@types/range-parser": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.3.tgz", - "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==" + "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==", + "dev": true }, "@types/serve-static": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.3.tgz", "integrity": "sha512-oprSwp094zOglVrXdlo/4bAHtKTAxX6VT8FOZlBKrmyLbNvE1zxZyJ6yikMVtHIvwP45+ZQGJn+FdXGKTozq0g==", + "dev": true, "requires": { "@types/express-serve-static-core": "*", "@types/mime": "*" @@ -508,7 +526,8 @@ "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true }, "accepts": { "version": "1.3.7", @@ -533,6 +552,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", "integrity": "sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==", + "dev": true, "requires": { "string-width": "^3.0.0" }, @@ -540,12 +560,14 @@ "ansi-regex": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true }, "string-width": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, "requires": { "emoji-regex": "^7.0.1", "is-fullwidth-code-point": "^2.0.0", @@ -556,6 +578,7 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, "requires": { "ansi-regex": "^4.1.0" } @@ -578,6 +601,7 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, "requires": { "color-convert": "^1.9.0" } @@ -586,6 +610,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "dev": true, "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -636,7 +661,8 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true }, "basic-auth": { "version": "2.0.1", @@ -654,7 +680,8 @@ "binary-extensions": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", - "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==" + "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==", + "dev": true }, "bluebird": { "version": "3.5.1", @@ -676,6 +703,16 @@ "qs": "6.7.0", "raw-body": "2.4.0", "type-is": "~1.6.17" + }, + "dependencies": { + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + } } }, "bowser": { @@ -687,6 +724,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", "integrity": "sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==", + "dev": true, "requires": { "ansi-align": "^3.0.0", "camelcase": "^5.3.1", @@ -701,12 +739,14 @@ "ansi-regex": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true }, "ansi-styles": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, "requires": { "@types/color-name": "^1.1.1", "color-convert": "^2.0.1" @@ -716,6 +756,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -725,6 +766,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "requires": { "color-name": "~1.1.4" } @@ -732,27 +774,32 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true }, "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true }, "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true }, "string-width": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -763,6 +810,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, "requires": { "ansi-regex": "^5.0.0" } @@ -771,6 +819,7 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, "requires": { "has-flag": "^4.0.0" } @@ -781,6 +830,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -790,6 +840,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, "requires": { "fill-range": "^7.0.1" } @@ -808,7 +859,8 @@ "builtin-modules": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", - "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=" + "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", + "dev": true }, "bytes": { "version": "3.1.0", @@ -819,6 +871,7 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "dev": true, "requires": { "clone-response": "^1.0.2", "get-stream": "^5.1.0", @@ -833,6 +886,7 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", + "dev": true, "requires": { "pump": "^3.0.0" } @@ -840,7 +894,8 @@ "lowercase-keys": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", - "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "dev": true } } }, @@ -864,7 +919,8 @@ "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true }, "camelize": { "version": "1.0.0", @@ -885,6 +941,7 @@ "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, "requires": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", @@ -895,6 +952,7 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.3.1.tgz", "integrity": "sha512-4QYCEWOcK3OJrxwvyyAOxFuhpvOVCYkr33LPfFNBjAD/w3sEzWsp2BUOkI4l9bHvWioAd0rc6NlHUOEaWkTeqg==", + "dev": true, "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", @@ -909,7 +967,8 @@ "ci-info": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", - "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==" + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "dev": true }, "clean-stack": { "version": "2.2.0", @@ -920,7 +979,8 @@ "cli-boxes": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.0.tgz", - "integrity": "sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w==" + "integrity": "sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w==", + "dev": true }, "cliui": { "version": "5.0.0", @@ -965,6 +1025,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "dev": true, "requires": { "mimic-response": "^1.0.0" } @@ -973,6 +1034,7 @@ "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, "requires": { "color-name": "1.1.3" } @@ -980,7 +1042,8 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true }, "combined-stream": { "version": "1.0.8", @@ -1040,12 +1103,14 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true }, "configstore": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", + "dev": true, "requires": { "dot-prop": "^5.2.0", "graceful-fs": "^4.1.2", @@ -1134,7 +1199,18 @@ "crypto-random-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", - "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==" + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", + "dev": true + }, + "csv-parser": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-2.3.3.tgz", + "integrity": "sha512-czcyxc4/3Tt63w0oiK1zsnRgRD4PkqWaRSJ6eef63xC0f+5LVLuGdSYEcJwGp2euPgRHx+jmlH2Lb49anb1CGQ==", + "dev": true, + "requires": { + "minimist": "^1.2.0", + "through2": "^3.0.1" + } }, "dasherize": { "version": "2.0.0", @@ -1159,6 +1235,7 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "dev": true, "requires": { "mimic-response": "^1.0.0" } @@ -1166,7 +1243,8 @@ "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true }, "default-require-extensions": { "version": "3.0.0", @@ -1180,7 +1258,8 @@ "defer-to-connect": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", - "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==" + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==", + "dev": true }, "define-properties": { "version": "1.1.3", @@ -1210,7 +1289,8 @@ "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true }, "dns-prefetch-control": { "version": "0.2.0", @@ -1226,6 +1306,7 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==", + "dev": true, "requires": { "is-obj": "^2.0.0" } @@ -1233,7 +1314,8 @@ "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", - "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", + "dev": true }, "ee-first": { "version": "1.1.1", @@ -1243,7 +1325,8 @@ "emoji-regex": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true }, "encodeurl": { "version": "1.0.2", @@ -1254,6 +1337,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, "requires": { "once": "^1.4.0" } @@ -1297,7 +1381,8 @@ "escape-goat": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz", - "integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==" + "integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==", + "dev": true }, "escape-html": { "version": "1.0.3", @@ -1307,7 +1392,8 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true }, "esprima": { "version": "4.0.1", @@ -1317,7 +1403,8 @@ "esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true }, "etag": { "version": "1.8.1", @@ -1381,6 +1468,7 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, "requires": { "to-regex-range": "^5.0.1" } @@ -1497,12 +1585,14 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true }, "fsevents": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz", "integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==", + "dev": true, "optional": true }, "function-bind": { @@ -1533,6 +1623,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, "requires": { "pump": "^3.0.0" } @@ -1541,6 +1632,7 @@ "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1554,6 +1646,7 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.0.tgz", "integrity": "sha512-qjtRgnIVmOfnKUE3NJAQEdk+lKrxfw8t5ke7SXtfMTHcjsBfOfWXCQfdb30zfDoZQ2IRSIiidmjtbHZPZ++Ihw==", + "dev": true, "requires": { "is-glob": "^4.0.1" } @@ -1562,6 +1655,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.0.1.tgz", "integrity": "sha512-5HqUqdhkEovj2Of/ms3IeS/EekcO54ytHRLV4PEY2rhRwrHXLQjeVEES0Lhka0xwNDtGYn58wyC4s5+MHsOO6A==", + "dev": true, "requires": { "ini": "^1.3.5" } @@ -1576,6 +1670,7 @@ "version": "9.6.0", "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", + "dev": true, "requires": { "@sindresorhus/is": "^0.14.0", "@szmarczak/http-timer": "^1.1.2", @@ -1593,7 +1688,8 @@ "graceful-fs": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", - "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==", + "dev": true }, "growl": { "version": "1.10.5", @@ -1613,7 +1709,8 @@ "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true }, "has-symbols": { "version": "1.0.1", @@ -1624,7 +1721,8 @@ "has-yarn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", - "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==" + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", + "dev": true }, "hasha": { "version": "5.2.0", @@ -1721,7 +1819,8 @@ "http-cache-semantics": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", - "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "dev": true }, "http-errors": { "version": "1.7.2", @@ -1736,9 +1835,10 @@ } }, "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.0.tgz", + "integrity": "sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==", + "dev": true, "requires": { "safer-buffer": ">= 2.1.2 < 3" } @@ -1751,17 +1851,26 @@ "ignore-by-default": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", - "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=" + "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=", + "dev": true + }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", + "dev": true }, "import-lazy": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", - "integrity": "sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM=" + "integrity": "sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM=", + "dev": true }, "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=" + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true }, "indent-string": { "version": "4.0.0", @@ -1773,6 +1882,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -1786,7 +1896,8 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true }, "ipaddr.js": { "version": "1.9.0", @@ -1797,6 +1908,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, "requires": { "binary-extensions": "^2.0.0" } @@ -1817,6 +1929,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dev": true, "requires": { "ci-info": "^2.0.0" } @@ -1830,17 +1943,20 @@ "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=" + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true }, "is-glob": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, "requires": { "is-extglob": "^2.1.1" } @@ -1849,6 +1965,7 @@ "version": "0.3.2", "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", "integrity": "sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==", + "dev": true, "requires": { "global-dirs": "^2.0.1", "is-path-inside": "^3.0.1" @@ -1857,22 +1974,26 @@ "is-npm": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", - "integrity": "sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==" + "integrity": "sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==", + "dev": true }, "is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true }, "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", - "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "dev": true }, "is-path-inside": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.2.tgz", - "integrity": "sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg==" + "integrity": "sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg==", + "dev": true }, "is-regex": { "version": "1.0.5", @@ -1901,7 +2022,8 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true }, "is-windows": { "version": "1.0.2", @@ -1912,7 +2034,8 @@ "is-yarn-global": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", - "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==" + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==", + "dev": true }, "isarray": { "version": "1.0.0", @@ -2051,7 +2174,8 @@ "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true }, "js-yaml": { "version": "3.13.1", @@ -2071,7 +2195,8 @@ "json-buffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", - "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=" + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=", + "dev": true }, "json-schema": { "version": "0.2.5", @@ -2087,6 +2212,18 @@ "minimist": "^1.2.5" } }, + "jszip": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.5.0.tgz", + "integrity": "sha512-WRtu7TPCmYePR1nazfrtuF216cIVon/3GWOvHS9QR5bIwSbnxtdpma6un3jyGGNhHsKCSzn5Ypk+EkDRvTGiFA==", + "dev": true, + "requires": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + } + }, "kareem": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.3.1.tgz", @@ -2096,6 +2233,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "dev": true, "requires": { "json-buffer": "3.0.0" } @@ -2104,10 +2242,20 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", + "dev": true, "requires": { "package-json": "^6.3.0" } }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "requires": { + "immediate": "~3.0.5" + } + }, "locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", @@ -2151,12 +2299,14 @@ "lowercase-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", - "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "dev": true }, "make-dir": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz", "integrity": "sha512-rYKABKutXa6vXTXhoV18cBE7PaewPXHe/Bdq4v+ZLMhxbWApkFFplT0LcbMW+6BbjnQXzZ/sAvSE/JdguApG5w==", + "dev": true, "requires": { "semver": "^6.0.0" }, @@ -2164,7 +2314,8 @@ "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true } } }, @@ -2210,12 +2361,14 @@ "mimic-response": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -2223,12 +2376,14 @@ "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true }, "mkdirp": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, "requires": { "minimist": "^1.2.5" } @@ -2447,6 +2602,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.3.tgz", "integrity": "sha512-lLQLPS90Lqwc99IHe0U94rDgvjo+G9I4uEIxRG3evSLROcqQ9hwc0AxlSHKS4T1JW/IMj/7N5mthiN58NL/5kw==", + "dev": true, "requires": { "chokidar": "^3.2.2", "debug": "^3.2.6", @@ -2464,6 +2620,7 @@ "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "dev": true, "requires": { "ms": "^2.1.1" } @@ -2471,7 +2628,8 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true } } }, @@ -2479,6 +2637,7 @@ "version": "1.0.10", "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", "integrity": "sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=", + "dev": true, "requires": { "abbrev": "1" } @@ -2486,12 +2645,14 @@ "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true }, "normalize-url": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", - "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==" + "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", + "dev": true }, "nyc": { "version": "15.0.1", @@ -2734,6 +2895,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "requires": { "wrappy": "1" } @@ -2743,10 +2905,17 @@ "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-1.3.5.tgz", "integrity": "sha512-11oi4zYorsgvg5yBarZplAqbpev5HkuVNPlZaPTknPDzAynq+lnJdXAmruGWP0s+dNYZS7bjM+xrTpJw7184Fg==" }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true + }, "p-cancelable": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", - "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==" + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", + "dev": true }, "p-limit": { "version": "2.3.0", @@ -2797,6 +2966,7 @@ "version": "6.5.0", "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", + "dev": true, "requires": { "got": "^9.6.0", "registry-auth-token": "^4.0.0", @@ -2807,10 +2977,17 @@ "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true } } }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2825,7 +3002,8 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true }, "path-key": { "version": "3.1.1", @@ -2836,17 +3014,76 @@ "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, + "pdf2json": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pdf2json/-/pdf2json-1.2.0.tgz", + "integrity": "sha512-Z/m+OFOe13Nn2SHQNSINZ6Mh2b8t2bK3whL3L6b5Av1wqDvotYvpMg1Zi8aEPV37jF0jG0yQ83c8XuuNbIsn6Q==", + "dev": true, + "requires": { + "async": "^3.2.0", + "lodash": "^4.17.13", + "optimist": "^0.6.1", + "xmldom": "^0.3.0" + }, + "dependencies": { + "async": { + "version": "3.2.0", + "bundled": true, + "dev": true + }, + "lodash": { + "version": "4.17.15", + "bundled": true, + "dev": true + }, + "minimist": { + "version": "0.0.10", + "bundled": true, + "dev": true + }, + "optimist": { + "version": "0.6.1", + "bundled": true, + "dev": true, + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + } + }, + "wordwrap": { + "version": "0.0.3", + "bundled": true, + "dev": true + }, + "xmldom": { + "version": "0.3.0", + "bundled": true, + "dev": true + } + } + }, + "pdfreader": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/pdfreader/-/pdfreader-1.0.7.tgz", + "integrity": "sha512-3hX/PpA/MQV2uvSiR2CH7isuyZXqYPoA6IXOxHd7hw9qS6Lz9RKYKu+iU369+OgkJKe/SHpxwEbgoHBV4L/76w==", + "dev": true, + "requires": { + "pdf2json": "^1.1.8" + } + }, "picomatch": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.1.tgz", - "integrity": "sha512-ISBaA8xQNmwELC7eOjqFKMESB2VIqt4PPDD0nsS95b/9dZXvVKOlz9keMSnoGGKcOHXfTvDD6WMaRoSc9UuhRA==" + "integrity": "sha512-ISBaA8xQNmwELC7eOjqFKMESB2VIqt4PPDD0nsS95b/9dZXvVKOlz9keMSnoGGKcOHXfTvDD6WMaRoSc9UuhRA==", + "dev": true }, "pkg-dir": { "version": "4.2.0", @@ -2901,7 +3138,8 @@ "prepend-http": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", - "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=" + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=", + "dev": true }, "process-nextick-args": { "version": "2.0.1", @@ -2930,12 +3168,14 @@ "pstree.remy": { "version": "1.1.7", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.7.tgz", - "integrity": "sha512-xsMgrUwRpuGskEzBFkH8NmTimbZ5PcPup0LA8JJkHIm2IMUbQcpo3yeLNWVrufEYjh8YwtSVh0xz6UeWc5Oh5A==" + "integrity": "sha512-xsMgrUwRpuGskEzBFkH8NmTimbZ5PcPup0LA8JJkHIm2IMUbQcpo3yeLNWVrufEYjh8YwtSVh0xz6UeWc5Oh5A==", + "dev": true }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -2945,6 +3185,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.0.1.tgz", "integrity": "sha512-hEJH0s8PXLY/cdXh66tNEQGndDrIKNqNC5xmrysZy3i5C3oEoLna7YAOad+7u125+zH1HNXUmGEkrhb3c2VriA==", + "dev": true, "requires": { "escape-goat": "^2.0.0" } @@ -2968,12 +3209,23 @@ "http-errors": "1.7.2", "iconv-lite": "0.4.24", "unpipe": "1.0.0" + }, + "dependencies": { + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + } } }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -3000,6 +3252,7 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.3.0.tgz", "integrity": "sha512-zz0pAkSPOXXm1viEwygWIPSPkcBYjW1xU5j/JBh5t9bGCJwa6f9+BJa6VaB2g+b55yVrmXzqkyLf4xaWYM0IkQ==", + "dev": true, "requires": { "picomatch": "^2.0.7" } @@ -3018,6 +3271,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.1.1.tgz", "integrity": "sha512-9bKS7nTl9+/A1s7tnPeGrUpRcVY+LUh7bfFgzpndALdPfXQBfQV77rQVtqgUV3ti4vc/Ik81Ex8UJDWDQ12zQA==", + "dev": true, "requires": { "rc": "^1.2.8" } @@ -3026,6 +3280,7 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", + "dev": true, "requires": { "rc": "^1.2.8" } @@ -3064,6 +3319,7 @@ "version": "1.14.2", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.14.2.tgz", "integrity": "sha512-EjlOBLBO1kxsUxsKjLt7TAECyKW6fOh1VRkykQkKGzcBbjjPIxBqGh0jf7GJ3k/f5mxMqW3htMD3WdTUVtW8HQ==", + "dev": true, "requires": { "path-parse": "^1.0.6" } @@ -3077,6 +3333,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "dev": true, "requires": { "lowercase-keys": "^1.0.0" } @@ -3109,6 +3366,28 @@ "sparse-bitfield": "^3.0.3" } }, + "selenium-webdriver": { + "version": "4.0.0-alpha.7", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.0.0-alpha.7.tgz", + "integrity": "sha512-D4qnTsyTr91jT8f7MfN+OwY0IlU5+5FmlO5xlgRUV6hDEV8JyYx2NerdTEqDDkNq7RZDYc4VoPALk8l578RBHw==", + "dev": true, + "requires": { + "jszip": "^3.2.2", + "rimraf": "^2.7.1", + "tmp": "0.0.30" + }, + "dependencies": { + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -3118,6 +3397,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-3.1.1.tgz", "integrity": "sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==", + "dev": true, "requires": { "semver": "^6.3.0" }, @@ -3125,7 +3405,8 @@ "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true } } }, @@ -3173,6 +3454,12 @@ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true }, + "set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", + "dev": true + }, "setprototypeof": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", @@ -3255,7 +3542,8 @@ "signal-exit": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", - "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", + "dev": true }, "sliced": { "version": "1.0.1", @@ -3391,7 +3679,8 @@ "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true }, "superagent": { "version": "3.8.3", @@ -3442,6 +3731,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, "requires": { "has-flag": "^3.0.0" } @@ -3462,7 +3752,8 @@ "term-size": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.0.tgz", - "integrity": "sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw==" + "integrity": "sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw==", + "dev": true }, "test-exclude": { "version": "6.0.0", @@ -3475,6 +3766,24 @@ "minimatch": "^3.0.4" } }, + "through2": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", + "dev": true, + "requires": { + "readable-stream": "2 || 3" + } + }, + "tmp": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.30.tgz", + "integrity": "sha1-ckGdSovn1s51FI/YsyTlk6cRwu0=", + "dev": true, + "requires": { + "os-tmpdir": "~1.0.1" + } + }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -3484,12 +3793,14 @@ "to-readable-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", - "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==" + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==", + "dev": true }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, "requires": { "is-number": "^7.0.0" } @@ -3503,6 +3814,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==", + "dev": true, "requires": { "nopt": "~1.0.10" } @@ -3510,12 +3822,14 @@ "tslib": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz", - "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==" + "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==", + "dev": true }, "tslint": { "version": "5.20.1", "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.20.1.tgz", "integrity": "sha512-EcMxhzCFt8k+/UP5r8waCf/lzmeSyVlqxqMEDQE7rWYiQky8KpIBz1JAoYXfROHrPZ1XXd43q8yQnULOLiBRQg==", + "dev": true, "requires": { "@babel/code-frame": "^7.0.0", "builtin-modules": "^1.1.1", @@ -3536,6 +3850,7 @@ "version": "2.29.0", "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz", "integrity": "sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==", + "dev": true, "requires": { "tslib": "^1.8.1" } @@ -3543,7 +3858,8 @@ "type-fest": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", - "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==" + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true }, "type-is": { "version": "1.6.18", @@ -3558,6 +3874,7 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, "requires": { "is-typedarray": "^1.0.0" } @@ -3565,12 +3882,14 @@ "typescript": { "version": "3.7.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.4.tgz", - "integrity": "sha512-A25xv5XCtarLwXpcDNZzCGvW2D1S3/bACratYBx2sax8PefsFhlYmkQicKHvpYflFS8if4zne5zT5kpJ7pzuvw==" + "integrity": "sha512-A25xv5XCtarLwXpcDNZzCGvW2D1S3/bACratYBx2sax8PefsFhlYmkQicKHvpYflFS8if4zne5zT5kpJ7pzuvw==", + "dev": true }, "undefsafe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz", "integrity": "sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A==", + "dev": true, "requires": { "debug": "^2.2.0" } @@ -3584,6 +3903,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "dev": true, "requires": { "crypto-random-string": "^2.0.0" } @@ -3597,6 +3917,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.0.tgz", "integrity": "sha512-w3doE1qtI0/ZmgeoDoARmI5fjDoT93IfKgEGqm26dGUOh8oNpaSTsGNdYRN/SjOuo10jcJGwkEL3mroKzktkew==", + "dev": true, "requires": { "boxen": "^4.2.0", "chalk": "^3.0.0", @@ -3617,6 +3938,7 @@ "version": "4.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, "requires": { "@types/color-name": "^1.1.1", "color-convert": "^2.0.1" @@ -3626,6 +3948,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -3635,6 +3958,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "requires": { "color-name": "~1.1.4" } @@ -3642,17 +3966,20 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true }, "supports-color": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, "requires": { "has-flag": "^4.0.0" } @@ -3663,6 +3990,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", + "dev": true, "requires": { "prepend-http": "^2.0.0" } @@ -3722,6 +4050,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "dev": true, "requires": { "string-width": "^4.0.0" }, @@ -3729,22 +4058,26 @@ "ansi-regex": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true }, "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true }, "string-width": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -3755,6 +4088,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, "requires": { "ansi-regex": "^5.0.0" } @@ -3803,12 +4137,14 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true }, "write-file-atomic": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, "requires": { "imurmurhash": "^0.1.4", "is-typedarray": "^1.0.0", @@ -3824,7 +4160,8 @@ "xdg-basedir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", - "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==" + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==", + "dev": true }, "y18n": { "version": "4.0.0", diff --git a/package.json b/package.json index 777e274..e5ca620 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "start": "node index.js", "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"", "loadDev": "node dist/test/loadDev.js", - "coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000" + "coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000", + "import": "node data_import/import.js" }, "keywords": [], "author": "", @@ -45,9 +46,13 @@ "@types/node": "^13.1.6", "@types/qs": "^6.9.1", "@types/serve-static": "^1.13.3", + "csv-parser": "^2.3.3", + "iconv-lite": "^0.6.0", "mocha": "^7.1.2", "nodemon": "^2.0.3", "nyc": "^15.0.1", + "pdfreader": "^1.0.7", + "selenium-webdriver": "^4.0.0-alpha.7", "should": "^13.2.3", "supertest": "^4.0.2", "tslint": "^5.20.1", diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 9a483a4..4af6d55 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -10,6 +10,7 @@ import mongoose from 'mongoose'; // TODO: generate output for ML in format DPT -> data, implement filtering, field selection // TODO: generate csv // TODO: write script for data import +// TODO: allowed types: tension rod, part, granulate, other describe('/sample', () => { let server; @@ -18,6 +19,7 @@ describe('/sample', () => { afterEach(done => TestHelper.afterEach(server, done)); after(done => TestHelper.after(done)); + // TODO: sort, added date filter, has measurements/condition filter describe('GET /samples', () => { it('returns all samples', done => { TestHelper.request(server, done, { @@ -30,7 +32,7 @@ describe('/sample', () => { const json = require('../test/db.json'); should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.validated).length); should(res.body).matchEach(sample => { - should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(sample).have.property('_id').be.type('string'); should(sample).have.property('number').be.type('string'); should(sample).have.property('type').be.type('string'); @@ -41,6 +43,7 @@ describe('/sample', () => { should(sample).have.property('material_id').be.type('string'); should(sample).have.property('note_id'); should(sample).have.property('user_id').be.type('string'); + should(sample).have.property('added').be.type('string'); }); done(); }); @@ -56,7 +59,7 @@ describe('/sample', () => { const json = require('../test/db.json'); should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.validated).length); should(res.body).matchEach(sample => { - should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(sample).have.property('_id').be.type('string'); should(sample).have.property('number').be.type('string'); should(sample).have.property('type').be.type('string'); @@ -67,6 +70,7 @@ describe('/sample', () => { should(sample).have.property('material_id').be.type('string'); should(sample).have.property('note_id'); should(sample).have.property('user_id').be.type('string'); + should(sample).have.property('added').be.type('string'); }); done(); }); @@ -82,7 +86,7 @@ describe('/sample', () => { const json = require('../test/db.json'); should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.new).length); should(res.body).matchEach(sample => { - should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(sample).have.property('_id').be.type('string'); should(sample).have.property('number').be.type('string'); should(sample).have.property('type').be.type('string'); @@ -92,10 +96,114 @@ describe('/sample', () => { should(sample).have.property('material_id').be.type('string'); should(sample).have.property('note_id'); should(sample).have.property('user_id').be.type('string'); + should(sample).have.property('added').be.type('string'); }); done(); }); }); + it('uses the given page size', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&page-size=3', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(3); + done(); + }); + }); + it('returns results starting after last-id', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&last-id=400000000000000000000002', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.property('_id', '400000000000000000000003'); + should(res.body[1]).have.property('_id', '400000000000000000000004'); + done(); + }); + }); + it('returns the right page number', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&to-page=2&page-size=2', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.property('_id', '400000000000000000000006'); + done(); + }); + }); + it('works with negative page numbers', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&to-page=-1&page-size=2&last-id=400000000000000000000004', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.property('_id', '400000000000000000000001'); + should(res.body[1]).have.property('_id', '400000000000000000000002'); + done(); + }); + }); + it('returns an empty array for a page number out of range', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&to-page=100&page-size=2', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(0); + should(res.body).be.eql([]); + done(); + }); + }); + it('returns an empty array for a page number out of negative range', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&to-page=-100&page-size=3&last-id=400000000000000000000004', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(0); + should(res.body).be.eql([]); + done(); + }); + }); + it('rejects a negative page size', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?page-size=-3', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"page-size" must be larger than or equal to 1'} + }); + }); + it('rejects an invalid last-id', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?last-id=40000000000h000000000002', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"last-id" with value "40000000000h000000000002" fails to match the required pattern: /[0-9a-f]{24}/'} + }); + }); + it('rejects a to-page without page-size', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?to-page=3', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"to-page" missing required peer "page-size"'} + }); + }); it('rejects an invalid state name', done => { TestHelper.request(server, done, { method: 'get', @@ -127,7 +235,7 @@ describe('/sample', () => { let asyncCounter = res.body.length; should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.new).length); should(res.body).matchEach(sample => { - should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(sample).have.property('_id').be.type('string'); should(sample).have.property('number').be.type('string'); should(sample).have.property('type').be.type('string'); @@ -140,6 +248,7 @@ describe('/sample', () => { should(sample).have.property('material_id').be.type('string'); should(sample).have.property('note_id'); should(sample).have.property('user_id').be.type('string'); + should(sample).have.property('added').be.type('string'); SampleModel.findById(sample._id).lean().exec((err, data) => { should(data).have.property('status',globals.status.new); if (--asyncCounter === 0) { @@ -161,7 +270,7 @@ describe('/sample', () => { let asyncCounter = res.body.length; should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status === -1).length); should(res.body).matchEach(sample => { - should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(sample).have.property('_id').be.type('string'); should(sample).have.property('number').be.type('string'); should(sample).have.property('type').be.type('string'); @@ -174,6 +283,7 @@ describe('/sample', () => { should(sample).have.property('material_id').be.type('string'); should(sample).have.property('note_id'); should(sample).have.property('user_id').be.type('string'); + should(sample).have.property('added').be.type('string'); SampleModel.findById(sample._id).lean().exec((err, data) => { should(data).have.property('status',globals.status.deleted); if (--asyncCounter === 0) { @@ -269,7 +379,7 @@ describe('/sample', () => { }); }); - describe('PUT /sample/{id}', () => { + describe('PUT /sample/{id}', () => { // TODO: fix tests, work on /samples it('returns the right sample', done => { TestHelper.request(server, done, { method: 'put', @@ -277,7 +387,7 @@ describe('/sample', () => { auth: {basic: 'janedoe'}, httpStatus: 200, req: {}, - res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'} + res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'} }); }); it('keeps unchanged properties', done => { @@ -289,7 +399,7 @@ describe('/sample', () => { req: {type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', notes: {}} }).end((err, res) => { if (err) return done(err); - should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}); + should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}); SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => { if (err) return done (err); should(data).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'status', '__v'); @@ -316,7 +426,7 @@ describe('/sample', () => { req: {type: 'granulate'} }).end((err, res) => { if (err) return done(err); - should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}); + should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}); SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => { if (err) return done (err); should(data).have.property('status',globals.status.validated); @@ -333,7 +443,7 @@ describe('/sample', () => { req: {condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}} }).end((err, res) => { if (err) return done(err); - should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}); + should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}); SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => { if (err) return done (err); should(data).have.property('status',globals.status.validated); @@ -350,7 +460,7 @@ describe('/sample', () => { req: {notes: {comment: 'Stoff gesperrt', sample_references: []}} }).end((err, res) => { if (err) return done(err); - should(res.body).be.eql({_id: '400000000000000000000002', number: '21', type: 'granulate', color: 'natural', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', note_id: '500000000000000000000001', user_id: '000000000000000000000002'}); + should(res.body).be.eql({_id: '400000000000000000000002', number: '21', type: 'granulate', color: 'natural', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', note_id: '500000000000000000000001', user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}); SampleModel.findById('400000000000000000000002').lean().exec((err, data: any) => { if (err) return done (err); should(data).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'status', '__v'); @@ -627,7 +737,7 @@ describe('/sample', () => { auth: {basic: 'janedoe'}, httpStatus: 200, req: {condition: {}}, - res: {_id: '400000000000000000000006', number: 'Rng36', type: 'granulate', color: 'black', batch: '', condition: {}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'} + res: {_id: '400000000000000000000006', number: 'Rng36', type: 'granulate', color: 'black', batch: '', condition: {}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'} }); }); it('rejects an old version of a condition template', done => { @@ -647,7 +757,7 @@ describe('/sample', () => { auth: {basic: 'admin'}, httpStatus: 200, req: {condition: {p1: 36, condition_template: '200000000000000000000004'}}, - res: {_id: '400000000000000000000004', number: '32', type: 'granulate', color: 'black', batch: '1653000308', condition: {p1: 36, condition_template: '200000000000000000000004'}, material_id: '100000000000000000000005', note_id: '500000000000000000000003', user_id: '000000000000000000000003'} + res: {_id: '400000000000000000000004', number: '32', type: 'granulate', color: 'black', batch: '1653000308', condition: {p1: 36, condition_template: '200000000000000000000004'}, material_id: '100000000000000000000005', note_id: '500000000000000000000003', user_id: '000000000000000000000003', added: '2004-01-10T13:37:04.000Z'} }); }); it('rejects an changing back to an empty condition', done => { @@ -694,7 +804,7 @@ describe('/sample', () => { auth: {basic: 'admin'}, httpStatus: 200, req: {}, - res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {condition_template: '200000000000000000000001', material: 'copper', weeks: 3}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'} + res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {condition_template: '200000000000000000000001', material: 'copper', weeks: 3}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'} }); }); it('rejects requests from a read user', done => { @@ -1085,7 +1195,7 @@ describe('/sample', () => { req: {color: 'black', type: 'granulate', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}} }).end((err, res) => { if (err) return done (err); - should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(res.body).have.property('_id').be.type('string'); should(res.body).have.property('number', 'Rng37'); should(res.body).have.property('color', 'black'); @@ -1095,6 +1205,8 @@ describe('/sample', () => { should(res.body).have.property('material_id', '100000000000000000000001'); should(res.body).have.property('note_id').be.type('string'); should(res.body).have.property('user_id', '000000000000000000000002'); + should(res.body).have.property('added').be.type('string'); + should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000); done(); }); }); @@ -1198,7 +1310,7 @@ describe('/sample', () => { req: {color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}} }).end((err, res) => { if (err) return done (err); - should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(res.body).have.property('_id').be.type('string'); should(res.body).have.property('number', 'Fe1'); should(res.body).have.property('color', 'black'); @@ -1207,6 +1319,8 @@ describe('/sample', () => { should(res.body).have.property('material_id', '100000000000000000000001'); should(res.body).have.property('note_id').be.type('string'); should(res.body).have.property('user_id', '000000000000000000000004'); + should(res.body).have.property('added').be.type('string'); + should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1500); done(); }); }); @@ -1219,7 +1333,7 @@ describe('/sample', () => { req: {color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}} }).end((err, res) => { if (err) return done (err); - should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(res.body).have.property('_id').be.type('string'); should(res.body).have.property('number', 'Rng37'); should(res.body).have.property('color', 'black'); @@ -1229,6 +1343,8 @@ describe('/sample', () => { should(res.body).have.property('material_id', '100000000000000000000001'); should(res.body).have.property('note_id').be.type('string'); should(res.body).have.property('user_id', '000000000000000000000002'); + should(res.body).have.property('added').be.type('string'); + should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000); done(); }); }); @@ -1271,7 +1387,7 @@ describe('/sample', () => { req: {number: 'Rng34', color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}}, }).end((err, res) => { if (err) return done (err); - should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id'); + should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added'); should(res.body).have.property('_id').be.type('string'); should(res.body).have.property('number', 'Rng34'); should(res.body).have.property('color', 'black'); @@ -1281,6 +1397,8 @@ describe('/sample', () => { should(res.body).have.property('material_id', '100000000000000000000001'); should(res.body).have.property('note_id').be.type('string'); should(res.body).have.property('user_id', '000000000000000000000003'); + should(res.body).have.property('added').be.type('string'); + should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000); done(); }); }); diff --git a/src/routes/sample.ts b/src/routes/sample.ts index d28e725..24f33cf 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -25,22 +25,43 @@ router.get('/samples', (req, res, next) => { const {error, value: filters} = SampleValidate.query(req.query); if (error) return res400(error, res); - let conditions; - + let status; if (filters.hasOwnProperty('status')) { if(filters.status === 'all') { - conditions = {$or: [{status: globals.status.validated}, {status: globals.status.new}]} + status = {$or: [{status: globals.status.validated}, {status: globals.status.new}]} } else { - conditions = {status: globals.status[filters.status]}; + status = {status: globals.status[filters.status]}; } } else { // default - conditions = {status: globals.status.validated}; + status = {status: globals.status.validated}; + } + const query = SampleModel.find(status); + + if (filters['page-size']) { + query.limit(filters['page-size']); } - SampleModel.find(conditions).lean().exec((err, data) => { + if (filters['last-id']) { + if (filters['to-page'] && filters['to-page'] < 0) { + query.lte('_id', mongoose.Types.ObjectId(filters['last-id'])); // TODO: consider sorting + query.sort({_id: -1}); + } + else { + query.gt('_id', mongoose.Types.ObjectId(filters['last-id'])); // TODO: consider sorting + } + } + + if (filters['to-page']) { + query.skip(Math.abs(filters['to-page']) * filters['page-size']); // TODO: check order for negative numbers + } + + query.lean().exec((err, data) => { if (err) return next(err); + if (filters['to-page'] && filters['to-page'] < 0) { + data.reverse(); + } res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors }) }); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 1da56da..f1f17f9 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -44,7 +44,11 @@ export default class SampleValidate { Joi.date() ) ) - }) + }), + + added: Joi.date() + .iso() + .min('1970-01-01T00:00:00.000Z') }; static input (data, param) { // validate input, set param to 'new' to make all attributes required @@ -85,6 +89,7 @@ export default class SampleValidate { } static output (data, param = 'refs') { // validate output and strip unwanted properties, returns null if not valid + data.added = data._id.getTimestamp(); data = IdValidate.stringify(data); let joiObject; if (param === 'refs') { @@ -97,7 +102,8 @@ export default class SampleValidate { condition: this.sample.condition, material_id: IdValidate.get(), note_id: IdValidate.get().allow(null), - user_id: IdValidate.get() + user_id: IdValidate.get(), + added: this.sample.added }; } else if(param === 'details') { @@ -123,7 +129,10 @@ export default class SampleValidate { static query (data) { return Joi.object({ - status: Joi.string().valid('validated', 'new', 'all') - }).validate(data); + status: Joi.string().valid('validated', 'new', 'all'), + 'last-id': IdValidate.get(), + 'to-page': Joi.number().integer(), + 'page-size': Joi.number().integer().min(1) + }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file From 4dad680edf63d7a6ae954f99a030773badffe317 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 25 Jun 2020 11:59:36 +0200 Subject: [PATCH 05/19] changed last-id behaviour to from-id --- api/sample.yaml | 6 +++--- src/routes/sample.spec.ts | 22 +++++++++++----------- src/routes/sample.ts | 9 +++++---- src/routes/validate/sample.ts | 2 +- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index 4ebb61f..b37702d 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -12,14 +12,14 @@ schema: type: string example: all - - name: last-id - description: last id of current page, if not given the results are displayed from start + - name: from-id + description: first id of the requested page, if not given the results are displayed from start in: query schema: type: string example: 5ea0450ed851c30a90e70894 - name: to-page - description: relative change of pages, use negative values to get back, defaults to 0 (if last-id is given, the sample after is the first of the result, so the next page is selected automatically), works only together with page-size + description: relative change of pages, use negative values to get back, defaults to 0, works only together with page-size in: query schema: type: string diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 4af6d55..bd3cb79 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -113,16 +113,16 @@ describe('/sample', () => { done(); }); }); - it('returns results starting after last-id', done => { + it('returns results starting from first-id', done => { TestHelper.request(server, done, { method: 'get', - url: '/samples?status=all&last-id=400000000000000000000002', + url: '/samples?status=all&from-id=400000000000000000000002', auth: {basic: 'janedoe'}, httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.body[0]).have.property('_id', '400000000000000000000003'); - should(res.body[1]).have.property('_id', '400000000000000000000004'); + should(res.body[0]).have.property('_id', '400000000000000000000002'); + should(res.body[1]).have.property('_id', '400000000000000000000003'); done(); }); }); @@ -141,13 +141,13 @@ describe('/sample', () => { it('works with negative page numbers', done => { TestHelper.request(server, done, { method: 'get', - url: '/samples?status=all&to-page=-1&page-size=2&last-id=400000000000000000000004', + url: '/samples?status=all&to-page=-1&page-size=2&from-id=400000000000000000000004', auth: {basic: 'janedoe'}, httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.body[0]).have.property('_id', '400000000000000000000001'); - should(res.body[1]).have.property('_id', '400000000000000000000002'); + should(res.body[0]).have.property('_id', '400000000000000000000002'); + should(res.body[1]).have.property('_id', '400000000000000000000003'); done(); }); }); @@ -167,7 +167,7 @@ describe('/sample', () => { it('returns an empty array for a page number out of negative range', done => { TestHelper.request(server, done, { method: 'get', - url: '/samples?status=all&to-page=-100&page-size=3&last-id=400000000000000000000004', + url: '/samples?status=all&to-page=-100&page-size=3&from-id=400000000000000000000004', auth: {basic: 'janedoe'}, httpStatus: 200 }).end((err, res) => { @@ -186,13 +186,13 @@ describe('/sample', () => { res: {status: 'Invalid body format', details: '"page-size" must be larger than or equal to 1'} }); }); - it('rejects an invalid last-id', done => { + it('rejects an invalid from-id', done => { TestHelper.request(server, done, { method: 'get', - url: '/samples?last-id=40000000000h000000000002', + url: '/samples?from-id=40000000000h000000000002', auth: {basic: 'janedoe'}, httpStatus: 400, - res: {status: 'Invalid body format', details: '"last-id" with value "40000000000h000000000002" fails to match the required pattern: /[0-9a-f]{24}/'} + res: {status: 'Invalid body format', details: '"from-id" with value "40000000000h000000000002" fails to match the required pattern: /[0-9a-f]{24}/'} }); }); it('rejects a to-page without page-size', done => { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 24f33cf..156725b 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -43,18 +43,18 @@ router.get('/samples', (req, res, next) => { query.limit(filters['page-size']); } - if (filters['last-id']) { + if (filters['from-id']) { if (filters['to-page'] && filters['to-page'] < 0) { - query.lte('_id', mongoose.Types.ObjectId(filters['last-id'])); // TODO: consider sorting + query.lt('_id', mongoose.Types.ObjectId(filters['from-id'])); // TODO: consider sorting query.sort({_id: -1}); } else { - query.gt('_id', mongoose.Types.ObjectId(filters['last-id'])); // TODO: consider sorting + query.gte('_id', mongoose.Types.ObjectId(filters['from-id'])); // TODO: consider sorting } } if (filters['to-page']) { - query.skip(Math.abs(filters['to-page']) * filters['page-size']); // TODO: check order for negative numbers + query.skip(Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size']); // TODO: check order for negative numbers } query.lean().exec((err, data) => { @@ -62,6 +62,7 @@ router.get('/samples', (req, res, next) => { if (filters['to-page'] && filters['to-page'] < 0) { data.reverse(); } + console.log(data); res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors }) }); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index f1f17f9..4b10a7c 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -130,7 +130,7 @@ export default class SampleValidate { static query (data) { return Joi.object({ status: Joi.string().valid('validated', 'new', 'all'), - 'last-id': IdValidate.get(), + 'from-id': IdValidate.get(), 'to-page': Joi.number().integer(), 'page-size': Joi.number().integer().min(1) }).with('to-page', 'page-size').validate(data); From 49f7a475b714303de23ace51142bc29835c82888 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 25 Jun 2020 14:29:54 +0200 Subject: [PATCH 06/19] added /samples/count --- api/sample.yaml | 19 +++++++++++++++++++ src/routes/sample.spec.ts | 36 ++++++++++++++++++++++++++++++++++++ src/routes/sample.ts | 10 +++++++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/api/sample.yaml b/api/sample.yaml index b37702d..bb5d9be 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -71,6 +71,25 @@ 500: $ref: 'api.yaml#/components/responses/500' +/samples/count: + get: + summary: total number of samples + description: 'Auth: all, levels: read, write, maintain, dev, admin' + tags: + - /sample + responses: + 200: + description: sample count + content: + application/json: + schema: + properties: + count: + type: number + example: 864 + 500: + $ref: 'api.yaml#/components/responses/500' + /sample/{id}: parameters: - $ref: 'api.yaml#/components/parameters/Id' diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index bd3cb79..bdda00e 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -318,6 +318,42 @@ describe('/sample', () => { }); }); + describe('GET /samples/count', () => { + it('returns the correct number of samples', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples/count', + auth: {basic: 'admin'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body.count).be.eql(json.collections.samples.length); + done(); + }); + }); + it('works with an API key', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples/count', + auth: {key: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body.count).be.eql(json.collections.samples.length); + done(); + }); + }); + it('rejects unauthorized requests', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples/count', + httpStatus: 401 + }); + }); + }); + describe('GET /sample/{id}', () => { it('returns the right sample', done => { TestHelper.request(server, done, { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 156725b..19ec993 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -62,7 +62,6 @@ router.get('/samples', (req, res, next) => { if (filters['to-page'] && filters['to-page'] < 0) { data.reverse(); } - console.log(data); res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors }) }); @@ -76,6 +75,15 @@ router.get('/samples/:state(new|deleted)', (req, res, next) => { }); }); +router.get('/samples/count', (req, res, next) => { + if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; + + SampleModel.estimatedDocumentCount((err, data) => { + if (err) return next(err); + res.json({count: data}); + }); +}); + router.get('/sample/' + IdValidate.parameter(), (req, res, next) => { if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; From 43413001e9c8c569b4f0b77885fed67201a015c8 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Fri, 26 Jun 2020 09:38:28 +0200 Subject: [PATCH 07/19] sorting for direct sample properties added --- api/sample.yaml | 6 ++++ src/routes/sample.spec.ts | 53 ++++++++++++++++++++++++++++++++++ src/routes/sample.ts | 54 ++++++++++++++++++++++++++--------- src/routes/validate/sample.ts | 3 +- 4 files changed, 101 insertions(+), 15 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index bb5d9be..5f07b78 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -30,6 +30,12 @@ schema: type: string example: 30 + - name: sort + description: sorting of results, in format 'key-asc/desc' + in: query + schema: + type: string + example: color-asc responses: 200: description: samples overview diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index bdda00e..5a32356 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -177,6 +177,59 @@ describe('/sample', () => { done(); }); }); + it('sorts the samples ascending', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&sort=color-asc', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.property('color', 'black'); + should(res.body[res.body.length - 1]).have.property('color', 'natural'); + done(); + }); + }); + it('sorts the samples descending', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&sort=number-desc', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.property('number', 'Rng36'); + should(res.body[1]).have.property('number', '33'); + should(res.body[res.body.length - 1]).have.property('number', '1'); + done(); + }); + }); + it('sorts the samples correctly in combination with paging', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&sort=color-asc&page-size=2&from-id=400000000000000000000006', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.property('_id', '400000000000000000000006'); + should(res.body[1]).have.property('_id', '400000000000000000000002'); + done(); + }); + }); + it('sorts the samples correctly in combination with going pages backward', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&sort=color-desc&page-size=2&from-id=400000000000000000000004&to-page=-1', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.property('_id', '400000000000000000000002'); + should(res.body[1]).have.property('_id', '400000000000000000000006'); + done(); + }); + }); it('rejects a negative page size', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 19ec993..7bc1b67 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -19,7 +19,7 @@ import db from '../db'; const router = express.Router(); -router.get('/samples', (req, res, next) => { +router.get('/samples', async (req, res, next) => { if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; const {error, value: filters} = SampleValidate.query(req.query); @@ -37,29 +37,55 @@ router.get('/samples', (req, res, next) => { else { // default status = {status: globals.status.validated}; } - const query = SampleModel.find(status); + + + const sort = []; + let paging = {} + + // sorting + filters.sort = filters.sort.split('-'); + filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id + filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1; + + if (!filters['to-page']) { // set to-page default + filters['to-page'] = 0; + } + + if (filters['from-id']) { // from-id specified + const fromSample = SampleValidate.output(await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);})); + if (fromSample instanceof Error) return; + + if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc + paging = {$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: mongoose.Types.ObjectId(filters['from-id'])}}]}]}; + sort.push([filters.sort[0], 1]); + sort.push(['_id', 1]); + } + else { + paging = {$or: [{[filters.sort[0]]: {$lt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$lte: mongoose.Types.ObjectId(filters['from-id'])}}]}]}; + sort.push([filters.sort[0], -1]); + sort.push(['_id', -1]); + } + } + else { // sort from beginning + sort.push([filters.sort[0], filters.sort[1]]); // set _id as secondary sort + sort.push(['_id', filters.sort[1]]); // set _id as secondary sort + } + + const query = SampleModel.find({$and: [status, paging]}); if (filters['page-size']) { query.limit(filters['page-size']); } - if (filters['from-id']) { - if (filters['to-page'] && filters['to-page'] < 0) { - query.lt('_id', mongoose.Types.ObjectId(filters['from-id'])); // TODO: consider sorting - query.sort({_id: -1}); - } - else { - query.gte('_id', mongoose.Types.ObjectId(filters['from-id'])); // TODO: consider sorting - } + if (filters['to-page']) { + query.skip(Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)); } - if (filters['to-page']) { - query.skip(Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size']); // TODO: check order for negative numbers - } + query.sort(sort); query.lean().exec((err, data) => { if (err) return next(err); - if (filters['to-page'] && filters['to-page'] < 0) { + if (filters['to-page'] < 0) { data.reverse(); } res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 4b10a7c..7a706c2 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -132,7 +132,8 @@ export default class SampleValidate { status: Joi.string().valid('validated', 'new', 'all'), 'from-id': IdValidate.get(), 'to-page': Joi.number().integer(), - 'page-size': Joi.number().integer().min(1) + 'page-size': Joi.number().integer().min(1), + sort: Joi.string().pattern(/^(_id|color|number|type|batch|added)-(asc|desc)$/m).default('_id-asc') // TODO: material keys }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file From 8aa051f0bdb35c85209c274742ab31b9d64345eb Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Fri, 26 Jun 2020 15:23:29 +0200 Subject: [PATCH 08/19] switched to aggregation, included material sort keys --- src/routes/sample.spec.ts | 14 +++++++++ src/routes/sample.ts | 54 +++++++++++++++++++---------------- src/routes/validate/sample.ts | 2 +- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 5a32356..0aa1c1f 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -230,6 +230,20 @@ describe('/sample', () => { done(); }); }); + it('sorts the samples correctly for material keys', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&sort=material.name-desc', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.property('_id', '400000000000000000000002'); + should(res.body[1]).have.property('_id', '400000000000000000000006'); + should(res.body[2]).have.property('_id', '400000000000000000000001'); + done(); + }); + }); it('rejects a negative page size', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 7bc1b67..4a04fc3 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -25,65 +25,69 @@ router.get('/samples', async (req, res, next) => { const {error, value: filters} = SampleValidate.query(req.query); if (error) return res400(error, res); - let status; + const query = []; + query.push({$match: {$and: []}}); if (filters.hasOwnProperty('status')) { if(filters.status === 'all') { - status = {$or: [{status: globals.status.validated}, {status: globals.status.new}]} + query[0].$match.$and.push({$or: [{status: globals.status.validated}, {status: globals.status.new}]}); } else { - status = {status: globals.status[filters.status]}; + query[0].$match.$and.push({status: globals.status[filters.status]}); } } else { // default - status = {status: globals.status.validated}; + query[0].$match.$and.push({status: globals.status.validated}); } - - const sort = []; - let paging = {} - // sorting filters.sort = filters.sort.split('-'); filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1; - if (!filters['to-page']) { // set to-page default filters['to-page'] = 0; } + if (filters.sort[0].indexOf('material.') >= 0) { // need to populate materials, material supplier and group + query.push( + {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, + {$set: {material: { $arrayElemAt: ['$material', 0]}}}, + {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, + {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}, + {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + ); + } + if (filters['from-id']) { // from-id specified - const fromSample = SampleValidate.output(await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);})); + const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); if (fromSample instanceof Error) return; if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc - paging = {$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: mongoose.Types.ObjectId(filters['from-id'])}}]}]}; - sort.push([filters.sort[0], 1]); - sort.push(['_id', 1]); + query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); + query.push({$sort: {[filters.sort[0]]: 1, _id: 1}}); } else { - paging = {$or: [{[filters.sort[0]]: {$lt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$lte: mongoose.Types.ObjectId(filters['from-id'])}}]}]}; - sort.push([filters.sort[0], -1]); - sort.push(['_id', -1]); + query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$lt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); + query.push({$sort: {[filters.sort[0]]: -1, _id: -1}}); } } else { // sort from beginning - sort.push([filters.sort[0], filters.sort[1]]); // set _id as secondary sort - sort.push(['_id', filters.sort[1]]); // set _id as secondary sort + query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}}); // set _id as secondary sort } - const query = SampleModel.find({$and: [status, paging]}); - - if (filters['page-size']) { - query.limit(filters['page-size']); + if (filters.sort[0].indexOf('material.') >= 0) { // unpopulate materials again + query.push({$unset: 'material'}); } if (filters['to-page']) { - query.skip(Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)); + query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more } - query.sort(sort); + if (filters['page-size']) { + query.push({$limit: filters['page-size']}); + } - query.lean().exec((err, data) => { + SampleModel.aggregate(query).exec((err, data) => { if (err) return next(err); if (filters['to-page'] < 0) { data.reverse(); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 7a706c2..b55b847 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -133,7 +133,7 @@ export default class SampleValidate { 'from-id': IdValidate.get(), 'to-page': Joi.number().integer(), 'page-size': Joi.number().integer().min(1), - sort: Joi.string().pattern(/^(_id|color|number|type|batch|added)-(asc|desc)$/m).default('_id-asc') // TODO: material keys + sort: Joi.string().pattern(/^(_id|color|number|type|batch|added|material\.name|material\.supplier|material\.group|material\.mineral|material\.glass_fiber|material\.carbon_fiber)-(asc|desc)$/m).default('_id-asc') // TODO: material keys }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file From e5cc661928f7b844f631c5b228513835329710d8 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 29 Jun 2020 12:27:39 +0200 Subject: [PATCH 09/19] base for csv export --- api/sample.yaml | 6 ++++++ package-lock.json | 22 ++++++++++++++++++++++ package.json | 1 + src/helpers/authorize.ts | 4 +++- src/helpers/csv.ts | 7 +++++++ src/index.ts | 3 +-- src/routes/sample.spec.ts | 15 +++++++++++++++ src/routes/sample.ts | 17 +++++++++++++++-- src/routes/validate/sample.ts | 3 ++- src/test/helper.ts | 26 +++++++++++++++----------- 10 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 src/helpers/csv.ts diff --git a/api/sample.yaml b/api/sample.yaml index 5f07b78..c6e59a1 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -36,6 +36,12 @@ schema: type: string example: color-asc + - name: csv + description: output as csv + in: query + schema: + type: boolean + example: false responses: 200: description: samples overview diff --git a/package-lock.json b/package-lock.json index 93fdea0..5478eef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2203,6 +2203,23 @@ "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.5.tgz", "integrity": "sha512-gWJOWYFrhQ8j7pVm0EM8Slr+EPVq1Phf6lvzvD/WCeqkrx/f2xBI0xOsRRS9xCn3I4vKtP519dvs3TP09r24wQ==" }, + "json2csv": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json2csv/-/json2csv-5.0.1.tgz", + "integrity": "sha512-QFMifUX1y8W2tKi2TwZpnzf2rHdZvzdmgZUMEMDF46F90f4a9mUeWfx/qg4kzXSZYJYc3cWA5O+eLXk5lj9g8g==", + "requires": { + "commander": "^5.0.0", + "jsonparse": "^1.3.1", + "lodash.get": "^4.4.2" + }, + "dependencies": { + "commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" + } + } + }, "json5": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.3.tgz", @@ -2212,6 +2229,11 @@ "minimist": "^1.2.5" } }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" + }, "jszip": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.5.0.tgz", diff --git a/package.json b/package.json index e5ca620..4b04218 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "express": "^4.17.1", "helmet": "^3.22.0", "json-schema": "^0.2.5", + "json2csv": "^5.0.1", "lodash": "^4.17.15", "mongo-sanitize": "^1.1.0", "mongoose": "^5.8.7", diff --git a/src/helpers/authorize.ts b/src/helpers/authorize.ts index 71a42c2..03d344b 100644 --- a/src/helpers/authorize.ts +++ b/src/helpers/authorize.ts @@ -89,7 +89,9 @@ function key (req, next): any { // checks API key and returns changed user obje if (err) return next(err); if (data.length === 1) { // one user found resolve({level: data[0].level, name: data[0].name, id: data[0]._id.toString(), location: data[0].location}); - delete req.query.key; // delete query parameter to avoid interference with later validation + if (!/^\/api/m.test(req.url)){ + delete req.query.key; // delete query parameter to avoid interference with later validation + } } else { resolve(null); diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts new file mode 100644 index 0000000..dbeb213 --- /dev/null +++ b/src/helpers/csv.ts @@ -0,0 +1,7 @@ +import {parseAsync} from 'json2csv'; + +export default function csv(input: object, fields: string[], f: (err, data) => void) { + parseAsync(input) + .then(csv => f(null, csv)) + .catch(err => f(err, null)); +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 9af77cf..97a080e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -48,9 +48,8 @@ app.use(require('./helpers/authorize')); // handle authentication // redirect /api routes for Angular proxy in development if (process.env.NODE_ENV !== 'production') { - app.use('/api/:url([^]+)', (req, res) => { + app.use('/api/:url([^]+)', (req, ignore) => { req.url = '/' + req.params.url; - app.handle(req, res); }); } diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 0aa1c1f..e7f6cfb 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -244,6 +244,21 @@ describe('/sample', () => { done(); }); }); + it('returns a correct csv file if specified', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&page-size=2&csv=true', + contentType: /text\/csv/, + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.text).be.eql('"_id","number","type","color","batch","condition","material_id","note_id","user_id","added"\r\n' + + '"400000000000000000000001","1","granulate","black","","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' + + '"400000000000000000000002","21","granulate","natural","1560237365","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"'); + done(); + }); + }); it('rejects a negative page size', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 4a04fc3..82a4eea 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -15,6 +15,7 @@ import ConditionTemplateModel from '../models/condition_template'; import ParametersValidate from './validate/parameters'; import globals from '../globals'; import db from '../db'; +import csv from '../helpers/csv'; const router = express.Router(); @@ -54,7 +55,10 @@ router.get('/samples', async (req, res, next) => { {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}, {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}, + {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]} + } + } ); } @@ -92,7 +96,16 @@ router.get('/samples', async (req, res, next) => { if (filters['to-page'] < 0) { data.reverse(); } - res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors + if (filters.csv) { // output as csv // TODO: csv example in OAS + csv(_.compact(data.map(e => SampleValidate.output(e))), ['_id', 'number'], (err, data) => { + if (err) return next(err); + res.set('Content-Type', 'text/csv'); + res.send(data); + }); + } + else { + res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors + } }) }); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index b55b847..d6c77a2 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -133,7 +133,8 @@ export default class SampleValidate { 'from-id': IdValidate.get(), 'to-page': Joi.number().integer(), 'page-size': Joi.number().integer().min(1), - sort: Joi.string().pattern(/^(_id|color|number|type|batch|added|material\.name|material\.supplier|material\.group|material\.mineral|material\.glass_fiber|material\.carbon_fiber)-(asc|desc)$/m).default('_id-asc') // TODO: material keys + sort: Joi.string().pattern(/^(_id|color|number|type|batch|added|material\.name|material\.supplier|material\.group|material\.mineral|material\.glass_fiber|material\.carbon_fiber|material\.number)-(asc|desc)$/m).default('_id-asc'), + csv: Joi.boolean().default(false) }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file diff --git a/src/test/helper.ts b/src/test/helper.ts index e1e8eec..44085f7 100644 --- a/src/test/helper.ts +++ b/src/test/helper.ts @@ -38,15 +38,7 @@ export default class TestHelper { return server } - static afterEach (server, done) { - server.close(done); - } - - static after(done) { - db.disconnect(done); - } - - static request (server, done, options) { // options in form: {method, url, auth: {key/basic: 'name' or 'key'/{name, pass}}, httpStatus, req, res, default (set to false if you want to dismiss default .end handling)} + static request (server, done, options) { // options in form: {method, url, contentType, auth: {key/basic: 'name' or 'key'/{name, pass}}, httpStatus, req, res, default (set to false if you want to dismiss default .end handling)} let st = supertest(server); if (options.hasOwnProperty('auth') && options.auth.hasOwnProperty('key')) { // resolve API key options.url += '?key=' + (this.auth.hasOwnProperty(options.auth.key)? this.auth[options.auth.key].key : options.auth.key); @@ -79,8 +71,12 @@ export default class TestHelper { st = st.auth(options.auth.basic.name, options.auth.basic.pass) } } - st = st.expect('Content-type', /json/) - .expect(options.httpStatus); + if (options.hasOwnProperty('contentType')) { + st = st.expect('Content-type', options.contentType).expect(options.httpStatus); + } + else { + st = st.expect('Content-type', /json/).expect(options.httpStatus); + } if (options.hasOwnProperty('res')) { // evaluate result return st.end((err, res) => { if (err) return done (err); @@ -128,4 +124,12 @@ export default class TestHelper { return st; } } + + static afterEach (server, done) { + server.close(done); + } + + static after(done) { + db.disconnect(done); + } } \ No newline at end of file From 52eb828beadba70b490595ef377ffee10fd49258 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 29 Jun 2020 15:50:24 +0200 Subject: [PATCH 10/19] first implementation of fields --- api/sample.yaml | 8 ++++++- src/db.ts | 2 +- src/index.ts | 3 ++- src/routes/sample.spec.ts | 29 +++++++++++++++++++++++- src/routes/sample.ts | 42 +++++++++++++++++++++-------------- src/routes/validate/sample.ts | 39 ++++++++++++++++++++++++++++---- 6 files changed, 98 insertions(+), 25 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index c6e59a1..f527513 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -42,9 +42,15 @@ schema: type: boolean example: false + - name: fields + description: the fields to include in the output as array, defaults to ['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'] + in: query + schema: + type: string + example: '&fields[]=number&fields[]=batch' responses: 200: - description: samples overview + description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) content: application/json: schema: diff --git a/src/db.ts b/src/db.ts index 60dadf9..cb11af5 100644 --- a/src/db.ts +++ b/src/db.ts @@ -3,7 +3,7 @@ import cfenv from 'cfenv'; import _ from 'lodash'; import ChangelogModel from './models/changelog'; -// mongoose.set('debug', true); // enable mongoose debug +mongoose.set('debug', true); // enable mongoose debug // database urls, prod db url is retrieved automatically const TESTING_URL = 'mongodb://localhost/dfopdb_test'; diff --git a/src/index.ts b/src/index.ts index 97a080e..9af77cf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -48,8 +48,9 @@ app.use(require('./helpers/authorize')); // handle authentication // redirect /api routes for Angular proxy in development if (process.env.NODE_ENV !== 'production') { - app.use('/api/:url([^]+)', (req, ignore) => { + app.use('/api/:url([^]+)', (req, res) => { req.url = '/' + req.params.url; + app.handle(req, res); }); } diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index e7f6cfb..29e35b7 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -259,6 +259,33 @@ describe('/sample', () => { done(); }); }); + it('returns only the fields specified', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&page-size=1&fields[]=number&fields[]=condition&fields[]=color&fields[]=material.name&fields[]=material.mineral', + auth: {basic: 'janedoe'}, + httpStatus: 200, + res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', mineral: 0}}] + }); + }); + it('rejects an invalid fields parameter', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&page-size=1&fields=number', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"fields" must be an array'} + }); + }); + it('rejects an unknown field name', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&page-size=1&fields[]=xx', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"fields[0]" must be one of [_id, color, number, type, batch, added, material.name, material.supplier, material.group, material.mineral, material.glass_fiber, material.carbon_fiber, material.number, condition, material_id, note_id, user_id, material._id, material.numbers]'} + }); + }); it('rejects a negative page size', done => { TestHelper.request(server, done, { method: 'get', @@ -302,7 +329,7 @@ describe('/sample', () => { httpStatus: 401 }); }); - }); + }); // TODO: measurement fields describe('GET /samples/{state}', () => { it('returns all new samples', done => { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 82a4eea..f356211 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -48,19 +48,17 @@ router.get('/samples', async (req, res, next) => { filters['to-page'] = 0; } - if (filters.sort[0].indexOf('material.') >= 0) { // need to populate materials, material supplier and group - query.push( - {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, - {$set: {material: { $arrayElemAt: ['$material', 0]}}}, - {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}, - {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}, - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]} - } + query.push( + {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, + {$set: {material: { $arrayElemAt: ['$material', 0]}}}, + {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, + {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}, + {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}, + {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]} } - ); - } + } + ); if (filters['from-id']) { // from-id specified const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); @@ -79,9 +77,9 @@ router.get('/samples', async (req, res, next) => { query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}}); // set _id as secondary sort } - if (filters.sort[0].indexOf('material.') >= 0) { // unpopulate materials again - query.push({$unset: 'material'}); - } + // if (filters.sort[0].indexOf('material.') >= 0) { // unpopulate materials again + // query.push({$unset: 'material'}); + // } if (filters['to-page']) { query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more @@ -90,6 +88,16 @@ router.get('/samples', async (req, res, next) => { if (filters['page-size']) { query.push({$limit: filters['page-size']}); } + console.log(filters.fields); + const projection = filters.fields.reduce((s, e) => {s[e] = true; return s; }, {}); + if (filters.fields.indexOf('added') >= 0) { // add added date + projection.added = {$toDate: '$_id'}; + } + if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly + console.log('disable id'); + projection._id = false; + } + query.push({$project: projection}); SampleModel.aggregate(query).exec((err, data) => { if (err) return next(err); @@ -97,14 +105,14 @@ router.get('/samples', async (req, res, next) => { data.reverse(); } if (filters.csv) { // output as csv // TODO: csv example in OAS - csv(_.compact(data.map(e => SampleValidate.output(e))), ['_id', 'number'], (err, data) => { + csv(_.compact(data.map(e => SampleValidate.output(e, 'refs'))), ['_id', 'number'], (err, data) => { if (err) return next(err); res.set('Content-Type', 'text/csv'); res.send(data); }); } else { - res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors + res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs')))); // validate all and filter null values from validation errors } }) }); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index d6c77a2..520f91b 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -51,6 +51,32 @@ export default class SampleValidate { .min('1970-01-01T00:00:00.000Z') }; + private static sortKeys = [ + '_id', + 'color', + 'number', + 'type', + 'batch', + 'added', + 'material.name', + 'material.supplier', + 'material.group', + 'material.mineral', + 'material.glass_fiber', + 'material.carbon_fiber', + 'material.number' + ]; + + private static fieldKeys = [ + ...SampleValidate.sortKeys, + 'condition', + 'material_id', + 'note_id', + 'user_id', + 'material._id', + 'material.numbers' + ]; + static input (data, param) { // validate input, set param to 'new' to make all attributes required if (param === 'new') { return Joi.object({ @@ -88,8 +114,11 @@ export default class SampleValidate { } } - static output (data, param = 'refs') { // validate output and strip unwanted properties, returns null if not valid - data.added = data._id.getTimestamp(); + static output (data, param = 'refs+added') { // validate output and strip unwanted properties, returns null if not valid + if (param === 'refs+added') { + param = 'refs'; + data.added = data._id.getTimestamp(); + } data = IdValidate.stringify(data); let joiObject; if (param === 'refs') { @@ -101,6 +130,7 @@ export default class SampleValidate { batch: this.sample.batch, condition: this.sample.condition, material_id: IdValidate.get(), + material: MaterialValidate.outputV(), note_id: IdValidate.get().allow(null), user_id: IdValidate.get(), added: this.sample.added @@ -133,8 +163,9 @@ export default class SampleValidate { 'from-id': IdValidate.get(), 'to-page': Joi.number().integer(), 'page-size': Joi.number().integer().min(1), - sort: Joi.string().pattern(/^(_id|color|number|type|batch|added|material\.name|material\.supplier|material\.group|material\.mineral|material\.glass_fiber|material\.carbon_fiber|material\.number)-(asc|desc)$/m).default('_id-asc'), - csv: Joi.boolean().default(false) + sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'), + csv: Joi.boolean().default(false), + fields: Joi.array().items(Joi.string().valid(...this.fieldKeys)).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file From 8cf1c14d887811e48247f08239e7e6b20817fd93 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Tue, 30 Jun 2020 14:16:37 +0200 Subject: [PATCH 11/19] implementation of measurement fields --- api/sample.yaml | 8 +++-- data_import/import.js | 4 +-- src/helpers/csv.ts | 31 ++++++++++++++++++-- src/routes/sample.spec.ts | 55 +++++++++++++++++++++++++++++++---- src/routes/sample.ts | 46 ++++++++++++++++++++++------- src/routes/validate/sample.ts | 13 ++++++--- src/test/db.json | 14 +++++++++ 7 files changed, 145 insertions(+), 26 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index f527513..91c57e0 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -42,12 +42,14 @@ schema: type: boolean example: false - - name: fields + - name: fields[] description: the fields to include in the output as array, defaults to ['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'] in: query schema: - type: string - example: '&fields[]=number&fields[]=batch' + type: array + items: + type: string + example: ['number', 'batch'] responses: 200: description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) diff --git a/data_import/import.js b/data_import/import.js index 28614c8..a72ee0c 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -32,10 +32,10 @@ async function main() { await allSamples(); await saveSamples(); } - else if (0) { // DPT + else if (1) { // DPT await allDpts(); } - else if (1) { // KF/VZ + else if (0) { // KF/VZ await importCsv(); await allKfVz(); } diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts index dbeb213..a307ca5 100644 --- a/src/helpers/csv.ts +++ b/src/helpers/csv.ts @@ -1,7 +1,34 @@ import {parseAsync} from 'json2csv'; -export default function csv(input: object, fields: string[], f: (err, data) => void) { - parseAsync(input) +export default function csv(input: any[], f: (err, data) => void) { + parseAsync(input.map(e => flatten(e))) .then(csv => f(null, csv)) .catch(err => f(err, null)); +} + +function flatten (data) { // flatten object: {a: {b: true}} -> {a.b: true} + const result = {}; + function recurse (cur, prop) { + if (Object(cur) !== cur || Object.keys(cur).length === 0) { + result[prop] = cur; + } + else if (Array.isArray(cur)) { + let l = 0; + for(let i = 0, l = cur.length; i < l; i++) + recurse(cur[i], prop + "[" + i + "]"); + if (l == 0) + result[prop] = []; + } + else { + let isEmpty = true; + for (let p in cur) { + isEmpty = false; + recurse(cur[p], prop ? prop+"."+p : p); + } + if (isEmpty && prop) + result[prop] = {}; + } + } + recurse(data, ''); + return result; } \ No newline at end of file diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 29e35b7..df7c242 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -244,6 +244,42 @@ describe('/sample', () => { done(); }); }); + it('adds the specified measurements', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=measurements.kf', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body.find(e => e.number === '1')).have.property('kf', {}); + should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null}); + done(); + }); + }); + it('multiplies the sample information for each spectrum', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=measurements.spectrum', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(2); + should(res.body[0]).have.property('spectrum', [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]); + should(res.body[1]).have.property('spectrum', [[3996.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]); + done(); + }); + }); + it('rejects unknown measurement names', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=measurements.xx', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: 'Measurement key not found'} + }); + }); it('returns a correct csv file if specified', done => { TestHelper.request(server, done, { method: 'get', @@ -253,9 +289,9 @@ describe('/sample', () => { httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.text).be.eql('"_id","number","type","color","batch","condition","material_id","note_id","user_id","added"\r\n' + - '"400000000000000000000001","1","granulate","black","","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' + - '"400000000000000000000002","21","granulate","natural","1560237365","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"'); + should(res.text).be.eql('"_id","number","type","color","batch","condition.material","condition.weeks","condition.condition_template","material_id","note_id","user_id","added"\r\n' + + '"400000000000000000000001","1","granulate","black","","copper",3,"200000000000000000000001","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' + + '"400000000000000000000002","21","granulate","natural","1560237365","copper",3,"200000000000000000000001","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"'); done(); }); }); @@ -268,6 +304,15 @@ describe('/sample', () => { res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', mineral: 0}}] }); }); + it('rejects a from-id not in the database', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?from-id=5ea0450ed851c30a90e70894&sort=color-asc', + auth: {basic: 'admin'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: 'from-id not found'} + }); + }); it('rejects an invalid fields parameter', done => { TestHelper.request(server, done, { method: 'get', @@ -283,7 +328,7 @@ describe('/sample', () => { url: '/samples?status=all&page-size=1&fields[]=xx', auth: {basic: 'janedoe'}, httpStatus: 400, - res: {status: 'Invalid body format', details: '"fields[0]" must be one of [_id, color, number, type, batch, added, material.name, material.supplier, material.group, material.mineral, material.glass_fiber, material.carbon_fiber, material.number, condition, material_id, note_id, user_id, material._id, material.numbers]'} + res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\..+)$/m'} }); }); it('rejects a negative page size', done => { @@ -329,7 +374,7 @@ describe('/sample', () => { httpStatus: 401 }); }); - }); // TODO: measurement fields + }); describe('GET /samples/{state}', () => { it('returns all new samples', done => { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index f356211..eef30f9 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -6,6 +6,7 @@ import NoteFieldValidate from './validate/note_field'; import res400 from './validate/res400'; import SampleModel from '../models/sample' import MeasurementModel from '../models/measurement'; +import MeasurementTemplateModel from '../models/measurement_template'; import MaterialModel from '../models/material'; import NoteModel from '../models/note'; import NoteFieldModel from '../models/note_field'; @@ -63,6 +64,9 @@ router.get('/samples', async (req, res, next) => { if (filters['from-id']) { // from-id specified const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); if (fromSample instanceof Error) return; + if (!fromSample) { + return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); + } if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); @@ -77,10 +81,6 @@ router.get('/samples', async (req, res, next) => { query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}}); // set _id as secondary sort } - // if (filters.sort[0].indexOf('material.') >= 0) { // unpopulate materials again - // query.push({$unset: 'material'}); - // } - if (filters['to-page']) { query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more } @@ -88,13 +88,38 @@ router.get('/samples', async (req, res, next) => { if (filters['page-size']) { query.push({$limit: filters['page-size']}); } - console.log(filters.fields); - const projection = filters.fields.reduce((s, e) => {s[e] = true; return s; }, {}); + + let measurementFields = []; + if (filters.fields.find(e => /measurements\./.test(e))) { // joining measurements is required + query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); + measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', '')); + const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); + if (measurementTemplates instanceof Error) return; + if (measurementTemplates.length < measurementFields.length) { + return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); + } + measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later + query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, + in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} + }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}}); + }); + console.log(measurementFields); + if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well + query.push( + {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, + {$set: {spectrum: '$spectrum.values.dpt'}}, + {$unwind: '$spectrum'} + ); + } + query.push({$unset: 'measurements'}); + } + + const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date projection.added = {$toDate: '$_id'}; } if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly - console.log('disable id'); projection._id = false; } query.push({$project: projection}); @@ -104,15 +129,16 @@ router.get('/samples', async (req, res, next) => { if (filters['to-page'] < 0) { data.reverse(); } - if (filters.csv) { // output as csv // TODO: csv example in OAS - csv(_.compact(data.map(e => SampleValidate.output(e, 'refs'))), ['_id', 'number'], (err, data) => { + console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); + if (filters.csv) { // output as csv + csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { if (err) return next(err); res.set('Content-Type', 'text/csv'); res.send(data); }); } else { - res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs')))); // validate all and filter null values from validation errors + res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors } }) }); diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 520f91b..41435dd 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -71,10 +71,12 @@ export default class SampleValidate { ...SampleValidate.sortKeys, 'condition', 'material_id', + 'material', 'note_id', 'user_id', 'material._id', - 'material.numbers' + 'material.numbers', + 'measurements.*' ]; static input (data, param) { // validate input, set param to 'new' to make all attributes required @@ -114,7 +116,7 @@ export default class SampleValidate { } } - static output (data, param = 'refs+added') { // validate output and strip unwanted properties, returns null if not valid + static output (data, param = 'refs+added', additionalParams = []) { // validate output and strip unwanted properties, returns null if not valid if (param === 'refs+added') { param = 'refs'; data.added = data._id.getTimestamp(); @@ -130,7 +132,7 @@ export default class SampleValidate { batch: this.sample.batch, condition: this.sample.condition, material_id: IdValidate.get(), - material: MaterialValidate.outputV(), + material: MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}), note_id: IdValidate.get().allow(null), user_id: IdValidate.get(), added: this.sample.added @@ -153,6 +155,9 @@ export default class SampleValidate { else { return null; } + additionalParams.forEach(param => { + joiObject[param] = Joi.any(); + }); const {value, error} = Joi.object(joiObject).validate(data, {stripUnknown: true}); return error !== undefined? null : value; } @@ -165,7 +170,7 @@ export default class SampleValidate { 'page-size': Joi.number().integer().min(1), sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'), csv: Joi.boolean().default(false), - fields: Joi.array().items(Joi.string().valid(...this.fieldKeys)).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) + fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file diff --git a/src/test/db.json b/src/test/db.json index ef26a63..aa68283 100644 --- a/src/test/db.json +++ b/src/test/db.json @@ -417,6 +417,20 @@ "status": 0, "measurement_template": {"$oid":"300000000000000000000002"}, "__v": 0 + }, + { + "_id": {"$oid":"800000000000000000000007"}, + "sample_id": {"$oid":"400000000000000000000001"}, + "values": { + "dpt": [ + [3996.12558,98.00555], + [3995.08519,98.03253], + [3993.04480,98.02657] + ] + }, + "status": 10, + "measurement_template": {"$oid":"300000000000000000000001"}, + "__v": 0 } ], "condition_templates": [ From e4bc5a77f1f4f1e071a10c0c7c70b63be88f735e Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 2 Jul 2020 12:18:01 +0200 Subject: [PATCH 12/19] restructured aggregation --- data_import/import.js | 1 + src/helpers/csv.ts | 8 +- src/index.ts | 1 + src/models/material.ts | 2 + src/models/measurement.ts | 2 + src/models/sample.ts | 3 + src/routes/sample.spec.ts | 4 +- src/routes/sample.ts | 213 +++++++++++++++++++++++++--------- src/routes/validate/sample.ts | 7 +- 9 files changed, 180 insertions(+), 61 deletions(-) diff --git a/data_import/import.js b/data_import/import.js index a72ee0c..3d84160 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -16,6 +16,7 @@ let normMaster = {}; // TODO: integrate measurement device information from DPT names using different users // TODO: supplier: other for supplierless samples +// TODO: BASF twice, BASF as color main(); diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts index a307ca5..18e633c 100644 --- a/src/helpers/csv.ts +++ b/src/helpers/csv.ts @@ -1,7 +1,13 @@ import {parseAsync} from 'json2csv'; export default function csv(input: any[], f: (err, data) => void) { - parseAsync(input.map(e => flatten(e))) + console.log(input[1000]); + console.log(flatten(input[1000])); + parseAsync([flatten(input[1000])]).then(csv => console.log(csv)); + console.log(input[1]); + console.log(flatten(input[1])); + parseAsync([flatten(input[1])]).then(csv => console.log(csv)); + parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true}) .then(csv => f(null, csv)) .catch(err => f(err, null)); } diff --git a/src/index.ts b/src/index.ts index 9af77cf..8116de7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import helmet from 'helmet'; import api from './api'; import db from './db'; +// TODO: working demo branch // tell if server is running in debug or production environment console.info(process.env.NODE_ENV === 'production' ? '===== PRODUCTION =====' : process.env.NODE_ENV === 'test' ? '' :'===== DEVELOPMENT ====='); diff --git a/src/models/material.ts b/src/models/material.ts index bcebb83..d7d5eb9 100644 --- a/src/models/material.ts +++ b/src/models/material.ts @@ -22,5 +22,7 @@ MaterialSchema.query.log = function > db.log(req, this); return this; } +MaterialSchema.index({supplier_id: 1}); +MaterialSchema.index({group_id: 1}); export default mongoose.model>('material', MaterialSchema); \ No newline at end of file diff --git a/src/models/measurement.ts b/src/models/measurement.ts index 1136e6b..55267ec 100644 --- a/src/models/measurement.ts +++ b/src/models/measurement.ts @@ -17,5 +17,7 @@ MeasurementSchema.query.log = function >('measurement', MeasurementSchema); \ No newline at end of file diff --git a/src/models/sample.ts b/src/models/sample.ts index 0e457d8..8eec7bd 100644 --- a/src/models/sample.ts +++ b/src/models/sample.ts @@ -22,5 +22,8 @@ SampleSchema.query.log = function > ( db.log(req, this); return this; } +SampleSchema.index({material_id: 1}); +SampleSchema.index({note_id: 1}); +SampleSchema.index({user_id: 1}); export default mongoose.model>('sample', SampleSchema); \ No newline at end of file diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index df7c242..2f82bc9 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -252,7 +252,7 @@ describe('/sample', () => { httpStatus: 200 }).end((err, res) => { if (err) return done(err); - should(res.body.find(e => e.number === '1')).have.property('kf', {}); + should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null}); should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null}); done(); }); @@ -328,7 +328,7 @@ describe('/sample', () => { url: '/samples?status=all&page-size=1&fields[]=xx', auth: {basic: 'janedoe'}, httpStatus: 400, - res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\..+)$/m'} + res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'} }); }); it('rejects a negative page size', done => { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index eef30f9..7b2af04 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -27,21 +27,10 @@ router.get('/samples', async (req, res, next) => { const {error, value: filters} = SampleValidate.query(req.query); if (error) return res400(error, res); - const query = []; - query.push({$match: {$and: []}}); - if (filters.hasOwnProperty('status')) { - if(filters.status === 'all') { - query[0].$match.$and.push({$or: [{status: globals.status.validated}, {status: globals.status.new}]}); - } - else { - query[0].$match.$and.push({status: globals.status[filters.status]}); - } - } - else { // default - query[0].$match.$and.push({status: globals.status.validated}); - } + // TODO: find a better place for these + const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', 'user_id']; - // sorting + // evaluate sort parameter from 'color-asc' to ['color', 1] filters.sort = filters.sort.split('-'); filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1; @@ -49,62 +38,139 @@ router.get('/samples', async (req, res, next) => { filters['to-page'] = 0; } - query.push( - {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, - {$set: {material: { $arrayElemAt: ['$material', 0]}}}, - {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}, - {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}, - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]} + let collection; + const query = []; + query.push({$match: {$and: []}}); + + if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection + collection = MeasurementModel; + const measurementName = filters.sort[0].replace('measurements.', ''); + const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);}); + if (measurementTemplate instanceof Error) return; + if (!measurementTemplate) { + return res.status(400).json({status: 'Invalid body format', details: filters.sort[0] + ' not found'}); + } + let sortStartValue = null; + if (filters['from-id']) { // from-id specified, fetch values for sorting + const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}).lean().exec().catch(err => {next(err);}); // TODO: what if more than one measurement for sample? + if (fromSample instanceof Error) return; + if (!fromSample) { + return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); } + sortStartValue = fromSample.values[measurementTemplate.parameters[0].name]; } - ); + query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort + query.push( + sortQuery(query, filters, ['values.' + measurementTemplate.parameters[0].name, 'sample_id'], sortStartValue), // sort measurements + {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure + {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, + {$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added + ); + addSkipLimit(query, filters); // skip and limit to select right page + query.push( + {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring + {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} + ); - if (filters['from-id']) { // from-id specified - const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); - if (fromSample instanceof Error) return; - if (!fromSample) { - return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); - } + } + else { // sorting with samples as starting collection + collection = SampleModel; + // filter for status + query[0].$match.$and.push(statusQuery(filters, 'status')); - if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc - query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); - query.push({$sort: {[filters.sort[0]]: 1, _id: 1}}); + // differentiate by sort key to do sorting, skip and limit as early as possible + if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys + let sortStartValue = null; + if (filters['from-id']) { // from-id specified + const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); + if (fromSample instanceof Error) return; + if (!fromSample) { + return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); + } + sortStartValue = fromSample[filters.sort[0]]; + } + query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + addSkipLimit(query, filters); } - else { - query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$lt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); - query.push({$sort: {[filters.sort[0]]: -1, _id: -1}}); + else { // sorting for material keys + let materialQuery = [] + materialQuery.push( // add material properties + {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, + {$set: {material: { $arrayElemAt: ['$material', 0]}}} + ); + if (filters.sort[0] === 'material.supplier') { // add supplier if needed + materialQuery.push( + {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + ); + } + if (filters.sort[0] === 'material.group') { // add group if needed + materialQuery.push( + {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, + {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + ); + } + if (filters.sort[0] === 'material.number') { // add material number if needed + materialQuery.push( + {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + ); + } + query.push(...materialQuery); + let sortStartValue = null; + if (filters['from-id']) { // from-id specified + const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);}); + if (fromSample instanceof Error) return; + if (!fromSample) { + return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); + } + sortStartValue = fromSample[filters.sort[0]]; + } + query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + addSkipLimit(query, filters); } } - else { // sort from beginning - query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}}); // set _id as secondary sort + + const fieldsNoSort = filters.fields.filter(e => e !== filters.sort[0]); // sort field was definitely added already, exclude from further field operations + if (fieldsNoSort.find(e => /material\./.test(e))) { // add material fields + query.push( + {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, + {$set: {material: { $arrayElemAt: ['$material', 0]}}} + ); + } + if (fieldsNoSort.indexOf('material.supplier') >= 0) { // add supplier if needed + query.push( + {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + ); + } + if (fieldsNoSort.indexOf('material.group') >= 0) { // add group if needed + query.push( + {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, + {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + ); + } + if (fieldsNoSort.indexOf('material.number') >= 0) { // add material number if needed + query.push( + {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + ); } - if (filters['to-page']) { - query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more - } - - if (filters['page-size']) { - query.push({$limit: filters['page-size']}); - } - - let measurementFields = []; - if (filters.fields.find(e => /measurements\./.test(e))) { // joining measurements is required + let measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', '')); + console.log(fieldsNoSort); + console.log(measurementFields); + if (fieldsNoSort.find(e => /measurements\./.test(e))) { // add measurement fields query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); - measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', '')); - const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); + const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); if (measurementTemplates instanceof Error) return; if (measurementTemplates.length < measurementFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values - vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, - in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} - }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}}); + vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, + in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} + }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); - console.log(measurementFields); if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well query.push( {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, @@ -124,12 +190,11 @@ router.get('/samples', async (req, res, next) => { } query.push({$project: projection}); - SampleModel.aggregate(query).exec((err, data) => { + collection.aggregate(query).exec((err, data) => { if (err) return next(err); if (filters['to-page'] < 0) { data.reverse(); } - console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); if (filters.csv) { // output as csv csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { if (err) return next(err); @@ -498,4 +563,42 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and } }); }); +} + +function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] + if (filters['from-id']) { // from-id specified + if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc + query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); + return {$sort: {[sortKeys[0]]: 1, _id: 1}}; + } else { + query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); + return {$sort: {[sortKeys[0]]: -1, _id: -1}}; + } + } else { // sort from beginning + return {$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}; // set _id as secondary sort + } +} + +function addSkipLimit(query, filters) { + if (filters['to-page']) { + query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more + } + + if (filters['page-size']) { + query.push({$limit: filters['page-size']}); + } +} + +function statusQuery(filters, field) { + if (filters.hasOwnProperty('status')) { + if(filters.status === 'all') { + return {$or: [{[field]: globals.status.validated}, {[field]: globals.status.new}]}; + } + else { + return {[field]: globals.status[filters.status]}; + } + } + else { // default + return {[field]: globals.status.validated}; + } } \ No newline at end of file diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 41435dd..16bc8a1 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -64,7 +64,8 @@ export default class SampleValidate { 'material.mineral', 'material.glass_fiber', 'material.carbon_fiber', - 'material.number' + 'material.number', + 'measurements.(?!spectrum)*' ]; private static fieldKeys = [ @@ -76,7 +77,7 @@ export default class SampleValidate { 'user_id', 'material._id', 'material.numbers', - 'measurements.*' + 'measurements.spectrum' ]; static input (data, param) { // validate input, set param to 'new' to make all attributes required @@ -168,7 +169,7 @@ export default class SampleValidate { 'from-id': IdValidate.get(), 'to-page': Joi.number().integer(), 'page-size': Joi.number().integer().min(1), - sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'), + sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'), csv: Joi.boolean().default(false), fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) }).with('to-page', 'page-size').validate(data); From 29eefce0c9b6a641690a1089fe43f20a80a8f0ee Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 6 Jul 2020 09:43:04 +0200 Subject: [PATCH 13/19] added filters --- api/sample.yaml | 8 +++ data_import/import.js | 2 + src/routes/sample.spec.ts | 105 +++++++++++++++++++++++++++++++++- src/routes/sample.ts | 57 +++++++++++++----- src/routes/validate/sample.ts | 19 +++++- src/test/db.json | 2 +- 6 files changed, 176 insertions(+), 17 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index 91c57e0..acdd33c 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -50,6 +50,14 @@ items: type: string example: ['number', 'batch'] + - name: filters[] + description: "the filters to apply as an array of URIComponent encoded objects in the form {mode: 'eq/ne/lt/lte/gt/gte/in/nin', field: 'material.m', values: ['15']} using encodeURIComponent(JSON.stringify({}))" + in: query + schema: + type: array + items: + type: string + example: ["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D", "%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22%5D%7D"] responses: 200: description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) diff --git a/data_import/import.js b/data_import/import.js index 3d84160..4f31f8b 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -17,6 +17,8 @@ let normMaster = {}; // TODO: integrate measurement device information from DPT names using different users // TODO: supplier: other for supplierless samples // TODO: BASF twice, BASF as color +// TODO: trim color names +// TODO: duplicate kf values main(); diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 2f82bc9..d15200b 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -20,6 +20,7 @@ describe('/sample', () => { after(done => TestHelper.after(done)); // TODO: sort, added date filter, has measurements/condition filter + // TODO: check if conditions work in sort/fields/filters describe('GET /samples', () => { it('returns all samples', done => { TestHelper.request(server, done, { @@ -253,7 +254,7 @@ describe('/sample', () => { }).end((err, res) => { if (err) return done(err); should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null}); - should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null}); + should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.6, 'standard deviation': null}); done(); }); }); @@ -271,6 +272,108 @@ describe('/sample', () => { done(); }); }); + it('filters a sample property', done => { // TODO: implement filters + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=type&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22type%22%2C%22values%22%3A%5B%22part%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.samples.filter(e => e.type === 'part').length); + should(res.body).matchEach(sample => { + should(sample).have.property('type', 'part'); + }); + done(); + }); + }); + it('filters a material property', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22in%22%2C%22field%22%3A%22material.name%22%2C%22values%22%3A%5B%22Schulamid%2066%20GF%2025%20H%22%2C%22Stanyl%20TW%20200%20F8%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.samples.filter(e => e.material_id == '100000000000000000000004' || e.material_id == '100000000000000000000001').length); + should(res.body).matchEach(sample => { + should(sample.material.name).be.equalOneOf('Schulamid 66 GF 25 H', 'Stanyl TW 200 F8'); + }); + done(); + }); + }); + it('filters by measurement value', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=measurements.kf.weight%20%25&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length); + should(res.body).matchEach(sample => { + should(sample.kf['weight %']).be.above(0.5); + }); + done(); + }); + }); + it('filters by measurement value not in the fields', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + const json = require('../test/db.json'); + should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length); + should(res.body[0]).have.property('number', 'Rng36'); + done(); + }); + }); + it('filters multiple properties', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.glass_fiber%22%2C%22values%22%3A%5B33%5D%7D&filters[]=%7B%22mode%22%3A%22lte%22%2C%22field%22%3A%22number%22%2C%22values%22%3A%5B%22Rng33%22%5D%7D&filters[]=%7B%22mode%22%3A%22nin%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body).have.lengthOf(1); + should(res.body[0]).be.eql({number: '32', material: {glass_fiber: 33}, batch: '1653000308'}); + done(); + }); + }); // TODO: do measurement pipeline, check if it works with UI + it('rejects an invalid JSON string as a filters parameter', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=xx', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: 'Invalid JSON string for filter parameter'} + }); + }); + it('rejects an invalid filter mode', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'} + }); + }); + it('rejects an filter field not existing', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22xx%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', + auth: {basic: 'janedoe'}, + httpStatus: 400, + res: {status: 'Invalid body format', details: '"filters[0].field" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'} + }); + }); it('rejects unknown measurement names', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 7b2af04..240c156 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -44,7 +44,7 @@ router.get('/samples', async (req, res, next) => { if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection collection = MeasurementModel; - const measurementName = filters.sort[0].replace('measurements.', ''); + const [,measurementName, measurementParam] = filters.sort[0].split('.'); const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);}); if (measurementTemplate instanceof Error) return; if (!measurementTemplate) { @@ -57,11 +57,14 @@ router.get('/samples', async (req, res, next) => { if (!fromSample) { return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); } - sortStartValue = fromSample.values[measurementTemplate.parameters[0].name]; + sortStartValue = fromSample.values[measurementParam]; } query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort + if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered + query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); + } query.push( - sortQuery(query, filters, ['values.' + measurementTemplate.parameters[0].name, 'sample_id'], sortStartValue), // sort measurements + sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, {$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added @@ -71,12 +74,13 @@ router.get('/samples', async (req, res, next) => { {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); - + addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters } else { // sorting with samples as starting collection collection = SampleModel; // filter for status query[0].$match.$and.push(statusQuery(filters, 'status')); + addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters // differentiate by sort key to do sorting, skip and limit as early as possible if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys @@ -90,6 +94,7 @@ router.get('/samples', async (req, res, next) => { sortStartValue = fromSample[filters.sort[0]]; } query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + // material filters addSkipLimit(query, filters); } else { // sorting for material keys @@ -130,41 +135,54 @@ router.get('/samples', async (req, res, next) => { } } - const fieldsNoSort = filters.fields.filter(e => e !== filters.sort[0]); // sort field was definitely added already, exclude from further field operations - if (fieldsNoSort.find(e => /material\./.test(e))) { // add material fields + const fieldsToAdd = [ + ...filters.fields, + ...filters.filters.map(e => e.field) // add filter fields in case they were not specified to display + ].filter(e => e !== filters.sort[0]) // sort field was definitely added already, exclude from further field operations + .filter((e, i, self) => self.indexOf(e) === i); // remove duplicates + if (fieldsToAdd.find(e => /material\./.test(e))) { // add material fields query.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$set: {material: { $arrayElemAt: ['$material', 0]}}} ); } - if (fieldsNoSort.indexOf('material.supplier') >= 0) { // add supplier if needed + if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed query.push( {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); } - if (fieldsNoSort.indexOf('material.group') >= 0) { // add group if needed + if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed query.push( {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } - if (fieldsNoSort.indexOf('material.number') >= 0) { // add material number if needed + if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed query.push( {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} ); } - let measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', '')); - console.log(fieldsNoSort); - console.log(measurementFields); - if (fieldsNoSort.find(e => /measurements\./.test(e))) { // add measurement fields - query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); + addFilterQueries(query, filters.filters.filter(e => /material\./.test(e.field))); // material filters + + let measurementFields = fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]).filter((e, i, self) => self.indexOf(e) === i); // filter measurement names and remove duplicates from parameters + if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); if (measurementTemplates instanceof Error) return; if (measurementTemplates.length < measurementFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } + if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance + query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); + } + else { + query.push({$lookup: { + from: 'measurements', let: {sId: '$_id'}, + pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], + as: 'measurements' + }}); + } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, @@ -180,6 +198,7 @@ router.get('/samples', async (req, res, next) => { } query.push({$unset: 'measurements'}); } + addFilterQueries(query, filters.filters.filter(e => /measurements\./.test(e.field)).map(e => {e.field = e.field.replace('measurements.', ''); return e; })); // measurement filters const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date @@ -601,4 +620,14 @@ function statusQuery(filters, field) { else { // default return {[field]: globals.status.validated}; } +} + +function addFilterQueries (query, filters) { // returns array of match queries from given filters + if (filters.length) { + query.push({$match: {$and: filterQueries(filters)}}); + } +} + +function filterQueries (filters) { + return filters.map(e => ({[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}})) // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin } \ No newline at end of file diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 16bc8a1..b0cae01 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -164,6 +164,18 @@ export default class SampleValidate { } static query (data) { + if (data.filters && data.filters.length) { + const filterValidation = Joi.array().items(Joi.string()).validate(data.filters); + if (filterValidation.error) return filterValidation; + try { + for (let i in data.filters) { + data.filters[i] = JSON.parse(data.filters[i]); + } + } + catch { + return {error: {details: [{message: 'Invalid JSON string for filter parameter'}]}, value: null} + } + } return Joi.object({ status: Joi.string().valid('validated', 'new', 'all'), 'from-id': IdValidate.get(), @@ -171,7 +183,12 @@ export default class SampleValidate { 'page-size': Joi.number().integer().min(1), sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'), csv: Joi.boolean().default(false), - fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) + fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']), + filters: Joi.array().items(Joi.object({ + mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'), + field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')), + values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean())).min(1) + })).default([]) }).with('to-page', 'page-size').validate(data); } } \ No newline at end of file diff --git a/src/test/db.json b/src/test/db.json index aa68283..99ae417 100644 --- a/src/test/db.json +++ b/src/test/db.json @@ -411,7 +411,7 @@ "_id": {"$oid":"800000000000000000000006"}, "sample_id": {"$oid":"400000000000000000000006"}, "values": { - "weight %": 0.5, + "weight %": 0.6, "standard deviation":null }, "status": 0, From 6a02f09e7f1689c9a6ee7cff8f77a44b70bf3c32 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 6 Jul 2020 16:57:09 +0200 Subject: [PATCH 14/19] reworked filters --- src/routes/sample.ts | 144 ++++++++++++++++++++++++++----------------- 1 file changed, 88 insertions(+), 56 deletions(-) diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 240c156..11080f7 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -38,6 +38,8 @@ router.get('/samples', async (req, res, next) => { filters['to-page'] = 0; } + const sortFilterKeys = filters.filters.map(e => e.field); + let collection; const query = []; query.push({$match: {$and: []}}); @@ -64,13 +66,10 @@ router.get('/samples', async (req, res, next) => { query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); } query.push( - sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements + ...sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, - {$match: statusQuery(filters, 'sample.status')} // filter out wrong status once samples were added - ); - addSkipLimit(query, filters); // skip and limit to select right page - query.push( + {$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); @@ -78,49 +77,60 @@ router.get('/samples', async (req, res, next) => { } else { // sorting with samples as starting collection collection = SampleModel; - // filter for status query[0].$match.$and.push(statusQuery(filters, 'status')); - addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters - // differentiate by sort key to do sorting, skip and limit as early as possible if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys let sortStartValue = null; if (filters['from-id']) { // from-id specified - const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);}); + const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => { + next(err); + }); if (fromSample instanceof Error) return; if (!fromSample) { return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'}); } sortStartValue = fromSample[filters.sort[0]]; } - query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); - // material filters - addSkipLimit(query, filters); + query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); } - else { // sorting for material keys - let materialQuery = [] - materialQuery.push( // add material properties - {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, - {$set: {material: { $arrayElemAt: ['$material', 0]}}} + else { // add sort key to list to add field later + sortFilterKeys.push(filters.sort[0]); + } + } + + addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters + + let materialQuery = []; // put material query together separate first to reuse for first-id + let materialAdded = false; + if (sortFilterKeys.find(e => /material\./.test(e))) { // add material fields + materialAdded = true; + materialQuery.push( // add material properties + {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields + {$set: {material: {$arrayElemAt: ['$material', 0]}}} + ); + const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0); + addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters + if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed + materialQuery.push( + {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, + {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); - if (filters.sort[0] === 'material.supplier') { // add supplier if needed - materialQuery.push( - {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} - ); - } - if (filters.sort[0] === 'material.group') { // add group if needed - materialQuery.push( - {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} - ); - } - if (filters.sort[0] === 'material.number') { // add material number if needed - materialQuery.push( - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} - ); - } - query.push(...materialQuery); + } + if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed + materialQuery.push( + {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, + {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + ); + } + if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed + materialQuery.push( + {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + ); + } + const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); + addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); // base material filters + query.push(...materialQuery); + if (/material\./.test(filters.sort[0])) { // sort by material key let sortStartValue = null; if (filters['from-id']) { // from-id specified const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);}); @@ -130,17 +140,41 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample[filters.sort[0]]; } - query.push(sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); - addSkipLimit(query, filters); + query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); } } - const fieldsToAdd = [ - ...filters.fields, - ...filters.filters.map(e => e.field) // add filter fields in case they were not specified to display - ].filter(e => e !== filters.sort[0]) // sort field was definitely added already, exclude from further field operations - .filter((e, i, self) => self.indexOf(e) === i); // remove duplicates - if (fieldsToAdd.find(e => /material\./.test(e))) { // add material fields + const measurementFilterFields = _.uniq(sortFilterKeys.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters + if (sortFilterKeys.find(e => /measurements\./.test(e))) { // add measurement fields + const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFilterFields}}).lean().exec().catch(err => {next(err);}); + if (measurementTemplates instanceof Error) return; + if (measurementTemplates.length < measurementFilterFields.length) { + return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); + } + query.push({$lookup: { + from: 'measurements', let: {sId: '$_id'}, + pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], + as: 'measurements' + }}); + measurementTemplates.forEach(template => { + query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, + in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} + }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); + }); + addFilterQueries(query, filters.filters + .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) + .map(e => {e.field = e.field.replace('measurements.', ''); return e; }) + ); // measurement filters + } + addSkipLimit(query, filters); + + const fieldsToAdd = filters.fields.filter(e => // fields to add + sortFilterKeys.indexOf(e) < 0 // field was not in filter + && e !== filters.sort[0] // field was not in sort + ); + + if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already query.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$set: {material: { $arrayElemAt: ['$material', 0]}}} @@ -164,13 +198,11 @@ router.get('/samples', async (req, res, next) => { ); } - addFilterQueries(query, filters.filters.filter(e => /material\./.test(e.field))); // material filters - - let measurementFields = fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]).filter((e, i, self) => self.indexOf(e) === i); // filter measurement names and remove duplicates from parameters + let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields - const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.filter(e => e !== filters.sort[0].replace('measurements.', '')).map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);}); + const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);}); if (measurementTemplates instanceof Error) return; - if (measurementTemplates.length < measurementFields.length) { + if (measurementTemplates.length < measurementFieldsFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance @@ -181,7 +213,7 @@ router.get('/samples', async (req, res, next) => { from: 'measurements', let: {sId: '$_id'}, pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], as: 'measurements' - }}); + }}); } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values @@ -189,7 +221,7 @@ router.get('/samples', async (req, res, next) => { in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); - if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well + if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well query.push( {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, {$set: {spectrum: '$spectrum.values.dpt'}}, @@ -198,7 +230,6 @@ router.get('/samples', async (req, res, next) => { } query.push({$unset: 'measurements'}); } - addFilterQueries(query, filters.filters.filter(e => /measurements\./.test(e.field)).map(e => {e.field = e.field.replace('measurements.', ''); return e; })); // measurement filters const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date @@ -214,6 +245,7 @@ router.get('/samples', async (req, res, next) => { if (filters['to-page'] < 0) { data.reverse(); } + const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]); if (filters.csv) { // output as csv csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { if (err) return next(err); @@ -587,14 +619,14 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] if (filters['from-id']) { // from-id specified if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc - query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); - return {$sort: {[sortKeys[0]]: 1, _id: 1}}; + return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}}, + {$sort: {[sortKeys[0]]: 1, _id: 1}}]; } else { - query[0].$match.$and.push({$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}); - return {$sort: {[sortKeys[0]]: -1, _id: -1}}; + return [{$match: {$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}}, + {$sort: {[sortKeys[0]]: -1, _id: -1}}]; } } else { // sort from beginning - return {$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}; // set _id as secondary sort + return [{$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}]; // set _id as secondary sort } } From 1ddc2b617aeb858024be386ea1d7effd2c107355 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 9 Jul 2020 13:48:27 +0200 Subject: [PATCH 15/19] spectrum field working again --- api/api.yaml | 4 +- api/sample.yaml | 8 +- data_import/import.js | 209 +++++++++++++++++++++++----------- mainfest.yml => manifest.yml | 5 +- package-lock.json | 14 +++ package.json | 4 +- src/api.ts | 1 + src/helpers/csv.ts | 6 - src/helpers/mail.ts | 2 +- src/index.ts | 3 + src/routes/sample.spec.ts | 1 + src/routes/sample.ts | 95 ++++++++++------ src/routes/validate/sample.ts | 32 +++++- 13 files changed, 269 insertions(+), 115 deletions(-) rename mainfest.yml => manifest.yml (75%) diff --git a/api/api.yaml b/api/api.yaml index d281206..a1966fa 100644 --- a/api/api.yaml +++ b/api/api.yaml @@ -39,10 +39,10 @@ info: servers: + - url: https://definma-api.apps.de1.bosch-iot-cloud.com + description: server on the BIC - url: http://localhost:3000 description: local server - - url: https://digital-fingerprint-of-plastics-api.apps.de1.bosch-iot-cloud.com/ - description: server on the BIC security: diff --git a/api/sample.yaml b/api/sample.yaml index acdd33c..17df4c3 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -2,7 +2,7 @@ get: summary: all samples in overview description: 'Auth: all, levels: read, write, maintain, dev, admin' - x-doc: returns only samples with status 10 + x-doc: 'Limitations: paging and csv output does not work when including the spectrum measurement fields as well as the returned number of total samples' tags: - /sample parameters: @@ -61,6 +61,12 @@ responses: 200: description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) + headers: + X-Total-Items: + description: Total number of available items when page is specified + schema: + type: integer + example: 243 content: application/json: schema: diff --git a/data_import/import.js b/data_import/import.js index 4f31f8b..627e1b8 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -1,55 +1,82 @@ const csv = require('csv-parser'); const fs = require('fs'); const axios = require('axios'); -const {Builder} = require('selenium-webdriver'); +const {Builder} = require('selenium-webdriver'); // selenium and the chrome driver must be installed and configured separately const chrome = require('selenium-webdriver/chrome'); const pdfReader = require('pdfreader'); const iconv = require('iconv-lite'); -const metadata = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\VZ.csv'; // metadata file -const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\nmDocs'; // NormMaster Documents -const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\DPT'; // Spectrum files +const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv'; // metadata files +const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv'; +const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv'; +const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs'; // NormMaster Documents +const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT'; // Spectrum files +// const host = 'http://localhost:3000'; +const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; let data = []; // metadata contents let materials = {}; let samples = []; let normMaster = {}; +let sampleDevices = {}; -// TODO: integrate measurement device information from DPT names using different users -// TODO: supplier: other for supplierless samples // TODO: BASF twice, BASF as color -// TODO: trim color names // TODO: duplicate kf values +// TODO: conditions +// TODO: comment and reference handling + + +// TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC main(); async function main() { if (0) { // materials await getNormMaster(); - await importCsv(); + await importCsv(metaDoc); + await allMaterials(); + await saveMaterials(); + await importCsv(kfDoc); + await allMaterials(); + await saveMaterials(); + await importCsv(vzDoc); await allMaterials(); - fs.writeFileSync('./data_import/materials.json', JSON.stringify(materials)); await saveMaterials(); } - else if (0) { // samples - await importCsv(); - await allSamples(); - await saveSamples(); + if (0) { // samples + sampleDeviceMap(); + if (1) { + console.log('-------- META ----------'); + await importCsv(metaDoc); + await allSamples(); + await saveSamples(); + } + if (1) { + console.log('-------- KF ----------'); + await importCsv(kfDoc); + await allSamples(); + await saveSamples(); + await allKfVz(); + } + if (1) { + console.log('-------- VZ ----------'); + await importCsv(vzDoc); + await allSamples(); + await saveSamples(); + await allKfVz(); + } } - else if (1) { // DPT + if (1) { // DPT await allDpts(); } - else if (0) { // KF/VZ - await importCsv(); - await allKfVz(); - } - else if (0) { // pdf test - console.log(await readPdf('N28_BN22-O010_2018-03-08.pdf')); + if (0) { // pdf test + console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf')); } } -async function importCsv() { +async function importCsv(doc) { + data = []; await new Promise(resolve => { - fs.createReadStream(metadata) + fs.createReadStream(doc) .pipe(iconv.decodeStream('win1252')) .pipe(csv()) .on('data', (row) => { @@ -57,6 +84,9 @@ async function importCsv() { }) .on('end', () => { console.info('CSV file successfully processed'); + if (data[0]['Farbe']) { // fix German column names + data.map(e => {e['Color'] = e['Farbe']; return e; }); + } resolve(); }); }); @@ -65,7 +95,7 @@ async function importCsv() { async function allDpts() { let res = await axios({ method: 'get', - url: 'http://localhost:3000/template/measurements', + url: host + '/template/measurements', auth: { username: 'admin', password: 'Abc123!#' @@ -74,7 +104,7 @@ async function allDpts() { const measurement_template = res.data.find(e => e.name === 'spectrum')._id; res = await axios({ method: 'get', - url: 'http://localhost:3000/samples?status=all', + url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' @@ -84,10 +114,10 @@ async function allDpts() { res.data.forEach(sample => { sampleIds[sample.number] = sample._id; }); - const regex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/; + const dptRegex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/; const dpts = fs.readdirSync(dptFiles); for (let i in dpts) { - const regexRes = regex.exec(dpts[i]) + const regexRes = dptRegex.exec(dpts[i]) if (regexRes && sampleIds[regexRes[1]]) { // found matching sample console.log(dpts[i]); const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8'); @@ -99,7 +129,7 @@ async function allDpts() { data.values.dpt = f.split('\r\n').map(e => e.split(',')); await axios({ method: 'post', - url: 'http://localhost:3000/measurement/new', + url: host + '/measurement/new', auth: { username: 'admin', password: 'Abc123!#' @@ -110,13 +140,16 @@ async function allDpts() { console.error(err.response.data); }); } + else { + console.log(`Could not find sample for ${dpts[i]} !!!!!!`); + } } } async function allKfVz() { let res = await axios({ method: 'get', - url: 'http://localhost:3000/template/measurements', + url: host + '/template/measurements', auth: { username: 'admin', password: 'Abc123!#' @@ -126,7 +159,7 @@ async function allKfVz() { const vz_template = res.data.find(e => e.name === 'vz')._id; res = await axios({ method: 'get', - url: 'http://localhost:3000/samples?status=all', + url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' @@ -140,13 +173,17 @@ async function allKfVz() { console.info(`${index}/${data.length}`); let sample = data[index]; if (sample['Sample number'] !== '') { + let credentials = ['admin', 'Abc123!#']; + if (sampleDevices[sample['Sample number']]) { + credentials = [sampleDevices[sample['Sample number']], '2020DeFinMachen!'] + } if (sample['KF in Gew%']) { await axios({ method: 'post', - url: 'http://localhost:3000/measurement/new', + url: host + '/measurement/new', auth: { - username: 'admin', - password: 'Abc123!#' + username: credentials[0], + password: credentials[1] }, data: { sample_id: sampleIds[sample['Sample number']], @@ -164,10 +201,10 @@ async function allKfVz() { if (sample['VZ (ml/g)']) { await axios({ method: 'post', - url: 'http://localhost:3000/measurement/new', + url: host + '/measurement/new', auth: { - username: 'admin', - password: 'Abc123!#' + username: credentials[0], + password: credentials[1] }, data: { sample_id: sampleIds[sample['Sample number']], @@ -186,9 +223,10 @@ async function allKfVz() { } async function allSamples() { + samples = []; let res = await axios({ method: 'get', - url: 'http://localhost:3000/materials?status=all', + url: host + '/materials?status=all', auth: { username: 'admin', password: 'Abc123!#' @@ -200,7 +238,7 @@ async function allSamples() { }) res = await axios({ method: 'get', - url: 'http://localhost:3000/samples?status=all', + url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' @@ -215,7 +253,13 @@ async function allSamples() { for (let index in data) { console.info(`${index}/${data.length}`); let sample = data[index]; - if (sample['Sample number'] !== '' && sample['Supplier'] !== '' && sample['Granulate/Part'] !== '') { // TODO: wait for decision about samples without suppliers/color/type + if (sample['Sample number'] !== '') { // TODO: what about samples without color + if (sample['Supplier'] === '') { // empty supplier fields + sample['Supplier'] = 'unknown'; + } + if (sample['Granulate/Part'] === '') { // empty supplier fields + sample['Granulate/Part'] = 'unknown'; + } const material = dbMaterials[trim(sample['Material name'])]; if (!material) { // could not find material, skipping sample continue; @@ -236,13 +280,20 @@ async function allSamples() { samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color; } else if (sample['Color'] && sample['Color'] !== '') { - samples[si].color = material.numbers.find(e => e.color.indexOf(sample['Color']) >= 0).color; + let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0); + if (!number && /black/.test(sample['Color'])) { // special case bk for black + number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0); + if (!number) { // try German word + number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0); + } + } + samples[si].color = number.color; } else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz samples[si].color = sampleColors[sample['Sample number'].split('_')[0]]; } - else { // TODO: no color information at all - samples.pop(); + else { + samples[si].color = ''; } } } @@ -251,41 +302,57 @@ async function allSamples() { async function saveSamples() { for (let i in samples) { console.info(`${i}/${samples.length}`); + let credentials = ['admin', 'Abc123!#']; + if (sampleDevices[samples[i].number]) { + credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!'] + } await axios({ method: 'post', - url: 'http://localhost:3000/sample/new', + url: host + '/sample/new', auth: { - username: 'admin', - password: 'Abc123!#' + username: credentials[0], + password: credentials[1] }, data: samples[i] }).catch(err => { - console.log(samples[i]); - console.error(err.response.data); + if (err.response.data.status && err.response.data.status !== 'Sample number already taken') { + console.log(samples[i]); + console.error(err.response.data); + } }); } console.info('saved all samples'); } async function allMaterials() { + materials = {}; for (let index in data) { let sample = data[index]; - if (sample['Sample number'] !== '' && sample['Supplier'] !== '') { // TODO: wait for decision about supplierless samples + if (sample['Sample number'] && sample['Sample number'] !== '') { + if (sample['Supplier'] === '') { // empty supplier fields + sample['Supplier'] = 'unknown'; + } + if (sample['Material name'] === '') { // empty name fields + sample['Material name'] = sample['Material']; + } + if (!sample['Material']) { // column Material is named Plastic in VZ metadata + sample['Material'] = sample['Plastic']; + } sample['Material name'] = trim(sample['Material name']); if (materials.hasOwnProperty(sample['Material name'])) { // material already found at least once - if (sample['Material number'] !== '') { + if (sample['Material number'] && sample['Material number'] !== '') { if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) { // new material number if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) { // color already in list, only number missing materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']); } else { - materials[sample['Material name']].numbers.push({color: sample['Color'], number: stripSpaces(sample['Material number'])}); + materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])}); } } } - else if (sample['Color'] !== '') { + else if (sample['Color'] && sample['Color'] !== '') { if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) { // new material color - materials[sample['Material name']].numbers.push({color: sample['Color'], number: ''}); + materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''}); } } } @@ -293,8 +360,8 @@ async function allMaterials() { console.info(`${index}/${data.length} ${sample['Material name']}`); materials[sample['Material name']] = { name: sample['Material name'], - supplier: sample['Supplier'], - group: sample['Material'] + supplier: trim(sample['Supplier']), + group: trim(sample['Material']) }; let tmp = /M(\d+)/.exec(sample['Reinforcing material']); materials[sample['Material name']].mineral = tmp ? tmp[1] : 0; @@ -312,17 +379,20 @@ async function allMaterials() { async function saveMaterials() { const mKeys = Object.keys(materials) for (let i in mKeys) { + console.info(`${i}/${mKeys.length}`); await axios({ method: 'post', - url: 'http://localhost:3000/material/new', + url: host + '/material/new', auth: { username: 'admin', password: 'Abc123!#' }, data: materials[mKeys[i]] }).catch(err => { - console.log(materials[mKeys[i]]); - console.error(err.response.data); + if (err.response.data.status && err.response.data.status !== 'Material name already taken') { + console.info(materials[mKeys[i]]); + console.error(err.response.data); + } }); } console.info('saved all materials'); @@ -362,16 +432,16 @@ async function numbersFetch(sample) { } } if (res.length === 0) { // no results - if (sample['Color'] !== '' || sample['Material number'] !== '') { - return [{color: sample['Color'], number: sample['Material number']}]; + if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) { + return [{color: trim(sample['Color']), number: sample['Material number']}]; } else { return []; } } else { - if (!res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed - res.push({color: sample['Color'], number: sample['Material number']}); + if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed + res.push({color: trim(sample['Color']), number: sample['Material number']}); } return res; } @@ -403,7 +473,7 @@ async function getNormMaster(fetchAgain = false) { } function getNormMasterDoc(url, timing = 1) { - console.log(url); + console.info(url); return new Promise(async resolve => { const options = new chrome.Options(); options.setUserPreferences({ @@ -453,7 +523,7 @@ function readPdf(file) { rows.push(item.text); } else { // still the same row row - rows[rows.length - 1] += (item.x - lastX > 1.1 ? '$' : '') + item.text; // push to row, detect if still same cell + rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text; // push to row, detect if still same cell } lastX = (item.w * 0.055) + item.x; @@ -465,7 +535,7 @@ function readPdf(file) { table = -1; // console.log(rows); rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows - resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; })); + resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; })); } } lastLastText = lastText; @@ -473,12 +543,23 @@ function readPdf(file) { } if (!item && table !== -1) { // document ended rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows - resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; })); + resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; })); } }); }); } +function sampleDeviceMap() { + const dpts = fs.readdirSync(dptFiles); + const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/; + for (let i in dpts) { + const regexRes = regex.exec(dpts[i]) + if (regexRes) { // found matching sample + sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase(); + } + } +} + function stripSpaces(s) { return s ? s.replace(/ /g,'') : ''; } diff --git a/mainfest.yml b/manifest.yml similarity index 75% rename from mainfest.yml rename to manifest.yml index 16e5924..0e8c57d 100644 --- a/mainfest.yml +++ b/manifest.yml @@ -1,6 +1,7 @@ --- applications: - - name: digital-fingerprint-of-plastics-api + - name: definma-api + path: dist/ instances: 1 memory: 256M stack: cflinuxfs3 @@ -10,4 +11,4 @@ applications: NODE_ENV: production OPTIMIZE_MEMORY: true services: - - dfopdb + - definmadb diff --git a/package-lock.json b/package-lock.json index 5478eef..34fb53e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1174,6 +1174,15 @@ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2866,6 +2875,11 @@ } } }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, "object-inspect": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", diff --git a/package.json b/package.json index 4b04218..f9494d3 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "tsc": "tsc", "tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc", "build": "build.bat", + "build-push": "build.bat && cf push", "test": "mocha dist/**/**.spec.js", "start": "node index.js", "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"", @@ -28,6 +29,7 @@ "cfenv": "^1.2.2", "compression": "^1.7.4", "content-filter": "^1.1.2", + "cors": "^2.8.5", "express": "^4.17.1", "helmet": "^3.22.0", "json-schema": "^0.2.5", @@ -35,7 +37,7 @@ "lodash": "^4.17.15", "mongo-sanitize": "^1.1.0", "mongoose": "^5.8.7", - "swagger-ui-express": "^4.1.2" + "swagger-ui-express": "4.1.2" }, "devDependencies": { "@types/bcrypt": "^3.0.0", diff --git a/src/api.ts b/src/api.ts index 0867bc1..aab7b80 100644 --- a/src/api.ts +++ b/src/api.ts @@ -18,6 +18,7 @@ export default class api { jsonRefParser.bundle('api/api.yaml', (err, doc) => { // parse yaml if (err) throw err; apiDoc = doc; + apiDoc.servers.splice(process.env.NODE_ENV === 'production', 1); apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e)); // bundle routes apiDoc = this.resolveXDoc(apiDoc); oasParser.validate(apiDoc, (err, api) => { // validate oas schema diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts index 18e633c..38c487a 100644 --- a/src/helpers/csv.ts +++ b/src/helpers/csv.ts @@ -1,12 +1,6 @@ import {parseAsync} from 'json2csv'; export default function csv(input: any[], f: (err, data) => void) { - console.log(input[1000]); - console.log(flatten(input[1000])); - parseAsync([flatten(input[1000])]).then(csv => console.log(csv)); - console.log(input[1]); - console.log(flatten(input[1])); - parseAsync([flatten(input[1])]).then(csv => console.log(csv)); parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true}) .then(csv => f(null, csv)) .catch(err => f(err, null)); diff --git a/src/helpers/mail.ts b/src/helpers/mail.ts index a3d79c1..8ec71c8 100644 --- a/src/helpers/mail.ts +++ b/src/helpers/mail.ts @@ -17,7 +17,7 @@ export default (mailAddress, subject, content, f) => { // callback, executed em contentType: "text/html" }, from: { - eMail: "dfop@bosch-iot.com", + eMail: "definma@bosch-iot.com", password: "PlasticsOfFingerprintDigital" } } diff --git a/src/index.ts b/src/index.ts index 8116de7..4051f23 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import compression from 'compression'; import contentFilter from 'content-filter'; import mongoSanitize from 'mongo-sanitize'; import helmet from 'helmet'; +import cors from 'cors'; import api from './api'; import db from './db'; @@ -42,9 +43,11 @@ app.use((req, res, next) => { // no database connection error next(); } else { + console.error('No database connection'); res.status(500).send({status: 'Internal server error'}); } }); +app.use(cors()); // CORS headers app.use(require('./helpers/authorize')); // handle authentication // redirect /api routes for Angular proxy in development diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index d15200b..7dc5f24 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -21,6 +21,7 @@ describe('/sample', () => { // TODO: sort, added date filter, has measurements/condition filter // TODO: check if conditions work in sort/fields/filters + // TODO: test for numbers as strings in glass_fiber describe('GET /samples', () => { it('returns all samples', done => { TestHelper.request(server, done, { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 11080f7..bf741c2 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -21,6 +21,12 @@ import csv from '../helpers/csv'; const router = express.Router(); +// TODO: check added filter +// TODO: return total number of pages -> use facet +// TODO: use query pointer +// TODO: convert filter value to number according to table model +// TODO: validation for filter parameters +// TODO: location/device sort/filter router.get('/samples', async (req, res, next) => { if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; @@ -37,6 +43,7 @@ router.get('/samples', async (req, res, next) => { if (!filters['to-page']) { // set to-page default filters['to-page'] = 0; } + console.log(filters); const sortFilterKeys = filters.filters.map(e => e.field); @@ -70,7 +77,7 @@ router.get('/samples', async (req, res, next) => { {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, {$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added - {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring + {$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters @@ -106,25 +113,25 @@ router.get('/samples', async (req, res, next) => { materialAdded = true; materialQuery.push( // add material properties {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields - {$set: {material: {$arrayElemAt: ['$material', 0]}}} + {$addFields: {material: {$arrayElemAt: ['$material', 0]}}} ); const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0); addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed materialQuery.push( {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); } if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed materialQuery.push( {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed materialQuery.push( - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} ); } const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); @@ -157,10 +164,10 @@ router.get('/samples', async (req, res, next) => { as: 'measurements' }}); measurementTemplates.forEach(template => { - query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} - }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); + }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); addFilterQueries(query, filters.filters .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) @@ -173,39 +180,40 @@ router.get('/samples', async (req, res, next) => { sortFilterKeys.indexOf(e) < 0 // field was not in filter && e !== filters.sort[0] // field was not in sort ); + console.log(fieldsToAdd); if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already query.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, - {$set: {material: { $arrayElemAt: ['$material', 0]}}} + {$addFields: {material: { $arrayElemAt: ['$material', 0]}}} ); } if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed query.push( {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); } if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed query.push( {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed query.push( - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} ); } - let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters + let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);}); if (measurementTemplates instanceof Error) return; if (measurementTemplates.length < measurementFieldsFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } - if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance + if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); } else { @@ -216,15 +224,15 @@ router.get('/samples', async (req, res, next) => { }}); } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later - query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} - }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); + }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well query.push( - {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, - {$set: {spectrum: '$spectrum.values.dpt'}}, + {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, + {$addFields: {spectrum: '$spectrum.values'}}, {$unwind: '$spectrum'} ); } @@ -233,30 +241,45 @@ router.get('/samples', async (req, res, next) => { const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date - projection.added = {$toDate: '$_id'}; + // projection.added = {$toDate: '$_id'}; + // projection.added = { $convert: { input: '$_id', to: "date" } } // TODO } if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly projection._id = false; } query.push({$project: projection}); - collection.aggregate(query).exec((err, data) => { - if (err) return next(err); - if (filters['to-page'] < 0) { - data.reverse(); - } - const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]); - if (filters.csv) { // output as csv - csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { - if (err) return next(err); - res.set('Content-Type', 'text/csv'); - res.send(data); - }); - } - else { - res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors - } - }) + if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files + collection.aggregate(query).exec((err, data) => { + if (err) return next(err); + console.log(data.length); + if (filters['to-page'] < 0) { + data.reverse(); + } + const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]); + if (filters.csv) { // output as csv + csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { + if (err) return next(err); + res.set('Content-Type', 'text/csv'); + res.send(data); + }); + } + else { + res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors + } + }); + } + else { + res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'}); + res.write('['); + let count = 0; + const stream = collection.aggregate(query).cursor().exec(); + stream.on('data', data => { res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; }); + stream.on('close', () => { + res.write(']'); + res.end(); + }); + } }); router.get('/samples/:state(new|deleted)', (req, res, next) => { @@ -537,7 +560,7 @@ async function materialCheck (sample, res, next, id = sample.material_id) { // res.status(400).json({status: 'Material not available'}); return false; } - if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified + if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified res.status(400).json({status: 'Color not available for material'}); return false; } diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index b0cae01..f84a5be 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -11,7 +11,8 @@ export default class SampleValidate { .max(128), color: Joi.string() - .max(128), + .max(128) + .allow(''), type: Joi.string() .max(128), @@ -77,7 +78,7 @@ export default class SampleValidate { 'user_id', 'material._id', 'material.numbers', - 'measurements.spectrum' + 'measurements.spectrum.dpt' ]; static input (data, param) { // validate input, set param to 'new' to make all attributes required @@ -170,6 +171,33 @@ export default class SampleValidate { try { for (let i in data.filters) { data.filters[i] = JSON.parse(data.filters[i]); + data.filters[i].values = data.filters[i].values.map(e => { // validate filter values + let validator; + let field = data.filters[i].field + if (/material\./.test(field)) { // select right validation model + validator = MaterialValidate.outputV(); + field = field.replace('material.', ''); + } + else if (/measurements\./.test(field)) { + validator = Joi.object({ + value: Joi.alternatives() + .try( + Joi.string().max(128), + Joi.number(), + Joi.boolean(), + Joi.array() + ) + .allow(null) + }); + field = 'value'; + } + else { + validator = Joi.object(this.sample); + } + const {value, error} = validator.validate({[field]: e}); + if (error) throw error; // reject invalid values + return value[field]; + }); } } catch { From f41498da53a274dda6ebe64beb951a7ea40b10bf Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 9 Jul 2020 16:30:10 +0200 Subject: [PATCH 16/19] implemented x-total-items header --- api/sample.yaml | 4 +- manifest.yml | 2 +- package.json | 2 +- src/db.ts | 6 ++- src/index.ts | 2 +- src/routes/sample.ts | 89 +++++++++++++++++++++++--------------------- 6 files changed, 57 insertions(+), 48 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index 17df4c3..2b0ce31 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -62,8 +62,8 @@ 200: description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) headers: - X-Total-Items: - description: Total number of available items when page is specified + x-total-items: + description: Total number of available items when from-id is not specified and spectrum field is not included schema: type: integer example: 243 diff --git a/manifest.yml b/manifest.yml index 0e8c57d..dd7e0f1 100644 --- a/manifest.yml +++ b/manifest.yml @@ -3,7 +3,7 @@ applications: - name: definma-api path: dist/ instances: 1 - memory: 256M + memory: 512M stack: cflinuxfs3 buildpacks: - nodejs_buildpack diff --git a/package.json b/package.json index f9494d3..bfcdcca 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "build": "build.bat", "build-push": "build.bat && cf push", "test": "mocha dist/**/**.spec.js", - "start": "node index.js", + "start": "sleep 5s && node index.js", "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"", "loadDev": "node dist/test/loadDev.js", "coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000", diff --git a/src/db.ts b/src/db.ts index cb11af5..03c56e1 100644 --- a/src/db.ts +++ b/src/db.ts @@ -3,11 +3,15 @@ import cfenv from 'cfenv'; import _ from 'lodash'; import ChangelogModel from './models/changelog'; -mongoose.set('debug', true); // enable mongoose debug // database urls, prod db url is retrieved automatically const TESTING_URL = 'mongodb://localhost/dfopdb_test'; const DEV_URL = 'mongodb://localhost/dfopdb'; +const debugging = false; + +if (process.env.NODE_ENV !== 'production' && debugging) { + mongoose.set('debug', true); // enable mongoose debug +} export default class db { private static state = { // db object and current mode (test, dev, prod) diff --git a/src/index.ts b/src/index.ts index 4051f23..d6ea865 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,11 +26,11 @@ const port = process.env.PORT || 3000; //middleware app.use(helmet()); +app.use(contentFilter()); // filter URL query attacks app.use(express.json({ limit: '5mb'})); app.use(express.urlencoded({ extended: false, limit: '5mb' })); app.use(compression()); // compress responses app.use(bodyParser.json()); -app.use(contentFilter()); // filter URL query attacks app.use((req, res, next) => { // filter body query attacks req.body = mongoSanitize(req.body); next(); diff --git a/src/routes/sample.ts b/src/routes/sample.ts index bf741c2..f63b0b6 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -43,13 +43,13 @@ router.get('/samples', async (req, res, next) => { if (!filters['to-page']) { // set to-page default filters['to-page'] = 0; } - console.log(filters); const sortFilterKeys = filters.filters.map(e => e.field); let collection; const query = []; - query.push({$match: {$and: []}}); + let queryPtr = query; + queryPtr.push({$match: {$and: []}}); if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection collection = MeasurementModel; @@ -68,23 +68,23 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample.values[measurementParam]; } - query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort + queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered - query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); + queryPtr[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); } - query.push( - ...sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements + queryPtr.push( + ...sortQuery(filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, {$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added {$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); - addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters + addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters } else { // sorting with samples as starting collection collection = SampleModel; - query[0].$match.$and.push(statusQuery(filters, 'status')); + queryPtr[0].$match.$and.push(statusQuery(filters, 'status')); if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys let sortStartValue = null; @@ -98,14 +98,14 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample[filters.sort[0]]; } - query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue)); } else { // add sort key to list to add field later sortFilterKeys.push(filters.sort[0]); } } - addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters + addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters let materialQuery = []; // put material query together separate first to reuse for first-id let materialAdded = false; @@ -136,7 +136,7 @@ router.get('/samples', async (req, res, next) => { } const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); // base material filters - query.push(...materialQuery); + queryPtr.push(...materialQuery); if (/material\./.test(filters.sort[0])) { // sort by material key let sortStartValue = null; if (filters['from-id']) { // from-id specified @@ -147,7 +147,7 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample[filters.sort[0]]; } - query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue)); } } @@ -158,50 +158,61 @@ router.get('/samples', async (req, res, next) => { if (measurementTemplates.length < measurementFilterFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } - query.push({$lookup: { + queryPtr.push({$lookup: { from: 'measurements', let: {sId: '$_id'}, pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], as: 'measurements' }}); measurementTemplates.forEach(template => { - query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); - addFilterQueries(query, filters.filters + addFilterQueries(queryPtr, filters.filters .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) .map(e => {e.field = e.field.replace('measurements.', ''); return e; }) ); // measurement filters } - addSkipLimit(query, filters); + + if (!filters.fields.find(e => /spectrum\./.test(e)) && !filters['from-id']) { // count total number of items before $skip and $limit, only works when from-id is not specified and spectra are not included + queryPtr.push({$facet: {count: [{$count: 'count'}], samples: []}}); + queryPtr = queryPtr[queryPtr.length - 1].$facet.samples; // add rest of aggregation pipeline into $facet + } + + // paging + if (filters['to-page']) { + queryPtr.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more + } + if (filters['page-size']) { + queryPtr.push({$limit: filters['page-size']}); + } const fieldsToAdd = filters.fields.filter(e => // fields to add sortFilterKeys.indexOf(e) < 0 // field was not in filter && e !== filters.sort[0] // field was not in sort ); - console.log(fieldsToAdd); if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already - query.push( + queryPtr.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$addFields: {material: { $arrayElemAt: ['$material', 0]}}} ); } if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed - query.push( + queryPtr.push( {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); } if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed - query.push( + queryPtr.push( {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed - query.push( + queryPtr.push( {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} ); } @@ -214,45 +225,49 @@ router.get('/samples', async (req, res, next) => { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance - query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); + queryPtr.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); } else { - query.push({$lookup: { + queryPtr.push({$lookup: { from: 'measurements', let: {sId: '$_id'}, pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], as: 'measurements' }}); } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later - query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well - query.push( + queryPtr.push( {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, {$addFields: {spectrum: '$spectrum.values'}}, {$unwind: '$spectrum'} ); } - query.push({$unset: 'measurements'}); + // queryPtr.push({$unset: 'measurements'}); + queryPtr.push({$project: {measurements: 0}}); } const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date // projection.added = {$toDate: '$_id'}; - // projection.added = { $convert: { input: '$_id', to: "date" } } // TODO + // projection.added = { $convert: { input: '$_id', to: "date" } } // TODO: upgrade MongoDB version or find alternative } if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly projection._id = false; } - query.push({$project: projection}); + queryPtr.push({$project: projection}); if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files collection.aggregate(query).exec((err, data) => { if (err) return next(err); - console.log(data.length); + if (data[0].count) { + res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0); + data = data[0].samples; + } if (filters['to-page'] < 0) { data.reverse(); } @@ -639,7 +654,7 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and }); } -function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] +function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] if (filters['from-id']) { // from-id specified if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}}, @@ -653,16 +668,6 @@ function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = [' } } -function addSkipLimit(query, filters) { - if (filters['to-page']) { - query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more - } - - if (filters['page-size']) { - query.push({$limit: filters['page-size']}); - } -} - function statusQuery(filters, field) { if (filters.hasOwnProperty('status')) { if(filters.status === 'all') { @@ -677,9 +682,9 @@ function statusQuery(filters, field) { } } -function addFilterQueries (query, filters) { // returns array of match queries from given filters +function addFilterQueries (queryPtr, filters) { // returns array of match queries from given filters if (filters.length) { - query.push({$match: {$and: filterQueries(filters)}}); + queryPtr.push({$match: {$and: filterQueries(filters)}}); } } From 523b2c9b68fd3930759f76ba240e48e54ee98ddf Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Fri, 10 Jul 2020 09:42:05 +0200 Subject: [PATCH 17/19] added workaround for 'added' field compatible to MongoDB 3.6 --- manifest.yml | 2 +- package.json | 2 +- src/db.ts | 7 ++++++- src/routes/sample.ts | 22 ++++++++++++++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/manifest.yml b/manifest.yml index dd7e0f1..1791c58 100644 --- a/manifest.yml +++ b/manifest.yml @@ -3,7 +3,7 @@ applications: - name: definma-api path: dist/ instances: 1 - memory: 512M + memory: 1024M stack: cflinuxfs3 buildpacks: - nodejs_buildpack diff --git a/package.json b/package.json index bfcdcca..f9494d3 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "build": "build.bat", "build-push": "build.bat && cf push", "test": "mocha dist/**/**.spec.js", - "start": "sleep 5s && node index.js", + "start": "node index.js", "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"", "loadDev": "node dist/test/loadDev.js", "coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000", diff --git a/src/db.ts b/src/db.ts index 03c56e1..2b1f409 100644 --- a/src/db.ts +++ b/src/db.ts @@ -47,10 +47,15 @@ export default class db { if (err) done(err); }); mongoose.connection.on('error', console.error.bind(console, 'connection error:')); + mongoose.connection.on('connected', () => { // evaluation connection behaviour on prod + if (process.env.NODE_ENV !== 'test') { // Do not interfere with testing + console.info('Database connected'); + } + }); mongoose.connection.on('disconnected', () => { // reset state on disconnect if (process.env.NODE_ENV !== 'test') { // Do not interfere with testing console.info('Database disconnected'); - this.state.db = 0; + // this.state.db = 0; // prod database connects and disconnects automatically } }); process.on('SIGINT', () => { // close connection when app is terminated diff --git a/src/routes/sample.ts b/src/routes/sample.ts index f63b0b6..f3395c8 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -256,7 +256,7 @@ router.get('/samples', async (req, res, next) => { // projection.added = {$toDate: '$_id'}; // projection.added = { $convert: { input: '$_id', to: "date" } } // TODO: upgrade MongoDB version or find alternative } - if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly + if (filters.fields.indexOf('_id') < 0 && filters.fields.indexOf('added') < 0) { // disable _id explicitly projection._id = false; } queryPtr.push({$project: projection}); @@ -266,8 +266,18 @@ router.get('/samples', async (req, res, next) => { if (err) return next(err); if (data[0].count) { res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0); + res.header('Access-Control-Expose-Headers', 'x-total-items'); data = data[0].samples; } + if (filters.fields.indexOf('added') >= 0) { // add added date + data.map(e => { + e.added = e._id.getTimestamp(); + if (filters.fields.indexOf('_id') < 0) { + delete e._id; + } + return e + }); + } if (filters['to-page'] < 0) { data.reverse(); } @@ -289,7 +299,15 @@ router.get('/samples', async (req, res, next) => { res.write('['); let count = 0; const stream = collection.aggregate(query).cursor().exec(); - stream.on('data', data => { res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; }); + stream.on('data', data => { + if (filters.fields.indexOf('added') >= 0) { // add added date + data.added = data._id.getTimestamp(); + if (filters.fields.indexOf('_id') < 0) { + delete data._id; + } + } + res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; + }); stream.on('close', () => { res.write(']'); res.end(); From 758eb0e143c6b14b9bb42cd9c4a5e06cfa62df4d Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Fri, 10 Jul 2020 13:09:15 +0200 Subject: [PATCH 18/19] implemented added filters --- src/db.ts | 2 +- src/routes/sample.ts | 80 ++++++++++++++++++++++++++++++++--- src/routes/validate/sample.ts | 6 +-- 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/src/db.ts b/src/db.ts index 2b1f409..2bab005 100644 --- a/src/db.ts +++ b/src/db.ts @@ -7,7 +7,7 @@ import ChangelogModel from './models/changelog'; // database urls, prod db url is retrieved automatically const TESTING_URL = 'mongodb://localhost/dfopdb_test'; const DEV_URL = 'mongodb://localhost/dfopdb'; -const debugging = false; +const debugging = true; if (process.env.NODE_ENV !== 'production' && debugging) { mongoose.set('debug', true); // enable mongoose debug diff --git a/src/routes/sample.ts b/src/routes/sample.ts index f3395c8..ef87ab3 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -43,6 +43,54 @@ router.get('/samples', async (req, res, next) => { if (!filters['to-page']) { // set to-page default filters['to-page'] = 0; } + const addedFilter = filters.filters.find(e => e.field === 'added'); + if (addedFilter) { // convert added filter to object id + filters.filters.splice(filters.filters.findIndex(e => e.field === 'added'), 1); + if (addedFilter.mode === 'in') { + const v = []; // query value + addedFilter.values.forEach(value => { + const date = [new Date(value).setHours(0,0,0,0), new Date(value).setHours(23,59,59,999)]; + v.push({$and: [{ _id: { '$gte': dateToOId(date[0])}}, { _id: { '$lte': dateToOId(date[1])}}]}); + }); + filters.filters.push({mode: 'or', field: '_id', values: v}); + } + else if (addedFilter.mode === 'nin') { + addedFilter.values = addedFilter.values.sort(); + const v = []; // query value + + for (let i = 0; i <= addedFilter.values.length; i ++) { + v[i] = {$and: []}; + if (i > 0) { + const date = new Date(addedFilter.values[i - 1]).setHours(23,59,59,999); + v[i].$and.push({ _id: { '$gt': dateToOId(date)}}) ; + } + if (i < addedFilter.values.length) { + const date = new Date(addedFilter.values[i]).setHours(0,0,0,0); + v[i].$and.push({ _id: { '$lt': dateToOId(date)}}) ; + } + } + filters.filters.push({mode: 'or', field: '_id', values: v}); + } + else { + // start and end of day + const date = [new Date(addedFilter.values[0]).setHours(0,0,0,0), new Date(addedFilter.values[0]).setHours(23,59,59,999)]; + if (addedFilter.mode === 'lt') { // lt start + filters.filters.push({mode: 'lt', field: '_id', values: [dateToOId(date[0])]}); + } + if (addedFilter.mode === 'eq' || addedFilter.mode === 'lte') { // lte end + filters.filters.push({mode: 'lte', field: '_id', values: [dateToOId(date[1])]}); + } + if (addedFilter.mode === 'gt') { // gt end + filters.filters.push({mode: 'gt', field: '_id', values: [dateToOId(date[1])]}); + } + if (addedFilter.mode === 'eq' || addedFilter.mode === 'gte') { // gte start + filters.filters.push({mode: 'gte', field: '_id', values: [dateToOId(date[0])]}); + } + if (addedFilter.mode === 'ne') { + filters.filters.push({mode: 'or', field: '_id', values: [{ _id: { '$lt': dateToOId(date[0])}}, { _id: { '$gt': dateToOId(date[1])}}]}); + } + } + } const sortFilterKeys = filters.filters.map(e => e.field); @@ -80,7 +128,6 @@ router.get('/samples', async (req, res, next) => { {$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); - addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters } else { // sorting with samples as starting collection collection = SampleModel; @@ -568,13 +615,23 @@ module.exports = router; async function numberGenerate (sample, req, res, next) { // generate number in format Location32, returns false on error const sampleData = await SampleModel - .findOne({number: new RegExp('^' + req.authDetails.location + '[0-9]+$', 'm')}) - .sort({number: -1}) - .lean() + // .findOne({number: new RegExp('^' + req.authDetails.location + '[0-9]+$', 'm')}) + // .sort({number: -1}) + // .lean() + .aggregate([ + {$match: {number: new RegExp('^' + 'Rng' + '[0-9]+$', 'm')}}, + // {$addFields: {number2: {$toDecimal: {$arrayElemAt: [{$split: [{$arrayElemAt: [{$split: ['$number', 'Rng']}, 1]}, '_']}, 0]}}}}, // not working with MongoDb 3.6 + {$addFields: {sortNumber: {$let: { + vars: {tmp: {$concat: ['000000000000000000000000000000', {$arrayElemAt: [{$split: [{$arrayElemAt: [{$split: ['$number', 'Rng']}, 1]}, '_']}, 0]}]}}, + in: {$substrCP: ['$$tmp', {$subtract: [{$strLenCP: '$$tmp'}, 30]}, {$strLenCP: '$$tmp'}]} + }}}}, + {$sort: {sortNumber: -1}}, + {$limit: 1} + ]) .exec() .catch(err => next(err)); if (sampleData instanceof Error) return false; - return req.authDetails.location + (sampleData ? Number(sampleData.number.replace(/[^0-9]+/g, '')) + 1 : 1); + return req.authDetails.location + (sampleData[0] ? Number(sampleData[0].number.replace(/[^0-9]+/g, '')) + 1 : 1); } async function numberCheck(sample, res, next) { @@ -707,5 +764,16 @@ function addFilterQueries (queryPtr, filters) { // returns array of match queri } function filterQueries (filters) { - return filters.map(e => ({[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}})) // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin + return filters.map(e => { + if (e.mode === 'or') { // allow or queries (needed for $ne added) + return {['$' + e.mode]: e.values}; + } + else { + return {[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}}; // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin + } + }); +} + +function dateToOId (date) { // convert date to ObjectId + return mongoose.Types.ObjectId(Math.floor(date / 1000).toString(16) + '0000000000000000'); } \ No newline at end of file diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index f84a5be..ef0fa0a 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -175,7 +175,7 @@ export default class SampleValidate { let validator; let field = data.filters[i].field if (/material\./.test(field)) { // select right validation model - validator = MaterialValidate.outputV(); + validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}); field = field.replace('material.', ''); } else if (/measurements\./.test(field)) { @@ -195,7 +195,7 @@ export default class SampleValidate { validator = Joi.object(this.sample); } const {value, error} = validator.validate({[field]: e}); - if (error) throw error; // reject invalid values + if (error) throw error; // reject invalid values // TODO: return exact error description, handle in frontend filters return value[field]; }); } @@ -215,7 +215,7 @@ export default class SampleValidate { filters: Joi.array().items(Joi.object({ mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'), field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')), - values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean())).min(1) + values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso())).min(1) })).default([]) }).with('to-page', 'page-size').validate(data); } From 3dda3d77a13f84a40f8132f49f4ad704f2810a75 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Tue, 14 Jul 2020 12:07:43 +0200 Subject: [PATCH 19/19] minor fixes --- data_import/import.js | 14 ++++++++++++-- src/routes/root.ts | 1 + src/routes/sample.ts | 3 ++- src/routes/validate/sample.ts | 3 ++- src/routes/validate/template.ts | 1 + 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/data_import/import.js b/data_import/import.js index 627e1b8..dc8c8d8 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -11,8 +11,8 @@ const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv'; const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv'; const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs'; // NormMaster Documents const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT'; // Spectrum files -// const host = 'http://localhost:3000'; -const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; +const host = 'http://localhost:3000'; +// const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; let data = []; // metadata contents let materials = {}; let samples = []; @@ -127,6 +127,16 @@ async function allDpts() { measurement_template }; data.values.dpt = f.split('\r\n').map(e => e.split(',')); + let rescale = false; + for (let i in data.values.dpt) { + if (data.values.dpt[i][1] > 2) { + rescale = true; + break; + } + } + if (rescale) { + data.values.dpt = data.values.dpt.map(e => [e[0], e[1] / 100]); + } await axios({ method: 'post', url: host + '/measurement/new', diff --git a/src/routes/root.ts b/src/routes/root.ts index 946948f..1547844 100644 --- a/src/routes/root.ts +++ b/src/routes/root.ts @@ -17,6 +17,7 @@ router.get('/authorized', (req, res) => { res.json({status: 'Authorization successful', method: req.authDetails.method}); }); +// TODO: evaluate exact changelog functionality (restoring, delting after time, etc.) router.get('/changelog/:timestamp/:page?/:pagesize?', (req, res, next) => { if (!req.auth(res, ['maintain', 'admin'], 'basic')) return; diff --git a/src/routes/sample.ts b/src/routes/sample.ts index ef87ab3..91ada86 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -118,7 +118,7 @@ router.get('/samples', async (req, res, next) => { } queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered - queryPtr[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); + queryPtr[0].$match.$and.push(...filterQueries(filters.filters.filter(e => e.field === filters.sort[0]).map(e => {e.field = 'values.' + e.field.split('.')[2]; return e; }))); } queryPtr.push( ...sortQuery(filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements @@ -764,6 +764,7 @@ function addFilterQueries (queryPtr, filters) { // returns array of match queri } function filterQueries (filters) { + console.log(filters); return filters.map(e => { if (e.mode === 'or') { // allow or queries (needed for $ne added) return {['$' + e.mode]: e.values}; diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index ef0fa0a..3fb28d9 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -182,8 +182,8 @@ export default class SampleValidate { validator = Joi.object({ value: Joi.alternatives() .try( - Joi.string().max(128), Joi.number(), + Joi.string().max(128), Joi.boolean(), Joi.array() ) @@ -195,6 +195,7 @@ export default class SampleValidate { validator = Joi.object(this.sample); } const {value, error} = validator.validate({[field]: e}); + console.log(value); if (error) throw error; // reject invalid values // TODO: return exact error description, handle in frontend filters return value[field]; }); diff --git a/src/routes/validate/template.ts b/src/routes/validate/template.ts index 7a63d1d..ae9426a 100644 --- a/src/routes/validate/template.ts +++ b/src/routes/validate/template.ts @@ -1,6 +1,7 @@ import Joi from '@hapi/joi'; import IdValidate from './id'; +// TODO: do not allow a . in the name export default class TemplateValidate { private static template = { name: Joi.string()