From 4dede74f499ed90475ce74618a36be037ea53876 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 10 Aug 2020 08:09:46 +0200 Subject: [PATCH 1/2] csv only for dev/admin, mail change notice --- api/sample.yaml | 15 ++++++++------- src/routes/sample.spec.ts | 14 ++++++++++---- src/routes/sample.ts | 14 ++++++-------- src/routes/user.ts | 11 +++++++++++ 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index e39a122..ba1df37 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -40,12 +40,6 @@ schema: type: string example: color-asc - - name: csv - description: output as csv - in: query - schema: - type: boolean - example: false - name: fields[] description: "the fields to include in the output as array, defaults to ['_id', 'number', 'type', 'batch', 'material_id', 'color', 'condition', 'note_id', 'user_id', 'added']" @@ -57,7 +51,8 @@ example: ['number', 'batch'] - name: filters[] description: "the filters to apply as an array of URIComponent encoded objects in the form {mode: - 'eq/ne/lt/lte/gt/gte/in/nin/stringin', field: 'material.m', values: ['15']} using encodeURIComponent(JSON.stringify({}))" + 'eq/ne/lt/lte/gt/gte/in/nin/stringin', field: 'material.m', values: ['15']} using + encodeURIComponent(JSON.stringify({}))" in: query schema: type: array @@ -66,6 +61,12 @@ example: '["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D", "%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22 %5D%7D"]' + - name: csv + description: output as csv, only available for dev and admin + in: query + schema: + type: boolean + example: false responses: 200: description: samples overview (output depends on the fields specified)
diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index bd798a4..5911876 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -7,8 +7,6 @@ import TestHelper from "../test/helper"; import mongoose from 'mongoose'; -// TODO: allowed types: tension rod, part, granulate, other -// TODO: filter by conditions and material properties describe('/sample', () => { let server; @@ -451,12 +449,12 @@ describe('/sample', () => { res: {status: 'Invalid body format', details: 'Measurement key not found'} }); }); - it('returns a correct csv file if specified', done => { + it('returns a correct csv file for admins if specified', done => { TestHelper.request(server, done, { method: 'get', url: '/samples?status[]=new&status[]=validated&page-size=2&csv=true', contentType: /text\/csv/, - auth: {basic: 'janedoe'}, + auth: {basic: 'admin'}, httpStatus: 200 }).end((err, res) => { if (err) return done(err); @@ -466,6 +464,14 @@ describe('/sample', () => { done(); }); }); + it('rejects returning a csv file for a write user', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status[]=new&status[]=validated&page-size=2&csv=true', + auth: {basic: 'janedoe'}, + httpStatus: 403 + }); + }); it('returns only the fields specified', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index dc29af9..6694ad6 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -34,8 +34,8 @@ router.get('/samples', async (req, res, next) => { const {error, value: filters} = SampleValidate.query(req.query, ['dev', 'admin'].indexOf(req.authDetails.level) >= 0); if (error) return res400(error, res); - // spectral data not allowed for read/write users - if (filters.fields.find(e => /\.dpt$/.test(e)) && !req.auth(res, ['dev', 'admin'], 'all')) return; + // spectral data and csv not allowed for read/write users + if ((filters.fields.find(e => /\.dpt$/.test(e)) || filters.csv) && !req.auth(res, ['dev', 'admin'], 'all')) return; // TODO: find a better place for these const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', @@ -195,7 +195,8 @@ router.get('/samples', async (req, res, next) => { {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } - // if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed // TODO: adapt code to new numbers format + // TODO: adapt code to new numbers format + // if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed // materialQuery.push( // {$addFields: {'material.number': { $arrayElemAt: [ // '$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']} @@ -478,9 +479,6 @@ router.put('/sample/' + IdValidate.parameter(), (req, res, next) => { if (sample.hasOwnProperty('material_id')) { if (!await materialCheck(sample, res, next)) return; } - else if (sample.hasOwnProperty('color')) { - if (!await materialCheck(sample, res, next, sampleData.material_id)) return; - } // do not execute check if condition is and was empty if (sample.hasOwnProperty('condition') && !(_.isEmpty(sample.condition) && _.isEmpty(sampleData.condition))) { if (!await conditionCheck(sample.condition, 'change', res, next, @@ -706,8 +704,8 @@ async function numberCheck(sample, res, next) { } // validate material_id and color, returns false if invalid -async function materialCheck (sample, res, next, id = sample.material_id) { - const materialData = await MaterialModel.findById(id).lean().exec().catch(err => next(err)) as any; +async function materialCheck (sample, res, next) { + const materialData = await MaterialModel.findById(sample.material_id).lean().exec().catch(err => next(err)) as any; if (materialData instanceof Error) return false; if (!materialData) { // could not find material_id res.status(400).json({status: 'Material not available'}); diff --git a/src/routes/user.ts b/src/routes/user.ts index c8ebdc2..e90f1a0 100644 --- a/src/routes/user.ts +++ b/src/routes/user.ts @@ -59,9 +59,20 @@ router.put('/user:username([/](?!key|new).?*|/?)', async (req, res, next) => { if (!await usernameCheck(user.name, res, next)) return; } + // get current mail address to compare to given address + const {email: oldMail} = await UserModel.findOne({name: username}).lean().exec().catch(err => next(err)); + await UserModel.findOneAndUpdate({name: username}, user, {new: true}).log(req).lean().exec( (err, data:any) => { if (err) return next(err); if (data) { + if (data.mail !== oldMail) { // mail address was changed, send notice to old address + Mail.send(oldMail, 'Email change in your DeFinMa database account', + 'Hi,

Your email address of your DeFinMa account was changed to ' + data.mail + + '

If you actually did this, just delete this email.' + + '

If you did not change your email, someone might be messing around with your account, ' + + 'so talk to the sysadmin quickly!

Have a nice day.' + + '

The DeFinMa team'); + } res.json(UserValidate.output(data)); } else { From ed8b549752cbfbeba497adda8427f34c8b8497a4 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Mon, 10 Aug 2020 12:35:08 +0200 Subject: [PATCH 2/2] flattened samples result --- api/sample.yaml | 7 +++--- src/helpers/csv.ts | 32 +-------------------------- src/helpers/flatten.ts | 41 +++++++++++++++++++++++++++++++++++ src/routes/sample.spec.ts | 20 +++++++++++++++-- src/routes/sample.ts | 12 ++++++++-- src/routes/validate/sample.ts | 2 +- 6 files changed, 75 insertions(+), 39 deletions(-) create mode 100644 src/helpers/flatten.ts diff --git a/api/sample.yaml b/api/sample.yaml index ba1df37..1359d23 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -61,12 +61,13 @@ example: '["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D", "%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22 %5D%7D"]' - - name: csv - description: output as csv, only available for dev and admin + - name: output + description: 'output format, available values are csv, json, flatten (converts material: {number: x} to + material.number: x), defaults to json' in: query schema: type: boolean - example: false + example: csv responses: 200: description: samples overview (output depends on the fields specified)
diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts index e6f07b2..d7774d7 100644 --- a/src/helpers/csv.ts +++ b/src/helpers/csv.ts @@ -1,4 +1,5 @@ import {parseAsync} from 'json2csv'; +import flatten from './flatten'; export default function csv(input: any[], f: (err, data) => void) { parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true}) @@ -6,34 +7,3 @@ export default function csv(input: any[], f: (err, data) => void) { .catch(err => f(err, null)); } -function flatten (data) { // flatten object: {a: {b: true}} -> {a.b: true} - const result = {}; - function recurse (cur, prop) { - if (Object(cur) !== cur || Object.keys(cur).length === 0) { - result[prop] = cur; - } - else if (Array.isArray(cur)) { - if (cur.length && (Object(cur[0]) !== cur || Object.keys(cur[0]).length === 0)) { // array of non-objects - result[prop] = '[' + cur.join(', ') + ']'; - } - else { - let l = 0; - for(let i = 0, l = cur.length; i < l; i++) - recurse(cur[i], prop + "[" + i + "]"); - if (l == 0) - result[prop] = []; - } - } - else { - let isEmpty = true; - for (let p in cur) { - isEmpty = false; - recurse(cur[p], prop ? prop+"."+p : p); - } - if (isEmpty && prop) - result[prop] = {}; - } - } - recurse(data, ''); - return result; -} \ No newline at end of file diff --git a/src/helpers/flatten.ts b/src/helpers/flatten.ts new file mode 100644 index 0000000..5c2d7d5 --- /dev/null +++ b/src/helpers/flatten.ts @@ -0,0 +1,41 @@ +export default function flatten (data, keepArray = false) { // flatten object: {a: {b: true}} -> {a.b: true} + const result = {}; + function recurse (cur, prop) { + if (Object(cur) !== cur || Object.keys(cur).length === 0) { + result[prop] = cur; + } + else if (prop === 'spectrum.dpt') { + console.log('dpt'); + result[prop + '.labels'] = cur.map(e => e[0]); + result[prop + '.values'] = cur.map(e => e[1]); + } + else if (Array.isArray(cur)) { + if (keepArray) { + result[prop] = cur; + } + else { // array to string + if (cur.length && (Object(cur[0]) !== cur || Object.keys(cur[0]).length === 0)) { // array of non-objects + result[prop] = '[' + cur.join(', ') + ']'; + } + else { + let l = 0; + for(let i = 0, l = cur.length; i < l; i++) + recurse(cur[i], prop + "[" + i + "]"); + if (l == 0) + result[prop] = []; + } + } + } + else { + let isEmpty = true; + for (let p in cur) { + isEmpty = false; + recurse(cur[p], prop ? prop+"."+p : p); + } + if (isEmpty && prop) + result[prop] = {}; + } + } + recurse(data, ''); + return result; +} \ No newline at end of file diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 5911876..3d038c2 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -452,7 +452,7 @@ describe('/sample', () => { it('returns a correct csv file for admins if specified', done => { TestHelper.request(server, done, { method: 'get', - url: '/samples?status[]=new&status[]=validated&page-size=2&csv=true', + url: '/samples?status[]=new&status[]=validated&page-size=2&output=csv', contentType: /text\/csv/, auth: {basic: 'admin'}, httpStatus: 200 @@ -467,11 +467,27 @@ describe('/sample', () => { it('rejects returning a csv file for a write user', done => { TestHelper.request(server, done, { method: 'get', - url: '/samples?status[]=new&status[]=validated&page-size=2&csv=true', + url: '/samples?status[]=new&status[]=validated&page-size=2&output=csv', auth: {basic: 'janedoe'}, httpStatus: 403 }); }); + it('returns the object flattened if specified', done => { + TestHelper.request(server, done, { + method: 'get', + url: '/samples?status[]=new&status[]=validated&fields[]=number&fields[]=measurements.spectrum.device&fields[]=measurements.spectrum.dpt&page-size=1&output=flatten', + auth: {basic: 'admin'}, + httpStatus: 200 + }).end((err, res) => { + if (err) return done(err); + should(res.body[0]).have.only.keys('number', 'spectrum.device', 'spectrum.dpt.labels', 'spectrum.dpt.values'); + should(res.body[0]).have.property('number', '1'); + should(res.body[0]).have.property('spectrum.device', 'Alpha I'); + should(res.body[0]).have.property('spectrum.dpt.labels', [3997.12558, 3995.08519, 3993.0448]); + should(res.body[0]).have.property('spectrum.dpt.values', [98.00555, 98.03253, 98.02657]); + done(); + }); + }); it('returns only the fields specified', done => { TestHelper.request(server, done, { method: 'get', diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 6694ad6..8a82071 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -16,6 +16,7 @@ import ConditionTemplateModel from '../models/condition_template'; import ParametersValidate from './validate/parameters'; import db from '../db'; import csv from '../helpers/csv'; +import flatten from '../helpers/flatten'; const router = express.Router(); @@ -35,7 +36,8 @@ router.get('/samples', async (req, res, next) => { if (error) return res400(error, res); // spectral data and csv not allowed for read/write users - if ((filters.fields.find(e => /\.dpt$/.test(e)) || filters.csv) && !req.auth(res, ['dev', 'admin'], 'all')) return; + if ((filters.fields.find(e => /\.dpt$/.test(e)) || filters.output !== 'json') && + !req.auth(res, ['dev', 'admin'], 'all')) return; // TODO: find a better place for these const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', @@ -394,13 +396,16 @@ router.get('/samples', async (req, res, next) => { [filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields] ); - if (filters.csv) { // output as csv + if (filters.output === 'csv') { // output as csv csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { if (err) return next(err); res.set('Content-Type', 'text/csv'); res.send(data); }); } + else if (filters.output === 'flatten') { + res.json(_.compact(data.map(e => flatten(SampleValidate.output(e, 'refs', measurementFields), true)))); + } else { // validate all and filter null values from validation errors res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); } @@ -418,6 +423,9 @@ router.get('/samples', async (req, res, next) => { delete data._id; } } + if (filters.output === 'flatten') { + data = flatten(data, true); + } res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; }); stream.on('error', err => { diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index 96cbc9c..baabecb 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -227,7 +227,7 @@ export default class SampleValidate { sort: Joi.string().pattern( new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm') ).default('_id-asc'), - csv: Joi.boolean().default(false), + output: Joi.string().valid('json', 'flatten', 'csv').default('json'), fields: Joi.array().items(Joi.string().pattern( new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm') )).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])