Archived
2

flattened samples result

This commit is contained in:
VLE2FE 2020-08-10 12:35:08 +02:00
parent 4dede74f49
commit ed8b549752
6 changed files with 75 additions and 39 deletions

View File

@ -61,12 +61,13 @@
example: '["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D", example: '["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D",
"%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22 "%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22
%5D%7D"]' %5D%7D"]'
- name: csv - name: output
description: output as csv, only available for dev and admin description: 'output format, available values are csv, json, flatten (converts material: {number: x} to
material.number: x), defaults to json'
in: query in: query
schema: schema:
type: boolean type: boolean
example: false example: csv
responses: responses:
200: 200:
description: samples overview (output depends on the fields specified)<br> description: samples overview (output depends on the fields specified)<br>

View File

@ -1,4 +1,5 @@
import {parseAsync} from 'json2csv'; import {parseAsync} from 'json2csv';
import flatten from './flatten';
export default function csv(input: any[], f: (err, data) => void) { export default function csv(input: any[], f: (err, data) => void) {
parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true}) parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
@ -6,34 +7,3 @@ export default function csv(input: any[], f: (err, data) => void) {
.catch(err => f(err, null)); .catch(err => f(err, null));
} }
function flatten (data) { // flatten object: {a: {b: true}} -> {a.b: true}
const result = {};
function recurse (cur, prop) {
if (Object(cur) !== cur || Object.keys(cur).length === 0) {
result[prop] = cur;
}
else if (Array.isArray(cur)) {
if (cur.length && (Object(cur[0]) !== cur || Object.keys(cur[0]).length === 0)) { // array of non-objects
result[prop] = '[' + cur.join(', ') + ']';
}
else {
let l = 0;
for(let i = 0, l = cur.length; i < l; i++)
recurse(cur[i], prop + "[" + i + "]");
if (l == 0)
result[prop] = [];
}
}
else {
let isEmpty = true;
for (let p in cur) {
isEmpty = false;
recurse(cur[p], prop ? prop+"."+p : p);
}
if (isEmpty && prop)
result[prop] = {};
}
}
recurse(data, '');
return result;
}

41
src/helpers/flatten.ts Normal file
View File

@ -0,0 +1,41 @@
export default function flatten (data, keepArray = false) { // flatten object: {a: {b: true}} -> {a.b: true}
const result = {};
function recurse (cur, prop) {
if (Object(cur) !== cur || Object.keys(cur).length === 0) {
result[prop] = cur;
}
else if (prop === 'spectrum.dpt') {
console.log('dpt');
result[prop + '.labels'] = cur.map(e => e[0]);
result[prop + '.values'] = cur.map(e => e[1]);
}
else if (Array.isArray(cur)) {
if (keepArray) {
result[prop] = cur;
}
else { // array to string
if (cur.length && (Object(cur[0]) !== cur || Object.keys(cur[0]).length === 0)) { // array of non-objects
result[prop] = '[' + cur.join(', ') + ']';
}
else {
let l = 0;
for(let i = 0, l = cur.length; i < l; i++)
recurse(cur[i], prop + "[" + i + "]");
if (l == 0)
result[prop] = [];
}
}
}
else {
let isEmpty = true;
for (let p in cur) {
isEmpty = false;
recurse(cur[p], prop ? prop+"."+p : p);
}
if (isEmpty && prop)
result[prop] = {};
}
}
recurse(data, '');
return result;
}

View File

@ -452,7 +452,7 @@ describe('/sample', () => {
it('returns a correct csv file for admins if specified', done => { it('returns a correct csv file for admins if specified', done => {
TestHelper.request(server, done, { TestHelper.request(server, done, {
method: 'get', method: 'get',
url: '/samples?status[]=new&status[]=validated&page-size=2&csv=true', url: '/samples?status[]=new&status[]=validated&page-size=2&output=csv',
contentType: /text\/csv/, contentType: /text\/csv/,
auth: {basic: 'admin'}, auth: {basic: 'admin'},
httpStatus: 200 httpStatus: 200
@ -467,11 +467,27 @@ describe('/sample', () => {
it('rejects returning a csv file for a write user', done => { it('rejects returning a csv file for a write user', done => {
TestHelper.request(server, done, { TestHelper.request(server, done, {
method: 'get', method: 'get',
url: '/samples?status[]=new&status[]=validated&page-size=2&csv=true', url: '/samples?status[]=new&status[]=validated&page-size=2&output=csv',
auth: {basic: 'janedoe'}, auth: {basic: 'janedoe'},
httpStatus: 403 httpStatus: 403
}); });
}); });
it('returns the object flattened if specified', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status[]=new&status[]=validated&fields[]=number&fields[]=measurements.spectrum.device&fields[]=measurements.spectrum.dpt&page-size=1&output=flatten',
auth: {basic: 'admin'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.only.keys('number', 'spectrum.device', 'spectrum.dpt.labels', 'spectrum.dpt.values');
should(res.body[0]).have.property('number', '1');
should(res.body[0]).have.property('spectrum.device', 'Alpha I');
should(res.body[0]).have.property('spectrum.dpt.labels', [3997.12558, 3995.08519, 3993.0448]);
should(res.body[0]).have.property('spectrum.dpt.values', [98.00555, 98.03253, 98.02657]);
done();
});
});
it('returns only the fields specified', done => { it('returns only the fields specified', done => {
TestHelper.request(server, done, { TestHelper.request(server, done, {
method: 'get', method: 'get',

View File

@ -16,6 +16,7 @@ import ConditionTemplateModel from '../models/condition_template';
import ParametersValidate from './validate/parameters'; import ParametersValidate from './validate/parameters';
import db from '../db'; import db from '../db';
import csv from '../helpers/csv'; import csv from '../helpers/csv';
import flatten from '../helpers/flatten';
const router = express.Router(); const router = express.Router();
@ -35,7 +36,8 @@ router.get('/samples', async (req, res, next) => {
if (error) return res400(error, res); if (error) return res400(error, res);
// spectral data and csv not allowed for read/write users // spectral data and csv not allowed for read/write users
if ((filters.fields.find(e => /\.dpt$/.test(e)) || filters.csv) && !req.auth(res, ['dev', 'admin'], 'all')) return; if ((filters.fields.find(e => /\.dpt$/.test(e)) || filters.output !== 'json') &&
!req.auth(res, ['dev', 'admin'], 'all')) return;
// TODO: find a better place for these // TODO: find a better place for these
const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id',
@ -394,13 +396,16 @@ router.get('/samples', async (req, res, next) => {
[filters.sort[0].split('.')[1], [filters.sort[0].split('.')[1],
...measurementFilterFields, ...measurementFieldsFields] ...measurementFilterFields, ...measurementFieldsFields]
); );
if (filters.csv) { // output as csv if (filters.output === 'csv') { // output as csv
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
if (err) return next(err); if (err) return next(err);
res.set('Content-Type', 'text/csv'); res.set('Content-Type', 'text/csv');
res.send(data); res.send(data);
}); });
} }
else if (filters.output === 'flatten') {
res.json(_.compact(data.map(e => flatten(SampleValidate.output(e, 'refs', measurementFields), true))));
}
else { // validate all and filter null values from validation errors else { // validate all and filter null values from validation errors
res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));
} }
@ -418,6 +423,9 @@ router.get('/samples', async (req, res, next) => {
delete data._id; delete data._id;
} }
} }
if (filters.output === 'flatten') {
data = flatten(data, true);
}
res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++;
}); });
stream.on('error', err => { stream.on('error', err => {

View File

@ -227,7 +227,7 @@ export default class SampleValidate {
sort: Joi.string().pattern( sort: Joi.string().pattern(
new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm') new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')
).default('_id-asc'), ).default('_id-asc'),
csv: Joi.boolean().default(false), output: Joi.string().valid('json', 'flatten', 'csv').default('json'),
fields: Joi.array().items(Joi.string().pattern( fields: Joi.array().items(Joi.string().pattern(
new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm') new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')
)).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']) )).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])