banana/definma-api
Archived
2
Fork 0
Code Releases Activity

implementation of measurement fields

Browse Source
This commit is contained in:
VLE2FE 2020-06-30 14:16:37 +02:00
parent 52eb828bea
commit 8cf1c14d88
7 changed files with 145 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/sample.yaml
View File

@ -42,12 +42,14 @@
schema:
type: boolean
example: false
- name: fields
- name: fields[]
description: the fields to include in the output as array, defaults to ['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']
in: query
schema:
type: array
items:
type: string
example: '&fields[]=number&fields[]=batch'
example: ['number', 'batch']
responses:
200:
description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)

4
data_import/import.js
View File

@ -32,10 +32,10 @@ async function main() {
await allSamples();
await saveSamples();
}
else if (0) { // DPT
else if (1) { // DPT
await allDpts();
}
else if (1) { // KF/VZ
else if (0) { // KF/VZ
await importCsv();
await allKfVz();
}

31
src/helpers/csv.ts
View File

@ -1,7 +1,34 @@
import {parseAsync} from 'json2csv';
export default function csv(input: object, fields: string[], f: (err, data) => void) {
parseAsync(input)
export default function csv(input: any[], f: (err, data) => void) {
parseAsync(input.map(e => flatten(e)))
.then(csv => f(null, csv))
.catch(err => f(err, null));
}
function flatten (data) { // flatten object: {a: {b: true}} -> {a.b: true}
const result = {};
function recurse (cur, prop) {
if (Object(cur) !== cur || Object.keys(cur).length === 0) {
result[prop] = cur;
}
else if (Array.isArray(cur)) {
let l = 0;
for(let i = 0, l = cur.length; i < l; i++)
recurse(cur[i], prop + "[" + i + "]");
if (l == 0)
result[prop] = [];
}
else {
let isEmpty = true;
for (let p in cur) {
isEmpty = false;
recurse(cur[p], prop ? prop+"."+p : p);
}
if (isEmpty && prop)
result[prop] = {};
}
}
recurse(data, '');
return result;
}

55
src/routes/sample.spec.ts
View File

@ -244,6 +244,42 @@ describe('/sample', () => {
done();
});
});
it('adds the specified measurements', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=measurements.kf',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body.find(e => e.number === '1')).have.property('kf', {});
should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.5, 'standard deviation': null});
done();
});
});
it('multiplies the sample information for each spectrum', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=measurements.spectrum',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(2);
should(res.body[0]).have.property('spectrum', [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
should(res.body[1]).have.property('spectrum', [[3996.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
done();
});
});
it('rejects unknown measurement names', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=measurements.xx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: 'Measurement key not found'}
});
});
it('returns a correct csv file if specified', done => {
TestHelper.request(server, done, {
method: 'get',
@ -253,9 +289,9 @@ describe('/sample', () => {
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.text).be.eql('"_id","number","type","color","batch","condition","material_id","note_id","user_id","added"\r\n' +
'"400000000000000000000001","1","granulate","black","","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' +
'"400000000000000000000002","21","granulate","natural","1560237365","{""material"":""copper"",""weeks"":3,""condition_template"":""200000000000000000000001""}","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"');
should(res.text).be.eql('"_id","number","type","color","batch","condition.material","condition.weeks","condition.condition_template","material_id","note_id","user_id","added"\r\n' +
'"400000000000000000000001","1","granulate","black","","copper",3,"200000000000000000000001","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' +
'"400000000000000000000002","21","granulate","natural","1560237365","copper",3,"200000000000000000000001","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"');
done();
});
});
@ -268,6 +304,15 @@ describe('/sample', () => {
res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', mineral: 0}}]
});
});
it('rejects a from-id not in the database', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?from-id=5ea0450ed851c30a90e70894&sort=color-asc',
auth: {basic: 'admin'},
httpStatus: 400,
res: {status: 'Invalid body format', details: 'from-id not found'}
});
});
it('rejects an invalid fields parameter', done => {
TestHelper.request(server, done, {
method: 'get',
@ -283,7 +328,7 @@ describe('/sample', () => {
url: '/samples?status=all&page-size=1&fields[]=xx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"fields[0]" must be one of [_id, color, number, type, batch, added, material.name, material.supplier, material.group, material.mineral, material.glass_fiber, material.carbon_fiber, material.number, condition, material_id, note_id, user_id, material._id, material.numbers]'}
res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\..+)$/m'}
});
});
it('rejects a negative page size', done => {
@ -329,7 +374,7 @@ describe('/sample', () => {
httpStatus: 401
});
});
}); // TODO: measurement fields
});
describe('GET /samples/{state}', () => {
it('returns all new samples', done => {

46
src/routes/sample.ts
View File

@ -6,6 +6,7 @@ import NoteFieldValidate from './validate/note_field';
import res400 from './validate/res400';
import SampleModel from '../models/sample'
import MeasurementModel from '../models/measurement';
import MeasurementTemplateModel from '../models/measurement_template';
import MaterialModel from '../models/material';
import NoteModel from '../models/note';
import NoteFieldModel from '../models/note_field';
@ -63,6 +64,9 @@ router.get('/samples', async (req, res, next) => {
if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);});
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc
query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
@ -77,10 +81,6 @@ router.get('/samples', async (req, res, next) => {
query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}}); // set _id as secondary sort
}
// if (filters.sort[0].indexOf('material.') >= 0) { // unpopulate materials again
// query.push({$unset: 'material'});
// }
if (filters['to-page']) {
query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more
}
@ -88,13 +88,38 @@ router.get('/samples', async (req, res, next) => {
if (filters['page-size']) {
query.push({$limit: filters['page-size']});
}
console.log(filters.fields);
const projection = filters.fields.reduce((s, e) => {s[e] = true; return s; }, {});
let measurementFields = [];
if (filters.fields.find(e => /measurements\./.test(e))) { // joining measurements is required
query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', ''));
const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
if (measurementTemplates instanceof Error) return;
if (measurementTemplates.length < measurementFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}});
});
console.log(measurementFields);
if (measurementFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
query.push(
{$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
{$set: {spectrum: '$spectrum.values.dpt'}},
{$unwind: '$spectrum'}
);
}
query.push({$unset: 'measurements'});
}
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
if (filters.fields.indexOf('added') >= 0) { // add added date
projection.added = {$toDate: '$_id'};
}
if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly
console.log('disable id');
projection._id = false;
}
query.push({$project: projection});
@ -104,15 +129,16 @@ router.get('/samples', async (req, res, next) => {
if (filters['to-page'] < 0) {
data.reverse();
}
if (filters.csv) { // output as csv // TODO: csv example in OAS
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs'))), ['_id', 'number'], (err, data) => {
console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));
if (filters.csv) { // output as csv
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
if (err) return next(err);
res.set('Content-Type', 'text/csv');
res.send(data);
});
}
else {
res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs')))); // validate all and filter null values from validation errors
res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors
}
})
});

13
src/routes/validate/sample.ts
View File

@ -71,10 +71,12 @@ export default class SampleValidate {
...SampleValidate.sortKeys,
'condition',
'material_id',
'material',
'note_id',
'user_id',
'material._id',
'material.numbers'
'material.numbers',
'measurements.*'
];
static input (data, param) { // validate input, set param to 'new' to make all attributes required
@ -114,7 +116,7 @@ export default class SampleValidate {
}
}
static output (data, param = 'refs+added') { // validate output and strip unwanted properties, returns null if not valid
static output (data, param = 'refs+added', additionalParams = []) { // validate output and strip unwanted properties, returns null if not valid
if (param === 'refs+added') {
param = 'refs';
data.added = data._id.getTimestamp();
@ -130,7 +132,7 @@ export default class SampleValidate {
batch: this.sample.batch,
condition: this.sample.condition,
material_id: IdValidate.get(),
material: MaterialValidate.outputV(),
material: MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}),
note_id: IdValidate.get().allow(null),
user_id: IdValidate.get(),
added: this.sample.added
@ -153,6 +155,9 @@ export default class SampleValidate {
else {
return null;
}
additionalParams.forEach(param => {
joiObject[param] = Joi.any();
});
const {value, error} = Joi.object(joiObject).validate(data, {stripUnknown: true});
return error !== undefined? null : value;
}
@ -165,7 +170,7 @@ export default class SampleValidate {
'page-size': Joi.number().integer().min(1),
sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.') + ')-(asc|desc)$', 'm')).default('_id-asc'),
csv: Joi.boolean().default(false),
fields: Joi.array().items(Joi.string().valid(...this.fieldKeys)).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])
fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added'])
}).with('to-page', 'page-size').validate(data);
}
}

14
src/test/db.json
View File

@ -417,6 +417,20 @@
"status": 0,
"measurement_template": {"$oid":"300000000000000000000002"},
"__v": 0
},
{
"_id": {"$oid":"800000000000000000000007"},
"sample_id": {"$oid":"400000000000000000000001"},
"values": {
"dpt": [
[3996.12558,98.00555],
[3995.08519,98.03253],
[3993.04480,98.02657]
]
},
"status": 10,
"measurement_template": {"$oid":"300000000000000000000001"},
"__v": 0
}
],
"condition_templates": [