banana/definma-api
Archived
2
Fork 0
Code Releases Activity

spectrum field working again

Browse Source
This commit is contained in:
VLE2FE
2020-07-09 13:48:27 +02:00
parent 6a02f09e7f
commit 1ddc2b617a
13 changed files with 269 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/api.ts
View File

@@ -18,6 +18,7 @@ export default class api {
jsonRefParser.bundle('api/api.yaml', (err, doc) => { // parse yaml
if (err) throw err;
apiDoc = doc;
apiDoc.servers.splice(process.env.NODE_ENV === 'production', 1);
apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e)); // bundle routes
apiDoc = this.resolveXDoc(apiDoc);
oasParser.validate(apiDoc, (err, api) => { // validate oas schema

6
src/helpers/csv.ts
View File

@@ -1,12 +1,6 @@
import {parseAsync} from 'json2csv';
export default function csv(input: any[], f: (err, data) => void) {
console.log(input[1000]);
console.log(flatten(input[1000]));
parseAsync([flatten(input[1000])]).then(csv => console.log(csv));
console.log(input[1]);
console.log(flatten(input[1]));
parseAsync([flatten(input[1])]).then(csv => console.log(csv));
parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
.then(csv => f(null, csv))
.catch(err => f(err, null));

2
src/helpers/mail.ts
View File

@@ -17,7 +17,7 @@ export default (mailAddress, subject, content, f) => { // callback, executed em
contentType: "text/html"
},
from: {
eMail: "dfop@bosch-iot.com",
eMail: "definma@bosch-iot.com",
password: "PlasticsOfFingerprintDigital"
}
}

3
src/index.ts
View File

@@ -4,6 +4,7 @@ import compression from 'compression';
import contentFilter from 'content-filter';
import mongoSanitize from 'mongo-sanitize';
import helmet from 'helmet';
import cors from 'cors';
import api from './api';
import db from './db';
@@ -42,9 +43,11 @@ app.use((req, res, next) => { // no database connection error
next();
}
else {
console.error('No database connection');
res.status(500).send({status: 'Internal server error'});
}
});
app.use(cors()); // CORS headers
app.use(require('./helpers/authorize')); // handle authentication
// redirect /api routes for Angular proxy in development

1
src/routes/sample.spec.ts
View File

@@ -21,6 +21,7 @@ describe('/sample', () => {
// TODO: sort, added date filter, has measurements/condition filter
// TODO: check if conditions work in sort/fields/filters
// TODO: test for numbers as strings in glass_fiber
describe('GET /samples', () => {
it('returns all samples', done => {
TestHelper.request(server, done, {

95
src/routes/sample.ts
View File

@@ -21,6 +21,12 @@ import csv from '../helpers/csv';
const router = express.Router();
// TODO: check added filter
// TODO: return total number of pages -> use facet
// TODO: use query pointer
// TODO: convert filter value to number according to table model
// TODO: validation for filter parameters
// TODO: location/device sort/filter
router.get('/samples', async (req, res, next) => {
if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
@@ -37,6 +43,7 @@ router.get('/samples', async (req, res, next) => {
if (!filters['to-page']) { // set to-page default
filters['to-page'] = 0;
}
console.log(filters);
const sortFilterKeys = filters.filters.map(e => e.field);
@@ -70,7 +77,7 @@ router.get('/samples', async (req, res, next) => {
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
{$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added
{$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
{$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
);
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
@@ -106,25 +113,25 @@ router.get('/samples', async (req, res, next) => {
materialAdded = true;
materialQuery.push( // add material properties
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields
{$set: {material: {$arrayElemAt: ['$material', 0]}}}
{$addFields: {material: {$arrayElemAt: ['$material', 0]}}}
);
const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters
if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed
materialQuery.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
{$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
);
}
if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed
materialQuery.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed
materialQuery.push(
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
{$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
);
}
const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
@@ -157,10 +164,10 @@ router.get('/samples', async (req, res, next) => {
as: 'measurements'
}});
measurementTemplates.forEach(template => {
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
}}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
});
addFilterQueries(query, filters.filters
.filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
@@ -173,39 +180,40 @@ router.get('/samples', async (req, res, next) => {
sortFilterKeys.indexOf(e) < 0 // field was not in filter
&& e !== filters.sort[0] // field was not in sort
);
console.log(fieldsToAdd);
if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already
query.push(
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$set: {material: { $arrayElemAt: ['$material', 0]}}}
{$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed
query.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
{$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed
query.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed
query.push(
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
{$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
);
}
let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields
const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
if (measurementTemplates instanceof Error) return;
if (measurementTemplates.length < measurementFieldsFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance
if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance
query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
}
else {
@@ -216,15 +224,15 @@ router.get('/samples', async (req, res, next) => {
}});
}
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
}}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
});
if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
query.push(
{$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
{$set: {spectrum: '$spectrum.values.dpt'}},
{$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
{$addFields: {spectrum: '$spectrum.values'}},
{$unwind: '$spectrum'}
);
}
@@ -233,30 +241,45 @@ router.get('/samples', async (req, res, next) => {
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
if (filters.fields.indexOf('added') >= 0) { // add added date
projection.added = {$toDate: '$_id'};
// projection.added = {$toDate: '$_id'};
// projection.added = { $convert: { input: '$_id', to: "date" } } // TODO
}
if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly
projection._id = false;
}
query.push({$project: projection});
collection.aggregate(query).exec((err, data) => {
if (err) return next(err);
if (filters['to-page'] < 0) {
data.reverse();
}
const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]);
if (filters.csv) { // output as csv
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
if (err) return next(err);
res.set('Content-Type', 'text/csv');
res.send(data);
});
}
else {
res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors
}
})
if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files
collection.aggregate(query).exec((err, data) => {
if (err) return next(err);
console.log(data.length);
if (filters['to-page'] < 0) {
data.reverse();
}
const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]);
if (filters.csv) { // output as csv
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
if (err) return next(err);
res.set('Content-Type', 'text/csv');
res.send(data);
});
}
else {
res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors
}
});
}
else {
res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
res.write('[');
let count = 0;
const stream = collection.aggregate(query).cursor().exec();
stream.on('data', data => { res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; });
stream.on('close', () => {
res.write(']');
res.end();
});
}
});
router.get('/samples/:state(new|deleted)', (req, res, next) => {
@@ -537,7 +560,7 @@ async function materialCheck (sample, res, next, id = sample.material_id) { //
res.status(400).json({status: 'Material not available'});
return false;
}
if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified
if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified
res.status(400).json({status: 'Color not available for material'});
return false;
}

32
src/routes/validate/sample.ts
View File

@@ -11,7 +11,8 @@ export default class SampleValidate {
.max(128),
color: Joi.string()
.max(128),
.max(128)
.allow(''),
type: Joi.string()
.max(128),
@@ -77,7 +78,7 @@ export default class SampleValidate {
'user_id',
'material._id',
'material.numbers',
'measurements.spectrum'
'measurements.spectrum.dpt'
];
static input (data, param) { // validate input, set param to 'new' to make all attributes required
@@ -170,6 +171,33 @@ export default class SampleValidate {
try {
for (let i in data.filters) {
data.filters[i] = JSON.parse(data.filters[i]);
data.filters[i].values = data.filters[i].values.map(e => { // validate filter values
let validator;
let field = data.filters[i].field
if (/material\./.test(field)) { // select right validation model
validator = MaterialValidate.outputV();
field = field.replace('material.', '');
}
else if (/measurements\./.test(field)) {
validator = Joi.object({
value: Joi.alternatives()
.try(
Joi.string().max(128),
Joi.number(),
Joi.boolean(),
Joi.array()
)
.allow(null)
});
field = 'value';
}
else {
validator = Joi.object(this.sample);
}
const {value, error} = validator.validate({[field]: e});
if (error) throw error; // reject invalid values
return value[field];
});
}
}
catch {