banana/definma-api
Archived
2
Fork 0
Code Releases Activity

implemented x-total-items header

Browse Source
This commit is contained in:
VLE2FE 2020-07-09 16:30:10 +02:00
parent 1ddc2b617a
commit f41498da53
6 changed files with 57 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/sample.yaml
View File

@ -62,8 +62,8 @@
200:
description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)
headers:
X-Total-Items:
description: Total number of available items when page is specified
x-total-items:
description: Total number of available items when from-id is not specified and spectrum field is not included
schema:
type: integer
example: 243

2
manifest.yml
View File

@ -3,7 +3,7 @@ applications:
- name: definma-api
path: dist/
instances: 1
memory: 256M
memory: 512M
stack: cflinuxfs3
buildpacks:
- nodejs_buildpack

2
package.json
View File

@ -9,7 +9,7 @@
"build": "build.bat",
"build-push": "build.bat && cf push",
"test": "mocha dist/**/**.spec.js",
"start": "node index.js",
"start": "sleep 5s && node index.js",
"dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"",
"loadDev": "node dist/test/loadDev.js",
"coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000",

6
src/db.ts
View File

@ -3,11 +3,15 @@ import cfenv from 'cfenv';
import _ from 'lodash';
import ChangelogModel from './models/changelog';
mongoose.set('debug', true); // enable mongoose debug
// database urls, prod db url is retrieved automatically
const TESTING_URL = 'mongodb://localhost/dfopdb_test';
const DEV_URL = 'mongodb://localhost/dfopdb';
const debugging = false;
if (process.env.NODE_ENV !== 'production' && debugging) {
mongoose.set('debug', true); // enable mongoose debug
}
export default class db {
private static state = { // db object and current mode (test, dev, prod)

2
src/index.ts
View File

@ -26,11 +26,11 @@ const port = process.env.PORT || 3000;
//middleware
app.use(helmet());
app.use(contentFilter()); // filter URL query attacks
app.use(express.json({ limit: '5mb'}));
app.use(express.urlencoded({ extended: false, limit: '5mb' }));
app.use(compression()); // compress responses
app.use(bodyParser.json());
app.use(contentFilter()); // filter URL query attacks
app.use((req, res, next) => { // filter body query attacks
req.body = mongoSanitize(req.body);
next();

89
src/routes/sample.ts
View File

@ -43,13 +43,13 @@ router.get('/samples', async (req, res, next) => {
if (!filters['to-page']) { // set to-page default
filters['to-page'] = 0;
}
console.log(filters);
const sortFilterKeys = filters.filters.map(e => e.field);
let collection;
const query = [];
query.push({$match: {$and: []}});
let queryPtr = query;
queryPtr.push({$match: {$and: []}});
if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection
collection = MeasurementModel;
@ -68,23 +68,23 @@ router.get('/samples', async (req, res, next) => {
}
sortStartValue = fromSample.values[measurementParam];
}
query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort
queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort
if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered
query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0])));
queryPtr[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0])));
}
query.push(
...sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements
queryPtr.push(
...sortQuery(filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
{$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added
{$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
);
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
}
else { // sorting with samples as starting collection
collection = SampleModel;
query[0].$match.$and.push(statusQuery(filters, 'status'));
queryPtr[0].$match.$and.push(statusQuery(filters, 'status'));
if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys
let sortStartValue = null;
@ -98,14 +98,14 @@ router.get('/samples', async (req, res, next) => {
}
sortStartValue = fromSample[filters.sort[0]];
}
query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
}
else { // add sort key to list to add field later
sortFilterKeys.push(filters.sort[0]);
}
}
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
let materialQuery = []; // put material query together separate first to reuse for first-id
let materialAdded = false;
@ -136,7 +136,7 @@ router.get('/samples', async (req, res, next) => {
}
const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); // base material filters
query.push(...materialQuery);
queryPtr.push(...materialQuery);
if (/material\./.test(filters.sort[0])) { // sort by material key
let sortStartValue = null;
if (filters['from-id']) { // from-id specified
@ -147,7 +147,7 @@ router.get('/samples', async (req, res, next) => {
}
sortStartValue = fromSample[filters.sort[0]];
}
query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
}
}
@ -158,50 +158,61 @@ router.get('/samples', async (req, res, next) => {
if (measurementTemplates.length < measurementFilterFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
query.push({$lookup: {
queryPtr.push({$lookup: {
from: 'measurements', let: {sId: '$_id'},
pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
as: 'measurements'
}});
measurementTemplates.forEach(template => {
query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
});
addFilterQueries(query, filters.filters
addFilterQueries(queryPtr, filters.filters
.filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
.map(e => {e.field = e.field.replace('measurements.', ''); return e; })
); // measurement filters
}
addSkipLimit(query, filters);
if (!filters.fields.find(e => /spectrum\./.test(e)) && !filters['from-id']) { // count total number of items before $skip and $limit, only works when from-id is not specified and spectra are not included
queryPtr.push({$facet: {count: [{$count: 'count'}], samples: []}});
queryPtr = queryPtr[queryPtr.length - 1].$facet.samples; // add rest of aggregation pipeline into $facet
}
// paging
if (filters['to-page']) {
queryPtr.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more
}
if (filters['page-size']) {
queryPtr.push({$limit: filters['page-size']});
}
const fieldsToAdd = filters.fields.filter(e => // fields to add
sortFilterKeys.indexOf(e) < 0 // field was not in filter
&& e !== filters.sort[0] // field was not in sort
);
console.log(fieldsToAdd);
if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already
query.push(
queryPtr.push(
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed
query.push(
queryPtr.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed
query.push(
queryPtr.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed
query.push(
queryPtr.push(
{$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
);
}
@ -214,45 +225,49 @@ router.get('/samples', async (req, res, next) => {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance
query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
queryPtr.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
}
else {
query.push({$lookup: {
queryPtr.push({$lookup: {
from: 'measurements', let: {sId: '$_id'},
pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
as: 'measurements'
}});
}
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
});
if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
query.push(
queryPtr.push(
{$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
{$addFields: {spectrum: '$spectrum.values'}},
{$unwind: '$spectrum'}
);
}
query.push({$unset: 'measurements'});
// queryPtr.push({$unset: 'measurements'});
queryPtr.push({$project: {measurements: 0}});
}
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
if (filters.fields.indexOf('added') >= 0) { // add added date
// projection.added = {$toDate: '$_id'};
// projection.added = { $convert: { input: '$_id', to: "date" } } // TODO
// projection.added = { $convert: { input: '$_id', to: "date" } } // TODO: upgrade MongoDB version or find alternative
}
if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly
projection._id = false;
}
query.push({$project: projection});
queryPtr.push({$project: projection});
if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files
collection.aggregate(query).exec((err, data) => {
if (err) return next(err);
console.log(data.length);
if (data[0].count) {
res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0);
data = data[0].samples;
}
if (filters['to-page'] < 0) {
data.reverse();
}
@ -639,7 +654,7 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and
});
}
function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key']
function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key']
if (filters['from-id']) { // from-id specified
if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc
return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
@ -653,16 +668,6 @@ function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['
}
}
function addSkipLimit(query, filters) {
if (filters['to-page']) {
query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more
}
if (filters['page-size']) {
query.push({$limit: filters['page-size']});
}
}
function statusQuery(filters, field) {
if (filters.hasOwnProperty('status')) {
if(filters.status === 'all') {
@ -677,9 +682,9 @@ function statusQuery(filters, field) {
}
}
function addFilterQueries (query, filters) { // returns array of match queries from given filters
function addFilterQueries (queryPtr, filters) { // returns array of match queries from given filters
if (filters.length) {
query.push({$match: {$and: filterQueries(filters)}});
queryPtr.push({$match: {$and: filterQueries(filters)}});
}
}