From f41498da53a274dda6ebe64beb951a7ea40b10bf Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 9 Jul 2020 16:30:10 +0200 Subject: [PATCH] implemented x-total-items header --- api/sample.yaml | 4 +- manifest.yml | 2 +- package.json | 2 +- src/db.ts | 6 ++- src/index.ts | 2 +- src/routes/sample.ts | 89 +++++++++++++++++++++++--------------------- 6 files changed, 57 insertions(+), 48 deletions(-) diff --git a/api/sample.yaml b/api/sample.yaml index 17df4c3..2b0ce31 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -62,8 +62,8 @@ 200: description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) headers: - X-Total-Items: - description: Total number of available items when page is specified + x-total-items: + description: Total number of available items when from-id is not specified and spectrum field is not included schema: type: integer example: 243 diff --git a/manifest.yml b/manifest.yml index 0e8c57d..dd7e0f1 100644 --- a/manifest.yml +++ b/manifest.yml @@ -3,7 +3,7 @@ applications: - name: definma-api path: dist/ instances: 1 - memory: 256M + memory: 512M stack: cflinuxfs3 buildpacks: - nodejs_buildpack diff --git a/package.json b/package.json index f9494d3..bfcdcca 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "build": "build.bat", "build-push": "build.bat && cf push", "test": "mocha dist/**/**.spec.js", - "start": "node index.js", + "start": "sleep 5s && node index.js", "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"", "loadDev": "node dist/test/loadDev.js", "coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000", diff --git a/src/db.ts b/src/db.ts index cb11af5..03c56e1 100644 --- a/src/db.ts +++ b/src/db.ts @@ -3,11 +3,15 @@ import cfenv from 'cfenv'; import _ from 'lodash'; import ChangelogModel from './models/changelog'; -mongoose.set('debug', true); // enable mongoose debug // database urls, prod db url is retrieved automatically const TESTING_URL = 'mongodb://localhost/dfopdb_test'; const DEV_URL = 'mongodb://localhost/dfopdb'; +const debugging = false; + +if (process.env.NODE_ENV !== 'production' && debugging) { + mongoose.set('debug', true); // enable mongoose debug +} export default class db { private static state = { // db object and current mode (test, dev, prod) diff --git a/src/index.ts b/src/index.ts index 4051f23..d6ea865 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,11 +26,11 @@ const port = process.env.PORT || 3000; //middleware app.use(helmet()); +app.use(contentFilter()); // filter URL query attacks app.use(express.json({ limit: '5mb'})); app.use(express.urlencoded({ extended: false, limit: '5mb' })); app.use(compression()); // compress responses app.use(bodyParser.json()); -app.use(contentFilter()); // filter URL query attacks app.use((req, res, next) => { // filter body query attacks req.body = mongoSanitize(req.body); next(); diff --git a/src/routes/sample.ts b/src/routes/sample.ts index bf741c2..f63b0b6 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -43,13 +43,13 @@ router.get('/samples', async (req, res, next) => { if (!filters['to-page']) { // set to-page default filters['to-page'] = 0; } - console.log(filters); const sortFilterKeys = filters.filters.map(e => e.field); let collection; const query = []; - query.push({$match: {$and: []}}); + let queryPtr = query; + queryPtr.push({$match: {$and: []}}); if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection collection = MeasurementModel; @@ -68,23 +68,23 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample.values[measurementParam]; } - query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort + queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered - query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); + queryPtr[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0]))); } - query.push( - ...sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements + queryPtr.push( + ...sortQuery(filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, {$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added {$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); - addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters + addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters } else { // sorting with samples as starting collection collection = SampleModel; - query[0].$match.$and.push(statusQuery(filters, 'status')); + queryPtr[0].$match.$and.push(statusQuery(filters, 'status')); if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys let sortStartValue = null; @@ -98,14 +98,14 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample[filters.sort[0]]; } - query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue)); } else { // add sort key to list to add field later sortFilterKeys.push(filters.sort[0]); } } - addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters + addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters let materialQuery = []; // put material query together separate first to reuse for first-id let materialAdded = false; @@ -136,7 +136,7 @@ router.get('/samples', async (req, res, next) => { } const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); // base material filters - query.push(...materialQuery); + queryPtr.push(...materialQuery); if (/material\./.test(filters.sort[0])) { // sort by material key let sortStartValue = null; if (filters['from-id']) { // from-id specified @@ -147,7 +147,7 @@ router.get('/samples', async (req, res, next) => { } sortStartValue = fromSample[filters.sort[0]]; } - query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue)); + queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue)); } } @@ -158,50 +158,61 @@ router.get('/samples', async (req, res, next) => { if (measurementTemplates.length < measurementFilterFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } - query.push({$lookup: { + queryPtr.push({$lookup: { from: 'measurements', let: {sId: '$_id'}, pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], as: 'measurements' }}); measurementTemplates.forEach(template => { - query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); - addFilterQueries(query, filters.filters + addFilterQueries(queryPtr, filters.filters .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) .map(e => {e.field = e.field.replace('measurements.', ''); return e; }) ); // measurement filters } - addSkipLimit(query, filters); + + if (!filters.fields.find(e => /spectrum\./.test(e)) && !filters['from-id']) { // count total number of items before $skip and $limit, only works when from-id is not specified and spectra are not included + queryPtr.push({$facet: {count: [{$count: 'count'}], samples: []}}); + queryPtr = queryPtr[queryPtr.length - 1].$facet.samples; // add rest of aggregation pipeline into $facet + } + + // paging + if (filters['to-page']) { + queryPtr.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more + } + if (filters['page-size']) { + queryPtr.push({$limit: filters['page-size']}); + } const fieldsToAdd = filters.fields.filter(e => // fields to add sortFilterKeys.indexOf(e) < 0 // field was not in filter && e !== filters.sort[0] // field was not in sort ); - console.log(fieldsToAdd); if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already - query.push( + queryPtr.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$addFields: {material: { $arrayElemAt: ['$material', 0]}}} ); } if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed - query.push( + queryPtr.push( {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); } if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed - query.push( + queryPtr.push( {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed - query.push( + queryPtr.push( {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} ); } @@ -214,45 +225,49 @@ router.get('/samples', async (req, res, next) => { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance - query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); + queryPtr.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); } else { - query.push({$lookup: { + queryPtr.push({$lookup: { from: 'measurements', let: {sId: '$_id'}, pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}], as: 'measurements' }}); } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later - query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well - query.push( + queryPtr.push( {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, {$addFields: {spectrum: '$spectrum.values'}}, {$unwind: '$spectrum'} ); } - query.push({$unset: 'measurements'}); + // queryPtr.push({$unset: 'measurements'}); + queryPtr.push({$project: {measurements: 0}}); } const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date // projection.added = {$toDate: '$_id'}; - // projection.added = { $convert: { input: '$_id', to: "date" } } // TODO + // projection.added = { $convert: { input: '$_id', to: "date" } } // TODO: upgrade MongoDB version or find alternative } if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly projection._id = false; } - query.push({$project: projection}); + queryPtr.push({$project: projection}); if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files collection.aggregate(query).exec((err, data) => { if (err) return next(err); - console.log(data.length); + if (data[0].count) { + res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0); + data = data[0].samples; + } if (filters['to-page'] < 0) { data.reverse(); } @@ -639,7 +654,7 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and }); } -function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] +function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key'] if (filters['from-id']) { // from-id specified if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}}, @@ -653,16 +668,6 @@ function sortQuery(query, filters, sortKeys, sortStartValue) { // sortKeys = [' } } -function addSkipLimit(query, filters) { - if (filters['to-page']) { - query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more - } - - if (filters['page-size']) { - query.push({$limit: filters['page-size']}); - } -} - function statusQuery(filters, field) { if (filters.hasOwnProperty('status')) { if(filters.status === 'all') { @@ -677,9 +682,9 @@ function statusQuery(filters, field) { } } -function addFilterQueries (query, filters) { // returns array of match queries from given filters +function addFilterQueries (queryPtr, filters) { // returns array of match queries from given filters if (filters.length) { - query.push({$match: {$and: filterQueries(filters)}}); + queryPtr.push({$match: {$and: filterQueries(filters)}}); } }