implemented x-total-items header

api/sample.yaml

@@ -62,8 +62,8 @@
       200:
         description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)
         headers:
-          X-Total-Items:
+          x-total-items:
-            description: Total number of available items when page is specified
+            description: Total number of available items when from-id is not specified and spectrum field is not included
             schema:
               type: integer
               example: 243

manifest.yml

@@ -3,7 +3,7 @@ applications:
   - name: definma-api
     path: dist/
     instances: 1
-    memory: 256M
+    memory: 512M
     stack: cflinuxfs3
     buildpacks:
       - nodejs_buildpack

package.json

@@ -9,7 +9,7 @@
     "build": "build.bat",
     "build-push": "build.bat && cf push",
     "test": "mocha dist/**/**.spec.js",
-    "start": "node index.js",
+    "start": "sleep 5s && node index.js",
     "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"",
     "loadDev": "node dist/test/loadDev.js",
     "coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000",

src/db.ts

@@ -3,11 +3,15 @@ import cfenv from 'cfenv';
 import _ from 'lodash';
 import ChangelogModel from './models/changelog';
 
-mongoose.set('debug', true);  // enable mongoose debug
 
 // database urls, prod db url is retrieved automatically
 const TESTING_URL = 'mongodb://localhost/dfopdb_test';
 const DEV_URL = 'mongodb://localhost/dfopdb';
+const debugging = false;
+
+if (process.env.NODE_ENV !== 'production' && debugging) {
+  mongoose.set('debug', true);  // enable mongoose debug
+}
 
 export default class db {
   private static state = {  // db object and current mode (test, dev, prod)

src/index.ts

@@ -26,11 +26,11 @@ const port = process.env.PORT || 3000;
 
 //middleware
 app.use(helmet());
+app.use(contentFilter());  // filter URL query attacks
 app.use(express.json({ limit: '5mb'}));
 app.use(express.urlencoded({ extended: false, limit: '5mb' }));
 app.use(compression());  // compress responses
 app.use(bodyParser.json());
-app.use(contentFilter());  // filter URL query attacks
 app.use((req, res, next) => {  // filter body query attacks
   req.body = mongoSanitize(req.body);
   next();

src/routes/sample.ts

@@ -43,13 +43,13 @@ router.get('/samples', async (req, res, next) => {
   if (!filters['to-page']) {  // set to-page default
     filters['to-page'] = 0;
   }
-  console.log(filters);
 
   const sortFilterKeys = filters.filters.map(e => e.field);
 
   let collection;
   const query = [];
-  query.push({$match: {$and: []}});
+  let queryPtr = query;
+  queryPtr.push({$match: {$and: []}});
 
   if (filters.sort[0].indexOf('measurements.') >= 0) {  // sorting with measurements as starting collection
     collection = MeasurementModel;
@@ -68,23 +68,23 @@ router.get('/samples', async (req, res, next) => {
       }
       sortStartValue = fromSample.values[measurementParam];
     }
-    query[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)});  // find measurements to sort
+    queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)});  // find measurements to sort
     if (filters.filters.find(e => e.field === filters.sort[0])) {  // sorted measurement should also be filtered
-      query[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0])));
+      queryPtr[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0])));
     }
-    query.push(
+    queryPtr.push(
-      ...sortQuery(query, filters, ['values.' + measurementParam, 'sample_id'], sortStartValue),  // sort measurements
+      ...sortQuery(filters, ['values.' + measurementParam, 'sample_id'], sortStartValue),  // sort measurements
       {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}},                                  // fetch samples and restructure them to fit sample structure
       {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
       {$match: statusQuery(filters, 'sample.status')},  // filter out wrong status once samples were added
       {$addFields: {['sample.' + measurementName]: '$measurement.values'}},  // more restructuring
       {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
     );
-    addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0));  // sample filters
+    addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0));  // sample filters
   }
   else {  // sorting with samples as starting collection
     collection = SampleModel;
-    query[0].$match.$and.push(statusQuery(filters, 'status'));
+    queryPtr[0].$match.$and.push(statusQuery(filters, 'status'));
 
     if (sampleKeys.indexOf(filters.sort[0]) >= 0) {  // sorting for sample keys
       let sortStartValue = null;
@@ -98,14 +98,14 @@ router.get('/samples', async (req, res, next) => {
         }
         sortStartValue = fromSample[filters.sort[0]];
       }
-      query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
+      queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
     }
     else {  // add sort key to list to add field later
       sortFilterKeys.push(filters.sort[0]);
     }
   }
 
-  addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0));  // sample filters
+  addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0));  // sample filters
 
   let materialQuery = [];  // put material query together separate first to reuse for first-id
   let materialAdded = false;
@@ -136,7 +136,7 @@ router.get('/samples', async (req, res, next) => {
     }
     const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
     addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0));  // base material filters
-    query.push(...materialQuery);
+    queryPtr.push(...materialQuery);
     if (/material\./.test(filters.sort[0])) {  // sort by material key
       let sortStartValue = null;
       if (filters['from-id']) {  // from-id specified
@@ -147,7 +147,7 @@ router.get('/samples', async (req, res, next) => {
         }
         sortStartValue = fromSample[filters.sort[0]];
       }
-      query.push(...sortQuery(query, filters, [filters.sort[0], '_id'], sortStartValue));
+      queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
     }
   }
 
@@ -158,50 +158,61 @@ router.get('/samples', async (req, res, next) => {
     if (measurementTemplates.length < measurementFilterFields.length) {
       return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
     }
-    query.push({$lookup: {
+    queryPtr.push({$lookup: {
         from: 'measurements', let: {sId: '$_id'},
         pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
         as: 'measurements'
     }});
     measurementTemplates.forEach(template => {
-      query.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
+      queryPtr.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
         vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
         in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
       }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
     });
-    addFilterQueries(query, filters.filters
+    addFilterQueries(queryPtr, filters.filters
       .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
       .map(e => {e.field = e.field.replace('measurements.', ''); return e; })
     );  // measurement filters
   }
-  addSkipLimit(query, filters);
+
+  if (!filters.fields.find(e => /spectrum\./.test(e)) && !filters['from-id']) {  // count total number of items before $skip and $limit, only works when from-id is not specified and spectra are not included
+    queryPtr.push({$facet: {count: [{$count: 'count'}], samples: []}});
+    queryPtr = queryPtr[queryPtr.length - 1].$facet.samples;  // add rest of aggregation pipeline into $facet
+  }
+
+  // paging
+  if (filters['to-page']) {
+    queryPtr.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)})  // number to skip, if going back pages, one page has to be skipped less but on sample more
+  }
+  if (filters['page-size']) {
+    queryPtr.push({$limit: filters['page-size']});
+  }
 
   const fieldsToAdd = filters.fields.filter(e =>   // fields to add
     sortFilterKeys.indexOf(e) < 0                      // field was not in filter
     && e !== filters.sort[0]                           // field was not in sort
   );
-  console.log(fieldsToAdd);
 
   if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) {  // add material, was not added already
-    query.push(
+    queryPtr.push(
       {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
       {$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
     );
   }
   if (fieldsToAdd.indexOf('material.supplier') >= 0) {  // add supplier if needed
-    query.push(
+    queryPtr.push(
       {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
       {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
     );
   }
   if (fieldsToAdd.indexOf('material.group') >= 0) {  // add group if needed
-    query.push(
+    queryPtr.push(
       {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
       {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
     );
   }
   if (fieldsToAdd.indexOf('material.number') >= 0) {  // add material number if needed
-    query.push(
+    queryPtr.push(
       {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
     );
   }
@@ -214,45 +225,49 @@ router.get('/samples', async (req, res, next) => {
       return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
     }
     if (fieldsToAdd.find(e => /spectrum\./.test(e))) {  // use different lookup methods with and without spectrum for the best performance
-      query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
+      queryPtr.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
     }
     else {
-      query.push({$lookup: {
+      queryPtr.push({$lookup: {
           from: 'measurements', let: {sId: '$_id'},
           pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
           as: 'measurements'
         }});
     }
     measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => {  // TODO: hard coded dpt for special treatment, change later
-      query.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
+      queryPtr.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
               vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
               in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
             }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
     });
     if (measurementFieldsFields.find(e => e === 'spectrum')) {  // TODO: remove hardcoded as well
-      query.push(
+      queryPtr.push(
         {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
         {$addFields: {spectrum: '$spectrum.values'}},
         {$unwind: '$spectrum'}
       );
     }
-    query.push({$unset: 'measurements'});
+    // queryPtr.push({$unset: 'measurements'});
+    queryPtr.push({$project: {measurements: 0}});
   }
 
   const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
   if (filters.fields.indexOf('added') >= 0) {  // add added date
     // projection.added = {$toDate: '$_id'};
-    // projection.added = { $convert: { input: '$_id', to: "date" } }  // TODO
+    // projection.added = { $convert: { input: '$_id', to: "date" } }  // TODO: upgrade MongoDB version or find alternative
   }
   if (!(filters.fields.indexOf('_id') >= 0)) {  // disable _id explicitly
     projection._id = false;
   }
-  query.push({$project: projection});
+  queryPtr.push({$project: projection});
 
   if (!fieldsToAdd.find(e => /spectrum\./.test(e))) {  // use streaming when including spectrum files
     collection.aggregate(query).exec((err, data) => {
       if (err) return next(err);
-      console.log(data.length);
+      if (data[0].count) {
+        res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0);
+        data = data[0].samples;
+      }
       if (filters['to-page'] < 0) {
         data.reverse();
       }
@@ -639,7 +654,7 @@ function customFieldsChange (fields, amount, req) {  // update custom_fields and
   });
 }
 
-function sortQuery(query, filters, sortKeys, sortStartValue) {  // sortKeys = ['primary key', 'secondary key']
+function sortQuery(filters, sortKeys, sortStartValue) {  // sortKeys = ['primary key', 'secondary key']
   if (filters['from-id']) {  // from-id specified
     if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) {  // asc
       return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
@@ -653,16 +668,6 @@ function sortQuery(query, filters, sortKeys, sortStartValue) {  // sortKeys = ['
   }
 }
 
-function addSkipLimit(query, filters) {
-  if (filters['to-page']) {
-    query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)})  // number to skip, if going back pages, one page has to be skipped less but on sample more
-  }
-
-  if (filters['page-size']) {
-    query.push({$limit: filters['page-size']});
-  }
-}
-
 function statusQuery(filters, field) {
   if (filters.hasOwnProperty('status')) {
     if(filters.status === 'all') {
@@ -677,9 +682,9 @@ function statusQuery(filters, field) {
   }
 }
 
-function addFilterQueries (query, filters) {  // returns array of match queries from given filters
+function addFilterQueries (queryPtr, filters) {  // returns array of match queries from given filters
   if (filters.length) {
-    query.push({$match: {$and: filterQueries(filters)}});
+    queryPtr.push({$match: {$and: filterQueries(filters)}});
   }
 }