added model templates

api/model.yaml

@@ -1,4 +1,79 @@
-/model/{name}:
+/model/groups:
+  get:
+    summary: list all available groups
+    description: 'Auth: basic, levels: read, write, dev, admin'
+    tags:
+      - /model
+    responses:
+      200:
+        description: all groups
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                properties:
+                  group:
+                    type: string
+                    example: VN
+                  models:
+                    type: array
+                    items:
+                      $ref: 'api.yaml#/components/schemas/ModelItem'
+      401:
+        $ref: 'api.yaml#/components/responses/401'
+      500:
+        $ref: 'api.yaml#/components/responses/500'
+
+/model/{group}:
+  parameters:
+    - $ref: 'api.yaml#/components/parameters/Group'
+  post:
+    summary: add/replace model group item
+    description: 'Auth: basic, levels: dev, admin <br> If the given name exists, the item is replaced,
+    otherwise it is newly created'
+    tags:
+      - /model
+    requestBody:
+      required: true
+      content:
+        application/json:
+          schema:
+            $ref: 'api.yaml#/components/schemas/ModelItem'
+    responses:
+      200:
+        $ref: 'api.yaml#/components/responses/Ok'
+      400:
+        $ref: 'api.yaml#/components/responses/400'
+      401:
+        $ref: 'api.yaml#/components/responses/401'
+      403:
+        $ref: 'api.yaml#/components/responses/403'
+      500:
+        $ref: 'api.yaml#/components/responses/500'
+
+/model/{group}/{name}:
+  parameters:
+    - $ref: 'api.yaml#/components/parameters/Group'
+    - $ref: 'api.yaml#/components/parameters/Name'
+  delete:
+    summary: remove model group item
+    description: 'Auth: basic, levels: dev, admin'
+    tags:
+      - /model
+    responses:
+      200:
+        $ref: 'api.yaml#/components/responses/Ok'
+      401:
+        $ref: 'api.yaml#/components/responses/401'
+      403:
+        $ref: 'api.yaml#/components/responses/403'
+      404:
+        $ref: 'api.yaml#/components/responses/404'
+      500:
+        $ref: 'api.yaml#/components/responses/500'
+
+/model/file/{name}:
   parameters:
     - $ref: 'api.yaml#/components/parameters/Name'
   get:

api/parameters.yaml

@@ -39,3 +39,11 @@ Collection:
   schema:
     type: string
   example: condition
+
+Group:
+  name: group
+  in: path
+  required: true
+  schema:
+    type: string
+  example: vn

api/schemas.yaml

@@ -211,3 +211,15 @@ User:
       items:
         type: string
         example: Alpha II
+
+ModelItem:
+  properties:
+    name:
+      type: string
+      example: Model 1.1
+    url:
+      type: string
+      example: https://definma-model-test.apps.de1.bosch-iot-cloud.com/predict/model1-1
+    label:
+      type: string
+      example: 'ml/g'

src/models/model.ts

@@ -1,8 +1,20 @@
 import mongoose from 'mongoose';
+import db from '../db';
 
 const ModelSchema = new mongoose.Schema({
+  group: {type: String, index: {unique: true}},
+  models: [new mongoose.Schema({
     name: {type: String, index: {unique: true}},
-  data: Buffer
+    url: String,
+    label: String
+  } ,{ _id : false })]
 });
 
+// changelog query helper
+ModelSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>> (req) {
+  db.log(req, this);
+  return this;
+}
+ModelSchema.index({group: 1});
+
 export default mongoose.model<any, mongoose.Model<any, any>>('model', ModelSchema);

src/models/model_file.ts

@@ -0,0 +1,8 @@
+import mongoose from 'mongoose';
+
+const ModelFileSchema = new mongoose.Schema({
+  name: {type: String, index: {unique: true}},
+  data: Buffer
+});
+
+export default mongoose.model<any, mongoose.Model<any, any>>('model_file', ModelFileSchema);

src/routes/model.spec.ts

@@ -1,6 +1,8 @@
 import should from 'should/as-function';
-import ModelModel from '../models/model';
+import ModelFileModel from '../models/model_file';
 import TestHelper from "../test/helper";
+import ModelModel from '../models/model';
+import _ from 'lodash';
 
 
 describe('/model', () => {
@@ -10,11 +12,269 @@ describe('/model', () => {
   afterEach(done => TestHelper.afterEach(server, done));
   after(done => TestHelper.after(done));
 
-  describe('GET /model/{name}', (() => {
+  describe('GET /model/groups', () => {
+    it('returns all groups', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/model/groups',
+        auth: {basic: 'janedoe'},
+        httpStatus: 200,
+      }).end((err, res) => {
+        if (err) return done (err);
+        const json = require('../test/db.json');
+        should(res.body).have.lengthOf(json.collections.models.length);
+        should(res.body).matchEach(group => {
+          should(group).have.only.keys('group', 'models');
+          should(group).have.property('group').be.type('string');
+          should(group.models).matchEach(model => {
+            should(model).have.only.keys('name', 'url', 'label');
+            should(model).have.property('name').be.type('string');
+            should(model).have.property('url').be.type('string');
+            should(model).have.property('label').be.type('string');
+          });
+        });
+        done();
+      });
+    });
+    it('rejects an API key', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/model/groups',
+        auth: {key: 'janedoe'},
+        httpStatus: 401,
+      });
+    });
+    it('rejects an unauthorized request', done => {
+      TestHelper.request(server, done, {
+        method: 'get',
+        url: '/model/groups',
+        httpStatus: 401,
+      });
+    });
+  });
+
+  describe('POST /model/{group}', () => {
+    it('adds a new model', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {basic: 'admin'},
+        httpStatus: 200,
+        req: {name: 'Model C', url: 'http://model-c.com', label: 'ml/g'}
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).be.eql({status: 'OK'});
+        ModelModel.findOne({group: 'VN'}).lean().exec((err, res) => {
+          if (err) return done(err);
+          const model = res.models.find(e => e.name === 'Model C');
+          should(model).have.property('url', 'http://model-c.com');
+          should(model).have.property('label', 'ml/g');
+          done();
+        });
+      });
+    });
+    it('adds a new group', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/classification',
+        auth: {basic: 'admin'},
+        httpStatus: 200,
+        req: {name: 'Model 0.1', url: 'http://model-0-1.com', label: 'group'}
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).be.eql({status: 'OK'});
+        ModelModel.findOne({group: 'classification'}).lean().exec((err, res) => {
+          if (err) return done(err);
+          should(_.omit(res, ['_id', '__v'])).be.eql({group: 'classification', models: [{name: 'Model 0.1', url: 'http://model-0-1.com', label: 'group'}]});
+          done();
+        });
+      });
+    });
+    it('replaces a model', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {basic: 'admin'},
+        httpStatus: 200,
+        req: {name: 'Model A', url: 'http://model-a-new.com', label: 'ml/cm3'}
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).be.eql({status: 'OK'});
+        ModelModel.findOne({group: 'VN'}).lean().exec((err, res) => {
+          if (err) return done(err);
+          const model = res.models.find(e => e.name === 'Model A');
+          should(model).have.property('url', 'http://model-a-new.com');
+          should(model).have.property('label', 'ml/cm3');
+          done();
+        });
+      });
+    });
+    it('accepts an empty label', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {basic: 'admin'},
+        httpStatus: 200,
+        req: {name: 'Model C', url: 'http://model-c.com', label: ''}
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).be.eql({status: 'OK'});
+        ModelModel.findOne({group: 'VN'}).lean().exec((err, res) => {
+          if (err) return done(err);
+          const model = res.models.find(e => e.name === 'Model C');
+          should(model).have.property('url', 'http://model-c.com');
+          should(model).have.property('label', '');
+          done();
+        });
+      });
+    });
+    it('rejects an empty name', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {basic: 'admin'},
+        httpStatus: 400,
+        req: {name: '', url: 'http://model-c.com', label: 'ml/g'},
+        res:{status: 'Invalid body format', details: '"name" is not allowed to be empty'}
+      });
+    });
+    it('rejects a missing name', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {basic: 'admin'},
+        httpStatus: 400,
+        req: {url: 'http://model-c.com', label: 'ml/g'},
+        res:{status: 'Invalid body format', details: '"name" is required'}
+      });
+    });
+    it('rejects an invalid URL', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {basic: 'admin'},
+        httpStatus: 400,
+        req: {name: 'Model C', url: 'model-c', label: 'ml/g'},
+        res:{status: 'Invalid body format', details: '"url" must be a valid uri'}
+      });
+    });
+    it('rejects a missing URL', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {basic: 'admin'},
+        httpStatus: 400,
+        req: {name: 'Model C', label: 'ml/g'},
+        res:{status: 'Invalid body format', details: '"url" is required'}
+      });
+    });
+    it('rejects a write user', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {basic: 'janedoe'},
+        httpStatus: 403,
+        req: {name: 'Model C', url: 'http://model-c.com', label: 'ml/g'}
+      });
+    });
+    it('rejects an API key', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        auth: {key: 'admin'},
+        httpStatus: 401,
+        req: {name: 'Model C', url: 'http://model-c.com', label: 'ml/g'}
+      });
+    });
+    it('rejects an unauthorized request', done => {
+      TestHelper.request(server, done, {
+        method: 'post',
+        url: '/model/VN',
+        httpStatus: 401,
+        req: {name: 'Model C', url: 'http://model-c.com', label: 'ml/g'}
+      });
+    });
+  });
+
+  describe('DELETE /model/{group}/{name}', () => {
+    it('deletes the model', done => {
+      TestHelper.request(server, done, {
+        method: 'delete',
+        url: '/model/VN/Model%20A',
+        auth: {basic: 'admin'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).be.eql({status: 'OK'});
+        ModelModel.findOne({group: 'VN'}).lean().exec((err, res) => {
+          if (err) return done(err);
+          should(_.omit(res, ['_id'])).be.eql({group: 'VN', models: [{name: 'Model B', url: 'http://model-b.com', label: 'ml/g'}]});
+          done();
+        });
+      });
+    });
+    it('deletes the group, if empty afterwards', done => {
+      TestHelper.request(server, done, {
+        method: 'delete',
+        url: '/model/Moisture/Model%201',
+        auth: {basic: 'admin'},
+        httpStatus: 200
+      }).end((err, res) => {
+        if (err) return done(err);
+        should(res.body).be.eql({status: 'OK'});
+        ModelModel.find({group: 'Moisture'}).lean().exec((err, res) => {
+          if (err) return done(err);
+          should(res).have.lengthOf(0);
+          done();
+        });
+      });
+    });
+    it('returns 404 for an unknown group', done => {
+      TestHelper.request(server, done, {
+        method: 'delete',
+        url: '/model/xxx/Model%201',
+        auth: {basic: 'admin'},
+        httpStatus: 404
+      });
+    });
+    it('returns 404 for an unknown model', done => {
+      TestHelper.request(server, done, {
+        method: 'delete',
+        url: '/model/VN/xxx',
+        auth: {basic: 'admin'},
+        httpStatus: 404
+      });
+    });
+    it('rejects an API key', done => {
+      TestHelper.request(server, done, {
+        method: 'delete',
+        url: '/model/VN/Model%20A',
+        auth: {key: 'admin'},
+        httpStatus: 401
+      });
+    });
+    it('rejects a write user', done => {
+      TestHelper.request(server, done, {
+        method: 'delete',
+        url: '/model/VN/Model%20A',
+        auth: {basic: 'janedoe'},
+        httpStatus: 403
+      });
+    });
+    it('rejects an unauthorized request', done => {
+      TestHelper.request(server, done, {
+        method: 'delete',
+        url: '/model/VN/Model%20A',
+        httpStatus: 401
+      });
+    });
+  });
+
+  describe('GET /model/file/{name}', (() => {
     it('returns the binary data', done => {
       TestHelper.request(server, done, {
         method: 'get',
-        url: '/model/modela',
+        url: '/model/file/modela',
         auth: {basic: 'admin'},
         httpStatus: 200,
         contentType: 'application/octet-stream; charset=utf-8',
@@ -27,7 +287,7 @@ describe('/model', () => {
     it('returns the binary data for an API key', done => {
       TestHelper.request(server, done, {
         method: 'get',
-        url: '/model/modela',
+        url: '/model/file/modela',
         auth: {key: 'admin'},
         httpStatus: 200,
         contentType: 'application/octet-stream; charset=utf-8',
@@ -40,7 +300,7 @@ describe('/model', () => {
     it('returns 404 for an unknown name', done => {
       TestHelper.request(server, done, {
         method: 'get',
-        url: '/model/modelx',
+        url: '/model/file/modelx',
         auth: {basic: 'admin'},
         httpStatus: 404
       })
@@ -48,7 +308,7 @@ describe('/model', () => {
     it('rejects requests from a write user', done => {
       TestHelper.request(server, done, {
         method: 'get',
-        url: '/model/modela',
+        url: '/model/file/modela',
         auth: {basic: 'janedoe'},
         httpStatus: 403
       })
@@ -56,17 +316,17 @@ describe('/model', () => {
     it('rejects unauthorized requests', done => {
       TestHelper.request(server, done, {
         method: 'get',
-        url: '/model/modela',
+        url: '/model/file/modela',
         httpStatus: 401
       })
     });
   }));
 
-  describe('POST /model/{name}', () => {
+  describe('POST /model/file/{name}', () => {
     it('stores the data', done => {
       TestHelper.request(server, done, {
         method: 'post',
-        url: '/model/modelb',
+        url: '/model/file/modelb',
         auth: {basic: 'admin'},
         httpStatus: 200,
         reqContentType: 'application/octet-stream',
@@ -74,7 +334,7 @@ describe('/model', () => {
       }).end((err, res) => {
         if (err) return done (err);
         should(res.body).be.eql({status: 'OK'});
-        ModelModel.find({name: 'modelb'}).lean().exec((err, data) => {
+        ModelFileModel.find({name: 'modelb'}).lean().exec((err, data) => {
           if (err) return done (err);
           should(data).have.lengthOf(1);
           should(data[0]).have.only.keys('_id', 'name', 'data', '__v');
@@ -87,7 +347,7 @@ describe('/model', () => {
     it('stores the data with an API key', done => {
       TestHelper.request(server, done, {
         method: 'post',
-        url: '/model/modelb',
+        url: '/model/file/modelb',
         auth: {key: 'admin'},
         httpStatus: 200,
         reqContentType: 'application/octet-stream',
@@ -95,7 +355,7 @@ describe('/model', () => {
       }).end((err, res) => {
         if (err) return done (err);
         should(res.body).be.eql({status: 'OK'});
-        ModelModel.find({name: 'modelb'}).lean().exec((err, data) => {
+        ModelFileModel.find({name: 'modelb'}).lean().exec((err, data) => {
           if (err) return done (err);
           should(data).have.lengthOf(1);
           should(data[0]).have.only.keys('_id', 'name', 'data', '__v');
@@ -108,7 +368,7 @@ describe('/model', () => {
     it('overwrites existing data', done => {
       TestHelper.request(server, done, {
         method: 'post',
-        url: '/model/modela',
+        url: '/model/file/modela',
         auth: {basic: 'admin'},
         httpStatus: 200,
         reqContentType: 'application/octet-stream',
@@ -116,7 +376,7 @@ describe('/model', () => {
       }).end((err, res) => {
         if (err) return done (err);
         should(res.body).be.eql({status: 'OK'});
-        ModelModel.find({name: 'modela'}).lean().exec((err, data) => {
+        ModelFileModel.find({name: 'modela'}).lean().exec((err, data) => {
           if (err) return done (err);
           should(data).have.lengthOf(1);
           should(data[0]).have.only.keys('_id', 'name', 'data', '__v');
@@ -129,7 +389,7 @@ describe('/model', () => {
     it('rejects requests from a write user', done => {
       TestHelper.request(server, done, {
         method: 'post',
-        url: '/model/modelb',
+        url: '/model/file/modelb',
         auth: {basic: 'janedoe'},
         httpStatus: 403,
         req: 'another binary data'
@@ -138,24 +398,24 @@ describe('/model', () => {
     it('rejects unauthorized requests', done => {
       TestHelper.request(server, done, {
         method: 'post',
-        url: '/model/modelb',
+        url: '/model/file/modelb',
         httpStatus: 401,
         req: 'another binary data'
       });
     });
   });
 
-  describe('DELETE /model/{name}', () => {
+  describe('DELETE /model/file/{name}', () => {
     it('deletes the data', done => {
       TestHelper.request(server, done, {
         method: 'delete',
-        url: '/model/modela',
+        url: '/model/file/modela',
         auth: {basic: 'admin'},
         httpStatus: 200
       }).end((err, res) => {
         if (err) return done(err);
         should(res.body).be.eql({status: 'OK'});
-        ModelModel.find({name: 'modela'}).lean().exec((err, data) => {
+        ModelFileModel.find({name: 'modela'}).lean().exec((err, data) => {
           if (err) return done(err);
           should(data).have.lengthOf(0);
           done();
@@ -165,7 +425,7 @@ describe('/model', () => {
     it('returns 404 for an unknown name', done => {
       TestHelper.request(server, done, {
         method: 'delete',
-        url: '/model/modelx',
+        url: '/model/file/modelx',
         auth: {basic: 'admin'},
         httpStatus: 404
       });
@@ -173,7 +433,7 @@ describe('/model', () => {
     it('rejects an API key', done => {
       TestHelper.request(server, done, {
         method: 'delete',
-        url: '/model/modela',
+        url: '/model/file/modela',
         auth: {key: 'admin'},
         httpStatus: 401
       });
@@ -181,7 +441,7 @@ describe('/model', () => {
     it('rejects a write user', done => {
       TestHelper.request(server, done, {
         method: 'delete',
-        url: '/model/modela',
+        url: '/model/file/modela',
         auth: {basic: 'janedoe'},
         httpStatus: 403
       });
@@ -189,7 +449,7 @@ describe('/model', () => {
     it('rejects an unauthorized request', done => {
       TestHelper.request(server, done, {
         method: 'delete',
-        url: '/model/modela',
+        url: '/model/file/modela',
         httpStatus: 401
       });
     });

src/routes/model.ts

@@ -1,14 +1,97 @@
 import express from 'express';
 import bodyParser from 'body-parser';
 
+import ModelFileModel from '../models/model_file';
 import ModelModel from '../models/model';
+import _ from 'lodash';
+import ModelValidate from './validate/model';
+import res400 from './validate/res400';
+import db from '../db';
 
 const router = express.Router();
 
-router.get('/model/:name', (req, res, next) => {
+router.get('/model/groups', (req, res, next) => {
+  if (!req.auth(res, ['read', 'write', 'dev', 'admin'], 'basic')) return;
+
+  ModelModel.find().lean().exec((err, data) => {
+    if (err) return next(err);
+
+    // validate all and filter null values from validation errors
+    res.json(_.compact(data.map(e => ModelValidate.output(e))));
+  });
+});
+
+router.post('/model/:group', (req, res, next) => {
+  if (!req.auth(res, ['dev', 'admin'], 'basic')) return;
+
+  const {error, value: model} = ModelValidate.input(req.body);
+  console.log(error);
+  if (error) return res400(error, res);
+
+  ModelModel.findOne({group: req.params.group}).lean().exec((err, data) => {
+    if (err) return next(err);
+
+    if (data) {  // group exists
+      if (data.models.find(e => e.name === model.name)) {  // name exists, overwrite
+        ModelModel.findOneAndUpdate(
+          {$and: [{group: req.params.group}, {'models.name': model.name}]},
+          {'models.$': model},
+          {upsert: true}).log(req).lean().exec(err => {
+          if (err) return next(err);
+          res.json({status: 'OK'})
+        });
+      }
+      else {  // create new
+        ModelModel.findOneAndUpdate(
+          {group: req.params.group},
+          {$push: {models: model as never}}
+        ).log(req).lean().exec(err => {
+          if (err) return next(err);
+          res.json({status: 'OK'})
+        });
+      }
+    }
+    else {  // create new group
+      new ModelModel({group: req.params.group, models: [model]}).save((err, data) => {
+        if (err) return next(err);
+        db.log(req, 'models', {_id: data._id}, data.toObject());
+        res.json({status: 'OK'})
+      });
+    }
+  });
+});
+
+router.delete('/model/:group(((?!file)[^\\/]+?))/:name', (req, res, next) => {
+  if (!req.auth(res, ['dev', 'admin'], 'basic')) return;
+
+  ModelModel.findOne({group: req.params.group}).lean().exec((err, data) => {
+    if (err) return next(err);
+
+    if (!data || !data.models.find(e => e.name === req.params.name)) {
+      return res.status(404).json({status: 'Not found'});
+    }
+    if (data.models.length > 1) {  // only remove model
+      ModelModel.findOneAndUpdate(
+        {group: req.params.group},
+        {$pull: {models: data.models.find(e => e.name === req.params.name) as never}}
+      ).log(req).lean().exec(err => {
+        if (err) return next(err);
+        res.json({status: 'OK'})
+      });
+    }
+    else {  // remove document
+      ModelModel.findOneAndDelete({group: req.params.group}).log(req).lean().exec(err => {
+        if (err) return next(err);
+        res.json({status: 'OK'})
+      });
+    }
+  });
+});
+
+router.get('/model/file/:name', (req, res, next) => {
   if (!req.auth(res, ['dev', 'admin'], 'all')) return;
 
-  ModelModel.findOne({name: req.params.name}).lean().exec((err, data) => {
+  ModelFileModel.findOne({name: req.params.name}).lean().exec((err, data) => {
     if (err) return next(err);
     if (data) {
       res.set('Content-Type', 'application/octet-stream');
@@ -20,20 +103,20 @@ router.get('/model/:name', (req, res, next) => {
   });
 });
 
-router.post('/model/:name', bodyParser.raw({limit: '500kb'}), (req, res, next) => {
+router.post('/model/file/:name', bodyParser.raw({limit: '5mb'}), (req, res, next) => {
   if (!req.auth(res, ['dev', 'admin'], 'all')) return;
 
-  ModelModel.replaceOne({name: req.params.name}, {name: req.params.name, data: req.body}).setOptions({upsert: true})
+  ModelFileModel.replaceOne({name: req.params.name}, {name: req.params.name, data: req.body}).setOptions({upsert: true})
     .lean().exec(err => {
     if (err) return next(err);
     res.json({status: 'OK'});
   });
 });
 
-router.delete('/model/:name', (req, res, next) => {
+router.delete('/model/file/:name', (req, res, next) => {
   if (!req.auth(res, ['dev', 'admin'], 'basic')) return;
 
-  ModelModel.findOneAndDelete({name: req.params.name}).lean().exec((err, data) => {
+  ModelFileModel.findOneAndDelete({name: req.params.name}).lean().exec((err, data) => {
     if (err) return next(err);
     if (data) {
       res.json({status: 'OK'});

src/routes/sample.ts

@@ -22,6 +22,8 @@ import globals from '../globals';
 
 const router = express.Router();
 
+// TODO: do not use streaming for spectrum filenames
+
 
 router.get('/samples', async (req, res, next) => {
   if (!req.auth(res, ['read', 'write', 'dev', 'admin'], 'all')) return;
@@ -364,7 +366,7 @@ router.get('/samples', async (req, res, next) => {
   }
   queryPtr.push({$project: projection});
   // use streaming when including spectrum files
-  if (!fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.') >= 0)) {
+  if (!fieldsToAdd.find(e => e.indexOf(globals.spectrum.spectrum + '.' + globals.spectrum.dpt) >= 0)) {
     collection.aggregate(query).allowDiskUse(true).exec((err, data) => {
       if (err) return next(err);
       if (data[0] && data[0].count) {

src/routes/validate/model.ts

@@ -0,0 +1,38 @@
+import Joi from 'joi';
+
+
+export default class ModelValidate {  // validate input for model
+  private static model = {
+    group: Joi.string()
+      .disallow('file')
+      .max(128),
+
+    model: Joi.object({
+        name: Joi.string()
+          .max(128)
+          .required(),
+
+        url: Joi.string()
+          .uri()
+          .max(512)
+          .required(),
+
+        label: Joi.string()
+          .allow('')
+          .max(128)
+          .required()
+      })
+  };
+
+  static input (data) {  // validate input
+    return this.model.model.required().validate(data);
+  }
+
+  static output (data) {  // validate output and strip unwanted properties, returns null if not valid
+    const {value, error} = Joi.object({
+      group: this.model.group,
+      models: Joi.array().items(this.model.model)
+    }).validate(data, {stripUnknown: true});
+    return error !== undefined? null : value;
+  }
+}

src/test/db.json

@@ -678,13 +678,40 @@
         "__v": 0
       }
     ],
-    "models": [
+    "model_files": [
       {
         "_id": {"$oid":"140000000000000000000001"},
         "name": "modela",
         "data": {"buffer": "binary data"}
       }
     ],
+    "models": [
+      {
+        "group": "VN",
+        "models": [
+          {
+            "name": "Model A",
+            "url": "http://model-a.com",
+            "label": "ml/g"
+          },
+          {
+            "name": "Model B",
+            "url": "http://model-b.com",
+            "label": "ml/g"
+          }
+        ]
+      },
+      {
+        "group": "Moisture",
+        "models": [
+          {
+            "name": "Model 1",
+            "url": "http://model-1.com",
+            "label": "weight %"
+          }
+        ]
+      }
+    ],
     "users": [
       {
         "_id": {"$oid":"000000000000000000000001"},