definma-api/src/routes/sample.ts

import express from 'express';
import _ from 'lodash';

import SampleValidate from './validate/sample';
import NoteFieldValidate from './validate/note_field';
import res400 from './validate/res400';
import SampleModel from '../models/sample'
import MeasurementModel from '../models/measurement';
import MeasurementTemplateModel from '../models/measurement_template';
import MaterialModel from '../models/material';
import NoteModel from '../models/note';
import NoteFieldModel from '../models/note_field';
import IdValidate from './validate/id';
import mongoose from 'mongoose';
import ConditionTemplateModel from '../models/condition_template';
import ParametersValidate from './validate/parameters';
import globals from '../globals';
import db from '../db';
import csv from '../helpers/csv';


const router = express.Router();

// TODO: check added filter
// TODO: return total number of pages -> use facet
// TODO: use query pointer
// TODO: convert filter value to number according to table model
// TODO: validation for filter parameters
// TODO: location/device sort/filter
router.get('/samples', async (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

  const {error, value: filters} = SampleValidate.query(req.query);
  if (error) return res400(error, res);

  // TODO: find a better place for these
  const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', 'user_id'];

  // evaluate sort parameter from 'color-asc' to ['color', 1]
  filters.sort = filters.sort.split('-');
  filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0];  // route added sorting criteria to _id
  filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1;
  if (!filters['to-page']) {  // set to-page default
    filters['to-page'] = 0;
  }

  const sortFilterKeys = filters.filters.map(e => e.field);

  let collection;
  const query = [];
  let queryPtr = query;
  queryPtr.push({$match: {$and: []}});

  if (filters.sort[0].indexOf('measurements.') >= 0) {  // sorting with measurements as starting collection
    collection = MeasurementModel;
    const [,measurementName, measurementParam] = filters.sort[0].split('.');
    const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);});
    if (measurementTemplate instanceof Error) return;
    if (!measurementTemplate) {
      return res.status(400).json({status: 'Invalid body format', details: filters.sort[0] + ' not found'});
    }
    let sortStartValue = null;
    if (filters['from-id']) {  // from-id specified, fetch values for sorting
      const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}).lean().exec().catch(err => {next(err);});  // TODO: what if more than one measurement for sample?
      if (fromSample instanceof Error) return;
      if (!fromSample) {
        return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
      }
      sortStartValue = fromSample.values[measurementParam];
    }
    queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)});  // find measurements to sort
    if (filters.filters.find(e => e.field === filters.sort[0])) {  // sorted measurement should also be filtered
      queryPtr[0].$match.$and.push(...filterQueries(filters.filters.find(e => e.field === filters.sort[0])));
    }
    queryPtr.push(
      ...sortQuery(filters, ['values.' + measurementParam, 'sample_id'], sortStartValue),  // sort measurements
      {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}},                                  // fetch samples and restructure them to fit sample structure
      {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
      {$match: statusQuery(filters, 'sample.status')},  // filter out wrong status once samples were added
      {$addFields: {['sample.' + measurementName]: '$measurement.values'}},  // more restructuring
      {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
    );
    addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0));  // sample filters
  }
  else {  // sorting with samples as starting collection
    collection = SampleModel;
    queryPtr[0].$match.$and.push(statusQuery(filters, 'status'));

    if (sampleKeys.indexOf(filters.sort[0]) >= 0) {  // sorting for sample keys
      let sortStartValue = null;
      if (filters['from-id']) {  // from-id specified
        const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {
          next(err);
        });
        if (fromSample instanceof Error) return;
        if (!fromSample) {
          return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
        }
        sortStartValue = fromSample[filters.sort[0]];
      }
      queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
    }
    else {  // add sort key to list to add field later
      sortFilterKeys.push(filters.sort[0]);
    }
  }

  addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0));  // sample filters

  let materialQuery = [];  // put material query together separate first to reuse for first-id
  let materialAdded = false;
  if (sortFilterKeys.find(e => /material\./.test(e))) {  //  add material fields
    materialAdded = true;
    materialQuery.push(  // add material properties
      {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},  // TODO: project out unnecessary fields
      {$addFields: {material: {$arrayElemAt: ['$material', 0]}}}
    );
    const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
    addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0));  // base material filters
    if (sortFilterKeys.find(e => e === 'material.supplier')) {  // add supplier if needed
      materialQuery.push(
        {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
        {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
      );
    }
    if (sortFilterKeys.find(e => e === 'material.group')) {  // add group if needed
      materialQuery.push(
        {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
        {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
      );
    }
    if (sortFilterKeys.find(e => e === 'material.number')) {  // add material number if needed
      materialQuery.push(
        {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
      );
    }
    const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
    addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0));  // base material filters
    queryPtr.push(...materialQuery);
    if (/material\./.test(filters.sort[0])) {  // sort by material key
      let sortStartValue = null;
      if (filters['from-id']) {  // from-id specified
        const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);});
        if (fromSample instanceof Error) return;
        if (!fromSample) {
          return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
        }
        sortStartValue = fromSample[filters.sort[0]];
      }
      queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
    }
  }

  const measurementFilterFields = _.uniq(sortFilterKeys.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]));  // filter measurement names and remove duplicates from parameters
  if (sortFilterKeys.find(e => /measurements\./.test(e))) {  //  add measurement fields
    const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFilterFields}}).lean().exec().catch(err => {next(err);});
    if (measurementTemplates instanceof Error) return;
    if (measurementTemplates.length < measurementFilterFields.length) {
      return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
    }
    queryPtr.push({$lookup: {
        from: 'measurements', let: {sId: '$_id'},
        pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
        as: 'measurements'
    }});
    measurementTemplates.forEach(template => {
      queryPtr.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
        vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
        in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
      }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
    });
    addFilterQueries(queryPtr, filters.filters
      .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
      .map(e => {e.field = e.field.replace('measurements.', ''); return e; })
    );  // measurement filters
  }

  if (!filters.fields.find(e => /spectrum\./.test(e)) && !filters['from-id']) {  // count total number of items before $skip and $limit, only works when from-id is not specified and spectra are not included
    queryPtr.push({$facet: {count: [{$count: 'count'}], samples: []}});
    queryPtr = queryPtr[queryPtr.length - 1].$facet.samples;  // add rest of aggregation pipeline into $facet
  }

  // paging
  if (filters['to-page']) {
    queryPtr.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)})  // number to skip, if going back pages, one page has to be skipped less but on sample more
  }
  if (filters['page-size']) {
    queryPtr.push({$limit: filters['page-size']});
  }

  const fieldsToAdd = filters.fields.filter(e =>   // fields to add
    sortFilterKeys.indexOf(e) < 0                      // field was not in filter
    && e !== filters.sort[0]                           // field was not in sort
  );

  if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) {  // add material, was not added already
    queryPtr.push(
      {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
      {$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
    );
  }
  if (fieldsToAdd.indexOf('material.supplier') >= 0) {  // add supplier if needed
    queryPtr.push(
      {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
      {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
    );
  }
  if (fieldsToAdd.indexOf('material.group') >= 0) {  // add group if needed
    queryPtr.push(
      {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
      {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
    );
  }
  if (fieldsToAdd.indexOf('material.number') >= 0) {  // add material number if needed
    queryPtr.push(
      {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
    );
  }

  let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1]));  // filter measurement names and remove duplicates from parameters
  if (fieldsToAdd.find(e => /measurements\./.test(e))) {  // add measurement fields
    const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
    if (measurementTemplates instanceof Error) return;
    if (measurementTemplates.length < measurementFieldsFields.length) {
      return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
    }
    if (fieldsToAdd.find(e => /spectrum\./.test(e))) {  // use different lookup methods with and without spectrum for the best performance
      queryPtr.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
    }
    else {
      queryPtr.push({$lookup: {
          from: 'measurements', let: {sId: '$_id'},
          pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
          as: 'measurements'
        }});
    }
    measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => {  // TODO: hard coded dpt for special treatment, change later
      queryPtr.push({$addFields: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
              vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
              in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
            }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
    });
    if (measurementFieldsFields.find(e => e === 'spectrum')) {  // TODO: remove hardcoded as well
      queryPtr.push(
        {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
        {$addFields: {spectrum: '$spectrum.values'}},
        {$unwind: '$spectrum'}
      );
    }
    // queryPtr.push({$unset: 'measurements'});
    queryPtr.push({$project: {measurements: 0}});
  }

  const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
  if (filters.fields.indexOf('added') >= 0) {  // add added date
    // projection.added = {$toDate: '$_id'};
    // projection.added = { $convert: { input: '$_id', to: "date" } }  // TODO: upgrade MongoDB version or find alternative
  }
  if (filters.fields.indexOf('_id') < 0 && filters.fields.indexOf('added') < 0) {  // disable _id explicitly
    projection._id = false;
  }
  queryPtr.push({$project: projection});

  if (!fieldsToAdd.find(e => /spectrum\./.test(e))) {  // use streaming when including spectrum files
    collection.aggregate(query).exec((err, data) => {
      if (err) return next(err);
      if (data[0].count) {
        res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0);
        res.header('Access-Control-Expose-Headers', 'x-total-items');
        data = data[0].samples;
      }
      if (filters.fields.indexOf('added') >= 0) {  // add added date
        data.map(e => {
          e.added = e._id.getTimestamp();
          if (filters.fields.indexOf('_id') < 0) {
            delete e._id;
          }
          return e
        });
      }
      if (filters['to-page'] < 0) {
        data.reverse();
      }
      const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]);
      if (filters.csv) {  // output as csv
        csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
          if (err) return next(err);
          res.set('Content-Type', 'text/csv');
          res.send(data);
        });
      }
      else {
        res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));  // validate all and filter null values from validation errors
      }
    });
  }
  else {
    res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
    res.write('[');
    let count = 0;
    const stream = collection.aggregate(query).cursor().exec();
    stream.on('data', data => {
      if (filters.fields.indexOf('added') >= 0) {  // add added date
        data.added = data._id.getTimestamp();
        if (filters.fields.indexOf('_id') < 0) {
          delete data._id;
        }
      }
      res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++;
    });
    stream.on('close', () => {
      res.write(']');
      res.end();
    });
  }
});

router.get('/samples/:state(new|deleted)', (req, res, next) => {
  if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;

  SampleModel.find({status: globals.status[req.params.state]}).lean().exec((err, data) => {
    if (err) return next(err);
    res.json(_.compact(data.map(e => SampleValidate.output(e))));  // validate all and filter null values from validation errors
  });
});

router.get('/samples/count', (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

  SampleModel.estimatedDocumentCount((err, data) => {
    if (err) return next(err);
    res.json({count: data});
  });
});

router.get('/sample/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

  SampleModel.findById(req.params.id).populate('material_id').populate('user_id', 'name').populate('note_id').exec(async (err, sampleData: any) => {
    if (err) return next(err);

    if (sampleData) {
      await sampleData.populate('material_id.group_id').populate('material_id.supplier_id').execPopulate().catch(err => next(err));
      if (sampleData instanceof Error) return;
      sampleData = sampleData.toObject();

      if (sampleData.status === globals.status.deleted && !req.auth(res, ['maintain', 'admin'], 'all')) return;  // deleted samples only available for maintain/admin
      sampleData.material = sampleData.material_id;  // map data to right keys
      sampleData.material.group = sampleData.material.group_id.name;
      sampleData.material.supplier = sampleData.material.supplier_id.name;
      sampleData.user = sampleData.user_id.name;
      sampleData.notes = sampleData.note_id ? sampleData.note_id : {};
      MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id), status: {$ne: globals.status.deleted}}).lean().exec((err, data) => {
        sampleData.measurements = data;
        res.json(SampleValidate.output(sampleData, 'details'));
      });
    }
    else {
      res.status(404).json({status: 'Not found'});
    }
  });
});

router.put('/sample/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['write', 'maintain', 'dev', 'admin'], 'basic')) return;

  const {error, value: sample} = SampleValidate.input(req.body, 'change');
  if (error) return res400(error, res);

  SampleModel.findById(req.params.id).lean().exec(async (err, sampleData: any) => {  // check if id exists
    if (err) return next(err);
    if (!sampleData) {
      return res.status(404).json({status: 'Not found'});
    }
    if (sampleData.status === globals.status.deleted) {
      return res.status(403).json({status: 'Forbidden'});
    }

    // only maintain and admin are allowed to edit other user's data
    if (sampleData.user_id.toString() !== req.authDetails.id && !req.auth(res, ['maintain', 'admin'], 'basic')) return;

    if (sample.hasOwnProperty('material_id')) {
      if (!await materialCheck(sample, res, next)) return;
    }
    else if (sample.hasOwnProperty('color')) {
      if (!await materialCheck(sample, res, next, sampleData.material_id)) return;
    }

    if (sample.hasOwnProperty('condition') && !(_.isEmpty(sample.condition) && _.isEmpty(sampleData.condition))) {  // do not execute check if condition is and was empty
      if (!await conditionCheck(sample.condition, 'change', res, next, sampleData.condition.condition_template.toString() !== sample.condition.condition_template)) return;
    }

    if (sample.hasOwnProperty('notes')) {
      let newNotes = true;
      if (sampleData.note_id !== null) {  // old notes data exists
        const data = await NoteModel.findById(sampleData.note_id).lean().exec().catch(err => {next(err);}) as any;
        if (data instanceof Error) return;
        newNotes = !_.isEqual(_.pick(IdValidate.stringify(data), _.keys(sample.notes)), sample.notes);  // check if notes were changed
        if (newNotes) {
          if (data.hasOwnProperty('custom_fields')) {  // update note_fields
            customFieldsChange(Object.keys(data.custom_fields), -1, req);
          }
          await NoteModel.findByIdAndDelete(sampleData.note_id).log(req).lean().exec(err => {  // delete old notes
            if (err) return console.error(err);
          });
        }
      }

      if (_.keys(sample.notes).length > 0 && newNotes) {  // save new notes
        if (!await sampleRefCheck(sample, res, next)) return;
        if (sample.notes.hasOwnProperty('custom_fields') && Object.keys(sample.notes.custom_fields).length > 0) {  // new custom_fields
          customFieldsChange(Object.keys(sample.notes.custom_fields), 1, req);
        }
        let data = await new NoteModel(sample.notes).save().catch(err => { return next(err)});  // save new notes
        db.log(req, 'notes', {_id: data._id}, data.toObject());
        delete sample.notes;
        sample.note_id = data._id;
      }
    }

    // check for changes
    if (!_.isEqual(_.pick(IdValidate.stringify(sampleData), _.keys(sample)), _.omit(sample, ['notes']))) {
      sample.status = globals.status.new;
    }

    await SampleModel.findByIdAndUpdate(req.params.id, sample, {new: true}).log(req).lean().exec((err, data: any) => {
      if (err) return next(err);
      res.json(SampleValidate.output(data));
    });

  });
});

router.delete('/sample/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['write', 'maintain', 'dev', 'admin'], 'basic')) return;

  SampleModel.findById(req.params.id).lean().exec(async (err, sampleData: any) => {  // check if id exists
    if (err) return next(err);
    if (!sampleData) {
      return res.status(404).json({status: 'Not found'});
    }

    // only maintain and admin are allowed to edit other user's data
    if (sampleData.user_id.toString() !== req.authDetails.id && !req.auth(res, ['maintain', 'admin'], 'basic')) return;

    await SampleModel.findByIdAndUpdate(req.params.id, {status:globals.status.deleted}).log(req).lean().exec(err => {  // set sample status
      if (err) return next(err);

      // set status of associated measurements also to deleted
      MeasurementModel.updateMany({sample_id: mongoose.Types.ObjectId(req.params.id)}, {status: -1}).log(req).lean().exec(err => {
        if (err) return next(err);

        if (sampleData.note_id !== null) {  // handle notes
          NoteModel.findById(sampleData.note_id).lean().exec((err, data: any) => {  // find notes to update note_fields
            if (err) return next(err);
            if (data.hasOwnProperty('custom_fields')) {  // update note_fields
              customFieldsChange(Object.keys(data.custom_fields), -1, req);
            }
            res.json({status: 'OK'});
          });
        }
        else {
          res.json({status: 'OK'});
        }
      });
    });
  });
});

router.put('/sample/restore/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;

  SampleModel.findByIdAndUpdate(req.params.id, {status: globals.status.new}).log(req).lean().exec((err, data) => {
    if (err) return next(err);

    if (!data) {
      return res.status(404).json({status: 'Not found'});
    }
    res.json({status: 'OK'});
  });
});

router.put('/sample/validate/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;

  SampleModel.findById(req.params.id).lean().exec((err, data: any) => {
    if (err) return next(err);

    if (!data) {
      return res.status(404).json({status: 'Not found'});
    }
    if (Object.keys(data.condition).length === 0) {
      return res.status(400).json({status: 'Sample without condition cannot be valid'});
    }

    MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id)}).lean().exec((err, data) => {
      if (err) return next(err);

      if (data.length === 0) {
        return res.status(400).json({status: 'Sample without measurements cannot be valid'});
      }

      SampleModel.findByIdAndUpdate(req.params.id, {status: globals.status.validated}).log(req).lean().exec(err => {
        if (err) return next(err);
        res.json({status: 'OK'});
      });
    });
  });
});

router.post('/sample/new', async (req, res, next) => {
  if (!req.auth(res, ['write', 'maintain', 'dev', 'admin'], 'basic')) return;

  if (!req.body.hasOwnProperty('condition')) {  // add empty condition if not specified
    req.body.condition = {};
  }

  const {error, value: sample} = SampleValidate.input(req.body, 'new' + (req.authDetails.level === 'admin' ? '-admin' : ''));
  if (error) return res400(error, res);

  if (!await materialCheck(sample, res, next)) return;
  if (!await sampleRefCheck(sample, res, next)) return;

  if (sample.notes.hasOwnProperty('custom_fields') && Object.keys(sample.notes.custom_fields).length > 0) {  // new custom_fields
    customFieldsChange(Object.keys(sample.notes.custom_fields), 1, req);
  }

  if (!_.isEmpty(sample.condition)) {  // do not execute check if condition is empty
    if (!await conditionCheck(sample.condition, 'change', res, next)) return;
  }

  sample.status = globals.status.new;  // set status to new
  if (sample.hasOwnProperty('number')) {
    if (!await numberCheck(sample, res, next)) return;
  }
  else {
    sample.number = await numberGenerate(sample, req, res, next);
  }
  if (!sample.number) return;

  await new NoteModel(sample.notes).save((err, data) => {  // save notes
    if (err) return next(err);
    db.log(req, 'notes', {_id: data._id}, data.toObject());
    delete sample.notes;
    sample.note_id = data._id;
    sample.user_id = req.authDetails.id;

    new SampleModel(sample).save((err, data) => {
      if (err) return next(err);
      db.log(req, 'samples', {_id: data._id}, data.toObject());
      res.json(SampleValidate.output(data.toObject()));
    });
  });
});

router.get('/sample/notes/fields', (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

  NoteFieldModel.find({}).lean().exec((err, data) => {
    if (err) return next(err);
    res.json(_.compact(data.map(e => NoteFieldValidate.output(e))));  // validate all and filter null values from validation errors
  })
});


module.exports = router;


async function numberGenerate (sample, req, res, next) {  // generate number in format Location32, returns false on error
  const sampleData = await SampleModel
    .findOne({number: new RegExp('^' + req.authDetails.location + '[0-9]+$', 'm')})
    .sort({number: -1})
    .lean()
    .exec()
    .catch(err => next(err));
  if (sampleData instanceof Error) return false;
  return req.authDetails.location + (sampleData ? Number(sampleData.number.replace(/[^0-9]+/g, '')) + 1 : 1);
}

async function numberCheck(sample, res, next) {
  const sampleData = await SampleModel.findOne({number: sample.number}).lean().exec().catch(err => {next(err); return false;});
  if (sampleData) {  // found entry with sample number
    res.status(400).json({status: 'Sample number already taken'});
    return false
  }
  return true;
}

async function materialCheck (sample, res, next, id = sample.material_id) {  // validate material_id and color, returns false if invalid
  const materialData = await MaterialModel.findById(id).lean().exec().catch(err => next(err)) as any;
  if (materialData instanceof Error) return false;
  if (!materialData) {  // could not find material_id
    res.status(400).json({status: 'Material not available'});
    return false;
  }
  if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) {  // color for material not specified
    res.status(400).json({status: 'Color not available for material'});
    return false;
  }
  return true;
}

async function conditionCheck (condition, param, res, next, checkVersion = true) {  // validate treatment template, returns false if invalid, otherwise template data
  if (!condition.condition_template || !IdValidate.valid(condition.condition_template)) {  // template id not found
    res.status(400).json({status: 'Condition template not available'});
    return false;
  }
  const conditionData = await ConditionTemplateModel.findById(condition.condition_template).lean().exec().catch(err => next(err)) as any;
  if (conditionData instanceof Error) return false;
  if (!conditionData) {  // template not found
    res.status(400).json({status: 'Condition template not available'});
    return false;
  }

  if (checkVersion) {
    // get all template versions and check if given is latest
    const conditionVersions = await ConditionTemplateModel.find({first_id: conditionData.first_id}).sort({version: -1}).lean().exec().catch(err => next(err)) as any;
    if (conditionVersions instanceof Error) return false;
    if (condition.condition_template !== conditionVersions[0]._id.toString()) {  // template not latest
      res.status(400).json({status: 'Old template version not allowed'});
      return false;
    }
  }

  // validate parameters
  const {error, value: ignore} = ParametersValidate.input(_.omit(condition, 'condition_template'), conditionData.parameters, param);
  if (error) {res400(error, res); return false;}
  return conditionData;
}

function sampleRefCheck (sample, res, next) {  // validate sample_references, resolves false for invalid reference
  return new Promise(resolve => {
    if (sample.notes.hasOwnProperty('sample_references') && sample.notes.sample_references.length > 0) {  // there are sample_references
      let referencesCount = sample.notes.sample_references.length;  // count to keep track of running async operations

      sample.notes.sample_references.forEach(reference => {
        SampleModel.findById(reference.sample_id).lean().exec((err, data) => {
          if (err) {next(err); resolve(false)}
          if (!data) {
            res.status(400).json({status: 'Sample reference not available'});
            return resolve(false);
          }
          referencesCount --;
          if (referencesCount <= 0) {  // all async requests done
            resolve(true);
          }
        });
      });
    }
    else {
      resolve(true);
    }
  });
}

function customFieldsChange (fields, amount, req) {  // update custom_fields and respective quantities
  fields.forEach(field => {
    NoteFieldModel.findOneAndUpdate({name: field}, {$inc: {qty: amount}} as any, {new: true}).log(req).lean().exec((err, data: any) => {  // check if field exists
      if (err) return console.error(err);
      if (!data) {  // new field
        new NoteFieldModel({name: field, qty: 1}).save((err, data) => {
          if (err) return console.error(err);
          db.log(req, 'note_fields', {_id: data._id}, data.toObject());
        })
      }
      else if (data.qty <= 0) {  // delete document if field is not used anymore
        NoteFieldModel.findOneAndDelete({name: field}).log(req).lean().exec(err => {
          if (err) return console.error(err);
        });
      }
    });
  });
}

function sortQuery(filters, sortKeys, sortStartValue) {  // sortKeys = ['primary key', 'secondary key']
  if (filters['from-id']) {  // from-id specified
    if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) {  // asc
      return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
        {$sort: {[sortKeys[0]]: 1, _id: 1}}];
    } else {
      return [{$match: {$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
        {$sort: {[sortKeys[0]]: -1, _id: -1}}];
    }
  } else {  // sort from beginning
    return [{$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}];  // set _id as secondary sort
  }
}

function statusQuery(filters, field) {
  if (filters.hasOwnProperty('status')) {
    if(filters.status === 'all') {
      return {$or: [{[field]: globals.status.validated}, {[field]: globals.status.new}]};
    }
    else {
      return {[field]: globals.status[filters.status]};
    }
  }
  else {  // default
    return {[field]: globals.status.validated};
  }
}

function addFilterQueries (queryPtr, filters) {  // returns array of match queries from given filters
  if (filters.length) {
    queryPtr.push({$match: {$and: filterQueries(filters)}});
  }
}

function filterQueries (filters) {
  return filters.map(e => ({[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}}))  // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin
}