definma-api/src/routes/sample.ts

import express from 'express';
import _ from 'lodash';

import SampleValidate from './validate/sample';
import NoteFieldValidate from './validate/note_field';
import res400 from './validate/res400';
import SampleModel from '../models/sample'
import MeasurementModel from '../models/measurement';
import MeasurementTemplateModel from '../models/measurement_template';
import MaterialModel from '../models/material';
import NoteModel from '../models/note';
import NoteFieldModel from '../models/note_field';
import IdValidate from './validate/id';
import mongoose from 'mongoose';
import ConditionTemplateModel from '../models/condition_template';
import ParametersValidate from './validate/parameters';
import globals from '../globals';
import db from '../db';
import csv from '../helpers/csv';


const router = express.Router();

router.get('/samples', async (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

  const {error, value: filters} = SampleValidate.query(req.query);
  if (error) return res400(error, res);

  const query = [];
  query.push({$match: {$and: []}});
  if (filters.hasOwnProperty('status')) {
    if(filters.status === 'all') {
      query[0].$match.$and.push({$or: [{status: globals.status.validated}, {status: globals.status.new}]});
    }
    else {
      query[0].$match.$and.push({status: globals.status[filters.status]});
    }
  }
  else {  // default
    query[0].$match.$and.push({status: globals.status.validated});
  }

  // sorting
  filters.sort = filters.sort.split('-');
  filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0];  // route added sorting criteria to _id
  filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1;
  if (!filters['to-page']) {  // set to-page default
    filters['to-page'] = 0;
  }

  query.push(
    {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
    {$set: {material: { $arrayElemAt: ['$material', 0]}}},
    {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
    {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}},
    {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
    {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}},
    {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}
      }
    }
  );

  if (filters['from-id']) {  // from-id specified
    const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {next(err);});
    if (fromSample instanceof Error) return;
    if (!fromSample) {
      return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
    }

    if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) {  // asc
      query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$gt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
      query.push({$sort: {[filters.sort[0]]: 1, _id: 1}});
    }
    else {
      query[0].$match.$and.push({$or: [{[filters.sort[0]]: {$lt: fromSample[filters.sort[0]]}}, {$and: [{[filters.sort[0]]: fromSample[filters.sort[0]]}, {_id: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]});
      query.push({$sort: {[filters.sort[0]]: -1, _id: -1}});
    }
  }
  else {  // sort from beginning
    query.push({$sort: {[filters.sort[0]]: filters.sort[1], '_id': filters.sort[1]}});  // set _id as secondary sort
  }

  if (filters['to-page']) {
    query.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)})  // number to skip, if going back pages, one page has to be skipped less but on sample more
  }

  if (filters['page-size']) {
    query.push({$limit: filters['page-size']});
  }

  let measurementFields = [];
  if (filters.fields.find(e => /measurements\./.test(e))) {  // joining measurements is required
    query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
    measurementFields = filters.fields.filter(e => /measurements\./.test(e)).map(e => e.replace('measurements.', ''));
    const measurementTemplates = await MeasurementTemplateModel.find({$or: measurementFields.map(e => {return {name: e}})}).lean().exec().catch(err => {next(err);});
    if (measurementTemplates instanceof Error) return;
    if (measurementTemplates.length < measurementFields.length) {
      return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
    }
    measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => {  // TODO: hard coded dpt for special treatment, change later
      query.push({$set: {[template.name]: {$let: {  // add measurements as property [template.name], if one result, array is reduced to direct values
        vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
        in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
      }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', {}]}}});
    });
    console.log(measurementFields);
    if (measurementFields.find(e => e === 'spectrum')) {  // TODO: remove hardcoded as well
      query.push(
        {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
        {$set: {spectrum: '$spectrum.values.dpt'}},
        {$unwind: '$spectrum'}
      );
    }
    query.push({$unset: 'measurements'});
  }

  const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
  if (filters.fields.indexOf('added') >= 0) {  // add added date
    projection.added = {$toDate: '$_id'};
  }
  if (!(filters.fields.indexOf('_id') >= 0)) {  // disable _id explicitly
    projection._id = false;
  }
  query.push({$project: projection});

  SampleModel.aggregate(query).exec((err, data) => {
    if (err) return next(err);
    if (filters['to-page'] < 0) {
      data.reverse();
    }
    console.log(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));
    if (filters.csv) {  // output as csv
      csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
        if (err) return next(err);
        res.set('Content-Type', 'text/csv');
        res.send(data);
      });
    }
    else {
      res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))));  // validate all and filter null values from validation errors
    }
  })
});

router.get('/samples/:state(new|deleted)', (req, res, next) => {
  if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;

  SampleModel.find({status: globals.status[req.params.state]}).lean().exec((err, data) => {
    if (err) return next(err);
    res.json(_.compact(data.map(e => SampleValidate.output(e))));  // validate all and filter null values from validation errors
  });
});

router.get('/samples/count', (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

  SampleModel.estimatedDocumentCount((err, data) => {
    if (err) return next(err);
    res.json({count: data});
  });
});

router.get('/sample/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

  SampleModel.findById(req.params.id).populate('material_id').populate('user_id', 'name').populate('note_id').exec(async (err, sampleData: any) => {
    if (err) return next(err);

    if (sampleData) {
      await sampleData.populate('material_id.group_id').populate('material_id.supplier_id').execPopulate().catch(err => next(err));
      if (sampleData instanceof Error) return;
      sampleData = sampleData.toObject();

      if (sampleData.status === globals.status.deleted && !req.auth(res, ['maintain', 'admin'], 'all')) return;  // deleted samples only available for maintain/admin
      sampleData.material = sampleData.material_id;  // map data to right keys
      sampleData.material.group = sampleData.material.group_id.name;
      sampleData.material.supplier = sampleData.material.supplier_id.name;
      sampleData.user = sampleData.user_id.name;
      sampleData.notes = sampleData.note_id ? sampleData.note_id : {};
      MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id), status: {$ne: globals.status.deleted}}).lean().exec((err, data) => {
        sampleData.measurements = data;
        res.json(SampleValidate.output(sampleData, 'details'));
      });
    }
    else {
      res.status(404).json({status: 'Not found'});
    }
  });
});

router.put('/sample/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['write', 'maintain', 'dev', 'admin'], 'basic')) return;

  const {error, value: sample} = SampleValidate.input(req.body, 'change');
  if (error) return res400(error, res);

  SampleModel.findById(req.params.id).lean().exec(async (err, sampleData: any) => {  // check if id exists
    if (err) return next(err);
    if (!sampleData) {
      return res.status(404).json({status: 'Not found'});
    }
    if (sampleData.status === globals.status.deleted) {
      return res.status(403).json({status: 'Forbidden'});
    }

    // only maintain and admin are allowed to edit other user's data
    if (sampleData.user_id.toString() !== req.authDetails.id && !req.auth(res, ['maintain', 'admin'], 'basic')) return;

    if (sample.hasOwnProperty('material_id')) {
      if (!await materialCheck(sample, res, next)) return;
    }
    else if (sample.hasOwnProperty('color')) {
      if (!await materialCheck(sample, res, next, sampleData.material_id)) return;
    }

    if (sample.hasOwnProperty('condition') && !(_.isEmpty(sample.condition) && _.isEmpty(sampleData.condition))) {  // do not execute check if condition is and was empty
      if (!await conditionCheck(sample.condition, 'change', res, next, sampleData.condition.condition_template.toString() !== sample.condition.condition_template)) return;
    }

    if (sample.hasOwnProperty('notes')) {
      let newNotes = true;
      if (sampleData.note_id !== null) {  // old notes data exists
        const data = await NoteModel.findById(sampleData.note_id).lean().exec().catch(err => {next(err);}) as any;
        if (data instanceof Error) return;
        newNotes = !_.isEqual(_.pick(IdValidate.stringify(data), _.keys(sample.notes)), sample.notes);  // check if notes were changed
        if (newNotes) {
          if (data.hasOwnProperty('custom_fields')) {  // update note_fields
            customFieldsChange(Object.keys(data.custom_fields), -1, req);
          }
          await NoteModel.findByIdAndDelete(sampleData.note_id).log(req).lean().exec(err => {  // delete old notes
            if (err) return console.error(err);
          });
        }
      }

      if (_.keys(sample.notes).length > 0 && newNotes) {  // save new notes
        if (!await sampleRefCheck(sample, res, next)) return;
        if (sample.notes.hasOwnProperty('custom_fields') && Object.keys(sample.notes.custom_fields).length > 0) {  // new custom_fields
          customFieldsChange(Object.keys(sample.notes.custom_fields), 1, req);
        }
        let data = await new NoteModel(sample.notes).save().catch(err => { return next(err)});  // save new notes
        db.log(req, 'notes', {_id: data._id}, data.toObject());
        delete sample.notes;
        sample.note_id = data._id;
      }
    }

    // check for changes
    if (!_.isEqual(_.pick(IdValidate.stringify(sampleData), _.keys(sample)), _.omit(sample, ['notes']))) {
      sample.status = globals.status.new;
    }

    await SampleModel.findByIdAndUpdate(req.params.id, sample, {new: true}).log(req).lean().exec((err, data: any) => {
      if (err) return next(err);
      res.json(SampleValidate.output(data));
    });

  });
});

router.delete('/sample/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['write', 'maintain', 'dev', 'admin'], 'basic')) return;

  SampleModel.findById(req.params.id).lean().exec(async (err, sampleData: any) => {  // check if id exists
    if (err) return next(err);
    if (!sampleData) {
      return res.status(404).json({status: 'Not found'});
    }

    // only maintain and admin are allowed to edit other user's data
    if (sampleData.user_id.toString() !== req.authDetails.id && !req.auth(res, ['maintain', 'admin'], 'basic')) return;

    await SampleModel.findByIdAndUpdate(req.params.id, {status:globals.status.deleted}).log(req).lean().exec(err => {  // set sample status
      if (err) return next(err);

      // set status of associated measurements also to deleted
      MeasurementModel.updateMany({sample_id: mongoose.Types.ObjectId(req.params.id)}, {status: -1}).log(req).lean().exec(err => {
        if (err) return next(err);

        if (sampleData.note_id !== null) {  // handle notes
          NoteModel.findById(sampleData.note_id).lean().exec((err, data: any) => {  // find notes to update note_fields
            if (err) return next(err);
            if (data.hasOwnProperty('custom_fields')) {  // update note_fields
              customFieldsChange(Object.keys(data.custom_fields), -1, req);
            }
            res.json({status: 'OK'});
          });
        }
        else {
          res.json({status: 'OK'});
        }
      });
    });
  });
});

router.put('/sample/restore/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;

  SampleModel.findByIdAndUpdate(req.params.id, {status: globals.status.new}).log(req).lean().exec((err, data) => {
    if (err) return next(err);

    if (!data) {
      return res.status(404).json({status: 'Not found'});
    }
    res.json({status: 'OK'});
  });
});

router.put('/sample/validate/' + IdValidate.parameter(), (req, res, next) => {
  if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;

  SampleModel.findById(req.params.id).lean().exec((err, data: any) => {
    if (err) return next(err);

    if (!data) {
      return res.status(404).json({status: 'Not found'});
    }
    if (Object.keys(data.condition).length === 0) {
      return res.status(400).json({status: 'Sample without condition cannot be valid'});
    }

    MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id)}).lean().exec((err, data) => {
      if (err) return next(err);

      if (data.length === 0) {
        return res.status(400).json({status: 'Sample without measurements cannot be valid'});
      }

      SampleModel.findByIdAndUpdate(req.params.id, {status: globals.status.validated}).log(req).lean().exec(err => {
        if (err) return next(err);
        res.json({status: 'OK'});
      });
    });
  });
});

router.post('/sample/new', async (req, res, next) => {
  if (!req.auth(res, ['write', 'maintain', 'dev', 'admin'], 'basic')) return;

  if (!req.body.hasOwnProperty('condition')) {  // add empty condition if not specified
    req.body.condition = {};
  }

  const {error, value: sample} = SampleValidate.input(req.body, 'new' + (req.authDetails.level === 'admin' ? '-admin' : ''));
  if (error) return res400(error, res);

  if (!await materialCheck(sample, res, next)) return;
  if (!await sampleRefCheck(sample, res, next)) return;

  if (sample.notes.hasOwnProperty('custom_fields') && Object.keys(sample.notes.custom_fields).length > 0) {  // new custom_fields
    customFieldsChange(Object.keys(sample.notes.custom_fields), 1, req);
  }

  if (!_.isEmpty(sample.condition)) {  // do not execute check if condition is empty
    if (!await conditionCheck(sample.condition, 'change', res, next)) return;
  }

  sample.status = globals.status.new;  // set status to new
  if (sample.hasOwnProperty('number')) {
    if (!await numberCheck(sample, res, next)) return;
  }
  else {
    sample.number = await numberGenerate(sample, req, res, next);
  }
  if (!sample.number) return;

  await new NoteModel(sample.notes).save((err, data) => {  // save notes
    if (err) return next(err);
    db.log(req, 'notes', {_id: data._id}, data.toObject());
    delete sample.notes;
    sample.note_id = data._id;
    sample.user_id = req.authDetails.id;

    new SampleModel(sample).save((err, data) => {
      if (err) return next(err);
      db.log(req, 'samples', {_id: data._id}, data.toObject());
      res.json(SampleValidate.output(data.toObject()));
    });
  });
});

router.get('/sample/notes/fields', (req, res, next) => {
  if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;

  NoteFieldModel.find({}).lean().exec((err, data) => {
    if (err) return next(err);
    res.json(_.compact(data.map(e => NoteFieldValidate.output(e))));  // validate all and filter null values from validation errors
  })
});


module.exports = router;


async function numberGenerate (sample, req, res, next) {  // generate number in format Location32, returns false on error
  const sampleData = await SampleModel
    .findOne({number: new RegExp('^' + req.authDetails.location + '[0-9]+$', 'm')})
    .sort({number: -1})
    .lean()
    .exec()
    .catch(err => next(err));
  if (sampleData instanceof Error) return false;
  return req.authDetails.location + (sampleData ? Number(sampleData.number.replace(/[^0-9]+/g, '')) + 1 : 1);
}

async function numberCheck(sample, res, next) {
  const sampleData = await SampleModel.findOne({number: sample.number}).lean().exec().catch(err => {next(err); return false;});
  if (sampleData) {  // found entry with sample number
    res.status(400).json({status: 'Sample number already taken'});
    return false
  }
  return true;
}

async function materialCheck (sample, res, next, id = sample.material_id) {  // validate material_id and color, returns false if invalid
  const materialData = await MaterialModel.findById(id).lean().exec().catch(err => next(err)) as any;
  if (materialData instanceof Error) return false;
  if (!materialData) {  // could not find material_id
    res.status(400).json({status: 'Material not available'});
    return false;
  }
  if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) {  // color for material not specified
    res.status(400).json({status: 'Color not available for material'});
    return false;
  }
  return true;
}

async function conditionCheck (condition, param, res, next, checkVersion = true) {  // validate treatment template, returns false if invalid, otherwise template data
  if (!condition.condition_template || !IdValidate.valid(condition.condition_template)) {  // template id not found
    res.status(400).json({status: 'Condition template not available'});
    return false;
  }
  const conditionData = await ConditionTemplateModel.findById(condition.condition_template).lean().exec().catch(err => next(err)) as any;
  if (conditionData instanceof Error) return false;
  if (!conditionData) {  // template not found
    res.status(400).json({status: 'Condition template not available'});
    return false;
  }

  if (checkVersion) {
    // get all template versions and check if given is latest
    const conditionVersions = await ConditionTemplateModel.find({first_id: conditionData.first_id}).sort({version: -1}).lean().exec().catch(err => next(err)) as any;
    if (conditionVersions instanceof Error) return false;
    if (condition.condition_template !== conditionVersions[0]._id.toString()) {  // template not latest
      res.status(400).json({status: 'Old template version not allowed'});
      return false;
    }
  }

  // validate parameters
  const {error, value: ignore} = ParametersValidate.input(_.omit(condition, 'condition_template'), conditionData.parameters, param);
  if (error) {res400(error, res); return false;}
  return conditionData;
}

function sampleRefCheck (sample, res, next) {  // validate sample_references, resolves false for invalid reference
  return new Promise(resolve => {
    if (sample.notes.hasOwnProperty('sample_references') && sample.notes.sample_references.length > 0) {  // there are sample_references
      let referencesCount = sample.notes.sample_references.length;  // count to keep track of running async operations

      sample.notes.sample_references.forEach(reference => {
        SampleModel.findById(reference.sample_id).lean().exec((err, data) => {
          if (err) {next(err); resolve(false)}
          if (!data) {
            res.status(400).json({status: 'Sample reference not available'});
            return resolve(false);
          }
          referencesCount --;
          if (referencesCount <= 0) {  // all async requests done
            resolve(true);
          }
        });
      });
    }
    else {
      resolve(true);
    }
  });
}

function customFieldsChange (fields, amount, req) {  // update custom_fields and respective quantities
  fields.forEach(field => {
    NoteFieldModel.findOneAndUpdate({name: field}, {$inc: {qty: amount}} as any, {new: true}).log(req).lean().exec((err, data: any) => {  // check if field exists
      if (err) return console.error(err);
      if (!data) {  // new field
        new NoteFieldModel({name: field, qty: 1}).save((err, data) => {
          if (err) return console.error(err);
          db.log(req, 'note_fields', {_id: data._id}, data.toObject());
        })
      }
      else if (data.qty <= 0) {  // delete document if field is not used anymore
        NoteFieldModel.findOneAndDelete({name: field}).log(req).lean().exec(err => {
          if (err) return console.error(err);
        });
      }
    });
  });
}