Archived
2

Merge pull request #14 in ~VLE2FE/dfop-api from sample-filters to develop

* commit '3dda3d77a13f84a40f8132f49f4ad704f2810a75':
  minor fixes
  implemented added filters
  added workaround for 'added' field compatible to MongoDB 3.6
  implemented x-total-items header
  spectrum field working again
  reworked filters
  added filters
  restructured aggregation
  implementation of measurement fields
  first implementation of fields
  base for csv export
  switched to aggregation, included material sort keys
  sorting for direct sample properties added
  added /samples/count
  changed last-id behaviour to from-id
  implemented paging
  fixed validation to return measurements in /sample/{id}
  added status filter for materials
  added status filter
This commit is contained in:
Veit Lukas (PEA4-Fe) 2020-07-14 12:10:03 +02:00
commit e976d45ded
28 changed files with 2309 additions and 154 deletions

View File

@ -39,10 +39,10 @@ info:
servers:
- url: https://definma-api.apps.de1.bosch-iot-cloud.com
description: server on the BIC
- url: http://localhost:3000
description: local server
- url: https://digital-fingerprint-of-plastics-api.apps.de1.bosch-iot-cloud.com/
description: server on the BIC
security:

View File

@ -5,6 +5,13 @@
x-doc: returns only materials with status 10
tags:
- /material
parameters:
- name: status
description: 'values: validated|new|all, defaults to validated'
in: query
schema:
type: string
example: all
responses:
200:
description: all material details

View File

@ -2,20 +2,83 @@
get:
summary: all samples in overview
description: 'Auth: all, levels: read, write, maintain, dev, admin'
x-doc: returns only samples with status 10
x-doc: 'Limitations: paging and csv output does not work when including the spectrum measurement fields as well as the returned number of total samples'
tags:
- /sample
parameters:
- name: status
description: 'values: validated|new|all, defaults to validated'
in: query
schema:
type: string
example: all
- name: from-id
description: first id of the requested page, if not given the results are displayed from start
in: query
schema:
type: string
example: 5ea0450ed851c30a90e70894
- name: to-page
description: relative change of pages, use negative values to get back, defaults to 0, works only together with page-size
in: query
schema:
type: string
example: 1
- name: page-size
description: number of items per page
in: query
schema:
type: string
example: 30
- name: sort
description: sorting of results, in format 'key-asc/desc'
in: query
schema:
type: string
example: color-asc
- name: csv
description: output as csv
in: query
schema:
type: boolean
example: false
- name: fields[]
description: the fields to include in the output as array, defaults to ['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']
in: query
schema:
type: array
items:
type: string
example: ['number', 'batch']
- name: filters[]
description: "the filters to apply as an array of URIComponent encoded objects in the form {mode: 'eq/ne/lt/lte/gt/gte/in/nin', field: 'material.m', values: ['15']} using encodeURIComponent(JSON.stringify({}))"
in: query
schema:
type: array
items:
type: string
example: ["%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.m%22%2C%22values%22%3A%5B%2215%22%5D%7D", "%7B%22mode%22%3A%22isin%22%2C%22field%22%3A%22material.supplier%22%2C%22values%22%3A%5B%22BASF%22%2C%22DSM%22%5D%7D"]
responses:
200:
description: samples overview
description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)
headers:
x-total-items:
description: Total number of available items when from-id is not specified and spectrum field is not included
schema:
type: integer
example: 243
content:
application/json:
schema:
type: array
items:
$ref: 'api.yaml#/components/schemas/SampleRefs'
400:
$ref: 'api.yaml#/components/responses/400'
401:
$ref: 'api.yaml#/components/responses/401'
404:
$ref: 'api.yaml#/components/responses/404'
500:
$ref: 'api.yaml#/components/responses/500'
@ -42,12 +105,31 @@
500:
$ref: 'api.yaml#/components/responses/500'
/samples/count:
get:
summary: total number of samples
description: 'Auth: all, levels: read, write, maintain, dev, admin'
tags:
- /sample
responses:
200:
description: sample count
content:
application/json:
schema:
properties:
count:
type: number
example: 864
500:
$ref: 'api.yaml#/components/responses/500'
/sample/{id}:
parameters:
- $ref: 'api.yaml#/components/parameters/Id'
get:
summary: TODO sample details
description: 'Auth: all, levels: read, write, maintain, dev, admin'
summary: sample details
description: 'Auth: all, levels: read, write, maintain, dev, admin<br>Returns validated as well as new measurements'
x-doc: deleted samples are available only for maintain/admin
tags:
- /sample
@ -216,12 +298,14 @@
content:
application/json:
schema:
properties:
name:
type: string
qty:
type: number
example: 20
type: array
items:
properties:
name:
type: string
qty:
type: number
example: 20
401:
$ref: 'api.yaml#/components/responses/401'
500:

View File

@ -46,6 +46,9 @@ SampleRefs:
$ref: 'api.yaml#/components/schemas/Id'
user_id:
$ref: 'api.yaml#/components/schemas/Id'
added:
type: string
example: 1970-01-01T00:00:00.000Z
Sample:
allOf:
- $ref: 'api.yaml#/components/schemas/_Id'
@ -69,6 +72,8 @@ Sample:
relation:
type: string
example: part to this sample
custom_fields:
type: object
SampleDetail:
allOf:

View File

@ -0,0 +1,579 @@
const csv = require('csv-parser');
const fs = require('fs');
const axios = require('axios');
const {Builder} = require('selenium-webdriver'); // selenium and the chrome driver must be installed and configured separately
const chrome = require('selenium-webdriver/chrome');
const pdfReader = require('pdfreader');
const iconv = require('iconv-lite');
const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv'; // metadata files
const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv';
const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs'; // NormMaster Documents
const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT'; // Spectrum files
const host = 'http://localhost:3000';
// const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
let data = []; // metadata contents
let materials = {};
let samples = [];
let normMaster = {};
let sampleDevices = {};
// TODO: BASF twice, BASF as color
// TODO: duplicate kf values
// TODO: conditions
// TODO: comment and reference handling
// TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC
main();
async function main() {
if (0) { // materials
await getNormMaster();
await importCsv(metaDoc);
await allMaterials();
await saveMaterials();
await importCsv(kfDoc);
await allMaterials();
await saveMaterials();
await importCsv(vzDoc);
await allMaterials();
await saveMaterials();
}
if (0) { // samples
sampleDeviceMap();
if (1) {
console.log('-------- META ----------');
await importCsv(metaDoc);
await allSamples();
await saveSamples();
}
if (1) {
console.log('-------- KF ----------');
await importCsv(kfDoc);
await allSamples();
await saveSamples();
await allKfVz();
}
if (1) {
console.log('-------- VZ ----------');
await importCsv(vzDoc);
await allSamples();
await saveSamples();
await allKfVz();
}
}
if (1) { // DPT
await allDpts();
}
if (0) { // pdf test
console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf'));
}
}
async function importCsv(doc) {
data = [];
await new Promise(resolve => {
fs.createReadStream(doc)
.pipe(iconv.decodeStream('win1252'))
.pipe(csv())
.on('data', (row) => {
data.push(row);
})
.on('end', () => {
console.info('CSV file successfully processed');
if (data[0]['Farbe']) { // fix German column names
data.map(e => {e['Color'] = e['Farbe']; return e; });
}
resolve();
});
});
}
async function allDpts() {
let res = await axios({
method: 'get',
url: host + '/template/measurements',
auth: {
username: 'admin',
password: 'Abc123!#'
}
});
const measurement_template = res.data.find(e => e.name === 'spectrum')._id;
res = await axios({
method: 'get',
url: host + '/samples?status=all',
auth: {
username: 'admin',
password: 'Abc123!#'
}
});
const sampleIds = {};
res.data.forEach(sample => {
sampleIds[sample.number] = sample._id;
});
const dptRegex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/;
const dpts = fs.readdirSync(dptFiles);
for (let i in dpts) {
const regexRes = dptRegex.exec(dpts[i])
if (regexRes && sampleIds[regexRes[1]]) { // found matching sample
console.log(dpts[i]);
const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8');
const data = {
sample_id: sampleIds[regexRes[1]],
values: {},
measurement_template
};
data.values.dpt = f.split('\r\n').map(e => e.split(','));
let rescale = false;
for (let i in data.values.dpt) {
if (data.values.dpt[i][1] > 2) {
rescale = true;
break;
}
}
if (rescale) {
data.values.dpt = data.values.dpt.map(e => [e[0], e[1] / 100]);
}
await axios({
method: 'post',
url: host + '/measurement/new',
auth: {
username: 'admin',
password: 'Abc123!#'
},
data
}).catch(err => {
console.log(dpts[i]);
console.error(err.response.data);
});
}
else {
console.log(`Could not find sample for ${dpts[i]} !!!!!!`);
}
}
}
async function allKfVz() {
let res = await axios({
method: 'get',
url: host + '/template/measurements',
auth: {
username: 'admin',
password: 'Abc123!#'
}
});
const kf_template = res.data.find(e => e.name === 'kf')._id;
const vz_template = res.data.find(e => e.name === 'vz')._id;
res = await axios({
method: 'get',
url: host + '/samples?status=all',
auth: {
username: 'admin',
password: 'Abc123!#'
}
});
const sampleIds = {};
res.data.forEach(sample => {
sampleIds[sample.number] = sample._id;
});
for (let index in data) {
console.info(`${index}/${data.length}`);
let sample = data[index];
if (sample['Sample number'] !== '') {
let credentials = ['admin', 'Abc123!#'];
if (sampleDevices[sample['Sample number']]) {
credentials = [sampleDevices[sample['Sample number']], '2020DeFinMachen!']
}
if (sample['KF in Gew%']) {
await axios({
method: 'post',
url: host + '/measurement/new',
auth: {
username: credentials[0],
password: credentials[1]
},
data: {
sample_id: sampleIds[sample['Sample number']],
measurement_template: kf_template,
values: {
'weight %': sample['KF in Gew%'],
'standard deviation': sample['Stabwn']
}
}
}).catch(err => {
console.log(sample['Sample number']);
console.error(err.response.data);
});
}
if (sample['VZ (ml/g)']) {
await axios({
method: 'post',
url: host + '/measurement/new',
auth: {
username: credentials[0],
password: credentials[1]
},
data: {
sample_id: sampleIds[sample['Sample number']],
measurement_template: vz_template,
values: {
vz: sample['VZ (ml/g)']
}
}
}).catch(err => {
console.log(sample['Sample number']);
console.error(err.response.data);
});
}
}
}
}
async function allSamples() {
samples = [];
let res = await axios({
method: 'get',
url: host + '/materials?status=all',
auth: {
username: 'admin',
password: 'Abc123!#'
}
});
const dbMaterials = {}
res.data.forEach(m => {
dbMaterials[m.name] = m;
})
res = await axios({
method: 'get',
url: host + '/samples?status=all',
auth: {
username: 'admin',
password: 'Abc123!#'
}
});
const sampleColors = {};
res.data.forEach(sample => {
sampleColors[sample.number] = sample.color;
});
for (let index in data) {
console.info(`${index}/${data.length}`);
let sample = data[index];
if (sample['Sample number'] !== '') { // TODO: what about samples without color
if (sample['Supplier'] === '') { // empty supplier fields
sample['Supplier'] = 'unknown';
}
if (sample['Granulate/Part'] === '') { // empty supplier fields
sample['Granulate/Part'] = 'unknown';
}
const material = dbMaterials[trim(sample['Material name'])];
if (!material) { // could not find material, skipping sample
continue;
}
console.log(sample['Material name']);
console.log(material._id);
samples.push({
number: sample['Sample number'],
type: sample['Granulate/Part'],
batch: sample['Charge/batch granulate/part'] || '',
material_id: material._id,
notes: {
comment: sample['Comments']
}
});
const si = samples.length - 1;
if (sample['Material number'] !== '' && material.numbers.find(e => e.number === sample['Material number'])) { // TODO: fix because of false material/material number
samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
}
else if (sample['Color'] && sample['Color'] !== '') {
let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0);
if (!number && /black/.test(sample['Color'])) { // special case bk for black
number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
if (!number) { // try German word
number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
}
}
samples[si].color = number.color;
}
else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz
samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
}
else {
samples[si].color = '';
}
}
}
}
async function saveSamples() {
for (let i in samples) {
console.info(`${i}/${samples.length}`);
let credentials = ['admin', 'Abc123!#'];
if (sampleDevices[samples[i].number]) {
credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!']
}
await axios({
method: 'post',
url: host + '/sample/new',
auth: {
username: credentials[0],
password: credentials[1]
},
data: samples[i]
}).catch(err => {
if (err.response.data.status && err.response.data.status !== 'Sample number already taken') {
console.log(samples[i]);
console.error(err.response.data);
}
});
}
console.info('saved all samples');
}
async function allMaterials() {
materials = {};
for (let index in data) {
let sample = data[index];
if (sample['Sample number'] && sample['Sample number'] !== '') {
if (sample['Supplier'] === '') { // empty supplier fields
sample['Supplier'] = 'unknown';
}
if (sample['Material name'] === '') { // empty name fields
sample['Material name'] = sample['Material'];
}
if (!sample['Material']) { // column Material is named Plastic in VZ metadata
sample['Material'] = sample['Plastic'];
}
sample['Material name'] = trim(sample['Material name']);
if (materials.hasOwnProperty(sample['Material name'])) { // material already found at least once
if (sample['Material number'] && sample['Material number'] !== '') {
if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) { // new material number
if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) { // color already in list, only number missing
materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']);
}
else {
materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])});
}
}
}
else if (sample['Color'] && sample['Color'] !== '') {
if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) { // new material color
materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''});
}
}
}
else { // new material
console.info(`${index}/${data.length} ${sample['Material name']}`);
materials[sample['Material name']] = {
name: sample['Material name'],
supplier: trim(sample['Supplier']),
group: trim(sample['Material'])
};
let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
materials[sample['Material name']].mineral = tmp ? tmp[1] : 0;
tmp = /GF(\d+)/.exec(sample['Reinforcing material']);
materials[sample['Material name']].glass_fiber = tmp ? tmp[1] : 0;
tmp = /CF(\d+)/.exec(sample['Reinforcing material']);
materials[sample['Material name']].carbon_fiber = tmp ? tmp[1] : 0;
materials[sample['Material name']].numbers = await numbersFetch(sample);
console.log(materials[sample['Material name']]);
}
}
}
}
async function saveMaterials() {
const mKeys = Object.keys(materials)
for (let i in mKeys) {
console.info(`${i}/${mKeys.length}`);
await axios({
method: 'post',
url: host + '/material/new',
auth: {
username: 'admin',
password: 'Abc123!#'
},
data: materials[mKeys[i]]
}).catch(err => {
if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
console.info(materials[mKeys[i]]);
console.error(err.response.data);
}
});
}
console.info('saved all materials');
}
async function numbersFetch(sample) {
let nm = [];
let res = [];
if (sample['Material number']) { // sample has a material number
nm = normMaster[stripSpaces(sample['Material number'])]? [normMaster[stripSpaces(sample['Material number'])]] : [];
}
else { // try finding via material name
nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['Material name'])).map(e => normMaster[e]);
}
if (nm.length > 0) {
for (let i in nm) {
// if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded
// await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'));
// }
// if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded
// console.info('Retrying download...');
// await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 2.2);
// }
// if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded
// console.info('Retrying download again...');
// await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 5);
// }
if (fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document loaded
res = await readPdf(fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0));
}
if (res.length > 0) { // no results
break;
}
else if (i + 1 >= nm.length) {
console.error('Download failed!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
}
}
}
if (res.length === 0) { // no results
if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) {
return [{color: trim(sample['Color']), number: sample['Material number']}];
}
else {
return [];
}
}
else {
if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed
res.push({color: trim(sample['Color']), number: sample['Material number']});
}
return res;
}
}
async function getNormMaster(fetchAgain = false) {
if (fetchAgain) {
console.info('fetching norm master...');
const res = await axios({
method: 'get',
url: 'http://rb-normen.bosch.com/cgi-bin/searchRBNorm4TradeName'
});
console.info('finding documents...');
let match;
// const regex = /<tr>.*?<td>.*?<\/span>(.*?)<\/td><td>(\d+)<\/td>.*?<a href="(.*?)"/gm;
const regex = /<tr>.*?<td>.*?<\/span>(.*?)<\/td><td>(\d+)<\/td><td>40.*?<a href="(.*?)".*?<\/a>(.*?)<\/td>/gm; // only valid materials
do {
match = regex.exec(res.data);
if (match) {
normMaster[match[2]] = {name: match[1], nameSpaceless: stripSpaces(match[1]), number: match[2], url: match[3], doc: match[4]};
}
} while (match);
fs.writeFileSync('./data_import/normMaster.json', JSON.stringify(normMaster));
}
else {
normMaster = JSON.parse(fs.readFileSync('./data_import/normMaster.json'), 'utf-8');
}
}
function getNormMasterDoc(url, timing = 1) {
console.info(url);
return new Promise(async resolve => {
const options = new chrome.Options();
options.setUserPreferences({
"download.default_directory": nmDocs,
"download.prompt_for_download": false,
"download.directory_upgrade": true,
"plugins.always_open_pdf_externally": true
});
let driver = await new Builder().forBrowser('chrome').setChromeOptions(options).build();
let timeout = 7000 * timing;
try {
await driver.get(url);
if (await driver.getCurrentUrl() !== 'https://rb-wam-saml.bosch.com/tfim/sps/normmaster/saml20/login') { // got document selection page
timeout = 11000 * timing;
await driver.executeScript('Array.prototype.slice.call(document.querySelectorAll(\'.functionlink\')).filter(e => e.innerText === \'English\')[0].click()').catch(() => {timeout = 0; });
}
}
finally {
setTimeout(async () => { // wait until download is finished
await driver.quit();
resolve();
}, timeout);
}
});
}
function readPdf(file) {
return new Promise(async resolve => {
const countdown = 100; // value for text timeout
let table = 0; // > 0 when in correct table area
let rows = []; // found table rows
let lastY = 0; // y of last row
let lastX = 0; // right x of last item
let lastText = ''; // text of last item
let lastLastText = ''; // text of last last item
await new pdfReader.PdfReader().parseFileItems(nmDocs + '\\' + file, (err, item) => {
if (item && item.text) {
if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsupplier') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsupplier') >= 0)) { // table area starts
table = countdown;
}
if (table > 0) {
// console.log(item);
// console.log(item.y - lastY);
// console.log(item.text);
if (item.y - lastY > 0.8 && Math.abs(item.x - lastX) > 5) { // new row
lastY = item.y;
rows.push(item.text);
}
else { // still the same row row
rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text; // push to row, detect if still same cell
}
lastX = (item.w * 0.055) + item.x;
if (/\d \d\d\d \d\d\d \d\d\d/.test(item.text)) {
table = countdown;
}
table --;
if (table <= 0 || item.text.toLowerCase().indexOf('release document') >= 0 || item.text.toLowerCase().indexOf('normative references') >= 0) { // table area ended
table = -1;
// console.log(rows);
rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows
resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
}
}
lastLastText = lastText;
lastText = item.text;
}
if (!item && table !== -1) { // document ended
rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows
resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
}
});
});
}
function sampleDeviceMap() {
const dpts = fs.readdirSync(dptFiles);
const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/;
for (let i in dpts) {
const regexRes = regex.exec(dpts[i])
if (regexRes) { // found matching sample
sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase();
}
}
}
function stripSpaces(s) {
return s ? s.replace(/ /g,'') : '';
}
function trim(s) {
return s.replace(/(^\s+|\s+$)/gm, '');
}

View File

@ -1,8 +1,9 @@
---
applications:
- name: digital-fingerprint-of-plastics-api
- name: definma-api
path: dist/
instances: 1
memory: 256M
memory: 1024M
stack: cflinuxfs3
buildpacks:
- nodejs_buildpack
@ -10,4 +11,4 @@ applications:
NODE_ENV: production
OPTIMIZE_MEMORY: true
services:
- dfopdb
- definmadb

541
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -7,11 +7,13 @@
"tsc": "tsc",
"tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc",
"build": "build.bat",
"build-push": "build.bat && cf push",
"test": "mocha dist/**/**.spec.js",
"start": "node index.js",
"dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"",
"loadDev": "node dist/test/loadDev.js",
"coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000"
"coverage": "tsc && nyc --reporter=html --reporter=text mocha dist/**/**.spec.js --timeout 5000",
"import": "node data_import/import.js"
},
"keywords": [],
"author": "",
@ -27,13 +29,15 @@
"cfenv": "^1.2.2",
"compression": "^1.7.4",
"content-filter": "^1.1.2",
"cors": "^2.8.5",
"express": "^4.17.1",
"helmet": "^3.22.0",
"json-schema": "^0.2.5",
"json2csv": "^5.0.1",
"lodash": "^4.17.15",
"mongo-sanitize": "^1.1.0",
"mongoose": "^5.8.7",
"swagger-ui-express": "^4.1.2"
"swagger-ui-express": "4.1.2"
},
"devDependencies": {
"@types/bcrypt": "^3.0.0",
@ -45,9 +49,13 @@
"@types/node": "^13.1.6",
"@types/qs": "^6.9.1",
"@types/serve-static": "^1.13.3",
"csv-parser": "^2.3.3",
"iconv-lite": "^0.6.0",
"mocha": "^7.1.2",
"nodemon": "^2.0.3",
"nyc": "^15.0.1",
"pdfreader": "^1.0.7",
"selenium-webdriver": "^4.0.0-alpha.7",
"should": "^13.2.3",
"supertest": "^4.0.2",
"tslint": "^5.20.1",

View File

@ -18,6 +18,7 @@ export default class api {
jsonRefParser.bundle('api/api.yaml', (err, doc) => { // parse yaml
if (err) throw err;
apiDoc = doc;
apiDoc.servers.splice(process.env.NODE_ENV === 'production', 1);
apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e)); // bundle routes
apiDoc = this.resolveXDoc(apiDoc);
oasParser.validate(apiDoc, (err, api) => { // validate oas schema

View File

@ -3,11 +3,15 @@ import cfenv from 'cfenv';
import _ from 'lodash';
import ChangelogModel from './models/changelog';
// mongoose.set('debug', true); // enable mongoose debug
// database urls, prod db url is retrieved automatically
const TESTING_URL = 'mongodb://localhost/dfopdb_test';
const DEV_URL = 'mongodb://localhost/dfopdb';
const debugging = true;
if (process.env.NODE_ENV !== 'production' && debugging) {
mongoose.set('debug', true); // enable mongoose debug
}
export default class db {
private static state = { // db object and current mode (test, dev, prod)
@ -43,10 +47,15 @@ export default class db {
if (err) done(err);
});
mongoose.connection.on('error', console.error.bind(console, 'connection error:'));
mongoose.connection.on('connected', () => { // evaluation connection behaviour on prod
if (process.env.NODE_ENV !== 'test') { // Do not interfere with testing
console.info('Database connected');
}
});
mongoose.connection.on('disconnected', () => { // reset state on disconnect
if (process.env.NODE_ENV !== 'test') { // Do not interfere with testing
console.info('Database disconnected');
this.state.db = 0;
// this.state.db = 0; // prod database connects and disconnects automatically
}
});
process.on('SIGINT', () => { // close connection when app is terminated

View File

@ -89,6 +89,9 @@ function key (req, next): any { // checks API key and returns changed user obje
if (err) return next(err);
if (data.length === 1) { // one user found
resolve({level: data[0].level, name: data[0].name, id: data[0]._id.toString(), location: data[0].location});
if (!/^\/api/m.test(req.url)){
delete req.query.key; // delete query parameter to avoid interference with later validation
}
}
else {
resolve(null);

34
src/helpers/csv.ts Normal file
View File

@ -0,0 +1,34 @@
import {parseAsync} from 'json2csv';
export default function csv(input: any[], f: (err, data) => void) {
parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
.then(csv => f(null, csv))
.catch(err => f(err, null));
}
function flatten (data) { // flatten object: {a: {b: true}} -> {a.b: true}
const result = {};
function recurse (cur, prop) {
if (Object(cur) !== cur || Object.keys(cur).length === 0) {
result[prop] = cur;
}
else if (Array.isArray(cur)) {
let l = 0;
for(let i = 0, l = cur.length; i < l; i++)
recurse(cur[i], prop + "[" + i + "]");
if (l == 0)
result[prop] = [];
}
else {
let isEmpty = true;
for (let p in cur) {
isEmpty = false;
recurse(cur[p], prop ? prop+"."+p : p);
}
if (isEmpty && prop)
result[prop] = {};
}
}
recurse(data, '');
return result;
}

View File

@ -17,7 +17,7 @@ export default (mailAddress, subject, content, f) => { // callback, executed em
contentType: "text/html"
},
from: {
eMail: "dfop@bosch-iot.com",
eMail: "definma@bosch-iot.com",
password: "PlasticsOfFingerprintDigital"
}
}

View File

@ -4,9 +4,11 @@ import compression from 'compression';
import contentFilter from 'content-filter';
import mongoSanitize from 'mongo-sanitize';
import helmet from 'helmet';
import cors from 'cors';
import api from './api';
import db from './db';
// TODO: working demo branch
// tell if server is running in debug or production environment
console.info(process.env.NODE_ENV === 'production' ? '===== PRODUCTION =====' : process.env.NODE_ENV === 'test' ? '' :'===== DEVELOPMENT =====');
@ -24,11 +26,11 @@ const port = process.env.PORT || 3000;
//middleware
app.use(helmet());
app.use(contentFilter()); // filter URL query attacks
app.use(express.json({ limit: '5mb'}));
app.use(express.urlencoded({ extended: false, limit: '5mb' }));
app.use(compression()); // compress responses
app.use(bodyParser.json());
app.use(contentFilter()); // filter URL query attacks
app.use((req, res, next) => { // filter body query attacks
req.body = mongoSanitize(req.body);
next();
@ -41,14 +43,16 @@ app.use((req, res, next) => { // no database connection error
next();
}
else {
console.error('No database connection');
res.status(500).send({status: 'Internal server error'});
}
});
app.use(cors()); // CORS headers
app.use(require('./helpers/authorize')); // handle authentication
// redirect /api routes for Angular proxy in development
if (process.env.NODE_ENV !== 'production') {
app.use('/api/:url', (req, res) => {
app.use('/api/:url([^]+)', (req, res) => {
req.url = '/' + req.params.url;
app.handle(req, res);
});

View File

@ -22,5 +22,7 @@ MaterialSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>>
db.log(req, this);
return this;
}
MaterialSchema.index({supplier_id: 1});
MaterialSchema.index({group_id: 1});
export default mongoose.model<any, mongoose.Model<any, any>>('material', MaterialSchema);

View File

@ -17,5 +17,7 @@ MeasurementSchema.query.log = function <Q extends mongoose.DocumentQuery<any, an
db.log(req, this);
return this;
}
MeasurementSchema.index({sample_id: 1});
MeasurementSchema.index({measurement_template: 1});
export default mongoose.model<any, mongoose.Model<any, any>>('measurement', MeasurementSchema);

View File

@ -22,5 +22,8 @@ SampleSchema.query.log = function <Q extends mongoose.DocumentQuery<any, any>> (
db.log(req, this);
return this;
}
SampleSchema.index({material_id: 1});
SampleSchema.index({note_id: 1});
SampleSchema.index({user_id: 1});
export default mongoose.model<any, mongoose.Model<any, any>>('sample', SampleSchema);

View File

@ -72,6 +72,43 @@ describe('/material', () => {
done();
});
});
it('allows filtering by state', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/materials?status=new',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.materials.filter(e => e.status === globals.status.new).length);
should(res.body).matchEach(material => {
should(material).have.only.keys('_id', 'name', 'supplier', 'group', 'mineral', 'glass_fiber', 'carbon_fiber', 'numbers');
should(material).have.property('_id').be.type('string');
should(material).have.property('name').be.type('string');
should(material).have.property('supplier').be.type('string');
should(material).have.property('group').be.type('string');
should(material).have.property('mineral').be.type('number');
should(material).have.property('glass_fiber').be.type('number');
should(material).have.property('carbon_fiber').be.type('number');
should(material.numbers).matchEach(number => {
should(number).have.only.keys('color', 'number');
should(number).have.property('color').be.type('string');
should(number).have.property('number').be.type('string');
});
});
done();
});
});
it('rejects an invalid state name', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/materials?status=xxx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"status" must be one of [validated, new, all]'}
});
});
it('rejects unauthorized requests', done => {
TestHelper.request(server, done, {
method: 'get',

View File

@ -19,7 +19,24 @@ const router = express.Router();
router.get('/materials', (req, res, next) => {
if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
MaterialModel.find({status:globals.status.validated}).populate('group_id').populate('supplier_id').lean().exec((err, data) => {
const {error, value: filters} = MaterialValidate.query(req.query);
if (error) return res400(error, res);
let conditions;
if (filters.hasOwnProperty('status')) {
if(filters.status === 'all') {
conditions = {$or: [{status: globals.status.validated}, {status: globals.status.new}]}
}
else {
conditions = {status: globals.status[filters.status]};
}
}
else { // default
conditions = {status: globals.status.validated};
}
MaterialModel.find(conditions).populate('group_id').populate('supplier_id').lean().exec((err, data) => {
if (err) return next(err);
res.json(_.compact(data.map(e => MaterialValidate.output(e)))); // validate all and filter null values from validation errors

View File

@ -17,6 +17,7 @@ router.get('/authorized', (req, res) => {
res.json({status: 'Authorization successful', method: req.authDetails.method});
});
// TODO: evaluate exact changelog functionality (restoring, delting after time, etc.)
router.get('/changelog/:timestamp/:page?/:pagesize?', (req, res, next) => {
if (!req.auth(res, ['maintain', 'admin'], 'basic')) return;

View File

@ -10,6 +10,7 @@ import mongoose from 'mongoose';
// TODO: generate output for ML in format DPT -> data, implement filtering, field selection
// TODO: generate csv
// TODO: write script for data import
// TODO: allowed types: tension rod, part, granulate, other
describe('/sample', () => {
let server;
@ -18,6 +19,9 @@ describe('/sample', () => {
afterEach(done => TestHelper.afterEach(server, done));
after(done => TestHelper.after(done));
// TODO: sort, added date filter, has measurements/condition filter
// TODO: check if conditions work in sort/fields/filters
// TODO: test for numbers as strings in glass_fiber
describe('GET /samples', () => {
it('returns all samples', done => {
TestHelper.request(server, done, {
@ -30,7 +34,7 @@ describe('/sample', () => {
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.validated).length);
should(res.body).matchEach(sample => {
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(sample).have.property('_id').be.type('string');
should(sample).have.property('number').be.type('string');
should(sample).have.property('type').be.type('string');
@ -41,6 +45,7 @@ describe('/sample', () => {
should(sample).have.property('material_id').be.type('string');
should(sample).have.property('note_id');
should(sample).have.property('user_id').be.type('string');
should(sample).have.property('added').be.type('string');
});
done();
});
@ -56,7 +61,7 @@ describe('/sample', () => {
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.validated).length);
should(res.body).matchEach(sample => {
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(sample).have.property('_id').be.type('string');
should(sample).have.property('number').be.type('string');
should(sample).have.property('type').be.type('string');
@ -67,10 +72,405 @@ describe('/sample', () => {
should(sample).have.property('material_id').be.type('string');
should(sample).have.property('note_id');
should(sample).have.property('user_id').be.type('string');
should(sample).have.property('added').be.type('string');
});
done();
});
});
it('allows filtering by state', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=new',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.new).length);
should(res.body).matchEach(sample => {
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(sample).have.property('_id').be.type('string');
should(sample).have.property('number').be.type('string');
should(sample).have.property('type').be.type('string');
should(sample).have.property('color').be.type('string');
should(sample).have.property('batch').be.type('string');
should(sample).have.property('condition').be.type('object');
should(sample).have.property('material_id').be.type('string');
should(sample).have.property('note_id');
should(sample).have.property('user_id').be.type('string');
should(sample).have.property('added').be.type('string');
});
done();
});
});
it('uses the given page size', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&page-size=3',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(3);
done();
});
});
it('returns results starting from first-id', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&from-id=400000000000000000000002',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('_id', '400000000000000000000002');
should(res.body[1]).have.property('_id', '400000000000000000000003');
done();
});
});
it('returns the right page number', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&to-page=2&page-size=2',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('_id', '400000000000000000000006');
done();
});
});
it('works with negative page numbers', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&to-page=-1&page-size=2&from-id=400000000000000000000004',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('_id', '400000000000000000000002');
should(res.body[1]).have.property('_id', '400000000000000000000003');
done();
});
});
it('returns an empty array for a page number out of range', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&to-page=100&page-size=2',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(0);
should(res.body).be.eql([]);
done();
});
});
it('returns an empty array for a page number out of negative range', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&to-page=-100&page-size=3&from-id=400000000000000000000004',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(0);
should(res.body).be.eql([]);
done();
});
});
it('sorts the samples ascending', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&sort=color-asc',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('color', 'black');
should(res.body[res.body.length - 1]).have.property('color', 'natural');
done();
});
});
it('sorts the samples descending', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&sort=number-desc',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('number', 'Rng36');
should(res.body[1]).have.property('number', '33');
should(res.body[res.body.length - 1]).have.property('number', '1');
done();
});
});
it('sorts the samples correctly in combination with paging', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&sort=color-asc&page-size=2&from-id=400000000000000000000006',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('_id', '400000000000000000000006');
should(res.body[1]).have.property('_id', '400000000000000000000002');
done();
});
});
it('sorts the samples correctly in combination with going pages backward', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&sort=color-desc&page-size=2&from-id=400000000000000000000004&to-page=-1',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('_id', '400000000000000000000002');
should(res.body[1]).have.property('_id', '400000000000000000000006');
done();
});
});
it('sorts the samples correctly for material keys', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&sort=material.name-desc',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body[0]).have.property('_id', '400000000000000000000002');
should(res.body[1]).have.property('_id', '400000000000000000000006');
should(res.body[2]).have.property('_id', '400000000000000000000001');
done();
});
});
it('adds the specified measurements', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=measurements.kf',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body.find(e => e.number === '1')).have.property('kf', {'weight %': null, 'standard deviation': null});
should(res.body.find(e => e.number === 'Rng36')).have.property('kf', {'weight %': 0.6, 'standard deviation': null});
done();
});
});
it('multiplies the sample information for each spectrum', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=measurements.spectrum',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(2);
should(res.body[0]).have.property('spectrum', [[3997.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
should(res.body[1]).have.property('spectrum', [[3996.12558,98.00555],[3995.08519,98.03253],[3993.04480,98.02657]]);
done();
});
});
it('filters a sample property', done => { // TODO: implement filters
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=type&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22type%22%2C%22values%22%3A%5B%22part%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.type === 'part').length);
should(res.body).matchEach(sample => {
should(sample).have.property('type', 'part');
});
done();
});
});
it('filters a material property', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22in%22%2C%22field%22%3A%22material.name%22%2C%22values%22%3A%5B%22Schulamid%2066%20GF%2025%20H%22%2C%22Stanyl%20TW%20200%20F8%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.material_id == '100000000000000000000004' || e.material_id == '100000000000000000000001').length);
should(res.body).matchEach(sample => {
should(sample.material.name).be.equalOneOf('Schulamid 66 GF 25 H', 'Stanyl TW 200 F8');
});
done();
});
});
it('filters by measurement value', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.name&fields[]=measurements.kf.weight%20%25&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length);
should(res.body).matchEach(sample => {
should(sample.kf['weight %']).be.above(0.5);
});
done();
});
});
it('filters by measurement value not in the fields', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.name&filters[]=%7B%22mode%22%3A%22gt%22%2C%22field%22%3A%22measurements.kf.weight%20%25%22%2C%22values%22%3A%5B0.5%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body).have.lengthOf(json.collections.measurements.filter(e => e.measurement_template == '300000000000000000000002' && e.values['weight %'] > 0.5).length);
should(res.body[0]).have.property('number', 'Rng36');
done();
});
});
it('filters multiple properties', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22material.glass_fiber%22%2C%22values%22%3A%5B33%5D%7D&filters[]=%7B%22mode%22%3A%22lte%22%2C%22field%22%3A%22number%22%2C%22values%22%3A%5B%22Rng33%22%5D%7D&filters[]=%7B%22mode%22%3A%22nin%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.body).have.lengthOf(1);
should(res.body[0]).be.eql({number: '32', material: {glass_fiber: 33}, batch: '1653000308'});
done();
});
}); // TODO: do measurement pipeline, check if it works with UI
it('rejects an invalid JSON string as a filters parameter', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=xx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: 'Invalid JSON string for filter parameter'}
});
});
it('rejects an invalid filter mode', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'}
});
});
it('rejects an filter field not existing', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=material.glass_fiber&fields[]=batch&filters[]=%7B%22mode%22%3A%22eq%22%2C%22field%22%3A%22xx%22%2C%22values%22%3A%5B%221704-005%22%5D%7D',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"filters[0].field" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'}
});
});
it('rejects unknown measurement names', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&fields[]=number&fields[]=measurements.xx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: 'Measurement key not found'}
});
});
it('returns a correct csv file if specified', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&page-size=2&csv=true',
contentType: /text\/csv/,
auth: {basic: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
should(res.text).be.eql('"_id","number","type","color","batch","condition.material","condition.weeks","condition.condition_template","material_id","note_id","user_id","added"\r\n' +
'"400000000000000000000001","1","granulate","black","","copper",3,"200000000000000000000001","100000000000000000000004",,"000000000000000000000002","2004-01-10T13:37:04.000Z"\r\n' +
'"400000000000000000000002","21","granulate","natural","1560237365","copper",3,"200000000000000000000001","100000000000000000000001","500000000000000000000001","000000000000000000000002","2004-01-10T13:37:04.000Z"');
done();
});
});
it('returns only the fields specified', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&page-size=1&fields[]=number&fields[]=condition&fields[]=color&fields[]=material.name&fields[]=material.mineral',
auth: {basic: 'janedoe'},
httpStatus: 200,
res: [{number: '1', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, color: 'black', material: {name: 'Schulamid 66 GF 25 H', mineral: 0}}]
});
});
it('rejects a from-id not in the database', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?from-id=5ea0450ed851c30a90e70894&sort=color-asc',
auth: {basic: 'admin'},
httpStatus: 400,
res: {status: 'Invalid body format', details: 'from-id not found'}
});
});
it('rejects an invalid fields parameter', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&page-size=1&fields=number',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"fields" must be an array'}
});
});
it('rejects an unknown field name', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=all&page-size=1&fields[]=xx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"fields[0]" with value "xx" fails to match the required pattern: /^(_id|color|number|type|batch|added|material\\.name|material\\.supplier|material\\.group|material\\.mineral|material\\.glass_fiber|material\\.carbon_fiber|material\\.number|measurements\\.(?!spectrum).+|condition|material_id|material|note_id|user_id|material\\._id|material\\.numbers|measurements\\.spectrum)$/m'}
});
});
it('rejects a negative page size', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?page-size=-3',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"page-size" must be larger than or equal to 1'}
});
});
it('rejects an invalid from-id', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?from-id=40000000000h000000000002',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"from-id" with value "40000000000h000000000002" fails to match the required pattern: /[0-9a-f]{24}/'}
});
});
it('rejects a to-page without page-size', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?to-page=3',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"to-page" missing required peer "page-size"'}
});
});
it('rejects an invalid state name', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples?status=xxx',
auth: {basic: 'janedoe'},
httpStatus: 400,
res: {status: 'Invalid body format', details: '"status" must be one of [validated, new, all]'}
});
});
it('rejects unauthorized requests', done => {
TestHelper.request(server, done, {
method: 'get',
@ -93,7 +493,7 @@ describe('/sample', () => {
let asyncCounter = res.body.length;
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status ===globals.status.new).length);
should(res.body).matchEach(sample => {
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(sample).have.property('_id').be.type('string');
should(sample).have.property('number').be.type('string');
should(sample).have.property('type').be.type('string');
@ -106,6 +506,7 @@ describe('/sample', () => {
should(sample).have.property('material_id').be.type('string');
should(sample).have.property('note_id');
should(sample).have.property('user_id').be.type('string');
should(sample).have.property('added').be.type('string');
SampleModel.findById(sample._id).lean().exec((err, data) => {
should(data).have.property('status',globals.status.new);
if (--asyncCounter === 0) {
@ -127,7 +528,7 @@ describe('/sample', () => {
let asyncCounter = res.body.length;
should(res.body).have.lengthOf(json.collections.samples.filter(e => e.status === -1).length);
should(res.body).matchEach(sample => {
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
should(sample).have.only.keys('_id', 'number', 'type', 'color', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(sample).have.property('_id').be.type('string');
should(sample).have.property('number').be.type('string');
should(sample).have.property('type').be.type('string');
@ -140,6 +541,7 @@ describe('/sample', () => {
should(sample).have.property('material_id').be.type('string');
should(sample).have.property('note_id');
should(sample).have.property('user_id').be.type('string');
should(sample).have.property('added').be.type('string');
SampleModel.findById(sample._id).lean().exec((err, data) => {
should(data).have.property('status',globals.status.deleted);
if (--asyncCounter === 0) {
@ -174,6 +576,42 @@ describe('/sample', () => {
});
});
describe('GET /samples/count', () => {
it('returns the correct number of samples', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples/count',
auth: {basic: 'admin'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body.count).be.eql(json.collections.samples.length);
done();
});
});
it('works with an API key', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples/count',
auth: {key: 'janedoe'},
httpStatus: 200
}).end((err, res) => {
if (err) return done(err);
const json = require('../test/db.json');
should(res.body.count).be.eql(json.collections.samples.length);
done();
});
});
it('rejects unauthorized requests', done => {
TestHelper.request(server, done, {
method: 'get',
url: '/samples/count',
httpStatus: 401
});
});
});
describe('GET /sample/{id}', () => {
it('returns the right sample', done => {
TestHelper.request(server, done, {
@ -181,7 +619,7 @@ describe('/sample', () => {
url: '/sample/400000000000000000000003',
auth: {basic: 'janedoe'},
httpStatus: 200,
res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, user: 'admin'}
res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'}
});
});
it('works with an API key', done => {
@ -190,7 +628,7 @@ describe('/sample', () => {
url: '/sample/400000000000000000000003',
auth: {key: 'janedoe'},
httpStatus: 200,
res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, user: 'admin'}
res: {_id: '400000000000000000000003', number: '33', type: 'part', color: 'black', batch: '1704-005', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {comment: '', sample_references: [{sample_id: '400000000000000000000004', relation: 'granulate to sample'}], custom_fields: {'not allowed for new applications': true}}, measurements: [{_id: '800000000000000000000003', sample_id: '400000000000000000000003', values: {val1: 1}, measurement_template: '300000000000000000000003'}], user: 'admin'}
});
});
it('returns a deleted sample for a maintain/admin user', done => {
@ -199,7 +637,7 @@ describe('/sample', () => {
url: '/sample/400000000000000000000005',
auth: {basic: 'admin'},
httpStatus: 200,
res: {_id: '400000000000000000000005', number: 'Rng33', type: 'granulate', color: 'black', batch: '1653000308', condition: {condition_template: '200000000000000000000003'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {}, user: 'admin'}
res: {_id: '400000000000000000000005', number: 'Rng33', type: 'granulate', color: 'black', batch: '1653000308', condition: {condition_template: '200000000000000000000003'}, material: {_id: '100000000000000000000005', name: 'Amodel A 1133 HS', supplier: 'Solvay', group: 'PPA', mineral: 0, glass_fiber: 33, carbon_fiber: 0, numbers: [{color: 'black', number: '5514262406'}]}, notes: {}, measurements: [], user: 'admin'}
});
});
it('returns 403 for a write user when requesting a deleted sample', done => {
@ -235,7 +673,7 @@ describe('/sample', () => {
});
});
describe('PUT /sample/{id}', () => {
describe('PUT /sample/{id}', () => { // TODO: fix tests, work on /samples
it('returns the right sample', done => {
TestHelper.request(server, done, {
method: 'put',
@ -243,7 +681,7 @@ describe('/sample', () => {
auth: {basic: 'janedoe'},
httpStatus: 200,
req: {},
res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}
res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}
});
});
it('keeps unchanged properties', done => {
@ -255,7 +693,7 @@ describe('/sample', () => {
req: {type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', notes: {}}
}).end((err, res) => {
if (err) return done(err);
should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'});
should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'});
SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => {
if (err) return done (err);
should(data).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'status', '__v');
@ -282,7 +720,7 @@ describe('/sample', () => {
req: {type: 'granulate'}
}).end((err, res) => {
if (err) return done(err);
should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'});
should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'});
SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => {
if (err) return done (err);
should(data).have.property('status',globals.status.validated);
@ -299,7 +737,7 @@ describe('/sample', () => {
req: {condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}}
}).end((err, res) => {
if (err) return done(err);
should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'});
should(res.body).be.eql({_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'});
SampleModel.findById('400000000000000000000001').lean().exec((err, data: any) => {
if (err) return done (err);
should(data).have.property('status',globals.status.validated);
@ -316,7 +754,7 @@ describe('/sample', () => {
req: {notes: {comment: 'Stoff gesperrt', sample_references: []}}
}).end((err, res) => {
if (err) return done(err);
should(res.body).be.eql({_id: '400000000000000000000002', number: '21', type: 'granulate', color: 'natural', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', note_id: '500000000000000000000001', user_id: '000000000000000000000002'});
should(res.body).be.eql({_id: '400000000000000000000002', number: '21', type: 'granulate', color: 'natural', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', note_id: '500000000000000000000001', user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'});
SampleModel.findById('400000000000000000000002').lean().exec((err, data: any) => {
if (err) return done (err);
should(data).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'status', '__v');
@ -593,7 +1031,7 @@ describe('/sample', () => {
auth: {basic: 'janedoe'},
httpStatus: 200,
req: {condition: {}},
res: {_id: '400000000000000000000006', number: 'Rng36', type: 'granulate', color: 'black', batch: '', condition: {}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}
res: {_id: '400000000000000000000006', number: 'Rng36', type: 'granulate', color: 'black', batch: '', condition: {}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}
});
});
it('rejects an old version of a condition template', done => {
@ -613,7 +1051,7 @@ describe('/sample', () => {
auth: {basic: 'admin'},
httpStatus: 200,
req: {condition: {p1: 36, condition_template: '200000000000000000000004'}},
res: {_id: '400000000000000000000004', number: '32', type: 'granulate', color: 'black', batch: '1653000308', condition: {p1: 36, condition_template: '200000000000000000000004'}, material_id: '100000000000000000000005', note_id: '500000000000000000000003', user_id: '000000000000000000000003'}
res: {_id: '400000000000000000000004', number: '32', type: 'granulate', color: 'black', batch: '1653000308', condition: {p1: 36, condition_template: '200000000000000000000004'}, material_id: '100000000000000000000005', note_id: '500000000000000000000003', user_id: '000000000000000000000003', added: '2004-01-10T13:37:04.000Z'}
});
});
it('rejects an changing back to an empty condition', done => {
@ -660,7 +1098,7 @@ describe('/sample', () => {
auth: {basic: 'admin'},
httpStatus: 200,
req: {},
res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {condition_template: '200000000000000000000001', material: 'copper', weeks: 3}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002'}
res: {_id: '400000000000000000000001', number: '1', type: 'granulate', color: 'black', batch: '', condition: {condition_template: '200000000000000000000001', material: 'copper', weeks: 3}, material_id: '100000000000000000000004', note_id: null, user_id: '000000000000000000000002', added: '2004-01-10T13:37:04.000Z'}
});
});
it('rejects requests from a read user', done => {
@ -1051,7 +1489,7 @@ describe('/sample', () => {
req: {color: 'black', type: 'granulate', batch: '1560237365', condition: {material: 'copper', weeks: 3, condition_template: '200000000000000000000001'}, material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}}
}).end((err, res) => {
if (err) return done (err);
should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(res.body).have.property('_id').be.type('string');
should(res.body).have.property('number', 'Rng37');
should(res.body).have.property('color', 'black');
@ -1061,6 +1499,8 @@ describe('/sample', () => {
should(res.body).have.property('material_id', '100000000000000000000001');
should(res.body).have.property('note_id').be.type('string');
should(res.body).have.property('user_id', '000000000000000000000002');
should(res.body).have.property('added').be.type('string');
should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000);
done();
});
});
@ -1164,7 +1604,7 @@ describe('/sample', () => {
req: {color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}}
}).end((err, res) => {
if (err) return done (err);
should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(res.body).have.property('_id').be.type('string');
should(res.body).have.property('number', 'Fe1');
should(res.body).have.property('color', 'black');
@ -1173,6 +1613,8 @@ describe('/sample', () => {
should(res.body).have.property('material_id', '100000000000000000000001');
should(res.body).have.property('note_id').be.type('string');
should(res.body).have.property('user_id', '000000000000000000000004');
should(res.body).have.property('added').be.type('string');
should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1500);
done();
});
});
@ -1185,7 +1627,7 @@ describe('/sample', () => {
req: {color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}}
}).end((err, res) => {
if (err) return done (err);
should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(res.body).have.property('_id').be.type('string');
should(res.body).have.property('number', 'Rng37');
should(res.body).have.property('color', 'black');
@ -1195,6 +1637,8 @@ describe('/sample', () => {
should(res.body).have.property('material_id', '100000000000000000000001');
should(res.body).have.property('note_id').be.type('string');
should(res.body).have.property('user_id', '000000000000000000000002');
should(res.body).have.property('added').be.type('string');
should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000);
done();
});
});
@ -1237,7 +1681,7 @@ describe('/sample', () => {
req: {number: 'Rng34', color: 'black', type: 'granulate', batch: '1560237365', material_id: '100000000000000000000001', notes: {comment: 'Testcomment', sample_references: [{sample_id: '400000000000000000000003', relation: 'part to this sample'}]}},
}).end((err, res) => {
if (err) return done (err);
should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id');
should(res.body).have.only.keys('_id', 'number', 'color', 'type', 'batch', 'condition', 'material_id', 'note_id', 'user_id', 'added');
should(res.body).have.property('_id').be.type('string');
should(res.body).have.property('number', 'Rng34');
should(res.body).have.property('color', 'black');
@ -1247,6 +1691,8 @@ describe('/sample', () => {
should(res.body).have.property('material_id', '100000000000000000000001');
should(res.body).have.property('note_id').be.type('string');
should(res.body).have.property('user_id', '000000000000000000000003');
should(res.body).have.property('added').be.type('string');
should(new Date().getTime() - new Date(res.body.added).getTime()).be.below(1000);
done();
});
});

View File

@ -6,6 +6,7 @@ import NoteFieldValidate from './validate/note_field';
import res400 from './validate/res400';
import SampleModel from '../models/sample'
import MeasurementModel from '../models/measurement';
import MeasurementTemplateModel from '../models/measurement_template';
import MaterialModel from '../models/material';
import NoteModel from '../models/note';
import NoteFieldModel from '../models/note_field';
@ -15,17 +16,350 @@ import ConditionTemplateModel from '../models/condition_template';
import ParametersValidate from './validate/parameters';
import globals from '../globals';
import db from '../db';
import csv from '../helpers/csv';
const router = express.Router();
router.get('/samples', (req, res, next) => {
// TODO: check added filter
// TODO: return total number of pages -> use facet
// TODO: use query pointer
// TODO: convert filter value to number according to table model
// TODO: validation for filter parameters
// TODO: location/device sort/filter
router.get('/samples', async (req, res, next) => {
if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
SampleModel.find({status: globals.status.validated}).lean().exec((err, data) => {
if (err) return next(err);
res.json(_.compact(data.map(e => SampleValidate.output(e)))); // validate all and filter null values from validation errors
})
const {error, value: filters} = SampleValidate.query(req.query);
if (error) return res400(error, res);
// TODO: find a better place for these
const sampleKeys = ['_id', 'color', 'number', 'type', 'batch', 'added', 'condition', 'material_id', 'note_id', 'user_id'];
// evaluate sort parameter from 'color-asc' to ['color', 1]
filters.sort = filters.sort.split('-');
filters.sort[0] = filters.sort[0] === 'added' ? '_id' : filters.sort[0]; // route added sorting criteria to _id
filters.sort[1] = filters.sort[1] === 'desc' ? -1 : 1;
if (!filters['to-page']) { // set to-page default
filters['to-page'] = 0;
}
const addedFilter = filters.filters.find(e => e.field === 'added');
if (addedFilter) { // convert added filter to object id
filters.filters.splice(filters.filters.findIndex(e => e.field === 'added'), 1);
if (addedFilter.mode === 'in') {
const v = []; // query value
addedFilter.values.forEach(value => {
const date = [new Date(value).setHours(0,0,0,0), new Date(value).setHours(23,59,59,999)];
v.push({$and: [{ _id: { '$gte': dateToOId(date[0])}}, { _id: { '$lte': dateToOId(date[1])}}]});
});
filters.filters.push({mode: 'or', field: '_id', values: v});
}
else if (addedFilter.mode === 'nin') {
addedFilter.values = addedFilter.values.sort();
const v = []; // query value
for (let i = 0; i <= addedFilter.values.length; i ++) {
v[i] = {$and: []};
if (i > 0) {
const date = new Date(addedFilter.values[i - 1]).setHours(23,59,59,999);
v[i].$and.push({ _id: { '$gt': dateToOId(date)}}) ;
}
if (i < addedFilter.values.length) {
const date = new Date(addedFilter.values[i]).setHours(0,0,0,0);
v[i].$and.push({ _id: { '$lt': dateToOId(date)}}) ;
}
}
filters.filters.push({mode: 'or', field: '_id', values: v});
}
else {
// start and end of day
const date = [new Date(addedFilter.values[0]).setHours(0,0,0,0), new Date(addedFilter.values[0]).setHours(23,59,59,999)];
if (addedFilter.mode === 'lt') { // lt start
filters.filters.push({mode: 'lt', field: '_id', values: [dateToOId(date[0])]});
}
if (addedFilter.mode === 'eq' || addedFilter.mode === 'lte') { // lte end
filters.filters.push({mode: 'lte', field: '_id', values: [dateToOId(date[1])]});
}
if (addedFilter.mode === 'gt') { // gt end
filters.filters.push({mode: 'gt', field: '_id', values: [dateToOId(date[1])]});
}
if (addedFilter.mode === 'eq' || addedFilter.mode === 'gte') { // gte start
filters.filters.push({mode: 'gte', field: '_id', values: [dateToOId(date[0])]});
}
if (addedFilter.mode === 'ne') {
filters.filters.push({mode: 'or', field: '_id', values: [{ _id: { '$lt': dateToOId(date[0])}}, { _id: { '$gt': dateToOId(date[1])}}]});
}
}
}
const sortFilterKeys = filters.filters.map(e => e.field);
let collection;
const query = [];
let queryPtr = query;
queryPtr.push({$match: {$and: []}});
if (filters.sort[0].indexOf('measurements.') >= 0) { // sorting with measurements as starting collection
collection = MeasurementModel;
const [,measurementName, measurementParam] = filters.sort[0].split('.');
const measurementTemplate = await MeasurementTemplateModel.findOne({name: measurementName}).lean().exec().catch(err => {next(err);});
if (measurementTemplate instanceof Error) return;
if (!measurementTemplate) {
return res.status(400).json({status: 'Invalid body format', details: filters.sort[0] + ' not found'});
}
let sortStartValue = null;
if (filters['from-id']) { // from-id specified, fetch values for sorting
const fromSample = await MeasurementModel.findOne({sample_id: mongoose.Types.ObjectId(filters['from-id'])}).lean().exec().catch(err => {next(err);}); // TODO: what if more than one measurement for sample?
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
sortStartValue = fromSample.values[measurementParam];
}
queryPtr[0].$match.$and.push({measurement_template: mongoose.Types.ObjectId(measurementTemplate._id)}); // find measurements to sort
if (filters.filters.find(e => e.field === filters.sort[0])) { // sorted measurement should also be filtered
queryPtr[0].$match.$and.push(...filterQueries(filters.filters.filter(e => e.field === filters.sort[0]).map(e => {e.field = 'values.' + e.field.split('.')[2]; return e; })));
}
queryPtr.push(
...sortQuery(filters, ['values.' + measurementParam, 'sample_id'], sortStartValue), // sort measurements
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
{$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added
{$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
);
}
else { // sorting with samples as starting collection
collection = SampleModel;
queryPtr[0].$match.$and.push(statusQuery(filters, 'status'));
if (sampleKeys.indexOf(filters.sort[0]) >= 0) { // sorting for sample keys
let sortStartValue = null;
if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.findById(filters['from-id']).lean().exec().catch(err => {
next(err);
});
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
sortStartValue = fromSample[filters.sort[0]];
}
queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
}
else { // add sort key to list to add field later
sortFilterKeys.push(filters.sort[0]);
}
}
addFilterQueries(queryPtr, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
let materialQuery = []; // put material query together separate first to reuse for first-id
let materialAdded = false;
if (sortFilterKeys.find(e => /material\./.test(e))) { // add material fields
materialAdded = true;
materialQuery.push( // add material properties
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields
{$addFields: {material: {$arrayElemAt: ['$material', 0]}}}
);
const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters
if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed
materialQuery.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
);
}
if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed
materialQuery.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed
materialQuery.push(
{$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
);
}
const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
addFilterQueries(materialQuery, filters.filters.filter(e => specialMFilters.indexOf(e.field) >= 0)); // base material filters
queryPtr.push(...materialQuery);
if (/material\./.test(filters.sort[0])) { // sort by material key
let sortStartValue = null;
if (filters['from-id']) { // from-id specified
const fromSample = await SampleModel.aggregate([{$match: {_id: mongoose.Types.ObjectId(filters['from-id'])}}, ...materialQuery]).exec().catch(err => {next(err);});
if (fromSample instanceof Error) return;
if (!fromSample) {
return res.status(400).json({status: 'Invalid body format', details: 'from-id not found'});
}
sortStartValue = fromSample[filters.sort[0]];
}
queryPtr.push(...sortQuery(filters, [filters.sort[0], '_id'], sortStartValue));
}
}
const measurementFilterFields = _.uniq(sortFilterKeys.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
if (sortFilterKeys.find(e => /measurements\./.test(e))) { // add measurement fields
const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFilterFields}}).lean().exec().catch(err => {next(err);});
if (measurementTemplates instanceof Error) return;
if (measurementTemplates.length < measurementFilterFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
queryPtr.push({$lookup: {
from: 'measurements', let: {sId: '$_id'},
pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
as: 'measurements'
}});
measurementTemplates.forEach(template => {
queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
});
addFilterQueries(queryPtr, filters.filters
.filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
.map(e => {e.field = e.field.replace('measurements.', ''); return e; })
); // measurement filters
}
if (!filters.fields.find(e => /spectrum\./.test(e)) && !filters['from-id']) { // count total number of items before $skip and $limit, only works when from-id is not specified and spectra are not included
queryPtr.push({$facet: {count: [{$count: 'count'}], samples: []}});
queryPtr = queryPtr[queryPtr.length - 1].$facet.samples; // add rest of aggregation pipeline into $facet
}
// paging
if (filters['to-page']) {
queryPtr.push({$skip: Math.abs(filters['to-page'] + Number(filters['to-page'] < 0)) * filters['page-size'] + Number(filters['to-page'] < 0)}) // number to skip, if going back pages, one page has to be skipped less but on sample more
}
if (filters['page-size']) {
queryPtr.push({$limit: filters['page-size']});
}
const fieldsToAdd = filters.fields.filter(e => // fields to add
sortFilterKeys.indexOf(e) < 0 // field was not in filter
&& e !== filters.sort[0] // field was not in sort
);
if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already
queryPtr.push(
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed
queryPtr.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed
queryPtr.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
);
}
if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed
queryPtr.push(
{$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
);
}
let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields
const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
if (measurementTemplates instanceof Error) return;
if (measurementTemplates.length < measurementFieldsFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
}
if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance
queryPtr.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
}
else {
queryPtr.push({$lookup: {
from: 'measurements', let: {sId: '$_id'},
pipeline: [{$match: {$expr: {$and: [{$eq: ['$sample_id', '$$sId']}, {$in: ['$measurement_template', measurementTemplates.map(e => mongoose.Types.ObjectId(e._id))]}]}}}],
as: 'measurements'
}});
}
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
queryPtr.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
});
if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
queryPtr.push(
{$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
{$addFields: {spectrum: '$spectrum.values'}},
{$unwind: '$spectrum'}
);
}
// queryPtr.push({$unset: 'measurements'});
queryPtr.push({$project: {measurements: 0}});
}
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
if (filters.fields.indexOf('added') >= 0) { // add added date
// projection.added = {$toDate: '$_id'};
// projection.added = { $convert: { input: '$_id', to: "date" } } // TODO: upgrade MongoDB version or find alternative
}
if (filters.fields.indexOf('_id') < 0 && filters.fields.indexOf('added') < 0) { // disable _id explicitly
projection._id = false;
}
queryPtr.push({$project: projection});
if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files
collection.aggregate(query).exec((err, data) => {
if (err) return next(err);
if (data[0].count) {
res.header('x-total-items', data[0].count.length > 0 ? data[0].count[0].count : 0);
res.header('Access-Control-Expose-Headers', 'x-total-items');
data = data[0].samples;
}
if (filters.fields.indexOf('added') >= 0) { // add added date
data.map(e => {
e.added = e._id.getTimestamp();
if (filters.fields.indexOf('_id') < 0) {
delete e._id;
}
return e
});
}
if (filters['to-page'] < 0) {
data.reverse();
}
const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]);
if (filters.csv) { // output as csv
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
if (err) return next(err);
res.set('Content-Type', 'text/csv');
res.send(data);
});
}
else {
res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors
}
});
}
else {
res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
res.write('[');
let count = 0;
const stream = collection.aggregate(query).cursor().exec();
stream.on('data', data => {
if (filters.fields.indexOf('added') >= 0) { // add added date
data.added = data._id.getTimestamp();
if (filters.fields.indexOf('_id') < 0) {
delete data._id;
}
}
res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++;
});
stream.on('close', () => {
res.write(']');
res.end();
});
}
});
router.get('/samples/:state(new|deleted)', (req, res, next) => {
@ -37,6 +371,15 @@ router.get('/samples/:state(new|deleted)', (req, res, next) => {
});
});
router.get('/samples/count', (req, res, next) => {
if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
SampleModel.estimatedDocumentCount((err, data) => {
if (err) return next(err);
res.json({count: data});
});
});
router.get('/sample/' + IdValidate.parameter(), (req, res, next) => {
if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
@ -54,7 +397,7 @@ router.get('/sample/' + IdValidate.parameter(), (req, res, next) => {
sampleData.material.supplier = sampleData.material.supplier_id.name;
sampleData.user = sampleData.user_id.name;
sampleData.notes = sampleData.note_id ? sampleData.note_id : {};
MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id)}).lean().exec((err, data) => {
MeasurementModel.find({sample_id: mongoose.Types.ObjectId(req.params.id), status: {$ne: globals.status.deleted}}).lean().exec((err, data) => {
sampleData.measurements = data;
res.json(SampleValidate.output(sampleData, 'details'));
});
@ -272,13 +615,23 @@ module.exports = router;
async function numberGenerate (sample, req, res, next) { // generate number in format Location32, returns false on error
const sampleData = await SampleModel
.findOne({number: new RegExp('^' + req.authDetails.location + '[0-9]+$', 'm')})
.sort({number: -1})
.lean()
// .findOne({number: new RegExp('^' + req.authDetails.location + '[0-9]+$', 'm')})
// .sort({number: -1})
// .lean()
.aggregate([
{$match: {number: new RegExp('^' + 'Rng' + '[0-9]+$', 'm')}},
// {$addFields: {number2: {$toDecimal: {$arrayElemAt: [{$split: [{$arrayElemAt: [{$split: ['$number', 'Rng']}, 1]}, '_']}, 0]}}}}, // not working with MongoDb 3.6
{$addFields: {sortNumber: {$let: {
vars: {tmp: {$concat: ['000000000000000000000000000000', {$arrayElemAt: [{$split: [{$arrayElemAt: [{$split: ['$number', 'Rng']}, 1]}, '_']}, 0]}]}},
in: {$substrCP: ['$$tmp', {$subtract: [{$strLenCP: '$$tmp'}, 30]}, {$strLenCP: '$$tmp'}]}
}}}},
{$sort: {sortNumber: -1}},
{$limit: 1}
])
.exec()
.catch(err => next(err));
if (sampleData instanceof Error) return false;
return req.authDetails.location + (sampleData ? Number(sampleData.number.replace(/[^0-9]+/g, '')) + 1 : 1);
return req.authDetails.location + (sampleData[0] ? Number(sampleData[0].number.replace(/[^0-9]+/g, '')) + 1 : 1);
}
async function numberCheck(sample, res, next) {
@ -297,7 +650,7 @@ async function materialCheck (sample, res, next, id = sample.material_id) { //
res.status(400).json({status: 'Material not available'});
return false;
}
if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified
if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified
res.status(400).json({status: 'Color not available for material'});
return false;
}
@ -374,4 +727,54 @@ function customFieldsChange (fields, amount, req) { // update custom_fields and
}
});
});
}
function sortQuery(filters, sortKeys, sortStartValue) { // sortKeys = ['primary key', 'secondary key']
if (filters['from-id']) { // from-id specified
if ((filters['to-page'] === 0 && filters.sort[1] === 1) || (filters.sort[1] * filters['to-page'] > 0)) { // asc
return [{$match: {$or: [{[sortKeys[0]]: {$gt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$gte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
{$sort: {[sortKeys[0]]: 1, _id: 1}}];
} else {
return [{$match: {$or: [{[sortKeys[0]]: {$lt: sortStartValue}}, {$and: [{[sortKeys[0]]: sortStartValue}, {[sortKeys[1]]: {$lte: new mongoose.Types.ObjectId(filters['from-id'])}}]}]}},
{$sort: {[sortKeys[0]]: -1, _id: -1}}];
}
} else { // sort from beginning
return [{$sort: {[sortKeys[0]]: filters.sort[1], [sortKeys[1]]: filters.sort[1]}}]; // set _id as secondary sort
}
}
function statusQuery(filters, field) {
if (filters.hasOwnProperty('status')) {
if(filters.status === 'all') {
return {$or: [{[field]: globals.status.validated}, {[field]: globals.status.new}]};
}
else {
return {[field]: globals.status[filters.status]};
}
}
else { // default
return {[field]: globals.status.validated};
}
}
function addFilterQueries (queryPtr, filters) { // returns array of match queries from given filters
if (filters.length) {
queryPtr.push({$match: {$and: filterQueries(filters)}});
}
}
function filterQueries (filters) {
console.log(filters);
return filters.map(e => {
if (e.mode === 'or') { // allow or queries (needed for $ne added)
return {['$' + e.mode]: e.values};
}
else {
return {[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}}; // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin
}
});
}
function dateToOId (date) { // convert date to ObjectId
return mongoose.Types.ObjectId(Math.floor(date / 1000).toString(16) + '0000000000000000');
}

View File

@ -107,4 +107,10 @@ export default class MaterialValidate { // validate input for material
numbers: this.material.numbers
});
}
static query (data) {
return Joi.object({
status: Joi.string().valid('validated', 'new', 'all')
}).validate(data);
}
}

View File

@ -44,4 +44,13 @@ export default class MeasurementValidate {
}).validate(data, {stripUnknown: true});
return error !== undefined? null : value;
}
static outputV() { // return output validator
return Joi.object({
_id: IdValidate.get(),
sample_id: IdValidate.get(),
values: this.measurement.values,
measurement_template: IdValidate.get()
});
}
}

View File

@ -3,6 +3,7 @@ import Joi from '@hapi/joi';
import IdValidate from './id';
import UserValidate from './user';
import MaterialValidate from './material';
import MeasurementValidate from './measurement';
export default class SampleValidate {
private static sample = {
@ -10,7 +11,8 @@ export default class SampleValidate {
.max(128),
color: Joi.string()
.max(128),
.max(128)
.allow(''),
type: Joi.string()
.max(128),
@ -43,9 +45,42 @@ export default class SampleValidate {
Joi.date()
)
)
})
}),
added: Joi.date()
.iso()
.min('1970-01-01T00:00:00.000Z')
};
private static sortKeys = [
'_id',
'color',
'number',
'type',
'batch',
'added',
'material.name',
'material.supplier',
'material.group',
'material.mineral',
'material.glass_fiber',
'material.carbon_fiber',
'material.number',
'measurements.(?!spectrum)*'
];
private static fieldKeys = [
...SampleValidate.sortKeys,
'condition',
'material_id',
'material',
'note_id',
'user_id',
'material._id',
'material.numbers',
'measurements.spectrum.dpt'
];
static input (data, param) { // validate input, set param to 'new' to make all attributes required
if (param === 'new') {
return Joi.object({
@ -83,7 +118,11 @@ export default class SampleValidate {
}
}
static output (data, param = 'refs') { // validate output and strip unwanted properties, returns null if not valid
static output (data, param = 'refs+added', additionalParams = []) { // validate output and strip unwanted properties, returns null if not valid
if (param === 'refs+added') {
param = 'refs';
data.added = data._id.getTimestamp();
}
data = IdValidate.stringify(data);
let joiObject;
if (param === 'refs') {
@ -95,8 +134,10 @@ export default class SampleValidate {
batch: this.sample.batch,
condition: this.sample.condition,
material_id: IdValidate.get(),
material: MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')}),
note_id: IdValidate.get().allow(null),
user_id: IdValidate.get()
user_id: IdValidate.get(),
added: this.sample.added
};
}
else if(param === 'details') {
@ -108,6 +149,7 @@ export default class SampleValidate {
batch: this.sample.batch,
condition: this.sample.condition,
material: MaterialValidate.outputV(),
measurements: Joi.array().items(MeasurementValidate.outputV()),
notes: this.sample.notes,
user: UserValidate.username()
}
@ -115,7 +157,67 @@ export default class SampleValidate {
else {
return null;
}
additionalParams.forEach(param => {
joiObject[param] = Joi.any();
});
const {value, error} = Joi.object(joiObject).validate(data, {stripUnknown: true});
return error !== undefined? null : value;
}
static query (data) {
if (data.filters && data.filters.length) {
const filterValidation = Joi.array().items(Joi.string()).validate(data.filters);
if (filterValidation.error) return filterValidation;
try {
for (let i in data.filters) {
data.filters[i] = JSON.parse(data.filters[i]);
data.filters[i].values = data.filters[i].values.map(e => { // validate filter values
let validator;
let field = data.filters[i].field
if (/material\./.test(field)) { // select right validation model
validator = MaterialValidate.outputV().append({number: Joi.string().max(128).allow('')});
field = field.replace('material.', '');
}
else if (/measurements\./.test(field)) {
validator = Joi.object({
value: Joi.alternatives()
.try(
Joi.number(),
Joi.string().max(128),
Joi.boolean(),
Joi.array()
)
.allow(null)
});
field = 'value';
}
else {
validator = Joi.object(this.sample);
}
const {value, error} = validator.validate({[field]: e});
console.log(value);
if (error) throw error; // reject invalid values // TODO: return exact error description, handle in frontend filters
return value[field];
});
}
}
catch {
return {error: {details: [{message: 'Invalid JSON string for filter parameter'}]}, value: null}
}
}
return Joi.object({
status: Joi.string().valid('validated', 'new', 'all'),
'from-id': IdValidate.get(),
'to-page': Joi.number().integer(),
'page-size': Joi.number().integer().min(1),
sort: Joi.string().pattern(new RegExp('^(' + this.sortKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')-(asc|desc)$', 'm')).default('_id-asc'),
csv: Joi.boolean().default(false),
fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']),
filters: Joi.array().items(Joi.object({
mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'),
field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')),
values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso())).min(1)
})).default([])
}).with('to-page', 'page-size').validate(data);
}
}

View File

@ -1,6 +1,7 @@
import Joi from '@hapi/joi';
import IdValidate from './id';
// TODO: do not allow a . in the name
export default class TemplateValidate {
private static template = {
name: Joi.string()

View File

@ -411,12 +411,26 @@
"_id": {"$oid":"800000000000000000000006"},
"sample_id": {"$oid":"400000000000000000000006"},
"values": {
"weight %": 0.5,
"weight %": 0.6,
"standard deviation":null
},
"status": 0,
"measurement_template": {"$oid":"300000000000000000000002"},
"__v": 0
},
{
"_id": {"$oid":"800000000000000000000007"},
"sample_id": {"$oid":"400000000000000000000001"},
"values": {
"dpt": [
[3996.12558,98.00555],
[3995.08519,98.03253],
[3993.04480,98.02657]
]
},
"status": 10,
"measurement_template": {"$oid":"300000000000000000000001"},
"__v": 0
}
],
"condition_templates": [

View File

@ -38,15 +38,7 @@ export default class TestHelper {
return server
}
static afterEach (server, done) {
server.close(done);
}
static after(done) {
db.disconnect(done);
}
static request (server, done, options) { // options in form: {method, url, auth: {key/basic: 'name' or 'key'/{name, pass}}, httpStatus, req, res, default (set to false if you want to dismiss default .end handling)}
static request (server, done, options) { // options in form: {method, url, contentType, auth: {key/basic: 'name' or 'key'/{name, pass}}, httpStatus, req, res, default (set to false if you want to dismiss default .end handling)}
let st = supertest(server);
if (options.hasOwnProperty('auth') && options.auth.hasOwnProperty('key')) { // resolve API key
options.url += '?key=' + (this.auth.hasOwnProperty(options.auth.key)? this.auth[options.auth.key].key : options.auth.key);
@ -79,8 +71,12 @@ export default class TestHelper {
st = st.auth(options.auth.basic.name, options.auth.basic.pass)
}
}
st = st.expect('Content-type', /json/)
.expect(options.httpStatus);
if (options.hasOwnProperty('contentType')) {
st = st.expect('Content-type', options.contentType).expect(options.httpStatus);
}
else {
st = st.expect('Content-type', /json/).expect(options.httpStatus);
}
if (options.hasOwnProperty('res')) { // evaluate result
return st.end((err, res) => {
if (err) return done (err);
@ -128,4 +124,12 @@ export default class TestHelper {
return st;
}
}
static afterEach (server, done) {
server.close(done);
}
static after(done) {
db.disconnect(done);
}
}