banana/definma-api
Archived
2
Fork 0
Code Releases Activity

spectrum field working again

Browse Source
This commit is contained in:
VLE2FE 2020-07-09 13:48:27 +02:00
parent 6a02f09e7f
commit 1ddc2b617a
13 changed files with 269 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/api.yaml
View File

@ -39,10 +39,10 @@ info:
servers: servers:
- url: https://definma-api.apps.de1.bosch-iot-cloud.com
description: server on the BIC
- url: http://localhost:3000 - url: http://localhost:3000
description: local server description: local server
- url: https://digital-fingerprint-of-plastics-api.apps.de1.bosch-iot-cloud.com/
description: server on the BIC
security: security:

8
api/sample.yaml
View File

@ -2,7 +2,7 @@
get: get:
summary: all samples in overview summary: all samples in overview
description: 'Auth: all, levels: read, write, maintain, dev, admin' description: 'Auth: all, levels: read, write, maintain, dev, admin'
x-doc: returns only samples with status 10 x-doc: 'Limitations: paging and csv output does not work when including the spectrum measurement fields as well as the returned number of total samples'
tags: tags:
- /sample - /sample
parameters: parameters:
@ -61,6 +61,12 @@
responses: responses:
200: 200:
description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)
headers:
X-Total-Items:
description: Total number of available items when page is specified
schema:
type: integer
example: 243
content: content:
application/json: application/json:
schema: schema:

209
data_import/import.js
View File

@ -1,55 +1,82 @@
const csv = require('csv-parser'); const csv = require('csv-parser');
const fs = require('fs'); const fs = require('fs');
const axios = require('axios'); const axios = require('axios');
const {Builder} = require('selenium-webdriver'); const {Builder} = require('selenium-webdriver'); // selenium and the chrome driver must be installed and configured separately
const chrome = require('selenium-webdriver/chrome'); const chrome = require('selenium-webdriver/chrome');
const pdfReader = require('pdfreader'); const pdfReader = require('pdfreader');
const iconv = require('iconv-lite'); const iconv = require('iconv-lite');
const metadata = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\VZ.csv'; // metadata file const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv'; // metadata files
const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\nmDocs'; // NormMaster Documents const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\DPT'; // Spectrum files const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv';
const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs'; // NormMaster Documents
const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT'; // Spectrum files
// const host = 'http://localhost:3000';
const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
let data = []; // metadata contents let data = []; // metadata contents
let materials = {}; let materials = {};
let samples = []; let samples = [];
let normMaster = {}; let normMaster = {};
let sampleDevices = {};
// TODO: integrate measurement device information from DPT names using different users
// TODO: supplier: other for supplierless samples
// TODO: BASF twice, BASF as color // TODO: BASF twice, BASF as color
// TODO: trim color names
// TODO: duplicate kf values // TODO: duplicate kf values
// TODO: conditions
// TODO: comment and reference handling
// TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC
main(); main();
async function main() { async function main() {
if (0) { // materials if (0) { // materials
await getNormMaster(); await getNormMaster();
await importCsv(); await importCsv(metaDoc);
await allMaterials();
await saveMaterials();
await importCsv(kfDoc);
await allMaterials();
await saveMaterials();
await importCsv(vzDoc);
await allMaterials(); await allMaterials();
fs.writeFileSync('./data_import/materials.json', JSON.stringify(materials));
await saveMaterials(); await saveMaterials();
} }
else if (0) { // samples if (0) { // samples
await importCsv(); sampleDeviceMap();
await allSamples(); if (1) {
await saveSamples(); console.log('-------- META ----------');
await importCsv(metaDoc);
await allSamples();
await saveSamples();
}
if (1) {
console.log('-------- KF ----------');
await importCsv(kfDoc);
await allSamples();
await saveSamples();
await allKfVz();
}
if (1) {
console.log('-------- VZ ----------');
await importCsv(vzDoc);
await allSamples();
await saveSamples();
await allKfVz();
}
} }
else if (1) { // DPT if (1) { // DPT
await allDpts(); await allDpts();
} }
else if (0) { // KF/VZ if (0) { // pdf test
await importCsv(); console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf'));
await allKfVz();
}
else if (0) { // pdf test
console.log(await readPdf('N28_BN22-O010_2018-03-08.pdf'));
} }
} }
async function importCsv() { async function importCsv(doc) {
data = [];
await new Promise(resolve => { await new Promise(resolve => {
fs.createReadStream(metadata) fs.createReadStream(doc)
.pipe(iconv.decodeStream('win1252')) .pipe(iconv.decodeStream('win1252'))
.pipe(csv()) .pipe(csv())
.on('data', (row) => { .on('data', (row) => {
@ -57,6 +84,9 @@ async function importCsv() {
}) })
.on('end', () => { .on('end', () => {
console.info('CSV file successfully processed'); console.info('CSV file successfully processed');
if (data[0]['Farbe']) { // fix German column names
data.map(e => {e['Color'] = e['Farbe']; return e; });
}
resolve(); resolve();
}); });
}); });
@ -65,7 +95,7 @@ async function importCsv() {
async function allDpts() { async function allDpts() {
let res = await axios({ let res = await axios({
method: 'get', method: 'get',
url: 'http://localhost:3000/template/measurements', url: host + '/template/measurements',
auth: { auth: {
username: 'admin', username: 'admin',
password: 'Abc123!#' password: 'Abc123!#'
@ -74,7 +104,7 @@ async function allDpts() {
const measurement_template = res.data.find(e => e.name === 'spectrum')._id; const measurement_template = res.data.find(e => e.name === 'spectrum')._id;
res = await axios({ res = await axios({
method: 'get', method: 'get',
url: 'http://localhost:3000/samples?status=all', url: host + '/samples?status=all',
auth: { auth: {
username: 'admin', username: 'admin',
password: 'Abc123!#' password: 'Abc123!#'
@ -84,10 +114,10 @@ async function allDpts() {
res.data.forEach(sample => { res.data.forEach(sample => {
sampleIds[sample.number] = sample._id; sampleIds[sample.number] = sample._id;
}); });
const regex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/; const dptRegex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/;
const dpts = fs.readdirSync(dptFiles); const dpts = fs.readdirSync(dptFiles);
for (let i in dpts) { for (let i in dpts) {
const regexRes = regex.exec(dpts[i]) const regexRes = dptRegex.exec(dpts[i])
if (regexRes && sampleIds[regexRes[1]]) { // found matching sample if (regexRes && sampleIds[regexRes[1]]) { // found matching sample
console.log(dpts[i]); console.log(dpts[i]);
const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8'); const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8');
@ -99,7 +129,7 @@ async function allDpts() {
data.values.dpt = f.split('\r\n').map(e => e.split(',')); data.values.dpt = f.split('\r\n').map(e => e.split(','));
await axios({ await axios({
method: 'post', method: 'post',
url: 'http://localhost:3000/measurement/new', url: host + '/measurement/new',
auth: { auth: {
username: 'admin', username: 'admin',
password: 'Abc123!#' password: 'Abc123!#'
@ -110,13 +140,16 @@ async function allDpts() {
console.error(err.response.data); console.error(err.response.data);
}); });
} }
else {
console.log(`Could not find sample for ${dpts[i]} !!!!!!`);
}
} }
} }
async function allKfVz() { async function allKfVz() {
let res = await axios({ let res = await axios({
method: 'get', method: 'get',
url: 'http://localhost:3000/template/measurements', url: host + '/template/measurements',
auth: { auth: {
username: 'admin', username: 'admin',
password: 'Abc123!#' password: 'Abc123!#'
@ -126,7 +159,7 @@ async function allKfVz() {
const vz_template = res.data.find(e => e.name === 'vz')._id; const vz_template = res.data.find(e => e.name === 'vz')._id;
res = await axios({ res = await axios({
method: 'get', method: 'get',
url: 'http://localhost:3000/samples?status=all', url: host + '/samples?status=all',
auth: { auth: {
username: 'admin', username: 'admin',
password: 'Abc123!#' password: 'Abc123!#'
@ -140,13 +173,17 @@ async function allKfVz() {
console.info(`${index}/${data.length}`); console.info(`${index}/${data.length}`);
let sample = data[index]; let sample = data[index];
if (sample['Sample number'] !== '') { if (sample['Sample number'] !== '') {
let credentials = ['admin', 'Abc123!#'];
if (sampleDevices[sample['Sample number']]) {
credentials = [sampleDevices[sample['Sample number']], '2020DeFinMachen!']
}
if (sample['KF in Gew%']) { if (sample['KF in Gew%']) {
await axios({ await axios({
method: 'post', method: 'post',
url: 'http://localhost:3000/measurement/new', url: host + '/measurement/new',
auth: { auth: {
username: 'admin', username: credentials[0],
password: 'Abc123!#' password: credentials[1]
}, },
data: { data: {
sample_id: sampleIds[sample['Sample number']], sample_id: sampleIds[sample['Sample number']],
@ -164,10 +201,10 @@ async function allKfVz() {
if (sample['VZ (ml/g)']) { if (sample['VZ (ml/g)']) {
await axios({ await axios({
method: 'post', method: 'post',
url: 'http://localhost:3000/measurement/new', url: host + '/measurement/new',
auth: { auth: {
username: 'admin', username: credentials[0],
password: 'Abc123!#' password: credentials[1]
}, },
data: { data: {
sample_id: sampleIds[sample['Sample number']], sample_id: sampleIds[sample['Sample number']],
@ -186,9 +223,10 @@ async function allKfVz() {
} }
async function allSamples() { async function allSamples() {
samples = [];
let res = await axios({ let res = await axios({
method: 'get', method: 'get',
url: 'http://localhost:3000/materials?status=all', url: host + '/materials?status=all',
auth: { auth: {
username: 'admin', username: 'admin',
password: 'Abc123!#' password: 'Abc123!#'
@ -200,7 +238,7 @@ async function allSamples() {
}) })
res = await axios({ res = await axios({
method: 'get', method: 'get',
url: 'http://localhost:3000/samples?status=all', url: host + '/samples?status=all',
auth: { auth: {
username: 'admin', username: 'admin',
password: 'Abc123!#' password: 'Abc123!#'
@ -215,7 +253,13 @@ async function allSamples() {
for (let index in data) { for (let index in data) {
console.info(`${index}/${data.length}`); console.info(`${index}/${data.length}`);
let sample = data[index]; let sample = data[index];
if (sample['Sample number'] !== '' && sample['Supplier'] !== '' && sample['Granulate/Part'] !== '') { // TODO: wait for decision about samples without suppliers/color/type if (sample['Sample number'] !== '') { // TODO: what about samples without color
if (sample['Supplier'] === '') { // empty supplier fields
sample['Supplier'] = 'unknown';
}
if (sample['Granulate/Part'] === '') { // empty supplier fields
sample['Granulate/Part'] = 'unknown';
}
const material = dbMaterials[trim(sample['Material name'])]; const material = dbMaterials[trim(sample['Material name'])];
if (!material) { // could not find material, skipping sample if (!material) { // could not find material, skipping sample
continue; continue;
@ -236,13 +280,20 @@ async function allSamples() {
samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color; samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
} }
else if (sample['Color'] && sample['Color'] !== '') { else if (sample['Color'] && sample['Color'] !== '') {
samples[si].color = material.numbers.find(e => e.color.indexOf(sample['Color']) >= 0).color; let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0);
if (!number && /black/.test(sample['Color'])) { // special case bk for black
number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
if (!number) { // try German word
number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
}
}
samples[si].color = number.color;
} }
else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz
samples[si].color = sampleColors[sample['Sample number'].split('_')[0]]; samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
} }
else { // TODO: no color information at all else {
samples.pop(); samples[si].color = '';
} }
} }
} }
@ -251,41 +302,57 @@ async function allSamples() {
async function saveSamples() { async function saveSamples() {
for (let i in samples) { for (let i in samples) {
console.info(`${i}/${samples.length}`); console.info(`${i}/${samples.length}`);
let credentials = ['admin', 'Abc123!#'];
if (sampleDevices[samples[i].number]) {
credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!']
}
await axios({ await axios({
method: 'post', method: 'post',
url: 'http://localhost:3000/sample/new', url: host + '/sample/new',
auth: { auth: {
username: 'admin', username: credentials[0],
password: 'Abc123!#' password: credentials[1]
}, },
data: samples[i] data: samples[i]
}).catch(err => { }).catch(err => {
console.log(samples[i]); if (err.response.data.status && err.response.data.status !== 'Sample number already taken') {
console.error(err.response.data); console.log(samples[i]);
console.error(err.response.data);
}
}); });
} }
console.info('saved all samples'); console.info('saved all samples');
} }
async function allMaterials() { async function allMaterials() {
materials = {};
for (let index in data) { for (let index in data) {
let sample = data[index]; let sample = data[index];
if (sample['Sample number'] !== '' && sample['Supplier'] !== '') { // TODO: wait for decision about supplierless samples if (sample['Sample number'] && sample['Sample number'] !== '') {
if (sample['Supplier'] === '') { // empty supplier fields
sample['Supplier'] = 'unknown';
}
if (sample['Material name'] === '') { // empty name fields
sample['Material name'] = sample['Material'];
}
if (!sample['Material']) { // column Material is named Plastic in VZ metadata
sample['Material'] = sample['Plastic'];
}
sample['Material name'] = trim(sample['Material name']); sample['Material name'] = trim(sample['Material name']);
if (materials.hasOwnProperty(sample['Material name'])) { // material already found at least once if (materials.hasOwnProperty(sample['Material name'])) { // material already found at least once
if (sample['Material number'] !== '') { if (sample['Material number'] && sample['Material number'] !== '') {
if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) { // new material number if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) { // new material number
if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) { // color already in list, only number missing if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) { // color already in list, only number missing
materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']); materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']);
} }
else { else {
materials[sample['Material name']].numbers.push({color: sample['Color'], number: stripSpaces(sample['Material number'])}); materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])});
} }
} }
} }
else if (sample['Color'] !== '') { else if (sample['Color'] && sample['Color'] !== '') {
if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) { // new material color if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) { // new material color
materials[sample['Material name']].numbers.push({color: sample['Color'], number: ''}); materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''});
} }
} }
} }
@ -293,8 +360,8 @@ async function allMaterials() {
console.info(`${index}/${data.length} ${sample['Material name']}`); console.info(`${index}/${data.length} ${sample['Material name']}`);
materials[sample['Material name']] = { materials[sample['Material name']] = {
name: sample['Material name'], name: sample['Material name'],
supplier: sample['Supplier'], supplier: trim(sample['Supplier']),
group: sample['Material'] group: trim(sample['Material'])
}; };
let tmp = /M(\d+)/.exec(sample['Reinforcing material']); let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
materials[sample['Material name']].mineral = tmp ? tmp[1] : 0; materials[sample['Material name']].mineral = tmp ? tmp[1] : 0;
@ -312,17 +379,20 @@ async function allMaterials() {
async function saveMaterials() { async function saveMaterials() {
const mKeys = Object.keys(materials) const mKeys = Object.keys(materials)
for (let i in mKeys) { for (let i in mKeys) {
console.info(`${i}/${mKeys.length}`);
await axios({ await axios({
method: 'post', method: 'post',
url: 'http://localhost:3000/material/new', url: host + '/material/new',
auth: { auth: {
username: 'admin', username: 'admin',
password: 'Abc123!#' password: 'Abc123!#'
}, },
data: materials[mKeys[i]] data: materials[mKeys[i]]
}).catch(err => { }).catch(err => {
console.log(materials[mKeys[i]]); if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
console.error(err.response.data); console.info(materials[mKeys[i]]);
console.error(err.response.data);
}
}); });
} }
console.info('saved all materials'); console.info('saved all materials');
@ -362,16 +432,16 @@ async function numbersFetch(sample) {
} }
} }
if (res.length === 0) { // no results if (res.length === 0) { // no results
if (sample['Color'] !== '' || sample['Material number'] !== '') { if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) {
return [{color: sample['Color'], number: sample['Material number']}]; return [{color: trim(sample['Color']), number: sample['Material number']}];
} }
else { else {
return []; return [];
} }
} }
else { else {
if (!res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed
res.push({color: sample['Color'], number: sample['Material number']}); res.push({color: trim(sample['Color']), number: sample['Material number']});
} }
return res; return res;
} }
@ -403,7 +473,7 @@ async function getNormMaster(fetchAgain = false) {
} }
function getNormMasterDoc(url, timing = 1) { function getNormMasterDoc(url, timing = 1) {
console.log(url); console.info(url);
return new Promise(async resolve => { return new Promise(async resolve => {
const options = new chrome.Options(); const options = new chrome.Options();
options.setUserPreferences({ options.setUserPreferences({
@ -453,7 +523,7 @@ function readPdf(file) {
rows.push(item.text); rows.push(item.text);
} }
else { // still the same row row else { // still the same row row
rows[rows.length - 1] += (item.x - lastX > 1.1 ? '$' : '') + item.text; // push to row, detect if still same cell rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text; // push to row, detect if still same cell
} }
lastX = (item.w * 0.055) + item.x; lastX = (item.w * 0.055) + item.x;
@ -465,7 +535,7 @@ function readPdf(file) {
table = -1; table = -1;
// console.log(rows); // console.log(rows);
rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows
resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; })); resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
} }
} }
lastLastText = lastText; lastLastText = lastText;
@ -473,12 +543,23 @@ function readPdf(file) {
} }
if (!item && table !== -1) { // document ended if (!item && table !== -1) { // document ended
rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows
resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; })); resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
} }
}); });
}); });
} }
function sampleDeviceMap() {
const dpts = fs.readdirSync(dptFiles);
const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/;
for (let i in dpts) {
const regexRes = regex.exec(dpts[i])
if (regexRes) { // found matching sample
sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase();
}
}
}
function stripSpaces(s) { function stripSpaces(s) {
return s ? s.replace(/ /g,'') : ''; return s ? s.replace(/ /g,'') : '';
} }

5
mainfest.yml → manifest.yml
View File

@ -1,6 +1,7 @@
--- ---
applications: applications:
- name: digital-fingerprint-of-plastics-api - name: definma-api
path: dist/
instances: 1 instances: 1
memory: 256M memory: 256M
stack: cflinuxfs3 stack: cflinuxfs3
@ -10,4 +11,4 @@ applications:
NODE_ENV: production NODE_ENV: production
OPTIMIZE_MEMORY: true OPTIMIZE_MEMORY: true
services: services:
- dfopdb - definmadb

14
package-lock.json generated
View File

@ -1174,6 +1174,15 @@
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true "dev": true
}, },
"cors": {
"version": "2.8.5",
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
"integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
"requires": {
"object-assign": "^4",
"vary": "^1"
}
},
"cross-spawn": { "cross-spawn": {
"version": "7.0.3", "version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@ -2866,6 +2875,11 @@
} }
} }
}, },
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"object-inspect": { "object-inspect": {
"version": "1.7.0", "version": "1.7.0",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz",

4
package.json
View File

@ -7,6 +7,7 @@
"tsc": "tsc", "tsc": "tsc",
"tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc", "tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc",
"build": "build.bat", "build": "build.bat",
"build-push": "build.bat && cf push",
"test": "mocha dist/**/**.spec.js", "test": "mocha dist/**/**.spec.js",
"start": "node index.js", "start": "node index.js",
"dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"", "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"",
@ -28,6 +29,7 @@
"cfenv": "^1.2.2", "cfenv": "^1.2.2",
"compression": "^1.7.4", "compression": "^1.7.4",
"content-filter": "^1.1.2", "content-filter": "^1.1.2",
"cors": "^2.8.5",
"express": "^4.17.1", "express": "^4.17.1",
"helmet": "^3.22.0", "helmet": "^3.22.0",
"json-schema": "^0.2.5", "json-schema": "^0.2.5",
@ -35,7 +37,7 @@
"lodash": "^4.17.15", "lodash": "^4.17.15",
"mongo-sanitize": "^1.1.0", "mongo-sanitize": "^1.1.0",
"mongoose": "^5.8.7", "mongoose": "^5.8.7",
"swagger-ui-express": "^4.1.2" "swagger-ui-express": "4.1.2"
}, },
"devDependencies": { "devDependencies": {
"@types/bcrypt": "^3.0.0", "@types/bcrypt": "^3.0.0",

1
src/api.ts
View File

@ -18,6 +18,7 @@ export default class api {
jsonRefParser.bundle('api/api.yaml', (err, doc) => { // parse yaml jsonRefParser.bundle('api/api.yaml', (err, doc) => { // parse yaml
if (err) throw err; if (err) throw err;
apiDoc = doc; apiDoc = doc;
apiDoc.servers.splice(process.env.NODE_ENV === 'production', 1);
apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e)); // bundle routes apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e)); // bundle routes
apiDoc = this.resolveXDoc(apiDoc); apiDoc = this.resolveXDoc(apiDoc);
oasParser.validate(apiDoc, (err, api) => { // validate oas schema oasParser.validate(apiDoc, (err, api) => { // validate oas schema

6
src/helpers/csv.ts
View File

@ -1,12 +1,6 @@
import {parseAsync} from 'json2csv'; import {parseAsync} from 'json2csv';
export default function csv(input: any[], f: (err, data) => void) { export default function csv(input: any[], f: (err, data) => void) {
console.log(input[1000]);
console.log(flatten(input[1000]));
parseAsync([flatten(input[1000])]).then(csv => console.log(csv));
console.log(input[1]);
console.log(flatten(input[1]));
parseAsync([flatten(input[1])]).then(csv => console.log(csv));
parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true}) parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
.then(csv => f(null, csv)) .then(csv => f(null, csv))
.catch(err => f(err, null)); .catch(err => f(err, null));

2
src/helpers/mail.ts
View File

@ -17,7 +17,7 @@ export default (mailAddress, subject, content, f) => { // callback, executed em
contentType: "text/html" contentType: "text/html"
}, },
from: { from: {
eMail: "dfop@bosch-iot.com", eMail: "definma@bosch-iot.com",
password: "PlasticsOfFingerprintDigital" password: "PlasticsOfFingerprintDigital"
} }
} }

3
src/index.ts
View File

@ -4,6 +4,7 @@ import compression from 'compression';
import contentFilter from 'content-filter'; import contentFilter from 'content-filter';
import mongoSanitize from 'mongo-sanitize'; import mongoSanitize from 'mongo-sanitize';
import helmet from 'helmet'; import helmet from 'helmet';
import cors from 'cors';
import api from './api'; import api from './api';
import db from './db'; import db from './db';
@ -42,9 +43,11 @@ app.use((req, res, next) => { // no database connection error
next(); next();
} }
else { else {
console.error('No database connection');
res.status(500).send({status: 'Internal server error'}); res.status(500).send({status: 'Internal server error'});
} }
}); });
app.use(cors()); // CORS headers
app.use(require('./helpers/authorize')); // handle authentication app.use(require('./helpers/authorize')); // handle authentication
// redirect /api routes for Angular proxy in development // redirect /api routes for Angular proxy in development

1
src/routes/sample.spec.ts
View File

@ -21,6 +21,7 @@ describe('/sample', () => {
// TODO: sort, added date filter, has measurements/condition filter // TODO: sort, added date filter, has measurements/condition filter
// TODO: check if conditions work in sort/fields/filters // TODO: check if conditions work in sort/fields/filters
// TODO: test for numbers as strings in glass_fiber
describe('GET /samples', () => { describe('GET /samples', () => {
it('returns all samples', done => { it('returns all samples', done => {
TestHelper.request(server, done, { TestHelper.request(server, done, {

95
src/routes/sample.ts
View File

@ -21,6 +21,12 @@ import csv from '../helpers/csv';
const router = express.Router(); const router = express.Router();
// TODO: check added filter
// TODO: return total number of pages -> use facet
// TODO: use query pointer
// TODO: convert filter value to number according to table model
// TODO: validation for filter parameters
// TODO: location/device sort/filter
router.get('/samples', async (req, res, next) => { router.get('/samples', async (req, res, next) => {
if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
@ -37,6 +43,7 @@ router.get('/samples', async (req, res, next) => {
if (!filters['to-page']) { // set to-page default if (!filters['to-page']) { // set to-page default
filters['to-page'] = 0; filters['to-page'] = 0;
} }
console.log(filters);
const sortFilterKeys = filters.filters.map(e => e.field); const sortFilterKeys = filters.filters.map(e => e.field);
@ -70,7 +77,7 @@ router.get('/samples', async (req, res, next) => {
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
{$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added {$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added
{$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
); );
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
@ -106,25 +113,25 @@ router.get('/samples', async (req, res, next) => {
materialAdded = true; materialAdded = true;
materialQuery.push( // add material properties materialQuery.push( // add material properties
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields
{$set: {material: {$arrayElemAt: ['$material', 0]}}} {$addFields: {material: {$arrayElemAt: ['$material', 0]}}}
); );
const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0); const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters
if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed
materialQuery.push( materialQuery.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
); );
} }
if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed
materialQuery.push( materialQuery.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
); );
} }
if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed
materialQuery.push( materialQuery.push(
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
); );
} }
const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
@ -157,10 +164,10 @@ router.get('/samples', async (req, res, next) => {
as: 'measurements' as: 'measurements'
}}); }});
measurementTemplates.forEach(template => { measurementTemplates.forEach(template => {
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
}); });
addFilterQueries(query, filters.filters addFilterQueries(query, filters.filters
.filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
@ -173,39 +180,40 @@ router.get('/samples', async (req, res, next) => {
sortFilterKeys.indexOf(e) < 0 // field was not in filter sortFilterKeys.indexOf(e) < 0 // field was not in filter
&& e !== filters.sort[0] // field was not in sort && e !== filters.sort[0] // field was not in sort
); );
console.log(fieldsToAdd);
if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already
query.push( query.push(
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
{$set: {material: { $arrayElemAt: ['$material', 0]}}} {$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
); );
} }
if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed
query.push( query.push(
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
); );
} }
if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed
query.push( query.push(
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
); );
} }
if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed
query.push( query.push(
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
); );
} }
let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields
const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);}); const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
if (measurementTemplates instanceof Error) return; if (measurementTemplates instanceof Error) return;
if (measurementTemplates.length < measurementFieldsFields.length) { if (measurementTemplates.length < measurementFieldsFields.length) {
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
} }
if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance
query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
} }
else { else {
@ -216,15 +224,15 @@ router.get('/samples', async (req, res, next) => {
}}); }});
} }
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
}); });
if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
query.push( query.push(
{$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
{$set: {spectrum: '$spectrum.values.dpt'}}, {$addFields: {spectrum: '$spectrum.values'}},
{$unwind: '$spectrum'} {$unwind: '$spectrum'}
); );
} }
@ -233,30 +241,45 @@ router.get('/samples', async (req, res, next) => {
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
if (filters.fields.indexOf('added') >= 0) { // add added date if (filters.fields.indexOf('added') >= 0) { // add added date
projection.added = {$toDate: '$_id'}; // projection.added = {$toDate: '$_id'};
// projection.added = { $convert: { input: '$_id', to: "date" } } // TODO
} }
if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly
projection._id = false; projection._id = false;
} }
query.push({$project: projection}); query.push({$project: projection});
collection.aggregate(query).exec((err, data) => { if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files
if (err) return next(err); collection.aggregate(query).exec((err, data) => {
if (filters['to-page'] < 0) { if (err) return next(err);
data.reverse(); console.log(data.length);
} if (filters['to-page'] < 0) {
const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]); data.reverse();
if (filters.csv) { // output as csv }
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]);
if (err) return next(err); if (filters.csv) { // output as csv
res.set('Content-Type', 'text/csv'); csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
res.send(data); if (err) return next(err);
}); res.set('Content-Type', 'text/csv');
} res.send(data);
else { });
res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors }
} else {
}) res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors
}
});
}
else {
res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
res.write('[');
let count = 0;
const stream = collection.aggregate(query).cursor().exec();
stream.on('data', data => { res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; });
stream.on('close', () => {
res.write(']');
res.end();
});
}
}); });
router.get('/samples/:state(new|deleted)', (req, res, next) => { router.get('/samples/:state(new|deleted)', (req, res, next) => {
@ -537,7 +560,7 @@ async function materialCheck (sample, res, next, id = sample.material_id) { //
res.status(400).json({status: 'Material not available'}); res.status(400).json({status: 'Material not available'});
return false; return false;
} }
if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified
res.status(400).json({status: 'Color not available for material'}); res.status(400).json({status: 'Color not available for material'});
return false; return false;
} }

32
src/routes/validate/sample.ts
View File

@ -11,7 +11,8 @@ export default class SampleValidate {
.max(128), .max(128),
color: Joi.string() color: Joi.string()
.max(128), .max(128)
.allow(''),
type: Joi.string() type: Joi.string()
.max(128), .max(128),
@ -77,7 +78,7 @@ export default class SampleValidate {
'user_id', 'user_id',
'material._id', 'material._id',
'material.numbers', 'material.numbers',
'measurements.spectrum' 'measurements.spectrum.dpt'
]; ];
static input (data, param) { // validate input, set param to 'new' to make all attributes required static input (data, param) { // validate input, set param to 'new' to make all attributes required
@ -170,6 +171,33 @@ export default class SampleValidate {
try { try {
for (let i in data.filters) { for (let i in data.filters) {
data.filters[i] = JSON.parse(data.filters[i]); data.filters[i] = JSON.parse(data.filters[i]);
data.filters[i].values = data.filters[i].values.map(e => { // validate filter values
let validator;
let field = data.filters[i].field
if (/material\./.test(field)) { // select right validation model
validator = MaterialValidate.outputV();
field = field.replace('material.', '');
}
else if (/measurements\./.test(field)) {
validator = Joi.object({
value: Joi.alternatives()
.try(
Joi.string().max(128),
Joi.number(),
Joi.boolean(),
Joi.array()
)
.allow(null)
});
field = 'value';
}
else {
validator = Joi.object(this.sample);
}
const {value, error} = validator.validate({[field]: e});
if (error) throw error; // reject invalid values
return value[field];
});
} }
} }
catch { catch {