spectrum field working again
This commit is contained in:
VLE2FE
parent
6a02f09e7f
commit
1ddc2b617a
@ -39,10 +39,10 @@ info:
|
||||
|
||||
|
||||
servers:
|
||||
- url: https://definma-api.apps.de1.bosch-iot-cloud.com
|
||||
description: server on the BIC
|
||||
- url: http://localhost:3000
|
||||
description: local server
|
||||
- url: https://digital-fingerprint-of-plastics-api.apps.de1.bosch-iot-cloud.com/
|
||||
description: server on the BIC
|
||||
|
||||
|
||||
security:
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
get:
|
||||
summary: all samples in overview
|
||||
description: 'Auth: all, levels: read, write, maintain, dev, admin'
|
||||
x-doc: returns only samples with status 10
|
||||
x-doc: 'Limitations: paging and csv output does not work when including the spectrum measurement fields as well as the returned number of total samples'
|
||||
tags:
|
||||
- /sample
|
||||
parameters:
|
||||
@ -61,6 +61,12 @@
|
||||
responses:
|
||||
200:
|
||||
description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format)
|
||||
headers:
|
||||
X-Total-Items:
|
||||
description: Total number of available items when page is specified
|
||||
schema:
|
||||
type: integer
|
||||
example: 243
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
|
||||
@ -1,55 +1,82 @@
|
||||
const csv = require('csv-parser');
|
||||
const fs = require('fs');
|
||||
const axios = require('axios');
|
||||
const {Builder} = require('selenium-webdriver');
|
||||
const {Builder} = require('selenium-webdriver'); // selenium and the chrome driver must be installed and configured separately
|
||||
const chrome = require('selenium-webdriver/chrome');
|
||||
const pdfReader = require('pdfreader');
|
||||
const iconv = require('iconv-lite');
|
||||
|
||||
const metadata = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\VZ.csv'; // metadata file
|
||||
const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\nmDocs'; // NormMaster Documents
|
||||
const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\DPT'; // Spectrum files
|
||||
const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv'; // metadata files
|
||||
const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv';
|
||||
const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv';
|
||||
const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs'; // NormMaster Documents
|
||||
const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT'; // Spectrum files
|
||||
// const host = 'http://localhost:3000';
|
||||
const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
|
||||
let data = []; // metadata contents
|
||||
let materials = {};
|
||||
let samples = [];
|
||||
let normMaster = {};
|
||||
let sampleDevices = {};
|
||||
|
||||
// TODO: integrate measurement device information from DPT names using different users
|
||||
// TODO: supplier: other for supplierless samples
|
||||
// TODO: BASF twice, BASF as color
|
||||
// TODO: trim color names
|
||||
// TODO: duplicate kf values
|
||||
// TODO: conditions
|
||||
// TODO: comment and reference handling
|
||||
|
||||
|
||||
// TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC
|
||||
|
||||
main();
|
||||
|
||||
async function main() {
|
||||
if (0) { // materials
|
||||
await getNormMaster();
|
||||
await importCsv();
|
||||
await importCsv(metaDoc);
|
||||
await allMaterials();
|
||||
await saveMaterials();
|
||||
await importCsv(kfDoc);
|
||||
await allMaterials();
|
||||
await saveMaterials();
|
||||
await importCsv(vzDoc);
|
||||
await allMaterials();
|
||||
fs.writeFileSync('./data_import/materials.json', JSON.stringify(materials));
|
||||
await saveMaterials();
|
||||
}
|
||||
else if (0) { // samples
|
||||
await importCsv();
|
||||
if (0) { // samples
|
||||
sampleDeviceMap();
|
||||
if (1) {
|
||||
console.log('-------- META ----------');
|
||||
await importCsv(metaDoc);
|
||||
await allSamples();
|
||||
await saveSamples();
|
||||
}
|
||||
else if (1) { // DPT
|
||||
await allDpts();
|
||||
}
|
||||
else if (0) { // KF/VZ
|
||||
await importCsv();
|
||||
if (1) {
|
||||
console.log('-------- KF ----------');
|
||||
await importCsv(kfDoc);
|
||||
await allSamples();
|
||||
await saveSamples();
|
||||
await allKfVz();
|
||||
}
|
||||
else if (0) { // pdf test
|
||||
console.log(await readPdf('N28_BN22-O010_2018-03-08.pdf'));
|
||||
if (1) {
|
||||
console.log('-------- VZ ----------');
|
||||
await importCsv(vzDoc);
|
||||
await allSamples();
|
||||
await saveSamples();
|
||||
await allKfVz();
|
||||
}
|
||||
}
|
||||
if (1) { // DPT
|
||||
await allDpts();
|
||||
}
|
||||
if (0) { // pdf test
|
||||
console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf'));
|
||||
}
|
||||
}
|
||||
|
||||
async function importCsv() {
|
||||
async function importCsv(doc) {
|
||||
data = [];
|
||||
await new Promise(resolve => {
|
||||
fs.createReadStream(metadata)
|
||||
fs.createReadStream(doc)
|
||||
.pipe(iconv.decodeStream('win1252'))
|
||||
.pipe(csv())
|
||||
.on('data', (row) => {
|
||||
@ -57,6 +84,9 @@ async function importCsv() {
|
||||
})
|
||||
.on('end', () => {
|
||||
console.info('CSV file successfully processed');
|
||||
if (data[0]['Farbe']) { // fix German column names
|
||||
data.map(e => {e['Color'] = e['Farbe']; return e; });
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
@ -65,7 +95,7 @@ async function importCsv() {
|
||||
async function allDpts() {
|
||||
let res = await axios({
|
||||
method: 'get',
|
||||
url: 'http://localhost:3000/template/measurements',
|
||||
url: host + '/template/measurements',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
@ -74,7 +104,7 @@ async function allDpts() {
|
||||
const measurement_template = res.data.find(e => e.name === 'spectrum')._id;
|
||||
res = await axios({
|
||||
method: 'get',
|
||||
url: 'http://localhost:3000/samples?status=all',
|
||||
url: host + '/samples?status=all',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
@ -84,10 +114,10 @@ async function allDpts() {
|
||||
res.data.forEach(sample => {
|
||||
sampleIds[sample.number] = sample._id;
|
||||
});
|
||||
const regex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/;
|
||||
const dptRegex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/;
|
||||
const dpts = fs.readdirSync(dptFiles);
|
||||
for (let i in dpts) {
|
||||
const regexRes = regex.exec(dpts[i])
|
||||
const regexRes = dptRegex.exec(dpts[i])
|
||||
if (regexRes && sampleIds[regexRes[1]]) { // found matching sample
|
||||
console.log(dpts[i]);
|
||||
const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8');
|
||||
@ -99,7 +129,7 @@ async function allDpts() {
|
||||
data.values.dpt = f.split('\r\n').map(e => e.split(','));
|
||||
await axios({
|
||||
method: 'post',
|
||||
url: 'http://localhost:3000/measurement/new',
|
||||
url: host + '/measurement/new',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
@ -110,13 +140,16 @@ async function allDpts() {
|
||||
console.error(err.response.data);
|
||||
});
|
||||
}
|
||||
else {
|
||||
console.log(`Could not find sample for ${dpts[i]} !!!!!!`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function allKfVz() {
|
||||
let res = await axios({
|
||||
method: 'get',
|
||||
url: 'http://localhost:3000/template/measurements',
|
||||
url: host + '/template/measurements',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
@ -126,7 +159,7 @@ async function allKfVz() {
|
||||
const vz_template = res.data.find(e => e.name === 'vz')._id;
|
||||
res = await axios({
|
||||
method: 'get',
|
||||
url: 'http://localhost:3000/samples?status=all',
|
||||
url: host + '/samples?status=all',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
@ -140,13 +173,17 @@ async function allKfVz() {
|
||||
console.info(`${index}/${data.length}`);
|
||||
let sample = data[index];
|
||||
if (sample['Sample number'] !== '') {
|
||||
let credentials = ['admin', 'Abc123!#'];
|
||||
if (sampleDevices[sample['Sample number']]) {
|
||||
credentials = [sampleDevices[sample['Sample number']], '2020DeFinMachen!']
|
||||
}
|
||||
if (sample['KF in Gew%']) {
|
||||
await axios({
|
||||
method: 'post',
|
||||
url: 'http://localhost:3000/measurement/new',
|
||||
url: host + '/measurement/new',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
username: credentials[0],
|
||||
password: credentials[1]
|
||||
},
|
||||
data: {
|
||||
sample_id: sampleIds[sample['Sample number']],
|
||||
@ -164,10 +201,10 @@ async function allKfVz() {
|
||||
if (sample['VZ (ml/g)']) {
|
||||
await axios({
|
||||
method: 'post',
|
||||
url: 'http://localhost:3000/measurement/new',
|
||||
url: host + '/measurement/new',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
username: credentials[0],
|
||||
password: credentials[1]
|
||||
},
|
||||
data: {
|
||||
sample_id: sampleIds[sample['Sample number']],
|
||||
@ -186,9 +223,10 @@ async function allKfVz() {
|
||||
}
|
||||
|
||||
async function allSamples() {
|
||||
samples = [];
|
||||
let res = await axios({
|
||||
method: 'get',
|
||||
url: 'http://localhost:3000/materials?status=all',
|
||||
url: host + '/materials?status=all',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
@ -200,7 +238,7 @@ async function allSamples() {
|
||||
})
|
||||
res = await axios({
|
||||
method: 'get',
|
||||
url: 'http://localhost:3000/samples?status=all',
|
||||
url: host + '/samples?status=all',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
@ -215,7 +253,13 @@ async function allSamples() {
|
||||
for (let index in data) {
|
||||
console.info(`${index}/${data.length}`);
|
||||
let sample = data[index];
|
||||
if (sample['Sample number'] !== '' && sample['Supplier'] !== '' && sample['Granulate/Part'] !== '') { // TODO: wait for decision about samples without suppliers/color/type
|
||||
if (sample['Sample number'] !== '') { // TODO: what about samples without color
|
||||
if (sample['Supplier'] === '') { // empty supplier fields
|
||||
sample['Supplier'] = 'unknown';
|
||||
}
|
||||
if (sample['Granulate/Part'] === '') { // empty supplier fields
|
||||
sample['Granulate/Part'] = 'unknown';
|
||||
}
|
||||
const material = dbMaterials[trim(sample['Material name'])];
|
||||
if (!material) { // could not find material, skipping sample
|
||||
continue;
|
||||
@ -236,13 +280,20 @@ async function allSamples() {
|
||||
samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color;
|
||||
}
|
||||
else if (sample['Color'] && sample['Color'] !== '') {
|
||||
samples[si].color = material.numbers.find(e => e.color.indexOf(sample['Color']) >= 0).color;
|
||||
let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0);
|
||||
if (!number && /black/.test(sample['Color'])) { // special case bk for black
|
||||
number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0);
|
||||
if (!number) { // try German word
|
||||
number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0);
|
||||
}
|
||||
}
|
||||
samples[si].color = number.color;
|
||||
}
|
||||
else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz
|
||||
samples[si].color = sampleColors[sample['Sample number'].split('_')[0]];
|
||||
}
|
||||
else { // TODO: no color information at all
|
||||
samples.pop();
|
||||
else {
|
||||
samples[si].color = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -251,41 +302,57 @@ async function allSamples() {
|
||||
async function saveSamples() {
|
||||
for (let i in samples) {
|
||||
console.info(`${i}/${samples.length}`);
|
||||
let credentials = ['admin', 'Abc123!#'];
|
||||
if (sampleDevices[samples[i].number]) {
|
||||
credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!']
|
||||
}
|
||||
await axios({
|
||||
method: 'post',
|
||||
url: 'http://localhost:3000/sample/new',
|
||||
url: host + '/sample/new',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
username: credentials[0],
|
||||
password: credentials[1]
|
||||
},
|
||||
data: samples[i]
|
||||
}).catch(err => {
|
||||
if (err.response.data.status && err.response.data.status !== 'Sample number already taken') {
|
||||
console.log(samples[i]);
|
||||
console.error(err.response.data);
|
||||
}
|
||||
});
|
||||
}
|
||||
console.info('saved all samples');
|
||||
}
|
||||
|
||||
async function allMaterials() {
|
||||
materials = {};
|
||||
for (let index in data) {
|
||||
let sample = data[index];
|
||||
if (sample['Sample number'] !== '' && sample['Supplier'] !== '') { // TODO: wait for decision about supplierless samples
|
||||
if (sample['Sample number'] && sample['Sample number'] !== '') {
|
||||
if (sample['Supplier'] === '') { // empty supplier fields
|
||||
sample['Supplier'] = 'unknown';
|
||||
}
|
||||
if (sample['Material name'] === '') { // empty name fields
|
||||
sample['Material name'] = sample['Material'];
|
||||
}
|
||||
if (!sample['Material']) { // column Material is named Plastic in VZ metadata
|
||||
sample['Material'] = sample['Plastic'];
|
||||
}
|
||||
sample['Material name'] = trim(sample['Material name']);
|
||||
if (materials.hasOwnProperty(sample['Material name'])) { // material already found at least once
|
||||
if (sample['Material number'] !== '') {
|
||||
if (sample['Material number'] && sample['Material number'] !== '') {
|
||||
if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) { // new material number
|
||||
if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) { // color already in list, only number missing
|
||||
materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']);
|
||||
}
|
||||
else {
|
||||
materials[sample['Material name']].numbers.push({color: sample['Color'], number: stripSpaces(sample['Material number'])});
|
||||
materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])});
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (sample['Color'] !== '') {
|
||||
else if (sample['Color'] && sample['Color'] !== '') {
|
||||
if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) { // new material color
|
||||
materials[sample['Material name']].numbers.push({color: sample['Color'], number: ''});
|
||||
materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -293,8 +360,8 @@ async function allMaterials() {
|
||||
console.info(`${index}/${data.length} ${sample['Material name']}`);
|
||||
materials[sample['Material name']] = {
|
||||
name: sample['Material name'],
|
||||
supplier: sample['Supplier'],
|
||||
group: sample['Material']
|
||||
supplier: trim(sample['Supplier']),
|
||||
group: trim(sample['Material'])
|
||||
};
|
||||
let tmp = /M(\d+)/.exec(sample['Reinforcing material']);
|
||||
materials[sample['Material name']].mineral = tmp ? tmp[1] : 0;
|
||||
@ -312,17 +379,20 @@ async function allMaterials() {
|
||||
async function saveMaterials() {
|
||||
const mKeys = Object.keys(materials)
|
||||
for (let i in mKeys) {
|
||||
console.info(`${i}/${mKeys.length}`);
|
||||
await axios({
|
||||
method: 'post',
|
||||
url: 'http://localhost:3000/material/new',
|
||||
url: host + '/material/new',
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
},
|
||||
data: materials[mKeys[i]]
|
||||
}).catch(err => {
|
||||
console.log(materials[mKeys[i]]);
|
||||
if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
|
||||
console.info(materials[mKeys[i]]);
|
||||
console.error(err.response.data);
|
||||
}
|
||||
});
|
||||
}
|
||||
console.info('saved all materials');
|
||||
@ -362,16 +432,16 @@ async function numbersFetch(sample) {
|
||||
}
|
||||
}
|
||||
if (res.length === 0) { // no results
|
||||
if (sample['Color'] !== '' || sample['Material number'] !== '') {
|
||||
return [{color: sample['Color'], number: sample['Material number']}];
|
||||
if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) {
|
||||
return [{color: trim(sample['Color']), number: sample['Material number']}];
|
||||
}
|
||||
else {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (!res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed
|
||||
res.push({color: sample['Color'], number: sample['Material number']});
|
||||
if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed
|
||||
res.push({color: trim(sample['Color']), number: sample['Material number']});
|
||||
}
|
||||
return res;
|
||||
}
|
||||
@ -403,7 +473,7 @@ async function getNormMaster(fetchAgain = false) {
|
||||
}
|
||||
|
||||
function getNormMasterDoc(url, timing = 1) {
|
||||
console.log(url);
|
||||
console.info(url);
|
||||
return new Promise(async resolve => {
|
||||
const options = new chrome.Options();
|
||||
options.setUserPreferences({
|
||||
@ -453,7 +523,7 @@ function readPdf(file) {
|
||||
rows.push(item.text);
|
||||
}
|
||||
else { // still the same row row
|
||||
rows[rows.length - 1] += (item.x - lastX > 1.1 ? '$' : '') + item.text; // push to row, detect if still same cell
|
||||
rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text; // push to row, detect if still same cell
|
||||
}
|
||||
lastX = (item.w * 0.055) + item.x;
|
||||
|
||||
@ -465,7 +535,7 @@ function readPdf(file) {
|
||||
table = -1;
|
||||
// console.log(rows);
|
||||
rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows
|
||||
resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; }));
|
||||
resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
|
||||
}
|
||||
}
|
||||
lastLastText = lastText;
|
||||
@ -473,12 +543,23 @@ function readPdf(file) {
|
||||
}
|
||||
if (!item && table !== -1) { // document ended
|
||||
rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows
|
||||
resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; }));
|
||||
resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function sampleDeviceMap() {
|
||||
const dpts = fs.readdirSync(dptFiles);
|
||||
const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/;
|
||||
for (let i in dpts) {
|
||||
const regexRes = regex.exec(dpts[i])
|
||||
if (regexRes) { // found matching sample
|
||||
sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function stripSpaces(s) {
|
||||
return s ? s.replace(/ /g,'') : '';
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
---
|
||||
applications:
|
||||
- name: digital-fingerprint-of-plastics-api
|
||||
- name: definma-api
|
||||
path: dist/
|
||||
instances: 1
|
||||
memory: 256M
|
||||
stack: cflinuxfs3
|
||||
@ -10,4 +11,4 @@ applications:
|
||||
NODE_ENV: production
|
||||
OPTIMIZE_MEMORY: true
|
||||
services:
|
||||
- dfopdb
|
||||
- definmadb
|
||||
14
package-lock.json
generated
14
package-lock.json
generated
@ -1174,6 +1174,15 @@
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
|
||||
"dev": true
|
||||
},
|
||||
"cors": {
|
||||
"version": "2.8.5",
|
||||
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
|
||||
"integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
|
||||
"requires": {
|
||||
"object-assign": "^4",
|
||||
"vary": "^1"
|
||||
}
|
||||
},
|
||||
"cross-spawn": {
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
|
||||
@ -2866,6 +2875,11 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
|
||||
},
|
||||
"object-inspect": {
|
||||
"version": "1.7.0",
|
||||
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz",
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
"tsc": "tsc",
|
||||
"tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc",
|
||||
"build": "build.bat",
|
||||
"build-push": "build.bat && cf push",
|
||||
"test": "mocha dist/**/**.spec.js",
|
||||
"start": "node index.js",
|
||||
"dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"",
|
||||
@ -28,6 +29,7 @@
|
||||
"cfenv": "^1.2.2",
|
||||
"compression": "^1.7.4",
|
||||
"content-filter": "^1.1.2",
|
||||
"cors": "^2.8.5",
|
||||
"express": "^4.17.1",
|
||||
"helmet": "^3.22.0",
|
||||
"json-schema": "^0.2.5",
|
||||
@ -35,7 +37,7 @@
|
||||
"lodash": "^4.17.15",
|
||||
"mongo-sanitize": "^1.1.0",
|
||||
"mongoose": "^5.8.7",
|
||||
"swagger-ui-express": "^4.1.2"
|
||||
"swagger-ui-express": "4.1.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bcrypt": "^3.0.0",
|
||||
|
||||
@ -18,6 +18,7 @@ export default class api {
|
||||
jsonRefParser.bundle('api/api.yaml', (err, doc) => { // parse yaml
|
||||
if (err) throw err;
|
||||
apiDoc = doc;
|
||||
apiDoc.servers.splice(process.env.NODE_ENV === 'production', 1);
|
||||
apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e)); // bundle routes
|
||||
apiDoc = this.resolveXDoc(apiDoc);
|
||||
oasParser.validate(apiDoc, (err, api) => { // validate oas schema
|
||||
|
||||
@ -1,12 +1,6 @@
|
||||
import {parseAsync} from 'json2csv';
|
||||
|
||||
export default function csv(input: any[], f: (err, data) => void) {
|
||||
console.log(input[1000]);
|
||||
console.log(flatten(input[1000]));
|
||||
parseAsync([flatten(input[1000])]).then(csv => console.log(csv));
|
||||
console.log(input[1]);
|
||||
console.log(flatten(input[1]));
|
||||
parseAsync([flatten(input[1])]).then(csv => console.log(csv));
|
||||
parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true})
|
||||
.then(csv => f(null, csv))
|
||||
.catch(err => f(err, null));
|
||||
|
||||
@ -17,7 +17,7 @@ export default (mailAddress, subject, content, f) => { // callback, executed em
|
||||
contentType: "text/html"
|
||||
},
|
||||
from: {
|
||||
eMail: "dfop@bosch-iot.com",
|
||||
eMail: "definma@bosch-iot.com",
|
||||
password: "PlasticsOfFingerprintDigital"
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import compression from 'compression';
|
||||
import contentFilter from 'content-filter';
|
||||
import mongoSanitize from 'mongo-sanitize';
|
||||
import helmet from 'helmet';
|
||||
import cors from 'cors';
|
||||
import api from './api';
|
||||
import db from './db';
|
||||
|
||||
@ -42,9 +43,11 @@ app.use((req, res, next) => { // no database connection error
|
||||
next();
|
||||
}
|
||||
else {
|
||||
console.error('No database connection');
|
||||
res.status(500).send({status: 'Internal server error'});
|
||||
}
|
||||
});
|
||||
app.use(cors()); // CORS headers
|
||||
app.use(require('./helpers/authorize')); // handle authentication
|
||||
|
||||
// redirect /api routes for Angular proxy in development
|
||||
|
||||
@ -21,6 +21,7 @@ describe('/sample', () => {
|
||||
|
||||
// TODO: sort, added date filter, has measurements/condition filter
|
||||
// TODO: check if conditions work in sort/fields/filters
|
||||
// TODO: test for numbers as strings in glass_fiber
|
||||
describe('GET /samples', () => {
|
||||
it('returns all samples', done => {
|
||||
TestHelper.request(server, done, {
|
||||
|
||||
@ -21,6 +21,12 @@ import csv from '../helpers/csv';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// TODO: check added filter
|
||||
// TODO: return total number of pages -> use facet
|
||||
// TODO: use query pointer
|
||||
// TODO: convert filter value to number according to table model
|
||||
// TODO: validation for filter parameters
|
||||
// TODO: location/device sort/filter
|
||||
router.get('/samples', async (req, res, next) => {
|
||||
if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return;
|
||||
|
||||
@ -37,6 +43,7 @@ router.get('/samples', async (req, res, next) => {
|
||||
if (!filters['to-page']) { // set to-page default
|
||||
filters['to-page'] = 0;
|
||||
}
|
||||
console.log(filters);
|
||||
|
||||
const sortFilterKeys = filters.filters.map(e => e.field);
|
||||
|
||||
@ -70,7 +77,7 @@ router.get('/samples', async (req, res, next) => {
|
||||
{$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure
|
||||
{$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}},
|
||||
{$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added
|
||||
{$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
|
||||
{$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring
|
||||
{$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}}
|
||||
);
|
||||
addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters
|
||||
@ -106,25 +113,25 @@ router.get('/samples', async (req, res, next) => {
|
||||
materialAdded = true;
|
||||
materialQuery.push( // add material properties
|
||||
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields
|
||||
{$set: {material: {$arrayElemAt: ['$material', 0]}}}
|
||||
{$addFields: {material: {$arrayElemAt: ['$material', 0]}}}
|
||||
);
|
||||
const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0);
|
||||
addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters
|
||||
if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed
|
||||
materialQuery.push(
|
||||
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
|
||||
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
|
||||
{$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
|
||||
);
|
||||
}
|
||||
if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed
|
||||
materialQuery.push(
|
||||
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
|
||||
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
|
||||
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
|
||||
);
|
||||
}
|
||||
if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed
|
||||
materialQuery.push(
|
||||
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
|
||||
{$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
|
||||
);
|
||||
}
|
||||
const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0);
|
||||
@ -157,10 +164,10 @@ router.get('/samples', async (req, res, next) => {
|
||||
as: 'measurements'
|
||||
}});
|
||||
measurementTemplates.forEach(template => {
|
||||
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
|
||||
query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
|
||||
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
|
||||
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
|
||||
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
|
||||
}}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
|
||||
});
|
||||
addFilterQueries(query, filters.filters
|
||||
.filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0)
|
||||
@ -173,39 +180,40 @@ router.get('/samples', async (req, res, next) => {
|
||||
sortFilterKeys.indexOf(e) < 0 // field was not in filter
|
||||
&& e !== filters.sort[0] // field was not in sort
|
||||
);
|
||||
console.log(fieldsToAdd);
|
||||
|
||||
if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already
|
||||
query.push(
|
||||
{$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}},
|
||||
{$set: {material: { $arrayElemAt: ['$material', 0]}}}
|
||||
{$addFields: {material: { $arrayElemAt: ['$material', 0]}}}
|
||||
);
|
||||
}
|
||||
if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed
|
||||
query.push(
|
||||
{$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}},
|
||||
{$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
|
||||
{$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}}
|
||||
);
|
||||
}
|
||||
if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed
|
||||
query.push(
|
||||
{$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }},
|
||||
{$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
|
||||
{$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}}
|
||||
);
|
||||
}
|
||||
if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed
|
||||
query.push(
|
||||
{$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
|
||||
{$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}}
|
||||
);
|
||||
}
|
||||
|
||||
let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
|
||||
let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters
|
||||
if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields
|
||||
const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);});
|
||||
if (measurementTemplates instanceof Error) return;
|
||||
if (measurementTemplates.length < measurementFieldsFields.length) {
|
||||
return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'});
|
||||
}
|
||||
if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance
|
||||
if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance
|
||||
query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}});
|
||||
}
|
||||
else {
|
||||
@ -216,15 +224,15 @@ router.get('/samples', async (req, res, next) => {
|
||||
}});
|
||||
}
|
||||
measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later
|
||||
query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
|
||||
query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values
|
||||
vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}},
|
||||
in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']}
|
||||
}}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
|
||||
}}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}});
|
||||
});
|
||||
if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well
|
||||
query.push(
|
||||
{$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
|
||||
{$set: {spectrum: '$spectrum.values.dpt'}},
|
||||
{$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}},
|
||||
{$addFields: {spectrum: '$spectrum.values'}},
|
||||
{$unwind: '$spectrum'}
|
||||
);
|
||||
}
|
||||
@ -233,19 +241,22 @@ router.get('/samples', async (req, res, next) => {
|
||||
|
||||
const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {});
|
||||
if (filters.fields.indexOf('added') >= 0) { // add added date
|
||||
projection.added = {$toDate: '$_id'};
|
||||
// projection.added = {$toDate: '$_id'};
|
||||
// projection.added = { $convert: { input: '$_id', to: "date" } } // TODO
|
||||
}
|
||||
if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly
|
||||
projection._id = false;
|
||||
}
|
||||
query.push({$project: projection});
|
||||
|
||||
if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files
|
||||
collection.aggregate(query).exec((err, data) => {
|
||||
if (err) return next(err);
|
||||
console.log(data.length);
|
||||
if (filters['to-page'] < 0) {
|
||||
data.reverse();
|
||||
}
|
||||
const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]);
|
||||
const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]);
|
||||
if (filters.csv) { // output as csv
|
||||
csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => {
|
||||
if (err) return next(err);
|
||||
@ -256,7 +267,19 @@ router.get('/samples', async (req, res, next) => {
|
||||
else {
|
||||
res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
else {
|
||||
res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'});
|
||||
res.write('[');
|
||||
let count = 0;
|
||||
const stream = collection.aggregate(query).cursor().exec();
|
||||
stream.on('data', data => { res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; });
|
||||
stream.on('close', () => {
|
||||
res.write(']');
|
||||
res.end();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/samples/:state(new|deleted)', (req, res, next) => {
|
||||
@ -537,7 +560,7 @@ async function materialCheck (sample, res, next, id = sample.material_id) { //
|
||||
res.status(400).json({status: 'Material not available'});
|
||||
return false;
|
||||
}
|
||||
if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified
|
||||
if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified
|
||||
res.status(400).json({status: 'Color not available for material'});
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -11,7 +11,8 @@ export default class SampleValidate {
|
||||
.max(128),
|
||||
|
||||
color: Joi.string()
|
||||
.max(128),
|
||||
.max(128)
|
||||
.allow(''),
|
||||
|
||||
type: Joi.string()
|
||||
.max(128),
|
||||
@ -77,7 +78,7 @@ export default class SampleValidate {
|
||||
'user_id',
|
||||
'material._id',
|
||||
'material.numbers',
|
||||
'measurements.spectrum'
|
||||
'measurements.spectrum.dpt'
|
||||
];
|
||||
|
||||
static input (data, param) { // validate input, set param to 'new' to make all attributes required
|
||||
@ -170,6 +171,33 @@ export default class SampleValidate {
|
||||
try {
|
||||
for (let i in data.filters) {
|
||||
data.filters[i] = JSON.parse(data.filters[i]);
|
||||
data.filters[i].values = data.filters[i].values.map(e => { // validate filter values
|
||||
let validator;
|
||||
let field = data.filters[i].field
|
||||
if (/material\./.test(field)) { // select right validation model
|
||||
validator = MaterialValidate.outputV();
|
||||
field = field.replace('material.', '');
|
||||
}
|
||||
else if (/measurements\./.test(field)) {
|
||||
validator = Joi.object({
|
||||
value: Joi.alternatives()
|
||||
.try(
|
||||
Joi.string().max(128),
|
||||
Joi.number(),
|
||||
Joi.boolean(),
|
||||
Joi.array()
|
||||
)
|
||||
.allow(null)
|
||||
});
|
||||
field = 'value';
|
||||
}
|
||||
else {
|
||||
validator = Joi.object(this.sample);
|
||||
}
|
||||
const {value, error} = validator.validate({[field]: e});
|
||||
if (error) throw error; // reject invalid values
|
||||
return value[field];
|
||||
});
|
||||
}
|
||||
}
|
||||
catch {
|
||||
|
||||
Reference in New Issue
Block a user