From 1ddc2b617aeb858024be386ea1d7effd2c107355 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Thu, 9 Jul 2020 13:48:27 +0200 Subject: [PATCH] spectrum field working again --- api/api.yaml | 4 +- api/sample.yaml | 8 +- data_import/import.js | 209 +++++++++++++++++++++++----------- mainfest.yml => manifest.yml | 5 +- package-lock.json | 14 +++ package.json | 4 +- src/api.ts | 1 + src/helpers/csv.ts | 6 - src/helpers/mail.ts | 2 +- src/index.ts | 3 + src/routes/sample.spec.ts | 1 + src/routes/sample.ts | 95 ++++++++++------ src/routes/validate/sample.ts | 32 +++++- 13 files changed, 269 insertions(+), 115 deletions(-) rename mainfest.yml => manifest.yml (75%) diff --git a/api/api.yaml b/api/api.yaml index d281206..a1966fa 100644 --- a/api/api.yaml +++ b/api/api.yaml @@ -39,10 +39,10 @@ info: servers: + - url: https://definma-api.apps.de1.bosch-iot-cloud.com + description: server on the BIC - url: http://localhost:3000 description: local server - - url: https://digital-fingerprint-of-plastics-api.apps.de1.bosch-iot-cloud.com/ - description: server on the BIC security: diff --git a/api/sample.yaml b/api/sample.yaml index acdd33c..17df4c3 100644 --- a/api/sample.yaml +++ b/api/sample.yaml @@ -2,7 +2,7 @@ get: summary: all samples in overview description: 'Auth: all, levels: read, write, maintain, dev, admin' - x-doc: returns only samples with status 10 + x-doc: 'Limitations: paging and csv output does not work when including the spectrum measurement fields as well as the returned number of total samples' tags: - /sample parameters: @@ -61,6 +61,12 @@ responses: 200: description: samples overview (if the csv parameter is set, this is in CSV instead of JSON format) + headers: + X-Total-Items: + description: Total number of available items when page is specified + schema: + type: integer + example: 243 content: application/json: schema: diff --git a/data_import/import.js b/data_import/import.js index 4f31f8b..627e1b8 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -1,55 +1,82 @@ const csv = require('csv-parser'); const fs = require('fs'); const axios = require('axios'); -const {Builder} = require('selenium-webdriver'); +const {Builder} = require('selenium-webdriver'); // selenium and the chrome driver must be installed and configured separately const chrome = require('selenium-webdriver/chrome'); const pdfReader = require('pdfreader'); const iconv = require('iconv-lite'); -const metadata = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\VZ.csv'; // metadata file -const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\nmDocs'; // NormMaster Documents -const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200622\\DPT'; // Spectrum files +const metaDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\metadata.csv'; // metadata files +const kfDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\kf.csv'; +const vzDoc = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\vz.csv'; +const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\nmDocs'; // NormMaster Documents +const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\Rng_200707\\DPT'; // Spectrum files +// const host = 'http://localhost:3000'; +const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; let data = []; // metadata contents let materials = {}; let samples = []; let normMaster = {}; +let sampleDevices = {}; -// TODO: integrate measurement device information from DPT names using different users -// TODO: supplier: other for supplierless samples // TODO: BASF twice, BASF as color -// TODO: trim color names // TODO: duplicate kf values +// TODO: conditions +// TODO: comment and reference handling + + +// TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC main(); async function main() { if (0) { // materials await getNormMaster(); - await importCsv(); + await importCsv(metaDoc); + await allMaterials(); + await saveMaterials(); + await importCsv(kfDoc); + await allMaterials(); + await saveMaterials(); + await importCsv(vzDoc); await allMaterials(); - fs.writeFileSync('./data_import/materials.json', JSON.stringify(materials)); await saveMaterials(); } - else if (0) { // samples - await importCsv(); - await allSamples(); - await saveSamples(); + if (0) { // samples + sampleDeviceMap(); + if (1) { + console.log('-------- META ----------'); + await importCsv(metaDoc); + await allSamples(); + await saveSamples(); + } + if (1) { + console.log('-------- KF ----------'); + await importCsv(kfDoc); + await allSamples(); + await saveSamples(); + await allKfVz(); + } + if (1) { + console.log('-------- VZ ----------'); + await importCsv(vzDoc); + await allSamples(); + await saveSamples(); + await allKfVz(); + } } - else if (1) { // DPT + if (1) { // DPT await allDpts(); } - else if (0) { // KF/VZ - await importCsv(); - await allKfVz(); - } - else if (0) { // pdf test - console.log(await readPdf('N28_BN22-O010_2018-03-08.pdf')); + if (0) { // pdf test + console.log(await readPdf('N28_BN05-OX013_2016-03-11.pdf')); } } -async function importCsv() { +async function importCsv(doc) { + data = []; await new Promise(resolve => { - fs.createReadStream(metadata) + fs.createReadStream(doc) .pipe(iconv.decodeStream('win1252')) .pipe(csv()) .on('data', (row) => { @@ -57,6 +84,9 @@ async function importCsv() { }) .on('end', () => { console.info('CSV file successfully processed'); + if (data[0]['Farbe']) { // fix German column names + data.map(e => {e['Color'] = e['Farbe']; return e; }); + } resolve(); }); }); @@ -65,7 +95,7 @@ async function importCsv() { async function allDpts() { let res = await axios({ method: 'get', - url: 'http://localhost:3000/template/measurements', + url: host + '/template/measurements', auth: { username: 'admin', password: 'Abc123!#' @@ -74,7 +104,7 @@ async function allDpts() { const measurement_template = res.data.find(e => e.name === 'spectrum')._id; res = await axios({ method: 'get', - url: 'http://localhost:3000/samples?status=all', + url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' @@ -84,10 +114,10 @@ async function allDpts() { res.data.forEach(sample => { sampleIds[sample.number] = sample._id; }); - const regex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/; + const dptRegex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/; const dpts = fs.readdirSync(dptFiles); for (let i in dpts) { - const regexRes = regex.exec(dpts[i]) + const regexRes = dptRegex.exec(dpts[i]) if (regexRes && sampleIds[regexRes[1]]) { // found matching sample console.log(dpts[i]); const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8'); @@ -99,7 +129,7 @@ async function allDpts() { data.values.dpt = f.split('\r\n').map(e => e.split(',')); await axios({ method: 'post', - url: 'http://localhost:3000/measurement/new', + url: host + '/measurement/new', auth: { username: 'admin', password: 'Abc123!#' @@ -110,13 +140,16 @@ async function allDpts() { console.error(err.response.data); }); } + else { + console.log(`Could not find sample for ${dpts[i]} !!!!!!`); + } } } async function allKfVz() { let res = await axios({ method: 'get', - url: 'http://localhost:3000/template/measurements', + url: host + '/template/measurements', auth: { username: 'admin', password: 'Abc123!#' @@ -126,7 +159,7 @@ async function allKfVz() { const vz_template = res.data.find(e => e.name === 'vz')._id; res = await axios({ method: 'get', - url: 'http://localhost:3000/samples?status=all', + url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' @@ -140,13 +173,17 @@ async function allKfVz() { console.info(`${index}/${data.length}`); let sample = data[index]; if (sample['Sample number'] !== '') { + let credentials = ['admin', 'Abc123!#']; + if (sampleDevices[sample['Sample number']]) { + credentials = [sampleDevices[sample['Sample number']], '2020DeFinMachen!'] + } if (sample['KF in Gew%']) { await axios({ method: 'post', - url: 'http://localhost:3000/measurement/new', + url: host + '/measurement/new', auth: { - username: 'admin', - password: 'Abc123!#' + username: credentials[0], + password: credentials[1] }, data: { sample_id: sampleIds[sample['Sample number']], @@ -164,10 +201,10 @@ async function allKfVz() { if (sample['VZ (ml/g)']) { await axios({ method: 'post', - url: 'http://localhost:3000/measurement/new', + url: host + '/measurement/new', auth: { - username: 'admin', - password: 'Abc123!#' + username: credentials[0], + password: credentials[1] }, data: { sample_id: sampleIds[sample['Sample number']], @@ -186,9 +223,10 @@ async function allKfVz() { } async function allSamples() { + samples = []; let res = await axios({ method: 'get', - url: 'http://localhost:3000/materials?status=all', + url: host + '/materials?status=all', auth: { username: 'admin', password: 'Abc123!#' @@ -200,7 +238,7 @@ async function allSamples() { }) res = await axios({ method: 'get', - url: 'http://localhost:3000/samples?status=all', + url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' @@ -215,7 +253,13 @@ async function allSamples() { for (let index in data) { console.info(`${index}/${data.length}`); let sample = data[index]; - if (sample['Sample number'] !== '' && sample['Supplier'] !== '' && sample['Granulate/Part'] !== '') { // TODO: wait for decision about samples without suppliers/color/type + if (sample['Sample number'] !== '') { // TODO: what about samples without color + if (sample['Supplier'] === '') { // empty supplier fields + sample['Supplier'] = 'unknown'; + } + if (sample['Granulate/Part'] === '') { // empty supplier fields + sample['Granulate/Part'] = 'unknown'; + } const material = dbMaterials[trim(sample['Material name'])]; if (!material) { // could not find material, skipping sample continue; @@ -236,13 +280,20 @@ async function allSamples() { samples[si].color = material.numbers.find(e => e.number === sample['Material number']).color; } else if (sample['Color'] && sample['Color'] !== '') { - samples[si].color = material.numbers.find(e => e.color.indexOf(sample['Color']) >= 0).color; + let number = material.numbers.find(e => e.color.indexOf(trim(sample['Color'])) >= 0); + if (!number && /black/.test(sample['Color'])) { // special case bk for black + number = material.numbers.find(e => e.color.toLowerCase().indexOf('bk') >= 0); + if (!number) { // try German word + number = material.numbers.find(e => e.color.toLowerCase().indexOf('schwarz') >= 0); + } + } + samples[si].color = number.color; } else if (sampleColors[sample['Sample number'].split('_')[0]]) { // derive color from main sample for kf/vz samples[si].color = sampleColors[sample['Sample number'].split('_')[0]]; } - else { // TODO: no color information at all - samples.pop(); + else { + samples[si].color = ''; } } } @@ -251,41 +302,57 @@ async function allSamples() { async function saveSamples() { for (let i in samples) { console.info(`${i}/${samples.length}`); + let credentials = ['admin', 'Abc123!#']; + if (sampleDevices[samples[i].number]) { + credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!'] + } await axios({ method: 'post', - url: 'http://localhost:3000/sample/new', + url: host + '/sample/new', auth: { - username: 'admin', - password: 'Abc123!#' + username: credentials[0], + password: credentials[1] }, data: samples[i] }).catch(err => { - console.log(samples[i]); - console.error(err.response.data); + if (err.response.data.status && err.response.data.status !== 'Sample number already taken') { + console.log(samples[i]); + console.error(err.response.data); + } }); } console.info('saved all samples'); } async function allMaterials() { + materials = {}; for (let index in data) { let sample = data[index]; - if (sample['Sample number'] !== '' && sample['Supplier'] !== '') { // TODO: wait for decision about supplierless samples + if (sample['Sample number'] && sample['Sample number'] !== '') { + if (sample['Supplier'] === '') { // empty supplier fields + sample['Supplier'] = 'unknown'; + } + if (sample['Material name'] === '') { // empty name fields + sample['Material name'] = sample['Material']; + } + if (!sample['Material']) { // column Material is named Plastic in VZ metadata + sample['Material'] = sample['Plastic']; + } sample['Material name'] = trim(sample['Material name']); if (materials.hasOwnProperty(sample['Material name'])) { // material already found at least once - if (sample['Material number'] !== '') { + if (sample['Material number'] && sample['Material number'] !== '') { if (materials[sample['Material name']].numbers.length === 0 || !materials[sample['Material name']].numbers.find(e => e.number === stripSpaces(sample['Material number']))) { // new material number if (materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '')) { // color already in list, only number missing materials[sample['Material name']].numbers.find(e => e.color === sample['Color'] && e.number === '').number = stripSpaces(sample['Material number']); } else { - materials[sample['Material name']].numbers.push({color: sample['Color'], number: stripSpaces(sample['Material number'])}); + materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: stripSpaces(sample['Material number'])}); } } } - else if (sample['Color'] !== '') { + else if (sample['Color'] && sample['Color'] !== '') { if (!materials[sample['Material name']].numbers.find(e => e.color === stripSpaces(sample['Color']))) { // new material color - materials[sample['Material name']].numbers.push({color: sample['Color'], number: ''}); + materials[sample['Material name']].numbers.push({color: trim(sample['Color']), number: ''}); } } } @@ -293,8 +360,8 @@ async function allMaterials() { console.info(`${index}/${data.length} ${sample['Material name']}`); materials[sample['Material name']] = { name: sample['Material name'], - supplier: sample['Supplier'], - group: sample['Material'] + supplier: trim(sample['Supplier']), + group: trim(sample['Material']) }; let tmp = /M(\d+)/.exec(sample['Reinforcing material']); materials[sample['Material name']].mineral = tmp ? tmp[1] : 0; @@ -312,17 +379,20 @@ async function allMaterials() { async function saveMaterials() { const mKeys = Object.keys(materials) for (let i in mKeys) { + console.info(`${i}/${mKeys.length}`); await axios({ method: 'post', - url: 'http://localhost:3000/material/new', + url: host + '/material/new', auth: { username: 'admin', password: 'Abc123!#' }, data: materials[mKeys[i]] }).catch(err => { - console.log(materials[mKeys[i]]); - console.error(err.response.data); + if (err.response.data.status && err.response.data.status !== 'Material name already taken') { + console.info(materials[mKeys[i]]); + console.error(err.response.data); + } }); } console.info('saved all materials'); @@ -362,16 +432,16 @@ async function numbersFetch(sample) { } } if (res.length === 0) { // no results - if (sample['Color'] !== '' || sample['Material number'] !== '') { - return [{color: sample['Color'], number: sample['Material number']}]; + if ((sample['Color'] && sample['Color'] !== '') || (sample['Material number'] &&sample['Material number'] !== '')) { + return [{color: trim(sample['Color']), number: sample['Material number']}]; } else { return []; } } else { - if (!res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed - res.push({color: sample['Color'], number: sample['Material number']}); + if (sample['Material number'] && !res.find(e => e.number === sample['Material number'])) { // sometimes norm master does not include sample number even if listed + res.push({color: trim(sample['Color']), number: sample['Material number']}); } return res; } @@ -403,7 +473,7 @@ async function getNormMaster(fetchAgain = false) { } function getNormMasterDoc(url, timing = 1) { - console.log(url); + console.info(url); return new Promise(async resolve => { const options = new chrome.Options(); options.setUserPreferences({ @@ -453,7 +523,7 @@ function readPdf(file) { rows.push(item.text); } else { // still the same row row - rows[rows.length - 1] += (item.x - lastX > 1.1 ? '$' : '') + item.text; // push to row, detect if still same cell + rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text; // push to row, detect if still same cell } lastX = (item.w * 0.055) + item.x; @@ -465,7 +535,7 @@ function readPdf(file) { table = -1; // console.log(rows); rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows - resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; })); + resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; })); } } lastLastText = lastText; @@ -473,12 +543,23 @@ function readPdf(file) { } if (!item && table !== -1) { // document ended rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows - resolve(rows.map(e => {return {color: e.split('$')[3], number: stripSpaces(e.split('$')[0])}; })); + resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; })); } }); }); } +function sampleDeviceMap() { + const dpts = fs.readdirSync(dptFiles); + const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/; + for (let i in dpts) { + const regexRes = regex.exec(dpts[i]) + if (regexRes) { // found matching sample + sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase(); + } + } +} + function stripSpaces(s) { return s ? s.replace(/ /g,'') : ''; } diff --git a/mainfest.yml b/manifest.yml similarity index 75% rename from mainfest.yml rename to manifest.yml index 16e5924..0e8c57d 100644 --- a/mainfest.yml +++ b/manifest.yml @@ -1,6 +1,7 @@ --- applications: - - name: digital-fingerprint-of-plastics-api + - name: definma-api + path: dist/ instances: 1 memory: 256M stack: cflinuxfs3 @@ -10,4 +11,4 @@ applications: NODE_ENV: production OPTIMIZE_MEMORY: true services: - - dfopdb + - definmadb diff --git a/package-lock.json b/package-lock.json index 5478eef..34fb53e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1174,6 +1174,15 @@ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2866,6 +2875,11 @@ } } }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, "object-inspect": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", diff --git a/package.json b/package.json index 4b04218..f9494d3 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "tsc": "tsc", "tsc-full": "del /q dist\\* & (for /d %x in (dist\\*) do @rd /s /q \"%x\") & tsc", "build": "build.bat", + "build-push": "build.bat && cf push", "test": "mocha dist/**/**.spec.js", "start": "node index.js", "dev": "nodemon -e ts,yaml --exec \"tsc && node dist/index.js || exit 1\"", @@ -28,6 +29,7 @@ "cfenv": "^1.2.2", "compression": "^1.7.4", "content-filter": "^1.1.2", + "cors": "^2.8.5", "express": "^4.17.1", "helmet": "^3.22.0", "json-schema": "^0.2.5", @@ -35,7 +37,7 @@ "lodash": "^4.17.15", "mongo-sanitize": "^1.1.0", "mongoose": "^5.8.7", - "swagger-ui-express": "^4.1.2" + "swagger-ui-express": "4.1.2" }, "devDependencies": { "@types/bcrypt": "^3.0.0", diff --git a/src/api.ts b/src/api.ts index 0867bc1..aab7b80 100644 --- a/src/api.ts +++ b/src/api.ts @@ -18,6 +18,7 @@ export default class api { jsonRefParser.bundle('api/api.yaml', (err, doc) => { // parse yaml if (err) throw err; apiDoc = doc; + apiDoc.servers.splice(process.env.NODE_ENV === 'production', 1); apiDoc.paths = apiDoc.paths.allOf.reduce((s, e) => Object.assign(s, e)); // bundle routes apiDoc = this.resolveXDoc(apiDoc); oasParser.validate(apiDoc, (err, api) => { // validate oas schema diff --git a/src/helpers/csv.ts b/src/helpers/csv.ts index 18e633c..38c487a 100644 --- a/src/helpers/csv.ts +++ b/src/helpers/csv.ts @@ -1,12 +1,6 @@ import {parseAsync} from 'json2csv'; export default function csv(input: any[], f: (err, data) => void) { - console.log(input[1000]); - console.log(flatten(input[1000])); - parseAsync([flatten(input[1000])]).then(csv => console.log(csv)); - console.log(input[1]); - console.log(flatten(input[1])); - parseAsync([flatten(input[1])]).then(csv => console.log(csv)); parseAsync(input.map(e => flatten(e)), {includeEmptyRows: true}) .then(csv => f(null, csv)) .catch(err => f(err, null)); diff --git a/src/helpers/mail.ts b/src/helpers/mail.ts index a3d79c1..8ec71c8 100644 --- a/src/helpers/mail.ts +++ b/src/helpers/mail.ts @@ -17,7 +17,7 @@ export default (mailAddress, subject, content, f) => { // callback, executed em contentType: "text/html" }, from: { - eMail: "dfop@bosch-iot.com", + eMail: "definma@bosch-iot.com", password: "PlasticsOfFingerprintDigital" } } diff --git a/src/index.ts b/src/index.ts index 8116de7..4051f23 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import compression from 'compression'; import contentFilter from 'content-filter'; import mongoSanitize from 'mongo-sanitize'; import helmet from 'helmet'; +import cors from 'cors'; import api from './api'; import db from './db'; @@ -42,9 +43,11 @@ app.use((req, res, next) => { // no database connection error next(); } else { + console.error('No database connection'); res.status(500).send({status: 'Internal server error'}); } }); +app.use(cors()); // CORS headers app.use(require('./helpers/authorize')); // handle authentication // redirect /api routes for Angular proxy in development diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index d15200b..7dc5f24 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -21,6 +21,7 @@ describe('/sample', () => { // TODO: sort, added date filter, has measurements/condition filter // TODO: check if conditions work in sort/fields/filters + // TODO: test for numbers as strings in glass_fiber describe('GET /samples', () => { it('returns all samples', done => { TestHelper.request(server, done, { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 11080f7..bf741c2 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -21,6 +21,12 @@ import csv from '../helpers/csv'; const router = express.Router(); +// TODO: check added filter +// TODO: return total number of pages -> use facet +// TODO: use query pointer +// TODO: convert filter value to number according to table model +// TODO: validation for filter parameters +// TODO: location/device sort/filter router.get('/samples', async (req, res, next) => { if (!req.auth(res, ['read', 'write', 'maintain', 'dev', 'admin'], 'all')) return; @@ -37,6 +43,7 @@ router.get('/samples', async (req, res, next) => { if (!filters['to-page']) { // set to-page default filters['to-page'] = 0; } + console.log(filters); const sortFilterKeys = filters.filters.map(e => e.field); @@ -70,7 +77,7 @@ router.get('/samples', async (req, res, next) => { {$replaceRoot: {newRoot: {measurement: '$$ROOT'}}}, // fetch samples and restructure them to fit sample structure {$lookup: {from: 'samples', localField: 'measurement.sample_id', foreignField: '_id', as: 'sample'}}, {$match: statusQuery(filters, 'sample.status')}, // filter out wrong status once samples were added - {$set: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring + {$addFields: {['sample.' + measurementName]: '$measurement.values'}}, // more restructuring {$replaceRoot: {newRoot: {$mergeObjects: [{$arrayElemAt: ['$sample', 0]}, {}]}}} ); addFilterQueries(query, filters.filters.filter(e => sampleKeys.indexOf(e.field) >= 0)); // sample filters @@ -106,25 +113,25 @@ router.get('/samples', async (req, res, next) => { materialAdded = true; materialQuery.push( // add material properties {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, // TODO: project out unnecessary fields - {$set: {material: {$arrayElemAt: ['$material', 0]}}} + {$addFields: {material: {$arrayElemAt: ['$material', 0]}}} ); const baseMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) < 0); addFilterQueries(materialQuery, filters.filters.filter(e => baseMFilters.indexOf(e.field) >= 0)); // base material filters if (sortFilterKeys.find(e => e === 'material.supplier')) { // add supplier if needed materialQuery.push( {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); } if (sortFilterKeys.find(e => e === 'material.group')) { // add group if needed materialQuery.push( {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } if (sortFilterKeys.find(e => e === 'material.number')) { // add material number if needed materialQuery.push( - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} ); } const specialMFilters = sortFilterKeys.filter(e => /material\./.test(e)).filter(e => ['material.supplier', 'material.group', 'material.number'].indexOf(e) >= 0); @@ -157,10 +164,10 @@ router.get('/samples', async (req, res, next) => { as: 'measurements' }}); measurementTemplates.forEach(template => { - query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} - }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); + }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); addFilterQueries(query, filters.filters .filter(e => sortFilterKeys.filter(e => /measurements\./.test(e)).indexOf(e.field) >= 0) @@ -173,39 +180,40 @@ router.get('/samples', async (req, res, next) => { sortFilterKeys.indexOf(e) < 0 // field was not in filter && e !== filters.sort[0] // field was not in sort ); + console.log(fieldsToAdd); if (fieldsToAdd.find(e => /material\./.test(e)) && !materialAdded) { // add material, was not added already query.push( {$lookup: {from: 'materials', localField: 'material_id', foreignField: '_id', as: 'material'}}, - {$set: {material: { $arrayElemAt: ['$material', 0]}}} + {$addFields: {material: { $arrayElemAt: ['$material', 0]}}} ); } if (fieldsToAdd.indexOf('material.supplier') >= 0) { // add supplier if needed query.push( {$lookup: { from: 'material_suppliers', localField: 'material.supplier_id', foreignField: '_id', as: 'material.supplier'}}, - {$set: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} + {$addFields: {'material.supplier': {$arrayElemAt: ['$material.supplier.name', 0]}}} ); } if (fieldsToAdd.indexOf('material.group') >= 0) { // add group if needed query.push( {$lookup: { from: 'material_groups', localField: 'material.group_id', foreignField: '_id', as: 'material.group' }}, - {$set: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} + {$addFields: {'material.group': { $arrayElemAt: ['$material.group.name', 0]}}} ); } if (fieldsToAdd.indexOf('material.number') >= 0) { // add material number if needed query.push( - {$set: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} + {$addFields: {'material.number': { $arrayElemAt: ['$material.numbers.number', {$indexOfArray: ['$material.numbers.color', '$color']}]}}} ); } - let measurementFieldsFields = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters + let measurementFieldsFields: string[] = _.uniq(fieldsToAdd.filter(e => /measurements\./.test(e)).map(e => e.split('.')[1])); // filter measurement names and remove duplicates from parameters if (fieldsToAdd.find(e => /measurements\./.test(e))) { // add measurement fields const measurementTemplates = await MeasurementTemplateModel.find({name: {$in: measurementFieldsFields}}).lean().exec().catch(err => {next(err);}); if (measurementTemplates instanceof Error) return; if (measurementTemplates.length < measurementFieldsFields.length) { return res.status(400).json({status: 'Invalid body format', details: 'Measurement key not found'}); } - if (fieldsToAdd.find(e => e === 'measurements.spectrum')) { // use different lookup methods with and without spectrum for the best performance + if (fieldsToAdd.find(e => /spectrum\./.test(e))) { // use different lookup methods with and without spectrum for the best performance query.push({$lookup: {from: 'measurements', localField: '_id', foreignField: 'sample_id', as: 'measurements'}}); } else { @@ -216,15 +224,15 @@ router.get('/samples', async (req, res, next) => { }}); } measurementTemplates.filter(e => e.name !== 'spectrum').forEach(template => { // TODO: hard coded dpt for special treatment, change later - query.push({$set: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values + query.push({$addFields: {[template.name]: {$let: { // add measurements as property [template.name], if one result, array is reduced to direct values vars: {arr: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', mongoose.Types.ObjectId(template._id)]}}}}, in:{$cond: [{$lte: [{$size: '$$arr'}, 1]}, {$arrayElemAt: ['$$arr', 0]}, '$$arr']} - }}}}, {$set: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); + }}}}, {$addFields: {[template.name]: {$cond: ['$' + template.name + '.values', '$' + template.name + '.values', template.parameters.reduce((s, e) => {s[e.name] = null; return s;}, {})]}}}); }); if (measurementFieldsFields.find(e => e === 'spectrum')) { // TODO: remove hardcoded as well query.push( - {$set: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, - {$set: {spectrum: '$spectrum.values.dpt'}}, + {$addFields: {spectrum: {$filter: {input: '$measurements', cond: {$eq: ['$$this.measurement_template', measurementTemplates.filter(e => e.name === 'spectrum')[0]._id]}}}}}, + {$addFields: {spectrum: '$spectrum.values'}}, {$unwind: '$spectrum'} ); } @@ -233,30 +241,45 @@ router.get('/samples', async (req, res, next) => { const projection = filters.fields.map(e => e.replace('measurements.', '')).reduce((s, e) => {s[e] = true; return s; }, {}); if (filters.fields.indexOf('added') >= 0) { // add added date - projection.added = {$toDate: '$_id'}; + // projection.added = {$toDate: '$_id'}; + // projection.added = { $convert: { input: '$_id', to: "date" } } // TODO } if (!(filters.fields.indexOf('_id') >= 0)) { // disable _id explicitly projection._id = false; } query.push({$project: projection}); - collection.aggregate(query).exec((err, data) => { - if (err) return next(err); - if (filters['to-page'] < 0) { - data.reverse(); - } - const measurementFields = _.uniq([...measurementFilterFields, ...measurementFieldsFields]); - if (filters.csv) { // output as csv - csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { - if (err) return next(err); - res.set('Content-Type', 'text/csv'); - res.send(data); - }); - } - else { - res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors - } - }) + if (!fieldsToAdd.find(e => /spectrum\./.test(e))) { // use streaming when including spectrum files + collection.aggregate(query).exec((err, data) => { + if (err) return next(err); + console.log(data.length); + if (filters['to-page'] < 0) { + data.reverse(); + } + const measurementFields = _.uniq([filters.sort[0].split('.')[1], ...measurementFilterFields, ...measurementFieldsFields]); + if (filters.csv) { // output as csv + csv(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields))), (err, data) => { + if (err) return next(err); + res.set('Content-Type', 'text/csv'); + res.send(data); + }); + } + else { + res.json(_.compact(data.map(e => SampleValidate.output(e, 'refs', measurementFields)))); // validate all and filter null values from validation errors + } + }); + } + else { + res.writeHead(200, {'Content-Type': 'application/json; charset=utf-8'}); + res.write('['); + let count = 0; + const stream = collection.aggregate(query).cursor().exec(); + stream.on('data', data => { res.write((count === 0 ? '' : ',\n') + JSON.stringify(data)); count ++; }); + stream.on('close', () => { + res.write(']'); + res.end(); + }); + } }); router.get('/samples/:state(new|deleted)', (req, res, next) => { @@ -537,7 +560,7 @@ async function materialCheck (sample, res, next, id = sample.material_id) { // res.status(400).json({status: 'Material not available'}); return false; } - if (sample.hasOwnProperty('color') && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified + if (sample.hasOwnProperty('color') && sample.color !== '' && !materialData.numbers.find(e => e.color === sample.color)) { // color for material not specified res.status(400).json({status: 'Color not available for material'}); return false; } diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index b0cae01..f84a5be 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -11,7 +11,8 @@ export default class SampleValidate { .max(128), color: Joi.string() - .max(128), + .max(128) + .allow(''), type: Joi.string() .max(128), @@ -77,7 +78,7 @@ export default class SampleValidate { 'user_id', 'material._id', 'material.numbers', - 'measurements.spectrum' + 'measurements.spectrum.dpt' ]; static input (data, param) { // validate input, set param to 'new' to make all attributes required @@ -170,6 +171,33 @@ export default class SampleValidate { try { for (let i in data.filters) { data.filters[i] = JSON.parse(data.filters[i]); + data.filters[i].values = data.filters[i].values.map(e => { // validate filter values + let validator; + let field = data.filters[i].field + if (/material\./.test(field)) { // select right validation model + validator = MaterialValidate.outputV(); + field = field.replace('material.', ''); + } + else if (/measurements\./.test(field)) { + validator = Joi.object({ + value: Joi.alternatives() + .try( + Joi.string().max(128), + Joi.number(), + Joi.boolean(), + Joi.array() + ) + .allow(null) + }); + field = 'value'; + } + else { + validator = Joi.object(this.sample); + } + const {value, error} = validator.validate({[field]: e}); + if (error) throw error; // reject invalid values + return value[field]; + }); } } catch {