From a8465ff1285f00cb1f7ce5067edfda6fb3ca0340 Mon Sep 17 00:00:00 2001 From: VLE2FE Date: Wed, 22 Jul 2020 10:53:45 +0200 Subject: [PATCH] added stringin filter mode --- api/schemas.yaml | 7 +- data_import/import.js | 350 ++++++++++++++++++++-------------- src/routes/sample.spec.ts | 2 +- src/routes/sample.ts | 3 + src/routes/validate/sample.ts | 2 +- 5 files changed, 213 insertions(+), 151 deletions(-) diff --git a/api/schemas.yaml b/api/schemas.yaml index 3a332e0..1c844bb 100644 --- a/api/schemas.yaml +++ b/api/schemas.yaml @@ -92,7 +92,12 @@ SampleDetail: sample_references: type: array items: - $ref: 'api.yaml#/components/schemas/Id' + properties: + sample_id: + $ref: 'api.yaml#/components/schemas/Id' + relation: + type: string + example: part to this sample measurements: type: array items: diff --git a/data_import/import.js b/data_import/import.js index 8c4f924..8fac949 100644 --- a/data_import/import.js +++ b/data_import/import.js @@ -8,10 +8,9 @@ const iconv = require('iconv-lite'); const _ = require('lodash'); const stages = { - materials: false, + materials: true, samples: true, - measurements: false, - dpt: false + dpt: true } const docs = [ @@ -46,17 +45,12 @@ let samples = []; let normMaster = {}; let sampleDevices = {}; const sampleReferences = []; // references to other samples in format {sample, referencedSample, relation} -let comments = []; +let commentsLog = []; +let customFieldsLog = []; +const vzValues = {}; // vz values from comments +const dptLog = []; // TODO: conditions -// TODO: comment and reference handling - -// TODO: measurement device to spectrum - - -// TODO: check last color errors (filter out already taken) use location and device for user, upload to BIC - -// TODO: samples, conditions main(); @@ -77,32 +71,19 @@ async function main() { await importCsv(docs[i]); await allSamples(); await saveSamples(); - console.log(samples); - } // TODO: get sample by number, implement smple references - fs.writeFileSync('./data_import/comments.txt', comments.join('\r\n')); - // if (1) { // TODO: KfVz - // console.log('-------- META ----------'); - // await importCsv(metaDoc); - // await allSamples(); - // await saveSamples(); - // } - // if (1) { - // console.log('-------- KF ----------'); - // await importCsv(kfDoc); - // await allSamples(); - // await saveSamples(); - // await allKfVz(); - // } - // if (1) { - // console.log('-------- VZ ----------'); - // await importCsv(vzDoc); - // await allSamples(); - // await saveSamples(); - // await allKfVz(); - // } + await allKfVz(); + } + // write logs + fs.writeFileSync('./data_import/comments.txt', commentsLog.join('\r\n')); + fs.writeFileSync('./data_import/customFields.txt', customFieldsLog.join('\r\n')); + fs.writeFileSync('./data_import/sampleReferences.txt', sampleReferences.map(e => JSON.stringify(e)).join('\r\n')); + fs.writeFileSync('./data_import/sampleReferences.json', JSON.stringify(sampleReferences)); + + await sampleReferencesSave(); } if (stages.dpt) { // DPT await allDpts(); + fs.writeFileSync('./data_import/sdptLog.txt', dptLog.join('\r\n')); } if (0) { // pdf test console.log(await readPdf('N28_BN05-OX023_2019-07-16.pdf')); @@ -226,18 +207,20 @@ async function allDpts() { res.data.forEach(sample => { sampleIds[sample.number] = sample._id; }); - const dptRegex = /.*?_(.*?)_(\d+|\d+_\d+).DPT/; + const dptRegex = /(.*?)_(.*?)_(\d+|\d+_\d+).DPT/; const dpts = fs.readdirSync(dptFiles); for (let i in dpts) { const regexRes = dptRegex.exec(dpts[i]) - if (regexRes && sampleIds[regexRes[1]]) { // found matching sample - console.log(dpts[i]); + if (regexRes && sampleIds[regexRes[2]]) { // found matching sample + console.log(`${dpts[i]} -> ${regexRes[2]}`); + dptLog.push(`${dpts[i]}, ${regexRes[2]}`); const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8'); const data = { - sample_id: sampleIds[regexRes[1]], + sample_id: sampleIds[regexRes[2]], values: {}, measurement_template }; + data.values.device = regexRes[1]; data.values.dpt = f.split('\r\n').map(e => e.split(',')); let rescale = false; for (let i in data.values.dpt) { @@ -259,16 +242,23 @@ async function allDpts() { data }).catch(err => { console.log(dpts[i]); - console.error(err.response.data); + if (err.response) { + console.error(err.response.data); + errors.push(`Could not upload ${dpts[i]} for sample ${regexRes[2]}: ${err.response.data}`); + } + else { + console.error(err); + errors.push(`Could not upload ${dpts[i]} for sample ${regexRes[2]}: ${JSON.stringify(err)}`); + } }); } else { - console.log(`Could not find sample for ${dpts[i]} !!!!!!`); + console.log(`Could not find sample for ${dpts[i]}`); + errors.push(`Could not find sample for ${dpts[i]}`); } } } -// TODO: VZ from comments async function allKfVz() { let res = await axios({ method: 'get', @@ -293,54 +283,57 @@ async function allKfVz() { sampleIds[sample.number] = sample._id; }); for (let index in data) { - console.info(`${index}/${data.length}`); + console.info(`KF/VZ ${index}/${data.length}`); let sample = data[index]; - if (sample['Sample number'] !== '') { - let credentials = ['admin', 'Abc123!#']; - if (sampleDevices[sample['Sample number']]) { - credentials = [sampleDevices[sample['Sample number']], '2020DeFinMachen!'] - } - if (sample['KF in Gew%']) { - await axios({ - method: 'post', - url: host + '/measurement/new', - auth: { - username: credentials[0], - password: credentials[1] - }, - data: { - sample_id: sampleIds[sample['Sample number']], - measurement_template: kf_template, - values: { - 'weight %': sample['KF in Gew%'], - 'standard deviation': sample['Stabwn'] - } + let credentials = ['admin', 'Abc123!#']; + if (sampleDevices[sample['samplenumber']]) { + credentials = [sampleDevices[sample['samplenumber']], '2020DeFinMachen!'] + } + if (!sample['vz(ml/g)'] && vzValues[sample['samplenumber']]) { // fill in VZ values from comments + sample['vz(ml/g)'] = vzValues[sample['samplenumber']]; + } + if (sample['kfingew%']) { + await axios({ + method: 'post', + url: host + '/measurement/new', + auth: { + username: credentials[0], + password: credentials[1] + }, + data: { + sample_id: sampleIds[sample['samplenumber']], + measurement_template: kf_template, + values: { + 'weight %': sample['kfingew%'], + 'standard deviation': sample['stabwn'] } - }).catch(err => { - console.log(sample['Sample number']); - console.error(err.response.data); - }); - } - if (sample['VZ (ml/g)']) { - await axios({ - method: 'post', - url: host + '/measurement/new', - auth: { - username: credentials[0], - password: credentials[1] - }, - data: { - sample_id: sampleIds[sample['Sample number']], - measurement_template: vz_template, - values: { - vz: sample['VZ (ml/g)'] - } + } + }).catch(err => { + console.log(sample['samplenumber']); + console.error(err.response.data); + errors.push(`KF/VZ upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`); + }); + } + if (sample['VZ (ml/g)']) { + await axios({ + method: 'post', + url: host + '/measurement/new', + auth: { + username: credentials[0], + password: credentials[1] + }, + data: { + sample_id: sampleIds[sample['samplenumber']], + measurement_template: vz_template, + values: { + vz: sample['vz(ml/g)'] } - }).catch(err => { - console.log(sample['Sample number']); - console.error(err.response.data); - }); - } + } + }).catch(err => { + console.log(sample['samplenumber']); + console.error(err.response.data); + errors.push(`KF/VZ upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`); + }); } } } @@ -375,7 +368,7 @@ async function allSamples() { for (let index in data) { - console.info(`${index}/${data.length}`); + console.info(`SAMPLE LOAD ${index}/${data.length}`); let sample = data[index]; if (sample['granulate/Part'] === '') { // empty supplier fields sample['granulate/Part'] = 'unknown'; @@ -400,11 +393,15 @@ async function allSamples() { const si = samples.length - 1; // sample index if (samples[si].notes.custom_fields.hasOwnProperty('xRest')) { // reroute xRest property to comment samples[si].notes.comment = samples[si].notes.custom_fields.xRest; + commentsLog.push(sample['samplenumber'] + ' ' + samples[si].notes.comment); delete samples[si].notes.custom_fields.xRest; } - if (Object.keys(samples[si].notes.custom_fields). length === 0) { + if (Object.keys(samples[si].notes.custom_fields).length === 0) { // delete empty custom fields delete samples[si].notes.custom_fields; } + else { + customFieldsLog.push(sample['samplenumber'] + ' ' + JSON.stringify(samples[si].notes.custom_fields)); + } if (sample['materialnumber'] !== '' && material.numbers.find(e => e.number === sample['materialnumber'])) { samples[si].color = material.numbers.find(e => e.number === sample['materialnumber']).color; } @@ -432,7 +429,7 @@ async function allSamples() { async function saveSamples() { for (let i in samples) { - console.info(`${i}/${samples.length}`); + console.info(`SAMPLE SAVE ${i}/${samples.length}`); let credentials = ['admin', 'Abc123!#']; if (sampleDevices[samples[i].number]) { console.log(sampleDevices[samples[i].number]); @@ -450,13 +447,65 @@ async function saveSamples() { if (err.response.data.status && err.response.data.status !== 'Sample number already taken') { console.log(samples[i]); console.error(err.response.data); - errors.push(`Upload for ${JSON.stringify(samples[i])} failed: ${JSON.stringify(err.response.data)}`) + errors.push(`Upload for ${JSON.stringify(samples[i])} failed: ${JSON.stringify(err.response.data)}`); } }); } console.info('saved all samples'); } +async function sampleReferencesSave() { + for (let i in sampleReferences) { + console.info(`SAMPLE REFERENCES ${i}/${sampleReferences.length}`); + let refRes = await axios({ + method: 'get', + url: host + '/sample/number/' + sampleReferences[i].referencedSample, + auth: { + username: 'admin', + password: 'Abc123!#' + } + }).catch(err => { + console.log(sampleReferences[i].referencedSample); + console.error(err.response.data); + errors.push(`Getting reference id for ${JSON.stringify(sampleReferences[i])} failed: ${JSON.stringify(err.response.data)}`); + }); + if (!refRes) continue; + let sampleRes = await axios({ + method: 'get', + url: host + '/sample/number/' + sampleReferences[i].sample, + auth: { + username: 'admin', + password: 'Abc123!#' + } + }).catch(err => { + console.log(sampleReferences[i].sample); + console.error(err.response.data); + errors.push(`Getting sample id for ${JSON.stringify(sampleReferences[i])} failed: ${JSON.stringify(err.response.data)}`); + }); + if (!sampleRes) continue; + sampleRes.data.notes.sample_references.push({sample_id: refRes.data._id, relation: sampleReferences[i].relation}) + await axios({ + method: 'put', + url: host + '/sample/' + sampleRes.data._id, + auth: { + username: 'admin', + password: 'Abc123!#' + }, + data: {notes: {sample_references: sampleRes.data.notes.sample_references}} + }).catch(err => { + console.log(sampleRes.data.notes.sample_references); + if (err.response.data) { + console.error(err.response.data); + errors.push(`Saving references for ${JSON.stringify(sampleRes.data)} failed: ${JSON.stringify(err.response.data)}`); + } + else { + console.error(err.response); + errors.push(`Saving references for ${JSON.stringify(sampleRes.data)} failed: ${JSON.stringify(err.response)}`); + } + }); + } +} + async function allMaterials() { // materials = {}; let res = await axios({ @@ -497,7 +546,7 @@ async function allMaterials() { } } else { // new material - console.info(`${index}/${data.length} ${sample['materialname']}`); + console.info(`MATERIAL LOAD ${index}/${data.length} ${sample['materialname']}`); materials[sample['materialname']] = { name: trim(sample['materialname']), supplier: trim(sample['supplier']), @@ -529,7 +578,7 @@ async function allMaterials() { async function saveMaterials() { const mKeys = Object.keys(materials) for (let i in mKeys) { - console.info(`${i}/${mKeys.length}`); + console.info(`MATERIAL SAVE ${i}/${mKeys.length}`); const material = _.cloneDeep(materials[mKeys[i]]); material.numbers = material.numbers.map(e => e.number).filter(e => e !== '').map(e => e.replace(/ /g, '')); await axios({ @@ -716,60 +765,62 @@ function sampleDeviceMap() { function customFields (comment, sampleNumber) { const customFields = [ - {docKey: 'Versuchsreihe', dbKey: 'test series', regex: /Versuchsreihe (\d+),/, reference: false}, - {docKey: 'Stillstand', dbKey: 'idle', regex: /(\d+ min)/, reference: false}, - {docKey: 'Serienzyklus', dbKey: 'cycle', regex: /(\d+.) Serienzyklus (\(.*?\))/, reference: false}, - {docKey: 'Berstdruck', dbKey: 'bursting pressure', regex: /Berstdruck: (.*?bar);/, reference: false}, - {docKey: 'gemessen am', dbKey: 'measured at', regex: /gemessen am (.*20\d\d)/, reference: false}, - {docKey: 'used for', dbKey: 'used for', regex: /used for (.*)/, reference: false}, - {docKey: 'Stabilized', dbKey: 'stabilized', regex: null, reference: false}, - {docKey: 'parts from field', dbKey: 'parts from field', regex: null, reference: false}, - {docKey: 'side', dbKey: 'side', regex: /(\S*?) side/, reference: false}, - {docKey: 'Creep test', dbKey: 'creep test', regex: null, reference: false}, - {docKey: 'Variante', dbKey: 'variant', regex: /(.*)/, reference: false}, - {docKey: 'Parameter', dbKey: 'parameter', regex: /Parameter (\d)/, reference: false}, - {docKey: 'days without cooling', dbKey: 'days without cooling', regex: /(\d+) days without cooling/, reference: false}, - {docKey: 'Zyklus', dbKey: 'cycle', regex: /Zyklus (\d+ s)/, reference: false}, - {docKey: 'fast cure', dbKey: 'fast cure', regex: null, reference: false}, - {docKey: 'Stoff gesperrt', dbKey: 'material blocked', regex: null, reference: false}, - {docKey: 'anwendungsbeschränkt', dbKey: 'limited application', regex: null, reference: false}, - {docKey: 'für Neuanwendungen gesperrt', dbKey: 'blocked for new applications', regex: null, reference: false}, - {docKey: 'V', dbKey: 'test', regex: /V(\d+-\d+);/, reference: false}, - {docKey: 'Twz', dbKey: 'twz', regex: /Twz \(°C\): (\d+);/, reference: false}, - {docKey: 'Pnach', dbKey: 'pressure after', regex: /Pnach \(bar\): (\d+);/, reference: false}, - {docKey: 'Vein', dbKey: 'volume in', regex: /Vein \(ccm\/s\): (\d+)/, reference: false}, - {docKey: 'low emission', dbKey: 'low emission', regex: /low emission (.*?);/, reference: false}, - {docKey: 'aus', dbKey: 'from', regex: /aus (.*)/, reference: false}, - {docKey: 'Erprobung', dbKey: 'trial', regex: /Erprobung (.*?);/, reference: false}, - {docKey: 'Auftragsnummer', dbKey: 'job number', regex: /Auftragsnummer: (.*?);/, reference: false}, - {docKey: 'Wärmealterung', dbKey: 'heat aging', regex: /Wärmealterung: (.*)/, reference: false}, - {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall', regex: /Steg.*?A: (\d+)/, reference: false}, - {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall', regex: /Steg.*?I: (\d+)/, reference: false}, - {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall', regex: /Steg.*?S: (\d+)/, reference: false}, - {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall degraded', regex: /Degradation:.*?A: (\d+)/, reference: false}, - {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall degraded', regex: /Degradation:.*?I: (\d+)/, reference: false}, - {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall degraded', regex: /Degradation:.*?S: (\d+)/, reference: false}, - {docKey: 'Reines Polymer', dbKey: 'pure polymer', regex: null, reference: false}, - {docKey: 'Rücksendung erforderlich', dbKey: 'return needed', regex: /(.*?,) Rücksendung erforderlich, (.*)/, reference: false}, - {docKey: 'Prio', dbKey: 'priority', regex: /Prio (\d+)/, reference: false}, - {docKey: 'beanstandet', dbKey: 'faulty', regex: null, reference: false}, - {docKey: 'aged', dbKey: 'aged', regex: /aged: (.*)/, reference: false}, - {docKey: 'DOPPELT!!', dbKey: 'double', regex: null, reference: false}, - {docKey: 'Bauteil', dbKey: 'construction part', regex: /Bauteil (\S+)/, reference: false}, - {docKey: 'T =', dbKey: 'temperature', regex: /T = (\S+)/, reference: false}, - {docKey: 'nicht vorgealtert', dbKey: 'not preaged', regex: /nicht vorgealtert (.*)/, reference: false}, - {docKey: 'TS119', dbKey: 'TS119', regex: /TS119 (W\S+);/, reference: false}, - {docKey: 'GF vom Datenblatt', dbKey: 'glass fibre from data sheet', regex: null, reference: false}, - {docKey: 'nach Datensatz', dbKey: 'according to dataset', regex: null, reference: false}, - {docKey: 'Dosiergeschw', dbKey: 'metering speed', regex: /Dosiergeschw.*? (.*?min)/, reference: false}, - {docKey: 'Einspritzgeschw', dbKey: 'injection speed', regex: /Einspritzgeschw.*? (.*\/s)/, reference: false}, - {docKey: 'Heizbänder', dbKey: 'heating lines', regex: /Heizbänder (.*)/, reference: false}, - {docKey: 'Verweilzeit', dbKey: 'dwell time', regex: /Verweilzeit (.*?min)/, reference: false}, - {docKey: 'Probe', dbKey: 'belongs to', regex: /Probe (\S*\d+)/, reference: true}, - {docKey: 'zu', dbKey: 'belongs to', regex: /zu (\S*\d+)/, reference: true}, - {docKey: 'granulate zu', dbKey: 'granulate to', regex: /granulate zu.* (\S*\d+)/, reference: true}, - {docKey: 'construction part', dbKey: 'construction part', regex: /(? i > 0).join(' '); } diff --git a/src/routes/sample.spec.ts b/src/routes/sample.spec.ts index 1989746..ca62d16 100644 --- a/src/routes/sample.spec.ts +++ b/src/routes/sample.spec.ts @@ -394,7 +394,7 @@ describe('/sample', () => { url: '/samples?status=all&fields[]=number&fields[]=batch&filters[]=%7B%22mode%22%3A%22xx%22%2C%22field%22%3A%22batch%22%2C%22values%22%3A%5B%221704-005%22%5D%7D', auth: {basic: 'janedoe'}, httpStatus: 400, - res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin]'} + res: {status: 'Invalid body format', details: '"filters[0].mode" must be one of [eq, ne, lt, lte, gt, gte, in, nin, stringin]'} }); }); it('rejects an filter field not existing', done => { diff --git a/src/routes/sample.ts b/src/routes/sample.ts index 2d23d95..e468a41 100644 --- a/src/routes/sample.ts +++ b/src/routes/sample.ts @@ -758,6 +758,9 @@ function filterQueries (filters) { if (e.mode === 'or') { // allow or queries (needed for $ne added) return {['$' + e.mode]: e.values}; } + else if (e.mode === 'stringin') { + return {[e.field]: {['$in']: [new RegExp(e.values[0])]}}; + } else { return {[e.field]: {['$' + e.mode]: (e.mode.indexOf('in') >= 0 ? e.values : e.values[0])}}; // add filter criteria as {field: {$mode: value}}, only use first value when mode is not in/nin } diff --git a/src/routes/validate/sample.ts b/src/routes/validate/sample.ts index e5f8ffe..3e9aed3 100644 --- a/src/routes/validate/sample.ts +++ b/src/routes/validate/sample.ts @@ -213,7 +213,7 @@ export default class SampleValidate { csv: Joi.boolean().default(false), fields: Joi.array().items(Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm'))).default(['_id','number','type','batch','material_id','color','condition','note_id','user_id','added']).messages({'string.pattern.base': 'Invalid field name'}), filters: Joi.array().items(Joi.object({ - mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin'), + mode: Joi.string().valid('eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'in', 'nin', 'stringin'), field: Joi.string().pattern(new RegExp('^(' + this.fieldKeys.join('|').replace(/\./g, '\\.').replace(/\*/g, '.+') + ')$', 'm')).messages({'string.pattern.base': 'Invalid filter field name'}), values: Joi.array().items(Joi.alternatives().try(Joi.string().max(128), Joi.number(), Joi.boolean(), Joi.date().iso(), Joi.object())).min(1) })).default([])