const csv = require('csv-parser'); const fs = require('fs'); const axios = require('axios'); const {Builder} = require('selenium-webdriver'); // selenium and the chrome driver must be installed and configured separately const chrome = require('selenium-webdriver/chrome'); const pdfReader = require('pdfreader'); const iconv = require('iconv-lite'); const _ = require('lodash'); const stages = { materials: true, samples: true, dpt: true } const docs = [ "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2_A.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata__AnP2_B.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Ap.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Bj.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh_B.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Eh_Duroplasten.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell_A.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_Rng_aktuell_B.csv", "C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\Metadata_WaP.csv", ]; const errors = []; const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\nmDocs'; // NormMaster Documents const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200717\\DPT'; // Spectrum files const host = 'http://localhost:3000'; // const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com'; const requiredProperties = ['samplenumber','materialnumber','materialname','supplier','reinforcementmaterial','material','granulate/part','color','charge/batch','comments']; dict = { // dictionary 'Granulat': 'granulate', 'Zugstab': 'tension rod', 'Stecker': 'plug' }; let data = []; // metadata contents let materials = {}; let numberToColor = {}; let samples = []; let normMaster = {}; let sampleDevices = {}; const sampleReferences = []; // references to other samples in format {sample, referencedSample, relation} let commentsLog = []; let customFieldsLog = []; const vnValues = {}; // vn values from comments const dptLog = []; const dptSampleAddLog = []; // log samples created during dpt insertion const typeLog = []; // TODO: conditions main(); async function main() { if (stages.materials) { // materials await getNormMaster(); for (let i in docs) { await importCsv(docs[i]); await allMaterials(); await saveMaterials(); } fs.writeFileSync('./data_import/numberToColor.json', JSON.stringify(numberToColor)); } if (stages.samples) { // samples sampleDeviceMap(); numberToColor = JSON.parse(fs.readFileSync('./data_import/numberToColor.json'), 'utf-8'); for (let i in docs) { await importCsv(docs[i]); await allSamples(); await saveSamples(); await allMcVn(); } // write logs fs.writeFileSync('./data_import/comments.txt', commentsLog.join('\r\n')); fs.writeFileSync('./data_import/typeLog.txt', typeLog.join('\r\n')); fs.writeFileSync('./data_import/customFields.txt', customFieldsLog.join('\r\n')); fs.writeFileSync('./data_import/sampleReferences.txt', sampleReferences.map(e => JSON.stringify(e)).join('\r\n')); fs.writeFileSync('./data_import/sampleReferences.json', JSON.stringify(sampleReferences)); await sampleReferencesSave(); } if (stages.dpt) { // DPT await allDpts(); fs.writeFileSync('./data_import/sdptLog.txt', dptLog.join('\r\n')); fs.writeFileSync('./data_import/dptSampleAddLog.txt', dptSampleAddLog.join('\r\n')); } if (0) { // pdf test console.log(await readPdf('N28_BN05-OX023_2019-07-16.pdf')); } if (errors.length) { // console.log(errors); fs.writeFileSync('./data_import/errors/errors_' + new Date().getTime() + '.txt', errors.join('\r\n')); } } async function importCsv(doc) { // Uniform name samplenumber materialnumber materialname supplier material plastic reinforcingmaterial granulate/part color charge/batch comments vz(ml/g) kfingew% degradation(%) reinforcingmaterialcontent stabwn // Metadata__AnP2.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,granulate/Part,Color,Charge/ Batch, Comments // Metadata__AnP2_A.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part, Comments, Humidity [ppm] // Metadata__AnP2_B.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part, VZ [ml/g], glass fibre content // Metadata_Ap.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part,Color,Charge/Batch, Comments // Metadata_Bj.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Color,Charge/batch granulate/part,Comments // Metadata_Eh.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material,Granulate/Part,Color,Charge/Batch granulate/part,Comments, VZ [cm³/g], Spalte1 // Metadata_Eh_B.csv Sample number, Material name,Supplier, Plastic,Reinforcing material,Granulate/Part,Color, Comments, VZ [cm³/g] // Metadata_Eh_Duroplasten.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material,Granulate/Part,Color,Charge/Batch granulate/part,Comments // Metadata_Rng_aktuell.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Color,Charge/batch granulate/part,Comments, VZ (ml/g), Degradation(%),Glas fibre content (%) // Metadata_Rng_aktuell_A.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Farbe,Charge/batch granulate/part,Comments, KF in Gew%, Reinforcing material (content in %),Stabwn // Metadata_Rng_aktuell_B.csv Sample number, Material name,Supplier, Plastic, Granulate/Part, Comments, VZ (ml/g), Degradation (%), Alterungszeit in h // Metadata_WaP.csv Probennummer, Name, Firma, Material, Teil/Rohstoff, Charge, Anmerkung,VZ (ml/g), Abbau (%), Verstärkungsstoffgehalt (%), Versuchsnummer const nameCorrection = { // map to right column names 'probennummer': 'samplenumber', 'name': 'materialname', 'firma': 'supplier', 'teil/rohstoff': 'granulate/part', 'charge/batchgranulate/part': 'charge/batch', 'charge': 'charge/batch', 'anmerkung': 'comments', 'vz[ml/g]': 'vz(ml/g)', 'vz[cm³/g]': 'vz(ml/g)', 'abbau(%)': 'degradation(%)', 'glassfibrecontent': 'reinforcingmaterialcontent', 'glasfibrecontent(%)': 'reinforcingmaterialcontent', 'reinforcingmaterial(contentin%)': 'reinforcingmaterialcontent', 'verstärkungsstoffgehalt(%)': 'reinforcingmaterialcontent' }; const missingFieldsFill = [ // column names to fill if they do not exist 'color', 'charge/batch', 'comments', 'materialnumber', 'reinforcementmaterial' ] console.info('importing ' + doc); data = []; await new Promise(resolve => { fs.createReadStream(doc) .pipe(iconv.decodeStream('win1252')) .pipe(csv()) .on('data', (row) => { data.push(row); }) .on('end', () => { data = data.map(e => { const newE = {}; Object.keys(e).forEach(key => { newE[key.toLowerCase().replace(/ /g, '')] = e[key]; }); // replace wrong column names Object.keys(newE).forEach(key => { if (nameCorrection.hasOwnProperty(key)) { newE[nameCorrection[key]] = newE[key]; delete newE[key]; } }); // add missing fields with empty values missingFieldsFill.forEach(field => { if (!newE.hasOwnProperty(field)) { newE[field] = ''; } }); if(newE['materialname'] === '') { newE['materialname'] = newE['material']; } if (newE['supplier'] === '') { // empty supplier fields newE['supplier'] = 'unknown'; } if (!newE.hasOwnProperty('material')) { newE['material'] = newE['plastic'].indexOf(' GF') >= 0 ? newE['plastic'].split(' ')[0] : newE['plastic']; } return newE; }).filter(e => { const missingProperties = requiredProperties.filter(el => !e.hasOwnProperty(el)); if (e['materialname'] === '') { missingProperties.push('materialname'); } if (e['samplenumber'] === '') { // empty row return false; } else if (missingProperties.length > 0) { // incomplete sample errors.push(`${doc}: ${JSON.stringify(e)}is missing the required properties ${missingProperties}`); return false; } return true; }); console.info('CSV file successfully processed'); resolve(); }); }); } async function allDpts() { let res = await axios({ method: 'get', url: host + '/template/measurements', auth: { username: 'admin', password: 'Abc123!#' } }); const measurement_templates = res.data.filter(e => e.name === 'spectrum'); const measurement_template = measurement_templates[measurement_templates.length - 1]._id; res = await axios({ method: 'get', url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' } }); const sampleIds = {}; res.data.forEach(sample => { sampleIds[sample.number] = sample._id; }); const dptRegex = /(.*?)_(.*?)_(\d+|[a-zA-Z0-9]+[_.]\d+)(_JDX)?[.]{1,2}(DPT|csv|CSV|JDX)/; const dpts = fs.readdirSync(dptFiles); for (let i in dpts) { let regexInput; const bjRes = /^(Bj[FT]?)\s?([a-z0-9_]*)_JDX.DPT/.exec(dpts[i]); if (bjRes) { // correct Bj numbers with space regexInput = `Bj01_${bjRes[1]}${bjRes[2]}_0.DPT`; } else { // remove _JDX from name regexInput = dpts[i].replace(/_JDX.*\./, '.'); } const regexRes = dptRegex.exec(regexInput); if (regexRes && !sampleIds[regexRes[2]]) { // when sample number includes an additional _x instead of having _x_x for spectrum description regexRes[2] = `${regexRes[2]}_${regexRes[3].split('_')[0]}`; } let baseSample = null; if (regexRes) { baseSample = regexRes[2].split('_')[0]; if (baseSample === 'Wa11') { // as Wa11 samples use all the same material baseSample = 'Wa11_B0_1'; } } if (regexRes && !sampleIds[regexRes[2]] && sampleIds[baseSample]) { // when number_abx does not exist but number dptSampleAddLog.push(`Trying to find ${baseSample}`); dptSampleAddLog.push(host + '/sample/' + sampleIds[baseSample]); res = await axios({ // get base sample method: 'get', url: host + '/sample/' + stripSpaces(sampleIds[baseSample]), auth: { username: 'admin', password: 'Abc123!#' } }).catch(err => { if (err.response) { console.error(err.response.data); errors.push(`DPT Could not fetch sample ${baseSample}: ${JSON.stringify(err.response.data)}`); } }); if (res) { const data = _.merge(_.pick(res.data, ['color', 'type', 'batch']), {number: regexRes[2], condition: {}, notes: {}, material_id: res.data.material._id}); res = await axios({ method: 'post', url: host + '/sample/new', auth: { username: res.data.user, password: res.data.user === 'admin' ? 'Abc123!#' : '2020DeFinMachen!' }, data }).catch(err => { if (err.response) { console.error(err.response.data); errors.push(`DPT Could not save sample ${data}: ${err.response.data}`); } }); if (res.data) { dptSampleAddLog.push(`${regexRes[2]} from ${baseSample}`) sampleIds[regexRes[2]] = res.data._id; } } } if (regexRes && sampleIds[regexRes[2]]) { // found matching sample console.log(`${i}/${dpts.length} ${dpts[i]} -> ${regexRes[2]}`); dptLog.push(`${dpts[i]}, ${regexRes[2]}`); const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8'); const data = { sample_id: sampleIds[regexRes[2]], values: {}, measurement_template }; data.values.device = regexRes[1]; data.values.filename = dpts[i]; data.values.dpt = f.split('\r\n').map(e => e.split(',').map(e => parseFloat(e))); let rescale = false; for (let i in data.values.dpt) { if (data.values.dpt[i][1] > 10) { rescale = true; break; } } if (rescale) { data.values.dpt = data.values.dpt.map(e => [e[0], e[1] / 100]); } await axios({ method: 'post', url: host + '/measurement/new', auth: { username: 'admin', password: 'Abc123!#' }, data }).catch(err => { console.log(dpts[i]); if (err.response) { console.error(err.response.data); errors.push(`Could not upload ${dpts[i]} for sample ${regexRes[2]}: ${err.response.data}`); } else { console.error(err); errors.push(`Could not upload ${dpts[i]} for sample ${regexRes[2]}: ${JSON.stringify(err)}`); } }); } else { console.log(`Could not find sample for ${dpts[i]}`); if (regexRes) { errors.push(`Could not find sample for ${dpts[i]}; [DEBUG] ${regexRes[2]}, ${!sampleIds[regexRes[2]]}, ${sampleIds[baseSample]}`); } else { errors.push(`Could not find sample for ${dpts[i]} (did not match RegEx)`); } } } } async function allMcVn() { let res = await axios({ method: 'get', url: host + '/template/measurements', auth: { username: 'admin', password: 'Abc123!#' } }); const mc_template = res.data.filter(e => e.name === 'moisture content').sort((a, b) => b.version - a.version)[0]._id; const vn_template = res.data.filter(e => e.name === 'vn').sort((a, b) => b.version - a.version)[0]._id; const rmc_template = res.data.filter(e => e.name === 'reinforcement material content').sort((a, b) => b.version - a.version)[0]._id; res = await axios({ method: 'get', url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' } }); const sampleIds = {}; res.data.forEach(sample => { sampleIds[sample.number] = sample._id; }); for (let index in data) { console.info(`MC/VN ${index}/${data.length}`); let sample = data[index]; sample['samplenumber'] = sample['samplenumber'].replace(/[A-Z][a-z]0\d_/, ''); let credentials = ['admin', 'Abc123!#']; if (sampleDevices[sample['samplenumber']]) { credentials = [sampleDevices[sample['samplenumber']], '2020DeFinMachen!'] } if (!sample['vz(ml/g)'] && vnValues[sample['samplenumber']]) { // fill in VN values from comments sample['vz(ml/g)'] = vnValues[sample['samplenumber']]; } if (sample['kfingew%']) { await axios({ method: 'post', url: host + '/measurement/new', auth: { username: credentials[0], password: credentials[1] }, data: { sample_id: sampleIds[sample['samplenumber']], measurement_template: mc_template, values: { 'weight %': sample['kfingew%'], 'standard deviation': sample['stabwn'] } } }).catch(err => { console.log(sample['samplenumber']); console.error(err.response.data); errors.push(`MC/VN upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`); }); } if (sample['vz(ml/g)']) { await axios({ method: 'post', url: host + '/measurement/new', auth: { username: credentials[0], password: credentials[1] }, data: { sample_id: sampleIds[sample['samplenumber']], measurement_template: vn_template, values: { vn: sample['vz(ml/g)'] } } }).catch(err => { console.log(sample['samplenumber']); console.error(err.response.data); errors.push(`MC/VN upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`); }); } if (sample['reinforcingmaterialcontent']) { await axios({ method: 'post', url: host + '/measurement/new', auth: { username: credentials[0], password: credentials[1] }, data: { sample_id: sampleIds[sample['samplenumber']], measurement_template: rmc_template, values: { percentage: Number(sample['reinforcingmaterialcontent'].replace('%', '').replace(',', '.')) } } }).catch(err => { console.log(sample['samplenumber']); console.error(err.response.data); errors.push(`MC/VN upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`); }); } } } async function allSamples() { samples = []; let res = await axios({ method: 'get', url: host + '/materials?status=all', auth: { username: 'admin', password: 'Abc123!#' } }); const dbMaterials = {} res.data.forEach(m => { m.numbers = m.numbers.map(e => ({number: e, color: numberToColor[e]})); dbMaterials[m.name] = m; }) res = await axios({ method: 'get', url: host + '/samples?status=all', auth: { username: 'admin', password: 'Abc123!#' } }); const sampleColors = {}; res.data.forEach(sample => { sampleColors[sample.number] = sample.color; }); for (let index in data) { console.info(`SAMPLE LOAD ${index}/${data.length}`); let sample = data[index]; if (sample['granulate/Part'] === '') { // empty supplier fields sample['granulate/Part'] = 'unknown'; } const material = dbMaterials[trim(sample['materialname'])]; if (!material) { // could not find material, skipping sample errors.push(`Could not find a material for ${JSON.stringify(sample)}`); continue; } samples.push({ number: sample['samplenumber'].replace(/[A-Z][a-z]0\d_/, ''), // remove leading An_01 and Eh_01 type: sampleType(sample['granulate/part']), batch: sample['charge/batch'], material_id: material._id, notes: { custom_fields: customFields(sample['comments'], sample['samplenumber']) } }); // if (sample['comments']) { // comments.push(sample['samplenumber'] + ' ' + sample['comments']); // } const si = samples.length - 1; // sample index if (samples[si].notes.custom_fields.hasOwnProperty('xRest')) { // reroute xRest property to comment samples[si].notes.comment = samples[si].notes.custom_fields.xRest; commentsLog.push(sample['samplenumber'] + ' ' + samples[si].notes.comment); delete samples[si].notes.custom_fields.xRest; } if (Object.keys(samples[si].notes.custom_fields).length === 0) { // delete empty custom fields delete samples[si].notes.custom_fields; } else { customFieldsLog.push(sample['samplenumber'] + ' ' + JSON.stringify(samples[si].notes.custom_fields)); } if (sample['materialnumber'] !== '' && material.numbers.find(e => e.number === sample['materialnumber'])) { samples[si].color = material.numbers.find(e => e.number === sample['materialnumber']).color; } else if (sample['color'] !== '') { // find color with all edge cases let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['color'])) >= 0); if (!number && /black/.test(sample['color'])) { // special case bk for black console.log(material); number = material.numbers.find(e => e.color && e.color.toLowerCase().indexOf('bk') >= 0); if (!number) { // try German word number = material.numbers.find(e => e.color && e.color.toLowerCase().indexOf('schwarz') >= 0); } } if (number) { samples[si].color = number.color; } } else if (sampleColors[sample['samplenumber'].split('_')[0]]) { // derive color from main sample for mc/vn samples[si].color = sampleColors[sample['samplenumber'].split('_')[0]]; } if (!samples[si].color) { samples[si].color = sample['color']; } } } async function saveSamples() { for (let i in samples) { console.info(`SAMPLE SAVE ${i}/${samples.length}`); let credentials = ['admin', 'Abc123!#']; if (sampleDevices[samples[i].number]) { credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!'] } await axios({ method: 'post', url: host + '/sample/new', auth: { username: credentials[0], password: credentials[1] }, data: samples[i] }).catch(err => { if (err.response.data.status && err.response.data.status !== 'Sample number already taken') { console.log(samples[i]); console.error(err.response.data); errors.push(`Upload for ${JSON.stringify(samples[i])} failed: ${JSON.stringify(err.response.data)}`); } }); } console.info('saved all samples'); } async function sampleReferencesSave() { for (let i in sampleReferences) { console.info(`SAMPLE REFERENCES ${i}/${sampleReferences.length}`); let refRes = await axios({ method: 'get', url: host + '/sample/number/' + sampleReferences[i].referencedSample, auth: { username: 'admin', password: 'Abc123!#' } }).catch(err => { console.log(sampleReferences[i].referencedSample); console.error(err.response.data); errors.push(`Getting reference id for ${JSON.stringify(sampleReferences[i])} failed: ${JSON.stringify(err.response.data)}`); }); if (!refRes) continue; let sampleRes = await axios({ method: 'get', url: host + '/sample/number/' + sampleReferences[i].sample, auth: { username: 'admin', password: 'Abc123!#' } }).catch(err => { console.log(sampleReferences[i].sample); console.error(err.response.data); errors.push(`Getting sample id for ${JSON.stringify(sampleReferences[i])} failed: ${JSON.stringify(err.response.data)}`); }); if (!sampleRes) continue; sampleRes.data.notes.sample_references.push({sample_id: refRes.data._id, relation: sampleReferences[i].relation}) await axios({ method: 'put', url: host + '/sample/' + sampleRes.data._id, auth: { username: 'admin', password: 'Abc123!#' }, data: {notes: {sample_references: sampleRes.data.notes.sample_references}} }).catch(err => { console.log(sampleRes.data.notes.sample_references); if (err.response.data) { console.error(err.response.data); errors.push(`Saving references for ${JSON.stringify(sampleRes.data)} failed: ${JSON.stringify(err.response.data)}`); } else { console.error(err.response); errors.push(`Saving references for ${JSON.stringify(sampleRes.data)} failed: ${JSON.stringify(err.response)}`); } }); } } async function allMaterials() { // materials = {}; let res = await axios({ method: 'get', url: host + '/template/materials', auth: { username: 'admin', password: 'Abc123!#' } }); const materialTemplate = res.data.filter(e => e.name === 'plastic').sort((a, b) => b.version - a.version)[0]._id; // process all samples for (let index in data) { let sample = data[index]; if (sample['supplier'] === '') { // empty supplier fields sample['supplier'] = 'unknown'; } if (sample['materialname'] === '') { // empty name fields sample['materialname'] = sample['material']; } sample['materialname'] = trim(sample['materialname']); if (materials.hasOwnProperty(sample['materialname'])) { // material already found at least once if (sample['materialnumber'] !== '') { // material number given if (materials[sample['materialname']].numbers.length === 0 || !materials[sample['materialname']].numbers.find(e => e.number === stripSpaces(sample['materialnumber']))) { // new material number if (materials[sample['materialname']].numbers.find(e => e.color === sample['color'] && e.number === '')) { // color already in list, only number missing materials[sample['materialname']].numbers.find(e => e.color === sample['color'] && e.number === '').number = stripSpaces(sample['materialnumber']); } else { // completely new number entry materials[sample['materialname']].numbers.push({color: trim(sample['color']), number: stripSpaces(sample['materialnumber'])}); } } } else if (sample['color'] !== '') { // color given if (!materials[sample['materialname']].numbers.find(e => e.color === stripSpaces(sample['color']))) { // new material color materials[sample['materialname']].numbers.push({color: trim(sample['color']), number: ''}); } } } else { // new material console.info(`MATERIAL LOAD ${index}/${data.length} ${sample['materialname']}`); materials[sample['materialname']] = { name: trim(sample['materialname']), supplier: trim(sample['supplier']), group: trim(sample['material']) }; materials[sample['materialname']].numbers = await numbersFetch(sample); // material properties materials[sample['materialname']].properties = {material_template: materialTemplate}; let tmp = /M(\d+)/.exec(sample['reinforcingmaterial']); materials[sample['materialname']].properties.mineral = tmp ? tmp[1] : 0; tmp = /GF(\d+)/.exec(sample['reinforcingmaterial']); materials[sample['materialname']].properties.glass_fiber = tmp ? tmp[1] : 0; tmp = /CF(\d+)/.exec(sample['reinforcingmaterial']); materials[sample['materialname']].properties.carbon_fiber = tmp ? tmp[1] : 0; } } // Fill numberToColor array Object.keys(materials).forEach(mKey => { materials[mKey].numbers.forEach(number => { if (number.number && number.color) { numberToColor[number.number] = number.color; } }) }); } async function saveMaterials() { const mKeys = Object.keys(materials) for (let i in mKeys) { console.info(`MATERIAL SAVE ${i}/${mKeys.length}`); const material = _.cloneDeep(materials[mKeys[i]]); material.numbers = material.numbers.map(e => e.number).filter(e => e !== '').map(e => e.replace(/ /g, '')); await axios({ method: 'post', url: host + '/material/new', auth: { username: 'admin', password: 'Abc123!#' }, data: material }).catch(err => { if (err.response.data.status && err.response.data.status !== 'Material name already taken') { console.info(material); console.error(err.response.data); errors.push(`Upload for ${JSON.stringify(material)} failed: ${JSON.stringify(err.response.data)}`) } }); } console.info('saved all materials'); } async function numbersFetch(sample) { let nm = []; let res = []; if (sample['materialnumber']) { // sample has a material number nm = normMaster[stripSpaces(sample['materialnumber'])]? [normMaster[stripSpaces(sample['materialnumber'])]] : []; } else { // try finding via material name nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['materialnumber'])).map(e => normMaster[e]); } if (nm.length > 0) { for (let i in nm) { if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded await getNormMasterDoc(nm[i].url.replace(/ /g, '%20')); } // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded // console.info('Retrying download...'); // await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 2.2); // } // if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded // console.info('Retrying download again...'); // await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 5); // } if (fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document loaded res = await readPdf(fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)); } if (res.length > 0) { // no results break; } else if (i + 1 >= nm.length) { errors.push(`Download of ${nm[i].url.replace(/ /g, '%20')} for material number ${sample['materialnumber']} failed`); errors.push(nm[i].doc.replace(/ /g, '_')); } } } if (res.length === 0) { // no results if (sample['color'] !== '' || sample['materialnumber'] !== '') { // information in data available return [{color: trim(sample['color']), number: sample['materialnumber']}]; } else { return []; } } else { if (!res.find(e => e.number === sample['materialnumber'])) { // sometimes norm master does not include sample number even if listed res.push({color: trim(sample['color']), number: sample['materialnumber']}); } return res; } } async function getNormMaster(fetchAgain = false) { if (fetchAgain) { console.info('fetching norm master...'); const res = await axios({ method: 'get', url: 'http://rb-normen.bosch.com/cgi-bin/searchRBNorm4TradeName' }); console.info('finding documents...'); let match; // const regex = /.*?.*?<\/span>(.*?)<\/td>(\d+)<\/td>.*?.*?.*?<\/span>(.*?)<\/td>(\d+)<\/td>40.*?(.*?)<\/td>/gm; // only valid materials do { match = regex.exec(res.data); if (match) { normMaster[match[2]] = {name: match[1], nameSpaceless: stripSpaces(match[1]), number: match[2], url: match[3], doc: match[4]}; } } while (match); fs.writeFileSync('./data_import/normMaster.json', JSON.stringify(normMaster)); } else { normMaster = JSON.parse(fs.readFileSync('./data_import/normMaster.json'), 'utf-8'); } } function getNormMasterDoc(url, timing = 1) { console.info(url); return new Promise(async resolve => { const options = new chrome.Options(); options.setUserPreferences({ "download.default_directory": nmDocs, "download.prompt_for_download": false, "download.directory_upgrade": true, "plugins.always_open_pdf_externally": true }); let driver = await new Builder().forBrowser('chrome').setChromeOptions(options).build(); let timeout = 7000 * timing; try { await driver.get(url); if (await driver.getCurrentUrl() !== 'https://rb-wam-saml.bosch.com/tfim/sps/normmaster/saml20/login') { // got document selection page timeout = 11000 * timing; await driver.executeScript('Array.prototype.slice.call(document.querySelectorAll(\'.functionlink\')).filter(e => e.innerText === \'English\')[0].click()').catch(() => {timeout = 0; }); } } finally { setTimeout(async () => { // wait until download is finished await driver.quit(); resolve(); }, timeout); } }); } function readPdf(file) { return new Promise(async resolve => { const countdown = 100; // value for text timeout let table = 0; // > 0 when in correct table area let rows = []; // found table rows let lastY = 0; // y of last row let lastX = 0; // right x of last item let lastText = ''; // text of last item let lastLastText = ''; // text of last last item await new pdfReader.PdfReader().parseFileItems(nmDocs + '\\' + file, (err, item) => { if (item && item.text) { if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsuppl') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsuppl') >= 0)) { // table area starts table = countdown; } if (table > 0) { // console.log(item); // console.log(item.y - lastY); // console.log(item.text); if (item.y - lastY > 0.8 && Math.abs(item.x - lastX) > 5) { // new row lastY = item.y; rows.push(item.text); } else { // still the same row row rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text; // push to row, detect if still same cell } lastX = (item.w * 0.055) + item.x; if (/\d \d\d\d \d\d\d \d\d\d/.test(item.text)) { table = countdown; } table --; if (table <= 0 || item.text.toLowerCase().indexOf('release document') >= 0 || item.text.toLowerCase().indexOf('normative references') >= 0) { // table area ended table = -1; // console.log(rows); rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; })); } } lastLastText = lastText; lastText = item.text; } if (!item && table !== -1) { // document ended rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; })); } }); }); } function sampleDeviceMap() { const dpts = fs.readdirSync(dptFiles); const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/; for (let i in dpts) { const regexRes = regex.exec(dpts[i]) if (regexRes) { // found matching sample sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase(); } } } function customFields (comment, sampleNumber) { const customFields = [ {docKey: 'Versuchsreihe', dbKey: 'test series', regex: /Versuchsreihe (\d+),/, category: 'customField'}, {docKey: 'Stillstand', dbKey: 'idle', regex: /Stillstand (\d+ min):/, category: 'customField'}, {docKey: 'Serienzyklus', dbKey: 'cycle', regex: /(\d+.) Serienzyklus (\(.*?\))/, category: 'customField'}, {docKey: 'Berstdruck', dbKey: 'bursting pressure', regex: /Berstdruck: (.*?bar);/, category: 'customField'}, {docKey: 'gemessen am', dbKey: 'measured at', regex: /gemessen am (.*20\d\d)/, category: 'customField'}, {docKey: 'used for', dbKey: 'used for', regex: /used for (.*)/, category: 'customField'}, {docKey: 'Stabilized', dbKey: 'stabilized', regex: /Stabilized, (.*)/, category: 'customField'}, {docKey: 'parts from field', dbKey: 'parts from field', regex: null, category: 'customField'}, {docKey: 'side', dbKey: 'side', regex: /(\S*?) side/, category: 'customField'}, {docKey: 'Creep test', dbKey: 'creep test', regex: null, category: 'customField'}, {docKey: 'Variante', dbKey: 'variant', regex: /(.*)/, category: 'customField'}, {docKey: 'Parameter', dbKey: 'parameter', regex: /Parameter (\d)/, category: 'customField'}, {docKey: 'days without cooling', dbKey: 'days without cooling', regex: /(\d+) days without cooling/, category: 'customField'}, {docKey: 'Zyklus', dbKey: 'cycle', regex: /Zyklus (\d+ s)/, category: 'customField'}, {docKey: 'fast cure', dbKey: 'fast cure', regex: null, category: 'customField'}, {docKey: 'Stoff gesperrt', dbKey: 'material blocked', regex: null, category: 'customField'}, {docKey: 'anwendungsbeschränkt', dbKey: 'limited application', regex: null, category: 'customField'}, {docKey: 'für Neuanwendungen gesperrt', dbKey: 'blocked for new applications', regex: null, category: 'customField'}, {docKey: 'V', dbKey: 'test', regex: /V(\d+-\d+);/, category: 'customField'}, {docKey: 'Twz', dbKey: 'twz', regex: /Twz \(°C\): (\d+);/, category: 'customField'}, {docKey: 'Pnach', dbKey: 'pressure after', regex: /Pnach \(bar\): (\d+);/, category: 'customField'}, {docKey: 'Vein', dbKey: 'volume in', regex: /Vein \(ccm\/s\): (\d+)/, category: 'customField'}, {docKey: 'low emission', dbKey: 'low emission', regex: /low emission (\S+)[;]?/, category: 'customField'}, {docKey: 'aus', dbKey: 'from', regex: /aus (.*)/, category: 'customField'}, {docKey: 'Erprobung', dbKey: 'trial', regex: /Erprobung (.*?);/, category: 'customField'}, {docKey: 'Auftragsnummer', dbKey: 'job number', regex: /Auftragsnummer: (\S+)[;]?/, category: 'customField'}, {docKey: 'Wärmealterung', dbKey: 'heat aging', regex: /Wärmealterung: (.*)/, category: 'customField'}, {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall', regex: /Steg.*?A: (\d+)/, category: 'customField'}, {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall', regex: /Steg.*?I: (\d+)/, category: 'customField'}, {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall', regex: /Steg.*?S: (\d+)/, category: 'customField'}, {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall degraded', regex: /Degradation:.*?A: (\d+)/, category: 'customField'}, {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall degraded', regex: /Degradation:.*?I: (\d+)/, category: 'customField'}, {docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall degraded', regex: /Degradation:.*?S: (\d+)/, category: 'customField'}, {docKey: 'Reines Polymer', dbKey: 'pure polymer', regex: null, category: 'customField'}, {docKey: 'Rücksendung erforderlich', dbKey: 'return needed', regex: /(.*?,) Rücksendung erforderlich, (.*)/, category: 'customField'}, {docKey: 'Prio', dbKey: 'priority', regex: /Prio (\d+)/, category: 'customField'}, {docKey: 'beanstandet', dbKey: 'faulty', regex: null, category: 'customField'}, {docKey: 'aged', dbKey: 'aged', regex: /aged: (.*)/, category: 'customField'}, {docKey: 'DOPPELT!!', dbKey: 'double', regex: null, category: 'customField'}, {docKey: 'Bauteil', dbKey: 'construction part', regex: /Bauteil (\S+)/, category: 'customField'}, {docKey: 'T =', dbKey: 'temperature', regex: /T = (\S+)/, category: 'customField'}, {docKey: 'nicht vorgealtert', dbKey: 'not preaged', regex: /nicht vorgealtert (.*)/, category: 'customField'}, {docKey: 'TS119', dbKey: 'TS119', regex: /TS119 (W\S+);/, category: 'customField'}, {docKey: 'GF vom Datenblatt', dbKey: 'glass fibre from data sheet', regex: null, category: 'customField'}, {docKey: 'nach Datensatz', dbKey: 'according to dataset', regex: null, category: 'customField'}, {docKey: 'Dosiergeschw', dbKey: 'metering speed', regex: /Dosiergeschw.*? (.*?min)/, category: 'customField'}, {docKey: 'Einspritzgeschw', dbKey: 'injection speed', regex: /Einspritzgeschw.*? (.*\/s)/, category: 'customField'}, {docKey: 'Heizbänder', dbKey: 'heating lines', regex: /Heizbänder (.*)/, category: 'customField'}, {docKey: 'Verweilzeit', dbKey: 'dwell time', regex: /Verweilzeit (.*?min)/, category: 'customField'}, {docKey: 'Probe', dbKey: 'belongs to', regex: /Probe (\S*\d+)/, category: 'reference'}, {docKey: 'zu', dbKey: 'belongs to', regex: /zu (\S*\d+)/, category: 'reference'}, {docKey: 'granulate zu', dbKey: 'granulate to', regex: /granulate zu.* (\S*\d+)/, category: 'reference'}, {docKey: 'construction part', dbKey: 'construction part', regex: /(? { if (comment.indexOf(cField.docKey) >= 0) { // comment contains docKey if (cField.regex !== null) { const regexRes = cField.regex.exec(comment); if (regexRes) { usedParts.push(regexRes[0]); if (cField.category === 'reference') { sampleReferences.push({sample: sampleNumber, referencedSample: regexRes[1], relation: cField.dbKey}); } else if (cField.category === 'vn') { vnValues[sampleNumber] = regexRes[1]; } else { res[cField.dbKey] = regexRes.filter((e, i) => i > 0).join(' '); } } } else { usedParts.push(cField.docKey); res[cField.dbKey] = true; } } }); usedParts.forEach(part => { const index = comment.indexOf(part); if (index >= 0) { comment = comment.slice(0, index) + comment.slice(index + part.length); } }); if (/\w+/.test(comment)) { res.xRest = comment; } return res; } function sampleType (type) { type = stripSpaces(type).toLowerCase(); const allowedTypes = {'tension rod': 'tension rod', 'Zugstab': 'tension rod', 'part': 'part', 'granulate': 'granulate'}; if (!allowedTypes[type]) { typeLog.push(type); } return allowedTypes[type] ? allowedTypes[type] : 'part'; } function stripSpaces(s) { return s ? s.replace(/ /g,'') : ''; } function trim(s) { return s.replace(/(^\s+|\s+$)/gm, ''); }