1213 lines
49 KiB
JavaScript
1213 lines
49 KiB
JavaScript
const csv = require('csv-parser');
|
|
const fs = require('fs');
|
|
const axios = require('axios');
|
|
const {Builder} = require('selenium-webdriver'); // selenium and the chrome driver must be installed and configured separately
|
|
const chrome = require('selenium-webdriver/chrome');
|
|
const pdfReader = require('pdfreader');
|
|
const iconv = require('iconv-lite');
|
|
const _ = require('lodash');
|
|
|
|
const stages = {
|
|
materials: true,
|
|
samples: true,
|
|
dpt: true,
|
|
dptFromList: false // ignores dpts not on the dptList
|
|
}
|
|
|
|
const dptList = [
|
|
'Eh01_Eh3001_01_csv.DPT',
|
|
'Eh01_Eh3001_02_csv.DPT',
|
|
'Eh01_Eh3001_03_csv.DPT',
|
|
'Eh01_Eh3002_01_csv.DPT',
|
|
'Eh01_Eh3002_02_csv.DPT',
|
|
'Eh01_Eh3002_03_csv.DPT',
|
|
'Eh01_Eh3003_01_csv.DPT',
|
|
'Eh01_Eh3003_02_csv.DPT',
|
|
'Eh01_Eh3003_03_csv.DPT',
|
|
'Eh01_Eh3004_01_csv.DPT',
|
|
'Eh01_Eh3004_02_csv.DPT',
|
|
'Eh01_Eh3004_03_csv.DPT',
|
|
'Eh01_Eh3005_01_csv.DPT',
|
|
'Eh01_Eh3005_02_csv.DPT',
|
|
'Eh01_Eh3005_03_csv.DPT',
|
|
'Eh01_Eh3005_04_csv.DPT',
|
|
'Eh01_Eh3005_05_csv.DPT',
|
|
'Eh01_Eh3005_06_csv.DPT',
|
|
'Eh01_Eh3005_07_csv.DPT',
|
|
'Eh01_Eh3005_08_csv.DPT',
|
|
'Eh01_Eh3005_09_csv.DPT',
|
|
'Eh01_Eh3006_01_csv.DPT',
|
|
'Eh01_Eh3006_02_csv.DPT',
|
|
'Eh01_Eh3006_03_csv.DPT',
|
|
'Eh01_Eh3007_01_csv.DPT',
|
|
'Eh01_Eh3007_02_csv.DPT',
|
|
'Eh01_Eh3007_03_csv.DPT',
|
|
'Eh01_Eh3008_01_csv.DPT',
|
|
'Eh01_Eh3008_02_csv.DPT',
|
|
'Eh01_Eh3008_03_csv.DPT',
|
|
'Eh01_Eh3009_01_csv.DPT',
|
|
'Eh01_Eh3009_02_csv.DPT',
|
|
'Eh01_Eh3009_03_csv.DPT',
|
|
'Eh01_Eh3010_01_csv.DPT',
|
|
'Eh01_Eh3010_02_csv.DPT',
|
|
'Eh01_Eh3010_03_csv.DPT',
|
|
'Eh01_Eh3011_01_csv.DPT',
|
|
'Eh01_Eh3011_02_csv.DPT',
|
|
'Eh01_Eh3011_03_csv.DPT',
|
|
'Eh01_Eh3012_01_csv.DPT',
|
|
'Eh01_Eh3012_02_csv.DPT',
|
|
'Eh01_Eh3012_03_csv.DPT',
|
|
'Eh01_Eh3013_01_csv.DPT',
|
|
'Eh01_Eh3013_02_csv.DPT',
|
|
'Eh01_Eh3013_03_csv.DPT',
|
|
'Eh01_Eh3013_04_csv.DPT',
|
|
'Eh01_Eh3013_05_csv.DPT',
|
|
'Eh01_Eh3013_06_csv.DPT',
|
|
'Eh01_Eh3013_07_csv.DPT',
|
|
'Eh01_Eh3013_08_csv.DPT',
|
|
'Eh01_Eh3013_09_csv.DPT',
|
|
'Eh01_Eh3014_01_csv.DPT',
|
|
'Eh01_Eh3014_02_csv.DPT',
|
|
'Eh01_Eh3014_03_csv.DPT',
|
|
'Eh01_Eh3015_01_csv.DPT',
|
|
'Eh01_Eh3015_02_csv.DPT',
|
|
'Eh01_Eh3015_03_csv.DPT',
|
|
'Eh01_Eh3016_01_csv.DPT',
|
|
'Eh01_Eh3016_02_csv.DPT',
|
|
'Eh01_Eh3016_03_csv.DPT',
|
|
'Eh01_Eh3017_01_csv.DPT',
|
|
'Eh01_Eh3017_02_csv.DPT',
|
|
'Eh01_Eh3017_03_csv.DPT',
|
|
'Eh01_Eh3018_01_csv.DPT',
|
|
'Eh01_Eh3018_02_csv.DPT',
|
|
'Eh01_Eh3018_03_csv.DPT',
|
|
'Eh01_Eh3019_01_csv.DPT',
|
|
'Eh01_Eh3019_02_csv.DPT',
|
|
'Eh01_Eh3019_03_csv.DPT',
|
|
'Eh01_Eh3020_01_csv.DPT',
|
|
'Eh01_Eh3020_02_csv.DPT',
|
|
'Eh01_Eh3020_03_csv.DPT',
|
|
'Eh01_Eh3021_01_csv.DPT',
|
|
'Eh01_Eh3021_02_csv.DPT',
|
|
'Eh01_Eh3021_03_csv.DPT',
|
|
'Eh01_Eh3022_01_csv.DPT',
|
|
'Eh01_Eh3022_02_csv.DPT',
|
|
'Eh01_Eh3022_03_csv.DPT',
|
|
'Eh01_Eh4001_01_csv.DPT',
|
|
'Eh01_Eh4001_02_csv.DPT',
|
|
'Eh01_Eh4001_03_csv.DPT',
|
|
'Eh01_Eh4001_04_csv.DPT',
|
|
'Eh01_Eh4001_05_csv.DPT',
|
|
'Eh01_Eh4001_06_csv.DPT',
|
|
'Eh01_Eh4001_07_csv.DPT',
|
|
'Eh01_Eh4001_08_csv.DPT',
|
|
'Eh01_Eh4001_09_csv.DPT',
|
|
'Eh01_Eh4002_01_csv.DPT',
|
|
'Eh01_Eh4002_02_csv.DPT',
|
|
'Eh01_Eh4002_03_csv.DPT',
|
|
'Eh01_Eh4003_01_csv.DPT',
|
|
'Eh01_Eh4003_02_csv.DPT',
|
|
'Eh01_Eh4003_03_csv.DPT',
|
|
'Eh01_Eh4003_04_csv.DPT',
|
|
'Eh01_Eh4003_05_csv.DPT',
|
|
'Eh01_Eh4003_06_csv.DPT',
|
|
'Eh01_Eh4003_07_csv.DPT',
|
|
'Eh01_Eh4003_08_csv.DPT',
|
|
'Eh01_Eh4003_09_csv.DPT',
|
|
'Eh01_Eh4004_01_csv.DPT',
|
|
'Eh01_Eh4004_02_csv.DPT',
|
|
'Eh01_Eh4004_03_csv.DPT',
|
|
'Eh01_Eh4004_04_csv.DPT',
|
|
'Eh01_Eh4004_05_csv.DPT',
|
|
'Eh01_Eh4004_06_csv.DPT',
|
|
'Eh01_Eh4004_07_csv.DPT',
|
|
'Eh01_Eh4004_08_csv.DPT',
|
|
'Eh01_Eh4004_09_csv.DPT',
|
|
'Eh01_Eh4005_01_csv.DPT',
|
|
'Eh01_Eh4005_02_csv.DPT',
|
|
'Eh01_Eh4005_03_csv.DPT',
|
|
'Eh01_Eh4005_04_csv.DPT',
|
|
'Eh01_Eh4005_05_csv.DPT',
|
|
'Eh01_Eh4005_06_csv.DPT',
|
|
'Eh01_Eh4005_07_csv.DPT',
|
|
'Eh01_Eh4005_08_csv.DPT',
|
|
'Eh01_Eh4005_09_csv.DPT',
|
|
'Eh01_Eh4006_01_csv.DPT',
|
|
'Eh01_Eh4006_02_csv.DPT',
|
|
'Eh01_Eh4006_03_csv.DPT',
|
|
'Eh01_Eh4006_04_csv.DPT',
|
|
'Eh01_Eh4006_05_csv.DPT',
|
|
'Eh01_Eh4006_06_csv.DPT',
|
|
'Eh01_Eh4006_07_csv.DPT',
|
|
'Eh01_Eh4006_08_csv.DPT',
|
|
'Eh01_Eh4006_09_csv.DPT',
|
|
'Eh01_Eh4007_01_csv.DPT',
|
|
'Eh01_Eh4007_02_csv.DPT',
|
|
'Eh01_Eh4007_03_csv.DPT',
|
|
'Eh01_Eh4007_04_csv.DPT',
|
|
'Eh01_Eh4007_05_csv.DPT',
|
|
'Eh01_Eh4007_06_csv.DPT',
|
|
'Eh01_Eh4007_07_csv.DPT',
|
|
'Eh01_Eh4007_08_csv.DPT',
|
|
'Eh01_Eh4007_09_csv.DPT',
|
|
'Eh01_Eh4008_01_csv.DPT',
|
|
'Eh01_Eh4008_02_csv.DPT',
|
|
'Eh01_Eh4008_03_csv.DPT',
|
|
'Eh01_Eh4008_04_csv.DPT',
|
|
'Eh01_Eh4008_05_csv.DPT',
|
|
'Eh01_Eh4008_06_csv.DPT',
|
|
'Eh01_Eh4008_07_csv.DPT',
|
|
'Eh01_Eh4008_08_csv.DPT',
|
|
'Eh01_Eh4008_09_csv.DPT',
|
|
'Eh01_Eh4009_01_csv.DPT',
|
|
'Eh01_Eh4009_02_csv.DPT',
|
|
'Eh01_Eh4009_03_csv.DPT',
|
|
'Eh01_Eh4009_04_csv.DPT',
|
|
'Eh01_Eh4009_05_csv.DPT',
|
|
'Eh01_Eh4009_06_csv.DPT',
|
|
'Eh01_Eh4009_07_csv.DPT',
|
|
'Eh01_Eh4009_08_csv.DPT',
|
|
'Eh01_Eh4009_09_csv.DPT',
|
|
'Eh01_Eh4010_01_csv.DPT',
|
|
'Eh01_Eh4010_02_csv.DPT',
|
|
'Eh01_Eh4010_03_csv.DPT',
|
|
'Eh01_Eh4010_04_csv.DPT',
|
|
'Eh01_Eh4010_05_csv.DPT',
|
|
'Eh01_Eh4010_06_csv.DPT',
|
|
'Eh01_Eh4010_07_csv.DPT',
|
|
'Eh01_Eh4010_08_csv.DPT',
|
|
'Eh01_Eh4010_09_csv.DPT',
|
|
'Eh01_Eh4011_01_csv.DPT',
|
|
'Eh01_Eh4011_02_csv.DPT',
|
|
'Eh01_Eh4011_03_csv.DPT',
|
|
'Eh01_Eh4011_04_csv.DPT',
|
|
'Eh01_Eh4011_05_csv.DPT',
|
|
'Eh01_Eh4011_06_csv.DPT',
|
|
'Eh01_Eh4011_07_csv.DPT',
|
|
'Eh01_Eh4011_08_csv.DPT',
|
|
'Eh01_Eh4011_09_csv.DPT'
|
|
];
|
|
|
|
const docs = [
|
|
"C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\DPPD_DPPD-frei.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata__AnP2.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata__AnP2_A.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata__AnP2_B.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_Ap.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_Bj.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_Eh.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_Eh_B.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_Eh_Duroplasten.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_Rng_aktuell.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_Rng_aktuell_A.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_Rng_aktuell_B.csv",
|
|
// "C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\Metadata_WaP.csv",
|
|
];
|
|
const errors = [];
|
|
const nmDocs = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\nmDocs'; // NormMaster Documents
|
|
// const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\DPT'; // Spectrum files
|
|
const dptFiles = 'C:\\Users\\vle2fe\\Documents\\Data\\All_200807\\DPT_DPPD'; // Spectrum files
|
|
const host = 'http://localhost:3000';
|
|
// const host = 'https://definma-api.apps.de1.bosch-iot-cloud.com';
|
|
const requiredProperties = ['samplenumber','materialnumber','materialname','supplier','reinforcementmaterial','material','granulate/part','color','charge/batch','comments'];
|
|
dict = { // dictionary
|
|
'Granulat': 'granulate',
|
|
'Zugstab': 'tension rod',
|
|
'Stecker': 'plug'
|
|
};
|
|
let data = []; // metadata contents
|
|
let materials = {};
|
|
let numberToColor = {};
|
|
let samples = [];
|
|
let normMaster = {};
|
|
let sampleDevices = {};
|
|
const sampleReferences = []; // references to other samples in format {sample, referencedSample, relation}
|
|
let commentsLog = [];
|
|
let customFieldsLog = [];
|
|
const vnValues = {}; // vn values from comments
|
|
const dptLog = [];
|
|
const dptSampleAddLog = []; // log samples created during dpt insertion
|
|
const typeLog = [];
|
|
|
|
const duplicateMNames = [
|
|
"Badamid LA70 GF30 TM-Z2",
|
|
"Latamid 66 H2 G30",
|
|
"Latamid 66 H2 G35",
|
|
"Latamid 66 H2 G50",
|
|
"Radilon ARV350KB",
|
|
"Schulamid 66 GF 25H",
|
|
"Schulamid 66 GF25 H",
|
|
"Ultradur B 4330 G6 HR",
|
|
"Ultradur B 4300 G10",
|
|
"Ultradur B 4300 G 4",
|
|
"Ultradur B 4300 G6",
|
|
"Ultradur B 4315 G6 HR",
|
|
"Ultraform N 2200 G 43",
|
|
"Ultraform N 2200 G 43 AT",
|
|
"Ultrason E 2010 G6",
|
|
"Zytel 70G30HSLR"
|
|
];
|
|
const duplicateMNamesLog = [];
|
|
// TODO: conditions
|
|
|
|
main();
|
|
|
|
async function main() {
|
|
if (stages.materials) { // materials
|
|
await getNormMaster();
|
|
for (let i in docs) {
|
|
await importCsv(docs[i]);
|
|
await allMaterials();
|
|
await saveMaterials();
|
|
}
|
|
fs.writeFileSync('./data_import/numberToColor.json', JSON.stringify(numberToColor));
|
|
fs.writeFileSync('./data_import/duplicateMNames.json', JSON.stringify(duplicateMNamesLog));
|
|
}
|
|
if (stages.samples) { // samples
|
|
sampleDeviceMap();
|
|
numberToColor = JSON.parse(fs.readFileSync('./data_import/numberToColor.json'), 'utf-8');
|
|
for (let i in docs) {
|
|
await importCsv(docs[i]);
|
|
await allSamples();
|
|
await saveSamples();
|
|
await allMcVn();
|
|
}
|
|
// write logs
|
|
fs.writeFileSync('./data_import/comments.txt', commentsLog.join('\r\n'));
|
|
fs.writeFileSync('./data_import/typeLog.txt', typeLog.join('\r\n'));
|
|
fs.writeFileSync('./data_import/customFields.txt', customFieldsLog.join('\r\n'));
|
|
fs.writeFileSync('./data_import/sampleReferences.txt', sampleReferences.map(e => JSON.stringify(e)).join('\r\n'));
|
|
fs.writeFileSync('./data_import/sampleReferences.json', JSON.stringify(sampleReferences));
|
|
|
|
await sampleReferencesSave();
|
|
}
|
|
if (stages.dpt) { // DPT
|
|
await allDpts();
|
|
fs.writeFileSync('./data_import/sdptLog.txt', dptLog.join('\r\n'));
|
|
fs.writeFileSync('./data_import/dptSampleAddLog.txt', dptSampleAddLog.join('\r\n'));
|
|
}
|
|
if (0) { // pdf test
|
|
console.log(await readPdf('N28_BN05-OX023_2019-07-16.pdf'));
|
|
}
|
|
if (errors.length) {
|
|
// console.log(errors);
|
|
fs.writeFileSync('./data_import/errors/errors_' + new Date().getTime() + '.txt', errors.join('\r\n'));
|
|
}
|
|
}
|
|
|
|
async function importCsv(doc) {
|
|
// Uniform name samplenumber materialnumber materialname supplier material plastic reinforcingmaterial granulate/part color charge/batch comments vz(ml/g) kfingew% degradation(%) reinforcingmaterialcontent stabwn
|
|
// Metadata__AnP2.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,granulate/Part,Color,Charge/ Batch, Comments
|
|
// Metadata__AnP2_A.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part, Comments, Humidity [ppm]
|
|
// Metadata__AnP2_B.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part, VZ [ml/g], glass fibre content
|
|
// Metadata_Ap.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part,Color,Charge/Batch, Comments
|
|
// Metadata_Bj.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Color,Charge/batch granulate/part,Comments
|
|
// Metadata_Eh.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material,Granulate/Part,Color,Charge/Batch granulate/part,Comments, VZ [cm³/g], Spalte1
|
|
// Metadata_Eh_B.csv Sample number, Material name,Supplier, Plastic,Reinforcing material,Granulate/Part,Color, Comments, VZ [cm³/g]
|
|
// Metadata_Eh_Duroplasten.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material,Granulate/Part,Color,Charge/Batch granulate/part,Comments
|
|
// Metadata_Rng_aktuell.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Color,Charge/batch granulate/part,Comments, VZ (ml/g), Degradation(%),Glas fibre content (%)
|
|
// Metadata_Rng_aktuell_A.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Farbe,Charge/batch granulate/part,Comments, KF in Gew%, Reinforcing material (content in %),Stabwn
|
|
// Metadata_Rng_aktuell_B.csv Sample number, Material name,Supplier, Plastic, Granulate/Part, Comments, VZ (ml/g), Degradation (%), Alterungszeit in h
|
|
// Metadata_WaP.csv Probennummer, Name, Firma, Material, Teil/Rohstoff, Charge, Anmerkung,VZ (ml/g), Abbau (%), Verstärkungsstoffgehalt (%), Versuchsnummer
|
|
const nameCorrection = { // map to right column names
|
|
'probennummer': 'samplenumber',
|
|
'name': 'materialname',
|
|
'firma': 'supplier',
|
|
'teil/rohstoff': 'granulate/part',
|
|
'charge/batchgranulate/part': 'charge/batch',
|
|
'charge': 'charge/batch',
|
|
'anmerkung': 'comments',
|
|
'vz': 'vz(ml/g)',
|
|
'vz[ml/g]': 'vz(ml/g)',
|
|
'vz[cm³/g]': 'vz(ml/g)',
|
|
'abbau(%)': 'degradation(%)',
|
|
'glassfibrecontent': 'reinforcingmaterialcontent',
|
|
'glasfibrecontent(%)': 'reinforcingmaterialcontent',
|
|
'reinforcingmaterial(contentin%)': 'reinforcingmaterialcontent',
|
|
'verstärkungsstoffgehalt(%)': 'reinforcingmaterialcontent'
|
|
};
|
|
const missingFieldsFill = [ // column names to fill if they do not exist
|
|
'color',
|
|
'charge/batch',
|
|
'comments',
|
|
'materialnumber',
|
|
'reinforcementmaterial'
|
|
]
|
|
console.info('importing ' + doc);
|
|
data = [];
|
|
await new Promise(resolve => {
|
|
fs.createReadStream(doc)
|
|
.pipe(iconv.decodeStream('win1252'))
|
|
.pipe(csv())
|
|
.on('data', (row) => {
|
|
data.push(row);
|
|
})
|
|
.on('end', () => {
|
|
data = data.map(e => {
|
|
const newE = {};
|
|
Object.keys(e).forEach(key => {
|
|
newE[key.toLowerCase().replace(/ /g, '')] = e[key];
|
|
});
|
|
// replace wrong column names
|
|
Object.keys(newE).forEach(key => {
|
|
if (nameCorrection.hasOwnProperty(key)) {
|
|
newE[nameCorrection[key]] = newE[key];
|
|
delete newE[key];
|
|
}
|
|
});
|
|
|
|
// add missing fields with empty values
|
|
missingFieldsFill.forEach(field => {
|
|
if (!newE.hasOwnProperty(field)) {
|
|
newE[field] = '';
|
|
}
|
|
});
|
|
if(newE['materialname'] === '') {
|
|
newE['materialname'] = newE['material'];
|
|
}
|
|
if (newE['supplier'] === '') { // empty supplier fields
|
|
newE['supplier'] = 'unknown';
|
|
}
|
|
if (!newE.hasOwnProperty('material')) {
|
|
newE['material'] = newE['plastic'].indexOf(' GF') >= 0 ? newE['plastic'].split(' ')[0] : newE['plastic'];
|
|
}
|
|
return newE;
|
|
}).filter(e => {
|
|
const missingProperties = requiredProperties.filter(el => !e.hasOwnProperty(el));
|
|
if (e['materialname'] === '') {
|
|
missingProperties.push('materialname');
|
|
}
|
|
if (e['samplenumber'] === '') { // empty row
|
|
return false;
|
|
}
|
|
else if (missingProperties.length > 0) { // incomplete sample
|
|
errors.push(`${doc}: ${JSON.stringify(e)}is missing the required properties ${missingProperties}`);
|
|
return false;
|
|
}
|
|
return true;
|
|
});
|
|
console.info('CSV file successfully processed');
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
async function allDpts() {
|
|
let res = await axios({
|
|
method: 'get',
|
|
url: host + '/template/measurements',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not fetch /template/measurements: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
const measurement_templates = res.data.filter(e => e.name === 'spectrum');
|
|
const measurement_template = measurement_templates[measurement_templates.length - 1]._id;
|
|
res = await axios({
|
|
method: 'get',
|
|
url: host + '/samples?status[]=validated&status[]=new',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not fetch /samples?status[]=validated&status[]=new: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
const sampleIds = {};
|
|
res.data.forEach(sample => {
|
|
sampleIds[sample.number] = sample._id;
|
|
});
|
|
const dptRegex = /(.*?)_(.*?)_(\d+|[a-zA-Z0-9]+[_.]\d+)(_JDX|_csv)?[.]{1,2}(DPT|csv|CSV|JDX)/;
|
|
const dpts = fs.readdirSync(dptFiles);
|
|
for (let i in dpts) {
|
|
if (!stages.dptFromList || dptList.indexOf(dpts[i]) >= 0) { // if list option active, only use from filter
|
|
let regexInput;
|
|
const bjRes = /^(Bj[FT]?)\s?([a-z0-9_]*)_JDX.DPT/.exec(dpts[i]);
|
|
if (bjRes) { // correct Bj numbers with space
|
|
regexInput = `Bj01_${bjRes[1]}${bjRes[2]}_0.DPT`;
|
|
}
|
|
else { // remove _JDX from name
|
|
regexInput = dpts[i].replace(/_JDX.*\./, '.');
|
|
}
|
|
const regexRes = dptRegex.exec(regexInput);
|
|
if (regexRes && !sampleIds[regexRes[2]]) { // when sample number includes an additional _x instead of having _x_x for spectrum description
|
|
regexRes[2] = `${regexRes[2]}_${regexRes[3].split('_')[0]}`;
|
|
}
|
|
let baseSample = null;
|
|
if (regexRes) {
|
|
baseSample = regexRes[2].split('_')[0];
|
|
if (baseSample === 'Wa11') { // as Wa11 samples use all the same material
|
|
baseSample = 'Wa11_B0_1';
|
|
}
|
|
}
|
|
if (regexRes && !sampleIds[regexRes[2]] && sampleIds[baseSample]) { // when number_abx does not exist but number
|
|
dptSampleAddLog.push(`Trying to find ${baseSample}`);
|
|
dptSampleAddLog.push(host + '/sample/' + sampleIds[baseSample]);
|
|
res = await axios({ // get base sample
|
|
method: 'get',
|
|
url: host + '/sample/' + stripSpaces(sampleIds[baseSample]),
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`DPT Could not fetch sample ${baseSample}: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
if (res) {
|
|
const data = _.merge(_.pick(res.data, ['color', 'type', 'batch']),
|
|
{number: regexRes[2], condition: {}, notes: {}, material_id: res.data.material._id});
|
|
res = await axios({
|
|
method: 'post',
|
|
url: host + '/sample/new',
|
|
auth: {
|
|
username: res.data.user,
|
|
password: res.data.user === 'admin' ? 'Abc123!#' : '2020DeFinMachen!'
|
|
},
|
|
data
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`DPT Could not save sample ${data}: ${err.response.data}`);
|
|
}
|
|
});
|
|
if (res.data) {
|
|
dptSampleAddLog.push(`${regexRes[2]} from ${baseSample}`)
|
|
sampleIds[regexRes[2]] = res.data._id;
|
|
}
|
|
}
|
|
}
|
|
if (regexRes && sampleIds[regexRes[2]]) { // found matching sample
|
|
console.log(`${i}/${dpts.length} ${dpts[i]} -> ${regexRes[2]}`);
|
|
dptLog.push(`${dpts[i]}, ${regexRes[2]}`);
|
|
const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8');
|
|
const data = {
|
|
sample_id: sampleIds[regexRes[2]],
|
|
values: {},
|
|
measurement_template
|
|
};
|
|
data.values.device = regexRes[1];
|
|
data.values.filename = dpts[i];
|
|
data.values.dpt = f.split('\r\n').map(e => e.split(',').map(e => parseFloat(e)));
|
|
let rescale = false;
|
|
for (let i in data.values.dpt) {
|
|
if (data.values.dpt[i][1] > 10) {
|
|
rescale = true;
|
|
break;
|
|
}
|
|
}
|
|
if (rescale) {
|
|
data.values.dpt = data.values.dpt.map(e => [e[0], e[1] / 100]);
|
|
}
|
|
await axios({
|
|
method: 'post',
|
|
url: host + '/measurement/new',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
},
|
|
data
|
|
}).catch(err => {
|
|
console.log(dpts[i]);
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not upload ${dpts[i]} for sample ${regexRes[2]}: ${err.response.data}`);
|
|
}
|
|
else {
|
|
console.error(err);
|
|
errors.push(`Could not upload ${dpts[i]} for sample ${regexRes[2]}: ${JSON.stringify(err)}`);
|
|
}
|
|
});
|
|
}
|
|
else {
|
|
console.log(`Could not find sample for ${dpts[i]}`);
|
|
if (regexRes) {
|
|
errors.push(`Could not find sample for ${dpts[i]}; [DEBUG] ${regexRes[2]}, ${!sampleIds[regexRes[2]]}, ${sampleIds[baseSample]}`);
|
|
}
|
|
else {
|
|
errors.push(`Could not find sample for ${dpts[i]} (did not match RegEx)`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function allMcVn() {
|
|
let res = await axios({
|
|
method: 'get',
|
|
url: host + '/template/measurements',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not fetch /template/measurements: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
const mc_template = res.data.filter(e => e.name === 'moisture content').sort((a, b) => b.version - a.version)[0]._id;
|
|
const vn_template = res.data.filter(e => e.name === 'vn').sort((a, b) => b.version - a.version)[0]._id;
|
|
const rmc_template = res.data.filter(e => e.name === 'reinforcement material content').sort((a, b) => b.version - a.version)[0]._id;
|
|
res = await axios({
|
|
method: 'get',
|
|
url: host + '/samples?status[]=validated&status[]=new',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not fetch /samples?status[]=validated&status[]=new: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
const sampleIds = {};
|
|
res.data.forEach(sample => {
|
|
sampleIds[sample.number] = sample._id;
|
|
});
|
|
for (let index in data) {
|
|
console.info(`MC/VN ${index}/${data.length}`);
|
|
let sample = data[index];
|
|
console.log(sample);
|
|
sample['samplenumber'] = sample['samplenumber'].replace(/[A-Z][a-z]0\d_/, '');
|
|
let credentials = ['admin', 'Abc123!#'];
|
|
if (sampleDevices[sample['samplenumber']]) {
|
|
credentials = [sampleDevices[sample['samplenumber']], '2020DeFinMachen!']
|
|
}
|
|
if (!sample['vz(ml/g)'] && vnValues[sample['samplenumber']]) { // fill in VN values from comments
|
|
sample['vz(ml/g)'] = vnValues[sample['samplenumber']];
|
|
}
|
|
if (sample['kfingew%']) {
|
|
await axios({
|
|
method: 'post',
|
|
url: host + '/measurement/new',
|
|
auth: {
|
|
username: credentials[0],
|
|
password: credentials[1]
|
|
},
|
|
data: {
|
|
sample_id: sampleIds[sample['samplenumber']],
|
|
measurement_template: mc_template,
|
|
values: {
|
|
'weight %': sample['kfingew%'],
|
|
'standard deviation': sample['stabwn']
|
|
}
|
|
}
|
|
}).catch(err => {
|
|
console.log(sample['samplenumber']);
|
|
console.error(err.response.data);
|
|
errors.push(`MC/VN upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`);
|
|
});
|
|
}
|
|
if (sample['vz(ml/g)']) {
|
|
await axios({
|
|
method: 'post',
|
|
url: host + '/measurement/new',
|
|
auth: {
|
|
username: credentials[0],
|
|
password: credentials[1]
|
|
},
|
|
data: {
|
|
sample_id: sampleIds[sample['samplenumber']],
|
|
measurement_template: vn_template,
|
|
values: {
|
|
vn: sample['vz(ml/g)']
|
|
}
|
|
}
|
|
}).catch(err => {
|
|
console.log(sample['samplenumber']);
|
|
console.error(err.response.data);
|
|
errors.push(`MC/VN upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`);
|
|
});
|
|
}
|
|
if (sample['reinforcingmaterialcontent']) {
|
|
await axios({
|
|
method: 'post',
|
|
url: host + '/measurement/new',
|
|
auth: {
|
|
username: credentials[0],
|
|
password: credentials[1]
|
|
},
|
|
data: {
|
|
sample_id: sampleIds[sample['samplenumber']],
|
|
measurement_template: rmc_template,
|
|
values: {
|
|
percentage: Number(sample['reinforcingmaterialcontent'].replace('%', '').replace(',', '.'))
|
|
}
|
|
}
|
|
}).catch(err => {
|
|
console.log(sample['samplenumber']);
|
|
console.error(err.response.data);
|
|
errors.push(`MC/VN upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`);
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
async function allSamples() {
|
|
samples = [];
|
|
let res = await axios({
|
|
method: 'get',
|
|
url: host + '/materials?status=all',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not fetch /materials?status=all: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
const dbMaterials = {}
|
|
res.data.forEach(m => {
|
|
m.numbers = m.numbers.map(e => ({number: e, color: numberToColor[e]}));
|
|
dbMaterials[m.name] = m;
|
|
})
|
|
res = await axios({
|
|
method: 'get',
|
|
url: host + '/samples?status[]=validated&status[]=new',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not fetch /samples?status[]=validated&status[]=new: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
const sampleColors = {};
|
|
res.data.forEach(sample => {
|
|
sampleColors[sample.number] = sample.color;
|
|
});
|
|
|
|
|
|
for (let index in data) {
|
|
console.info(`SAMPLE LOAD ${index}/${data.length}`);
|
|
let sample = data[index];
|
|
if (sample['granulate/Part'] === '') { // empty supplier fields
|
|
sample['granulate/Part'] = 'unknown';
|
|
}
|
|
const material = dbMaterials[trim(sample['materialname'])];
|
|
if (!material) { // could not find material, skipping sample
|
|
errors.push(`Could not find a material for ${JSON.stringify(sample)}`);
|
|
continue;
|
|
}
|
|
samples.push({
|
|
number: sample['samplenumber'].replace(/[A-Z][a-z]0\d_/, ''), // remove leading An_01 and Eh_01
|
|
type: sampleType(sample['granulate/part']),
|
|
batch: sample['charge/batch'],
|
|
material_id: material._id,
|
|
notes: {
|
|
custom_fields: customFields(sample['comments'], sample['samplenumber'])
|
|
}
|
|
});
|
|
// if (sample['comments']) {
|
|
// comments.push(sample['samplenumber'] + ' ' + sample['comments']);
|
|
// }
|
|
const si = samples.length - 1; // sample index
|
|
if (samples[si].notes.custom_fields.hasOwnProperty('xRest')) { // reroute xRest property to comment
|
|
samples[si].notes.comment = samples[si].notes.custom_fields.xRest;
|
|
commentsLog.push(sample['samplenumber'] + ' ' + samples[si].notes.comment);
|
|
delete samples[si].notes.custom_fields.xRest;
|
|
}
|
|
if (Object.keys(samples[si].notes.custom_fields).length === 0) { // delete empty custom fields
|
|
delete samples[si].notes.custom_fields;
|
|
}
|
|
else {
|
|
customFieldsLog.push(sample['samplenumber'] + ' ' + JSON.stringify(samples[si].notes.custom_fields));
|
|
}
|
|
if (sample['materialnumber'] !== '' && material.numbers.find(e => e.number === sample['materialnumber'])) {
|
|
samples[si].color = material.numbers.find(e => e.number === sample['materialnumber']).color;
|
|
}
|
|
else if (sample['color'] !== '') { // find color with all edge cases
|
|
let number = material.numbers.find(e => e.color && e.color.indexOf(trim(sample['color'])) >= 0);
|
|
if (!number && /black/.test(sample['color'])) { // special case bk for black
|
|
console.log(material);
|
|
number = material.numbers.find(e => e.color && e.color.toLowerCase().indexOf('bk') >= 0);
|
|
if (!number) { // try German word
|
|
number = material.numbers.find(e => e.color && e.color.toLowerCase().indexOf('schwarz') >= 0);
|
|
}
|
|
}
|
|
if (number) {
|
|
samples[si].color = number.color;
|
|
}
|
|
}
|
|
else if (sampleColors[sample['samplenumber'].split('_')[0]]) { // derive color from main sample for mc/vn
|
|
samples[si].color = sampleColors[sample['samplenumber'].split('_')[0]];
|
|
}
|
|
if (!samples[si].color) {
|
|
samples[si].color = sample['color'];
|
|
}
|
|
}
|
|
}
|
|
|
|
async function saveSamples() {
|
|
for (let i in samples) {
|
|
console.info(`SAMPLE SAVE ${i}/${samples.length}`);
|
|
let credentials = ['admin', 'Abc123!#'];
|
|
if (sampleDevices[samples[i].number]) {
|
|
credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!']
|
|
}
|
|
await axios({
|
|
method: 'post',
|
|
url: host + '/sample/new',
|
|
auth: {
|
|
username: credentials[0],
|
|
password: credentials[1]
|
|
},
|
|
data: samples[i]
|
|
}).catch(err => {
|
|
if (err.response.data.status && err.response.data.status !== 'Sample number already taken') {
|
|
console.log(samples[i]);
|
|
console.error(err.response.data);
|
|
errors.push(`Upload for ${JSON.stringify(samples[i])} failed: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
}
|
|
console.info('saved all samples');
|
|
}
|
|
|
|
async function sampleReferencesSave() {
|
|
for (let i in sampleReferences) {
|
|
console.info(`SAMPLE REFERENCES ${i}/${sampleReferences.length}`);
|
|
let refRes = await axios({
|
|
method: 'get',
|
|
url: host + '/sample/number/' + sampleReferences[i].referencedSample,
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
console.log(sampleReferences[i].referencedSample);
|
|
console.error(err.response.data);
|
|
errors.push(`Getting reference id for ${JSON.stringify(sampleReferences[i])} failed: ${JSON.stringify(err.response.data)}`);
|
|
});
|
|
if (!refRes) continue;
|
|
let sampleRes = await axios({
|
|
method: 'get',
|
|
url: host + '/sample/number/' + sampleReferences[i].sample,
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
console.log(sampleReferences[i].sample);
|
|
console.error(err.response.data);
|
|
errors.push(`Getting sample id for ${JSON.stringify(sampleReferences[i])} failed: ${JSON.stringify(err.response.data)}`);
|
|
});
|
|
if (!sampleRes) continue;
|
|
sampleRes.data.notes.sample_references.push({sample_id: refRes.data._id, relation: sampleReferences[i].relation})
|
|
await axios({
|
|
method: 'put',
|
|
url: host + '/sample/' + sampleRes.data._id,
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
},
|
|
data: {notes: {sample_references: sampleRes.data.notes.sample_references}}
|
|
}).catch(err => {
|
|
console.log(sampleRes.data.notes.sample_references);
|
|
if (err.response.data) {
|
|
console.error(err.response.data);
|
|
errors.push(`Saving references for ${JSON.stringify(sampleRes.data)} failed: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
else {
|
|
console.error(err.response);
|
|
errors.push(`Saving references for ${JSON.stringify(sampleRes.data)} failed: ${JSON.stringify(err.response)}`);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
async function allMaterials() {
|
|
// materials = {};
|
|
let res = await axios({
|
|
method: 'get',
|
|
url: host + '/template/materials',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
}
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not fetch /template/materials: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
const materialTemplate = res.data.filter(e => e.name === 'plastic').sort((a, b) => b.version - a.version)[0]._id;
|
|
|
|
// process all samples
|
|
for (let index in data) {
|
|
let sample = data[index];
|
|
if (sample['supplier'] === '') { // empty supplier fields
|
|
sample['supplier'] = 'unknown';
|
|
}
|
|
if (sample['materialname'] === '') { // empty name fields
|
|
sample['materialname'] = sample['material'];
|
|
}
|
|
sample['materialname'] = trim(sample['materialname']);
|
|
if (materials.hasOwnProperty(sample['materialname'])) { // material already found at least once
|
|
if (sample['materialnumber'] !== '') { // material number given
|
|
if (materials[sample['materialname']].numbers.length === 0 || !materials[sample['materialname']].numbers.find(e => e.number === stripSpaces(sample['materialnumber']))) { // new material number
|
|
if (materials[sample['materialname']].numbers.find(e => e.color === sample['color'] && e.number === '')) { // color already in list, only number missing
|
|
materials[sample['materialname']].numbers.find(e => e.color === sample['color'] && e.number === '').number = stripSpaces(sample['materialnumber']);
|
|
}
|
|
else { // completely new number entry
|
|
materials[sample['materialname']].numbers.push({color: trim(sample['color']), number: stripSpaces(sample['materialnumber'])});
|
|
}
|
|
}
|
|
}
|
|
else if (sample['color'] !== '') { // color given
|
|
if (!materials[sample['materialname']].numbers.find(e => e.color === stripSpaces(sample['color']))) { // new material color
|
|
materials[sample['materialname']].numbers.push({color: trim(sample['color']), number: ''});
|
|
}
|
|
}
|
|
}
|
|
else { // new material
|
|
console.info(`MATERIAL LOAD ${index}/${data.length} ${sample['materialname']}`);
|
|
materials[sample['materialname']] = {
|
|
name: trim(sample['materialname']),
|
|
supplier: trim(sample['supplier']),
|
|
group: trim(sample['material'])
|
|
};
|
|
materials[sample['materialname']].numbers = await numbersFetch(sample);
|
|
|
|
// material properties
|
|
materials[sample['materialname']].properties = {material_template: materialTemplate};
|
|
let tmp = /M(\d+)/.exec(sample['reinforcingmaterial']);
|
|
materials[sample['materialname']].properties.mineral = tmp ? tmp[1] : 0;
|
|
tmp = /GF(\d+)/.exec(sample['reinforcingmaterial']);
|
|
materials[sample['materialname']].properties.glass_fiber = tmp ? tmp[1] : 0;
|
|
tmp = /CF(\d+)/.exec(sample['reinforcingmaterial']);
|
|
materials[sample['materialname']].properties.carbon_fiber = tmp ? tmp[1] : 0;
|
|
}
|
|
}
|
|
|
|
// Fill numberToColor array
|
|
Object.keys(materials).forEach(mKey => {
|
|
materials[mKey].numbers.forEach(number => {
|
|
if (number.number && number.color) {
|
|
numberToColor[number.number] = number.color;
|
|
}
|
|
})
|
|
});
|
|
}
|
|
|
|
async function saveMaterials() {
|
|
const mKeys = Object.keys(materials)
|
|
for (let i in mKeys) {
|
|
console.info(`MATERIAL SAVE ${i}/${mKeys.length}`);
|
|
const material = _.cloneDeep(materials[mKeys[i]]);
|
|
material.numbers = material.numbers.map(e => e.number).filter(e => e !== '').map(e => e.replace(/ /g, ''));
|
|
await axios({
|
|
method: 'post',
|
|
url: host + '/material/new',
|
|
auth: {
|
|
username: 'admin',
|
|
password: 'Abc123!#'
|
|
},
|
|
data: material
|
|
}).catch(err => {
|
|
if (err.response.data.status && err.response.data.status !== 'Material name already taken') {
|
|
console.info(material);
|
|
console.error(err.response.data);
|
|
errors.push(`Upload for ${JSON.stringify(material)} failed: ${JSON.stringify(err.response.data)}`)
|
|
}
|
|
});
|
|
}
|
|
console.info('saved all materials');
|
|
}
|
|
|
|
async function numbersFetch(sample) {
|
|
let nm = [];
|
|
let res = [];
|
|
if (sample['materialnumber']) { // sample has a material number
|
|
nm = normMaster[stripSpaces(sample['materialnumber'])]? [normMaster[stripSpaces(sample['materialnumber'])]] : [];
|
|
}
|
|
else { // try finding via material name
|
|
nm = Object.keys(normMaster).filter(e => normMaster[e].nameSpaceless === stripSpaces(sample['materialnumber'])).map(e => normMaster[e]);
|
|
}
|
|
if (nm.length > 0) {
|
|
for (let i in nm) {
|
|
if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded
|
|
await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'));
|
|
}
|
|
// if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded
|
|
// console.info('Retrying download...');
|
|
// await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 2.2);
|
|
// }
|
|
// if (!fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document not loaded
|
|
// console.info('Retrying download again...');
|
|
// await getNormMasterDoc(nm[i].url.replace(/ /g, '%20'), 5);
|
|
// }
|
|
if (fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0)) { // document loaded
|
|
res = await readPdf(fs.readdirSync(nmDocs).find(e => e.indexOf(nm[i].doc.replace(/ /g, '_')) >= 0));
|
|
}
|
|
if (res.length > 0) { // no results
|
|
break;
|
|
}
|
|
else if (i + 1 >= nm.length) {
|
|
errors.push(`Download of ${nm[i].url.replace(/ /g, '%20')} for material number ${sample['materialnumber']} failed`);
|
|
errors.push(nm[i].doc.replace(/ /g, '_'));
|
|
}
|
|
}
|
|
}
|
|
if (res.length === 0) { // no results
|
|
if (sample['color'] !== '' || sample['materialnumber'] !== '') { // information in data available
|
|
return [{color: trim(sample['color']), number: sample['materialnumber']}];
|
|
}
|
|
else {
|
|
return [];
|
|
}
|
|
}
|
|
else {
|
|
if (!res.find(e => e.number === sample['materialnumber'])) { // sometimes norm master does not include sample number even if listed
|
|
res.push({color: trim(sample['color']), number: sample['materialnumber']});
|
|
}
|
|
return res;
|
|
}
|
|
}
|
|
|
|
async function getNormMaster(fetchAgain = false) {
|
|
if (fetchAgain) {
|
|
console.info('fetching norm master...');
|
|
const res = await axios({
|
|
method: 'get',
|
|
url: 'http://rb-normen.bosch.com/cgi-bin/searchRBNorm4TradeName'
|
|
}).catch(err => {
|
|
if (err.response) {
|
|
console.error(err.response.data);
|
|
errors.push(`Could not fetch http://rb-normen.bosch.com/cgi-bin/searchRBNorm4TradeName: ${JSON.stringify(err.response.data)}`);
|
|
}
|
|
});
|
|
|
|
console.info('finding documents...');
|
|
let match;
|
|
// const regex = /<tr>.*?<td>.*?<\/span>(.*?)<\/td><td>(\d+)<\/td>.*?<a href="(.*?)"/gm;
|
|
const regex = /<tr>.*?<td>.*?<\/span>(.*?)<\/td><td>(\d+)<\/td><td>40.*?<a href="(.*?)".*?<\/a>(.*?)<\/td>/gm; // only valid materials
|
|
do {
|
|
match = regex.exec(res.data);
|
|
if (match) {
|
|
normMaster[match[2]] = {name: match[1], nameSpaceless: stripSpaces(match[1]), number: match[2], url: match[3], doc: match[4]};
|
|
}
|
|
} while (match);
|
|
fs.writeFileSync('./data_import/normMaster.json', JSON.stringify(normMaster));
|
|
}
|
|
else {
|
|
normMaster = JSON.parse(fs.readFileSync('./data_import/normMaster.json'), 'utf-8');
|
|
}
|
|
}
|
|
|
|
function getNormMasterDoc(url, timing = 1) {
|
|
console.info(url);
|
|
return new Promise(async resolve => {
|
|
const options = new chrome.Options();
|
|
options.setUserPreferences({
|
|
"download.default_directory": nmDocs,
|
|
"download.prompt_for_download": false,
|
|
"download.directory_upgrade": true,
|
|
"plugins.always_open_pdf_externally": true
|
|
});
|
|
let driver = await new Builder().forBrowser('chrome').setChromeOptions(options).build();
|
|
let timeout = 7000 * timing;
|
|
try {
|
|
await driver.get(url);
|
|
if (await driver.getCurrentUrl() !== 'https://rb-wam-saml.bosch.com/tfim/sps/normmaster/saml20/login') { // got document selection page
|
|
timeout = 11000 * timing;
|
|
await driver.executeScript('Array.prototype.slice.call(document.querySelectorAll(\'.functionlink\')).filter(e => e.innerText === \'English\')[0].click()').catch(() => {timeout = 0; });
|
|
}
|
|
}
|
|
finally {
|
|
setTimeout(async () => { // wait until download is finished
|
|
await driver.quit();
|
|
resolve();
|
|
}, timeout);
|
|
}
|
|
});
|
|
}
|
|
|
|
function readPdf(file) {
|
|
return new Promise(async resolve => {
|
|
const countdown = 100; // value for text timeout
|
|
let table = 0; // > 0 when in correct table area
|
|
let rows = []; // found table rows
|
|
let lastY = 0; // y of last row
|
|
let lastX = 0; // right x of last item
|
|
let lastText = ''; // text of last item
|
|
let lastLastText = ''; // text of last last item
|
|
await new pdfReader.PdfReader().parseFileItems(nmDocs + '\\' + file, (err, item) => {
|
|
if (item && item.text) {
|
|
if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsuppl') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsuppl') >= 0)) { // table area starts
|
|
table = countdown;
|
|
}
|
|
if (table > 0) {
|
|
// console.log(item);
|
|
// console.log(item.y - lastY);
|
|
// console.log(item.text);
|
|
if (item.y - lastY > 0.8 && Math.abs(item.x - lastX) > 5) { // new row
|
|
lastY = item.y;
|
|
rows.push(item.text);
|
|
}
|
|
else { // still the same row row
|
|
rows[rows.length - 1] += (item.x - lastX > 1.09 ? '$' : '') + item.text; // push to row, detect if still same cell
|
|
}
|
|
lastX = (item.w * 0.055) + item.x;
|
|
|
|
if (/\d \d\d\d \d\d\d \d\d\d/.test(item.text)) {
|
|
table = countdown;
|
|
}
|
|
table --;
|
|
if (table <= 0 || item.text.toLowerCase().indexOf('release document') >= 0 || item.text.toLowerCase().indexOf('normative references') >= 0) { // table area ended
|
|
table = -1;
|
|
// console.log(rows);
|
|
rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows
|
|
resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
|
|
}
|
|
}
|
|
lastLastText = lastText;
|
|
lastText = item.text;
|
|
}
|
|
if (!item && table !== -1) { // document ended
|
|
rows = rows.filter(e => /^\d{10}/m.test(stripSpaces(e))); // filter non-table rows
|
|
resolve(rows.map(e => {return {color: trim(e.split('$')[3]), number: stripSpaces(e.split('$')[0])}; }));
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
function sampleDeviceMap() {
|
|
const dpts = fs.readdirSync(dptFiles);
|
|
const regex = /(.*?)_(.*?)_(\d+|[^_]+_\d+).DPT/;
|
|
for (let i in dpts) {
|
|
const regexRes = regex.exec(dpts[i])
|
|
if (regexRes) { // found matching sample
|
|
sampleDevices[regexRes[2]] = regexRes[1] === 'plastics' ? 'rng01' : regexRes[1].toLowerCase();
|
|
}
|
|
}
|
|
}
|
|
|
|
function customFields (comment, sampleNumber) {
|
|
const customFields = [
|
|
{docKey: 'Versuchsreihe', dbKey: 'test series', regex: /Versuchsreihe (\d+),/, category: 'customField'},
|
|
{docKey: 'Stillstand', dbKey: 'idle', regex: /Stillstand (\d+ min):/, category: 'customField'},
|
|
{docKey: 'Serienzyklus', dbKey: 'cycle', regex: /(\d+.) Serienzyklus (\(.*?\))/, category: 'customField'},
|
|
{docKey: 'Berstdruck', dbKey: 'bursting pressure', regex: /Berstdruck: (.*?bar);/, category: 'customField'},
|
|
{docKey: 'gemessen am', dbKey: 'measured at', regex: /gemessen am (.*20\d\d)/, category: 'customField'},
|
|
{docKey: 'used for', dbKey: 'used for', regex: /used for (.*)/, category: 'customField'},
|
|
{docKey: 'Stabilized', dbKey: 'stabilized', regex: /Stabilized, (.*)/, category: 'customField'},
|
|
{docKey: 'parts from field', dbKey: 'parts from field', regex: null, category: 'customField'},
|
|
{docKey: 'side', dbKey: 'side', regex: /(\S*?) side/, category: 'customField'},
|
|
{docKey: 'Creep test', dbKey: 'creep test', regex: null, category: 'customField'},
|
|
{docKey: 'Variante', dbKey: 'variant', regex: /(.*)/, category: 'customField'},
|
|
{docKey: 'Parameter', dbKey: 'parameter', regex: /Parameter (\d)/, category: 'customField'},
|
|
{docKey: 'days without cooling', dbKey: 'days without cooling', regex: /(\d+) days without cooling/, category: 'customField'},
|
|
{docKey: 'Zyklus', dbKey: 'cycle', regex: /Zyklus (\d+ s)/, category: 'customField'},
|
|
{docKey: 'fast cure', dbKey: 'fast cure', regex: null, category: 'customField'},
|
|
{docKey: 'Stoff gesperrt', dbKey: 'material blocked', regex: null, category: 'customField'},
|
|
{docKey: 'anwendungsbeschränkt', dbKey: 'limited application', regex: null, category: 'customField'},
|
|
{docKey: 'für Neuanwendungen gesperrt', dbKey: 'blocked for new applications', regex: null, category: 'customField'},
|
|
{docKey: 'V', dbKey: 'test', regex: /V(\d+-\d+);/, category: 'customField'},
|
|
{docKey: 'Twz', dbKey: 'twz', regex: /Twz \(°C\): (\d+);/, category: 'customField'},
|
|
{docKey: 'Pnach', dbKey: 'pressure after', regex: /Pnach \(bar\): (\d+);/, category: 'customField'},
|
|
{docKey: 'Vein', dbKey: 'volume in', regex: /Vein \(ccm\/s\): (\d+)/, category: 'customField'},
|
|
{docKey: 'low emission', dbKey: 'low emission', regex: /low emission (\S+)[;]?/, category: 'customField'},
|
|
{docKey: 'aus', dbKey: 'from', regex: /aus (.*)/, category: 'customField'},
|
|
{docKey: 'Erprobung', dbKey: 'trial', regex: /Erprobung (.*?);/, category: 'customField'},
|
|
{docKey: 'Auftragsnummer', dbKey: 'job number', regex: /Auftragsnummer: (\S+)[;]?/, category: 'customField'},
|
|
{docKey: 'Wärmealterung', dbKey: 'heat aging', regex: /Wärmealterung: (.*)/, category: 'customField'},
|
|
{docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall', regex: /Steg.*?A: (\d+)/, category: 'customField'},
|
|
{docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall', regex: /Steg.*?I: (\d+)/, category: 'customField'},
|
|
{docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall', regex: /Steg.*?S: (\d+)/, category: 'customField'},
|
|
{docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'outer wall degraded', regex: /Degradation:.*?A: (\d+)/, category: 'customField'},
|
|
{docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'inner wall degraded', regex: /Degradation:.*?I: (\d+)/, category: 'customField'},
|
|
{docKey: 'A: Wandung außen / I: Wandung innen / S: Wandung Steg', dbKey: 'support wall degraded', regex: /Degradation:.*?S: (\d+)/, category: 'customField'},
|
|
{docKey: 'Reines Polymer', dbKey: 'pure polymer', regex: null, category: 'customField'},
|
|
{docKey: 'Rücksendung erforderlich', dbKey: 'return needed', regex: /(.*?,) Rücksendung erforderlich, (.*)/, category: 'customField'},
|
|
{docKey: 'Prio', dbKey: 'priority', regex: /Prio (\d+)/, category: 'customField'},
|
|
{docKey: 'beanstandet', dbKey: 'faulty', regex: null, category: 'customField'},
|
|
{docKey: 'aged', dbKey: 'aged', regex: /aged: (.*)/, category: 'customField'},
|
|
{docKey: 'DOPPELT!!', dbKey: 'double', regex: null, category: 'customField'},
|
|
{docKey: 'Bauteil', dbKey: 'construction part', regex: /Bauteil (\S+)/, category: 'customField'},
|
|
{docKey: 'T =', dbKey: 'temperature', regex: /T = (\S+)/, category: 'customField'},
|
|
{docKey: 'nicht vorgealtert', dbKey: 'not preaged', regex: /nicht vorgealtert (.*)/, category: 'customField'},
|
|
{docKey: 'TS119', dbKey: 'TS119', regex: /TS119 (W\S+);/, category: 'customField'},
|
|
{docKey: 'GF vom Datenblatt', dbKey: 'glass fibre from data sheet', regex: null, category: 'customField'},
|
|
{docKey: 'nach Datensatz', dbKey: 'according to dataset', regex: null, category: 'customField'},
|
|
{docKey: 'Dosiergeschw', dbKey: 'metering speed', regex: /Dosiergeschw.*? (.*?min)/, category: 'customField'},
|
|
{docKey: 'Einspritzgeschw', dbKey: 'injection speed', regex: /Einspritzgeschw.*? (.*\/s)/, category: 'customField'},
|
|
{docKey: 'Heizbänder', dbKey: 'heating lines', regex: /Heizbänder (.*)/, category: 'customField'},
|
|
{docKey: 'Verweilzeit', dbKey: 'dwell time', regex: /Verweilzeit (.*?min)/, category: 'customField'},
|
|
{docKey: 'Probe', dbKey: 'belongs to', regex: /Probe (\S*\d+)/, category: 'reference'},
|
|
{docKey: 'zu', dbKey: 'belongs to', regex: /zu (\S*\d+)/, category: 'reference'},
|
|
{docKey: 'granulate zu', dbKey: 'granulate to', regex: /granulate zu.* (\S*\d+)/, category: 'reference'},
|
|
{docKey: 'construction part', dbKey: 'construction part', regex: /(?<!granulate)construction part.* (\S*\d+)/, category: 'reference'},
|
|
{docKey: 'VZ =', dbKey: 'vn', regex: /VZ = (\d+) cm³\/g/, category: 'vn'},
|
|
{docKey: 'VWZ', dbKey: 'vwz', regex: /(\d+ min) VWZ \//, category: 'customField'},
|
|
{docKey: 'VZ:', dbKey: 'vn', regex: /VZ: ([0-9.,]+) mL\/g[;]?/, category: 'vn'}
|
|
];
|
|
const res = {}; // returned result
|
|
const usedParts = []; // all substrings used for custom fields, subtract at the end, as some parts are used multiple times
|
|
customFields.forEach(cField => {
|
|
if (comment.indexOf(cField.docKey) >= 0) { // comment contains docKey
|
|
if (cField.regex !== null) {
|
|
const regexRes = cField.regex.exec(comment);
|
|
if (regexRes) {
|
|
usedParts.push(regexRes[0]);
|
|
if (cField.category === 'reference') {
|
|
sampleReferences.push({sample: sampleNumber, referencedSample: regexRes[1], relation: cField.dbKey});
|
|
}
|
|
else if (cField.category === 'vn') {
|
|
vnValues[sampleNumber] = regexRes[1];
|
|
}
|
|
else {
|
|
res[cField.dbKey] = regexRes.filter((e, i) => i > 0).join(' ');
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
usedParts.push(cField.docKey);
|
|
res[cField.dbKey] = true;
|
|
}
|
|
}
|
|
});
|
|
usedParts.forEach(part => {
|
|
const index = comment.indexOf(part);
|
|
if (index >= 0) {
|
|
comment = comment.slice(0, index) + comment.slice(index + part.length);
|
|
}
|
|
});
|
|
if (/\w+/.test(comment)) {
|
|
res.xRest = comment;
|
|
}
|
|
return res;
|
|
}
|
|
|
|
function sampleType (type) {
|
|
type = stripSpaces(type).toLowerCase();
|
|
const allowedTypes = {'tension rod': 'processed', 'Zugstab': 'processed', 'part': 'processed', 'granulate': 'as-delivered/raw'};
|
|
if (!allowedTypes[type]) {
|
|
typeLog.push(type);
|
|
}
|
|
return allowedTypes[type] ? allowedTypes[type] : 'processed';
|
|
}
|
|
|
|
function stripSpaces(s) {
|
|
return s ? s.replace(/ /g,'') : '';
|
|
}
|
|
|
|
function trim(s) {
|
|
return s.replace(/(^\s+|\s+$)/gm, '');
|
|
} |