fixed testing cache
This commit is contained in:
VLE2FE
@ -49,6 +49,8 @@ let commentsLog = [];
|
||||
let customFieldsLog = [];
|
||||
const vzValues = {}; // vz values from comments
|
||||
const dptLog = [];
|
||||
const dptSampleAddLog = []; // log samples created during dpt insertion
|
||||
const typeLog = [];
|
||||
|
||||
// TODO: conditions
|
||||
|
||||
@ -75,6 +77,7 @@ async function main() {
|
||||
}
|
||||
// write logs
|
||||
fs.writeFileSync('./data_import/comments.txt', commentsLog.join('\r\n'));
|
||||
fs.writeFileSync('./data_import/typeLog.txt', typeLog.join('\r\n'));
|
||||
fs.writeFileSync('./data_import/customFields.txt', customFieldsLog.join('\r\n'));
|
||||
fs.writeFileSync('./data_import/sampleReferences.txt', sampleReferences.map(e => JSON.stringify(e)).join('\r\n'));
|
||||
fs.writeFileSync('./data_import/sampleReferences.json', JSON.stringify(sampleReferences));
|
||||
@ -84,6 +87,7 @@ async function main() {
|
||||
if (stages.dpt) { // DPT
|
||||
await allDpts();
|
||||
fs.writeFileSync('./data_import/sdptLog.txt', dptLog.join('\r\n'));
|
||||
fs.writeFileSync('./data_import/dptSampleAddLog.txt', dptSampleAddLog.join('\r\n'));
|
||||
}
|
||||
if (0) { // pdf test
|
||||
console.log(await readPdf('N28_BN05-OX023_2019-07-16.pdf'));
|
||||
@ -95,24 +99,23 @@ async function main() {
|
||||
}
|
||||
|
||||
async function importCsv(doc) {
|
||||
// Uniform name samplenumber materialnumber materialname supplier material plastic reinforcingmaterial granulate/part color charge/batch comments vz(ml/g) kfingew% degradation(%) glassfibrecontent(%) stabwn
|
||||
// Metadata__AnP2.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material, granulate/Part,Color,Charge/ Batch, Comments
|
||||
// Metadata__AnP2_A.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material, Granulate/Part, Comments, Humidity [ppm]
|
||||
// Metadata__AnP2_B.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material, Granulate/Part, VZ [ml/g], glass fibre content
|
||||
// Metadata_Ap.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material, Granulate/Part,Color,Charge/Batch, Comments
|
||||
// Metadata_Bj.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material, Granulate/Part,Color,Charge/batch granulate/part,Comments
|
||||
// Metadata_Eh.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material, Granulate/Part,Color,Charge/Batch granulate/part,Comments, VZ [cm³/g], Spalte1
|
||||
// Metadata_Eh_B.csv Sample number, Material name,Supplier, Plastic,Reinforcing material, Granulate/Part,Color, Comments, VZ [cm³/g]
|
||||
// Metadata_Eh_Duroplasten.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material, Granulate/Part,Color,Charge/Batch granulate/part,Comments
|
||||
// Metadata_Rng_aktuell.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material, Granulate/Part,Color,Charge/batch granulate/part,Comments, VZ (ml/g), Degradation(%),Glas fibre content (%)
|
||||
// Metadata_Rng_aktuell_A.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material, Granulate/Part,Farbe,Charge/batch granulate/part,Comments, KF in Gew%, Stabwn
|
||||
// Metadata_Rng_aktuell_B.csv Sample number, Material name,Supplier, Plastic,Reinforcing material (content in %),Granulate/Part, Comments, VZ (ml/g), Degradation (%), Alterungszeit in h
|
||||
// Metadata_WaP.csv Probennummer, Name, Firma, Material, Teil/Rohstoff, Charge, Anmerkung,VZ (ml/g), Abbau (%), Verstärkungsstoffgehalt (%), Versuchsnummer
|
||||
// Uniform name samplenumber materialnumber materialname supplier material plastic reinforcingmaterial granulate/part color charge/batch comments vz(ml/g) kfingew% degradation(%) reinforcingmaterialcontent stabwn
|
||||
// Metadata__AnP2.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,granulate/Part,Color,Charge/ Batch, Comments
|
||||
// Metadata__AnP2_A.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part, Comments, Humidity [ppm]
|
||||
// Metadata__AnP2_B.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part, VZ [ml/g], glass fibre content
|
||||
// Metadata_Ap.csv Sample number,Material number,Material name,Supplier, Plastic,Reinforcing material,Granulate/Part,Color,Charge/Batch, Comments
|
||||
// Metadata_Bj.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Color,Charge/batch granulate/part,Comments
|
||||
// Metadata_Eh.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material,Granulate/Part,Color,Charge/Batch granulate/part,Comments, VZ [cm³/g], Spalte1
|
||||
// Metadata_Eh_B.csv Sample number, Material name,Supplier, Plastic,Reinforcing material,Granulate/Part,Color, Comments, VZ [cm³/g]
|
||||
// Metadata_Eh_Duroplasten.csv Sample number,Material number,Material name,Supplier,Material, Reinforcing material,Granulate/Part,Color,Charge/Batch granulate/part,Comments
|
||||
// Metadata_Rng_aktuell.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Color,Charge/batch granulate/part,Comments, VZ (ml/g), Degradation(%),Glas fibre content (%)
|
||||
// Metadata_Rng_aktuell_A.csv Sample number,Material number,Material name,Supplier,Material,Plastic,Reinforcing material,Granulate/Part,Farbe,Charge/batch granulate/part,Comments, KF in Gew%, Reinforcing material (content in %),Stabwn
|
||||
// Metadata_Rng_aktuell_B.csv Sample number, Material name,Supplier, Plastic, Granulate/Part, Comments, VZ (ml/g), Degradation (%), Alterungszeit in h
|
||||
// Metadata_WaP.csv Probennummer, Name, Firma, Material, Teil/Rohstoff, Charge, Anmerkung,VZ (ml/g), Abbau (%), Verstärkungsstoffgehalt (%), Versuchsnummer
|
||||
const nameCorrection = { // map to right column names
|
||||
'probennummer': 'samplenumber',
|
||||
'name': 'materialname',
|
||||
'firma': 'supplier',
|
||||
'reinforcingmaterial(contentin%)': 'reinforcingmaterial',
|
||||
'teil/rohstoff': 'granulate/part',
|
||||
'charge/batchgranulate/part': 'charge/batch',
|
||||
'charge': 'charge/batch',
|
||||
@ -120,7 +123,10 @@ async function importCsv(doc) {
|
||||
'vz[ml/g]': 'vz(ml/g)',
|
||||
'vz[cm³/g]': 'vz(ml/g)',
|
||||
'abbau(%)': 'degradation(%)',
|
||||
'verstärkungsstoffgehalt(%)': 'glassfibrecontent(%)'
|
||||
'glassfibrecontent': 'reinforcingmaterialcontent',
|
||||
'glasfibrecontent(%)': 'reinforcingmaterialcontent',
|
||||
'reinforcingmaterial(contentin%)': 'reinforcingmaterialcontent',
|
||||
'verstärkungsstoffgehalt(%)': 'reinforcingmaterialcontent'
|
||||
};
|
||||
const missingFieldsFill = [ // column names to fill if they do not exist
|
||||
'color',
|
||||
@ -129,7 +135,7 @@ async function importCsv(doc) {
|
||||
'materialnumber',
|
||||
'reinforcementmaterial'
|
||||
]
|
||||
console.log('importing ' + doc);
|
||||
console.info('importing ' + doc);
|
||||
data = [];
|
||||
await new Promise(resolve => {
|
||||
fs.createReadStream(doc)
|
||||
@ -158,9 +164,9 @@ async function importCsv(doc) {
|
||||
newE[field] = '';
|
||||
}
|
||||
});
|
||||
// if(newE['materialname'] === '') { // TODO: is this replacement okay?
|
||||
// newE['materialname'] = newE['material'];
|
||||
// }
|
||||
if(newE['materialname'] === '') {
|
||||
newE['materialname'] = newE['material'];
|
||||
}
|
||||
if (newE['supplier'] === '') { // empty supplier fields
|
||||
newE['supplier'] = 'unknown';
|
||||
}
|
||||
@ -211,12 +217,68 @@ async function allDpts() {
|
||||
res.data.forEach(sample => {
|
||||
sampleIds[sample.number] = sample._id;
|
||||
});
|
||||
const dptRegex = /(.*?)_(.*?)_(\d+|[a-zA-Z0-9]+_\d+).DPT/;
|
||||
const dptRegex = /(.*?)_(.*?)_(\d+|[a-zA-Z0-9]+[_.]\d+)(_JDX)?[.]{1,2}(DPT|csv|CSV)/;
|
||||
const dpts = fs.readdirSync(dptFiles);
|
||||
for (let i in dpts) {
|
||||
const regexRes = dptRegex.exec(dpts[i])
|
||||
let regexInput;
|
||||
const bjRes = /^(Bj[FT]?)\s?([a-z0-9_]*)_JDX.DPT/.exec(dpts[i]);
|
||||
if (bjRes) {
|
||||
regexInput = `Bj01_${bjRes[1]}${bjRes[2]}_0.DPT`;
|
||||
}
|
||||
else {
|
||||
regexInput = dpts[i].replace(/_JDX.*\./, '.');
|
||||
}
|
||||
const regexRes = dptRegex.exec(regexInput);
|
||||
if (regexRes && !sampleIds[regexRes[2]]) { // when sample number includes an additional _x instead of having _x_x for spectrum description
|
||||
regexRes[2] = `${regexRes[2]}_${regexRes[3].split('_')[0]}`;
|
||||
}
|
||||
if (regexRes && !sampleIds[regexRes[2]] && sampleIds[regexRes[2].split('_')[0]]) { // when number_abx does not exist but number
|
||||
dptSampleAddLog.push(`Trying to find ${regexRes[2].split('_')[0]}`);
|
||||
dptSampleAddLog.push(host + '/sample/' + sampleIds[regexRes[2].split('_')[0]]);
|
||||
res = await axios({
|
||||
method: 'get',
|
||||
url: host + '/sample/number/' + sampleIds[regexRes[2].split('_')[0]],
|
||||
auth: {
|
||||
username: 'admin',
|
||||
password: 'Abc123!#'
|
||||
}
|
||||
}).catch(err => {
|
||||
if (err.response) {
|
||||
console.error(err.response.data);
|
||||
errors.push(`DPT Could not fetch sample ${regexRes[2].split('_')[0]}: ${err.response.data}`);
|
||||
}
|
||||
});
|
||||
if (res.data) {
|
||||
dptSampleAddLog.push(JSON.stringify(res.data));
|
||||
const data = _.merge(_.pick(res.data, ['color', 'type', 'batch', 'material_id']), {number: regexRes[2], condition: {}, notes: {}});
|
||||
res = await axios({
|
||||
method: 'get',
|
||||
url: host + '/sample/new',
|
||||
auth: {
|
||||
username: res.data.user,
|
||||
password: res.data.user === 'admin' ? 'Abc123!#' : '2020DeFinMachen!'
|
||||
},
|
||||
data
|
||||
}).catch(err => {
|
||||
if (err.response) {
|
||||
console.error(err.response.data);
|
||||
errors.push(`DPT Could not save sample ${data}: ${err.response.data}`);
|
||||
}
|
||||
});
|
||||
console.error(res);
|
||||
console.error(data);
|
||||
if (res.data) {
|
||||
dptSampleAddLog.push(`${regexRes[2]} from ${regexRes[2].split('_')[0]}`)
|
||||
sampleIds[regexRes[2]] = res.data._id;
|
||||
}
|
||||
else {
|
||||
console.error(res);
|
||||
console.error(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (regexRes && sampleIds[regexRes[2]]) { // found matching sample
|
||||
console.log(`${dpts[i]} -> ${regexRes[2]}`);
|
||||
console.log(`${i}/${dpts.length} ${dpts[i]} -> ${regexRes[2]}`);
|
||||
dptLog.push(`${dpts[i]}, ${regexRes[2]}`);
|
||||
const f = fs.readFileSync(dptFiles + '\\' + dpts[i], 'utf-8');
|
||||
const data = {
|
||||
@ -225,10 +287,10 @@ async function allDpts() {
|
||||
measurement_template
|
||||
};
|
||||
data.values.device = regexRes[1];
|
||||
data.values.dpt = f.split('\r\n').map(e => e.split(','));
|
||||
data.values.dpt = f.split('\r\n').map(e => e.split(',').map(e => parseFloat(e)));
|
||||
let rescale = false;
|
||||
for (let i in data.values.dpt) {
|
||||
if (data.values.dpt[i][1] > 2) {
|
||||
if (data.values.dpt[i][1] > 10) {
|
||||
rescale = true;
|
||||
break;
|
||||
}
|
||||
@ -258,7 +320,12 @@ async function allDpts() {
|
||||
}
|
||||
else {
|
||||
console.log(`Could not find sample for ${dpts[i]}`);
|
||||
errors.push(`Could not find sample for ${dpts[i]}`);
|
||||
if (regexRes) {
|
||||
errors.push(`Could not find sample for ${dpts[i]}; [DEBUG] ${regexRes[2]}, ${!sampleIds[regexRes[2]]}, ${sampleIds[regexRes[2].split('_')[0]]}`);
|
||||
}
|
||||
else {
|
||||
errors.push(`Could not find sample for ${dpts[i]} (did not match RegEx)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -274,6 +341,7 @@ async function allKfVz() {
|
||||
});
|
||||
const kf_template = res.data.filter(e => e.name === 'kf').sort((a, b) => b.version - a.version)[0]._id;
|
||||
const vz_template = res.data.filter(e => e.name === 'vz').sort((a, b) => b.version - a.version)[0]._id;
|
||||
const rmc_template = res.data.filter(e => e.name === 'reinforcement material content').sort((a, b) => b.version - a.version)[0]._id;
|
||||
res = await axios({
|
||||
method: 'get',
|
||||
url: host + '/samples?status=all',
|
||||
@ -289,6 +357,7 @@ async function allKfVz() {
|
||||
for (let index in data) {
|
||||
console.info(`KF/VZ ${index}/${data.length}`);
|
||||
let sample = data[index];
|
||||
sample['samplenumber'] = sample['samplenumber'].replace(/[A-Z][a-z]0\d_/, '');
|
||||
let credentials = ['admin', 'Abc123!#'];
|
||||
if (sampleDevices[sample['samplenumber']]) {
|
||||
credentials = [sampleDevices[sample['samplenumber']], '2020DeFinMachen!']
|
||||
@ -339,6 +408,27 @@ async function allKfVz() {
|
||||
errors.push(`KF/VZ upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`);
|
||||
});
|
||||
}
|
||||
if (sample['reinforcingmaterialcontent']) {
|
||||
await axios({
|
||||
method: 'post',
|
||||
url: host + '/measurement/new',
|
||||
auth: {
|
||||
username: credentials[0],
|
||||
password: credentials[1]
|
||||
},
|
||||
data: {
|
||||
sample_id: sampleIds[sample['samplenumber']],
|
||||
measurement_template: rmc_template,
|
||||
values: {
|
||||
percentage: Number(sample['reinforcingmaterialcontent'].replace('%', '').replace(',', '.'))
|
||||
}
|
||||
}
|
||||
}).catch(err => {
|
||||
console.log(sample['samplenumber']);
|
||||
console.error(err.response.data);
|
||||
errors.push(`KF/VZ upload for ${JSON.stringify(sample)} failed: ${JSON.stringify(err.response.data)}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -436,7 +526,6 @@ async function saveSamples() {
|
||||
console.info(`SAMPLE SAVE ${i}/${samples.length}`);
|
||||
let credentials = ['admin', 'Abc123!#'];
|
||||
if (sampleDevices[samples[i].number]) {
|
||||
console.log(sampleDevices[samples[i].number]);
|
||||
credentials = [sampleDevices[samples[i].number], '2020DeFinMachen!']
|
||||
}
|
||||
await axios({
|
||||
@ -520,7 +609,7 @@ async function allMaterials() {
|
||||
password: 'Abc123!#'
|
||||
}
|
||||
});
|
||||
const materialTemplate = res.data.find(e => e.name === 'plastic')._id;
|
||||
const materialTemplate = res.data.filter(e => e.name === 'plastic').sort((a, b) => b.version - a.version)[0]._id;
|
||||
|
||||
// process all samples
|
||||
for (let index in data) {
|
||||
@ -718,7 +807,7 @@ function readPdf(file) {
|
||||
let lastLastText = ''; // text of last last item
|
||||
await new pdfReader.PdfReader().parseFileItems(nmDocs + '\\' + file, (err, item) => {
|
||||
if (item && item.text) {
|
||||
if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsuppl') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsupplier') >= 0)) { // table area starts
|
||||
if ((stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignationsuppl') >= 0) || (stripSpaces(lastLastText + lastText + item.text).toLowerCase().indexOf('colordesignatiomsuppl') >= 0)) { // table area starts
|
||||
table = countdown;
|
||||
}
|
||||
if (table > 0) {
|
||||
@ -865,7 +954,10 @@ function customFields (comment, sampleNumber) {
|
||||
|
||||
function sampleType (type) {
|
||||
const allowedTypes = ['tension rod', 'part', 'granulate'];
|
||||
return allowedTypes.indexOf(type) >= 0 ? type : (type === '' ? 'unknown' : 'other');
|
||||
if (allowedTypes.indexOf(type) < 0) {
|
||||
typeLog.push(type);
|
||||
}
|
||||
return allowedTypes.indexOf(type) >= 0 ? type : 'part';
|
||||
}
|
||||
|
||||
function stripSpaces(s) {
|
||||
|
||||
Reference in New Issue
Block a user