in src/utils/configUtils.js [416:483]
async function validateSampleRowsAndStopIfTooManyErrors(
schema: Object,
config: Object,
) {
// Normalize some sample rows to see correct rate.
const previewTask = new SignalsUploaderPreviewV2Task(
config.inputFilePath,
config.delimiter,
{
schema,
mapping: config.mapping,
infoForNormalization: config.format,
customTypeInfo: config.customTypeInfo,
},
config.presetValues,
{
type: 'advanced',
maxNumInvalidSamples: NUM_SAMPLE_ROWS_TO_VERIFY_CONFIG,
},
NUM_SAMPLE_ROWS_TO_VERIFY_CONFIG,
config.header,
64 * 1024, // Chunk size.
16, // Max num chunks in buffer.
null, // Logger not used.
);
let previewResult;
try {
previewResult = await previewTask.run();
} catch (error) {
winston.error(`Preview failed with error: ${error}`);
process.exit(1);
throw error;
}
const numRowsWithError = previewResult.errorReport.invalidSamples.length;
const numProcessedRows = previewResult.numRowsProcessed;
if (numRowsWithError <= numProcessedRows * STOP_THRESHOLD) {
// Acceptable error rate. Pass.
return true;
}
const abortMessage =
`ABORTED because more than ${STOP_THRESHOLD * 100}% of the first `
+ `${numProcessedRows} rows have errors.`;
winston.error(abortMessage);
if (config.ignoreSampleErrors) {
return true;
}
const stream = fs.createWriteStream(config.reportOutputPath);
stream.once('open', () => {
printLine(stream, abortMessage);
stream.write(EOL);
printConfig(stream, 'ABORTED', config);
printAggregatedError(
stream,
previewResult.errorReport,
config.mapping,
);
printInvalidSamples(
stream,
previewResult.errorReport.invalidSamples,
config.mapping,
);
stream.end();
});
winston.info(`Report saved to: ${config.reportOutputPath}`);
return false;
}