in generators/csv-generator.ts [67:135]
for await (const line of rl) {
if (!skippedHeader) {
skippedHeader = true;
if (line === "message") {
timestamped = false;
}
continue;
}
let cell1: string;
let cell2: string;
try {
if (timestamped) {
[cell1, cell2] = line.split(/,(.+)/);
}
// overwrite cell 2 with cell if there is no timestamp
else {
cell2 = line;
}
// overwrite cell 2 with cell if there is no timestamp
if (cell2 === undefined) {
cell2 = cell1;
}
// remove quotes if needed
if (cell2.charAt(0) === "\"") {
cell2 = cell2.slice(1);
}
if (cell2.charAt(cell2.length - 1) === "\"") {
cell2 = cell2.slice(0, -1);
}
// substitute "" with "
try {
cell2 = cell2.replace(/\"\"/g, "\"");
}
catch (e) {
console.log(e)
}
if (config.isJson) {
parsed = JSON.parse(cell2); // will throw error
if (typeof parsed !== "object") {
continue;
}
}
} catch (e) {
continue;
}
if (config.isJson) {
batch.push(parsed);
}
else {
batch.push({
[config.logKey]: cell2,
});
}
if (batch.length === config.batchSize) {
yield batch;
batch = [];
}
// Each line in input.txt will be successively available here as `line`.
// console.log(`Line from file: ${line}`);
}