async function stageFile()

in ingestion/batch/index.js [169:201]

• 30 lines of code
• 8 McCabe index (conditional complexity)

async function stageFile(config) {
    console.log(`Using config ${JSON.stringify(config)}`);
    const dataset = bigqueryUtil.getDataset(config.datasetId);
    let today = new Date();
    today.setDate(today.getDate() + stagingTableExpiryDays);
    const expiryTime = today.getTime();
    console.log(`Setting expirationTime for staging table to ${expiryTime}`);

    const fields = (config.metadata && config.metadata.fields) || undefined;
    let options = { expirationTime: expiryTime };
    if (fields) {
        options.schema = fields;
    }
    else {
        options.autodetect = true;
    }

    await dataset.createTable(config.stagingTable, options);
    const table = dataset.table(config.stagingTable);
    console.log(`Created table ${config.stagingTable}`);
    console.log(`Executing load for ${config.sourceFile} with config: ${JSON.stringify(config)}`);

    try {
        let [job] = await table.load(storageUtil.getBucket(config.bucket).file(config.sourceFile), config.metadata || { autodetect: true });
        console.log(`${job.id} ${job.configuration.jobType} ${job.status.state} ${job.statistics.load.outputRows} rows`);
        return;
    }
    catch (ex) {
        console.error(`Errors encountered loading ${config.sourceFile} to ${config.stagingTable}`);
        logException(ex);
        throw (ex);
    }
}