in marketing-analytics/activation/data-tasks-coordinator/src/tasks/bigquery/load_task.js [128:181]
async doTask() {
/** @const {StorageFileConfig} */
const sourceFile = this.config.source.file;
/** @const {LoadOptions} */
const metadata = Object.assign({}, this.config.options);
/** @const {LoadTaskDestination} */
const destination = this.config.destination;
destination.tableSchema = destination.tableSchema || {};
// Needs to check the file is not empty for a single file and autodetect
// schema mode. Note: even with wildcard, there is still a chance that no
// files match and BigQuery fails to autodetect schema.
if (sourceFile.name.indexOf('*') === -1 && metadata.autodetect) {
/** @const {StorageFile} */
const storageFile = StorageFile.getInstance(
sourceFile.bucket, sourceFile.name,
{projectId: this.getCloudProject(sourceFile)});
const size = await storageFile.getFileSize();
if (size === 0) {
this.logger.info('Skip for empty file:', sourceFile);
return {
warning: `Skip for empty file: ${sourceFile.name}`,
parameters: this.appendParameter({
destinationTable: destination.table,
isEmptyFile: true,
}),
};
}
}
metadata.sourceUris = [`gs://${sourceFile.bucket}/${sourceFile.name}`];
metadata.compression = this.isCompressed_(sourceFile.name);
this.logger.debug('Load task metadata: ', metadata);
// @see the typedef of LoadTaskDestination
if (!metadata.autodetect && !destination.tableSchema.schema) {
destination.tableSchema.schema = await this.getSchemaForExternal_(
destination, metadata, sourceFile);
this.logger.debug('External schema: ', destination.tableSchema.schema);
}
const storeTable = await this.getTableForLoading_(destination.table,
destination.tableSchema);
const [job] = await storeTable.load(undefined, metadata);
const errors = job.status.errors;
if (errors && errors.length > 0) throw errors;
const { jobReference } = job;
const { jobId } = jobReference;
this.logger.debug(`Job ${jobId} status ${job.status.state}.`);
return {
jobId,
parameters: this.appendParameter({
destinationTable: destination.table,
jobReference,
}),
};
}