in api/v1/src/spots/dataManager.js [220:278]
async function createSpotJob(requestId, options, queryOptions) {
const bucketName = options.destination.bucketName;
const fileName = options.destination.fileName;
const projectId = options.config.destination.projectId;
const datasetId = options.config.destination.datasetId;
// Dynamically create unique tableIds for the extractToBucket method
const tableId = createTableId(requestId);
console.log(`Will execute query with options: ${JSON.stringify(queryOptions)}`);
// Set query options to execute BQ job and extract to table
const today = new Date();
today.setDate(today.getDate() + 1);
const expiryTime = today.getTime();
bigqueryUtil = new BigQueryUtil(projectId);
var results = await bigqueryUtil.executeQuerySync(queryOptions).then(() => {
const options = {
format: 'json',
gzip: true
}
console.log(`bigqueryUtil.extractTableToGCS: ${datasetId} ${tableId} ${bucketName} ${fileName}`);
return bigqueryUtil.extractTableToGCS(datasetId, tableId, bucketName, fileName, options);
}).then(() => {
// update with requestId
const fileMetadata = {
private: true,
metadata: {
requestId: requestId
}
}
console.log(`storageUtil.setFileMetadata: ${bucketName} ${fileName}`);
return storageUtil.setFileMetadata(bucketName, fileName, fileMetadata);
}).then(() => {
console.log(`storageUtil.getUrl: ${bucketName} ${fileName}`);
return storageUtil.getUrl(bucketName, fileName, true);
}).then((signedUrl) => {
console.log(`bigqueryUtil.setTableMetadata: ${datasetId} ${tableId} ${expiryTime}`);
const metadata = {
expirationTime: expiryTime
}
bigqueryUtil.setTableMetadata(datasetId, tableId, metadata);
return signedUrl;
}).catch(error => {
const message = `createSpotJob failed: ${error}`;
console.warn(message);
const metadata = {
expirationTime: expiryTime
}
console.log(`Finally bigqueryUtil.setTableMetadata: ${datasetId} ${tableId} ${expiryTime}`);
bigqueryUtil.setTableMetadata(datasetId, tableId, metadata);
return { success: false, errors: [message] };
});
if (results.success === false) {
return results;
}
return { signedUrl: results };
}