async getSchemaForExternal_()

in marketing-analytics/activation/data-tasks-coordinator/src/tasks/bigquery/load_task.js [242:288]

• 42 lines of code
• 7 McCabe index (conditional complexity)

  async getSchemaForExternal_(destination, metadata, sourceFile) {
    if (destination.schemaSource) {
      this.logger.debug('External schema from report: ',
          destination.schemaSource);
      const taskConfig = await this.options.taskConfigDao.load(
          destination.schemaSource);
      const task = new ReportTask(taskConfig, this.parameters);
      task.injectContext(this.options);
      return this.getSchemaFromReportTask_(task);
    }
    if (destination.schemaFile) {
      this.logger.debug('External schema from file: ', destination.schemaFile);
      const schemaFile = StorageFile.getInstance(
          destination.schemaFile.bucket, destination.schemaFile.name,
          {projectId: this.getCloudProject(destination.schemaFile)});
      return JSON.parse(await schemaFile.loadContent());
    }
    if (destination.csvSafeColumnNames) {
      if (metadata.compression
        || metadata.sourceFormat.toUpperCase() !== 'CSV') {
        this.logger.error('Can not figure column names for', metadata);
      } else {
        /** @const {StorageFile} */
        const storageFile = StorageFile.getInstance(
          sourceFile.bucket, sourceFile.name,
          { projectId: this.getCloudProject(sourceFile) });
        // Get first 100k bytes. This should be enough to cover the first line.
        const partialContent = await storageFile.loadContent(0, 100_000);
        const headers = partialContent.split('\n')[0]
          .split(metadata.fieldDelimiter || ',');
        const fields = headers.map((header) => {
          // replace all characters (expect digits, letters and underscore) with
          // an underscore; add a leading underscore if it starts with a digit.
          const name = header.trim().replace(/[^a-zA-Z0-9_]/g, '_')
            .replace(/^([0-9])/, '_$1');
          return {
            name,
            type: 'STRING',
            mode: 'NULLABLE',
          }

        })
        return { fields };
      }
    }
    throw new Error('Not schema defined in config or externally');
  }