public void runTemplate()

in java/src/main/java/com/google/cloud/dataproc/templates/bigquery/BigQueryToGCS.java [69:102]

30 lines of code
6 McCabe index (conditional complexity)


  public void runTemplate() {

    SparkSession spark = SparkSession.builder().appName("BigQuery to GCS").getOrCreate();

    // Set log level
    spark.sparkContext().setLogLevel(sparkLogLevel);

    Dataset<Row> inputData = spark.read().format(SPARK_READ_FORMAT_BIGQUERY).load(inputTableName);
    DataFrameWriter<Row> writer = inputData.write().mode(SaveMode.valueOf(outputMode));
    if (!StringUtils.isEmpty(partitionBy)) {
      writer.partitionBy(partitionBy.trim());
    }
    switch (outputFileFormat) {
      case BQ_GCS_OUTPUT_FORMAT_CSV:
        writer
            .format(GCS_BQ_CSV_FORMAT)
            .option(GCS_BQ_CSV_HEADER, true)
            .option(GCS_BQ_CSV_INFOR_SCHEMA, true)
            .save(outputFileLocation);
        break;
      case BQ_GCS_OUTPUT_FORMAT_JSON:
        writer.json(outputFileLocation);
        break;
      case BQ_GCS_OUTPUT_FORMAT_AVRO:
        writer.format(GCS_BQ_AVRO_EXTD_FORMAT).save(outputFileLocation);
        break;
      case BQ_GCS_OUTPUT_FORMAT_PARQUET:
        writer.parquet(outputFileLocation);
        break;
      default:
        throw new IllegalArgumentException(
            "Currently avro, parquet, csv and json are the only supported formats");
    }
  }