def _ci_step_for_platform_and_commits()

in buildkite/bazel-bench/bazel_bench.py [0:0]

• 55 lines of code
• 3 McCabe index (conditional complexity)

def _ci_step_for_platform_and_commits(
    bazel_commits, platform, project, extra_options, date, bucket, bigquery_table):
    """Perform bazel-bench for the platform-project combination.
    Uploads results to BigQuery.

    Args:
        bazel_commits: a list of strings: bazel commits to be benchmarked.
        platform: a string: the platform to benchmark on.
        project: an object: contains the information of the project to be
          tested on.
        extra_options: a string: extra bazel-bench options.
        date: the date of the commits.
        bucket: the GCP Storage bucket to upload data to.
        bigquery_table: the table to upload data to. In the form `project:table_identifier`.

    Return:
        An object: the result of applying bazelci.create_step to wrap the
          command to be executed by buildkite-agent.
    """
    project_clone_path = _get_clone_path(project["git_repository"], platform)
    bazel_clone_path = _get_clone_path(BAZEL_REPOSITORY, platform)

    bazel_bench_command = " ".join(
        [
            "bazel",
            "run",
            "benchmark",
            "--",
            "--bazel_commits=%s" % ",".join(bazel_commits),
            "--bazel_source=%s" % bazel_clone_path,
            "--project_source=%s" % project_clone_path,
            "--project_label=%s" % project["project_label"],
            "--platform=%s" % platform,
            "--collect_memory",
            "--data_directory=%s" % DATA_DIRECTORY,
            "--csv_file_name=%s" % BAZEL_BENCH_RESULT_FILENAME,
            "--collect_json_profile",
            "--aggregate_json_profiles",
            extra_options,
            "--",
            project["bazel_command"],
        ]
    )
    # TODO(leba): Use GCP Python client instead of gsutil.
    # TODO(https://github.com/bazelbuild/bazel-bench/issues/46): Include task-specific shell commands and build flags.

    # Upload everything under DATA_DIRECTORY to Storage.
    # This includes the raw data, aggr JSON profile and the JSON profiles
    # themselves.
    storage_subdir = "{}/{}/{}/".format(
        project["storage_subdir"], date.strftime("%Y/%m/%d"), platform
    )
    upload_output_files_storage_command = " ".join(
        [
            "gsutil",
            "-m",
            "cp",
            "-r",
            "{}/*".format(DATA_DIRECTORY),
            "gs://{}/{}".format(bucket, storage_subdir),
        ]
    )
    upload_to_big_query_command = " ".join(
        [
            "bq",
            "load",
            "--skip_leading_rows=1",
            "--source_format=CSV",
            bigquery_table,
            "{}/perf_data.csv".format(DATA_DIRECTORY),
        ]
    )

    commands = (
        [bazelci.fetch_bazelcipy_command()]
        + _bazel_bench_env_setup_command(platform, ",".join(bazel_commits))
        + [bazel_bench_command, upload_output_files_storage_command, upload_to_big_query_command]
    )
    label = bazelci.PLATFORMS[platform]["emoji-name"] + project["project_label"]
    return bazelci.create_step(label, commands, platform)