artifact_downloader.py [260:494]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        )
    return tmp_path


def move_file(abs_filepath, output_dir, count=0):
    tmp_path = os.path.join(output_dir, str(count))
    _, fname = os.path.split(abs_filepath)
    if not os.path.exists(tmp_path):
        make_count_dir(tmp_path)
    if os.path.exists(os.path.join(tmp_path, fname)):
        return

    shutil.copyfile(abs_filepath, os.path.join(tmp_path, fname))
    return tmp_path


def artifact_downloader(
    task_group_id,
    output_dir=os.getcwd(),
    test_suites=[],
    download_failures=False,
    artifact_to_get="grcov",
    unzip_artifact=True,
    platform="test-linux64-ccov",
    ingest_continue=False,
):
    global CURR_REQS
    global CURR_TASK
    global TOTAL_TASKS
    global FAILED
    global ALL_TASKS

    head_rev = ""
    all_tasks = False
    if "all" in test_suites:
        all_tasks = True

    # For compatibility
    if type(artifact_to_get) not in (list,):
        artifact_to_get = [artifact_to_get]

    # Make the data directories
    task_dir = os.path.join(output_dir, task_group_id)

    run_number = 0
    max_num = 0
    if not os.path.exists(task_dir):
        os.makedirs(task_dir, exist_ok=True)
    else:
        # Get current run number
        curr_dir = os.getcwd()
        os.chdir(task_dir)
        dir_list = next(os.walk("."))[1]
        max_num = 0
        for subdir in dir_list:
            run_num = int(subdir)
            if run_num > max_num:
                max_num = run_num
        os.chdir(curr_dir)

    if not ingest_continue:
        run_number = max_num + 1

    output_dir = os.path.join(task_dir, str(run_number))
    os.makedirs(output_dir, exist_ok=True)

    log("Artifacts will be stored in %s" % output_dir)
    config_json_path = os.path.join(output_dir, "config.json")
    with open(config_json_path, "w") as f:
        json.dump(
            {
                "test_suites": test_suites,
                "platform": platform,
                "artifact": artifact_to_get,
                "download_failures": download_failures,
                "task_group_id": task_group_id,
            },
            f,
            indent=4,
        )

    log("Saved run configuration to %s" % config_json_path)

    task_ids = []
    log("Getting task group information...")
    tgi_path = os.path.join(task_dir, "task-group-information.json")
    if os.path.exists(tgi_path):
        with open(tgi_path, "r") as f:
            tasks = json.load(f)
    else:
        tasks = get_tasks_in_group(task_group_id)
        with open(tgi_path, "w") as f:
            json.dump(tasks, f, indent=4)
    log("Obtained")

    # Used to keep track of how many grcov files
    # we are downloading per test.
    task_counters = {}
    taskid_to_file_map = {}

    # For each task in this group
    threads = []
    TOTAL_TASKS = len(tasks)
    for task in tasks:
        download_this_task = False
        # Get the test name
        if platform not in task["task"]["metadata"]["name"]:
            continue
        test_name = suite_name_from_task_name(task["task"]["metadata"]["name"])
        log(
            "Found %s with suite-name: %s"
            % (task["task"]["metadata"]["name"], test_name)
        )

        if (
            task.get("status", {}).get("state", "") in ("failed",)
            and not download_failures
        ):
            log("Skipped failed task")
            continue

        # If all tests weren't asked for but this test is
        # asked for, set the flag.
        if (not all_tasks) and test_name in test_suites:
            download_this_task = True

        if all_tasks or download_this_task:
            if "GECKO_HEAD_REV" in task["task"]["payload"]["env"]:
                # Some tasks are missing this variable
                head_rev = task["task"]["payload"]["env"]["GECKO_HEAD_REV"]

            # Make directories for this task
            grcov_dir = os.path.join(output_dir, test_name)
            downloads_dir = os.path.join(os.path.join(grcov_dir, "downloads"))
            data_dir = {
                aname: os.path.join(
                    os.path.join(grcov_dir, (aname.replace(".", "")) + "_data")
                )
                for aname in artifact_to_get
            }

            if test_name not in task_counters:
                os.makedirs(grcov_dir, exist_ok=True)
                os.makedirs(downloads_dir, exist_ok=True)
                for _, p in data_dir.items():
                    os.makedirs(p, exist_ok=True)
                task_counters[test_name] = 0
            else:
                task_counters[test_name] += 1
            task_id = task["status"]["taskId"]
            ALL_TASKS.append(task_id)

            def get_artifacts(
                task_id,
                downloads_dir,
                data_dir,
                unzip_artifact,
                test_counter,
                test_name,
                artifact_to_get,
                download_failures,
                taskid_to_file_map,
            ):
                global CURR_REQS

                try:

                    def _pattern_match(name, artifacts_to_get):
                        for aname in artifacts_to_get:
                            if aname in name:
                                return aname
                        return None

                    def _check_unzip(filen):
                        return unzip_artifact and (
                            filen.endswith(".zip") or filen.endswith(".tgz")
                        )

                    files = os.listdir(downloads_dir)
                    ffound = [
                        f
                        for f in files
                        if _pattern_match(f, artifact_to_get) and task_id in f
                    ]
                    if ffound:
                        log("File already exists.")
                        CURR_REQS -= 1
                        # There should only be file found
                        filen = ffound[0]
                        aname = _pattern_match(filen, artifact_to_get)

                        if aname == "grcov" or "grcov" in aname or _check_unzip(filen):
                            unzip_file(filen, data_dir[aname], test_counter)
                        else:
                            move_file(filen, data_dir[aname], test_counter)

                        taskid_to_file_map[task_id] = os.path.join(
                            data_dir[aname], str(test_counter)
                        )

                        return filen

                    CURR_REQS += 1
                    log("Getting task artifacts for %s" % task_id)
                    artifacts = get_task_artifacts(task_id)
                    CURR_REQS -= 1

                    # Check if the artifact to get exists before checking for
                    # failures in the task
                    exists = False
                    for artifact in artifacts:
                        if _pattern_match(artifact["name"], artifact_to_get):
                            exists = True
                    if not exists:
                        log("Missing %s in %s" % (artifact_to_get, task_id))
                        CURR_REQS -= 1
                        return

                    if not download_failures:
                        log("Checking for failures on %s" % task_id)
                        failed = None
                        for artifact in artifacts:
                            if "log_error" in artifact["name"]:
                                CURR_REQS += 1
                                filen = download_artifact(
                                    task_id, artifact, downloads_dir
                                )
                                CURR_REQS -= 1
                                if os.stat(filen).st_size != 0:
                                    failed = artifact["name"]
                        if failed:
                            log("Skipping a failed test: " + failed)
                            return

                    for artifact in artifacts:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



mozperftest_tools/mozperftest_tools/utils/artifact_downloader.py [241:475]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        )
    return tmp_path


def move_file(abs_filepath, output_dir, count=0):
    tmp_path = os.path.join(output_dir, str(count))
    _, fname = os.path.split(abs_filepath)
    if not os.path.exists(tmp_path):
        make_count_dir(tmp_path)
    if os.path.exists(os.path.join(tmp_path, fname)):
        return

    shutil.copyfile(abs_filepath, os.path.join(tmp_path, fname))
    return tmp_path


def artifact_downloader(
    task_group_id,
    output_dir=os.getcwd(),
    test_suites=[],
    download_failures=False,
    artifact_to_get="grcov",
    unzip_artifact=True,
    platform="test-linux64-ccov",
    ingest_continue=False,
):
    global CURR_REQS
    global CURR_TASK
    global TOTAL_TASKS
    global FAILED
    global ALL_TASKS

    head_rev = ""
    all_tasks = False
    if "all" in test_suites:
        all_tasks = True

    # For compatibility
    if type(artifact_to_get) not in (list,):
        artifact_to_get = [artifact_to_get]

    # Make the data directories
    task_dir = os.path.join(output_dir, task_group_id)

    run_number = 0
    max_num = 0
    if not os.path.exists(task_dir):
        os.makedirs(task_dir, exist_ok=True)
    else:
        # Get current run number
        curr_dir = os.getcwd()
        os.chdir(task_dir)
        dir_list = next(os.walk("."))[1]
        max_num = 0
        for subdir in dir_list:
            run_num = int(subdir)
            if run_num > max_num:
                max_num = run_num
        os.chdir(curr_dir)

    if not ingest_continue:
        run_number = max_num + 1

    output_dir = os.path.join(task_dir, str(run_number))
    os.makedirs(output_dir, exist_ok=True)

    log("Artifacts will be stored in %s" % output_dir)
    config_json_path = os.path.join(output_dir, "config.json")
    with open(config_json_path, "w") as f:
        json.dump(
            {
                "test_suites": test_suites,
                "platform": platform,
                "artifact": artifact_to_get,
                "download_failures": download_failures,
                "task_group_id": task_group_id,
            },
            f,
            indent=4,
        )

    log("Saved run configuration to %s" % config_json_path)

    task_ids = []
    log("Getting task group information...")
    tgi_path = os.path.join(task_dir, "task-group-information.json")
    if os.path.exists(tgi_path):
        with open(tgi_path, "r") as f:
            tasks = json.load(f)
    else:
        tasks = get_tasks_in_group(task_group_id)
        with open(tgi_path, "w") as f:
            json.dump(tasks, f, indent=4)
    log("Obtained")

    # Used to keep track of how many grcov files
    # we are downloading per test.
    task_counters = {}
    taskid_to_file_map = {}

    # For each task in this group
    threads = []
    TOTAL_TASKS = len(tasks)
    for task in tasks:
        download_this_task = False
        # Get the test name
        if platform not in task["task"]["metadata"]["name"]:
            continue
        test_name = suite_name_from_task_name(task["task"]["metadata"]["name"])
        log(
            "Found %s with suite-name: %s"
            % (task["task"]["metadata"]["name"], test_name)
        )

        if (
            task.get("status", {}).get("state", "") in ("failed",)
            and not download_failures
        ):
            log("Skipped failed task")
            continue
            
        # If all tests weren't asked for but this test is
        # asked for, set the flag.
        if (not all_tasks) and test_name in test_suites:
            download_this_task = True

        if all_tasks or download_this_task:
            if "GECKO_HEAD_REV" in task["task"]["payload"]["env"]:
                # Some tasks are missing this variable
                head_rev = task["task"]["payload"]["env"]["GECKO_HEAD_REV"]

            # Make directories for this task
            grcov_dir = os.path.join(output_dir, test_name)
            downloads_dir = os.path.join(os.path.join(grcov_dir, "downloads"))
            data_dir = {
                aname: os.path.join(
                    os.path.join(grcov_dir, (aname.replace(".", "")) + "_data")
                )
                for aname in artifact_to_get
            }

            if test_name not in task_counters:
                os.makedirs(grcov_dir, exist_ok=True)
                os.makedirs(downloads_dir, exist_ok=True)
                for _, p in data_dir.items():
                    os.makedirs(p, exist_ok=True)
                task_counters[test_name] = 0
            else:
                task_counters[test_name] += 1
            task_id = task["status"]["taskId"]
            ALL_TASKS.append(task_id)

            def get_artifacts(
                task_id,
                downloads_dir,
                data_dir,
                unzip_artifact,
                test_counter,
                test_name,
                artifact_to_get,
                download_failures,
                taskid_to_file_map,
            ):
                global CURR_REQS

                try:

                    def _pattern_match(name, artifacts_to_get):
                        for aname in artifacts_to_get:
                            if aname in name:
                                return aname
                        return None

                    def _check_unzip(filen):
                        return unzip_artifact and (
                            filen.endswith(".zip") or filen.endswith(".tgz")
                        )

                    files = os.listdir(downloads_dir)
                    ffound = [
                        f
                        for f in files
                        if _pattern_match(f, artifact_to_get) and task_id in f
                    ]
                    if ffound:
                        log("File already exists.")
                        CURR_REQS -= 1
                        # There should only be file found
                        filen = ffound[0]
                        aname = _pattern_match(filen, artifact_to_get)

                        if aname == "grcov" or "grcov" in aname or _check_unzip(filen):
                            unzip_file(filen, data_dir[aname], test_counter)
                        else:
                            move_file(filen, data_dir[aname], test_counter)

                        taskid_to_file_map[task_id] = os.path.join(
                            data_dir[aname], str(test_counter)
                        )

                        return filen

                    CURR_REQS += 1
                    log("Getting task artifacts for %s" % task_id)
                    artifacts = get_task_artifacts(task_id)
                    CURR_REQS -= 1

                    # Check if the artifact to get exists before checking for
                    # failures in the task
                    exists = False
                    for artifact in artifacts:
                        if _pattern_match(artifact["name"], artifact_to_get):
                            exists = True
                    if not exists:
                        log("Missing %s in %s" % (artifact_to_get, task_id))
                        CURR_REQS -= 1
                        return

                    if not download_failures:
                        log("Checking for failures on %s" % task_id)
                        failed = None
                        for artifact in artifacts:
                            if "log_error" in artifact["name"]:
                                CURR_REQS += 1
                                filen = download_artifact(
                                    task_id, artifact, downloads_dir
                                )
                                CURR_REQS -= 1
                                if os.stat(filen).st_size != 0:
                                    failed = artifact["name"]
                        if failed:
                            log("Skipping a failed test: " + failed)
                            return

                    for artifact in artifacts:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -