def collect_results()

in prediction_generation/make_table.py [0:0]

• 41 lines of code
• 16 McCabe index (conditional complexity)

def collect_results(summary_dir=None, metric=None, experiment=None):
    """Collect the results for the experiment on the specified metric.

    Returns a list of Result objects.
    """
    if not isinstance(metric, Metric):
        raise ValueError("Unknown metric: %s" % metric)
    if experiment not in ["default", "best"]:
        raise ValueError("Unknown experiment: %s" % experiment)
    if not os.path.isdir(summary_dir):
        raise FileNotFoundError(summary_dir)

    results = []
    for fname in sorted(os.listdir(summary_dir)):
        path = os.path.join(summary_dir, fname)
        summary_data = load_summary(path)

        dataset_name = summary_data["dataset"]
        if dataset_name not in DATASETS:
            raise ValueError("Unknown dataset: %s" % dataset_name)
        summary_results = summary_data["results"]

        for method in summary_results:
            # method names are prefixed with the experiment type, so we skip
            # the ones we don't want
            if not method.startswith(experiment + "_"):
                continue

            # extract the metric score for this experiment from the summary
            # results for the method
            score = extract_score(
                summary_results[method], metric=metric, experiment=experiment
            )

            # strip the experiment from the method name
            method_name = method[len(experiment + "_"):]

            # determine the placeholder value if there is no score.
            placeholder = set()
            if score is None:
                for result in summary_results[method]:
                    if result["status"] == "FAIL":
                        placeholder.add("F")
                    elif result["status"] == "TIMEOUT":
                        placeholder.add("T")
            placeholder = "/".join(sorted(placeholder))

            # create a Result object
            res = Result(
                dataset=dataset_name,
                experiment=Experiment(experiment),
                method=Method(method_name),
                metric=Metric(metric),
                score=score,
                summary_file=fname,
                placeholder=placeholder or None,
            )
            results.append(res)
    return results