def load_experiment()

in modelling/src/neuraldb/final_scoring_with_dbsize.py [0:0]

• 54 lines of code
• 13 McCabe index (conditional complexity)

def load_experiment(path):

    running_score = defaultdict(lambda: defaultdict(int))
    running_count = defaultdict(lambda: defaultdict(int))

    print(path)
    with open(path) as f:
        for line in f:
            instance = json.loads(line)
            actual = instance["actual"]
            prediction = instance["prediction"]

            if "dbsize" not in instance["metadata"]:
                db_idx, q_idx = (
                    instance["metadata"]["database_idx"],
                    instance["metadata"]["question_idx"],
                )
                dbsize = db_sizes[(db_idx, q_idx)]
            else:
                dbsize = instance["metadata"]["dbsize"]

            if dbsize == 0:
                dbsize = "0"
            elif dbsize == 1:
                dbsize = "1"
            elif dbsize < 5:
                dbsize = "2-4"
            elif dbsize < 10:
                dbsize = "5-9"
            elif dbsize < 20:
                dbsize = "10-19"
            else:
                dbsize = "20+"

            local_score = f1(set(actual), set(prediction))

            # relation = instance["metadata"]["relation"]
            # running_score["relation"][relation] += local_score
            # running_count["relation"][relation] += 1

            qtype = instance["metadata"]["type"]
            if qtype in {"argmin", "argmax", "min", "max"}:
                qtype = "minmax"

            running_score["type"][qtype] += local_score
            running_count["type"][qtype] += 1

            running_score["size"][dbsize] += local_score
            running_count["size"][dbsize] += 1

            running_score["all"][""] += local_score
            running_count["all"][""] += 1

    scores = {}
    for k, v in running_score.items():
        for attr, val in v.items():
            score = (
                running_score[k][attr] / running_count[k][attr]
                if running_count[k][attr]
                else 0
            )
            print(f"Running score: {k}\t{attr}\t\t{score}")
            scores["_".join([k, attr])] = (
                running_score[k][attr] / running_count[k][attr]
                if running_count[k][attr]
                else 0
            )

    return scores