experiments/scripts/eval_supervised.py [110:138]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    results_table = []
    # load all test data
    ev_exp = CheckpointableTestTube(
        config_id=big_config["representation"][mode][0][-1], seed=seed
    )
    ev_exp.config.general.test_rule = "rule_0"
    all_test_data = ev_exp.initialize_data(mode="test", override_mode="train")
    label2id = ev_exp.label2id
    del ev_exp
    for exp in big_config["representation"][mode]:
        print("evaluating experiment {}".format(json.dumps(exp)))
        res_row = copy.deepcopy(row)
        res_row["rep_fn"] = exp[0]
        res_row["comp_fn"] = exp[1]
        ev_exp = CheckpointableTestTube(config_id=exp[-1], seed=seed)
        # perform the modification with the worlds here
        train_worlds = ev_exp.config.general.train_rule.split(",")
        print("evaluating on {} train worlds".format(len(train_worlds)))
        for wi, current_world in enumerate(train_worlds):
            ev_exp = CheckpointableTestTube(config_id=exp[-1], seed=seed)
            ev_exp.config.general.test_rule = current_world + ","
            ev_exp.prepare_evaluator(
                epoch=wi,
                override_mode="train",
                test_data=all_test_data,
                label2id=label2id,
            )
            pr_current = ev_exp.evaluator.evaluate()
            if wi > 0:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



experiments/scripts/eval_supervised.py [162:190]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    results_table = []
    # load all test data
    ev_exp = CheckpointableTestTube(
        config_id=big_config["representation"][mode][0][-1], seed=seed
    )
    ev_exp.config.general.test_rule = "rule_0"
    all_test_data = ev_exp.initialize_data(mode="test", override_mode="train")
    label2id = ev_exp.label2id
    del ev_exp
    for exp in big_config["representation"][mode]:
        print("evaluating experiment {}".format(json.dumps(exp)))
        res_row = copy.deepcopy(row)
        res_row["rep_fn"] = exp[0]
        res_row["comp_fn"] = exp[1]
        ev_exp = CheckpointableTestTube(config_id=exp[-1], seed=seed)
        # perform the modification with the worlds here
        train_worlds = ev_exp.config.general.train_rule.split(",")
        print("evaluating on {} train worlds".format(len(train_worlds)))
        for wi, current_world in enumerate(train_worlds):
            ev_exp = CheckpointableTestTube(config_id=exp[-1], seed=seed)
            ev_exp.config.general.test_rule = current_world + ","
            ev_exp.prepare_evaluator(
                epoch=wi,
                override_mode="train",
                test_data=all_test_data,
                label2id=label2id,
            )
            pr_current = ev_exp.evaluator.evaluate()
            if wi > 0:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -