def print_stats()

in codes/word_randomization.py [0:0]

• 82 lines of code
• 6 McCabe index (conditional complexity)

    def print_stats(self, model_name, agg_out):
        """Compare stats with the aggregated output

        Args:
            model_name ([type]): [description]
            agg_out ([type]): [description]
        """
        stats = {}

        def cprint(key, val):
            stats[key] = val
            print(f"{key} : {val}")

        orig_out = pd.DataFrame(self.orig_data_outp)
        if model_name == "chinese-roberta-large":
            orig_out.uid = orig_out.uid.astype(int)
        # orig_acc = len(orig_out[orig_out.is_correct]) / len(orig_out)
        cprint("Model", model_name)
        print("========================")
        # cprint("Original Accuracy", orig_acc)
        rand_out = pd.DataFrame(agg_out)

        if self.args.hyp_distinct_perm:
            print(
                f"Warning: reducing eval data from {len(rand_out)} to {len(rand_out[rand_out.hyp_distinct_perm])}"
            )
            rand_out = rand_out[rand_out.hyp_distinct_perm]
        if self.args.prem_distinct_perm:
            print(
                f"Warning: reducing eval data from {len(rand_out)} to {len(rand_out[rand_out.prem_distinct_perm])}"
            )
            rand_out = rand_out[rand_out.prem_distinct_perm]
        rand_acc = len(rand_out[rand_out.exists]) / len(rand_out)
        orig_out_subset = orig_out[orig_out.uid.isin(rand_out.uid)]
        orig_acc = len(orig_out_subset[orig_out_subset.is_correct]) / len(
            orig_out_subset
        )
        cprint("Original Accuracy", orig_acc)
        print(f"Chosen number: {len(rand_out)}")
        cprint("Max Accuracy", rand_acc)
        print("==============")
        print("Originally Correct")
        print("-------------")
        for mode in ["orig_correct", "flipped"]:
            corr_num = len(rand_out[(rand_out.exists) & (rand_out.pred_mode == mode)])
            corr_per = np.round(
                len(rand_out[(rand_out.exists) & (rand_out.pred_mode == mode)])
                / len(rand_out),
                3,
            )
            cprint(f"{mode}_num", corr_num)
            cprint(f"{mode}_percent", corr_per)
            print(f"mode: {mode}")
            for target in ["e", "n", "c"]:
                t_mean = rand_out[
                    (rand_out.pred_mode == mode)
                    & (rand_out.label == target)
                    & rand_out.exists
                ][f"prob_{target}_mean"].mean()
                t_std = rand_out[
                    (rand_out.pred_mode == mode)
                    & (rand_out.label == target)
                    & rand_out.exists
                ][f"prob_{target}_mean"].std()
                cprint(
                    f"{mode}_{target}",
                    f"{np.round(t_mean, 3)} +/- {np.round(t_std, 3)}",
                )
            cor_mean = rand_out[
                (rand_out.pred_mode == mode) & (rand_out.exists)
            ].cor_mean.mean()
            cprint(f"{mode}_cor_mean", cor_mean)
            print("==============")
        cprint(
            "Majority Acc",
            len(rand_out[rand_out.is_correct_rand_majority]) / len(rand_out),
        )
        cprint(
            "Correct > Random Percentage",
            len(rand_out[rand_out.is_correct_gt_random]) / len(rand_out),
        )
        rand_out.uid = rand_out.uid.astype(str)
        orig_out.uid = orig_out.uid.astype(str)
        jdf = pd.merge(rand_out, orig_out, on="uid")
        cprint(
            "Combined Acc",
            len(jdf[(jdf.is_correct) | (jdf.is_correct_rand_majority)]) / len(jdf),
        )
        print("-------------")
        ## Save stats
        outp_path = Path(self.rand_data_loc) / "outputs"
        outp_path.mkdir(exist_ok=True, parents=True)
        save_jsonl([stats], outp_path / f"{model_name}.jsonl")
        jdf.to_csv(outp_path / f"{model_name}_filtered_rand_outs.csv")