def can_infer_option()

in lmms_eval/tasks/mmbench/mmbench_evals.py [0:0]

• 25 lines of code
• 14 McCabe index (conditional complexity)

    def can_infer_option(self, answer, num_choice=5):
        choices = string.ascii_uppercase[:num_choice]
        if "Failed to obtain answer via API" in answer:
            return False

        def count(splits, choices="ABCD", prefix="", suffix=""):
            cnt = 0
            for c in choices:
                if prefix + c + suffix in splits:
                    cnt += 1
            return cnt

        splits = [x.strip() for x in answer.split()]
        if count(splits, choices) == 1:
            for ch in choices:
                if "A" in splits and len(splits) > 3:
                    eval_logger.info(f"A might be a quantifier in the string: {answer}.")
                    break
                if ch in splits:
                    return ch
        tups = [("", "."), ("", ","), ("", ":"), ("", ")"), ("", ")."), ("(", ")"), ("(", ")."), (":", ""), (":", ","), (":", "."), (":", ")"), (":", ").")]
        for tup in tups:
            if count(splits, choices, prefix=tup[0], suffix=tup[1]) == 1:
                for ch in choices:
                    if tup[0] + ch + tup[1] in splits:
                        return ch
        return False