def __init__()

in simpleqa_eval.py [0:0]

• 11 lines of code
• 3 McCabe index (conditional complexity)

    def __init__(self, grader_model: SamplerBase, num_examples: int | None = None, n_repeats: int = 1):
        df = pandas.read_csv(
            "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv"
        )
        examples = [row.to_dict() for _, row in df.iterrows()]
        if num_examples:
            assert n_repeats == 1, "n_repeats only supported when max_examples = None"
            rng = random.Random(0)
            examples = rng.sample(examples, num_examples)
        self.examples = examples * n_repeats
        self.grader_model = grader_model