in evaluation/evaluate.py [0:0]
def evaluate(benchmark: str, dataset_id: str, dataset_config: str = None, dataset_split: str = "test", dataset_col: str = "pred", samples: list=None, max_num_samples=None):
samples = load_dataset(dataset_id, name=dataset_config, split=dataset_split)
# Sanity check we have unique number of MATH problems
if benchmark == "math" and len(samples.unique("problem")) != len(samples):
raise ValueError(f"Dataset contains duplicate math problems. Found {len(samples.unique('problem'))} unique problems out of {len(samples)} samples")
if "idx" not in samples.column_names:
samples = samples.map(lambda x, idx: {"idx": idx}, with_indices=True)
if max_num_samples:
print(f"max_num_samples: {max_num_samples} / {len(samples)}")
samples = samples[:max_num_samples]
def parse_gt(x):
x['gt_cot'], x['gt'] = parse_ground_truth(x, benchmark)
return x
samples = samples.map(parse_gt, desc="Parsing ground truth", num_proc=4, load_from_cache_file=False)
samples = samples.map(extract_answer_map, fn_kwargs={"data_name": benchmark, "col": dataset_col}, desc="Parsing predictions", num_proc=4, load_from_cache_file=False)
params = [(idx, pred, gt) for idx, pred, gt in zip(samples['idx'], samples['pred'], samples['gt'])]
scores = []
timeout_cnt = 0
with ProcessPool(max_workers=8) as pool:
future = pool.map(math_equal_process, params, timeout=3)
iterator = future.result()
with tqdm(total=len(samples), desc="Evaluate") as progress_bar:
while True:
try:
result = next(iterator)
scores.append(result)
except StopIteration:
break
except TimeoutError as error:
print(error)
scores.append(False)
timeout_cnt += 1
except Exception as error:
print(error.traceback)
exit()
progress_bar.update(1)
mean_score = np.mean(scores) * 100
result_json = {
"num_samples": len(samples),
"num_scores": len(scores),
"timeout_samples": timeout_cnt,
"acc": mean_score
}
print(result_json)
return samples, result_json