in expanded_checklist/checklist/eval_core.py [0:0]
def example_list_and_indices(self, n=None, seed=None):
"""Subsamples test cases
Parameters
----------
n : int
Number of testcases to sample
seed : int
Seed to use
Returns
-------
tuple(list, list)
First list is a list of examples
Second list maps examples to testcases.
For example, let's say we have two testcases: [a, b, c] and [d, e].
The first list will be [a, b, c, d, e]
the second list will be [0, 0, 0, 1, 1]
Also updates self.run_idxs if n is not None to indicate which testcases
were run. Also updates self.result_indexes with the second list.
"""
if seed is not None:
np.random.seed(seed)
self.run_idxs = None
idxs = list(range(len(self.data)))
if n is not None:
idxs = np.random.choice(idxs, min(n, len(idxs)), replace=False)
self.run_idxs = idxs
if is_1d_list(self.data[0]):
all_data = [
(i, y, m) for i in idxs
for (y, m) in zip(self.data[i], self.meta[i])]
result_indexes, examples, meta = map(list, list(zip(*all_data)))
# e.g. for each tempalte there are many groups and each group
# can be represented with many terms
elif is_2d_list(self.data[0]):
all_data = []
for i in idxs:
example_data = self.data[i]
example_meta = self.meta[i]
for group_idx, sent_list in enumerate(example_data):
for y in sent_list:
all_data.append(((i, group_idx), y, example_meta))
result_indexes, examples, meta = map(list, list(zip(*all_data)))
else:
examples = [self.data[i] for i in idxs]
meta = [self.meta[i] for i in idxs]
result_indexes = idxs # list(range(len(self.data)))
self.result_indexes = result_indexes
return examples, meta, result_indexes