in domainbed/model_selection.py [0:0]
def _step_acc(self, records):
"""Return the {val_acc, test_acc} for a group of records corresponding
to a single step."""
test_records = get_test_records(records)
if len(test_records) != 1:
return None
test_env = test_records[0]['args']['test_envs'][0]
n_envs = 0
for i in itertools.count():
if f'env{i}_out_acc' not in records[0]:
break
n_envs += 1
val_accs = np.zeros(n_envs) - 1
for r in records.filter(lambda r: len(r['args']['test_envs']) == 2):
val_env = (set(r['args']['test_envs']) - set([test_env])).pop()
val_accs[val_env] = r['env{}_in_acc'.format(val_env)]
val_accs = list(val_accs[:test_env]) + list(val_accs[test_env+1:])
if any([v==-1 for v in val_accs]):
return None
val_acc = np.sum(val_accs) / (n_envs-1)
return {
'val_acc': val_acc,
'test_acc': test_records[0]['env{}_in_acc'.format(test_env)]
}