in compare_models.py [0:0]
def __accumulate_metrics(self, inp, targ, vis, outputs, other_metrics):
threshold = 0.1 / self.args.divide_by # 10% of the value when 1
inp_np = inp.cpu().data.clone().numpy() # clone() in case we're on CPU already
targ_np = targ.cpu().data.clone().numpy() # clone() in case we're on CPU already
hid = 1 - vis
inp_hid, targ_hid = self.view_input_target(hid[:-1], hid[1:])
if self.state.delta:
targ_np = inp_np + targ_np
targ_hid = targ_hid + inp_hid
inp_hid = inp_hid > 0
targ_hid = targ_hid > 0
inf_mask = inp_hid
assert (inp_np < 0).sum() == 0, "Input should be all positive numbers..."
assert (targ_np < 0).sum() == 0, "Input should be all positive numbers..."
inp_our_units = inp_np[:, utm.our_units_inds, :, :]
inp_nmy = inp_np[:, utm.offset:]
inp_nmy_units = inp_np[:, utm.nmy_units_inds, :, :]
inp_nmy_bldgs = inp_np[:, utm.nmy_bldgs_inds, :, :]
inp_sum = inp_np.sum(-1).sum(-1)
inp_our_sum = inp_sum[:, :utm.offset]
inp_nmy_sum = inp_sum[:, utm.offset:]
inp_our_bldgs_sum = inp_sum[:, utm.our_bldgs_inds]
inp_nmy_bldgs_sum = inp_sum[:, utm.nmy_bldgs_inds]
our_units = targ_np[:, utm.our_units_inds, :, :]
nmy_units = targ_np[:, utm.nmy_units_inds, :, :]
nmy_bldgs = targ_np[:, utm.nmy_bldgs_inds, :, :]
nmy_hidden_units = (nmy_units * inf_mask) > 0
nmy_hidden_bldgs = (nmy_bldgs * inf_mask) > 0
targ_sum = targ_np.sum(-1).sum(-1)
our_bldgs_sum = targ_sum[:, utm.our_bldgs_inds]
nmy_bldgs_sum = targ_sum[:, utm.nmy_bldgs_inds]
gold_op_bt = (nmy_bldgs_sum > threshold)
#nmy_units_sum = targ_sum[:, utm.nmy_units_inds]
nmy_hidden_whole = (nmy_units * inf_mask) * self.args.divide_by
e_nmy_units = (nmy_units > 0)
hid_true_positives = []
for i, output in enumerate(outputs):
output_np = output[REGRESSION].cpu().clone().numpy()
# unit_output_np = output[UNIT_CLASS].cpu().clone().numpy()
# bldg_output_np = output[BLDG_CLASS].cpu().clone().numpy()
opbt_output_np = output[OPBT_CLASS].cpu().clone().numpy()
if self.state.delta:
output_np += inp_np
p_our_units = output_np[:, utm.our_units_inds, :, :]
p_nmy_units = output_np[:, utm.nmy_units_inds, :, :]
# p_nmy_bldgs = output_np[:, utm.nmy_bldgs_inds, :, :]
output_sum = output_np.sum(-1).sum(-1)
# p_our_bldgs_sum = output_sum[:, utm.our_bldgs_inds]
p_nmy_bldgs_sum = output_sum[:, utm.nmy_bldgs_inds]
#p_nmy_units_sum = output_sum[:, utm.nmy_units_inds]
logit = lambda x: np.log(x) - np.log(1-x)
if self.args.class_prob_thresh >= 1:
class_thresh = 1e100
elif self.args.class_prob_thresh <= -1:
class_thresh = -1e100
else:
class_thresh = logit(self.args.class_prob_thresh)
our_res = other_metrics[i]
our_res['op_bt'] += compute_prec_recall(p_nmy_bldgs_sum > self.args.n_unit_thresh, gold_op_bt)
p_nmy_units_e = p_nmy_units > self.args.n_unit_thresh
our_res['hid_u'] += compute_prec_recall(p_nmy_units_e * inf_mask, nmy_hidden_units)
our_res['nmy_u'] += compute_prec_recall(p_nmy_units_e, nmy_units > 0)
hid_true_positives.append((p_nmy_units_e * inf_mask * nmy_hidden_units) > 0)
tmp = p_nmy_units * self.args.regr_slope_scalar - self.args.n_unit_thresh
our_res[str(i) + '_L1'] += abs(tmp - nmy_units).sum()
our_res[str(i) + '_SmoothL1'] += HuberLoss(Variable(th.from_numpy(tmp), requires_grad=False), Variable(th.from_numpy(nmy_units), requires_grad=False)).data[0]
our_res[str(i)+'_L1'] += abs((p_nmy_units) - nmy_units).sum()
our_res['p_op_bt'] += compute_prec_recall(opbt_output_np > class_thresh, gold_op_bt)
# baselines
end = len(outputs)
bl_inp = other_metrics[end + 0]
bl_mem = other_metrics[end + 1]
bl_mem_r = other_metrics[end + 2]
bl_mem_p = other_metrics[end + 3]
# baseline_input
bl_inp['op_bt'] += compute_prec_recall(inp_nmy_bldgs_sum > threshold, gold_op_bt)
bl_inp['hid_u'] += compute_prec_recall((inp_nmy_units * inf_mask) > threshold, nmy_hidden_units)
for i, _ in enumerate(outputs):
bl_inp[str(i) + '_L1'] += abs(inp_nmy_units - nmy_units).sum()
bl_inp[str(i) + '_SmoothL1'] += HuberLoss(Variable(th.from_numpy(inp_nmy_units), requires_grad=False), Variable(th.from_numpy(nmy_units), requires_grad=False)).data[0]
bl_inp['p_op_bt'] = bl_inp['op_bt']
# baseline_memory
max_our_bldgs_sum = np.maximum.accumulate(inp_our_bldgs_sum)
max_nmy_bldgs_sum = np.maximum.accumulate(inp_nmy_bldgs_sum)
max_nmy_bldgs = np.maximum.accumulate(inp_nmy_bldgs)
max_nmy_units = np.maximum.accumulate(inp_nmy_units)
max_our_sum = np.maximum.accumulate(inp_our_sum)
max_nmy_sum = np.maximum.accumulate(inp_nmy_sum)
max_nmy = np.maximum.accumulate(inp_nmy)
bl_mem['op_bt'] += compute_prec_recall(max_nmy_bldgs_sum > threshold, gold_op_bt)
bl_mem['hid_u'] += compute_prec_recall((max_nmy_units * inf_mask) > threshold, nmy_hidden_units)
bl_mem['nmy_u'] += compute_prec_recall(max_nmy_units > threshold, nmy_units > 0)
for i, _ in enumerate(outputs):
bl_mem[str(i) + '_L1'] += abs(max_nmy_units - nmy_units).sum()
bl_mem[str(i) + '_SmoothL1'] += HuberLoss(Variable(th.from_numpy(max_nmy_units), requires_grad=False), Variable(th.from_numpy(nmy_units), requires_grad=False)).data[0]
bl_mem['p_op_bt'] = bl_mem['op_bt']
# baseline_memory_rules
inp_nmy_mem_r = utm.ttrules(max_nmy.reshape(-1, max_nmy.shape[1]) > threshold).reshape(max_nmy.shape)
bl_mem_r['op_bt'] += compute_prec_recall(utm.ttrules(max_nmy_sum > threshold)[:, utm.our_bldgs_inds], gold_op_bt)
bl_mem_r['hid_u'] += compute_prec_recall(inp_nmy_mem_r[:, utm.our_units_inds, :, :] * inf_mask > threshold, nmy_hidden_units) # fishy
bl_mem_r['nmy_u'] += compute_prec_recall(inp_nmy_mem_r[:, utm.our_units_inds, :, :] > threshold, nmy_units > 0)
bl_mem_r['0_L1'] += -1
bl_mem_r['1_L1'] += -1
bl_mem_r['0_SmoothL1'] += -1
bl_mem_r['1_SmoothL1'] += -1
bl_mem_r['p_op_bt'] = bl_mem_r['op_bt']
inp_mem_prev = inp_np.copy()
for i in range(1, inp_np.shape[0]):
inp_mem_prev[i] = (1 - inp_hid[i]) * inp_np[i] + inp_hid[i] * inp_mem_prev[i-1]
mem_prev_nmy_units = inp_mem_prev[:, utm.nmy_units_inds, :, :]
mem_prev_nmy_bldgs = inp_mem_prev[:, utm.nmy_bldgs_inds, :, :]
mem_prev_our_bldgs_sum = inp_mem_prev[:, utm.our_bldgs_inds, :, :].sum(-1).sum(-1)
mem_prev_nmy_bldgs_sum = inp_mem_prev[:, utm.nmy_bldgs_inds, :, :].sum(-1).sum(-1)
bl_mem_p['op_bt'] += compute_prec_recall(mem_prev_nmy_bldgs_sum > threshold, gold_op_bt)
bl_mem_p['hid_u'] += compute_prec_recall((mem_prev_nmy_units * inf_mask) > threshold, nmy_hidden_units)
bl_mem_p['nmy_u'] += compute_prec_recall(mem_prev_nmy_units > threshold, nmy_units > 0)
for i, _ in enumerate(outputs):
bl_mem_p[str(i) + '_L1'] += abs(mem_prev_nmy_units - nmy_units).sum()
bl_mem_p[str(i) + '_SmoothL1'] += HuberLoss(Variable(th.from_numpy(mem_prev_nmy_units), requires_grad=False), Variable(th.from_numpy(nmy_units), requires_grad=False)).data[0]
bl_mem_p['p_op_bt'] = bl_mem_p['op_bt']