in lib/modeling/VGG16_rel_softmaxed_triplet.py [0:0]
def add_embd_triplet_losses_labeled(model, label):
prefix = label + '_'
suffix = '_' + label
if label.find('rel') >= 0:
num_neg_classes = len(model.roi_data_loader._landb._predicate_categories) - 1
else:
num_neg_classes = len(model.roi_data_loader._landb._object_categories) - 1
model.net.ConstantFill(
[], 'reciprocal_num_neg_classes_blob' + suffix,
shape=[1], value=1.0 / float(num_neg_classes))
model.net.SquaredL2Distance(['xp' + suffix, 'yp' + suffix],
'dist_xp_yp' + suffix)
if label.find('rel') >= 0:
yall_name = 'all_prd_lan_embds'
else:
yall_name = 'all_obj_lan_embds'
model.net.MatMul(['scaled_xp' + suffix, 'scaled_' + yall_name],
'sim_xp_yall' + suffix, trans_b=1)
if (label.find('rel') >= 0 and cfg.TRAIN.ADD_LOSS_WEIGHTS) or \
((label.find('sbj') >= 0 or label.find('obj') >= 0) and
cfg.TRAIN.ADD_LOSS_WEIGHTS_SO):
_, loss_xp_yall = model.net.SoftmaxWithLoss(
['sim_xp_yall' + suffix,
prefix + 'pos_labels_int32',
prefix + 'pos_weights'],
['xp_yall_prob' + suffix, 'loss_xp_yall' + suffix],
scale=1. / cfg.NUM_DEVICES)
else:
_, loss_xp_yall = model.net.SoftmaxWithLoss(
['sim_xp_yall' + suffix, prefix + 'pos_labels_int32'],
['xp_yall_prob' + suffix, 'loss_xp_yall' + suffix],
scale=1. / cfg.NUM_DEVICES)
model.loss_set.extend([loss_xp_yall])
if cfg.MODEL.SPECS.find('no_xpypxn') < 0:
model.net.MatMul(['yp' + suffix, 'x' + suffix],
'sim_yp_xall_raw' + suffix, trans_b=1)
model.net.Sub(['sim_yp_xall_raw' + suffix, 'one_blob'],
'neg_dist_yp_xall' + suffix, broadcast=1)
model.net.Add(['neg_dist_yp_xall' + suffix, 'dist_xp_yp' + suffix],
'diff_dist_xp_yp_xall' + suffix, broadcast=1, axis=0)
model.net.Add(['diff_dist_xp_yp_xall' + suffix, 'margin_blob' + suffix],
'margin_xp_yp_xall' + suffix, broadcast=1)
model.net.Relu('margin_xp_yp_xall' + suffix, 'max_margin_xp_yp_xall' + suffix)
model.net.Mul(['max_margin_xp_yp_xall' + suffix, prefix + 'neg_affinity_mask'],
'max_margin_xp_yp_xn' + suffix)
mean_max_margin_xp_yp_xn = model.net.ReduceBackMean(
'max_margin_xp_yp_xn' + suffix, 'mean_max_margin_xp_yp_xn' + suffix)
if (label.find('rel') >= 0 and cfg.TRAIN.ADD_LOSS_WEIGHTS) or \
((label.find('sbj') >= 0 or label.find('obj') >= 0) and
cfg.TRAIN.ADD_LOSS_WEIGHTS_SO):
mean_max_margin_xp_yp_xn = model.net.Mul(
['mean_max_margin_xp_yp_xn' + suffix, prefix + 'pos_weights'],
'mean_max_margin_xp_yp_xn_weighted' + suffix)
loss_yp_xn = mean_max_margin_xp_yp_xn.AveragedLoss([], ['loss_yp_xn' + suffix])
model.loss_set.extend([loss_yp_xn])
if cfg.MODEL.SPECS.find('w_cluster') >= 0:
# 1. get neg_dist_xp_xn
model.net.MatMul(['xp' + suffix, 'x' + suffix],
'sim_xp_xall_raw' + suffix, trans_b=1)
model.net.Sub(['sim_xp_xall_raw' + suffix, 'one_blob'],
'neg_dist_xp_xall' + suffix, broadcast=1)
model.net.Mul(['neg_dist_xp_xall' + suffix, prefix + 'neg_affinity_mask'],
'neg_dist_xp_xn' + suffix)
# 2. get mean_max_dist_xp_xp
model.net.Negative('neg_dist_xp_xall' + suffix, 'dist_xp_xall' + suffix)
model.net.Sub([prefix + 'neg_affinity_mask', 'one_blob'],
prefix + 'neg_pos_affinity_mask', broadcast=1)
model.net.Negative(prefix + 'neg_pos_affinity_mask',
prefix + 'pos_affinity_mask')
model.net.Mul(['dist_xp_xall' + suffix, prefix + 'pos_affinity_mask'],
'dist_xp_xp' + suffix)
model.net.TopK('dist_xp_xp' + suffix,
['max_dist_xp_xp' + suffix, '_idx_max_dist_xp_xp' + suffix], k=1)
model.net.Squeeze('max_dist_xp_xp' + suffix,
'max_dist_xp_xp' + suffix, dims=[1])
model.net.Add(['neg_dist_xp_xn' + suffix, 'max_dist_xp_xp' + suffix],
'diff_dist_xp_xp_xn' + suffix, broadcast=1, axis=0)
model.net.Add(['diff_dist_xp_xp_xn' + suffix, 'margin_blob' + suffix],
'margin_xp_xp_xn' + suffix, broadcast=1)
model.net.Relu('margin_xp_xp_xn' + suffix, 'max_margin_xp_xp_xn' + suffix)
mean_max_margin_xp_xp_xn = model.net.ReduceBackMean(
'max_margin_xp_xp_xn' + suffix, 'mean_max_margin_xp_xp_xn' + suffix)
if (label.find('rel') >= 0 and cfg.TRAIN.ADD_LOSS_WEIGHTS) or \
((label.find('sbj') >= 0 or label.find('obj') >= 0) and
cfg.TRAIN.ADD_LOSS_WEIGHTS_SO):
mean_max_margin_xp_xp_xn = model.net.Mul(
['mean_max_margin_xp_xp_xn' + suffix, prefix + 'pos_weights'],
'mean_max_margin_xp_xp_xn_weighted' + suffix)
loss_xp_xn = mean_max_margin_xp_xp_xn.AveragedLoss([], ['loss_xp_xn' + suffix])
model.loss_set.extend([loss_xp_xn])