in lib/roi_data/fast_rcnn_rel.py [0:0]
def _sample_rois_softmax_yall(
unique_all_rois_sbj, unique_all_rois_obj,
unique_sbj_gt_boxes, unique_obj_gt_boxes,
unique_sbj_gt_vecs, unique_obj_gt_vecs,
unique_sbj_gt_labels, unique_obj_gt_labels,
sbj_gt_boxes, obj_gt_boxes,
sbj_gt_vecs, obj_gt_vecs,
rel_gt_vecs, rel_gt_labels,
low_shot_helper):
rois_sbj, pos_vecs_sbj, all_labels_sbj, _, _ = \
_sample_rois_pos_neg_for_one_branch(
unique_all_rois_sbj, unique_sbj_gt_boxes,
unique_sbj_gt_labels, unique_sbj_gt_vecs,
low_shot_helper, 'sbj')
fg_size_sbj = pos_vecs_sbj.shape[0]
pos_starts_sbj = np.array([0, 0], dtype=np.int32)
pos_ends_sbj = np.array([fg_size_sbj, -1], dtype=np.int32)
neg_starts_sbj = np.array([fg_size_sbj, 0], dtype=np.int32)
neg_ends_sbj = np.array([-1, -1], dtype=np.int32)
sbj_pos_labels = all_labels_sbj[:fg_size_sbj] - 1
rois_obj, pos_vecs_obj, all_labels_obj, _, _ = \
_sample_rois_pos_neg_for_one_branch(
unique_all_rois_obj, unique_obj_gt_boxes,
unique_obj_gt_labels, unique_obj_gt_vecs,
low_shot_helper, 'obj')
fg_size_obj = pos_vecs_obj.shape[0]
pos_starts_obj = np.array([0, 0], dtype=np.int32)
pos_ends_obj = np.array([fg_size_obj, -1], dtype=np.int32)
neg_starts_obj = np.array([fg_size_obj, 0], dtype=np.int32)
neg_ends_obj = np.array([-1, -1], dtype=np.int32)
obj_pos_labels = all_labels_obj[:fg_size_obj] - 1
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
fg_rois_per_image = int(
np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
fg_inds_sbj = np.where(all_labels_sbj != 0)[0]
# # Only consider positive sbj rois for rel
# # because during testing we assume each sbj roi is positive
bg_inds_sbj = np.where(all_labels_sbj == 0)[0]
# # we want the background amount to be equal to
# # 0.125 * fg_rois_per_image if not smaller
rel_bg_size_sbj = min(bg_inds_sbj.size, fg_rois_per_image)
if rel_bg_size_sbj < bg_inds_sbj.size:
rel_bg_inds_sbj = \
npr.choice(bg_inds_sbj, size=rel_bg_size_sbj, replace=False)
rel_keep_inds_sbj = np.append(fg_inds_sbj, rel_bg_inds_sbj)
unique_rel_roi_sbj = rois_sbj[rel_keep_inds_sbj]
fg_inds_obj = np.where(all_labels_obj != 0)[0]
# # Only consider positive obj rois for rel
# # because during testing we assume each obj roi is positive
bg_inds_obj = np.where(all_labels_obj == 0)[0]
# # we want the background amount to be equal to
# # 0.125 * fg_rois_per_image if not smaller
rel_bg_size_obj = min(bg_inds_obj.size, fg_rois_per_image)
if rel_bg_size_obj < bg_inds_obj.size:
rel_bg_inds_obj = \
npr.choice(bg_inds_obj, size=rel_bg_size_obj, replace=False)
rel_keep_inds_obj = np.append(fg_inds_obj, rel_bg_inds_obj)
unique_rel_roi_obj = rois_obj[rel_keep_inds_obj]
# create potential relationships by considering all pairs
rel_all_rois_sbj = np.repeat(unique_rel_roi_sbj, len(unique_rel_roi_obj), axis=0)
rel_all_rois_obj = np.tile(unique_rel_roi_obj, (len(unique_rel_roi_sbj), 1))
rel_all_rois_prd = box_union(rel_all_rois_sbj, rel_all_rois_obj)
rel_overlaps_sbj = box_utils.bbox_overlaps(
rel_all_rois_sbj[:, 1:5].astype(dtype=np.float32, copy=False),
sbj_gt_boxes[:, :4].astype(dtype=np.float32, copy=False))
rel_overlaps_obj = box_utils.bbox_overlaps(
rel_all_rois_obj[:, 1:5].astype(dtype=np.float32, copy=False),
obj_gt_boxes[:, :4].astype(dtype=np.float32, copy=False))
# sample foreground candidates
overlaps_pair_min = np.minimum(rel_overlaps_sbj, rel_overlaps_obj)
max_overlaps_pair_min = overlaps_pair_min.max(axis=1)
gt_assignment_pair_min = overlaps_pair_min.argmax(axis=1)
rel_gt_inds = np.where((max_overlaps_pair_min >= 0.99999))[0]
rel_pos_inds = np.where((max_overlaps_pair_min >= cfg.TRAIN.FG_THRESH) &
(max_overlaps_pair_min < 0.99999))[0]
rel_fg_rois_per_this_image = min(int(fg_rois_per_image),
rel_gt_inds.size + rel_pos_inds.size)
if rel_pos_inds.size > 0 and \
rel_pos_inds.size > fg_rois_per_image - rel_gt_inds.size:
rel_pos_inds = npr.choice(rel_pos_inds,
size=(rel_fg_rois_per_this_image - rel_gt_inds.size),
replace=False)
rel_fg_inds = np.append(rel_pos_inds, rel_gt_inds)
if rel_fg_inds.size > fg_rois_per_image:
rel_fg_inds = npr.choice(rel_fg_inds, size=fg_rois_per_image, replace=False)
rel_bg_inds = np.where((max_overlaps_pair_min < cfg.TRAIN.BG_THRESH_HI))[0]
rel_bg_rois_per_this_image = min(rois_per_image - rel_fg_inds.size,
rois_per_image - fg_rois_per_image,
rel_bg_inds.size)
if rel_bg_inds.size > 0:
rel_bg_inds = npr.choice(rel_bg_inds,
size=rel_bg_rois_per_this_image,
replace=False)
rel_fg_inds = np.append(rel_pos_inds, rel_gt_inds)
# duplicate low-shot predicates to increase their chances to be chosen
if cfg.TRAIN.OVERSAMPLE2:
rel_pos_labels = rel_gt_labels[gt_assignment_pair_min[rel_fg_inds]] - 1
low_shot_inds = \
np.array([rel_fg_inds[i] for i, p in enumerate(rel_pos_labels) if
low_shot_helper.check_low_shot_p([-1, p, -1])], dtype=np.int32)
rel_fg_inds = np.append(low_shot_inds, rel_fg_inds)
if rel_fg_inds.size > fg_rois_per_image:
rel_fg_inds = npr.choice(rel_fg_inds, size=fg_rois_per_image, replace=False)
rel_bg_inds = np.where((max_overlaps_pair_min < cfg.TRAIN.BG_THRESH_HI))[0]
rel_bg_rois_per_this_image = min(rois_per_image - rel_fg_inds.size,
rois_per_image - fg_rois_per_image,
rel_bg_inds.size)
if rel_bg_inds.size > 0:
rel_bg_inds = npr.choice(rel_bg_inds,
size=rel_bg_rois_per_this_image,
replace=False)
# This oversampling method has redundant computation on those
# low-shot ROIs, but it's flexible in that those low-shot ROIs
# can be fed into the oversampler immediately after ROI-pooling,
# instead of as late as after fc7
if cfg.TRAIN.OVERSAMPLE:
# Only consider low-shot on P
rel_pos_labels = rel_gt_labels[gt_assignment_pair_min[rel_fg_inds]] - 1
# low_shot_inds contains one dummy ROI at the very beginning
# This is to make sure that low_shot ROIs are never empty
low_shot_inds = \
np.array([rel_fg_inds[i] for i, p in enumerate(rel_pos_labels) if
low_shot_helper.check_low_shot_p([-1, p, -1])], dtype=np.int32)
rel_fg_inds = np.append(low_shot_inds, rel_fg_inds)
if cfg.TRAIN.ADD_LOSS_WEIGHTS:
# low-shot on P
rel_pos_labels = rel_gt_labels[gt_assignment_pair_min[rel_fg_inds]] - 1
rel_pos_weights = np.ones_like(rel_pos_labels, dtype=np.float32)
low_shot_idx = \
np.array([i for i, p in enumerate(rel_pos_labels) if
low_shot_helper.check_low_shot_p([-1, p, -1])], dtype=np.int32)
rel_pos_weights[low_shot_idx] *= 2.0
rel_pos_weights /= np.mean(rel_pos_weights)
if cfg.TRAIN.ADD_LOSS_WEIGHTS_SO:
# low-shot on S
sbj_pos_weights = np.ones_like(sbj_pos_labels, dtype=np.float32)
low_shot_idx = \
np.array([i for i, s in enumerate(sbj_pos_labels) if
low_shot_helper.check_low_shot_s([s, -1, -1])], dtype=np.int32)
sbj_pos_weights[low_shot_idx] *= 2.0
sbj_pos_weights /= np.mean(sbj_pos_weights)
# low-shot on O
obj_pos_weights = np.ones_like(obj_pos_labels, dtype=np.float32)
low_shot_idx = \
np.array([i for i, o in enumerate(obj_pos_labels) if
low_shot_helper.check_low_shot_o([-1, -1, o])], dtype=np.int32)
obj_pos_weights[low_shot_idx] *= 2.0
obj_pos_weights /= np.mean(obj_pos_weights)
rel_keep_inds = np.append(rel_fg_inds, rel_bg_inds)
rel_rois_sbj = rel_all_rois_sbj[rel_keep_inds]
rel_rois_obj = rel_all_rois_obj[rel_keep_inds]
rel_rois_prd = rel_all_rois_prd[rel_keep_inds]
all_labels_rel = rel_gt_labels[gt_assignment_pair_min[rel_keep_inds]]
rel_pos_labels = all_labels_rel[:rel_fg_inds.size] - 1
pos_starts_rel = np.array([0, 0], dtype=np.int32)
pos_ends_rel = np.array([rel_fg_inds.size, -1], dtype=np.int32)
neg_starts_rel = np.array([rel_fg_inds.size, 0], dtype=np.int32)
neg_ends_rel = np.array([-1, -1], dtype=np.int32)
blob = dict(
sbj_rois=rois_sbj,
obj_rois=rois_obj,
rel_rois_sbj=rel_rois_sbj,
rel_rois_obj=rel_rois_obj,
rel_rois_prd=rel_rois_prd,
sbj_pos_labels_int32=sbj_pos_labels.astype(np.int32, copy=False),
obj_pos_labels_int32=obj_pos_labels.astype(np.int32, copy=False),
rel_pos_labels_int32=rel_pos_labels.astype(np.int32, copy=False),
sbj_pos_starts=pos_starts_sbj,
obj_pos_starts=pos_starts_obj,
rel_pos_starts=pos_starts_rel,
sbj_pos_ends=pos_ends_sbj,
obj_pos_ends=pos_ends_obj,
rel_pos_ends=pos_ends_rel,
sbj_neg_starts=neg_starts_sbj,
obj_neg_starts=neg_starts_obj,
rel_neg_starts=neg_starts_rel,
sbj_neg_ends=neg_ends_sbj,
obj_neg_ends=neg_ends_obj,
rel_neg_ends=neg_ends_rel)
if cfg.TRAIN.ADD_LOSS_WEIGHTS:
blob['rel_pos_weights'] = rel_pos_weights
if cfg.TRAIN.ADD_LOSS_WEIGHTS_SO:
blob['sbj_pos_weights'] = sbj_pos_weights
blob['obj_pos_weights'] = obj_pos_weights
return blob