in lib/datasets/vg_wiki_and_relco.py [0:0]
def _load_vg_annotation(self, img_rels, index, cnt, length):
"""
Load image and bounding boxes info.
"""
print("Loading image %d/%d..." % (cnt + 1, length))
assert index == img_rels['image_id'] # sanity check
num_rels = len(img_rels['relationships'])
sbj_boxes = np.zeros((num_rels, 4), dtype=np.uint16)
obj_boxes = np.zeros((num_rels, 4), dtype=np.uint16)
rel_boxes = np.zeros((num_rels, 4), dtype=np.uint16)
sbj_names = np.zeros((num_rels), dtype='U100')
obj_names = np.zeros((num_rels), dtype='U100')
prd_names = np.zeros((num_rels), dtype='U100')
gt_sbj_classes = np.zeros((num_rels), dtype=np.int32)
gt_obj_classes = np.zeros((num_rels), dtype=np.int32)
gt_rel_classes = np.zeros((num_rels), dtype=np.int32)
sbj_overlaps = \
np.zeros((num_rels, self._num_object_classes), dtype=np.float32)
obj_overlaps = \
np.zeros((num_rels, self._num_object_classes), dtype=np.float32)
rel_overlaps = \
np.zeros((num_rels, self._num_predicate_classes), dtype=np.float32)
# "Seg" area for pascal is just the box area
sbj_seg_areas = np.zeros((num_rels), dtype=np.float32)
obj_seg_areas = np.zeros((num_rels), dtype=np.float32)
rel_seg_areas = np.zeros((num_rels), dtype=np.float32)
# variables for word vectors
half_dim = int(cfg.INPUT_LANG_EMBEDDING_DIM / 2)
sbj_vecs = np.zeros(
(num_rels, cfg.INPUT_LANG_EMBEDDING_DIM), dtype=np.float32)
obj_vecs = np.zeros(
(num_rels, cfg.INPUT_LANG_EMBEDDING_DIM), dtype=np.float32)
prd_vecs = np.zeros(
(num_rels, cfg.INPUT_LANG_EMBEDDING_DIM), dtype=np.float32)
# Load object bounding boxes into a data frame.
for ix, rel in enumerate(img_rels['relationships']):
sbj = rel['subject']
obj = rel['object']
prd = rel['predicate']
sbj_box = [sbj['x'], sbj['y'], sbj['x'] + sbj['w'], sbj['y'] + sbj['h']]
obj_box = [obj['x'], obj['y'], obj['x'] + obj['w'], obj['y'] + obj['h']]
rel_box = box_utils.box_union(sbj_box, obj_box)
sbj_boxes[ix, :] = sbj_box
obj_boxes[ix, :] = obj_box
rel_boxes[ix, :] = rel_box
sbj_names[ix] = sbj['name']
obj_names[ix] = obj['name']
prd_names[ix] = prd
sbj_cls = self._object_class_to_ind[str(sbj_names[ix])]
obj_cls = self._object_class_to_ind[str(obj_names[ix])]
prd_cls = self._predicate_class_to_ind[str(prd_names[ix])]
gt_sbj_classes[ix] = sbj_cls
gt_obj_classes[ix] = obj_cls
gt_rel_classes[ix] = prd_cls
sbj_overlaps[ix, sbj_cls] = 1.0
obj_overlaps[ix, obj_cls] = 1.0
rel_overlaps[ix, prd_cls] = 1.0
sbj_seg_areas[ix] = (sbj_box[2] - sbj_box[0] + 1) * \
(sbj_box[3] - sbj_box[1] + 1)
obj_seg_areas[ix] = (obj_box[2] - obj_box[0] + 1) * \
(obj_box[3] - obj_box[1] + 1)
rel_seg_areas[ix] = (rel_box[2] - rel_box[0] + 1) * \
(rel_box[3] - rel_box[1] + 1)
# add word vectors for sbjs, objs and rels
# sbj word2vec
sbj_vecs_wiki = np.zeros(half_dim, dtype=np.float32)
sbj_words = sbj_names[ix].split()
for word in sbj_words:
if word in self.model.vocab:
raw_word = self.model[word]
sbj_vecs_wiki += (raw_word / la.norm(raw_word))
else:
print('Singular word found: ', word)
raise NameError('Terminated.')
sbj_vecs_wiki /= len(sbj_words)
sbj_vecs_wiki /= la.norm(sbj_vecs_wiki)
sbj_vecs_relco = np.zeros(half_dim, dtype=np.float32)
sbj_words = sbj_names[ix].split()
for word in sbj_words:
if word in self.relco_model.wv.vocab:
raw_word = self.relco_model[word]
sbj_vecs_relco += (raw_word / la.norm(raw_word))
else:
sbj_vecs_relco += \
(self.relco_vec_mean / la.norm(self.relco_vec_mean))
sbj_vecs_relco /= len(sbj_words)
sbj_vecs_relco /= la.norm(sbj_vecs_relco)
sbj_vecs[ix][:half_dim] = sbj_vecs_wiki
sbj_vecs[ix][half_dim:] = sbj_vecs_relco
# obj word2vec
obj_vecs_wiki = np.zeros(half_dim, dtype=np.float32)
obj_words = obj_names[ix].split()
for word in obj_words:
if word in self.model.vocab:
raw_word = self.model[word]
obj_vecs_wiki += (raw_word / la.norm(raw_word))
else:
print('Singular word found: ', word)
raise NameError('Terminated.')
obj_vecs_wiki /= len(obj_words)
obj_vecs_wiki /= la.norm(obj_vecs_wiki)
obj_vecs_relco = np.zeros(half_dim, dtype=np.float32)
obj_words = obj_names[ix].split()
for word in obj_words:
if word in self.relco_model.wv.vocab:
raw_word = self.relco_model[word]
obj_vecs_relco += (raw_word / la.norm(raw_word))
else:
obj_vecs_relco += \
(self.relco_vec_mean / la.norm(self.relco_vec_mean))
obj_vecs_relco /= len(obj_words)
obj_vecs_relco /= la.norm(obj_vecs_relco)
obj_vecs[ix][:half_dim] = obj_vecs_wiki
obj_vecs[ix][half_dim:] = obj_vecs_relco
# prd word2vec
prd_vecs_wiki = np.zeros(half_dim, dtype=np.float32)
prd_words = prd_names[ix].split()
for word in prd_words:
if word in self.model.vocab:
raw_word = self.model[word]
prd_vecs_wiki += (raw_word / la.norm(raw_word))
else:
print('Singular word found: ', word)
raise NameError('Terminated.')
prd_vecs_wiki /= len(prd_words)
prd_vecs_wiki /= la.norm(prd_vecs_wiki)
prd_vecs_relco = np.zeros(half_dim, dtype=np.float32)
prd_words = prd_names[ix].split()
for word in prd_words:
if word in self.relco_model.wv.vocab:
raw_word = self.relco_model[word]
prd_vecs_relco += (raw_word / la.norm(raw_word))
else:
prd_vecs_relco += \
(self.relco_vec_mean / la.norm(self.relco_vec_mean))
prd_vecs_relco /= len(prd_words)
prd_vecs_relco /= la.norm(prd_vecs_relco)
prd_vecs[ix][:half_dim] = prd_vecs_wiki
prd_vecs[ix][half_dim:] = prd_vecs_relco
sbj_overlaps = scipy.sparse.csr_matrix(sbj_overlaps)
obj_overlaps = scipy.sparse.csr_matrix(obj_overlaps)
rel_overlaps = scipy.sparse.csr_matrix(rel_overlaps)
return {'sbj_boxes': sbj_boxes,
'obj_boxes': obj_boxes,
'rel_boxes': rel_boxes,
'sbj_names': sbj_names,
'obj_names': obj_names,
'prd_names': prd_names,
'gt_sbj_classes': gt_sbj_classes,
'gt_obj_classes': gt_obj_classes,
'gt_rel_classes': gt_rel_classes,
'gt_sbj_overlaps': sbj_overlaps,
'gt_obj_overlaps': obj_overlaps,
'gt_rel_overlaps': rel_overlaps,
'sbj_seg_areas': sbj_seg_areas,
'obj_seg_areas': obj_seg_areas,
'rel_seg_areas': rel_seg_areas,
'sbj_vecs': sbj_vecs,
'obj_vecs': obj_vecs,
'prd_vecs': prd_vecs,
'flipped': False}