vilbert/datasets/gqa_dataset.py [139:253]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - self.tokenize(max_seq_length) self.tensorize() cPickle.dump(self.entries, open(cache_path, "wb")) else: logger.info("Loading from %s" % cache_path) self.entries = cPickle.load(open(cache_path, "rb")) def tokenize(self, max_length=16): """Tokenizes the questions. This will add q_token in each entry of the dataset. -1 represent nil, and should be treated as padding_index in embedding """ for entry in self.entries: # tokens = self._tokenizer.tokenize(entry["question"]) # tokens = ["[CLS]"] + tokens + ["[SEP]"] # tokens = [ # self._tokenizer.vocab.get(w, self._tokenizer.vocab["[UNK]"]) # for w in tokens # ] tokens = self._tokenizer.encode(entry["question"]) tokens = tokens[: max_length - 2] tokens = self._tokenizer.add_special_tokens_single_sentence(tokens) segment_ids = [0] * len(tokens) input_mask = [1] * len(tokens) if len(tokens) < max_length: # Note here we pad in front of the sentence padding = [self._padding_index] * (max_length - len(tokens)) tokens = tokens + padding input_mask += padding segment_ids += padding assert_eq(len(tokens), max_length) entry["q_token"] = tokens entry["q_input_mask"] = input_mask entry["q_segment_ids"] = segment_ids def tensorize(self): for entry in self.entries: question = torch.from_numpy(np.array(entry["q_token"])) entry["q_token"] = question q_input_mask = torch.from_numpy(np.array(entry["q_input_mask"])) entry["q_input_mask"] = q_input_mask q_segment_ids = torch.from_numpy(np.array(entry["q_segment_ids"])) entry["q_segment_ids"] = q_segment_ids if "test" not in self.split: answer = entry["answer"] labels = np.array(answer["labels"]) scores = np.array(answer["scores"], dtype=np.float32) if len(labels): labels = torch.from_numpy(labels) scores = torch.from_numpy(scores) entry["answer"]["labels"] = labels entry["answer"]["scores"] = scores else: entry["answer"]["labels"] = None entry["answer"]["scores"] = None def __getitem__(self, index): entry = self.entries[index] image_id = entry["image_id"] question_id = entry["question_id"] features, num_boxes, boxes, _ = self._image_features_reader[image_id] mix_num_boxes = min(int(num_boxes), self._max_region_num) mix_boxes_pad = np.zeros((self._max_region_num, 5)) mix_features_pad = np.zeros((self._max_region_num, 2048)) image_mask = [1] * (int(mix_num_boxes)) while len(image_mask) < self._max_region_num: image_mask.append(0) mix_boxes_pad[:mix_num_boxes] = boxes[:mix_num_boxes] mix_features_pad[:mix_num_boxes] = features[:mix_num_boxes] features = torch.tensor(mix_features_pad).float() image_mask = torch.tensor(image_mask).long() spatials = torch.tensor(mix_boxes_pad).float() question = entry["q_token"] input_mask = entry["q_input_mask"] segment_ids = entry["q_segment_ids"] co_attention_mask = torch.zeros((self._max_region_num, self._max_seq_length)) target = torch.zeros(self.num_labels) if "test" not in self.split: answer = entry["answer"] labels = answer["labels"] scores = answer["scores"] if labels is not None: target.scatter_(0, labels, scores) return ( features, spatials, image_mask, question, target, input_mask, segment_ids, co_attention_mask, question_id, ) def __len__(self): return len(self.entries) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vilbert/datasets/guesswhat_dataset.py [99:213]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - self.tokenize(max_seq_length) self.tensorize() cPickle.dump(self.entries, open(cache_path, "wb")) else: logger.info("Loading from %s" % cache_path) self.entries = cPickle.load(open(cache_path, "rb")) def tokenize(self, max_length=16): """Tokenizes the questions. This will add q_token in each entry of the dataset. -1 represent nil, and should be treated as padding_index in embedding """ for entry in self.entries: # tokens = self._tokenizer.tokenize(entry["question"]) # tokens = ["[CLS]"] + tokens + ["[SEP]"] # tokens = [ # self._tokenizer.vocab.get(w, self._tokenizer.vocab["[UNK]"]) # for w in tokens # ] tokens = self._tokenizer.encode(entry["question"]) tokens = tokens[: max_length - 2] tokens = self._tokenizer.add_special_tokens_single_sentence(tokens) # tokens = tokens[:max_length] segment_ids = [0] * len(tokens) input_mask = [1] * len(tokens) if len(tokens) < max_length: # Note here we pad in front of the sentence padding = [self._padding_index] * (max_length - len(tokens)) tokens = tokens + padding input_mask += padding segment_ids += padding assert_eq(len(tokens), max_length) entry["q_token"] = tokens entry["q_input_mask"] = input_mask entry["q_segment_ids"] = segment_ids def tensorize(self): for entry in self.entries: question = torch.from_numpy(np.array(entry["q_token"])) entry["q_token"] = question q_input_mask = torch.from_numpy(np.array(entry["q_input_mask"])) entry["q_input_mask"] = q_input_mask q_segment_ids = torch.from_numpy(np.array(entry["q_segment_ids"])) entry["q_segment_ids"] = q_segment_ids if "test" not in self.split: answer = entry["answer"] labels = np.array(answer["labels"]) scores = np.array(answer["scores"], dtype=np.float32) if len(labels): labels = torch.from_numpy(labels) scores = torch.from_numpy(scores) entry["answer"]["labels"] = labels entry["answer"]["scores"] = scores else: entry["answer"]["labels"] = None entry["answer"]["scores"] = None def __getitem__(self, index): entry = self.entries[index] image_id = entry["image_id"] question_id = entry["question_id"] features, num_boxes, boxes, _ = self._image_features_reader[image_id] mix_num_boxes = min(int(num_boxes), self._max_region_num) mix_boxes_pad = np.zeros((self._max_region_num, 5)) mix_features_pad = np.zeros((self._max_region_num, 2048)) image_mask = [1] * (int(mix_num_boxes)) while len(image_mask) < self._max_region_num: image_mask.append(0) mix_boxes_pad[:mix_num_boxes] = boxes[:mix_num_boxes] mix_features_pad[:mix_num_boxes] = features[:mix_num_boxes] features = torch.tensor(mix_features_pad).float() image_mask = torch.tensor(image_mask).long() spatials = torch.tensor(mix_boxes_pad).float() question = entry["q_token"] input_mask = entry["q_input_mask"] segment_ids = entry["q_segment_ids"] co_attention_mask = torch.zeros((self._max_region_num, self._max_seq_length)) target = torch.zeros(self.num_labels) if "test" not in self.split: answer = entry["answer"] labels = answer["labels"] scores = answer["scores"] if labels is not None: target.scatter_(0, labels, scores) return ( features, spatials, image_mask, question, target, input_mask, segment_ids, co_attention_mask, question_id, ) def __len__(self): return len(self.entries) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -