src/nli/training.py [45:320]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "bert-base": { "model_name": "bert-base-uncased", "tokenizer": BertTokenizer, "sequence_classification": BertForSequenceClassification, # "padding_token_value": 0, "padding_segement_value": 0, "padding_att_value": 0, "do_lower_case": True, }, "bert-large": { "model_name": "bert-large-uncased", "tokenizer": BertTokenizer, "sequence_classification": BertForSequenceClassification, # "padding_token_value": 0, "padding_segement_value": 0, "padding_att_value": 0, "do_lower_case": True, "internal_model_name": "bert", 'insight_supported': True, }, "xlnet-base": { "model_name": "xlnet-base-cased", "tokenizer": XLNetTokenizer, "sequence_classification": XLNetForSequenceClassification, # "padding_token_value": 0, "padding_segement_value": 4, "padding_att_value": 0, "left_pad": True, "internal_model_name": ["transformer", "word_embedding"], }, "xlnet-large": { "model_name": "xlnet-large-cased", "tokenizer": XLNetTokenizer, "sequence_classification": XLNetForSequenceClassification, "padding_segement_value": 4, "padding_att_value": 0, "left_pad": True, "internal_model_name": ["transformer", "word_embedding"], 'insight_supported': True, }, "roberta-base": { "model_name": "roberta-base", "tokenizer": RobertaTokenizer, "sequence_classification": RobertaForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": "roberta", 'insight_supported': True, }, "roberta-large": { "model_name": "roberta-large", "tokenizer": RobertaTokenizer, "sequence_classification": RobertaForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": "roberta", 'insight_supported': True, }, "albert-xxlarge": { "model_name": "albert-xxlarge-v2", "tokenizer": AlbertTokenizer, "sequence_classification": AlbertForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "do_lower_case": True, "internal_model_name": "albert", 'insight_supported': True, }, "distilbert": { "model_name": "distilbert-base-cased", "tokenizer": DistilBertTokenizer, "sequence_classification": DistilBertForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, }, "bart-large": { "model_name": "facebook/bart-large", "tokenizer": BartTokenizer, "sequence_classification": BartForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": ["model", "encoder", "embed_tokens"], 'insight_supported': True, }, "electra-base": { "model_name": "google/electra-base-discriminator", "tokenizer": ElectraTokenizer, "sequence_classification": ElectraForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": "electra", 'insight_supported': True, }, "electra-large": { "model_name": "google/electra-large-discriminator", "tokenizer": ElectraTokenizer, "sequence_classification": ElectraForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": "electra", 'insight_supported': True, } } registered_path = { 'snli_train': config.PRO_ROOT / "data/build/snli/train.jsonl", 'snli_dev': config.PRO_ROOT / "data/build/snli/dev.jsonl", 'snli_test': config.PRO_ROOT / "data/build/snli/test.jsonl", 'mnli_train': config.PRO_ROOT / "data/build/mnli/train.jsonl", 'mnli_m_dev': config.PRO_ROOT / "data/build/mnli/m_dev.jsonl", 'mnli_mm_dev': config.PRO_ROOT / "data/build/mnli/mm_dev.jsonl", 'fever_train': config.PRO_ROOT / "data/build/fever_nli/train.jsonl", 'fever_dev': config.PRO_ROOT / "data/build/fever_nli/dev.jsonl", 'fever_test': config.PRO_ROOT / "data/build/fever_nli/test.jsonl", 'anli_r1_train': config.PRO_ROOT / "data/build/anli/r1/train.jsonl", 'anli_r1_dev': config.PRO_ROOT / "data/build/anli/r1/dev.jsonl", 'anli_r1_test': config.PRO_ROOT / "data/build/anli/r1/test.jsonl", 'anli_r2_train': config.PRO_ROOT / "data/build/anli/r2/train.jsonl", 'anli_r2_dev': config.PRO_ROOT / "data/build/anli/r2/dev.jsonl", 'anli_r2_test': config.PRO_ROOT / "data/build/anli/r2/test.jsonl", 'anli_r3_train': config.PRO_ROOT / "data/build/anli/r3/train.jsonl", 'anli_r3_dev': config.PRO_ROOT / "data/build/anli/r3/dev.jsonl", 'anli_r3_test': config.PRO_ROOT / "data/build/anli/r3/test.jsonl", } nli_label2index = { 'e': 0, 'n': 1, 'c': 2, 'h': -1, } def set_seed(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) class NLIDataset(Dataset): def __init__(self, data_list, transform) -> None: super().__init__() self.d_list = data_list self.len = len(self.d_list) self.transform = transform def __getitem__(self, index: int): return self.transform(self.d_list[index]) # you should write schema for each of the input elements def __len__(self) -> int: return self.len class NLITransform(object): def __init__(self, model_name, tokenizer, max_length=None): self.model_name = model_name self.tokenizer = tokenizer self.max_length = max_length def __call__(self, sample): processed_sample = dict() processed_sample['uid'] = sample['uid'] processed_sample['gold_label'] = sample['label'] processed_sample['y'] = nli_label2index[sample['label']] # premise: str = sample['premise'] premise: str = sample['context'] if 'context' in sample else sample['premise'] hypothesis: str = sample['hypothesis'] if premise.strip() == '': premise = 'empty' if hypothesis.strip() == '': hypothesis = 'empty' tokenized_input_seq_pair = self.tokenizer.encode_plus(premise, hypothesis, max_length=self.max_length, return_token_type_ids=True, truncation=True) processed_sample.update(tokenized_input_seq_pair) return processed_sample def build_eval_dataset_loader_and_sampler(d_list, data_transformer, batching_schema, batch_size_per_gpu_eval): d_dataset = NLIDataset(d_list, data_transformer) d_sampler = SequentialSampler(d_dataset) d_dataloader = DataLoader(dataset=d_dataset, batch_size=batch_size_per_gpu_eval, shuffle=False, # num_workers=0, pin_memory=True, sampler=d_sampler, collate_fn=BaseBatchBuilder(batching_schema)) # return d_dataset, d_sampler, d_dataloader def sample_data_list(d_list, ratio): if ratio <= 0: raise ValueError("Invalid training weight ratio. Please change --train_weights.") upper_int = int(math.ceil(ratio)) if upper_int == 1: return d_list # if ratio is 1 then we just return the data list else: sampled_d_list = [] for _ in range(upper_int): sampled_d_list.extend(copy.deepcopy(d_list)) if np.isclose(ratio, upper_int): return sampled_d_list else: sampled_length = int(ratio * len(d_list)) random.shuffle(sampled_d_list) return sampled_d_list[:sampled_length] def main(): parser = argparse.ArgumentParser() parser.add_argument("--cpu", action="store_true", help="If set, we only use CPU.") parser.add_argument("--single_gpu", action="store_true", help="If set, we only use single GPU.") parser.add_argument("--fp16", action="store_true", help="If set, we will use fp16.") parser.add_argument( "--fp16_opt_level", type=str, default="O1", help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." "See details at https://nvidia.github.io/apex/amp.html", ) # environment arguments parser.add_argument('-s', '--seed', default=1, type=int, metavar='N', help='manual random seed') parser.add_argument('-n', '--num_nodes', default=1, type=int, metavar='N', help='number of nodes') parser.add_argument('-g', '--gpus_per_node', default=1, type=int, help='number of gpus per node') parser.add_argument('-nr', '--node_rank', default=0, type=int, help='ranking within the nodes') # experiments specific arguments parser.add_argument('--debug_mode', action='store_true', dest='debug_mode', help='weather this is debug mode or normal') parser.add_argument( "--model_class_name", type=str, help="Set the model class of the experiment.", ) parser.add_argument( "--experiment_name", type=str, help="Set the name of the experiment. [model_name]/[data]/[task]/[other]", ) parser.add_argument( "--save_prediction", action='store_true', dest='save_prediction', help='Do we want to save prediction') - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - src/nli/training_extra.py [67:343]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "bert-base": { "model_name": "bert-base-uncased", "tokenizer": BertTokenizer, "sequence_classification": BertForSequenceClassification, # "padding_token_value": 0, "padding_segement_value": 0, "padding_att_value": 0, "do_lower_case": True, }, "bert-large": { "model_name": "bert-large-uncased", "tokenizer": BertTokenizer, "sequence_classification": BertForSequenceClassification, # "padding_token_value": 0, "padding_segement_value": 0, "padding_att_value": 0, "do_lower_case": True, "internal_model_name": "bert", 'insight_supported': True, }, "xlnet-base": { "model_name": "xlnet-base-cased", "tokenizer": XLNetTokenizer, "sequence_classification": XLNetForSequenceClassification, # "padding_token_value": 0, "padding_segement_value": 4, "padding_att_value": 0, "left_pad": True, "internal_model_name": ["transformer", "word_embedding"], }, "xlnet-large": { "model_name": "xlnet-large-cased", "tokenizer": XLNetTokenizer, "sequence_classification": XLNetForSequenceClassification, "padding_segement_value": 4, "padding_att_value": 0, "left_pad": True, "internal_model_name": ["transformer", "word_embedding"], 'insight_supported': True, }, "roberta-base": { "model_name": "roberta-base", "tokenizer": RobertaTokenizer, "sequence_classification": RobertaForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": "roberta", 'insight_supported': True, }, "roberta-large": { "model_name": "roberta-large", "tokenizer": RobertaTokenizer, "sequence_classification": RobertaForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": "roberta", 'insight_supported': True, }, "albert-xxlarge": { "model_name": "albert-xxlarge-v2", "tokenizer": AlbertTokenizer, "sequence_classification": AlbertForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "do_lower_case": True, "internal_model_name": "albert", 'insight_supported': True, }, "distilbert": { "model_name": "distilbert-base-cased", "tokenizer": DistilBertTokenizer, "sequence_classification": DistilBertForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, }, "bart-large": { "model_name": "facebook/bart-large", "tokenizer": BartTokenizer, "sequence_classification": BartForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": ["model", "encoder", "embed_tokens"], 'insight_supported': True, }, "electra-base": { "model_name": "google/electra-base-discriminator", "tokenizer": ElectraTokenizer, "sequence_classification": ElectraForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": "electra", 'insight_supported': True, }, "electra-large": { "model_name": "google/electra-large-discriminator", "tokenizer": ElectraTokenizer, "sequence_classification": ElectraForSequenceClassification, "padding_segement_value": 0, "padding_att_value": 0, "internal_model_name": "electra", 'insight_supported': True, } } registered_path = { 'snli_train': config.PRO_ROOT / "data/build/snli/train.jsonl", 'snli_dev': config.PRO_ROOT / "data/build/snli/dev.jsonl", 'snli_test': config.PRO_ROOT / "data/build/snli/test.jsonl", 'mnli_train': config.PRO_ROOT / "data/build/mnli/train.jsonl", 'mnli_m_dev': config.PRO_ROOT / "data/build/mnli/m_dev.jsonl", 'mnli_mm_dev': config.PRO_ROOT / "data/build/mnli/mm_dev.jsonl", 'fever_train': config.PRO_ROOT / "data/build/fever_nli/train.jsonl", 'fever_dev': config.PRO_ROOT / "data/build/fever_nli/dev.jsonl", 'fever_test': config.PRO_ROOT / "data/build/fever_nli/test.jsonl", 'anli_r1_train': config.PRO_ROOT / "data/build/anli/r1/train.jsonl", 'anli_r1_dev': config.PRO_ROOT / "data/build/anli/r1/dev.jsonl", 'anli_r1_test': config.PRO_ROOT / "data/build/anli/r1/test.jsonl", 'anli_r2_train': config.PRO_ROOT / "data/build/anli/r2/train.jsonl", 'anli_r2_dev': config.PRO_ROOT / "data/build/anli/r2/dev.jsonl", 'anli_r2_test': config.PRO_ROOT / "data/build/anli/r2/test.jsonl", 'anli_r3_train': config.PRO_ROOT / "data/build/anli/r3/train.jsonl", 'anli_r3_dev': config.PRO_ROOT / "data/build/anli/r3/dev.jsonl", 'anli_r3_test': config.PRO_ROOT / "data/build/anli/r3/test.jsonl", } nli_label2index = { 'e': 0, 'n': 1, 'c': 2, 'h': -1, } def set_seed(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) class NLIDataset(Dataset): def __init__(self, data_list, transform) -> None: super().__init__() self.d_list = data_list self.len = len(self.d_list) self.transform = transform def __getitem__(self, index: int): return self.transform(self.d_list[index]) # you should write schema for each of the input elements def __len__(self) -> int: return self.len class NLITransform(object): def __init__(self, model_name, tokenizer, max_length=None): self.model_name = model_name self.tokenizer = tokenizer self.max_length = max_length def __call__(self, sample): processed_sample = dict() processed_sample['uid'] = sample['uid'] processed_sample['gold_label'] = sample['label'] processed_sample['y'] = nli_label2index[sample['label']] # premise: str = sample['premise'] premise: str = sample['context'] if 'context' in sample else sample['premise'] hypothesis: str = sample['hypothesis'] if premise.strip() == '': premise = 'empty' if hypothesis.strip() == '': hypothesis = 'empty' tokenized_input_seq_pair = self.tokenizer.encode_plus(premise, hypothesis, max_length=self.max_length, return_token_type_ids=True, truncation=True) processed_sample.update(tokenized_input_seq_pair) return processed_sample def build_eval_dataset_loader_and_sampler(d_list, data_transformer, batching_schema, batch_size_per_gpu_eval): d_dataset = NLIDataset(d_list, data_transformer) d_sampler = SequentialSampler(d_dataset) d_dataloader = DataLoader(dataset=d_dataset, batch_size=batch_size_per_gpu_eval, shuffle=False, # num_workers=0, pin_memory=True, sampler=d_sampler, collate_fn=BaseBatchBuilder(batching_schema)) # return d_dataset, d_sampler, d_dataloader def sample_data_list(d_list, ratio): if ratio <= 0: raise ValueError("Invalid training weight ratio. Please change --train_weights.") upper_int = int(math.ceil(ratio)) if upper_int == 1: return d_list # if ratio is 1 then we just return the data list else: sampled_d_list = [] for _ in range(upper_int): sampled_d_list.extend(copy.deepcopy(d_list)) if np.isclose(ratio, upper_int): return sampled_d_list else: sampled_length = int(ratio * len(d_list)) random.shuffle(sampled_d_list) return sampled_d_list[:sampled_length] def main(): parser = argparse.ArgumentParser() parser.add_argument("--cpu", action="store_true", help="If set, we only use CPU.") parser.add_argument("--single_gpu", action="store_true", help="If set, we only use single GPU.") parser.add_argument("--fp16", action="store_true", help="If set, we will use fp16.") parser.add_argument( "--fp16_opt_level", type=str, default="O1", help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." "See details at https://nvidia.github.io/apex/amp.html", ) # environment arguments parser.add_argument('-s', '--seed', default=1, type=int, metavar='N', help='manual random seed') parser.add_argument('-n', '--num_nodes', default=1, type=int, metavar='N', help='number of nodes') parser.add_argument('-g', '--gpus_per_node', default=1, type=int, help='number of gpus per node') parser.add_argument('-nr', '--node_rank', default=0, type=int, help='ranking within the nodes') # experiments specific arguments parser.add_argument('--debug_mode', action='store_true', dest='debug_mode', help='weather this is debug mode or normal') parser.add_argument( "--model_class_name", type=str, help="Set the model class of the experiment.", ) parser.add_argument( "--experiment_name", type=str, help="Set the name of the experiment. [model_name]/[data]/[task]/[other]", ) parser.add_argument( "--save_prediction", action='store_true', dest='save_prediction', help='Do we want to save prediction') - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -