src/nli/training.py [430:534]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - model = model_class_item['sequence_classification'].from_pretrained(model_name, cache_dir=str(config.PRO_ROOT / "trans_cache"), num_labels=num_labels) padding_token_value = tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0] padding_segement_value = model_class_item["padding_segement_value"] padding_att_value = model_class_item["padding_att_value"] left_pad = model_class_item['left_pad'] if 'left_pad' in model_class_item else False batch_size_per_gpu_train = args.per_gpu_train_batch_size batch_size_per_gpu_eval = args.per_gpu_eval_batch_size if not args.cpu and not args.single_gpu: dist.init_process_group( backend='nccl', init_method='env://', world_size=args.world_size, rank=args.global_rank ) train_data_str = args.train_data train_data_weights_str = args.train_weights eval_data_str = args.eval_data train_data_name = [] train_data_path = [] train_data_list = [] train_data_weights = [] eval_data_name = [] eval_data_path = [] eval_data_list = [] train_data_named_path = train_data_str.split(',') weights_str = train_data_weights_str.split(',') if train_data_weights_str is not None else None eval_data_named_path = eval_data_str.split(',') for named_path in train_data_named_path: ind = named_path.find(':') name = named_path[:ind] path = name[ind + 1:] if name in registered_path: d_list = common.load_jsonl(registered_path[name]) else: d_list = common.load_jsonl(path) train_data_name.append(name) train_data_path.append(path) train_data_list.append(d_list) if weights_str is not None: for weights in weights_str: train_data_weights.append(float(weights)) else: for i in range(len(train_data_list)): train_data_weights.append(1) for named_path in eval_data_named_path: ind = named_path.find(':') name = named_path[:ind] path = name[ind + 1:] if name in registered_path: d_list = common.load_jsonl(registered_path[name]) else: d_list = common.load_jsonl(path) eval_data_name.append(name) eval_data_path.append(path) eval_data_list.append(d_list) assert len(train_data_weights) == len(train_data_list) batching_schema = { 'uid': RawFlintField(), 'y': LabelFlintField(), 'input_ids': ArrayIndexFlintField(pad_idx=padding_token_value, left_pad=left_pad), 'token_type_ids': ArrayIndexFlintField(pad_idx=padding_segement_value, left_pad=left_pad), 'attention_mask': ArrayIndexFlintField(pad_idx=padding_att_value, left_pad=left_pad), } data_transformer = NLITransform(model_name, tokenizer, max_length) # data_transformer = NLITransform(model_name, tokenizer, max_length, with_element=True) eval_data_loaders = [] for eval_d_list in eval_data_list: d_dataset, d_sampler, d_dataloader = build_eval_dataset_loader_and_sampler(eval_d_list, data_transformer, batching_schema, batch_size_per_gpu_eval) eval_data_loaders.append(d_dataloader) # Estimate the training size: training_list = [] for i in range(len(train_data_list)): print("Build Training Data ...") train_d_list = train_data_list[i] train_d_name = train_data_name[i] train_d_weight = train_data_weights[i] cur_train_list = sample_data_list(train_d_list, train_d_weight) # change later # we can apply different sample strategy here. print(f"Data Name:{train_d_name}; Weight: {train_d_weight}; " f"Original Size: {len(train_d_list)}; Sampled Size: {len(cur_train_list)}") training_list.extend(cur_train_list) estimated_training_size = len(training_list) print("Estimated training size:", estimated_training_size) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - src/nli/training_extra.py [449:553]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - model = model_class_item['sequence_classification'].from_pretrained(model_name, cache_dir=str(config.PRO_ROOT / "trans_cache"), num_labels=num_labels) padding_token_value = tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0] padding_segement_value = model_class_item["padding_segement_value"] padding_att_value = model_class_item["padding_att_value"] left_pad = model_class_item['left_pad'] if 'left_pad' in model_class_item else False batch_size_per_gpu_train = args.per_gpu_train_batch_size batch_size_per_gpu_eval = args.per_gpu_eval_batch_size if not args.cpu and not args.single_gpu: dist.init_process_group( backend='nccl', init_method='env://', world_size=args.world_size, rank=args.global_rank ) train_data_str = args.train_data train_data_weights_str = args.train_weights eval_data_str = args.eval_data train_data_name = [] train_data_path = [] train_data_list = [] train_data_weights = [] eval_data_name = [] eval_data_path = [] eval_data_list = [] train_data_named_path = train_data_str.split(',') weights_str = train_data_weights_str.split(',') if train_data_weights_str is not None else None eval_data_named_path = eval_data_str.split(',') for named_path in train_data_named_path: ind = named_path.find(':') name = named_path[:ind] path = name[ind + 1:] if name in registered_path: d_list = common.load_jsonl(registered_path[name]) else: d_list = common.load_jsonl(path) train_data_name.append(name) train_data_path.append(path) train_data_list.append(d_list) if weights_str is not None: for weights in weights_str: train_data_weights.append(float(weights)) else: for i in range(len(train_data_list)): train_data_weights.append(1) for named_path in eval_data_named_path: ind = named_path.find(':') name = named_path[:ind] path = name[ind + 1:] if name in registered_path: d_list = common.load_jsonl(registered_path[name]) else: d_list = common.load_jsonl(path) eval_data_name.append(name) eval_data_path.append(path) eval_data_list.append(d_list) assert len(train_data_weights) == len(train_data_list) batching_schema = { 'uid': RawFlintField(), 'y': LabelFlintField(), 'input_ids': ArrayIndexFlintField(pad_idx=padding_token_value, left_pad=left_pad), 'token_type_ids': ArrayIndexFlintField(pad_idx=padding_segement_value, left_pad=left_pad), 'attention_mask': ArrayIndexFlintField(pad_idx=padding_att_value, left_pad=left_pad), } data_transformer = NLITransform(model_name, tokenizer, max_length) # data_transformer = NLITransform(model_name, tokenizer, max_length, with_element=True) eval_data_loaders = [] for eval_d_list in eval_data_list: d_dataset, d_sampler, d_dataloader = build_eval_dataset_loader_and_sampler(eval_d_list, data_transformer, batching_schema, batch_size_per_gpu_eval) eval_data_loaders.append(d_dataloader) # Estimate the training size: training_list = [] for i in range(len(train_data_list)): print("Build Training Data ...") train_d_list = train_data_list[i] train_d_name = train_data_name[i] train_d_weight = train_data_weights[i] cur_train_list = sample_data_list(train_d_list, train_d_weight) # change later # we can apply different sample strategy here. print(f"Data Name:{train_d_name}; Weight: {train_d_weight}; " f"Original Size: {len(train_d_list)}; Sampled Size: {len(cur_train_list)}") training_list.extend(cur_train_list) estimated_training_size = len(training_list) print("Estimated training size:", estimated_training_size) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -