training/flax/run_pseudo_labelling_pt.py [352:467]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return batch def log_metric( accelerator, metrics: Dict, train_time: float, prefix: str = "eval", ): """Helper function to log all evaluation metrics with the correct prefixes and styling.""" log_metrics = {} for k, v in metrics.items(): log_metrics[f"{prefix}/{k}"] = v log_metrics[f"{prefix}/time"] = train_time accelerator.log(log_metrics) def log_pred( accelerator, pred_str: List[str], label_str: List[str], norm_pred_str: List[str], norm_label_str: List[str], prefix: str = "eval", num_lines: int = 200000, ): """Helper function to log target/predicted transcriptions to weights and biases (wandb).""" if accelerator.is_main_process: wandb_tracker = accelerator.get_tracker("wandb") # pretty name for split prefix = prefix.replace("/", "-") # convert str data to a wandb compatible format str_data = [[label_str[i], pred_str[i], norm_label_str[i], norm_pred_str[i]] for i in range(len(pred_str))] # log as a table with the appropriate headers wandb_tracker.log_table( table_name=f"{prefix}/all_predictions", columns=["Target", "Pred", "Norm Target", "Norm Pred"], data=str_data[:num_lines], ) # log incorrect normalised predictions str_data = np.asarray(str_data) str_data_incorrect = str_data[str_data[:, -2] != str_data[:, -1]] # log as a table with the appropriate headers wandb_tracker.log_table( table_name=f"{prefix}/incorrect_predictions", columns=["Target", "Pred", "Norm Target", "Norm Pred"], data=str_data_incorrect[:num_lines], ) def main(): # 1. Parse input arguments # We keep distinct sets of args, for cleaner separation of model/data/training related args parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments)) if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): # If we pass only one argument to the script and it's the path to a json file, # let's parse it to get our arguments. model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) else: model_args, data_args, training_args = parser.parse_args_into_dataclasses() # 2. Initialize the accelerator # We will let the accelerator handle device placement for us in this example # We simply have to specify the training precision and any trackers being used # We'll use the same dtype arguments as our JAX/Flax training script and convert # it to accelerate format if model_args.dtype == "float16": mixed_precision = "fp16" torch_dtype = torch.float16 elif model_args.dtype == "bfloat16": mixed_precision = "bf16" torch_dtype = torch.bfloat16 else: mixed_precision = "no" torch_dtype = torch.float32 kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=7200)) accelerator = Accelerator( gradient_accumulation_steps=training_args.gradient_accumulation_steps, mixed_precision=mixed_precision, log_with=training_args.report_to, project_dir=training_args.output_dir, kwargs_handlers=[kwargs], ) accelerator.init_trackers(project_name=data_args.wandb_project) # 3. Set-up basic logging # Create one log on every process with the configuration for debugging logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, ) # Log a small summary on each proces logger.warning( f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, " f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}" ) # Set the verbosity to info of the Transformers logger (on main process only) if accelerator.is_local_main_process: datasets.utils.logging.set_verbosity_warning() transformers.utils.logging.set_verbosity_info() else: datasets.utils.logging.set_verbosity_error() transformers.utils.logging.set_verbosity_error() logger.info("Training/evaluation parameters %s", training_args) # 3. Load dataset raw_datasets = IterableDatasetDict() if data_args.streaming else DatasetDict() token = model_args.token if model_args.token is not None else HfFolder().get_token() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - training/run_pseudo_labelling.py [383:498]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return batch def log_metric( accelerator, metrics: Dict, train_time: float, prefix: str = "eval", ): """Helper function to log all evaluation metrics with the correct prefixes and styling.""" log_metrics = {} for k, v in metrics.items(): log_metrics[f"{prefix}/{k}"] = v log_metrics[f"{prefix}/time"] = train_time accelerator.log(log_metrics) def log_pred( accelerator, pred_str: List[str], label_str: List[str], norm_pred_str: List[str], norm_label_str: List[str], prefix: str = "eval", num_lines: int = 200000, ): """Helper function to log target/predicted transcriptions to weights and biases (wandb).""" if accelerator.is_main_process: wandb_tracker = accelerator.get_tracker("wandb") # pretty name for split prefix = prefix.replace("/", "-") # convert str data to a wandb compatible format str_data = [[label_str[i], pred_str[i], norm_label_str[i], norm_pred_str[i]] for i in range(len(pred_str))] # log as a table with the appropriate headers wandb_tracker.log_table( table_name=f"{prefix}/all_predictions", columns=["Target", "Pred", "Norm Target", "Norm Pred"], data=str_data[:num_lines], ) # log incorrect normalised predictions str_data = np.asarray(str_data) str_data_incorrect = str_data[str_data[:, -2] != str_data[:, -1]] # log as a table with the appropriate headers wandb_tracker.log_table( table_name=f"{prefix}/incorrect_predictions", columns=["Target", "Pred", "Norm Target", "Norm Pred"], data=str_data_incorrect[:num_lines], ) def main(): # 1. Parse input arguments # We keep distinct sets of args, for cleaner separation of model/data/training related args parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments)) if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): # If we pass only one argument to the script and it's the path to a json file, # let's parse it to get our arguments. model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) else: model_args, data_args, training_args = parser.parse_args_into_dataclasses() # 2. Initialize the accelerator # We will let the accelerator handle device placement for us in this example # We simply have to specify the training precision and any trackers being used # We'll use the same dtype arguments as our JAX/Flax training script and convert # it to accelerate format if model_args.dtype == "float16": mixed_precision = "fp16" torch_dtype = torch.float16 elif model_args.dtype == "bfloat16": mixed_precision = "bf16" torch_dtype = torch.bfloat16 else: mixed_precision = "no" torch_dtype = torch.float32 kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=7200)) accelerator = Accelerator( gradient_accumulation_steps=training_args.gradient_accumulation_steps, mixed_precision=mixed_precision, log_with=training_args.report_to, project_dir=training_args.output_dir, kwargs_handlers=[kwargs], ) accelerator.init_trackers(project_name=data_args.wandb_project) # 3. Set-up basic logging # Create one log on every process with the configuration for debugging logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, ) # Log a small summary on each proces logger.warning( f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, " f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}" ) # Set the verbosity to info of the Transformers logger (on main process only) if accelerator.is_local_main_process: datasets.utils.logging.set_verbosity_warning() transformers.utils.logging.set_verbosity_info() else: datasets.utils.logging.set_verbosity_error() transformers.utils.logging.set_verbosity_error() logger.info("Training/evaluation parameters %s", training_args) # 3. Load dataset raw_datasets = IterableDatasetDict() if data_args.streaming else DatasetDict() token = model_args.token if model_args.token is not None else HfFolder().get_token() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -