in training/utils.py [0:0]
def save_codec_checkpoint(output_dir, dataset, step): checkpoint_path = f"{CHECKPOINT_CODEC_PREFIX}-{step}" output_path = os.path.join(output_dir, checkpoint_path) dataset.save_to_disk(output_path)