recipes_collection/recipes/fine-tuning/deepseek/hf_deepseek_r1_distilled_qwen_14b_seq16k_gpu_fine_tuning.yaml [6:94]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  results_dir: ${base_results_dir}/${.name}
  time_limit: "6-00:00:00"
  model_type: hf # huggingface for our recipes

# Basic pytorch lightning trainer config
trainer:
  devices: 8
  num_nodes: 2
  accelerator: gpu
  precision: bf16
  max_steps: 50
  log_every_n_steps: 1
  val_check_interval: 1
  limit_val_batches: 0 # Number of batches per each validation run, set to 0 to disable validation.

# Basic pytorch lightning experiment config
# Config for checkpoint/tensorboard etc
exp_manager:
  exp_dir: null
  name: experiment
  # experiment loggers
  create_tensorboard_logger: False
  summary_writer_kwargs: {"save_dir" : "${recipes.exp_manager.exp_dir}/tensorboard"}
  create_mlflow_logger: False
  mlflow_logger_kwargs: {"tracking_uri" : "${recipes.exp_manager.exp_dir}/mlflow"}
  create_wandb_logger: False
  wandb_logger_kwargs: {"save_dir" : "${recipes.exp_manager.exp_dir}"} # wandb creates a wandb folder by default
  create_checkpoint_callback: True
  # Configs to save checkpoint with a fixed interval
  # Note: These config will not work with auto checkpoint mode
  checkpoint_callback_params:
    # Set save_top_k = 0 to disable sharded checkpointing
    save_top_k: 0
    every_n_train_steps: 10
    monitor: "step"
    mode: "max"
    save_last: False
  checkpoint_dir: ${recipes.exp_manager.exp_dir}/checkpoints/
  resume_from_checkpoint: null
  # Enable auto_checkpoint to automatically calculate the checkpoint interval and resume from checkpoint
  auto_checkpoint:
    enabled: False
  export_full_model:
    # Set every_n_train_steps = 0 to disable full checkpointing
    every_n_train_steps: 0
    save_last: True

################# Predefined configs ##########################
use_smp_model: False # Enable sagemaker model parallelism
distributed_backend: nccl


# Model training configs
model:
  model_type: deepseek_r1_qwen_v2
  # Base configs
  train_batch_size: 1
  val_batch_size: 1
  seed: 12345
  grad_clip: 1.0
  log_reduced_training_loss: True

  # Memory saving / distributed training configs
  tensor_model_parallel_degree: 1
  expert_model_parallel_degree: 1
  context_parallel_degree: 1
  moe: False
  activation_checkpointing: True
  activation_loading_horizon: 2
  delayed_param: False
  offload_activations: False

  # FSDP Configs
  sharding_strategy: hybrid_shard
  forward_prefetch: True
  shard_degree: 16
  backward_fetch_policy: backward_pre
  auto_wrap_policy: transformer_auto_wrap_policy
  limit_all_gathers: true
  use_orig_param: False

  # FP8 config
  fp8: True
  fp8_amax_history_len: 1024
  fp8_amax_compute_algo: max

  # Model architecture
  max_context_width: 16384
  max_position_embeddings: ${.max_context_width}
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



recipes_collection/recipes/fine-tuning/deepseek/hf_deepseek_r1_distilled_qwen_32b_seq16k_gpu_lora.yaml [6:93]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  results_dir: ${base_results_dir}/${.name}
  time_limit: "6-00:00:00"
  model_type: hf # huggingface for our recipes

# Basic pytorch lightning trainer config
trainer:
  devices: 8
  num_nodes: 2
  accelerator: gpu
  precision: bf16
  max_steps: 50
  log_every_n_steps: 1
  val_check_interval: 1
  limit_val_batches: 0 # Number of batches per each validation run, set to 0 to disable validation.

# Basic pytorch lightning experiment config
# Config for checkpoint/tensorboard etc
exp_manager:
  exp_dir: null
  name: experiment
  # experiment loggers
  create_tensorboard_logger: False
  summary_writer_kwargs: {"save_dir" : "${recipes.exp_manager.exp_dir}/tensorboard"}
  create_mlflow_logger: False
  mlflow_logger_kwargs: {"tracking_uri" : "${recipes.exp_manager.exp_dir}/mlflow"}
  create_wandb_logger: False
  wandb_logger_kwargs: {"save_dir" : "${recipes.exp_manager.exp_dir}"} # wandb creates a wandb folder by default
  create_checkpoint_callback: True
  # Configs to save checkpoint with a fixed interval
  # Note: These config will not work with auto checkpoint mode
  checkpoint_callback_params:
    # Set save_top_k = 0 to disable sharded checkpointing
    save_top_k: 0
    every_n_train_steps: 10
    monitor: "step"
    mode: "max"
    save_last: False
  checkpoint_dir: ${recipes.exp_manager.exp_dir}/checkpoints/
  resume_from_checkpoint: null
  # Enable auto_checkpoint to automatically calculate the checkpoint interval and resume from checkpoint
  auto_checkpoint:
    enabled: False
  export_full_model:
    # Set every_n_train_steps = 0 to disable full checkpointing
    every_n_train_steps: 0
    save_last: True

################# Predefined configs ##########################
use_smp_model: False # Enable sagemaker model parallelism
distributed_backend: nccl

# Model training configs
model:
  model_type: deepseek_r1_qwen_v2
  # Base configs
  train_batch_size: 1
  val_batch_size: 1
  seed: 12345
  grad_clip: 1.0
  log_reduced_training_loss: True

  # Memory saving / distributed training configs
  tensor_model_parallel_degree: 1
  expert_model_parallel_degree: 1
  context_parallel_degree: 1
  moe: False
  activation_checkpointing: True
  activation_loading_horizon: 2
  delayed_param: False
  offload_activations: False

  # FSDP Configs
  sharding_strategy: hybrid_shard
  forward_prefetch: True
  shard_degree: 16
  backward_fetch_policy: backward_pre
  auto_wrap_policy: transformer_auto_wrap_policy
  limit_all_gathers: true
  use_orig_param: False

  # FP8 config
  fp8: True
  fp8_amax_history_len: 1024
  fp8_amax_compute_algo: max

  # Model architecture
  max_context_width: 16384
  max_position_embeddings: ${.max_context_width}
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



