scripts/lora_finetune_phi3.yaml [67:101]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
optimizer:
  _component_: torch.optim.AdamW
  fused: True
  weight_decay: 0.01
  lr: 3e-4
lr_scheduler:
  _component_: torchtune.training.get_cosine_schedule_with_warmup
  num_warmup_steps: 100
loss:
  _component_: torchtune.modules.loss.CEWithChunkedOutputLoss
clip_grad_norm: null
compile: False

# Training env
device: cuda

# Memory management
enable_activation_checkpointing: True
enable_activation_offloading: False

# Reduced precision
dtype: bf16

# Logging
output_dir: {{log_dir}}/lora_finetune_output
metric_logger:
  _component_: torchtune.training.metric_logging.{{metric_logger}}
  log_dir: {{log_dir}}/training_logs
log_every_n_steps: 1
log_peak_memory_stats: False

# Showcase the usage of PyTorch profiler
# Set enabled to False as it's only needed for debugging training
profiler:
  _component_: torchtune.training.setup_torch_profiler
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



scripts/lora_finetune_phi4.yaml [74:109]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
optimizer:
  _component_: torch.optim.AdamW
  fused: True
  weight_decay: 0.01
  lr: 3e-4
lr_scheduler:
  _component_: torchtune.training.get_cosine_schedule_with_warmup
  num_warmup_steps: 100
loss:
  _component_: torchtune.modules.loss.CEWithChunkedOutputLoss
clip_grad_norm: null
compile: False


# Training env
device: cuda

# Memory management
enable_activation_checkpointing: True  # True reduces memory
enable_activation_offloading: False  # True reduces memory

# Reduced precision
dtype: bf16

# Logging
output_dir: {{log_dir}}/lora_finetune_output
metric_logger:
  _component_: torchtune.training.metric_logging.{{metric_logger}}
  log_dir: {{log_dir}}/training_logs
log_every_n_steps: 1
log_peak_memory_stats: False

# Showcase the usage of PyTorch profiler
# Set enabled to False as it's only needed for debugging training
profiler:
  _component_: torchtune.training.setup_torch_profiler
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



