recipes_collection/recipes/training/mixtral/hf_mixtral_8x7b_seq16k_gpu_p5x16_pretrain.yaml [3:76]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
run:
  name: mixtral
  results_dir: ${base_results_dir}/${.name}
  time_limit: "6-00:00:00"
  model_type: hf # huggingface for our recipes


trainer:
  devices: 8
  num_nodes: 16
  accelerator: gpu
  precision: bf16
  max_steps: 50
  log_every_n_steps: 1
  val_check_interval: 1
  limit_val_batches: 0 # Number of batches per each validation run, set to 0 to disable validation.


exp_manager:
  exp_dir: null
  name: experiment
  # experiment loggers
  create_tensorboard_logger: False
  summary_writer_kwargs: {"save_dir" : "${recipes.exp_manager.exp_dir}/tensorboard"}
  create_mlflow_logger: False
  mlflow_logger_kwargs: {"tracking_uri" : "${recipes.exp_manager.exp_dir}/mlflow"}
  create_wandb_logger: False
  wandb_logger_kwargs: {"save_dir" : "${recipes.exp_manager.exp_dir}"} # wandb creates a wandb folder by default
  create_checkpoint_callback: False
  checkpoint_callback_params:
    # Set save_top_k = 0 to disable sharded checkpointing
    save_top_k: 0
    every_n_train_steps: 10
    monitor: "step"
    mode: "max"
    save_last: False
  checkpoint_dir: ${recipes.exp_manager.exp_dir}/checkpoints/
  resume_from_checkpoint: null
  # Set auto_checkpoint = False to disable auto resilience checkpointing
  auto_checkpoint:
    enabled: False
  export_full_model:
    # Set every_n_train_steps = 0 to disable full checkpointing
    every_n_train_steps: 0
    save_last: True

use_smp_model: True # Enable sagemaker model parallelism
distributed_backend: nccl

# Start training from pretrained model
model:
  model_type: mixtral

  train_batch_size: 2
  val_batch_size: 1
  tensor_model_parallel_degree: 1
  expert_model_parallel_degree: 1
  moe: True
  activation_checkpointing: True
  activation_loading_horizon: 2
  delayed_param: True
  offload_activations: False
  seed: 12345
  grad_clip: 1.0


  # FSDP Configs
  sharding_strategy: full_shard
  forward_prefetch: True
  shard_degree: 128
  backward_fetch_policy: backward_pre
  auto_wrap_policy: transformer_auto_wrap_policy
  limit_all_gathers: true
  use_orig_param: True
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



recipes_collection/recipes/training/mixtral/hf_mixtral_8x7b_seq8k_gpu_p5x16_pretrain.yaml [3:76]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
run:
  name: mixtral
  results_dir: ${base_results_dir}/${.name}
  time_limit: "6-00:00:00"
  model_type: hf # huggingface for our recipes


trainer:
  devices: 8
  num_nodes: 16
  accelerator: gpu
  precision: bf16
  max_steps: 50
  log_every_n_steps: 1
  val_check_interval: 1
  limit_val_batches: 0 # Number of batches per each validation run, set to 0 to disable validation.


exp_manager:
  exp_dir: null
  name: experiment
  # experiment loggers
  create_tensorboard_logger: False
  summary_writer_kwargs: {"save_dir" : "${recipes.exp_manager.exp_dir}/tensorboard"}
  create_mlflow_logger: False
  mlflow_logger_kwargs: {"tracking_uri" : "${recipes.exp_manager.exp_dir}/mlflow"}
  create_wandb_logger: False
  wandb_logger_kwargs: {"save_dir" : "${recipes.exp_manager.exp_dir}"} # wandb creates a wandb folder by default
  create_checkpoint_callback: False
  checkpoint_callback_params:
    # Set save_top_k = 0 to disable sharded checkpointing
    save_top_k: 0
    every_n_train_steps: 10
    monitor: "step"
    mode: "max"
    save_last: False
  checkpoint_dir: ${recipes.exp_manager.exp_dir}/checkpoints/
  resume_from_checkpoint: null
  # Set auto_checkpoint = False to disable auto resilience checkpointing
  auto_checkpoint:
    enabled: False
  export_full_model:
    # Set every_n_train_steps = 0 to disable full checkpointing
    every_n_train_steps: 0
    save_last: True

use_smp_model: True # Enable sagemaker model parallelism
distributed_backend: nccl

# Start training from pretrained model
model:
  model_type: mixtral

  train_batch_size: 2
  val_batch_size: 1
  tensor_model_parallel_degree: 1
  expert_model_parallel_degree: 1
  moe: True
  activation_checkpointing: True
  activation_loading_horizon: 2
  delayed_param: True
  offload_activations: False
  seed: 12345
  grad_clip: 1.0


  # FSDP Configs
  sharding_strategy: full_shard
  forward_prefetch: True
  shard_degree: 128
  backward_fetch_policy: backward_pre
  auto_wrap_policy: transformer_auto_wrap_policy
  limit_all_gathers: true
  use_orig_param: True
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



