infra/4-training/nemo-example/nemo-configurations/gpt-175b.yaml [76:90]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  kv_channels: null  # Projection weights dimension in multi-head attention. Set to hidden_size // num_attention_heads if null
  apply_query_key_layer_scaling: true  # scale Q * K^T by 1 / layer-number.
  layernorm_epsilon: 1e-5
  make_vocab_size_divisible_by: 128  # Pad the vocab size to be divisible by this value for computation efficiency.
  pre_process: true  # add embedding
  post_process: true  # add pooler
  persist_layer_norm: true  # Use of persistent fused layer norm kernel.
  gradient_as_bucket_view: true  # Allocate gradients in a contiguous bucket to save memory (less fragmentation and buffer memory)

  # Fusion
  grad_div_ar_fusion: true  # Fuse grad division into torch.distributed.all_reduce
  gradient_accumulation_fusion: true  # Fuse weight gradient accumulation to GEMMs
  bias_activation_fusion: true  # Use a kernel that fuses the bias addition from weight matrices with the subsequent activation function.
  bias_dropout_add_fusion: true  # Use a kernel that fuses the bias addition, dropout and residual connection addition.
  masked_softmax_fusion: true  # Use a kernel that fuses the attention softmax with it's mask.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



infra/4-training/nemo-example/selected-configuration.yaml [79:93]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  kv_channels: null  # Projection weights dimension in multi-head attention. Set to hidden_size // num_attention_heads if null
  apply_query_key_layer_scaling: true  # scale Q * K^T by 1 / layer-number.
  layernorm_epsilon: 1e-5
  make_vocab_size_divisible_by: 128  # Pad the vocab size to be divisible by this value for computation efficiency.
  pre_process: true  # add embedding
  post_process: true  # add pooler
  persist_layer_norm: true  # Use of persistent fused layer norm kernel.
  gradient_as_bucket_view: true  # Allocate gradients in a contiguous bucket to save memory (less fragmentation and buffer memory)

  # Fusion
  grad_div_ar_fusion: true  # Fuse grad division into torch.distributed.all_reduce
  gradient_accumulation_fusion: true  # Fuse weight gradient accumulation to GEMMs
  bias_activation_fusion: true  # Use a kernel that fuses the bias addition from weight matrices with the subsequent activation function.
  bias_dropout_add_fusion: true  # Use a kernel that fuses the bias addition, dropout and residual connection addition.
  masked_softmax_fusion: true  # Use a kernel that fuses the attention softmax with it's mask.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



