infra/4-training/nemo-example/nemo-configurations/gpt-175b.yaml [102:119]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  tokenizer:
    library: 'megatron'
    type: 'GPT2BPETokenizer'
    model: null
    delimiter: null  # only used for tabular tokenizer
    vocab_file: gpt2-vocab.json
    merge_file: gpt2-merges.txt

  # precision
  native_amp_init_scale: 4294967296  # 2 ** 32
  native_amp_growth_interval: 1000
  hysteresis: 2  # Gradient scale hysteresis
  fp32_residual_connection: false  # Move residual connections to fp32
  fp16_lm_cross_entropy: false  # Move the cross entropy unreduced loss calculation for lm head to fp16

  # Megatron O2-style half-precision
  megatron_amp_O2: true  # Enable O2-level automatic mixed precision using master parameters
  grad_allreduce_chunk_size_mb: 125
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



infra/4-training/nemo-example/selected-configuration.yaml [109:126]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  tokenizer:
    library: 'megatron'
    type: 'GPT2BPETokenizer'
    model: null
    delimiter: null  # only used for tabular tokenizer
    vocab_file: gpt2-vocab.json
    merge_file: gpt2-merges.txt

  # precision
  native_amp_init_scale: 4294967296  # 2 ** 32
  native_amp_growth_interval: 1000
  hysteresis: 2  # Gradient scale hysteresis
  fp32_residual_connection: false  # Move residual connections to fp32
  fp16_lm_cross_entropy: false  # Move the cross entropy unreduced loss calculation for lm head to fp16

  # Megatron O2-style half-precision
  megatron_amp_O2: true  # Enable O2-level automatic mixed precision using master parameters
  grad_allreduce_chunk_size_mb: 125
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



