configs/config/pretrain/dino/dino_16gpus_deits16.yaml [65:139]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    HEAD:
      PARAMS: [
              ["swav_head", {"use_weight_norm_prototypes": True, "dims": [384, 2048, 2048, 256], "use_bn": False, "return_embeddings": False, "activation_name": "GELU", "num_clusters": [65536]}],
      ]
    TEMP_FROZEN_PARAMS_ITER_MAP: [
      ['module.heads.0.prototypes0.weight_v', 1251],
      ['module.heads.0.prototypes0.weight_g', 1251],
      ]
    AMP_PARAMS:
      AMP_TYPE: pytorch
      USE_AMP: True
  LOSS:
    name: dino_loss
    dino_loss:
      momentum: 0.996
      teacher_temp_warmup_iters: 37530 # 30 epochs
      teacher_temp_min: 0.04
      teacher_temp_max: 0.07
      ema_center: 0.9
      normalize_last_layer: false
  OPTIMIZER:
      name: adamw
      momentum: 0.9
      nesterov: False
      num_epochs: 300
      regularize_bn: False
      regularize_bias: False
      param_schedulers:
        lr_head:
          name: composite
          schedulers:
            - name: linear
              start_value: 0.00001
              end_value: 0.002
            - name: cosine
              start_value: 0.002
              end_value: 0.00001
          update_interval: epoch
          interval_scaling: [rescaled, fixed]
          lengths: [0.0333, 0.9667]
        lr:
          name: composite
          schedulers:
            - name: linear
              start_value: 0.00001
              end_value: 0.002
            - name: cosine
              start_value: 0.002
              end_value: 0.00001
          update_interval: epoch
          interval_scaling: [rescaled, fixed]
          lengths: [0.0333, 0.9667]
        weight_decay:
          name: cosine
          start_value: 0.04
          end_value: 0.4
          update_interval: epoch
        weight_decay_head:
          name: cosine
          start_value: 0.04
          end_value: 0.4
          update_interval: epoch
  DISTRIBUTED:
    BACKEND: nccl
    NUM_NODES: 2
    NUM_PROC_PER_NODE: 8
    INIT_METHOD: tcp
    RUN_ID: auto
  MACHINE:
    DEVICE: gpu
  CHECKPOINT:
    DIR: "."
    AUTO_RESUME: True
    CHECKPOINT_FREQUENCY: 5
    OVERWRITE_EXISTING: true
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



configs/config/pretrain/dino/dino_16gpus_xcit_small_12_p16.yaml [66:140]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    HEAD:
      PARAMS: [
              ["swav_head", {"use_weight_norm_prototypes": True, "dims": [384, 2048, 2048, 256], "use_bn": False, "return_embeddings": False, "activation_name": "GELU", "num_clusters": [65536]}],
      ]
    TEMP_FROZEN_PARAMS_ITER_MAP: [
      ['module.heads.0.prototypes0.weight_v', 1251],
      ['module.heads.0.prototypes0.weight_g', 1251],
      ]
    AMP_PARAMS:
      AMP_TYPE: pytorch
      USE_AMP: True
  LOSS:
    name: dino_loss
    dino_loss:
      momentum: 0.996
      teacher_temp_warmup_iters: 37530 # 30 epochs
      teacher_temp_min: 0.04
      teacher_temp_max: 0.07
      ema_center: 0.9
      normalize_last_layer: false
  OPTIMIZER:
      name: adamw
      momentum: 0.9
      nesterov: False
      num_epochs: 300
      regularize_bn: False
      regularize_bias: False
      param_schedulers:
        lr_head:
          name: composite
          schedulers:
            - name: linear
              start_value: 0.00001
              end_value: 0.002
            - name: cosine
              start_value: 0.002
              end_value: 0.00001
          update_interval: epoch
          interval_scaling: [rescaled, fixed]
          lengths: [0.0333, 0.9667]
        lr:
          name: composite
          schedulers:
            - name: linear
              start_value: 0.00001
              end_value: 0.002
            - name: cosine
              start_value: 0.002
              end_value: 0.00001
          update_interval: epoch
          interval_scaling: [rescaled, fixed]
          lengths: [0.0333, 0.9667]
        weight_decay:
          name: cosine
          start_value: 0.04
          end_value: 0.4
          update_interval: epoch
        weight_decay_head:
          name: cosine
          start_value: 0.04
          end_value: 0.4
          update_interval: epoch
  DISTRIBUTED:
    BACKEND: nccl
    NUM_NODES: 2
    NUM_PROC_PER_NODE: 8
    INIT_METHOD: tcp
    RUN_ID: auto
  MACHINE:
    DEVICE: gpu
  CHECKPOINT:
    DIR: "."
    AUTO_RESUME: True
    CHECKPOINT_FREQUENCY: 5
    OVERWRITE_EXISTING: true
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



