configs/config/pretrain/vision_transformer/supervised/16gpu_convit_b.yaml [46:135]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        "cutmix_alpha": 1.0, # cutmix alpha value, cutmix is active if > 0.
        "prob": 1.0, # probability of applying mixup or cutmix per batch or element
        "switch_prob": 0.5, # probability of switching to cutmix instead of mixup when both are active
        "mode": "batch", # how to apply mixup/cutmix params (per 'batch', 'pair' (pair of elements), 'elem' (element)
        "correct_lam": True, # apply lambda correction when cutmix bbox clipped by image borders
        "label_smoothing": 0.1, # apply label smoothing to the mixed target tensor
        "num_classes": 1000 # number of classes for target
      }
    TEST:
      DATA_SOURCES: [disk_folder]
      LABEL_SOURCES: [disk_folder]
      DATASET_NAMES: [imagenet1k_folder]
      BATCHSIZE_PER_REPLICA: 64
      TRANSFORMS:
        - name: Resize
          size: 256
        - name: CenterCrop
          size: 224
        - name: ToTensor
        - name: Normalize
          mean: [0.485, 0.456, 0.406]
          std: [0.229, 0.224, 0.225]
  MODEL:
    TRUNK:
      NAME: convit
      VISION_TRANSFORMERS:
        IMAGE_SIZE: 224
        PATCH_SIZE: 16
        NUM_LAYERS: 12
        NUM_HEADS: 16
        HIDDEN_DIM: 1024
        MLP_DIM: 4096
        CLASSIFIER: token
        DROPOUT_RATE: 0
        ATTENTION_DROPOUT_RATE: 0
        DROP_PATH_RATE: 0.1 # stochastic depth dropout probability
        QKV_BIAS: False # Bias for QKV in attention layers.
        QK_SCALE: False # Scale
      CONVIT:
        N_GPSA_LAYERS: 10 # Number of gated positional self-attention layers
        CLASS_TOKEN_IN_LOCAL_LAYERS: False # Whether to add class token
        # Determines how much the positional attention is focused on the
        # patch of maximal attention. "Alpha" in the paper. Equivalent to
        # the temperature of positional attention softmax.
        LOCALITY_STRENGTH: 1.
        # Dimensionality of the relative positional embeddings * 1/3
        LOCALITY_DIM: 10
        # Whether to initialize the positional self-attention to be local
        # (equivalent to a convolution)
        USE_LOCAL_INIT: True
    HEAD:
      PARAMS: [
        ["mlp", {"dims": [1024, 1000]}],
      ]
  LOSS:
    name: cross_entropy_multiple_output_single_target
    cross_entropy_multiple_output_single_target:
      ignore_index: -1
  OPTIMIZER:
    name: adamw
    weight_decay: 0.05
    num_epochs: 100
    # We don't want to regularize the position embedding or classification token
    non_regularized_parameters: [pos_embedding, class_token]
    param_schedulers:
      lr:
        auto_lr_scaling:
          auto_scale: True
          base_value: 0.0005
          base_lr_batch_size: 512
        name: composite
        schedulers:
          - name: linear
            start_value: 0.00001
            end_value: 0.0005
          - name: cosine
            start_value: 0.0005
            end_value: 0.000001
        interval_scaling: [rescaled, fixed]
        update_interval: step
        lengths: [0.05, 0.95]
  METERS:
    name: accuracy_list_meter
    accuracy_list_meter:
      num_meters: 1
      topk_values: [1, 5]
  TRAINER:
    TRAIN_STEP_NAME: standard_train_step
  DISTRIBUTED:
    BACKEND: nccl
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



configs/config/pretrain/vision_transformer/supervised/1gpu_convit_b_example.yaml [45:135]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        "cutmix_alpha": 1.0, # cutmix alpha value, cutmix is active if > 0.
        "prob": 1.0, # probability of applying mixup or cutmix per batch or element
        "switch_prob": 0.5, # probability of switching to cutmix instead of mixup when both are active
        "mode": "batch", # how to apply mixup/cutmix params (per 'batch', 'pair' (pair of elements), 'elem' (element)
        "correct_lam": True, # apply lambda correction when cutmix bbox clipped by image borders
        "label_smoothing": 0.1, # apply label smoothing to the mixed target tensor
        "num_classes": 1000 # number of classes for target
      }

    TEST:
      DATA_SOURCES: [disk_folder]
      LABEL_SOURCES: [disk_folder]
      DATASET_NAMES: [imagenet1k_folder]
      BATCHSIZE_PER_REPLICA: 64
      TRANSFORMS:
        - name: Resize
          size: 256
        - name: CenterCrop
          size: 224
        - name: ToTensor
        - name: Normalize
          mean: [0.485, 0.456, 0.406]
          std: [0.229, 0.224, 0.225]
  MODEL:
    TRUNK:
      NAME: convit
      VISION_TRANSFORMERS:
        IMAGE_SIZE: 224
        PATCH_SIZE: 16
        NUM_LAYERS: 12
        NUM_HEADS: 16
        HIDDEN_DIM: 1024
        MLP_DIM: 4096
        CLASSIFIER: token
        DROPOUT_RATE: 0
        ATTENTION_DROPOUT_RATE: 0
        DROP_PATH_RATE: 0.1 # stochastic depth dropout probability
        QKV_BIAS: False # Bias for QKV in attention layers.
        QK_SCALE: False # Scale
      CONVIT:
        N_GPSA_LAYERS: 10 # Number of gated positional self-attention layers
        CLASS_TOKEN_IN_LOCAL_LAYERS: False # Whether to add class token
        # Determines how much the positional attention is focused on the
        # patch of maximal attention. "Alpha" in the paper. Equivalent to
        # the temperature of positional attention softmax.
        LOCALITY_STRENGTH: 1.
        # Dimensionality of the relative positional embeddings * 1/3
        LOCALITY_DIM: 10
        # Whether to initialize the positional self-attention to be local
        # (equivalent to a convolution)
        USE_LOCAL_INIT: True
    HEAD:
      PARAMS: [
        ["mlp", {"dims": [1024, 1000]}],
      ]
  LOSS:
    name: cross_entropy_multiple_output_single_target
    cross_entropy_multiple_output_single_target:
      ignore_index: -1
  OPTIMIZER:
    name: adamw
    weight_decay: 0.05
    num_epochs: 100
    # We don't want to regularize the position embedding or classification token
    non_regularized_parameters: [pos_embedding, class_token]
    param_schedulers:
      lr:
        auto_lr_scaling:
          auto_scale: True
          base_value: 0.0005
          base_lr_batch_size: 512
        name: composite
        schedulers:
          - name: linear
            start_value: 0.00001
            end_value: 0.0005
          - name: cosine
            start_value: 0.0005
            end_value: 0.000001
        interval_scaling: [rescaled, fixed]
        update_interval: step
        lengths: [0.05, 0.95]
  METERS:
    name: accuracy_list_meter
    accuracy_list_meter:
      num_meters: 1
      topk_values: [1, 5]
  TRAINER:
    TRAIN_STEP_NAME: standard_train_step
  DISTRIBUTED:
    BACKEND: nccl
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



