mmf/projects/vilt/configs/vqa2/defaults.yaml [1:97]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
includes:
- ../projects/unit/configs/vqa2_dataset_cfg.yaml

other_configs:
  image_w: 384
  image_h: 384
  hidden_size: 768

dataset_config:
  vqa2:
    processors:
      image_processor:
        type: vilt_image_processor
        params:
          size:
          - ${other_configs.image_w}
          - ${other_configs.image_h}
      text_processor:
        type: vilt_text_tokenizer
        params:
          mask_probability: 0
          max_seq_length: 40

model_config:
  vilt:
    tasks: ${datasets}
    heads:
      vqa2:
        - type: mlp
          num_layers: 1
          loss: vqa2_mlp
          freeze: false
          lr_multiplier: 10.0
          in_dim: 768
          hidden_size: 1536
          num_labels: 3129
          pooler_name: bert_pooler
    losses:
      vqa2_mlp: logit_bce

    image_encoder:
      type: vit
      params:
        random_init: True
        pretrained_model_name: google/vit-base-patch32-384
        gradient_checkpointing: false
        do_pooling: false
        hidden_dropout_prob: 0.1
        add_pooling_layer: False
        do_patch_embeddings: False
        image_size:
        - ${other_configs.image_w}
        - ${other_configs.image_h}
        regularize_bn: False
        regularize_bias: True
        hidden_dim: ${other_configs.hidden_size}
        pretrained_model: vit_base_patch32_384
        mlp_dim: 3072

    text_embeddings:
      type: vilt_text_embedding
      bert_model_name: bert-base-uncased
      hidden_dim: ${other_configs.hidden_size}
      hidden_size: 768
      max_position_embeddings: 40
      random_init: True

training:
  clip_gradients: false
  lr_scheduler: true
  max_updates: 44000
  checkpoint_interval: 4000
  evaluation_interval: 4000
  batch_size: 256 # 32 per GPU * 8 GPU
  find_unused_parameters: false
  log_interval: 1000

optimizer:
  type: adam_w
  params:
    lr: 1e-4
    eps: 1e-8
    weight_decay: 1e-2


scheduler:
  type: warmup_cosine
  params:
    num_warmup_steps: 4400
    num_training_steps: ${training.max_updates}


evaluation:
  metrics:
    - type: vqa_accuracy
      datasets:
      - vqa2
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



projects/vilt/configs/vqa2/defaults.yaml [1:97]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
includes:
- ../projects/unit/configs/vqa2_dataset_cfg.yaml

other_configs:
  image_w: 384
  image_h: 384
  hidden_size: 768

dataset_config:
  vqa2:
    processors:
      image_processor:
        type: vilt_image_processor
        params:
          size:
          - ${other_configs.image_w}
          - ${other_configs.image_h}
      text_processor:
        type: vilt_text_tokenizer
        params:
          mask_probability: 0
          max_seq_length: 40

model_config:
  vilt:
    tasks: ${datasets}
    heads:
      vqa2:
        - type: mlp
          num_layers: 1
          loss: vqa2_mlp
          freeze: false
          lr_multiplier: 10.0
          in_dim: 768
          hidden_size: 1536
          num_labels: 3129
          pooler_name: bert_pooler
    losses:
      vqa2_mlp: logit_bce

    image_encoder:
      type: vit
      params:
        random_init: True
        pretrained_model_name: google/vit-base-patch32-384
        gradient_checkpointing: false
        do_pooling: false
        hidden_dropout_prob: 0.1
        add_pooling_layer: False
        do_patch_embeddings: False
        image_size:
        - ${other_configs.image_w}
        - ${other_configs.image_h}
        regularize_bn: False
        regularize_bias: True
        hidden_dim: ${other_configs.hidden_size}
        pretrained_model: vit_base_patch32_384
        mlp_dim: 3072

    text_embeddings:
      type: vilt_text_embedding
      bert_model_name: bert-base-uncased
      hidden_dim: ${other_configs.hidden_size}
      hidden_size: 768
      max_position_embeddings: 40
      random_init: True

training:
  clip_gradients: false
  lr_scheduler: true
  max_updates: 44000
  checkpoint_interval: 4000
  evaluation_interval: 4000
  batch_size: 256 # 32 per GPU * 8 GPU
  find_unused_parameters: false
  log_interval: 1000

optimizer:
  type: adam_w
  params:
    lr: 1e-4
    eps: 1e-8
    weight_decay: 1e-2


scheduler:
  type: warmup_cosine
  params:
    num_warmup_steps: 4400
    num_training_steps: ${training.max_updates}


evaluation:
  metrics:
    - type: vqa_accuracy
      datasets:
      - vqa2
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



