mmf/projects/unit/configs/all_8_datasets/shared_dec.yaml [1:105]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
includes:
- ../vqa2_dataset_cfg.yaml
- ../visual_entailment_dataset_cfg.yaml

model_config:
  unit:
    base_args:
      num_queries:
        detection:
          detection_coco: 100
          detection_visual_genome: 100
        vl:
          vqa2: 25
          # hateful_memes: 25
          visual_entailment: 25
        glue:
          glue_qnli: 25
          glue_qqp: 25
          glue_sst2: 25
          glue_mnli_mismatched: 25
      share_decoders: true
      decoder_hidden_dim: 768
      dilation: true
      use_task_embedding_in_img_encoder: true
      use_task_embedding_in_lang_encoder: true
    losses:
    - logit_bce
    # initialize the ResNet convnet backbone from DETR
    base_ckpt_path: https://dl.fbaipublicfiles.com/detr/detr-r50-dc5-f0fb7ef5.pth
    base_ckpt_load_backbone_only: true

evaluation:
  metrics:
  - type: vqa_accuracy
    datasets:
    - vqa2
  - type: detection_mean_ap
    key: detection_mean_ap
    datasets:
    - detection_coco
    - detection_visual_genome
    params:
      dataset_json_files:
        detection_coco:
          val: ${env.data_dir}/datasets/coco/detection_2017/annotations/annotations/instances_val2017.json
        detection_visual_genome:
          val: ${env.data_dir}/datasets/visual_genome/detection_split_by_coco_2017/annotations/instances_val_split_by_coco_2017.json
  - type: accuracy
    key: accuracy
    datasets:
    - visual_entailment
    - glue_qnli
    - glue_qqp
    - glue_sst2
    - glue_mnli_mismatched

optimizer:
  type: adam_w  # HuggingFace transformer's AdamW
  params:
    lr: 5e-5
    eps: 1e-8
    weight_decay: 1e-4

scheduler:
  type: warmup_cosine
  params:
    num_warmup_steps: 2000
    num_training_steps: ${training.max_updates}

training:
  num_workers: 2
  # these are mostly the same as in COCO detection training
  clip_norm_mode: all
  clip_gradients: true
  max_grad_l2_norm: 0.1
  lr_scheduler: true
  lr_ratio: 0.1
  batch_size: 64
  max_updates: 500000
  checkpoint_interval: 10000
  evaluation_interval: 10000
  dataset_size_proportional_sampling: false
  early_stop:
    enabled: false
    criteria: vqa2/vqa_accuracy
    minimize: false
  stdout_capture: false
  find_unused_parameters: true

multitasking:
  enabled: true
  type: ratios
  params:
    sampling_ratios:
      detection_coco: 0.2
      detection_visual_genome: 0.07
      visual_entailment: 0.12
      vqa2: 0.26
      glue_qnli: 0.1
      glue_mnli_mismatched: 0.1
      glue_qqp: 0.1
      glue_sst2: 0.05

checkpoint:
  max_to_keep: 5
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



projects/unit/configs/all_8_datasets/shared_dec.yaml [1:105]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
includes:
- ../vqa2_dataset_cfg.yaml
- ../visual_entailment_dataset_cfg.yaml

model_config:
  unit:
    base_args:
      num_queries:
        detection:
          detection_coco: 100
          detection_visual_genome: 100
        vl:
          vqa2: 25
          # hateful_memes: 25
          visual_entailment: 25
        glue:
          glue_qnli: 25
          glue_qqp: 25
          glue_sst2: 25
          glue_mnli_mismatched: 25
      share_decoders: true
      decoder_hidden_dim: 768
      dilation: true
      use_task_embedding_in_img_encoder: true
      use_task_embedding_in_lang_encoder: true
    losses:
    - logit_bce
    # initialize the ResNet convnet backbone from DETR
    base_ckpt_path: https://dl.fbaipublicfiles.com/detr/detr-r50-dc5-f0fb7ef5.pth
    base_ckpt_load_backbone_only: true

evaluation:
  metrics:
  - type: vqa_accuracy
    datasets:
    - vqa2
  - type: detection_mean_ap
    key: detection_mean_ap
    datasets:
    - detection_coco
    - detection_visual_genome
    params:
      dataset_json_files:
        detection_coco:
          val: ${env.data_dir}/datasets/coco/detection_2017/annotations/annotations/instances_val2017.json
        detection_visual_genome:
          val: ${env.data_dir}/datasets/visual_genome/detection_split_by_coco_2017/annotations/instances_val_split_by_coco_2017.json
  - type: accuracy
    key: accuracy
    datasets:
    - visual_entailment
    - glue_qnli
    - glue_qqp
    - glue_sst2
    - glue_mnli_mismatched

optimizer:
  type: adam_w  # HuggingFace transformer's AdamW
  params:
    lr: 5e-5
    eps: 1e-8
    weight_decay: 1e-4

scheduler:
  type: warmup_cosine
  params:
    num_warmup_steps: 2000
    num_training_steps: ${training.max_updates}

training:
  num_workers: 2
  # these are mostly the same as in COCO detection training
  clip_norm_mode: all
  clip_gradients: true
  max_grad_l2_norm: 0.1
  lr_scheduler: true
  lr_ratio: 0.1
  batch_size: 64
  max_updates: 500000
  checkpoint_interval: 10000
  evaluation_interval: 10000
  dataset_size_proportional_sampling: false
  early_stop:
    enabled: false
    criteria: vqa2/vqa_accuracy
    minimize: false
  stdout_capture: false
  find_unused_parameters: true

multitasking:
  enabled: true
  type: ratios
  params:
    sampling_ratios:
      detection_coco: 0.2
      detection_visual_genome: 0.07
      visual_entailment: 0.12
      vqa2: 0.26
      glue_qnli: 0.1
      glue_mnli_mismatched: 0.1
      glue_qqp: 0.1
      glue_sst2: 0.05

checkpoint:
  max_to_keep: 5
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



