configs/imagenet.yaml [37:83]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        use_encoder_layernorm: True
        use_mlm_layer: True
        use_mlm_layernorm: True
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0

    gradient_checkpointing: True
    enable_xformers_memory_efficient_attention: True


dataset:
    params:
        train_shards_path_or_url: "pipe:aws s3 cp s3://muse-datasets/imagenet-wds/imagenet-train-{000000..000320}.tar -"
        eval_shards_path_or_url: "pipe:aws s3 cp s3://muse-datasets/imagenet-wds/imagenet-val-{000000..000012}.tar -"
        batch_size: ${training.batch_size}
        shuffle_buffer_size: 1000
        num_workers: 4
        resolution: 256
        pin_memory: True
        persistent_workers: True
    preprocessing:
        resolution: 256
        center_crop: True
        random_flip: False


optimizer:
    name: fused_adamw 
    params: # default adamw params
        learning_rate: 1e-4
        scale_lr: False # scale learning rate by total batch size
        beta1: 0.9
        beta2: 0.999
        weight_decay: 0.01
        epsilon: 1e-8


lr_scheduler:
    scheduler: "constant_with_warmup"
    params:
        learning_rate: ${optimizer.params.learning_rate}
        warmup_steps: 1000


training:
    gradient_accumulation_steps: 1
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



configs/imagenet_movq.yaml [37:83]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        use_encoder_layernorm: True
        use_mlm_layer: True
        use_mlm_layernorm: True
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0

    gradient_checkpointing: True
    enable_xformers_memory_efficient_attention: True


dataset:
    params:
        train_shards_path_or_url: "pipe:aws s3 cp s3://muse-datasets/imagenet-wds/imagenet-train-{000000..000320}.tar -"
        eval_shards_path_or_url: "pipe:aws s3 cp s3://muse-datasets/imagenet-wds/imagenet-val-{000000..000012}.tar -"
        batch_size: ${training.batch_size}
        shuffle_buffer_size: 1000
        num_workers: 4
        resolution: 256
        pin_memory: True
        persistent_workers: True
    preprocessing:
        resolution: 256
        center_crop: True
        random_flip: False


optimizer:
    name: fused_adamw 
    params: # default adamw params
        learning_rate: 1e-4
        scale_lr: False # scale learning rate by total batch size
        beta1: 0.9
        beta2: 0.999
        weight_decay: 0.01
        epsilon: 1e-8


lr_scheduler:
    scheduler: "constant_with_warmup"
    params:
        learning_rate: ${optimizer.params.learning_rate}
        warmup_steps: 1000


training:
    gradient_accumulation_steps: 1
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



