configs/cc12m_uvit.yaml [42:74]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        add_cross_attention: True
        project_encoder_hidden_states: False
        codebook_size: 8192
        num_vq_tokens: 256
        initializer_range: 0.02
        norm_type: "rmsnorm"
        layer_norm_eps: 1e-6
        use_normformer: False
        use_encoder_layernorm: True
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0
        use_codebook_size_for_output: True

    gradient_checkpointing: True
    enable_xformers_memory_efficient_attention: True


dataset:
    type: "text2image"
    params:
        train_shards_path_or_url: 
            - "pipe:aws s3 cp s3://muse-datasets/conceptual-captions/CC12M/{00000..02484}.tar -"
            - "pipe:aws s3 cp s3://muse-datasets/conceptual-captions/CC3M/train/{00000..00663}.tar -"
        eval_shards_path_or_url: "pipe:aws s3 cp s3://muse-datasets/conceptual-captions/CC3M/valid/{00000..00003}.tar -"
        validation_prompts_file: "validation_prompts/cc_validation_prompts.txt"
        batch_size: ${training.batch_size}
        shuffle_buffer_size: 1000
        num_workers: 4
        resolution: 256
        pin_memory: True
        persistent_workers: True
    preprocessing:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



configs/cc12m_uvit_clip.yaml [42:74]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        add_cross_attention: True
        project_encoder_hidden_states: False
        codebook_size: 8192
        num_vq_tokens: 256
        initializer_range: 0.02
        norm_type: "rmsnorm"
        layer_norm_eps: 1e-6
        use_normformer: False
        use_encoder_layernorm: True
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0
        use_codebook_size_for_output: True

    gradient_checkpointing: True
    enable_xformers_memory_efficient_attention: True


dataset:
    type: "text2image"
    params:
        train_shards_path_or_url: 
            - "pipe:aws s3 cp s3://muse-datasets/conceptual-captions/CC12M/{00000..02484}.tar -"
            - "pipe:aws s3 cp s3://muse-datasets/conceptual-captions/CC3M/train/{00000..00663}.tar -"
        eval_shards_path_or_url: "pipe:aws s3 cp s3://muse-datasets/conceptual-captions/CC3M/valid/{00000..00003}.tar -"
        validation_prompts_file: "validation_prompts/cc_validation_prompts.txt"
        batch_size: ${training.batch_size}
        shuffle_buffer_size: 1000
        num_workers: 4
        resolution: 256
        pin_memory: True
        persistent_workers: True
    preprocessing:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



