configs/laiona6plus_prencode_random_replace.yaml [1:62]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
wandb:
  entity: null

experiment:
    project: "muse"
    name: "cc12m"
    output_dir: "cc12m"
    max_train_examples: 8974320 # toal sucessfully downloaded images for laiona6plus
    max_eval_examples: 8118
    save_every: 1000
    eval_every: 1000
    generate_every: 1000
    log_every: 50
    log_grad_norm_every: 500
    resume_from_checkpoint: False
    resume_lr_scheduler: True

model:
    vq_model:
        type: "vqgan"
        pretrained: "openMUSE/vqgan-f16-8192-laion"
    
    text_encoder:
        type: "clip"
        pretrained: "openMUSE/CLIP-ViT-L-14-DataComp.XL-s13B-b90K-penultimate"
    
    architecture: "uvit"

    transformer:
        vocab_size: 8256 # (8192 + 1 for <mask> = 8193 but 8256 is the next multiple of 8)
        hidden_size: 1024
        intermediate_size: 4096
        num_hidden_layers: 22
        num_attention_heads: 16
        max_position_embeddings: 256
        in_channels: 512
        block_out_channels:
            - 1024
        num_res_blocks: 3
        patch_size: 1
        encoder_hidden_size: 768
        add_cross_attention: True
        project_encoder_hidden_states: False
        codebook_size: 8192
        num_vq_tokens: 256
        initializer_range: 0.02
        norm_type: "rmsnorm"
        layer_norm_eps: 1e-6
        use_normformer: False
        use_encoder_layernorm: True
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0
        use_codebook_size_for_output: True

    gradient_checkpointing: True
    enable_xformers_memory_efficient_attention: True


dataset:
    type: "text2image"
    params:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



configs/laiona6plus_uvit_clip.yaml [1:62]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
wandb:
  entity: null

experiment:
    project: "muse"
    name: "cc12m"
    output_dir: "cc12m"
    max_train_examples: 8974320 # toal sucessfully downloaded images for laiona6plus
    max_eval_examples: 8118
    save_every: 1000
    eval_every: 1000
    generate_every: 1000
    log_every: 50
    log_grad_norm_every: 500
    resume_from_checkpoint: False
    resume_lr_scheduler: True

model:
    vq_model:
        type: "vqgan"
        pretrained: "openMUSE/vqgan-f16-8192-laion"
    
    text_encoder:
        type: "clip"
        pretrained: "openMUSE/CLIP-ViT-L-14-DataComp.XL-s13B-b90K-penultimate"
    
    architecture: "uvit"

    transformer:
        vocab_size: 8256 # (8192 + 1 for <mask> = 8193 but 8256 is the next multiple of 8)
        hidden_size: 1024
        intermediate_size: 4096
        num_hidden_layers: 22
        num_attention_heads: 16
        max_position_embeddings: 256
        in_channels: 512
        block_out_channels:
            - 1024
        num_res_blocks: 3
        patch_size: 1
        encoder_hidden_size: 768
        add_cross_attention: True
        project_encoder_hidden_states: False
        codebook_size: 8192
        num_vq_tokens: 256
        initializer_range: 0.02
        norm_type: "rmsnorm"
        layer_norm_eps: 1e-6
        use_normformer: False
        use_encoder_layernorm: True
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0
        use_codebook_size_for_output: True

    gradient_checkpointing: True
    enable_xformers_memory_efficient_attention: True


dataset:
    type: "text2image"
    params:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



