configs/cc12m_movq.yaml [14:57]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    log_grad_norm_every: 500
    resume_from_checkpoint: False
    resume_lr_scheduler: True

model:
    vq_model:
        type: "movq"
        pretrained: "openMUSE/movq-lion-high-res-f8-16384"
    
    text_encoder:
        type: "t5"
        pretrained: "openMUSE/t5-v1_1-large-enc"

    transformer:
        vocab_size: 16400 # (8192 + 1 for <mask> = 8193 but 8256 is the next multiple of 8)
        max_position_embeddings: 256
        hidden_size: 1024
        num_hidden_layers: 24
        num_attention_heads: 16
        intermediate_size: 4096
        add_cross_attention: True
        encoder_hidden_size: 1024
        project_encoder_hidden_states: False
        codebook_size: 16384
        num_vq_tokens: 1024
        initializer_range: 0.02
        norm_type: "rmsnorm"
        layer_norm_eps: 1e-6
        use_normformer: False
        use_encoder_layernorm: True
        use_mlm_layer: True
        use_mlm_layernorm: True
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0
        use_codebook_size_for_output: True
        use_conv_in_out: True
        patch_size: 2

    gradient_checkpointing: True
    enable_xformers_memory_efficient_attention: True


dataset:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



configs/imagenet_text2image_movq_conv.yaml [14:57]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    log_grad_norm_every: 500
    resume_from_checkpoint: False
    resume_lr_scheduler: True

model:
    vq_model:
        type: "movq"
        pretrained: "openMUSE/movq-lion-high-res-f8-16384"
    
    text_encoder:
        type: "t5"
        pretrained: "openMUSE/t5-v1_1-large-enc"

    transformer:
        vocab_size: 16400 # (16384 + + 1 = 16385 -> Vq +  <mask>, use 16400 for even division by 8)
        max_position_embeddings: 256
        hidden_size: 1024
        num_hidden_layers: 24
        num_attention_heads: 16
        intermediate_size: 4096
        add_cross_attention: True
        encoder_hidden_size: 1024
        project_encoder_hidden_states: False
        codebook_size: 16384
        num_vq_tokens: 1024
        initializer_range: 0.02
        norm_type: "rmsnorm"
        layer_norm_eps: 1e-6
        use_normformer: False
        use_encoder_layernorm: True
        use_mlm_layer: True
        use_mlm_layernorm: True
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0
        use_codebook_size_for_output: True
        use_conv_in_out: True
        patch_size: 2

    gradient_checkpointing: True
    enable_xformers_memory_efficient_attention: True


dataset:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



