configs/research_run_512.yaml [8:65]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    max_train_examples: 1000000000
    max_eval_examples: 5000
    save_every: 2000
    eval_every: 1000
    generate_every: 1000
    log_every: 50
    log_grad_norm_every: 500
    resume_from_checkpoint: latest
    resume_lr_scheduler: True
    checkpoints_total_limit: 10

model:
    vq_model:
        type: "vqgan"
        pretrained: "openMUSE/vqgan-f16-8192-laion"
    
    text_encoder:
        type: "clip"
        pretrained: "openMUSE/clip-vit-large-patch14-text-enc"
        pad_token_id: 49407
    
    architecture: "uvit"

    transformer:
        vocab_size: 8256 # (8192 + 1 for <mask> = 8193 but 8256 is the next multiple of 8)
        hidden_size: 1024
        intermediate_size: 2816
        num_hidden_layers: 22
        num_attention_heads: 16
        in_channels: 768
        block_out_channels:
            - 768
        block_has_attention:
            - True
        block_num_heads: 12
        num_res_blocks: 3
        res_ffn_factor: 4
        patch_size: 1
        encoder_hidden_size: 768
        add_cross_attention: True
        project_encoder_hidden_states: True
        codebook_size: 8192
        num_vq_tokens: 512
        initializer_range: 0.02
        norm_type: "rmsnorm"
        layer_norm_eps: 1e-6
        ln_elementwise_affine: True
        use_encoder_layernorm: False
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0
        use_codebook_size_for_output: True
        use_empty_embeds_for_uncond: True
        add_cond_embeds: True
        cond_embed_dim: 768
        add_micro_cond_embeds: True
        micro_cond_encode_dim: 256
        micro_cond_embed_dim: 1280
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



configs/research_run_512_with_downsample.yaml [8:65]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    max_train_examples: 1000000000
    max_eval_examples: 5000
    save_every: 2000
    eval_every: 1000
    generate_every: 1000
    log_every: 50
    log_grad_norm_every: 500
    resume_from_checkpoint: latest
    resume_lr_scheduler: True
    checkpoints_total_limit: 10

model:
    vq_model:
        type: "vqgan"
        pretrained: "openMUSE/vqgan-f16-8192-laion"
    
    text_encoder:
        type: "clip"
        pretrained: "openMUSE/clip-vit-large-patch14-text-enc"
        pad_token_id: 49407
    
    architecture: "uvit"

    transformer:
        vocab_size: 8256 # (8192 + 1 for <mask> = 8193 but 8256 is the next multiple of 8)
        hidden_size: 1024
        intermediate_size: 2816
        num_hidden_layers: 22
        num_attention_heads: 16
        in_channels: 768
        block_out_channels:
            - 768
        block_has_attention:
            - True
        block_num_heads: 12
        num_res_blocks: 3
        res_ffn_factor: 4
        patch_size: 1
        encoder_hidden_size: 768
        add_cross_attention: True
        project_encoder_hidden_states: True
        codebook_size: 8192
        num_vq_tokens: 512
        initializer_range: 0.02
        norm_type: "rmsnorm"
        layer_norm_eps: 1e-6
        ln_elementwise_affine: True
        use_encoder_layernorm: False
        use_bias: False
        hidden_dropout: 0.0
        attention_dropout: 0.0
        use_codebook_size_for_output: True
        use_empty_embeds_for_uncond: True
        add_cond_embeds: True
        cond_embed_dim: 768
        add_micro_cond_embeds: True
        micro_cond_encode_dim: 256
        micro_cond_embed_dim: 1280
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



