configs/anycontrol_local.yaml

model:
 target: models.anycontrol.AnyControlNet
 params:
 linear_start: 0.00085
 linear_end: 0.0120
 num_timesteps_cond: 1
 log_every_t: 200
 timesteps: 1000
 first_stage_key: "jpg"
 cond_stage_key: "txt"
 image_size: 64
 channels: 4
 cond_stage_trainable: false
 conditioning_key: crossattn
 monitor: val/loss_simple_ema
 scale_factor: 0.18215
 use_ema: False
 mode: local

 qformer_config:
 target: models.q_formers.blip2_qformer.Blip2Qformer
 model_name: "blip2"
 model_type: "pretrain"
 pretrained: "ckpts/blip2_pretrained.pth"
 params:
 img_size: 224
 drop_path_rate: 0
 use_grad_checkpoint: False
 vit_precision: "fp16"
 num_query_token: 256
 max_txt_len: 32
 query_token_init_type: "uniform"
 max_position_embeddings: 512
 multilevels: [3, 10, 17, 24, 31, 38]

 local_control_config:
 target: models.local_adapter.LocalAdapter
 params:
 in_channels: 4
 model_channels: 320
 local_channels: 3
 inject_channels: [192, 256, 384, 512]
 inject_layers: [1, 4, 7, 10]
 query_channels: [768, 768, 768, 768]
 query_layers: [4, 6, 8, 12]
 query_scales: [4, 2, 1, 0.5] 
 num_res_blocks: 2
 attention_resolutions: [4, 2, 1]
 channel_mult: [1, 2, 4, 4]
 use_checkpoint: False
 num_heads: 8
 use_spatial_transformer: True
 transformer_depth: 1
 context_dim: 768
 legacy: False

 global_control_config:
 target: models.global_adapter.GlobalAdapter
 params:
 cross_attention_dim: 768
 clip_embeddings_dim: 768
 context_tokens: 4
 color_in_dim: 180

 unet_config:
 target: models.local_adapter.LocalControlUNetModel
 params:
 image_size: 32
 in_channels: 4
 model_channels: 320
 out_channels: 4
 num_res_blocks: 2
 attention_resolutions: [4, 2, 1]
 channel_mult: [1, 2, 4, 4]
 use_checkpoint: False
 num_heads: 8
 use_spatial_transformer: True
 transformer_depth: 1
 context_dim: 768
 legacy: False

 first_stage_config:
 target: ldm.models.autoencoder.AutoencoderKL
 params:
 embed_dim: 4
 monitor: val/rec_loss
 ddconfig:
 double_z: true
 z_channels: 4
 resolution: 256
 in_channels: 3
 out_ch: 3
 ch: 128
 ch_mult:
 - 1
 - 2
 - 4
 - 4
 num_res_blocks: 2
 attn_resolutions: []
 dropout: 0.0
 lossconfig:
 target: torch.nn.Identity

 cond_stage_config:
 target: ldm.modules.encoders.modules.FrozenCLIPEmbedder


data:
 target: src.train.dataset.CustomDataset
 local_tasks: [canny, hed, depth, seg, openpose]
 datasets: [multigen, coco, openimages]
 json_files: 
 multigen: "./datasets/MultiGen-20M/anycontrol_annotations.jsonl"
 coco: "./datasets/MSCOCO/anycontrol_annotations.jsonl"
 openimages: "./datasets/OpenImages/anycontrol_annotations.jsonl" 
 params:
 data_root:
 multigen: ./datasets/MultiGen-20M
 coco: ./datasets/MSCOCO
 openimages: ./datasets/OpenImages
 image_dir:
 multigen: ./datasets/MultiGen-20M/images 
 coco: ./datasets/MSCOCO/train2017
 openimages: ./datasets/OpenImages/train
 condition_root:
 multigen: conditions
 coco: conditions
 openimages: conditions
 resolution: 512
 drop_txt_prob: 0.05
 drop_all_prob: 0.05
 keep_all_local_prob: 0.0
 drop_all_local_prob: 0.0
 drop_each_cond_prob: 
 canny: 0.0
 hed: 0.0
 depth: 0.0
 seg: 0.0
 openpose: 0.0

logger:
 sample: false
 N: 4
 n_row: 4
 ddim_steps: 50
 ddim_eta: 0.0
 plot_denoise_rows: false
 plot_diffusion_rows: false
 unconditional_guidance_scale: 7.5