Upload folder using huggingface_hub

.gitattributes

@@ -0,0 +1,3 @@
+text_encoder/model.safetensors filter=lfs diff=lfs merge=lfs -text
+unet/diffusion_pytorch_model.safetensors filter=lfs diff=lfs merge=lfs -text
+vae/diffusion_pytorch_model.safetensors filter=lfs diff=lfs merge=lfs -text

config.yaml

@@ -0,0 +1,83 @@
+pretrained_model_path: output/latent/animate_anything_512_v1.02
+output_dir: ./output/latent
+train_data:
+  width: 512
+  height: 512
+  use_bucketing: false
+  return_mask: true
+  return_motion: true
+  sample_start_idx: 1
+  fps: 8
+  n_sample_frames: 16
+  json_path: /webvid/animation0.json
+validation_data:
+  prompt: a girl moves hands
+  prompt_image: output/example/girl.jpg
+  sample_preview: true
+  num_frames: 16
+  width: 512
+  height: 512
+  num_inference_steps: 25
+  guidance_scale: 9
+dataset_types:
+- json
+shuffle: true
+validation_steps: 200
+trainable_modules:
+- all
+- attn1
+- conv_in
+- temp_conv
+- motion
+not_trainable_modules: []
+trainable_text_modules: null
+extra_unet_params: null
+extra_text_encoder_params: null
+train_batch_size: 4
+max_train_steps: 10000
+learning_rate: 5.0e-06
+scale_lr: false
+lr_scheduler: constant
+lr_warmup_steps: 0
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_weight_decay: 0
+adam_epsilon: 1.0e-08
+max_grad_norm: 1.0
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+text_encoder_gradient_checkpointing: false
+checkpointing_steps: 2000
+resume_from_checkpoint: null
+resume_step: null
+mixed_precision: fp16
+use_8bit_adam: false
+enable_xformers_memory_efficient_attention: false
+enable_torch_2_attn: true
+seed: null
+train_text_encoder: false
+use_offset_noise: false
+rescale_schedule: false
+offset_noise_strength: 0.1
+extend_dataset: false
+cache_latents: false
+cached_latent_dir: null
+lora_version: cloneofsimo
+save_lora_for_webui: true
+only_lora_for_webui: false
+lora_bias: none
+use_unet_lora: false
+use_text_lora: false
+unet_lora_modules:
+- UNet3DConditionModel
+text_encoder_lora_modules:
+- CLIPEncoderLayer
+save_pretrained_model: true
+lora_rank: 16
+lora_path: ''
+lora_unet_dropout: 0.1
+lora_text_dropout: 0.1
+logger_type: tensorboard
+motion_mask: true
+motion_strength: true
+kwargs: {}

model_index.json

@@ -0,0 +1,25 @@
+{
+  "_class_name": "LatentToVideoPipeline",
+  "_diffusers_version": "0.24.0",
+  "_name_or_path": "output/latent/train_2023-12-15T12-02-48/checkpoint-2000",
+  "scheduler": [
+    "diffusers",
+    "DDIMScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "unet": [
+    "models.unet_3d_condition_mask",
+    "UNet3DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ]
+}

scheduler/scheduler_config.json

@@ -0,0 +1,20 @@
+{
+  "_class_name": "DDIMScheduler",
+  "_diffusers_version": "0.24.0",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "clip_sample_range": 1.0,
+  "dynamic_thresholding_ratio": 0.995,
+  "num_train_timesteps": 1000,
+  "prediction_type": "epsilon",
+  "rescale_betas_zero_snr": false,
+  "sample_max_value": 1.0,
+  "set_alpha_to_one": false,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "thresholding": false,
+  "timestep_spacing": "leading",
+  "trained_betas": null
+}

text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "output/latent/train_2023-12-15T12-02-48/checkpoint-2000",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1024,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 23,
+  "pad_token_id": 1,
+  "projection_dim": 512,
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.1",
+  "vocab_size": 49408
+}

text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb24b36c82491320c0a990bdbb3de4cf4f7e4819aaff390d596a0f41d75bdd84
+size 1361596304

tokenizer/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "!",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "!",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "!",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>"
+}

unet/config.json

@@ -0,0 +1,36 @@
+{
+  "_class_name": "UNet3DConditionModel",
+  "_diffusers_version": "0.24.0",
+  "_name_or_path": "output/latent/train_2023-12-15T12-02-48/checkpoint-2000",
+  "act_fn": "silu",
+  "attention_head_dim": 64,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "cross_attention_dim": 1024,
+  "down_block_types": [
+    "CrossAttnDownBlock3D",
+    "CrossAttnDownBlock3D",
+    "CrossAttnDownBlock3D",
+    "DownBlock3D"
+  ],
+  "downsample_padding": 1,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "motion_mask": true,
+  "motion_strength": true,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "out_channels": 4,
+  "sample_size": 32,
+  "up_block_types": [
+    "UpBlock3D",
+    "CrossAttnUpBlock3D",
+    "CrossAttnUpBlock3D",
+    "CrossAttnUpBlock3D"
+  ]
+}

unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c83d7d4abbc91386c8f29b493b7b8939e38860109d2859d98ba757a690b6a6a
+size 5653789792

vae/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.24.0",
+  "_name_or_path": "output/latent/train_2023-12-15T12-02-48/checkpoint-2000",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 512,
+  "scaling_factor": 0.18215,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc436097e5cbb105c44df4403ae84acf7e81de1f64acfedc334888e9a8eea223
+size 334643268