Upload 66 files

.gitattributes

@@ -33,3 +33,32 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/0_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/0.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/1_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/1.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/2_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/2.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/3_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/3.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/4_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/4.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/5_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/5.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/6_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/6.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/7_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/7.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/8_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/8.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/9_edit.png filter=lfs diff=lfs merge=lfs -text
+assets/9.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/outputvideo/output_0.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/outputvideo/output_1.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/outputvideo/output_2.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/outputvideo/output_3.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/outputvideo/output_5.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/outputvideo/output_6.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/outputvideo/output_7.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/outputvideo/output_8.mp4 filter=lfs diff=lfs merge=lfs -text
+control_cogvideox/__pycache__/embeddings.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text

app_video_image_guidence.py

@@ -0,0 +1,258 @@
+import gradio as gr
+
+import cv2
+import torch
+import numpy as np
+import os
+from control_cogvideox.cogvideox_transformer_3d import CogVideoXTransformer3DModel
+from control_cogvideox.controlnet_cogvideox_transformer_3d import ControlCogVideoXTransformer3DModel
+from pipeline_cogvideox_controlnet_5b_i2v_instruction2 import ControlCogVideoXPipeline
+from diffusers.utils import export_to_video
+from diffusers import AutoencoderKLCogVideoX
+from transformers import T5EncoderModel, T5Tokenizer
+from diffusers.schedulers import CogVideoXDDIMScheduler
+
+from omegaconf import OmegaConf
+from transformers import T5EncoderModel
+from einops import rearrange
+import decord
+from typing import List
+from tqdm import tqdm
+
+import PIL
+import torch.nn.functional as F
+from torchvision import transforms
+
+def get_prompt(file:str):
+    with open(file,'r') as f:
+        a=f.readlines()
+    return a #a[0]:positive prompt, a[1] negative prompt
+
+def init_pipe():
+    def unwarp_model(state_dict):
+        new_state_dict = {}
+        for key in state_dict:
+            new_state_dict[key.split('module.')[1]] = state_dict[key]
+        return new_state_dict
+
+    i2v=True
+    root_path="./cogvideox_instructions_lr_1e_5_bs_48_2_epoch_params_controlnet_first_frame_5b_mixed_precision_480_896/cogvideox-2025-01-15T12-42-19/"
+    training_steps=30001
+
+    if i2v:
+        key = "i2v"
+    else:
+        key = "t2v"
+    noise_scheduler = CogVideoXDDIMScheduler(
+        **OmegaConf.to_container(
+            OmegaConf.load(f"./cogvideox-5b-{key}/scheduler/scheduler_config.json")
+        )
+    )
+
+    text_encoder = T5EncoderModel.from_pretrained(f"./cogvideox-5b-{key}/", subfolder="text_encoder", torch_dtype=torch.float16)#.to("cuda:0")
+    vae = AutoencoderKLCogVideoX.from_pretrained(f"./cogvideox-5b-{key}/", subfolder="vae", torch_dtype=torch.float16).to("cuda:0")
+    tokenizer = T5Tokenizer.from_pretrained(f"./cogvideox-5b-{key}/tokenizer", torch_dtype=torch.float16)
+
+
+    config = OmegaConf.to_container(
+        OmegaConf.load(f"./cogvideox-5b-{key}/transformer/config.json")
+    )
+    if i2v:
+        config["in_channels"] = 32
+    else:
+        config["in_channels"] = 16
+    transformer = CogVideoXTransformer3DModel(**config)
+
+    control_config = OmegaConf.to_container(
+        OmegaConf.load(f"./cogvideox-5b-{key}/transformer/config.json")
+    )
+    if i2v:
+        control_config["in_channels"] = 32
+    else:
+        control_config["in_channels"] = 16
+    control_config['num_layers'] = 6
+    control_config['control_in_channels'] = 16
+    controlnet_transformer = ControlCogVideoXTransformer3DModel(**control_config)
+
+    all_state_dicts = torch.load(f"{root_path}/checkpoints/checkpoint{training_steps}.ckpt", map_location="cpu",weights_only=True)
+    transformer_state_dict = unwarp_model(all_state_dicts["transformer_state_dict"])
+    controlnet_transformer_state_dict = unwarp_model(all_state_dicts["controlnet_transformer_state_dict"])
+
+    transformer.load_state_dict(transformer_state_dict, strict=True)
+    controlnet_transformer.load_state_dict(controlnet_transformer_state_dict, strict=True)
+
+    transformer = transformer.half().to("cuda:0")
+    controlnet_transformer = controlnet_transformer.half().to("cuda:0")
+
+    vae = vae.eval()
+    text_encoder = text_encoder.eval()
+    transformer = transformer.eval()
+    controlnet_transformer = controlnet_transformer.eval()
+
+    pipe = ControlCogVideoXPipeline(tokenizer,
+            text_encoder,
+            vae,
+            transformer,
+            noise_scheduler,
+            controlnet_transformer,
+    )
+
+    pipe.vae.enable_slicing()
+    pipe.vae.enable_tiling()
+    pipe.enable_model_cpu_offload()
+
+    return pipe
+
+def inference(source_images, 
+        target_images, 
+        text_prompt, negative_prompt, 
+        pipe, vae, 
+        step, guidance_scale, 
+        h, w, random_seed)->List[PIL.Image.Image]:
+    torch.manual_seed(random_seed)
+
+    source_pixel_values = source_images/127.5 - 1.0
+    source_pixel_values = source_pixel_values.to(torch.float16).to("cuda:0")
+    if target_images is not None:
+        target_pixel_values = target_images/127.5 - 1.0
+        target_pixel_values = target_pixel_values.to(torch.float16).to("cuda:0")
+    bsz,f,h,w,c = source_pixel_values.shape
+
+    with torch.no_grad():
+        source_pixel_values = rearrange(source_pixel_values, "b f w h c -> b c f w h")
+        source_latents = vae.encode(source_pixel_values).latent_dist.sample()
+        source_latents = source_latents.to(torch.float16)
+        source_latents = source_latents * vae.config.scaling_factor
+        source_latents = rearrange(source_latents, "b c f h w -> b f c h w")
+
+        if target_images is not None:
+            target_pixel_values = rearrange(target_pixel_values, "b f w h c -> b c f w h")
+            images = target_pixel_values[:,:,:1,...]
+            image_latents = vae.encode(images).latent_dist.sample()
+            image_latents = image_latents.to(torch.float16)
+            image_latents = image_latents * vae.config.scaling_factor
+            image_latents = rearrange(image_latents, "b c f h w -> b f c h w")
+            image_latents = torch.cat([image_latents, torch.zeros_like(source_latents)[:,1:]],dim=1)
+            latents = torch.cat([image_latents, source_latents], dim=2)
+        else:
+            image_latents = None
+            latents = source_latents
+
+    video = pipe(
+        prompt = text_prompt,
+        negative_prompt = negative_prompt,
+        video_condition = source_latents, # input to controlnet
+        video_condition2 = image_latents, # concat with latents
+        height = h,
+        width = w,
+        num_frames = f,
+        num_inference_steps = 50,
+        interval = 6,
+        guidance_scale = guidance_scale,
+        generator = torch.Generator(device=f"cuda:0").manual_seed(random_seed)
+    ).frames[0]
+
+    return video
+
+def process_video(video_file, image_file, positive_prompt, negative_prompt, guidance, random_seed, choice, progress=gr.Progress(track_tqdm=True))->str:
+    if choice==33:
+        video_shard=1
+    elif choice==65:
+        video_shard=2
+    
+    pipe=PIPE
+
+    h = 448
+    w = 768
+    step=30001 #checkpoint
+    frames_per_shard=33
+
+    #get image
+    image = cv2.imread(image_file)
+    resized_image = cv2.resize(image, (768, 448))
+    resized_image = cv2.cvtColor(resized_image, cv2.COLOR_BGR2RGB)
+    image=torch.from_numpy(resized_image)
+    #get mp4
+    vr = decord.VideoReader(video_file)
+    frames = vr.get_batch(list(range(33))).asnumpy()
+    _,src_h,src_w,_=frames.shape
+    resized_frames = [cv2.resize(frame, (768, 448)) for frame in frames]
+    images=torch.from_numpy(np.array(resized_frames))
+
+    target_path="outputvideo"
+    source_images = images[None,...]
+    target_images = image[None,None,...]
+    
+    video:List[PIL.Image.Image]=[]
+
+    
+    for i in progress.tqdm(range(video_shard)):
+        if i>0: #first frame guidence
+            first_frame=transforms.ToTensor()(video[-1])
+            first_frame = first_frame*255.0
+            first_frame = rearrange(first_frame,"c w h -> w h c")
+            source_images=source_images#仍用原视频引导
+            target_images=first_frame[None,None,...]
+
+        video+=inference(source_images, \
+            target_images, positive_prompt, \
+            negative_prompt, pipe, pipe.vae, \
+            step, guidance, \
+            h, w, random_seed)
+        i+=1
+
+    video=[image.resize((int(src_w/src_h*448),448))for image in video]
+
+    os.makedirs(f"./{target_path}", exist_ok=True)
+    output_path:str=f"./{target_path}/output_{video_file[-5]}.mp4"
+    export_to_video(video, output_path, fps=8)
+    return output_path
+    
+
+PIPE=init_pipe()
+
+with gr.Blocks() as demo:
+    gr.Markdown("""
+    # Señorita-2M: A High-Quality Instruction-based Dataset for General Video Editing by Video Specialists
+    
+    [Paper](https://arxiv.org/bas/2502.06734) | [Code](https://127.0.0.1:7860) | [Huggingface](https://127.0.0.1:7860)
+    """)
+    #gr.HTML(open("gradio_title.md",'r').read())
+    
+    with gr.Row():
+        video_input = gr.Video(label="Video input")
+        image_input = gr.Image(type="filepath", label="First frame guidence")
+    with gr.Row():
+        with gr.Column():
+            positive_prompt = gr.Textbox(label="Positive prompt",value="")
+            negative_prompt = gr.Textbox(label="Negative prompt",value="")
+            seed = gr.Slider(minimum=0, maximum=2147483647, step=1, value=0, label="Seed")
+            guidance_slider = gr.Slider(minimum=1, maximum=10, value=4, label="Guidance")
+            choice=gr.Radio(choices=[33,65],label="Frame number",value=33)
+        with gr.Column():
+            video_output = gr.Video(label="Video output")
+            
+    with gr.Row():
+        submit_button = gr.Button("Generate")
+        submit_button.click(fn=process_video, inputs=[video_input, image_input, positive_prompt, negative_prompt, guidance_slider, seed, choice], outputs=video_output)
+    with gr.Row():
+        gr.Examples(
+            [
+                ["assets/0.mp4","assets/0_edit.png",get_prompt("assets/0.txt")[0],get_prompt("assets/0.txt")[1],4,0,33],
+                ["assets/1.mp4","assets/1_edit.png",get_prompt("assets/1.txt")[0],get_prompt("assets/1.txt")[1],4,0,33],
+                ["assets/2.mp4","assets/2_edit.png",get_prompt("assets/2.txt")[0],get_prompt("assets/2.txt")[1],4,0,33],
+                ["assets/3.mp4","assets/3_edit.png",get_prompt("assets/3.txt")[0],get_prompt("assets/3.txt")[1],4,0,33],
+                ["assets/4.mp4","assets/4_edit.png",get_prompt("assets/4.txt")[0],get_prompt("assets/4.txt")[1],4,0,33],
+                ["assets/5.mp4","assets/5_edit.png",get_prompt("assets/5.txt")[0],get_prompt("assets/5.txt")[1],4,0,33],
+                ["assets/6.mp4","assets/6_edit.png",get_prompt("assets/6.txt")[0],get_prompt("assets/6.txt")[1],4,0,33],
+                ["assets/7.mp4","assets/7_edit.png",get_prompt("assets/7.txt")[0],get_prompt("assets/7.txt")[1],4,0,33],
+                ["assets/8.mp4","assets/8_edit.png",get_prompt("assets/8.txt")[0],get_prompt("assets/8.txt")[1],4,0,33]
+            ],
+            inputs=[video_input, image_input, positive_prompt, negative_prompt, guidance_slider, seed, choice],
+            outputs=video_output,
+            fn=process_video,
+            cache_examples=False
+        )
+    
+if __name__ == "__main__":
+    demo.queue().launch()

assets/0.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9b4cbbc26fd2c76e5339fc9868b97cb3a9e6dfd394a99a6217f8f2070ad4af
+size 1234363

assets/0.txt

@@ -0,0 +1,4 @@
+Swap dog for fox. prompt: the fox with two ears. the motion is clear. The background is strictly aligned.
+The fox with three ears.
+
+

assets/1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b5bc1baee8501d33e58ea3a914b90f8737b3dc692cd21c313a790b7c90d2ed1
+size 583649

assets/1.txt

@@ -0,0 +1,2 @@
+Make it anime style. prompt: the flower is swaying in the wind. the video is captured by professional camera. The motion is stable. Best quality.
+bad quality.

assets/2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0bff917b61848877637c5e9e9c8f8a7ff246ecf02b91b6d5b496737f7ea9ddc
+size 3359278

assets/2.txt

@@ -0,0 +1,2 @@
+Add a hat on girl's head.
+Bad quality.

assets/3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea5c36673ade253be717ee8a7b06599f7108a03cd75ef6a954c32f1a05fde812
+size 1294568

assets/3.txt

@@ -0,0 +1,2 @@
+Make it oil painting style. The color is bright and beautiful. the video is captured by professional camera. The motion is stable. Best quality.
+bad quality.

assets/4.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a57e8216cdb4b2bc990a380d72844e2fe6594820ff31993330718c9315c0167a
+size 460228

assets/4.txt

@@ -0,0 +1,2 @@
+Remove the girl. the video is captured by professional camera. The motion is stable. Best quality.
+bad quality.

assets/5.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92dbba09c23b2dc594a84505dc7575bf2111b5af3a2636723533282d349e6e86
+size 513130

assets/5.txt

@@ -0,0 +1,2 @@
+Make it water color style. prompt: the flowers with green leaves. The color is bright and beautiful. the video is captured by professional camera. The motion is stable. Best quality.
+bad quality.

assets/6.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afb4aadab4339e508e66e1630225191f859dd34902c7f165284070241b853eff
+size 119857

assets/6.txt

@@ -0,0 +1,2 @@
+Make it anime style. prompt: the butterfly in on the flower. The color is bright and beautiful. the video is captured by professional camera. The motion is stable. Best quality.
+bad quality.

assets/7.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ade27a4aed77a41c809a82768c2006179718d2779b8391e5b74d8bf7a44aecf
+size 403370

assets/7.txt

@@ -0,0 +1,2 @@
+Make it anime style. prompt: white swan, autumn. The color is bright and beautiful. the video is captured by professional camera. The motion is stable. Best quality.
+bad quality.

assets/8.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfe7c713e94c8f85bb174326db147942d906973d49c4dd1a0412a81a7a7a1d93
+size 690881

assets/8.txt

@@ -0,0 +1,2 @@
+Swap bird for squirrel. prompt: the squirrel is standing on the column. Squirrel are looking around. two ears.
+Static Squirrel.

assets/9.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0073470e8507542945226a48e3af1e72656cdb5df331449374955be07fef4871
+size 539224

assets/9.txt

@@ -0,0 +1,3 @@
+Swap black dog for white pig. prompt: the pig is standing between two trees.
+black dog.
+

assets/outputvideo/output_0.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ba1cf2d89579e0d9e0d12cf4ebc06c0756725f048c6c2b269d52a99f6185477
+size 1282150

assets/outputvideo/output_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc843ae155c3c3198e147b882fe2460c18d05e0b387136925df828edd83587fc
+size 266780

assets/outputvideo/output_2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee9743f7d86ff59dc3b19cbfaf2677aad38340cc89c6856f353da1aa69a226fe
+size 912515

assets/outputvideo/output_3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07928cc806c93453a99e6de1ece37d949d71fbf33bbd20ea2c7116ec988f1d17
+size 1058274

assets/outputvideo/output_5.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3e32b887b2084b9ce04bc77c94da9c068018432d1b15de7f93435dff9e65ee3
+size 237088

assets/outputvideo/output_6.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fff03bc3efbb20752cae96fb8da35b5fee292596ad00d588a09fabb20c9aeef
+size 141423

assets/outputvideo/output_7.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0910239dd1f4b40a49602bdc27af42bd394c3d55724a3665d731471fe20989db
+size 294492

assets/outputvideo/output_8.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8abf76c7fffe1f58445a24bfe203073c094dbc44ec9934ccaff71c31e645a7f9
+size 358596