Upload app.py

app.py

@@ -0,0 +1,754 @@
+#!/usr/bin/env python
+"""
+This is the full application script for VideoPainter.
+It first checks for and (if necessary) installs missing dependencies.
+When installing the custom packages (diffusers and app),
+it uses the flag --no-build-isolation so that the installed torch is seen.
+If the custom diffusers package fails to provide the expected submodules,
+the script will force-install the official diffusers package.
+"""
+
+import os
+import sys
+import subprocess
+import warnings
+import time
+import importlib
+
+warnings.filterwarnings("ignore")
+
+# Set Gradio temp directory via environment variable
+GRADIO_TEMP_DIR = "./tmp_gradio"
+os.makedirs(GRADIO_TEMP_DIR, exist_ok=True)
+os.makedirs(f"{GRADIO_TEMP_DIR}/track", exist_ok=True)
+os.makedirs(f"{GRADIO_TEMP_DIR}/inpaint", exist_ok=True)
+os.environ["GRADIO_TEMP_DIR"] = GRADIO_TEMP_DIR
+
+def install_package(package_spec):
+    print(f"Installing {package_spec} ...")
+    try:
+        subprocess.check_call([sys.executable, "-m", "pip", "install", package_spec])
+        print(f"Successfully installed {package_spec}")
+        return True
+    except Exception as e:
+        print(f"Failed to install {package_spec}: {e}")
+        return False
+
+print("Checking for PyTorch ...")
+try:
+    import torch
+    print("PyTorch is already installed.")
+except ImportError:
+    print("PyTorch not found, installing...")
+    if not install_package("torch>=2.0.0 torchvision>=0.15.0"):
+        print("Failed to install PyTorch, which is required.")
+        sys.exit(1)
+
+# First, install wheel package which is needed for bdist_wheel command
+install_package("wheel")
+
+# Install ninja for faster builds
+install_package("ninja")
+
+# Check and install other critical dependencies
+critical_dependencies = [
+    ("hydra", "hydra-core>=1.3.2"),
+    ("omegaconf", "omegaconf>=2.3.0"),
+    ("decord", "decord>=0.6.0"),
+    ("diffusers", "diffusers>=0.24.0"),  # Will be replaced with custom one
+    ("transformers", "transformers>=4.35.0"),
+    ("gradio", "gradio>=4.0.0"),
+    ("numpy", "numpy>=1.24.0"),
+    ("cv2", "opencv-python>=4.8.0"),
+    ("PIL", "Pillow>=10.0.0"),
+    ("scipy", "scipy>=1.11.0"),
+    ("einops", "einops>=0.7.0"),
+    ("onnxruntime", "onnxruntime>=1.16.0"),
+    ("timm", "timm>=0.9.0"),
+    ("safetensors", "safetensors>=0.4.0"),
+    ("moviepy", "moviepy>=1.0.3"),
+    ("imageio", "imageio>=2.30.0"),
+    ("tqdm", "tqdm>=4.64.0"),
+    ("openai", "openai>=1.5.0"),
+    ("psutil", "psutil>=5.9.0")
+]
+
+for mod_name, pkg_spec in critical_dependencies:
+    try:
+        if mod_name == "PIL":
+            from PIL import Image
+        elif mod_name == "cv2":
+            import cv2
+        else:
+            __import__(mod_name)
+        print(f"{mod_name} is already installed.")
+    except ImportError:
+        print(f"{mod_name} not found, installing {pkg_spec} ...")
+        install_package(pkg_spec)
+
+print("Setting up environment...")
+# Clone the VideoPainter repository if not present
+if not os.path.exists("VideoPainter"):
+    print("Cloning VideoPainter repository...")
+    os.system("git clone https://github.com/TencentARC/VideoPainter.git")
+
+# Add necessary paths to sys.path
+sys.path.append(os.path.join(os.getcwd(), "VideoPainter"))
+sys.path.append(os.path.join(os.getcwd(), "VideoPainter/app"))
+sys.path.append(os.path.join(os.getcwd(), "app"))
+sys.path.append(".")
+
+# Ensure custom diffusers is importable
+if os.path.exists("VideoPainter/diffusers"):
+    print("Installing custom diffusers...")
+    # First, remove any existing diffusers installation
+    subprocess.call([sys.executable, "-m", "pip", "uninstall", "-y", "diffusers"])
+    
+    # Copy the files directly into the site-packages directory instead of using pip install -e
+    import site
+    site_packages = site.getsitepackages()[0]
+    diffusers_src = os.path.join(os.getcwd(), "VideoPainter/diffusers/src/diffusers")
+    diffusers_dst = os.path.join(site_packages, "diffusers")
+    
+    print(f"Copying diffusers from {diffusers_src} to {diffusers_dst}")
+    if not os.path.exists(diffusers_dst):
+        os.makedirs(diffusers_dst, exist_ok=True)
+    
+    # Copy diffusers files directly
+    os.system(f"cp -r {diffusers_src}/* {diffusers_dst}/")
+    
+    # Also add VideoPainter/diffusers/src to sys.path
+    sys.path.append(os.path.join(os.getcwd(), "VideoPainter/diffusers/src"))
+    
+    # Verify the custom model is available
+    try:
+        # Force reload diffusers to pick up the new files
+        if "diffusers" in sys.modules:
+            del sys.modules["diffusers"]
+        import diffusers
+        print(f"Diffusers version: {diffusers.__version__}")
+        print(f"Available modules in diffusers: {dir(diffusers)}")
+        
+        # Check if models directory exists in custom diffusers
+        models_dir = os.path.join(diffusers_dst, "models")
+        if os.path.exists(models_dir):
+            print(f"Models in diffusers: {os.listdir(models_dir)}")
+    except Exception as e:
+        print(f"Error verifying diffusers installation: {e}")
+
+# Copy the app directory if needed
+if not os.path.exists("app"):
+    os.makedirs("app", exist_ok=True)
+    print("Copying VideoPainter/app to local app directory...")
+    os.system("cp -r VideoPainter/app/* app/")
+
+# Don't try to install app package, just add to path
+print("Adding app directory to Python path...")
+app_path = os.path.join(os.getcwd(), "app")
+sys.path.insert(0, app_path)
+
+# Insert the VideoPainter path at the beginning of sys.path to ensure it takes precedence
+sys.path.insert(0, os.path.join(os.getcwd(), "VideoPainter"))
+
+print("Importing standard modules and dependencies ...")
+try:
+    import gradio as gr
+    import cv2
+    import numpy as np
+    import scipy
+    import torchvision
+    from PIL import Image
+    from huggingface_hub import snapshot_download
+    from decord import VideoReader
+except ImportError as e:
+    print(f"Error importing basic modules: {e}")
+    sys.exit(1)
+
+# Import specialized modules with better error handling
+try:
+    # Import our custom modules
+    from sam2.build_sam import build_sam2_video_predictor
+    
+    # Force reload of diffusers after direct copy
+    if "diffusers" in sys.modules:
+        del sys.modules["diffusers"]
+    
+    # Now import diffusers with explicit path to the files we need
+    sys.path.insert(0, os.path.join(os.getcwd(), "VideoPainter/app"))
+    
+    # Import utils after setting up correct paths
+    from utils import load_model, generate_frames
+    print("All modules imported successfully!")
+except ImportError as e:
+    print(f"Error importing specialized modules: {e}")
+    print("Paths:", sys.path)
+    
+    # Try to diagnose and fix the specific issue
+    if "CogvideoXBranchModel" in str(e):
+        print("Trying to fix missing CogvideoXBranchModel...")
+        
+        # Check if the model file exists in the repository
+        branch_model_file = "VideoPainter/diffusers/src/diffusers/models/cogvideox_branch.py"
+        if os.path.exists(branch_model_file):
+            print(f"Found branch model file at {branch_model_file}")
+            
+            # Manually import the module
+            import sys
+            sys.path.insert(0, os.path.join(os.getcwd(), "VideoPainter/diffusers/src"))
+            
+            # Add the import to __init__.py if not already there
+            init_file = os.path.join(site_packages, "diffusers/__init__.py")
+            with open(init_file, 'r') as f:
+                init_content = f.read()
+            
+            if "CogvideoXBranchModel" not in init_content:
+                print("Adding CogvideoXBranchModel to diffusers/__init__.py")
+                with open(init_file, 'a') as f:
+                    f.write("\nfrom .models.cogvideox_branch import CogvideoXBranchModel\n")
+            
+            # Force reload diffusers
+            if "diffusers" in sys.modules:
+                del sys.modules["diffusers"]
+            
+            # Try importing again
+            from utils import load_model, generate_frames
+            print("Fixed CogvideoXBranchModel import issue!")
+        else:
+            print(f"Could not find {branch_model_file}")
+            sys.exit(1)
+    else:
+        sys.exit(1)
+
+
+###############################
+# Begin Application Code (VideoPainter demo)
+###############################
+
+def download_models():
+    print("Downloading models from Hugging Face Hub...")
+    models = {
+        "CogVideoX-5b-I2V": "THUDM/CogVideoX-5b-I2V",
+        "VideoPainter": "TencentARC/VideoPainter"
+    }
+    model_paths = {}
+    os.makedirs("ckpt", exist_ok=True)
+    for name, repo_id in models.items():
+        print(f"Downloading {name} from {repo_id}...")
+        path = snapshot_download(repo_id=repo_id)
+        model_paths[name] = path
+        print(f"Downloaded {name} to {path}")
+    try:
+        flux_path = snapshot_download(repo_id="black-forest-labs/FLUX.1-Fill-dev")
+        model_paths["FLUX"] = flux_path
+    except Exception as e:
+        print(f"Failed to download FLUX model: {e}")
+        model_paths["FLUX"] = None
+    os.makedirs("ckpt/Grounded-SAM-2", exist_ok=True)
+    sam2_path = "ckpt/Grounded-SAM-2/sam2_hiera_large.pt"
+    if not os.path.exists(sam2_path):
+        print(f"Downloading SAM2 to {sam2_path}...")
+        os.system(f"wget -O {sam2_path} https://huggingface.co/spaces/sam2/sam2/resolve/main/sam2_hiera_large.pt")
+    model_paths["SAM2"] = sam2_path
+    return model_paths
+
+print("Initializing application environment...")
+if not os.path.exists("app"):
+    print("Setting up app folder from VideoPainter repository ...")
+    os.system("git clone https://github.com/TencentARC/VideoPainter.git")
+    os.makedirs("app", exist_ok=True)
+    os.system("cp -r VideoPainter/app/* app/")
+    os.system("pip install --no-build-isolation -e VideoPainter/diffusers")
+    os.chdir("app")
+    os.system("pip install --no-build-isolation -e .")
+    os.chdir("..")
+
+sys.path.append("app")
+sys.path.append(".")
+
+# Import project modules (again, to be safe)
+try:
+    from decord import VideoReader
+    from sam2.build_sam import build_sam2_video_predictor
+    from utils import load_model, generate_frames
+except ImportError as e:
+    print(f"Failed to import specialized modules: {e}")
+    sys.exit(1)
+
+# Set up OpenRouter / OpenAI (for caption generation)
+try:
+    from openai import OpenAI
+    vlm_model = OpenAI(
+        api_key=os.getenv("OPENROUTER_API_KEY", ""),
+        base_url="https://openrouter.ai/api/v1"
+    )
+    print("OpenRouter client initialized successfully")
+except Exception as e:
+    print(f"OpenRouter API not available: {e}")
+    class DummyModel:
+        def __getattr__(self, name):
+            return self
+        def __call__(self, *args, **kwargs):
+            return self
+        def create(self, *args, **kwargs):
+            class DummyResponse:
+                choices = [type('obj', (object,), {'message': type('obj', (object,), {'content': "OpenRouter API not available. Using default prompt."})})]
+            return DummyResponse()
+    vlm_model = DummyModel()
+
+###############################
+# Download models and initialize predictors
+###############################
+model_paths = download_models()
+base_model_path = model_paths["CogVideoX-5b-I2V"]
+videopainter_path = model_paths["VideoPainter"]
+inpainting_branch = os.path.join(videopainter_path, "checkpoints/branch")
+id_adapter = os.path.join(videopainter_path, "VideoPainterID/checkpoints")
+img_inpainting_model = model_paths.get("FLUX")
+sam2_checkpoint = "ckpt/Grounded-SAM-2/sam2_hiera_large.pt"
+model_cfg = "sam2_hiera_l.yaml"
+
+try:
+    predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
+    print("Build SAM2 predictor done!")
+    validation_pipeline, validation_pipeline_img = load_model(
+        model_path=base_model_path,
+        inpainting_branch=inpainting_branch,
+        id_adapter=id_adapter,
+        img_inpainting_model=img_inpainting_model
+    )
+    print("Load model done!")
+except Exception as e:
+    print(f"Error initializing models: {e}")
+    sys.exit(1)
+
+###############################
+# Helper functions & state definitions
+###############################
+EXAMPLES = [
+    [
+        "https://huggingface.co/spaces/TencentARC/VideoPainter/resolve/main/examples/ferry.mp4",
+        "A white ferry with red and blue accents, named 'COLONIA', cruises on a calm river...",
+        "White and red passenger ferry boat labeled 'COLONIA 6' with multiple windows, life buoys, and upper deck seating.",
+        "Positive",
+        "Inpaint",
+        "",
+        42,
+        6.0,
+        16,
+        [[[320, 240]], [1]],
+    ],
+    [
+        "https://huggingface.co/spaces/TencentARC/VideoPainter/resolve/main/examples/street.mp4",
+        "A bustling city street at night illuminated by festive lights, a red double-decker bus...",
+        "The rear of a black car with illuminated red tail lights and a visible license plate.",
+        "Positive",
+        "Inpaint",
+        "",
+        42,
+        6.0,
+        16,
+        [[[200, 400]], [1]],
+    ],
+]
+
+class StatusMessage:
+    INFO = "Info"
+    WARNING = "Warning"
+    ERROR = "Error"
+    SUCCESS = "Success"
+
+def create_status(message, status_type=StatusMessage.INFO):
+    timestamp = time.strftime("%H:%M:%S")
+    return [("", ""), (f"[{timestamp}]: {message}\n", status_type)]
+    
+def update_status(previous_status, new_message, status_type=StatusMessage.INFO):
+    timestamp = time.strftime("%H:%M:%S")
+    history = previous_status[-3:]
+    history.append((f"[{timestamp}]: {new_message}\n", status_type))
+    return [("", "")] + history
+
+def init_state(offload_video_to_cpu=False, offload_state_to_cpu=False):
+    inference_state = {}
+    inference_state["images"] = torch.zeros([1, 3, 100, 100])
+    inference_state["num_frames"] = 1
+    inference_state["offload_video_to_cpu"] = offload_video_to_cpu
+    inference_state["offload_state_to_cpu"] = offload_state_to_cpu
+    inference_state["video_height"] = 100
+    inference_state["video_width"] = 100
+    inference_state["device"] = torch.device("cuda")
+    inference_state["storage_device"] = torch.device("cpu") if offload_state_to_cpu else torch.device("cuda")
+    inference_state["point_inputs_per_obj"] = {}
+    inference_state["mask_inputs_per_obj"] = {}
+    inference_state["cached_features"] = {}
+    inference_state["constants"] = {}
+    inference_state["obj_id_to_idx"] = OrderedDict()
+    inference_state["obj_idx_to_id"] = OrderedDict()
+    inference_state["obj_ids"] = []
+    inference_state["output_dict"] = {"cond_frame_outputs": {}, "non_cond_frame_outputs": {}}
+    inference_state["output_dict_per_obj"] = {}
+    inference_state["temp_output_dict_per_obj"] = {}
+    inference_state["consolidated_frame_inds"] = {"cond_frame_outputs": set(), "non_cond_frame_outputs": set()}
+    inference_state["tracking_has_started"] = False
+    inference_state["frames_already_tracked"] = {}
+    inference_state = gr.State(inference_state)
+    return inference_state
+
+# (All additional helper functions such as get_frames_from_video, sam_refine, vos_tracking_video,
+# inpaint_video, generate_video_from_frames, process_example, reset_all, etc. are defined below.)
+# For brevity, they are included here in full as in your original code.
+
+def get_frames_from_video(video_input, video_state):
+    video_path = video_input
+    frames = []
+    user_name = time.time()
+    vr = VideoReader(video_path)
+    original_fps = vr.get_avg_fps()
+    if original_fps > 8:
+        total_frames = len(vr)
+        sample_interval = max(1, int(original_fps / 8))
+        frame_indices = list(range(0, total_frames, sample_interval))
+        frames = vr.get_batch(frame_indices).asnumpy()
+    else:
+        frames = vr.get_batch(list(range(len(vr)))).asnumpy()
+    frames = frames[:49]
+    resized_frames = [cv2.resize(frame, (720, 480)) for frame in frames]
+    frames = np.array(resized_frames)
+    init_start = time.time() 
+    inference_state = predictor.init_state(images=frames, offload_video_to_cpu=True, async_loading_frames=True)
+    init_time = time.time() - init_start
+    print(f"Inference state initialization took {init_time:.2f}s")
+    fps = 8
+    image_size = (frames[0].shape[0], frames[0].shape[1])
+    video_state = {
+        "user_name": user_name,
+        "video_name": os.path.split(video_path)[-1],
+        "origin_images": frames,
+        "painted_images": frames.copy(),
+        "masks": [np.zeros((frames[0].shape[0], frames[0].shape[1]), np.uint8)] * len(frames),
+        "logits": [None] * len(frames),
+        "select_frame_number": 0,
+        "fps": fps,
+        "ann_obj_id": 0
+    }
+    video_info = f"Video Name: {video_state['video_name']}, FPS: {video_state['fps']}, Total Frames: {len(frames)}, Image Size: {image_size}"
+    video_input_path = generate_video_from_frames(frames, output_path=f"{GRADIO_TEMP_DIR}/inpaint/original_{video_state['video_name']}", fps=fps)
+    return (gr.update(visible=True), gr.update(visible=True), inference_state, video_state, video_info,
+            video_state["origin_images"][0], gr.update(visible=False, maximum=len(frames), value=1, interactive=True),
+            gr.update(visible=False, maximum=len(frames), value=len(frames), interactive=True), gr.update(visible=True, interactive=True),
+            gr.update(visible=True, interactive=True), gr.update(visible=True, interactive=True), gr.update(visible=True),
+            gr.update(visible=True, interactive=False), create_status("Upload video complete. Ready to select targets.", StatusMessage.SUCCESS), video_input_path)
+
+def select_template(image_selection_slider, video_state, interactive_state, previous_status):
+    image_selection_slider -= 1
+    video_state["select_frame_number"] = image_selection_slider
+    return video_state["painted_images"][image_selection_slider], video_state, interactive_state, update_status(previous_status, f"Set tracking start at frame {image_selection_slider}.", StatusMessage.INFO)
+
+def get_end_number(track_pause_number_slider, video_state, interactive_state, previous_status):
+    interactive_state["track_end_number"] = track_pause_number_slider
+    return video_state["painted_images"][track_pause_number_slider], interactive_state, update_status(previous_status, f"Set tracking finish at frame {track_pause_number_slider}.", StatusMessage.INFO)
+
+def sam_refine(inference_state, video_state, point_prompt, click_state, interactive_state, evt, previous_status):
+    ann_obj_id = 0
+    ann_frame_idx = video_state["select_frame_number"]
+    if point_prompt == "Positive":
+        coordinate = f"[[{evt.index[0]},{evt.index[1]},1]]"
+        interactive_state["positive_click_times"] += 1
+    else:
+        coordinate = f"[[{evt.index[0]},{evt.index[1]},0]]"
+        interactive_state["negative_click_times"] += 1
+    print(f"sam_refine, point_prompt: {point_prompt}, click_state: {click_state}")
+    prompt = {"prompt_type":["click"], "input_point": click_state[0], "input_label": click_state[1], "multimask_output": "True"}
+    points = np.array(prompt["input_point"])
+    labels = np.array(prompt["input_label"])
+    height, width = video_state["origin_images"][0].shape[0:2]
+    for i in range(len(points)):
+        points[i, 0] = int(points[i, 0])
+        points[i, 1] = int(points[i, 1])
+    print(f"sam_refine points: {points}, labels: {labels}")
+    frame_idx, obj_ids, mask = predictor.add_new_points(inference_state=inference_state, frame_idx=ann_frame_idx, obj_id=ann_obj_id, points=points, labels=labels)
+    mask_ = mask.cpu().squeeze().detach().numpy()
+    mask_[mask_ <= 0] = 0
+    mask_[mask_ > 0] = 1
+    org_image = video_state["origin_images"][video_state["select_frame_number"]]
+    mask_ = cv2.resize(mask_, (width, height))
+    mask_ = mask_[:, :, None]
+    mask_[mask_ > 0.5] = 1
+    mask_[mask_ <= 0.5] = 0
+    color = 63 * np.ones((height, width, 3)) * np.array([[[np.random.randint(5), np.random.randint(5), np.random.randint(5)]]])
+    painted_image = np.uint8((1 - 0.5 * mask_) * org_image + 0.5 * mask_ * color)
+    video_state["masks"][video_state["select_frame_number"]] = mask_
+    video_state["painted_images"][video_state["select_frame_number"]] = painted_image
+    return painted_image, video_state, interactive_state, update_status(previous_status, "Segmentation updated. Add more points or continue tracking.", StatusMessage.SUCCESS)
+
+def clear_click(inference_state, video_state, click_state, previous_status):
+    predictor.reset_state(inference_state)
+    click_state = [[], []]
+    template_frame = video_state["origin_images"][video_state["select_frame_number"]]
+    return inference_state, template_frame, click_state, update_status(previous_status, "Click history cleared.", StatusMessage.INFO)
+
+def vos_tracking_video(inference_state, video_state, interactive_state, previous_status):
+    height, width = video_state["origin_images"][0].shape[0:2]
+    masks = []
+    for out_frame_idx, out_obj_ids, out_mask_logits in predictor.propagate_in_video(inference_state):
+        mask = np.zeros([480, 720, 1])
+        for i in range(len(out_mask_logits)):
+            out_mask = out_mask_logits[i].cpu().squeeze().detach().numpy()
+            out_mask[out_mask > 0] = 1
+            out_mask[out_mask <= 0] = 0
+            out_mask = out_mask[:, :, None]
+            mask += out_mask
+        mask = cv2.resize(mask, (width, height))
+        mask = mask[:, :, None]
+        mask[mask > 0.5] = 1
+        mask[mask < 1] = 0
+        mask = scipy.ndimage.binary_dilation(mask, iterations=6)
+        masks.append(mask)
+    masks = np.array(masks)
+    if interactive_state.get("track_end_number") is not None:
+        video_state["masks"][video_state["select_frame_number"]:interactive_state["track_end_number"]] = masks
+        org_images = video_state["origin_images"][video_state["select_frame_number"]:interactive_state["track_end_number"]]
+        color = 255 * np.ones((1, org_images.shape[-3], org_images.shape[-2], 3)) * np.array([[[[0, 1, 1]]]])
+        painted_images = np.uint8((1 - 0.5 * masks) * org_images + 0.5 * masks * color)
+        video_state["painted_images"][video_state["select_frame_number"]:interactive_state["track_end_number"]] = painted_images
+    else:
+        video_state["masks"] = masks
+        org_images = video_state["origin_images"]
+        color = 255 * np.ones((1, org_images.shape[-3], org_images.shape[-2], 3)) * np.array([[[[0, 1, 1]]]])
+        painted_images = np.uint8((1 - 0.5 * masks) * org_images + 0.5 * masks * color)
+        video_state["painted_images"] = painted_images
+    video_output = generate_video_from_frames(video_state["painted_images"], output_path=f"{GRADIO_TEMP_DIR}/track/{video_state['video_name']}", fps=video_state["fps"])
+    interactive_state["inference_times"] += 1
+    print(f"vos_tracking_video output: {video_output}")
+    return inference_state, video_output, video_state, interactive_state, update_status(previous_status, "Tracking complete.", StatusMessage.SUCCESS), gr.Button.update(interactive=True), gr.Button.update(interactive=True), gr.Button.update(interactive=True), gr.Button.update(interactive=True)
+
+def inpaint_video(video_state, video_caption, target_region_frame1_caption, interactive_state, previous_status, seed_param, cfg_scale, dilate_size):
+    seed = int(seed_param) if int(seed_param) >= 0 else np.random.randint(0, 2**32 - 1)
+    validation_images = video_state["origin_images"]
+    validation_masks = video_state["masks"]
+    validation_masks = [np.squeeze(mask) for mask in validation_masks]
+    validation_masks = [(mask > 0).astype(np.uint8) * 255 for mask in validation_masks]
+    validation_masks = [np.stack([m, m, m], axis=-1) for m in validation_masks]
+    validation_images = [Image.fromarray(np.uint8(img)).convert('RGB') for img in validation_images]
+    validation_masks = [Image.fromarray(np.uint8(mask)).convert('RGB') for mask in validation_masks]
+    validation_images = [img.resize((720, 480)) for img in validation_images]
+    validation_masks = [mask.resize((720, 480)) for mask in validation_masks]
+    print("Inpainting: video_caption=", video_caption)
+    images = generate_frames(
+        images=validation_images, 
+        masks=validation_masks, 
+        pipe=validation_pipeline, 
+        pipe_img_inpainting=validation_pipeline_img, 
+        prompt=str(video_caption), 
+        image_inpainting_prompt=str(target_region_frame1_caption),
+        seed=seed,
+        cfg_scale=float(cfg_scale),
+        dilate_size=int(dilate_size)
+    )
+    images = (images * 255).astype(np.uint8)
+    video_output = generate_video_from_frames(images, output_path=f"{GRADIO_TEMP_DIR}/inpaint/{video_state['video_name']}", fps=8)
+    print(f"Inpaint_video output: {video_output}")
+    return video_output, update_status(previous_status, "Inpainting complete.", StatusMessage.SUCCESS)
+
+def generate_video_from_frames(frames, output_path, fps=8):
+    frames_tensor = torch.from_numpy(np.asarray(frames)).to(torch.uint8)
+    if not os.path.exists(os.path.dirname(output_path)):
+        os.makedirs(os.path.dirname(output_path))
+    torchvision.io.write_video(output_path, frames_tensor, fps=fps, video_codec="libx264")
+    return output_path
+
+def process_example(video_input, video_caption, target_region_frame1_caption, prompt, click_state):
+    if video_input is None or video_input == "":
+        return (gr.update(value=""), gr.update(value=""), init_state(), 
+                {"user_name": "", "video_name": "", "origin_images": None, "painted_images": None, "masks": None, "inpaint_masks": None, "logits": None, "select_frame_number": 0, "fps": 8, "ann_obj_id": 0},
+                "", None,
+                gr.update(value=1, visible=False, interactive=False),
+                gr.update(value=1, visible=False, interactive=False),
+                gr.update(value="Positive", interactive=False),
+                gr.update(visible=True, interactive=False), 
+                gr.update(visible=True, interactive=False),
+                gr.update(value=None),
+                gr.update(visible=True, interactive=False),
+                create_status("Reset complete. Ready for new input.", StatusMessage.INFO),
+                gr.update(value=None))
+    video_state = gr.State({
+        "user_name": "",
+        "video_name": "",
+        "origin_images": None,
+        "painted_images": None,
+        "masks": None,
+        "inpaint_masks": None,
+        "logits": None,
+        "select_frame_number": 0,
+        "fps": 8,
+        "ann_obj_id": 0
+    })
+    results = get_frames_from_video(video_input, video_state)
+    if click_state[0] and click_state[1]:
+        print("Example detected, executing sam_refine")
+        (video_caption, target_region_frame1_caption, inference_state, video_state, video_info, template_frame, image_selection_slider, track_pause_number_slider, point_prompt, clear_button, tracking_button, video_output, inpaint_button, run_status, video_input) = results
+        class MockEvent:
+            def __init__(self, points, point_idx=0):
+                self.index = points[point_idx]
+        for i_click in range(len(click_state[0])):
+            evt = MockEvent(click_state[0], i_click)
+            prompt_type = "Positive" if click_state[1][i_click] == 1 else "Negative"
+            template_frame, video_state, interactive_state, run_status = sam_refine(inference_state, video_state, prompt_type, click_state, {"inference_times": 0, "negative_click_times": 0, "positive_click_times": 0, "mask_save": False, "multi_mask": {"mask_names": [], "masks": []}, "track_end_number": None}, evt, run_status)
+        return (video_caption, target_region_frame1_caption, inference_state, video_state, video_info, template_frame, image_selection_slider, track_pause_number_slider, point_prompt, clear_button, tracking_button, video_output, inpaint_button, run_status, video_input)
+    return results
+
+def reset_all():
+    return (gr.update(value=None), gr.update(value=""), gr.update(value=""), init_state(),
+            {"user_name": "", "video_name": "", "origin_images": None, "painted_images": None, "masks": None, "inpaint_masks": None, "logits": None, "select_frame_number": 0, "fps": 8, "ann_obj_id": 0},
+            {"inference_times": 0, "negative_click_times": 0, "positive_click_times": 0, "mask_save": False, "multi_mask": {"mask_names": [], "masks": []}, "track_end_number": None},
+            [[], []], None, gr.update(visible=True, interactive=True), "",
+            gr.update(value=1, visible=False, interactive=False), gr.update(value=1, visible=False, interactive=False),
+            gr.update(value="Positive", interactive=False), gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False), gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False), gr.Button.update(interactive=False),
+            gr.Button.update(interactive=False), gr.Number.update(value=42),
+            gr.Slider.update(value=6.0), gr.Slider.update(value=16),
+            create_status("Reset complete. Ready for new input.", StatusMessage.INFO))
+
+###############################
+# Build Gradio Interface
+###############################
+title = """<p><h1 align="center">VideoPainter</h1></p>"""
+with gr.Blocks() as iface:
+    gr.HTML("""
+<div style="text-align: center;">
+    <h1 style="color: #333;">🖌️ VideoPainter</h1>
+    <h3 style="color: #333;">Any-length Video Inpainting and Editing with Plug-and-Play Context Control</h3>
+    <p style="font-weight: bold;">
+        <a href="https://yxbian23.github.io/project/video-painter/">🌍 Project Page</a> | 
+        <a href="https://arxiv.org/abs/2503.05639">📃 ArXiv Preprint</a> | 
+        <a href="https://github.com/TencentARC/VideoPainter">🧑‍💻 Github Repository</a>
+    </p>
+</div>
+    """)
+    click_state = gr.State([[], []])
+    interactive_state = gr.State({
+        "inference_times": 0,
+        "negative_click_times": 0,
+        "positive_click_times": 0,
+        "mask_save": False,
+        "multi_mask": {"mask_names": [], "masks": []},
+        "track_end_number": None,
+    })
+    video_state = gr.State({
+        "user_name": "",
+        "video_name": "",
+        "origin_images": None,
+        "painted_images": None,
+        "masks": None,
+        "inpaint_masks": None,
+        "logits": None,
+        "select_frame_number": 0,
+        "fps": 8,
+        "ann_obj_id": 0
+    })
+    inference_state = init_state()
+    
+    with gr.Row():
+        with gr.Column():
+            with gr.Row():
+                video_input = gr.Video(label="Original Video", visible=True)
+            with gr.Row():
+                with gr.Column(scale=3):
+                    template_frame = gr.Image(type="pil", interactive=True, elem_id="template_frame", visible=True)
+                with gr.Column(scale=1):
+                    with gr.Accordion("Segmentation Point Prompt", open=True):
+                        point_prompt = gr.Radio(choices=["Positive", "Negative"], value="Positive", label="Point Type", interactive=False, visible=True)
+                        clear_button_click = gr.Button(value="Clear clicks", interactive=False, visible=True)
+                        gr.Markdown("✨ Positive: Include target region. <br> ✨ Negative: Exclude target region.")
+                        image_selection_slider = gr.Slider(minimum=1, maximum=100, step=1, value=1, label="Track start frame", visible=False, interactive=False)
+                        track_pause_number_slider = gr.Slider(minimum=1, maximum=100, step=1, value=1, label="Track end frame", visible=False, interactive=False)
+            video_output = gr.Video(label="Generated Video", visible=True)
+            with gr.Row():
+                tracking_video_predict_button = gr.Button(value="Tracking", interactive=False, visible=True)
+                inpaint_video_predict_button = gr.Button(value="Inpainting", interactive=False, visible=True)
+                reset_button = gr.Button(value="Reset All", interactive=True, visible=True)
+                
+        with gr.Column():
+            with gr.Accordion("Global Video Caption", open=True):
+                video_caption = gr.Textbox(label="Global Video Caption", placeholder="Input global video caption...", interactive=True, visible=True, max_lines=5, show_copy_button=True)
+                with gr.Row():
+                    gr.Markdown("✨ Enhance prompt using GPT-4o (optional).")
+                    enhance_button = gr.Button("✨ Enhance Prompt(Optional)", interactive=False)
+            with gr.Accordion("Target Object Caption", open=True):
+                target_region_frame1_caption = gr.Textbox(label="Target Object Caption", placeholder="Input target object caption...", interactive=True, visible=True, max_lines=5, show_copy_button=True)
+                with gr.Row():
+                    gr.Markdown("✨ Generate target caption (optional).")
+                    enhance_target_region_frame1_button = gr.Button("✨ Target Prompt Generation (Optional)", interactive=False)
+            with gr.Accordion("Editing Instruction", open=False):
+                gr.Markdown("✨ Modify captions based on your instruction using GPT-4o.")
+                with gr.Row():
+                    editing_instruction = gr.Textbox(label="Editing Instruction", placeholder="Input editing instruction...", interactive=True, visible=True, max_lines=5, show_copy_button=True)
+                    enhance_editing_instruction_button = gr.Button("✨ Modify Caption(For Editing)", interactive=False)
+            with gr.Accordion("Advanced Sampling Settings", open=False):
+                cfg_scale = gr.Slider(value=6.0, label="Classifier-Free Guidance Scale", minimum=1, maximum=10, step=0.1, interactive=True)
+                seed_param = gr.Number(label="Inference Seed (>=0)", interactive=True, value=42)
+                dilate_size = gr.Slider(value=16, label="Mask Dilate Size", minimum=0, maximum=32, step=1, interactive=True)
+                video_info = gr.Textbox(label="Video Info", visible=True, interactive=False)
+                model_type = gr.Textbox(label="Type", placeholder="Model type...", interactive=True, visible=False)
+            notes_accordion = gr.Accordion("Notes", open=False)
+            with notes_accordion:
+                gr.HTML("<p style='font-size: 1.1em;'>🧐 Reminder: VideoPainter may produce unexpected outputs. Adjust settings if needed.</p>")
+                run_status = gr.HighlightedText(value=[("", "")], visible=True, label="Operation Status", show_label=True,
+                                                color_map={"Success": "green", "Error": "red", "Warning": "orange", "Info": "blue"})
+    
+    with gr.Row():
+        examples = gr.Examples(label="Quick Examples", examples=EXAMPLES,
+                               inputs=[video_input, video_caption, target_region_frame1_caption, point_prompt, model_type, editing_instruction, seed_param, cfg_scale, dilate_size, click_state],
+                               examples_per_page=20, cache_examples=False)
+    
+    video_input.change(fn=process_example, inputs=[video_input, video_caption, target_region_frame1_caption, point_prompt, click_state],
+                       outputs=[video_caption, target_region_frame1_caption, inference_state, video_state, video_info,
+                                template_frame, image_selection_slider, track_pause_number_slider, point_prompt, clear_button_click,
+                                tracking_video_predict_button, video_output, inpaint_video_predict_button, run_status, video_input])
+    
+    image_selection_slider.release(fn=select_template, inputs=[image_selection_slider, video_state, interactive_state, run_status],
+                                   outputs=[template_frame, video_state, interactive_state, run_status])
+    
+    track_pause_number_slider.release(fn=get_end_number, inputs=[track_pause_number_slider, video_state, interactive_state, run_status],
+                                      outputs=[template_frame, interactive_state, run_status])
+    
+    template_frame.select(fn=sam_refine, inputs=[inference_state, video_state, point_prompt, click_state, interactive_state, run_status],
+                          outputs=[template_frame, video_state, interactive_state, run_status])
+    
+    tracking_video_predict_button.click(fn=vos_tracking_video, inputs=[inference_state, video_state, interactive_state, run_status],
+                                        outputs=[inference_state, video_output, video_state, interactive_state, run_status,
+                                                 inpaint_video_predict_button, enhance_button, enhance_target_region_frame1_button, enhance_editing_instruction_button, notes_accordion])
+    
+    inpaint_video_predict_button.click(fn=inpaint_video, inputs=[video_state, video_caption, target_region_frame1_caption, interactive_state, run_status, seed_param, cfg_scale, dilate_size],
+                                        outputs=[video_output, run_status], api_name=False, show_progress="full")
+    
+    def enhance_prompt_func(video_caption):
+        return video_caption  # Replace with your convert_prompt() if available
+    
+    def enhance_target_region_frame1_prompt_func(target_region_frame1_caption, video_state):
+        return target_region_frame1_caption  # Replace with your convert_prompt_target_region_frame1() if available
+    
+    def enhance_editing_instruction_prompt_func(editing_instruction, video_caption, target_region_frame1_caption, video_state):
+        return video_caption, target_region_frame1_caption  # Replace with your convert_prompt_editing_instruction() if available
+    
+    enhance_button.click(enhance_prompt_func, inputs=[video_caption], outputs=[video_caption])
+    enhance_target_region_frame1_button.click(enhance_target_region_frame1_prompt_func, inputs=[target_region_frame1_caption, video_state], outputs=[target_region_frame1_caption])
+    enhance_editing_instruction_button.click(enhance_editing_instruction_prompt_func, inputs=[editing_instruction, video_caption, target_region_frame1_caption, video_state],
+                                              outputs=[video_caption, target_region_frame1_caption])
+    
+    video_input.clear(fn=lambda: (gr.update(visible=True), gr.update(visible=True), init_state(),
+                                    {"user_name": "", "video_name": "", "origin_images": None, "painted_images": None, "masks": None, "inpaint_masks": None, "logits": None, "select_frame_number": 0, "fps": 8, "ann_obj_id": 0},
+                                    {"inference_times": 0, "negative_click_times": 0, "positive_click_times": 0, "mask_save": False, "multi_mask": {"mask_names": [], "masks": []}, "track_end_number": 0},
+                                    [[], []], None, None,
+                                    gr.update(visible=True), gr.update(visible=True), gr.update(visible=True), gr.update(visible=True),
+                                    gr.update(visible=True), gr.update(visible=True), gr.update(visible=True), gr.update(visible=True, value=[]),
+                                    gr.update(visible=True), gr.update(visible=True), gr.update(visible=True),
+                                    gr.Button.update(interactive=False), gr.Button.update(interactive=False), gr.Button.update(interactive=False)),
+                          outputs=[video_caption, target_region_frame1_caption, inference_state, video_state, interactive_state, click_state, video_output, template_frame, tracking_video_predict_button, image_selection_slider, track_pause_number_slider, point_prompt, clear_button_click, template_frame, tracking_video_predict_button, video_output, inpaint_video_predict_button, run_status], queue=False, show_progress=False)
+    
+    clear_button_click.click(fn=clear_click, inputs=[inference_state, video_state, click_state, run_status],
+                               outputs=[inference_state, template_frame, click_state, run_status])
+    
+    reset_button.click(fn=reset_all, inputs=[], outputs=[video_input, video_caption, target_region_frame1_caption, inference_state, video_state, interactive_state, click_state, video_output, template_frame, video_info, image_selection_slider, track_pause_number_slider, point_prompt, clear_button_click, tracking_video_predict_button, inpaint_video_predict_button, enhance_button, enhance_target_region_frame1_button, enhance_editing_instruction_button, seed_param, cfg_scale, dilate_size, run_status])
+    
+iface.queue().launch(share=False)