from dataclasses import dataclass from pathlib import Path import pathlib from typing import Dict, Any, Optional, Tuple import asyncio import base64 import io import logging import random import traceback import os import numpy as np import torch from diffusers import LTXPipeline, LTXImageToVideoPipeline from PIL import Image from varnish import Varnish # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # Constraints MAX_WIDTH = 1280 MAX_HEIGHT = 720 MAX_FRAMES = 257 # this is only a temporary solution (famous last words) def apply_dirty_hack_to_patch_file_extensions_and_bypass_filter(directory): """ Recursively rename all '.wut' files to '.pth' in the given directory Args: directory (str): Path to the directory to process """ # Convert the directory path to absolute path directory = os.path.abspath(directory) # Walk through directory and its subdirectories for root, _, files in os.walk(directory): for filename in files: if filename.endswith('.wut'): # Get full path of the file old_path = os.path.join(root, filename) # Create new filename by replacing the extension new_filename = filename.replace('.wut', '.pth') new_path = os.path.join(root, new_filename) try: os.rename(old_path, new_path) print(f"Renamed: {old_path} -> {new_path}") except OSError as e: print(f"Error renaming {old_path}: {e}") def print_directory_structure(startpath): """Print the directory structure starting from the given path.""" for root, dirs, files in os.walk(startpath): level = root.replace(startpath, '').count(os.sep) indent = ' ' * 4 * level logger.info(f"{indent}{os.path.basename(root)}/") subindent = ' ' * 4 * (level + 1) for f in files: logger.info(f"{subindent}{f}") logger.info("💡 Applying a dirty hack (patch ""/repository"" to fix file extensions):") apply_dirty_hack_to_patch_file_extensions_and_bypass_filter("/repository") logger.info("💡 Printing directory structure of ""/repository"":") print_directory_structure("/repository") @dataclass class GenerationConfig: """Configuration for video generation""" width: int = 768 height: int = 512 fps: int = 24 duration_sec: float = 4.0 num_inference_steps: int = 30 guidance_scale: float = 7.5 upscale_factor: float = 2.0 enable_interpolation: bool = False seed: int = -1 # -1 means random seed @property def num_frames(self) -> int: """Calculate number of frames based on fps and duration""" return int(self.duration_sec * self.fps) + 1 def validate_and_adjust(self) -> 'GenerationConfig': """Validate and adjust parameters to meet constraints""" # Round dimensions to nearest multiple of 32 self.width = max(32, min(MAX_WIDTH, round(self.width / 32) * 32)) self.height = max(32, min(MAX_HEIGHT, round(self.height / 32) * 32)) # Adjust number of frames to be in format 8k + 1 k = (self.num_frames - 1) // 8 num_frames = min((k * 8) + 1, MAX_FRAMES) self.duration_sec = (num_frames - 1) / self.fps # Set random seed if not specified if self.seed == -1: self.seed = random.randint(0, 2**32 - 1) return self class EndpointHandler: """Handles video generation requests using LTX models and Varnish post-processing""" def __init__(self, model_path: str = ""): """Initialize the handler with LTX models and Varnish Args: model_path: Path to LTX model weights """ # Enable TF32 for potential speedup on Ampere GPUs #torch.backends.cuda.matmul.allow_tf32 = True # Initialize models with bfloat16 precision self.text_to_video = LTXPipeline.from_pretrained( model_path, torch_dtype=torch.bfloat16 ).to("cuda") self.image_to_video = LTXImageToVideoPipeline.from_pretrained( model_path, torch_dtype=torch.bfloat16 ).to("cuda") # Enable CPU offload for memory efficiency #self.text_to_video.enable_model_cpu_offload() #self.image_to_video.enable_model_cpu_offload() # Initialize Varnish for post-processing self.varnish = Varnish( device="cuda" if torch.cuda.is_available() else "cpu", output_format="mp4", output_codec="h264", output_quality=23, enable_mmaudio=False, #model_base_dir=os.path.abspath(os.path.join(os.getcwd(), "varnish")) model_base_dir="/repository/varnish", ) async def process_frames( self, frames: torch.Tensor, config: GenerationConfig ) -> tuple[str, dict]: """Post-process generated frames using Varnish Args: frames: Generated video frames tensor config: Generation configuration Returns: Tuple of (video data URI, metadata dictionary) """ try: logger.info(f"Original frames shape: {frames.shape}") # Remove batch dimension if present if len(frames.shape) == 5: frames = frames.squeeze(0) # Remove batch dimension logger.info(f"Processed frames shape: {frames.shape}") # Process video with Varnish result = await self.varnish( input_data=frames, input_fps=config.fps, output_fps=config.fps, upscale_factor=config.upscale_factor if config.upscale_factor > 1 else None, enable_interpolation=config.enable_interpolation ) # Convert to data URI video_uri = await result.write( output_type="data-uri", output_format="mp4", output_codec="h264", output_quality=23 ) # Collect metadata metadata = { "width": result.metadata.width, "height": result.metadata.height, "num_frames": result.metadata.frame_count, "fps": result.metadata.fps, "duration": result.metadata.duration, "num_inference_steps": config.num_inference_steps, "seed": config.seed, "upscale_factor": config.upscale_factor, "interpolation_enabled": config.enable_interpolation } return video_uri, metadata except Exception as e: logger.error(f"Error in process_frames: {str(e)}") raise RuntimeError(f"Failed to process frames: {str(e)}") def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]: """Process incoming requests for video generation Args: data: Request data containing: - inputs (str): Text prompt or image - width (optional): Video width - height (optional): Video height - fps (optional): Frames per second - duration_sec (optional): Video duration - num_inference_steps (optional): Inference steps - guidance_scale (optional): Guidance scale - upscale_factor (optional): Upscaling factor - enable_interpolation (optional): Enable frame interpolation - seed (optional): Random seed Returns: Dictionary containing: - video: Base64 encoded MP4 data URI - content-type: MIME type - metadata: Generation metadata """ # Extract prompt prompt = data.get("inputs") if not prompt: raise ValueError("No prompt provided in the 'inputs' field") # Create and validate configuration config = GenerationConfig( width=data.get("width", GenerationConfig.width), height=data.get("height", GenerationConfig.height), fps=data.get("fps", GenerationConfig.fps), duration_sec=data.get("duration_sec", GenerationConfig.duration_sec), num_inference_steps=data.get("num_inference_steps", GenerationConfig.num_inference_steps), guidance_scale=data.get("guidance_scale", GenerationConfig.guidance_scale), upscale_factor=data.get("upscale_factor", GenerationConfig.upscale_factor), enable_interpolation=data.get("enable_interpolation", GenerationConfig.enable_interpolation), seed=data.get("seed", GenerationConfig.seed) ).validate_and_adjust() try: with torch.no_grad(): # Set random seeds random.seed(config.seed) np.random.seed(config.seed) generator = torch.manual_seed(config.seed) # Prepare generation parameters generation_kwargs = { "prompt": prompt, "height": config.height, "width": config.width, "num_frames": config.num_frames, "guidance_scale": config.guidance_scale, "num_inference_steps": config.num_inference_steps, "output_type": "pt", "generator": generator } # Check if image-to-video generation is requested image_data = data.get("image") if image_data: # Process base64 image if image_data.startswith('data:'): image_data = image_data.split(',', 1)[1] image_bytes = base64.b64decode(image_data) image = Image.open(io.BytesIO(image_bytes)).convert("RGB") generation_kwargs["image"] = image frames = self.image_to_video(**generation_kwargs).frames else: frames = self.text_to_video(**generation_kwargs).frames # Log original shape logger.info(f"Original frames shape: {frames.shape}") # Remove batch dimension if present if len(frames.shape) == 5: frames = frames.squeeze(0) # Remove batch dimension logger.info(f"Processed frames shape: {frames.shape}") # Ensure we have the correct shape if len(frames.shape) != 4: raise ValueError(f"Expected tensor of shape [frames, channels, height, width], got shape {frames.shape}") # Post-process frames try: loop = asyncio.get_event_loop() except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) video_uri, metadata = loop.run_until_complete( self.process_frames(frames, config) ) return { "video": video_uri, "content-type": "video/mp4", "metadata": metadata } except Exception as e: message = f"Error generating video ({str(e)})\n{traceback.format_exc()}" print(message) raise RuntimeError(message)