import os import torch import numpy as np from diffusers import DiffusionPipeline, AutoencoderKL, UNet2DConditionModel, DDPMScheduler from transformers import CLIPVisionModelWithProjection, CLIPFeatureExtractor from diffusers.utils import load_image from pipeline_zero1to3_stable import Zero1to3StableDiffusionPipeline, CCProjection import math import imageio import gradio as gr from PIL import Image import cv2 # Define the background removal function def preprocess_image(input_im): ''' :param input_im (PIL Image). :return input_im (H, W, 3) array in [0, 1]. ''' input_im = input_im.convert('RGB') print("shape1 = ",input_im.size) input_im = input_im.resize([256, 256], Image.Resampling.LANCZOS) input_im = np.asarray(input_im, dtype=np.float32) / 255.0 # input_im[input_im[:, :, -1] <= 0.9] = [1., 1., 1.] return input_im return input_im # Load model and set paths model_id = "mirza152/zero123-face" cc_projection = CCProjection.from_pretrained(model_id, subfolder="cc_projection", use_safetensors=True) unet = UNet2DConditionModel.from_pretrained(model_id, subfolder="unet", use_safetensors=True) feature_extractor = CLIPFeatureExtractor.from_pretrained(model_id, subfolder="feature_extractor", use_safetensors=True) image_encoder = CLIPVisionModelWithProjection.from_pretrained(model_id, subfolder="image_encoder") scheduler = DDPMScheduler.from_pretrained(model_id, subfolder="scheduler", use_safetensors=True) vae = AutoencoderKL.from_pretrained(model_id, subfolder="vae", use_safetensors=True) # Instantiate pipeline pipe = Zero1to3StableDiffusionPipeline( unet=unet, cc_projection=cc_projection, vae=vae, scheduler=scheduler, feature_extractor=feature_extractor, image_encoder=image_encoder, safety_checker=None, ) pipe.enable_vae_tiling() pipe.enable_attention_slicing() # Define the function to process and generate GIFs def process_image(input_image): input_image = preprocess_image(input_image) H, W = input_image.shape[:2] input_image = Image.fromarray((input_image * 255.0).astype(np.uint8)) total_frames = 8 input_images = [input_image]*total_frames pitch_range, yaw_range = 0.20, 0.20 avg_polar, avg_azimuth = 1.52, 1.57 all_poses = [] # Generate poses for GIF frames for frame_idx in range(total_frames): theta_target = 3.14 / 2 + yaw_range * np.sin(2 * 3.14 * frame_idx / total_frames) polar = avg_polar - theta_target azimuth_cond = 3.14 / 2 - 0.05 + pitch_range * np.cos(2 * 3.14 * frame_idx / total_frames) azimuth = avg_azimuth - azimuth_cond query_pose = torch.tensor([(1.5708 - theta_target) - (1.5708 - avg_polar), math.sin(azimuth), math.cos(azimuth), 1.5708 - avg_azimuth]) all_poses.append(query_pose) query_poses = torch.stack(all_poses) images = pipe(input_imgs=input_images, prompt_imgs=input_images, poses=query_poses, height=H, width=W, guidance_scale=4, num_images_per_prompt=1, num_inference_steps=1).images # Save images to GIF gif_path = "output.gif" images[0].save(gif_path, save_all=True, append_images=images[1:], duration=100, loop=0) return gif_path # Create Gradio Interface iface = gr.Interface( fn=process_image, inputs=gr.Image(type="pil", label="Input Image"), outputs=gr.Image(type="filepath", label="Output GIF"), title="Image to GIF Pipeline", description="Upload an image to generate a GIF.", allow_flagging="never", ) iface.launch()