import imageio import numpy as np import spaces import torch from diffusers import UniPCMultistepScheduler, StableDiffusionControlNetPipeline, StableDiffusionInpaintPipeline, ConsistencyDecoderVAE from diffusers.utils import get_class_from_dynamic_module from tqdm import tqdm device = torch.device('cpu') dtype = torch.float32 if torch.cuda.is_available(): device = torch.device('cuda') dtype = torch.float16 NeuralTextureControlNetModel = get_class_from_dynamic_module( "dilightnet/model_helpers", "neuraltexture_controlnet.py", "NeuralTextureControlNetModel" ) controlnet = NeuralTextureControlNetModel.from_pretrained( "DiLightNet/DiLightNet", torch_dtype=dtype, ) vae = ConsistencyDecoderVAE.from_pretrained("openai/consistency-decoder", torch_dtype=dtype) pipe = StableDiffusionControlNetPipeline.from_pretrained( "stabilityai/stable-diffusion-2-1", vae=vae, controlnet=controlnet, torch_dtype=dtype ).to(device) pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) pipe.set_progress_bar_config(disable=True) inpainting_pipe = StableDiffusionInpaintPipeline.from_pretrained( "stabilityai/stable-diffusion-2-inpainting", torch_dtype=dtype ).to(device) inpainting_pipe.set_progress_bar_config(disable=True) @spaces.GPU def relighting_gen(masked_ref_img, mask, cond_path, frames, prompt, steps, seed, cfg, inpaint=False): mask = mask[..., :1] / 255. for i in tqdm(range(frames)): source_image = masked_ref_img[..., :3] / 255. hint_types = ['diffuse', 'ggx0.05', 'ggx0.13', 'ggx0.34'] images = [mask, source_image] for hint_type in hint_types: image_path = f'{cond_path}/hint{i:02d}_{hint_type}.png' image = imageio.v3.imread(image_path) / 255. if image.shape[-1] == 4: # Check if the image has an alpha channel image = image[..., :3] * image[..., 3:] # Premultiply RGB by Alpha images.append(image) hint = np.concatenate(images, axis=2).astype(np.float32)[None] hint = torch.from_numpy(hint).to(dtype).permute(0, 3, 1, 2).to(device) generator = torch.manual_seed(seed) image = pipe( prompt, num_inference_steps=steps, generator=generator, image=hint, num_images_per_prompt=1, guidance_scale=cfg, output_type='np', ).images[0] # [H, W, C] if inpaint: mask_image = (1. - mask)[None] image = inpainting_pipe(prompt=prompt, image=image[None], mask_image=mask_image, generator=generator, output_type='np', cfg=3.0, strength=1.0).images[0] imageio.imwrite(f'{cond_path}/relighting{i:02d}.png', (image * 255).clip(0, 255).astype(np.uint8))