Correct example with new pipeline

README.md

@@ -63,57 +63,73 @@ It is recommended to use the checkpoint with [Stable Diffusion v1-5](https://hug
 has been trained on it.
 Experimentally, the checkpoint can be used with other diffusion models such as dreamboothed stable diffusion.
 
-```
+
 1. Let's install `diffusers` and related packages:
+2. 
 ```
 $ pip install diffusers transformers accelerate
 ```
+
 2. Run code:
+3. 
 ```python
-import torch
-import os
+# !pip install transformers accelerate
+from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel
 from diffusers.utils import load_image
-from PIL import Image
 import numpy as np
-from diffusers import (
-    ControlNetModel,
-    StableDiffusionControlNetPipeline,
-    UniPCMultistepScheduler,
-)
-checkpoint = "lllyasviel/control_v11p_sd15_inpaint"
-original_image = load_image(
-    "https://huggingface.co/lllyasviel/control_v11p_sd15_inpaint/resolve/main/images/original.png"
+import torch
+
+init_image = load_image(
+    "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy.png"
 )
+init_image = init_image.resize((512, 512))
+
+generator = torch.Generator(device="cpu").manual_seed(1)
+
 mask_image = load_image(
-    "https://huggingface.co/lllyasviel/control_v11p_sd15_inpaint/resolve/main/images/mask.png"
+    "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy_mask.png"
 )
+mask_image = mask_image.resize((512, 512))
+
 
 def make_inpaint_condition(image, image_mask):
     image = np.array(image.convert("RGB")).astype(np.float32) / 255.0
-    image_mask = np.array(image_mask.convert("L"))
+    image_mask = np.array(image_mask.convert("L")).astype(np.float32) / 255.0
+
     assert image.shape[0:1] == image_mask.shape[0:1], "image and image_mask must have the same image size"
-    image[image_mask < 128] = -1.0 # set as masked pixel 
+    image[image_mask > 0.5] = -1.0  # set as masked pixel
     image = np.expand_dims(image, 0).transpose(0, 3, 1, 2)
     image = torch.from_numpy(image)
     return image
 
-control_image = make_inpaint_condition(original_image, mask_image)
-prompt = "best quality"
-negative_prompt="lowres, bad anatomy, bad hands, cropped, worst quality"
-controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
-pipe = StableDiffusionControlNetPipeline.from_pretrained(
+
+control_image = make_inpaint_condition(init_image, mask_image)
+
+controlnet = ControlNetModel.from_pretrained(
+    "lllyasviel/control_v11p_sd15_inpaint", torch_dtype=torch.float16
+)
+pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
     "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
 )
-pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+
+pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
 pipe.enable_model_cpu_offload()
-generator = torch.manual_seed(2)
-image = pipe(prompt, negative_prompt=negative_prompt, num_inference_steps=30, 
-             generator=generator, image=control_image).images[0]
-image.save('images/output.png')
+
+# generate image
+image = pipe(
+    "a handsome man with ray-ban sunglasses",
+    num_inference_steps=20,
+    generator=generator,
+    eta=1.0,
+    image=init_image,
+    mask_image=mask_image,
+    control_image=control_image,
+).images[0]
 ```
-![original](./images/original.png)
-![mask](./images/mask.png)
-![inpaint_output](./images/output.png)
+
+![original](https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy.png)
+![img](https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy_with_mask.png)
+![img](https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/boy_ray_ban.png)
 
 ## Other released checkpoints v1-1
 The authors released 14 different checkpoints, each trained with [Stable Diffusion v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5)