{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "GjF0Vw1G3CsS" }, "outputs": [], "source": [ "# @title Install requirements\n", "import os\n", "#!git clone https://github.com/huggingface/diffusers\n", "!git clone https://huggingface.co/waveydaveygravy/controlnetexpts\n", "!pip install controlnet-aux==0.0.7\n", "#!pip install -U openmim\n", "#!pip install cog\n", "#!pip install mediapipe\n", "#!mim install mmengine\n", "#!mim install \"mmcv>=2.0.1\"\n", "#!mim install \"mmdet>=3.1.0\"\n", "#!mim install \"mmpose>=1.1.0\"\n", "!pip install diffusers\n", "!pip install moviepy\n", "!pip install argparse\n", "!pip install transformers\n", "!pip install pillow\n", "!pip install accelerate\n", "!pip install xformers\n", "#!pip install https://github.com/karaokenerds/python-audio-separator/releases/download/v0.12.1/onnxruntime_gpu-1.17.0-cp310-cp310-linux_x86_64.whl\n", "#!git clone https://github.com/danielgatis/rembg.git\n", "#!git clone https://huggingface.co/spaces/LiheYoung/Depth-Anything\n", "\n", "# Create the directory /content/test\n", "import os\n", "os.makedirs(\"/content/test\", exist_ok=True)\n", "os.makedirs(\"/content/frames\", exist_ok=True)\n", "os.makedirs(\"/content/op\", exist_ok=True)\n", "os.makedirs(\"/content/dp\", exist_ok=True)\n", "os.makedirs(\"/content/checkpoints\")\n", "os.makedirs(\"/content/checkpoints/openpose\")\n", "os.makedirs(\"/content/checkpoints/depth\")\n", "os.makedirs(\"/content/checkpoints/realisticvision\")\n", "#INPUT_DIR = \"/content/frames\" # replace with your input directory\n", "#OUTPUT_DIR = \"/content/test\" # replace with your output directory" ] }, { "cell_type": "code", "source": [ "#@title upload\n", "from google.colab import files\n", "uploaded = files.upload()" ], "metadata": { "cellView": "form", "id": "uLaX0p173O-A" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title break video down into frames\n", "import cv2\n", "import os\n", "\n", "# Create the directory /content/test\n", "os.makedirs(\"/content/test\", exist_ok=True)\n", "os.makedirs(\"/content/frames\", exist_ok=True)\n", "\n", "#INPUT_DIR = \"/content/frames\" # replace with your input directory\n", "#OUTPUT_DIR = \"/content/testIP\" # replace with your output directory\n", "\n", "\n", "# Open the video file\n", "cap = cv2.VideoCapture('/content/trumpoverlay_1.mp4')\n", "\n", "i = 0\n", "while(cap.isOpened()):\n", " ret, frame = cap.read()\n", "\n", " if ret == False:\n", " break\n", "\n", " # Save each frame of the video\n", " cv2.imwrite('/content/testIP/frame_' + str(i) + '.jpg', frame)\n", "\n", " i += 1\n", "\n", "cap.release()\n", "cv2.destroyAllWindows()" ], "metadata": { "cellView": "form", "id": "LqdApe0Y3VtH" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title COMMENT OUT PROCESSORS YOU DONT WANT TO USE ALSO COMMENT OUT ONES WITH LARGE MODELS IF YOU WANT TO SAVE SPACE\n", "### based on https://github.com/patrickvonplaten/controlnet_aux\n", "### which is derived from https://github.com/lllyasviel/ControlNet/tree/main/annotator and connected to the 🤗 Hub.\n", "#All credit & copyright goes to https://github.com/lllyasviel .\n", "#some of the models are large comment them out to save space if not needed\n", "\n", "import torch\n", "import os\n", "import shutil\n", "import logging\n", "import math\n", "import numpy as np\n", "from PIL import Image\n", "from tqdm.auto import tqdm\n", "from PIL import Image\n", "from tqdm import tqdm\n", "import matplotlib.pyplot as plt\n", "import matplotlib.image as mpimg\n", "from transformers import AutoModel\n", "from diffusers import DiffusionPipeline\n", "#from depth_anything.dpt import DepthAnything\n", "#from depth_anything.util.transform import Resize, NormalizeImage, PrepareForNet\n", "from controlnet_aux import (CannyDetector, ContentShuffleDetector, HEDdetector,\n", " LeresDetector, LineartAnimeDetector,\n", " LineartDetector, MediapipeFaceDetector,\n", " MidasDetector, MLSDdetector, NormalBaeDetector,\n", " OpenposeDetector, PidiNetDetector, SamDetector,\n", " ZoeDetector, DWposeDetector)\n", "\n", "# Create the directory /content/test\n", "os.makedirs(\"/content/test\", exist_ok=True)\n", "os.makedirs(\"/content/frames\", exist_ok=True)\n", "\n", "INPUT_DIR = \"/content/frames\" # replace with your input directory\n", "OUTPUT_DIR = \"/content/test\" # replace with your output directory\n", "\n", "#controlnet_model_path = \"/content/checkpoints\"\n", "#controlnet = ControlNetModel.from_pretrained(controlnet_model_path, torch_dtype=torch.float16)\n", "\n", "# Check if CUDA is available and set the device accordingly\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", "\n", "def output(filename, img):\n", " img.save(os.path.join(OUTPUT_DIR, filename))\n", "\n", "def process_image(processor, img):\n", " return processor(img)\n", "\n", "def load_images():\n", " if os.path.exists(OUTPUT_DIR):\n", " shutil.rmtree(OUTPUT_DIR)\n", " os.mkdir(OUTPUT_DIR)\n", " images = []\n", " filenames = []\n", " for filename in os.listdir(INPUT_DIR):\n", " if filename.endswith(\".png\") or filename.endswith(\".jpg\"):\n", " img_path = os.path.join(INPUT_DIR, filename)\n", " img = Image.open(img_path).convert(\"RGB\").resize((512, 512))\n", " images.append(img)\n", " filenames.append(filename)\n", " return images, filenames\n", "\n", "def process_images(processor):\n", " images, filenames = load_images()\n", " for img, filename in tqdm(zip(images, filenames), total=len(images), desc=\"Processing images\"):\n", " output_img = process_image(processor, img)\n", " output(filename, output_img)\n", "\n", "# Initialize the detectors\n", "\n", "#canny = CannyDetector()\n", "#hed = HEDdetector.from_pretrained(\"lllyasviel/Annotators\")\n", "#shuffle = ContentShuffleDetector()\n", "leres = LeresDetector.from_pretrained(\"lllyasviel/Annotators\")\n", "#lineart_anime = LineartAnimeDetector.from_pretrained(\"lllyasviel/Annotators\")\n", "#lineart = LineartDetector.from_pretrained(\"lllyasviel/Annotators\")\n", "#mediapipe_face = MediapipeFaceDetector()\n", "#midas = MidasDetector.from_pretrained(\"lllyasviel/Annotators\").to('cuda')\n", "#mlsd = MLSDdetector.from_pretrained(\"lllyasviel/Annotators\")\n", "#normal_bae = NormalBaeDetector.from_pretrained(\"lllyasviel/Annotators\")\n", "openpose = OpenposeDetector.from_pretrained(\"lllyasviel/Annotators\")\n", "#pidi_net = PidiNetDetector.from_pretrained(\"lllyasviel/Annotators\")\n", "#sam = SamDetector.from_pretrained(\"ybelkada/segment-anything\", subfolder=\"checkpoints\")\n", "#zoe = ZoeDetector\n", "#depth_anything = AutoModel.from_pretrained(\"waveydaveygravy/depth-anything_pruned\")\n", "\n", "\n", "\n", "# Run the image processing\n", "# Uncomment the line for the detector you want to use\n", "#process_images(canny)\n", "#process_images(hed)\n", "process_images(openpose)\n", "#process_images(midas)\n", "process_images(leres)" ], "metadata": { "cellView": "form", "id": "ExeWABuE3vjo" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title zip frames\n", "!zip -r (name) path" ], "metadata": { "cellView": "form", "id": "hJzsEihl4BVx" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title download models\n", "#%cd /content/checkpoints\n", "#!wget https://huggingface.co/lllyasviel/control_v11p_sd15_openpose/resolve/main/diffusion_pytorch_model.fp16.bin?download=true\n", "#!wget https://huggingface.co/lllyasviel/control_v11f1p_sd15_depth/resolve/main/diffusion_pytorch_model.fp16.bin?download=true\n", "%cd /content/checkpoints/realisticvision\n", "!wget https://huggingface.co/SG161222/Realistic_Vision_V4.0_noVAE/resolve/main/Realistic_Vision_V4.0_fp16-no-ema-inpainting.ckpt?download=true\n", "%cd /content/" ], "metadata": { "cellView": "form", "id": "evRfuqs74HqW" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title config file for openpose\n", "{\n", " \"_class_name\": \"ControlNetModel\",\n", " \"_diffusers_version\": \"0.16.0.dev0\",\n", " \"_name_or_path\": \"/home/patrick/controlnet_v1_1/control_v11p_sd15_openpose\",\n", " \"act_fn\": \"silu\",\n", " \"attention_head_dim\": 8,\n", " \"block_out_channels\": [\n", " 320,\n", " 640,\n", " 1280,\n", " 1280\n", " ],\n", " \"class_embed_type\": null,\n", " \"conditioning_embedding_out_channels\": [\n", " 16,\n", " 32,\n", " 96,\n", " 256\n", " ],\n", " \"controlnet_conditioning_channel_order\": \"rgb\",\n", " \"cross_attention_dim\": 768,\n", " \"down_block_types\": [\n", " \"CrossAttnDownBlock2D\",\n", " \"CrossAttnDownBlock2D\",\n", " \"CrossAttnDownBlock2D\",\n", " \"DownBlock2D\"\n", " ],\n", " \"downsample_padding\": 1,\n", " \"flip_sin_to_cos\": true,\n", " \"freq_shift\": 0,\n", " \"in_channels\": 4,\n", " \"layers_per_block\": 2,\n", " \"mid_block_scale_factor\": 1,\n", " \"norm_eps\": 1e-05,\n", " \"norm_num_groups\": 32,\n", " \"num_class_embeds\": null,\n", " \"only_cross_attention\": false,\n", " \"projection_class_embeddings_input_dim\": null,\n", " \"resnet_time_scale_shift\": \"default\",\n", " \"upcast_attention\": false,\n", " \"use_linear_projection\": false\n", "}" ], "metadata": { "cellView": "form", "id": "ZpkAQLlO4RVf" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title WORKING MUTICONTROLNET SCRIPT DO NOT CHANGE! save as multi1.py and use as shown in next cell\n", "\n", "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler\n", "import torch\n", "import os\n", "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler\n", "from diffusers.utils import load_image\n", "import argparse\n", "from PIL import Image\n", "import cv2\n", "import numpy as np\n", "import torch\n", "import os\n", "import shutil\n", "from tqdm import tqdm\n", "import numpy as np\n", "\n", "device = (\"cuda\")\n", "\n", "# Initialize the argument parser\n", "parser = argparse.ArgumentParser(description='Choose a processor to run.')\n", "parser.add_argument('--op_image', type=str, help='path to pose image')\n", "parser.add_argument('--dp_image', type=str, help='path to depth image')\n", "parser.add_argument('--output_dir', type=str, default='/content/multi', help='The directory to save the output.')\n", "# Parse the arguments\n", "args = parser.parse_args()\n", "\n", "op_image = load_image(args.op_image)\n", "dp_image = load_image(args.dp_image)\n", "\n", "controlnet = [\n", " ControlNetModel.from_pretrained(\"/content/checkpoints/openpose\", torch_dtype=torch.float16).to('cuda'),\n", " ControlNetModel.from_pretrained(\"/content/checkpoints/depth\", torch_dtype=torch.float16).to('cuda'),\n", "]\n", "\n", "pipe = StableDiffusionControlNetPipeline.from_pretrained(\n", " \"SG161222/Realistic_Vision_V4.0_noVAE\", controlnet=controlnet, torch_dtype=torch.float16\n", ").to('cuda')\n", "\n", "pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)\n", "\n", "prompt = \"a boxer in a boxing ring, best quality\"\n", "negative_prompt = \"monochrome, lowres, bad anatomy, worst quality, low quality\"\n", "\n", "images = [op_image, dp_image]\n", "\n", "image = pipe(\n", " prompt,\n", " images,\n", " num_inference_steps=20,\n", " negative_prompt=negative_prompt,\n", " controlnet_conditioning_scale=[1.0, 0.8],\n", ").images[0]\n", "\n", "# Extract the filename and extension from args.op_image\n", "filename, extension = os.path.splitext(os.path.basename(args.op_image))\n", "\n", "# Construct the full output path with the specified output directory\n", "output_path = os.path.join(args.output_dir, filename + extension)\n", "\n", "print(type(image))\n", "# Save the image using PIL\n", "image.save(output_path) # Assuming image is from PIL\n", "print(\"saved in output directory!\")\n" ], "metadata": { "cellView": "form", "id": "55GnGKo74VLg" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title list of commands for multi1.py change to whichever paths needed\n", "#!python /content/multi1.py --op_image \"/content/op/test/frame_0.jpg\" --dp_image \"/content/dp/test/frame_0.jpg\"\n", "\n", "#!python /content/multi1.py --op_image \"/content/op/test/frame_1.jpg\" --dp_image \"/content/dp/test/frame_1.jpg\"\n", "\n", "#!python /content/multi1.py --op_image \"/content/op/test/frame_2.jpg\" --dp_image \"/content/dp/test/frame_2.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_3.jpg\" --dp_image \"/content/dp/test/frame_3.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_4.jpg\" --dp_image \"/content/dp/test/frame_4.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_5.jpg\" --dp_image \"/content/dp/test/frame_5.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_6.jpg\" --dp_image \"/content/dp/test/frame_6.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_7.jpg\" --dp_image \"/content/dp/test/frame_7.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_8.jpg\" --dp_image \"/content/dp/test/frame_8.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_9.jpg\" --dp_image \"/content/dp/test/frame_9.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_10.jpg\" --dp_image \"/content/dp/test/frame_10.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_11.jpg\" --dp_image \"/content/dp/test/frame_11.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_12.jpg\" --dp_image \"/content/dp/test/frame_12.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_13.jpg\" --dp_image \"/content/dp/test/frame_13.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_14.jpg\" --dp_image \"/content/dp/test/frame_14.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_15.jpg\" --dp_image \"/content/dp/test/frame_15.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_16.jpg\" --dp_image \"/content/dp/test/frame_16.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_17.jpg\" --dp_image \"/content/dp/test/frame_17.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_18.jpg\" --dp_image \"/content/dp/test/frame_18.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_19.jpg\" --dp_image \"/content/dp/test/frame_19.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_20.jpg\" --dp_image \"/content/dp/test/frame_20.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_21.jpg\" --dp_image \"/content/dp/test/frame_21.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_22.jpg\" --dp_image \"/content/dp/test/frame_22.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_23.jpg\" --dp_image \"/content/dp/test/frame_23.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_24.jpg\" --dp_image \"/content/dp/test/frame_24.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_25.jpg\" --dp_image \"/content/dp/test/frame_25.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_26.jpg\" --dp_image \"/content/dp/test/frame_26.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_27.jpg\" --dp_image \"/content/dp/test/frame_27.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_28.jpg\" --dp_image \"/content/dp/test/frame_28.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_29.jpg\" --dp_image \"/content/dp/test/frame_29.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_30.jpg\" --dp_image \"/content/dp/test/frame_30.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_31.jpg\" --dp_image \"/content/dp/test/frame_31.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_32.jpg\" --dp_image \"/content/dp/test/frame_32.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_33.jpg\" --dp_image \"/content/dp/test/frame_33.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_34.jpg\" --dp_image \"/content/dp/test/frame_34.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_35.jpg\" --dp_image \"/content/dp/test/frame_35.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_36.jpg\" --dp_image \"/content/dp/test/frame_36.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_37.jpg\" --dp_image \"/content/dp/test/frame_37.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_38.jpg\" --dp_image \"/content/dp/test/frame_38.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_39.jpg\" --dp_image \"/content/dp/test/frame_39.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_40.jpg\" --dp_image \"/content/dp/test/frame_40.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_41.jpg\" --dp_image \"/content/dp/test/frame_41.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_42.jpg\" --dp_image \"/content/dp/test/frame_42.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_43.jpg\" --dp_image \"/content/dp/test/frame_43.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_44.jpg\" --dp_image \"/content/dp/test/frame_44.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_45.jpg\" --dp_image \"/content/dp/test/frame_45.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_46.jpg\" --dp_image \"/content/dp/test/frame_46.jpg\"\n", "\n", "!python /content/multi1.py --op_image \"/content/op/test/frame_47.jpg\" --dp_image \"/content/dp/test/frame_47.jpg\"" ], "metadata": { "cellView": "form", "id": "IQ4wkiE04sN3" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title interpolate processed frames (best to keep fps same as input video)\n", "!ffmpeg -r 12 -i /content/output_processed/frame_%d.jpg -vf \"format=yuv420p\" -c:v libx264 -crf 1 boxermulti1.mp4\n", "\n" ], "metadata": { "cellView": "form", "id": "fjpga-Ra5-RN" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title display video\n", "from IPython.display import HTML\n", "from base64 import b64encode\n", "\n", "# Open the video file and read its contents\n", "mp4 = open('/content/boxermulti1.mp4', 'rb').read()\n", "\n", "# Encode the video data as a base64 string\n", "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n", "\n", "# Display the video using an HTML video element\n", "HTML(f\"\"\"\n", "\n", "\"\"\")" ], "metadata": { "cellView": "form", "id": "4PaTDmZw5_2P" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title clear variables and empty vram cache\n", "import gc\n", "gc.collect()\n", "torch.cuda.empty_cache()" ], "metadata": { "cellView": "form", "id": "30XaQxxq5cU5" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title zip up frames\n", "!zip -r multiframes.zip /content/multi" ], "metadata": { "cellView": "form", "id": "Qjocs_K45A1u" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FaF3RdKdaFa8", "cellView": "form" }, "outputs": [], "source": [ "#@title Login to HuggingFace 🤗\n", "\n", "#@markdown You need to accept the model license before downloading or using the Stable Diffusion weights. Please, visit the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5), read the license and tick the checkbox if you agree. You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need to use an access token for the code to work.\n", "# https://huggingface.co/settings/tokens\n", "!mkdir -p ~/.huggingface\n", "HUGGINGFACE_TOKEN = \"\" #@param {type:\"string\"}\n", "!echo -n \"{HUGGINGFACE_TOKEN}\" > ~/.huggingface/token" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "aEJZoFQ2YHIb", "cellView": "form" }, "outputs": [], "source": [ "#@title upload to huggingface\n", "from huggingface_hub import HfApi\n", "api = HfApi()\n", "api.upload_file(\n", " path_or_fileobj=\"\",\n", " path_in_repo=\"\",\n", " repo_id=\"\",\n", " repo_type=\"model\",\n", ")" ] }, { "cell_type": "code", "source": [ "#==============================" ], "metadata": { "id": "LFm40CCy5Upw" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "#@title OG multi code for reference do not change https://huggingface.co/blog/controlnet\n", "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler\n", "import torch\n", "\n", "controlnet = [\n", " ControlNetModel.from_pretrained(\"lllyasviel/sd-controlnet-openpose\", torch_dtype=torch.float16),\n", " ControlNetModel.from_pretrained(\"lllyasviel/sd-controlnet-canny\", torch_dtype=torch.float16),\n", "]\n", "\n", "pipe = StableDiffusionControlNetPipeline.from_pretrained(\n", " \"runwayml/stable-diffusion-v1-5\", controlnet=controlnet, torch_dtype=torch.float16\n", ")\n", "pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)\n", "\n", "pipe.enable_xformers_memory_efficient_attention()\n", "pipe.enable_model_cpu_offload()\n", "\n", "prompt = \"a giant standing in a fantasy landscape, best quality\"\n", "negative_prompt = \"monochrome, lowres, bad anatomy, worst quality, low quality\"\n", "\n", "generator = torch.Generator(device=\"cpu\").manual_seed(1)\n", "\n", "images = [openpose_image, canny_image]\n", "\n", "image = pipe(\n", " prompt,\n", " images,\n", " num_inference_steps=20,\n", " generator=generator,\n", " negative_prompt=negative_prompt,\n", " controlnet_conditioning_scale=[1.0, 0.8],\n", ").images[0]\n", "\n", "image.save(\"./multi_controlnet_output.png\")\n", "\n", "\n" ], "metadata": { "cellView": "form", "id": "a44MnBt-5N6d" }, "execution_count": null, "outputs": [] } ] }