Add application file

.dockerignore

@@ -0,0 +1,6 @@
+env
+__pycache__
+*.pyc
+*.pyo
+.git
+.vscode

.gitignore

@@ -0,0 +1,5 @@
+env
+__pycache__
+*.pyc
+*.pyo
+.vscode

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11
+
+RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "app.cloudgate:app", "--host", "0.0.0.0", "--port", "7860"]

app/cgt2im/feature_extractor/preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "feature_extractor_type": "CLIPFeatureExtractor",
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 224
+  }
+}

app/cgt2im/model_index.json

@@ -0,0 +1,33 @@
+{
+  "_class_name": "LatentConsistencyModelPipeline",
+  "_diffusers_version": "0.22.0.dev0",
+  "feature_extractor": [
+    "transformers",
+    "CLIPImageProcessor"
+  ],
+  "requires_safety_checker": true,
+  "safety_checker": [
+    "stable_diffusion",
+    "StableDiffusionSafetyChecker"
+  ],
+  "scheduler": [
+    "diffusers",
+    "LCMScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "unet": [
+    "diffusers",
+    "UNet2DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ]
+}

app/cgt2im/safety_checker/config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "digiplay/DreamShaper_7",
+  "architectures": [
+    "StableDiffusionSafetyChecker"
+  ],
+  "initializer_factor": 1.0,
+  "logit_scale_init_value": 2.6592,
+  "model_type": "clip",
+  "projection_dim": 768,
+  "text_config": {
+    "bos_token_id": 0,
+    "dropout": 0.0,
+    "eos_token_id": 2,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "model_type": "clip_text_model",
+    "num_attention_heads": 12
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.0.dev0",
+  "vision_config": {
+    "dropout": 0.0,
+    "hidden_size": 1024,
+    "intermediate_size": 4096,
+    "model_type": "clip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "patch_size": 14
+  }
+}

app/cgt2im/safety_checker/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb351a5ded815c3ff744968ad9c6b218d071b9d313d04f35e813b84b4c0ffde8
+size 1215979664

app/cgt2im/scheduler/scheduler_config.json

@@ -0,0 +1,20 @@
+{
+  "_class_name": "LCMScheduler",
+  "_diffusers_version": "0.22.0.dev0",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "clip_sample_range": 1.0,
+  "dynamic_thresholding_ratio": 0.995,
+  "num_train_timesteps": 1000,
+  "original_inference_steps": 50,
+  "prediction_type": "epsilon",
+  "rescale_betas_zero_snr": false,
+  "sample_max_value": 1.0,
+  "set_alpha_to_one": true,
+  "steps_offset": 0,
+  "thresholding": false,
+  "timestep_spacing": "leading",
+  "trained_betas": null
+}

app/cgt2im/text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "digiplay/DreamShaper_7",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.0.dev0",
+  "vocab_size": 49408
+}

app/cgt2im/text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66262104f5099c84ad0e6ec156acae57b2292caebd4d7b8699327fa745145b76
+size 492265168

app/cgt2im/tokenizer/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

app/cgt2im/tokenizer/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "tokenizer_file": null,
+  "unk_token": "<|endoftext|>"
+}

app/cgt2im/unet/config.json

@@ -0,0 +1,66 @@
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.22.0.dev0",
+  "act_fn": "silu",
+  "addition_embed_type": null,
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": null,
+  "attention_head_dim": 8,
+  "attention_type": "default",
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "class_embeddings_concat": false,
+  "conv_in_kernel": 3,
+  "conv_out_kernel": 3,
+  "cross_attention_dim": 768,
+  "cross_attention_norm": null,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dropout": 0.0,
+  "dual_cross_attention": false,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_only_cross_attention": null,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 4,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_out_scale_factor": 1.0,
+  "resnet_skip_time_act": false,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 96,
+  "time_cond_proj_dim": 256,
+  "time_embedding_act_fn": null,
+  "time_embedding_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "transformer_layers_per_block": 1,
+  "up_block_types": [
+    "UpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D"
+  ],
+  "upcast_attention": null,
+  "use_linear_projection": false
+}

app/cgt2im/unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7be468469c513f8614a73423aa009581295fbd8c903703450632b2275c0145a9
+size 3438495328

app/cgt2im/vae/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.22.0.dev0",
+  "_name_or_path": "digiplay/DreamShaper_7",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 768,
+  "scaling_factor": 0.18215,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

app/cgt2im/vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bb865b3478d73053007b8a76f114010e26f1a1d2ea02dc0261464caa4289c2d
+size 334643268

app/cloudgate.py

@@ -0,0 +1,99 @@
+from typing import Annotated
+from fastapi import FastAPI, Path, Query, Response
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
+import torch
+from torch import autocast
+from diffusers import DiffusionPipeline
+from io import BytesIO
+import base64
+from os.path import dirname
+
+# class Prompt(BaseModel):
+#     prompt: str
+#     steps: Annotated[int, Path(title="No of steps", ge=4, le=10)] = 8
+#     guide: Annotated[float, Path(title="Guidance scale", ge=0.5, le=2)] = 0.8
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware, 
+    allow_credentials=True, 
+    allow_origins=["*"], 
+    allow_methods=["*"], 
+    allow_headers=["*"]
+)
+
+device = "cuda"
+pipe = DiffusionPipeline.from_pretrained(f'{dirname(__file__)}/cgt2im', 
+                                        #  use_auth_token=auth_token,
+                                        # use_safetensors=True
+                                         )
+pipe = pipe.to(device, dtype=torch.float16)
+
+# @app.get("/")
+# def generate(prompt: str): 
+#     with autocast(device): 
+#         image = pipe(
+#                     prompt=prompt,
+#                     num_inference_steps=8,
+#                     guidance_scale=8.0,
+#                     lcm_origin_steps=50,
+#                     output_type="pil",
+#                     ).images[0]
+
+#     # image.save("testimage.png")
+#     buffer = BytesIO()
+#     image.save(buffer, format="PNG")
+#     imgstr = base64.b64encode(buffer.getvalue())
+
+#     return Response(content=imgstr, media_type="image/png")
+
+@app.get("/t2i")
+def generate(prompt: str, 
+             steps: Annotated[int, Query(ge=4, le=10)] = 8,
+             guide: Annotated[float, Query(ge=0.5, le=2)] = 0.8,
+             ): 
+    with autocast(device): 
+        image = pipe(
+                    prompt=prompt,
+                    num_inference_steps=steps,
+                    guidance_scale=guide,
+                    lcm_origin_steps=50,
+                    output_type="pil",
+                    ).images[0]
+
+    # image.save("testimage.png")
+    buffer = BytesIO()
+    image.save(buffer, format="PNG")
+    imgstr = base64.b64encode(buffer.getvalue())
+
+    return Response(content=imgstr, media_type="image/png")
+
+
+@app.get("/", response_class=HTMLResponse)
+async def read_home():
+    with open("app/static/index.html", "r") as file:
+        content = file.read()
+    return HTMLResponse(content=content)
+
+
+# @app.post("/t2i")
+# def generate(prompt: Prompt): 
+#     with autocast(device): 
+#         image = pipe(
+#                     prompt=prompt.prompt,
+#                     num_inference_steps=prompt.steps,
+#                     guidance_scale=prompt.guide,
+#                     lcm_origin_steps=50,
+#                     output_type="pil",
+#                     ).images[0]
+
+#     # image.save("testimage.png")
+#     buffer = BytesIO()
+#     image.save(buffer, format="PNG")
+#     imgstr = base64.b64encode(buffer.getvalue())
+
+#     return Response(content=imgstr, media_type="image/png")

app/static/index.html

@@ -0,0 +1,79 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="utf-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1">
+	<title>CloudGate Text-2-Image Ver 1.0</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+</head>
+<body>
+    <main>
+        <section id="img-gen" class="container mx-auto bg-slate-50">
+            <h1 class="text-4xl font-bold text-center py-6 mb-6">Text to Image Generation</h1>
+            <form action="" class="img-gen-form p-6">
+                <div class="mb-4">
+                    <label for="img-gen-prompt" class="block mb-2 text-2xl">Prompt</label>
+                    <input 
+                        type="text" 
+                        id="img-gen-prompt" 
+                        class="w-full px-3 py-2 placeholder-gray-300 border border-gray-300 rounded-md focus:outline-none focus:ring focus:ring-indigo-100 focus:border-indigo-300" 
+                        value="A serene night scene featuring a house, majestic mountains, graceful trees, a tranquil lake, a glowing moon, and a shimmering sky adorned with countless stars." 
+                    />
+                </div>
+                <button class="img-gen-submit px-3 py-2 rounded-md bg-indigo-500 text-white">Submit</button>
+            </form>
+
+            <article>
+            <!-- <h2 class="text-2xl font-bold text-center py-6 mb-6 text-indigo-500 italic">Output</h2> -->
+            <div id="img-gen-output" class="py-6 px-6 rounded-md">
+                <p id="loading" class="text-center italic text-2xl text-green-500"></p>
+                <img src="" alt="" id="img-tgt" class="mx-auto">
+            </div>
+            </article>
+            
+        </section>
+    </main>
+    <script type="text/javascript">
+        const imgGenForm = document.querySelector('.img-gen-form');
+
+        const genImg = async (data) => {
+            const response = await fetch(`t2i?prompt=${encodeURIComponent(data.prompt)}`);
+            
+            if (!response.ok) {
+                console.error('Failed to fetch image:', response.statusText);
+                return null;
+            }
+
+            return response.text();
+        }
+
+        imgGenForm.addEventListener('submit', async (event) => {
+            event.preventDefault();
+            const imgElement = document.getElementById('img-tgt')
+            imgElement.src = "";
+            let loading_ = document.getElementById('loading');
+            loading_.textContent = "Generating an awesome image for you ...!";
+            const imgGenPrompt = document.getElementById('img-gen-prompt');
+            // const imgGenOutput = document.getElementById('img-gen-output');
+            
+        
+            try {
+                genImg({ "prompt": imgGenPrompt.value,
+                        // "steps": 8,
+                        // "guide": 1.5
+                        }).then((base64String) => {
+                            if (base64String) {
+                                // const imgElement = document.createElement('img');
+                                loading_.textContent = "";
+                                console.log(loading_)
+                                imgElement.src = `data:image/png;base64,${base64String}`;
+                                // imgGenOutput.appendChild(imgElement);
+                            }
+                });
+            } catch (err) {
+            console.error(err);
+            }
+        });
+    </script>
+</body>
+</html>

app/static/indexlocalgpt.html

@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Display FastAPI Image</title>
+</head>
+<body>
+  <div id="imageContainer"></div>
+
+  <script type="text/javascript">
+    async function query(data) {
+      const response = await fetch(
+        `http://127.0.0.1:7860/t2i/?prompt=${encodeURIComponent(data.prompt)}`,
+      );
+
+      if (!response.ok) {
+        console.error('Failed to fetch image:', response.statusText);
+        return null;
+      }
+
+      return response.text();
+    }
+
+    query({ "prompt": "a night painting with house, mountains, trees, lake, moon, stars",
+            "steps": 8,
+            "guide": 1.5
+            }).then((base64String) => {
+      if (base64String) {
+        const imageElement = document.createElement('img');
+        imageElement.src = `data:image/png;base64,${base64String}`;
+
+        const imageContainer = document.getElementById('imageContainer');
+        imageContainer.appendChild(imageElement);
+      }
+    });
+  </script>
+</body>
+</html>

requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+uvicorn
+diffusers
+transformers
+accelerate