add demo

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "MiDaS"]
+	path = live2diff/MiDaS
+	url = [email protected]:lewiji/MiDaS.git

Dockerfile

@@ -0,0 +1,62 @@
+FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV PYTHONUNBUFFERED=1
+ENV NODE_MAJOR=20
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    build-essential \
+    python3.9 \
+    python3-pip \
+    python3-dev \
+    git \
+    ffmpeg \
+    google-perftools \
+    ca-certificates curl gnupg \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /code
+
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list > /dev/null
+RUN apt-get update && apt-get install nodejs -y
+
+COPY ./requirements.txt /code/requirements.txt
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+    PYTHONUNBUFFERED=1 \
+    SYSTEM=spaces
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
+
+# install dependencies
+RUN git submodule update --init --recursive
+RUN pip install -e ."[tensorrt]"
+
+# download models`
+RUN mkdir models
+RUN huggingface-cli download Leoxing/Live2Diff live2diff.ckpt --local-dir ./models
+RUN huggingface-cli download runwayml/stable-diffusion-v1-5 \
+    --local-dir ./models/Model/stable-diffusion-v1-5
+RUN bash scripts/download.sh felted
+RUN wget https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt -P models
+
+RUN cd demo
+RUN pip install -r requirements.txt
+CMD ["bash ./start.sh"]
+

demo/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

demo/.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+venv/
+public/
+*.pem
+!lib/
+!static/

demo/README.md

@@ -0,0 +1,70 @@
+# Video2Video Example
+
+<div align="center">
+    <table align="center">
+        <tbody>
+        <tr align="center">
+            <td>
+                <p> Human Face (Web Camera Input) </p>
+            </td>
+            <td>
+                <p> Anime Character (Screen Video Input) </p>
+            </td>
+        </tr>
+        <tr align="center">
+            <td>
+                <video controls autoplay src="https://github-production-user-asset-6210df.s3.amazonaws.com/28132635/346031564-a56bea4f-9aed-497c-9cb9-868db72599fb.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240705%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240705T064755Z&X-Amz-Expires=300&X-Amz-Signature=3d0890fa6af8b86d10c69de9c31b8f97ff1d35cdbcdd4b1a9e465e49dd8d8194&X-Amz-SignedHeaders=host&actor_id=28132635&key_id=0&repo_id=819189683" width="100%">
+            </td>
+            <td>
+                <video controls autoplay src="https://github-production-user-asset-6210df.s3.amazonaws.com/28132635/346055688-00f873dc-7348-458d-a814-7a7e8d158db5.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240705%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240705T081427Z&X-Amz-Expires=300&X-Amz-Signature=609199998d9f10a814d7a65e6dbff980a84bba77f25f43f99a1607bd0cee2f01&X-Amz-SignedHeaders=host&actor_id=28132635&key_id=0&repo_id=819189683" width="80%">
+            </td>
+        </tr>
+        </tbody>
+    </table>
+
+</div>
+
+This example, based on this [MJPEG server](https://github.com/radames/Real-Time-Latent-Consistency-Model/), runs image-to-image with a live webcam feed or screen capture on a web browser.
+
+## Usage
+
+### 1. Prepare Dependencies
+
+You need Node.js 18+ and Python 3.10 to run this example. Please make sure you've installed all dependencies according to the [installation instructions](../README.md#installation).
+
+```bash
+cd frontend
+npm i
+npm run build
+cd ..
+pip install -r requirements.txt
+```
+
+If you face some difficulties in install `npm`, you can try to install it via `conda`:
+
+```bash
+conda install -c conda-forge nodejs
+```
+
+### 2. Run Demo
+
+If you run the demo with default [setting](./demo_cfg.yaml), you should download the model for style `felted`.
+
+```bash
+bash ../scripts/download_model.sh felted
+```
+
+Then, you can run the demo with the following command, and open `http://127.0.0.1:7860` in your browser:
+
+```bash
+# with TensorRT acceleration, please pay patience for the first time, may take more than 20 minutes
+python main.py --port 7860 --host 127.0.0.1 --acceleration tensorrt
+# if you don't have TensorRT, you can run it with `none` acceleration
+python main.py --port 7860 --host 127.0.0.1 --acceleration none
+```
+
+If you want to run this demo on a remote server, you can set host to `0.0.0.0`, e.g.
+
+```bash
+python main.py --port 7860 --host 0.0.0.0 --acceleration tensorrt
+```

demo/app.py

@@ -0,0 +1,149 @@
+import logging
+import mimetypes
+import os
+import time
+import uuid
+from types import SimpleNamespace
+
+import markdown2
+import torch
+from config import Args, config
+from connection_manager import ConnectionManager, ServerFullException
+from fastapi import FastAPI, HTTPException, Request, WebSocket
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+from util import bytes_to_pil, pil_to_frame
+from vid2vid import Pipeline
+
+
+# fix mime error on windows
+mimetypes.add_type("application/javascript", ".js")
+
+THROTTLE = 1.0 / 120
+# logging.basicConfig(level=logging.DEBUG)
+
+
+class App:
+    def __init__(self, config: Args):
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        torch_dtype = torch.float16
+        pipeline = Pipeline(config, device, torch_dtype)
+        self.args = config
+        self.pipeline = pipeline
+        self.app = FastAPI()
+        self.conn_manager = ConnectionManager()
+        self.init_app()
+
+    def init_app(self):
+        self.app.add_middleware(
+            CORSMiddleware,
+            allow_origins=["*"],
+            allow_credentials=True,
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )
+
+        @self.app.websocket("/api/ws/{user_id}")
+        async def websocket_endpoint(user_id: uuid.UUID, websocket: WebSocket):
+            try:
+                await self.conn_manager.connect(user_id, websocket, self.args.max_queue_size)
+                await handle_websocket_data(user_id)
+            except ServerFullException as e:
+                logging.error(f"Server Full: {e}")
+            finally:
+                await self.conn_manager.disconnect(user_id)
+                logging.info(f"User disconnected: {user_id}")
+
+        async def handle_websocket_data(user_id: uuid.UUID):
+            if not self.conn_manager.check_user(user_id):
+                return HTTPException(status_code=404, detail="User not found")
+            last_time = time.time()
+            try:
+                while True:
+                    if self.args.timeout > 0 and time.time() - last_time > self.args.timeout:
+                        await self.conn_manager.send_json(
+                            user_id,
+                            {
+                                "status": "timeout",
+                                "message": "Your session has ended",
+                            },
+                        )
+                        await self.conn_manager.disconnect(user_id)
+                        return
+                    data = await self.conn_manager.receive_json(user_id)
+                    if data["status"] == "next_frame":
+                        info = self.pipeline.Info()
+                        params = await self.conn_manager.receive_json(user_id)
+                        params = self.pipeline.InputParams(**params)
+                        params = SimpleNamespace(**params.model_dump())
+                        if info.input_mode == "image":
+                            image_data = await self.conn_manager.receive_bytes(user_id)
+                            if len(image_data) == 0:
+                                await self.conn_manager.send_json(user_id, {"status": "send_frame"})
+                                continue
+                            params.image = bytes_to_pil(image_data)
+                        await self.conn_manager.update_data(user_id, params)
+
+            except Exception as e:
+                logging.error(f"Websocket Error: {e}, {user_id} ")
+                await self.conn_manager.disconnect(user_id)
+
+        @self.app.get("/api/queue")
+        async def get_queue_size():
+            queue_size = self.conn_manager.get_user_count()
+            return JSONResponse({"queue_size": queue_size})
+
+        @self.app.get("/api/stream/{user_id}")
+        async def stream(user_id: uuid.UUID, request: Request):
+            try:
+
+                async def generate():
+                    while True:
+                        last_time = time.time()
+                        await self.conn_manager.send_json(user_id, {"status": "send_frame"})
+                        params = await self.conn_manager.get_latest_data(user_id)
+                        if params is None:
+                            continue
+                        image = self.pipeline.predict(params)
+                        if image is None:
+                            continue
+                        frame = pil_to_frame(image)
+                        yield frame
+                        if self.args.debug:
+                            print(f"Time taken: {time.time() - last_time}")
+
+                return StreamingResponse(
+                    generate(),
+                    media_type="multipart/x-mixed-replace;boundary=frame",
+                    headers={"Cache-Control": "no-cache"},
+                )
+            except Exception as e:
+                logging.error(f"Streaming Error: {e}, {user_id} ")
+                return HTTPException(status_code=404, detail="User not found")
+
+        # route to setup frontend
+        @self.app.get("/api/settings")
+        async def settings():
+            info_schema = self.pipeline.Info.model_json_schema()
+            info = self.pipeline.Info()
+            if info.page_content:
+                page_content = markdown2.markdown(info.page_content)
+
+            input_params = self.pipeline.InputParams.model_json_schema()
+            return JSONResponse(
+                {
+                    "info": info_schema,
+                    "input_params": input_params,
+                    "max_queue_size": self.args.max_queue_size,
+                    "page_content": page_content if info.page_content else "",
+                }
+            )
+
+        if not os.path.exists("public"):
+            os.makedirs("public")
+
+        self.app.mount("/", StaticFiles(directory="./frontend/public", html=True), name="public")
+
+
+app = App(config).app

demo/config.py

@@ -0,0 +1,121 @@
+import argparse
+import os
+from typing import List, NamedTuple
+
+
+class Args(NamedTuple):
+    host: str
+    port: int
+    reload: bool
+    max_queue_size: int
+    timeout: float
+    safety_checker: bool
+    taesd: bool
+    ssl_certfile: str
+    ssl_keyfile: str
+    debug: bool
+    acceleration: str
+    engine_dir: str
+    config: str
+    seed: int
+    num_inference_steps: int
+    strength: float
+    t_index_list: List[int]
+    prompt: str
+
+    def pretty_print(self):
+        print("\n")
+        for field, value in self._asdict().items():
+            print(f"{field}: {value}")
+        print("\n")
+
+
+MAX_QUEUE_SIZE = int(os.environ.get("MAX_QUEUE_SIZE", 0))
+TIMEOUT = float(os.environ.get("TIMEOUT", 0))
+SAFETY_CHECKER = os.environ.get("SAFETY_CHECKER", None) == "True"
+USE_TAESD = os.environ.get("USE_TAESD", "True") == "True"
+ENGINE_DIR = os.environ.get("ENGINE_DIR", "engines")
+ACCELERATION = os.environ.get("ACCELERATION", "tensorrt")
+
+default_host = os.getenv("HOST", "0.0.0.0")
+default_port = int(os.getenv("PORT", "7860"))
+default_mode = os.getenv("MODE", "default")
+
+parser = argparse.ArgumentParser(description="Run the app")
+parser.add_argument("--host", type=str, default=default_host, help="Host address")
+parser.add_argument("--port", type=int, default=default_port, help="Port number")
+parser.add_argument("--reload", action="store_true", help="Reload code on change")
+parser.add_argument(
+    "--max-queue-size",
+    dest="max_queue_size",
+    type=int,
+    default=MAX_QUEUE_SIZE,
+    help="Max Queue Size",
+)
+parser.add_argument("--timeout", type=float, default=TIMEOUT, help="Timeout")
+parser.add_argument(
+    "--safety-checker",
+    dest="safety_checker",
+    action="store_true",
+    default=SAFETY_CHECKER,
+    help="Safety Checker",
+)
+parser.add_argument(
+    "--taesd",
+    dest="taesd",
+    action="store_true",
+    help="Use Tiny Autoencoder",
+)
+parser.add_argument(
+    "--no-taesd",
+    dest="taesd",
+    action="store_false",
+    help="Use Tiny Autoencoder",
+)
+parser.add_argument(
+    "--ssl-certfile",
+    dest="ssl_certfile",
+    type=str,
+    default=None,
+    help="SSL certfile",
+)
+parser.add_argument(
+    "--ssl-keyfile",
+    dest="ssl_keyfile",
+    type=str,
+    default=None,
+    help="SSL keyfile",
+)
+parser.add_argument(
+    "--debug",
+    action="store_true",
+    default=False,
+    help="Debug",
+)
+parser.add_argument(
+    "--acceleration",
+    type=str,
+    default=ACCELERATION,
+    choices=["none", "xformers", "tensorrt"],
+    help="Acceleration",
+)
+parser.add_argument(
+    "--engine-dir",
+    dest="engine_dir",
+    type=str,
+    default=ENGINE_DIR,
+    help="Engine Dir",
+)
+parser.add_argument(
+    "--config",
+    default="./demo_cfg.yaml",
+)
+parser.add_argument("--num-inference-steps", type=int, default=None)
+parser.add_argument("--strength", type=float, default=None)
+parser.add_argument("--t-index-list", type=list)
+parser.add_argument("--seed", default=42)
+parser.add_argument("--prompt", type=str)
+
+parser.set_defaults(taesd=USE_TAESD)
+config = Args(**vars(parser.parse_args()))
+config.pretty_print()

demo/connection_manager.py

@@ -0,0 +1,111 @@
+import asyncio
+import logging
+from types import SimpleNamespace
+from typing import Dict, Union
+from uuid import UUID
+
+from fastapi import WebSocket
+from starlette.websockets import WebSocketState
+
+
+Connections = Dict[UUID, Dict[str, Union[WebSocket, asyncio.Queue]]]
+
+
+class ServerFullException(Exception):
+    """Exception raised when the server is full."""
+
+    pass
+
+
+class ConnectionManager:
+    def __init__(self):
+        self.active_connections: Connections = {}
+
+    async def connect(self, user_id: UUID, websocket: WebSocket, max_queue_size: int = 0):
+        await websocket.accept()
+        user_count = self.get_user_count()
+        print(f"User count: {user_count}")
+        if max_queue_size > 0 and user_count >= max_queue_size:
+            print("Server is full")
+            await websocket.send_json({"status": "error", "message": "Server is full"})
+            await websocket.close()
+            raise ServerFullException("Server is full")
+        print(f"New user connected: {user_id}")
+        self.active_connections[user_id] = {
+            "websocket": websocket,
+            "queue": asyncio.Queue(),
+        }
+        await websocket.send_json(
+            {"status": "connected", "message": "Connected"},
+        )
+        await websocket.send_json({"status": "wait"})
+        await websocket.send_json({"status": "send_frame"})
+
+    def check_user(self, user_id: UUID) -> bool:
+        return user_id in self.active_connections
+
+    async def update_data(self, user_id: UUID, new_data: SimpleNamespace):
+        user_session = self.active_connections.get(user_id)
+        if user_session:
+            queue = user_session["queue"]
+            await queue.put(new_data)
+
+    async def get_latest_data(self, user_id: UUID) -> SimpleNamespace:
+        user_session = self.active_connections.get(user_id)
+        if user_session:
+            queue = user_session["queue"]
+            try:
+                return await queue.get()
+            except asyncio.QueueEmpty:
+                return None
+
+    def delete_user(self, user_id: UUID):
+        user_session = self.active_connections.pop(user_id, None)
+        if user_session:
+            queue = user_session["queue"]
+            while not queue.empty():
+                try:
+                    queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    continue
+
+    def get_user_count(self) -> int:
+        return len(self.active_connections)
+
+    def get_websocket(self, user_id: UUID) -> WebSocket:
+        user_session = self.active_connections.get(user_id)
+        if user_session:
+            websocket = user_session["websocket"]
+            if websocket.client_state == WebSocketState.CONNECTED:
+                return user_session["websocket"]
+        return None
+
+    async def disconnect(self, user_id: UUID):
+        websocket = self.get_websocket(user_id)
+        if websocket:
+            await websocket.close()
+        self.delete_user(user_id)
+
+    async def send_json(self, user_id: UUID, data: Dict):
+        try:
+            websocket = self.get_websocket(user_id)
+            if websocket:
+                await websocket.send_json(data)
+        except Exception as e:
+            logging.error(f"Error: Send json: {e}")
+
+    async def receive_json(self, user_id: UUID) -> Dict:
+        try:
+            websocket = self.get_websocket(user_id)
+            if websocket:
+                return await websocket.receive_json()
+        except Exception as e:
+            logging.error(f"Error: Receive json: {e}")
+
+    async def receive_bytes(self, user_id: UUID) -> bytes:
+        try:
+            websocket = self.get_websocket(user_id)
+            if websocket:
+                return await websocket.receive_bytes()
+        except Exception as e:
+            logging.error(f"Error: Receive bytes: {e}")

demo/demo_cfg.yaml

@@ -0,0 +1,47 @@
+pretrained_model_path: "../models/Model/stable-diffusion-v1-5"
+
+motion_module_path: '../models/live2diff.ckpt'
+depth_model_path: '../models/dpt_hybrid_384.pt'
+
+unet_additional_kwargs:
+  cond_mapping: true
+  use_inflated_groupnorm:          true
+  use_motion_module              : true
+  motion_module_resolutions      : [ 1,2,4,8 ]
+  unet_use_cross_frame_attention : false
+  unet_use_temporal_attention    : false
+
+  motion_module_type: Streaming
+  motion_module_kwargs:
+    num_attention_heads                : 8
+    num_transformer_block              : 1
+    attention_block_types              : [ "Temporal_Self", "Temporal_Self" ]
+    temporal_position_encoding         : true
+    temporal_position_encoding_max_len : 24
+    temporal_attention_dim_div         : 1
+    zero_initialize                    : true
+
+    attention_class_name               : 'stream'
+
+    attention_kwargs:
+      window_size: 16
+      sink_size: 8
+
+noise_scheduler_kwargs:
+  num_train_timesteps: 1000
+  beta_start:    0.00085
+  beta_end:      0.012
+  beta_schedule: "linear"
+  steps_offset:  1
+  clip_sample:   False
+
+third_party_dict:
+  dreambooth: "../models/Model/revAnimated_v2RebirthVAE.safetensors"
+  lora_list:
+    - lora: '../models/LoRA/kFeltedReV.safetensors'
+      lora_alpha: 1
+  clip_skip: 2
+
+num_inference_steps: 50
+t_index_list: [30, 40]
+prompt: "masterpiece, best quality, felted, 1man with glasses, glasses, play with his pen"

demo/demo_cfg_arknight.yaml

@@ -0,0 +1,48 @@
+pretrained_model_path: "../models/Model/stable-diffusion-v1-5"
+
+motion_module_path: '../models/live2diff.ckpt'
+depth_model_path: '../models/dpt_hybrid_384.pt'
+
+unet_additional_kwargs:
+  cond_mapping: true
+  use_inflated_groupnorm:          true
+  use_motion_module              : true
+  motion_module_resolutions      : [ 1,2,4,8 ]
+  unet_use_cross_frame_attention : false
+  unet_use_temporal_attention    : false
+
+  motion_module_type: Streaming
+  motion_module_kwargs:
+    num_attention_heads                : 8
+    num_transformer_block              : 1
+    attention_block_types              : [ "Temporal_Self", "Temporal_Self" ]
+    temporal_position_encoding         : true
+    temporal_position_encoding_max_len : 24
+    temporal_attention_dim_div         : 1
+    zero_initialize                    : true
+
+    attention_class_name               : 'stream'
+
+    attention_kwargs:
+      window_size: 16
+      sink_size: 8
+
+noise_scheduler_kwargs:
+  num_train_timesteps: 1000
+  beta_start:    0.00085
+  beta_end:      0.012
+  beta_schedule: "linear"
+  steps_offset:  1
+  clip_sample:   False
+
+third_party_dict:
+  dreambooth: "../models/Model/aziibpixelmix_v10.safetensors"
+  lora_list:
+    - lora: '../models/LoRA/kFeltedReV.safetensors'
+      lora_alpha: 1
+  clip_skip: 2
+
+num_inference_steps: 50
+t_index_list: [35, 40, 45]
+
+prompt: "masterpiece, best quality, 1gril"

demo/frontend/.eslintignore

@@ -0,0 +1,13 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+
+# Ignore files for PNPM, NPM and YARN
+pnpm-lock.yaml
+package-lock.json
+yarn.lock

demo/frontend/.eslintrc.cjs

@@ -0,0 +1,30 @@
+module.exports = {
+  root: true,
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:svelte/recommended',
+    'prettier'
+  ],
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint'],
+  parserOptions: {
+    sourceType: 'module',
+    ecmaVersion: 2020,
+    extraFileExtensions: ['.svelte']
+  },
+  env: {
+    browser: true,
+    es2017: true,
+    node: true
+  },
+  overrides: [
+    {
+      files: ['*.svelte'],
+      parser: 'svelte-eslint-parser',
+      parserOptions: {
+        parser: '@typescript-eslint/parser'
+      }
+    }
+  ]
+};

demo/frontend/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+vite.config.js.timestamp-*
+vite.config.ts.timestamp-*

demo/frontend/.npmrc

@@ -0,0 +1 @@
+engine-strict=true

demo/frontend/.prettierignore

@@ -0,0 +1,13 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+
+# Ignore files for PNPM, NPM and YARN
+pnpm-lock.yaml
+package-lock.json
+yarn.lock

demo/frontend/.prettierrc

@@ -0,0 +1,19 @@
+{
+  "useTabs": false,
+  "singleQuote": true,
+  "trailingComma": "none",
+  "printWidth": 100,
+  "plugins": [
+    "prettier-plugin-svelte",
+    "prettier-plugin-organize-imports",
+    "prettier-plugin-tailwindcss"
+  ],
+  "overrides": [
+    {
+      "files": "*.svelte",
+      "options": {
+        "parser": "svelte"
+      }
+    }
+  ]
+}

demo/frontend/README.md

@@ -0,0 +1,38 @@
+# create-svelte
+
+Everything you need to build a Svelte project, powered by [`create-svelte`](https://github.com/sveltejs/kit/tree/master/packages/create-svelte).
+
+## Creating a project
+
+If you're seeing this, you've probably already done this step. Congrats!
+
+```bash
+# create a new project in the current directory
+npm create svelte@latest
+
+# create a new project in my-app
+npm create svelte@latest my-app
+```
+
+## Developing
+
+Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
+
+```bash
+npm run dev
+
+# or start the server and open the app in a new browser tab
+npm run dev -- --open
+```
+
+## Building
+
+To create a production version of your app:
+
+```bash
+npm run build
+```
+
+You can preview the production build with `npm run preview`.
+
+> To deploy your app, you may need to install an [adapter](https://kit.svelte.dev/docs/adapters) for your target environment.

demo/frontend/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "frontend",
+  "version": "0.0.1",
+  "private": true,
+  "scripts": {
+    "dev": "vite dev",
+    "build": "vite build",
+    "preview": "vite preview",
+    "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
+    "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
+    "lint": "prettier --check . && eslint .",
+    "format": "prettier --write ."
+  },
+  "devDependencies": {
+    "@sveltejs/adapter-auto": "^2.0.0",
+    "@sveltejs/adapter-static": "^2.0.3",
+    "@sveltejs/kit": "^1.20.4",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "autoprefixer": "^10.4.16",
+    "eslint": "^8.28.0",
+    "eslint-config-prettier": "^9.0.0",
+    "eslint-plugin-svelte": "^2.30.0",
+    "postcss": "^8.4.31",
+    "prettier": "^3.1.0",
+    "prettier-plugin-organize-imports": "^3.2.4",
+    "prettier-plugin-svelte": "^3.1.0",
+    "prettier-plugin-tailwindcss": "^0.5.7",
+    "svelte": "^4.0.5",
+    "svelte-check": "^3.4.3",
+    "tailwindcss": "^3.3.5",
+    "tslib": "^2.4.1",
+    "typescript": "^5.0.0",
+    "vite": "^4.4.2"
+  },
+  "type": "module",
+  "dependencies": {
+    "piexifjs": "^1.0.6",
+    "rvfc-polyfill": "^1.0.7"
+  }
+}

demo/frontend/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {}
+  }
+};

demo/frontend/src/app.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

demo/frontend/src/app.d.ts

@@ -0,0 +1,12 @@
+// See https://kit.svelte.dev/docs/types#app
+// for information about these interfaces
+declare global {
+  namespace App {
+    // interface Error {}
+    // interface Locals {}
+    // interface PageData {}
+    // interface Platform {}
+  }
+}
+
+export {};

demo/frontend/src/app.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="%sveltekit.assets%/favicon.png" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    %sveltekit.head%
+  </head>
+  <body data-sveltekit-preload-data="hover">
+    <div style="display: contents">%sveltekit.body%</div>
+  </body>
+</html>

demo/frontend/src/lib/components/Button.svelte

@@ -0,0 +1,15 @@
+<script lang="ts">
+  export let classList: string = 'p-2';
+  export let disabled: boolean = false;
+  export let title: string = '';
+</script>
+
+<button class="button {classList}" on:click {disabled} {title}>
+  <slot />
+</button>
+
+<style lang="postcss" scoped>
+  .button {
+    @apply rounded bg-gray-700 font-normal text-white hover:bg-gray-800 disabled:cursor-not-allowed disabled:bg-gray-300 dark:disabled:bg-gray-700 dark:disabled:text-black;
+  }
+</style>

demo/frontend/src/lib/components/Checkbox.svelte

@@ -0,0 +1,15 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = false;
+  export let params: FieldProps;
+  export let disabled: boolean = false;
+  onMount(() => {
+    value = Boolean(params?.default) ?? 8.0;
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center justify-items-start gap-3">
+  <label class="text-sm font-medium" for={params.id}>{params?.title}</label>
+  <input bind:checked={value} disabled={disabled} type="checkbox" id={params.id} class="cursor-pointer" />
+</div>

demo/frontend/src/lib/components/ImagePlayer.svelte

@@ -0,0 +1,50 @@
+<script lang="ts">
+  import { lcmLiveStatus, LCMLiveStatus, streamId } from '$lib/lcmLive';
+  import { getPipelineValues } from '$lib/store';
+
+  import Button from '$lib/components/Button.svelte';
+  import Floppy from '$lib/icons/floppy.svelte';
+  import { snapImage } from '$lib/utils';
+
+  $: isLCMRunning = $lcmLiveStatus !== LCMLiveStatus.DISCONNECTED;
+  $: console.log('isLCMRunning', isLCMRunning);
+  let imageEl: HTMLImageElement;
+  async function takeSnapshot() {
+    if (isLCMRunning) {
+      await snapImage(imageEl, {
+        prompt: getPipelineValues()?.prompt,
+        negative_prompt: getPipelineValues()?.negative_prompt,
+        seed: getPipelineValues()?.seed,
+        guidance_scale: getPipelineValues()?.guidance_scale
+      });
+    }
+  }
+</script>
+
+<div
+  class="relative mx-auto aspect-square max-w-lg self-center overflow-hidden rounded-lg border border-slate-300"
+>
+  <!-- svelte-ignore a11y-missing-attribute -->
+  {#if isLCMRunning && $streamId}
+    <img
+      bind:this={imageEl}
+      class="aspect-square w-full rounded-lg"
+      src={'/api/stream/' + $streamId}
+    />
+    <div class="absolute bottom-1 right-1">
+      <Button
+        on:click={takeSnapshot}
+        disabled={!isLCMRunning}
+        title={'Take Snapshot'}
+        classList={'text-sm ml-auto text-white p-1 shadow-lg rounded-lg opacity-50'}
+      >
+        <Floppy classList={''} />
+      </Button>
+    </div>
+  {:else}
+    <img
+      class="aspect-square w-full rounded-lg"
+      src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
+    />
+  {/if}
+</div>

demo/frontend/src/lib/components/InputRange.svelte

@@ -0,0 +1,52 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = 8.0;
+  export let params: FieldProps;
+  export let disabled: boolean = false;
+  onMount(() => {
+    value = Number(params?.default) ?? 8.0;
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center gap-3">
+  <label class="text-sm font-medium" for={params.id}>{params?.title}</label>
+  <input
+    class="col-span-2 h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-300 dark:bg-gray-500"
+    bind:value
+    type="range"
+    id={params.id}
+    name={params.id}
+    min={params?.min}
+    max={params?.max}
+    step={params?.step ?? 1}
+  />
+  <input
+    type="number"
+    step={params?.step ?? 1}
+    bind:value
+    disabled={disabled}
+    class="rounded-md border px-1 py-1 text-center text-xs font-bold dark:text-black"
+  />
+</div>
+<!--
+<style lang="postcss" scoped>
+  input[type='range']::-webkit-slider-runnable-track {
+    @apply h-2 cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-webkit-slider-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-moz-range-track {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-moz-range-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-ms-track {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-ms-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+</style> -->

demo/frontend/src/lib/components/MediaListSwitcher.svelte

@@ -0,0 +1,40 @@
+<script lang="ts">
+  import { mediaDevices, mediaStreamActions } from '$lib/mediaStream';
+  import Screen from '$lib/icons/screen.svelte';
+  import { onMount } from 'svelte';
+
+  let deviceId: string = '';
+  $: {
+    console.log($mediaDevices);
+  }
+  $: {
+    console.log(deviceId);
+  }
+  onMount(() => {
+    deviceId = $mediaDevices[0].deviceId;
+  });
+</script>
+
+<div class="flex items-center justify-center text-xs">
+  <button
+    title="Share your screen"
+    class="border-1 my-1 flex cursor-pointer gap-1 rounded-md border-gray-500 border-opacity-50 bg-slate-100 bg-opacity-30 p-1 font-medium text-white"
+    on:click={() => mediaStreamActions.startScreenCapture()}
+  >
+    <span>Share</span>
+
+    <Screen classList={''} />
+  </button>
+  {#if $mediaDevices}
+    <select
+      bind:value={deviceId}
+      on:change={() => mediaStreamActions.switchCamera(deviceId)}
+      id="devices-list"
+      class="border-1 block cursor-pointer rounded-md border-gray-800 border-opacity-50 bg-slate-100 bg-opacity-30 p-1 font-medium text-white"
+    >
+      {#each $mediaDevices as device, i}
+        <option value={device.deviceId}>{device.label}</option>
+      {/each}
+    </select>
+  {/if}
+</div>

demo/frontend/src/lib/components/PipelineOptions.svelte

@@ -0,0 +1,62 @@
+<script lang="ts">
+  import type { Fields } from '$lib/types';
+  import { FieldType } from '$lib/types';
+  import InputRange from './InputRange.svelte';
+  import SeedInput from './SeedInput.svelte';
+  import TextArea from './TextArea.svelte';
+  import Checkbox from './Checkbox.svelte';
+  import Selectlist from './Selectlist.svelte';
+  import { pipelineValues } from '$lib/store';
+
+  export let pipelineParams: Fields;
+  export let disabled: boolean = false;
+
+  $: advanceOptions = Object.values(pipelineParams)?.filter(
+    (e) => e?.hide == true && e?.disabled !== true
+  );
+  $: featuredOptions = Object.values(pipelineParams)?.filter((e) => e?.hide !== true);
+</script>
+
+<div class="flex flex-col gap-3">
+  <div class="grid grid-cols-1 items-center gap-3">
+    {#if featuredOptions}
+      {#each featuredOptions as params}
+        {#if params.field === FieldType.RANGE}
+          <InputRange {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></InputRange>
+        {:else if params.field === FieldType.SEED}
+          <SeedInput {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></SeedInput>
+        {:else if params.field === FieldType.TEXTAREA}
+          <TextArea {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></TextArea>
+        {:else if params.field === FieldType.CHECKBOX}
+          <Checkbox {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></Checkbox>
+        {:else if params.field === FieldType.SELECT}
+          <Selectlist {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></Selectlist>
+        {/if}
+      {/each}
+    {/if}
+  </div>
+  {#if advanceOptions && advanceOptions.length > 0}
+    <details>
+      <summary class="cursor-pointer font-medium">Advanced Options</summary>
+      <div
+        class="grid grid-cols-1 items-center gap-3 {Object.values(pipelineParams).length > 5
+          ? 'sm:grid-cols-2'
+          : ''}"
+      >
+        {#each advanceOptions as params}
+          {#if params.field === FieldType.RANGE}
+            <InputRange {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></InputRange>
+          {:else if params.field === FieldType.SEED}
+            <SeedInput {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></SeedInput>
+          {:else if params.field === FieldType.TEXTAREA}
+            <TextArea {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></TextArea>
+          {:else if params.field === FieldType.CHECKBOX}
+            <Checkbox {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></Checkbox>
+          {:else if params.field === FieldType.SELECT}
+            <Selectlist {params} bind:value={$pipelineValues[params.id]} disabled={disabled}></Selectlist>
+          {/if}
+        {/each}
+      </div>
+    </details>
+  {/if}
+</div>

demo/frontend/src/lib/components/SeedInput.svelte

@@ -0,0 +1,28 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  import Button from './Button.svelte';
+  export let value = 299792458;
+  export let params: FieldProps;
+  export let disabled: boolean = false;
+
+  onMount(() => {
+    value = Number(params?.default ?? '');
+  });
+  function randomize() {
+    value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+  }
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center gap-3">
+  <label class="text-sm font-medium" for="seed">Seed</label>
+  <input
+    bind:value
+    type="number"
+    id="seed"
+    name="seed"
+    class="col-span-2 rounded-md border border-gray-700 p-2 text-right font-light dark:text-black"
+    disabled={disabled}
+  />
+  <Button on:click={randomize}>Rand</Button>
+</div>

demo/frontend/src/lib/components/Selectlist.svelte

@@ -0,0 +1,26 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = '';
+  export let params: FieldProps;
+  export let disabled: boolean = false;
+  onMount(() => {
+    value = String(params?.default);
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center justify-items-start gap-3">
+  <label for="model-list" class="text-sm font-medium">{params?.title} </label>
+  {#if params?.values}
+    <select
+      bind:value
+      disabled={disabled}
+      id="model-list"
+      class="cursor-pointer rounded-md border-2 border-gray-500 p-2 font-light dark:text-black"
+    >
+      {#each params.values as model, i}
+        <option value={model} selected={i === 0}>{model}</option>
+      {/each}
+    </select>
+  {/if}
+</div>

demo/frontend/src/lib/components/TextArea.svelte

@@ -0,0 +1,25 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value: string;
+  export let params: FieldProps;
+  export let disabled: boolean = false;
+  onMount(() => {
+    value = String(params?.default ?? '');
+  });
+</script>
+
+<div class="">
+  <label class="text-sm font-medium" for={params?.title}>
+    {params?.title}
+  </label>
+  <div class="text-normal flex items-center rounded-md border border-gray-700">
+    <textarea
+      class="mx-1 w-full px-3 py-2 font-light outline-none dark:text-black"
+      title={params?.title}
+      placeholder="Add your prompt here..."
+      bind:value
+      disabled={disabled}
+    ></textarea>
+  </div>
+</div>

demo/frontend/src/lib/components/VideoInput.svelte

@@ -0,0 +1,139 @@
+<script lang="ts">
+  import 'rvfc-polyfill';
+
+  import { onDestroy, onMount } from 'svelte';
+  import {
+    mediaStreamStatus,
+    MediaStreamStatusEnum,
+    onFrameChangeStore,
+    mediaStream,
+    mediaDevices,
+    mediaStreamActions
+  } from '$lib/mediaStream';
+  import Button from './Button.svelte';
+  import MediaListSwitcher from './MediaListSwitcher.svelte';
+  export let width = 512;
+  export let height = 512;
+  const size = { width, height };
+
+  let videoEl: HTMLVideoElement;
+  let canvasEl: HTMLCanvasElement;
+  let ctx: CanvasRenderingContext2D;
+  let videoFrameCallbackId: number;
+
+  let isWebCamActive: boolean = false;
+
+  // adjust the throttle time to your needs
+  const THROTTLE = 1000 / 120;
+  let selectedDevice: string = '';
+  let videoIsReady = false;
+
+  onMount(() => {
+    ctx = canvasEl.getContext('2d') as CanvasRenderingContext2D;
+    canvasEl.width = size.width;
+    canvasEl.height = size.height;
+  });
+  $: {
+    console.log(selectedDevice);
+  }
+  onDestroy(() => {
+    if (videoFrameCallbackId) videoEl.cancelVideoFrameCallback(videoFrameCallbackId);
+  });
+
+  $: if (videoEl) {
+    videoEl.srcObject = $mediaStream;
+  }
+  let lastMillis = 0;
+  async function onFrameChange(now: DOMHighResTimeStamp, metadata: VideoFrameCallbackMetadata) {
+    if (now - lastMillis < THROTTLE) {
+      videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+      return;
+    }
+    const videoWidth = videoEl.videoWidth;
+    const videoHeight = videoEl.videoHeight;
+    let height0 = videoHeight;
+    let width0 = videoWidth;
+    let x0 = 0;
+    let y0 = 0;
+    if (videoWidth > videoHeight) {
+      width0 = videoHeight;
+      x0 = (videoWidth - videoHeight) / 2;
+    } else {
+      height0 = videoWidth;
+      y0 = (videoHeight - videoWidth) / 2;
+    }
+    ctx.drawImage(videoEl, x0, y0, width0, height0, 0, 0, size.width, size.height);
+    const blob = await new Promise<Blob>((resolve) => {
+      canvasEl.toBlob(
+        (blob) => {
+          resolve(blob as Blob);
+        },
+        'image/jpeg',
+        1
+      );
+    });
+    onFrameChangeStore.set({ blob });
+    videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+  }
+
+  $: if ($mediaStreamStatus == MediaStreamStatusEnum.CONNECTED && videoIsReady) {
+    videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+  }
+
+  $: isWebCamActive = $mediaStreamStatus === MediaStreamStatusEnum.CONNECTED;
+  async function startWebCam() {
+    await mediaStreamActions.enumerateDevices();
+    await mediaStreamActions.start();
+  }
+
+  function stopWebCam() {
+    mediaStreamActions.stop();
+  }
+
+  async function toggleWebCam() {
+    if (isWebCamActive) {
+      stopWebCam();
+    } else {
+      await startWebCam();
+    }
+  }
+</script>
+
+<div class="relative mx-auto max-w-lg overflow-hidden rounded-lg border border-slate-300">
+  <div class="relative z-10 aspect-square w-full object-cover">
+    {#if $mediaDevices.length > 0}
+      <div class="absolute bottom-0 right-0 z-10">
+        <MediaListSwitcher />
+      </div>
+    {/if}
+    <video
+      class="pointer-events-none aspect-square w-full object-cover"
+      bind:this={videoEl}
+      on:loadeddata={() => {
+        videoIsReady = true;
+      }}
+      playsinline
+      autoplay
+      muted
+      loop
+    ></video>
+    <canvas bind:this={canvasEl} class="absolute left-0 top-0 aspect-square w-full object-cover"
+    ></canvas>
+  </div>
+  <div class="absolute left-0 top-0 flex aspect-square w-full items-center justify-center">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 448" class="w-40 p-5 opacity-20">
+      <path
+        fill="currentColor"
+        d="M224 256a128 128 0 1 0 0-256 128 128 0 1 0 0 256zm-45.7 48A178.3 178.3 0 0 0 0 482.3 29.7 29.7 0 0 0 29.7 512h388.6a29.7 29.7 0 0 0 29.7-29.7c0-98.5-79.8-178.3-178.3-178.3h-91.4z"
+      />
+    </svg>
+  </div>
+</div>
+
+<Button on:click={toggleWebCam}>
+  {#if isWebCamActive}
+    <span>Stop WebCam</span>
+  {:else}
+    <span>Start WebCam</span>
+  {/if}
+</Button>

demo/frontend/src/lib/components/Warning.svelte

@@ -0,0 +1,27 @@
+<script lang="ts">
+  export let message: string = '';
+
+  let timeout = 0;
+  $: if (message !== '') {
+    console.log('message', message);
+    clearTimeout(timeout);
+    timeout = setTimeout(() => {
+      message = '';
+    }, 5000);
+  }
+</script>
+
+{#if message}
+  <div class="fixed right-0 top-0 m-4 cursor-pointer" on:click={() => (message = '')}>
+    <div class="rounded bg-red-800 p-4 text-white">
+      {message}
+    </div>
+    <div class="bar transition-all duration-500" style="width: 0;"></div>
+  </div>
+{/if}
+
+<style lang="postcss" scoped>
+  .button {
+    @apply rounded bg-gray-700 font-normal text-white hover:bg-gray-800 disabled:cursor-not-allowed disabled:bg-gray-300 dark:disabled:bg-gray-700 dark:disabled:text-black;
+  }
+</style>

demo/frontend/src/lib/icons/floppy.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string;
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" height="1em" viewBox="0 0 448 512" class={classList}>
+  <path
+    fill="currentColor"
+    d="M48 96v320a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16V170.5a16 16 0 0 0-4.7-11.3l33.9-33.9a64 64 0 0 1 18.7 45.3V416a64 64 0 0 1-64 64H64a64 64 0 0 1-64-64V96a64 64 0 0 1 64-64h245.5a64 64 0 0 1 45.3 18.7l74.5 74.5-33.9 33.9-74.6-74.4-.8-.8V184a24 24 0 0 1-24 24H104a24 24 0 0 1-24-24V80H64a16 16 0 0 0-16 16zm80-16v80h144V80H128zm32 240a64 64 0 1 1 128 0 64 64 0 1 1-128 0z"
+  />
+</svg>

demo/frontend/src/lib/icons/screen.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string = '';
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -32 576 576" height="16px" class={classList}>
+  <path
+    fill="currentColor"
+    d="M64 0A64 64 0 0 0 0 64v288a64 64 0 0 0 64 64h176l-10.7 32H160a32 32 0 1 0 0 64h256a32 32 0 1 0 0-64h-69.3L336 416h176a64 64 0 0 0 64-64V64a64 64 0 0 0-64-64H64zm448 64v288H64V64h448z"
+  />
+</svg>

demo/frontend/src/lib/icons/spinner.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string;
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" height="1em" viewBox="0 0 512 512" class={classList}>
+  <path
+    fill="currentColor"
+    d="M304 48a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zm0 416a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zM48 304a48 48 0 1 0 0-96 48 48 0 1 0 0 96zm464-48a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zM142.9 437A48 48 0 1 0 75 369.1 48 48 0 1 0 142.9 437zm0-294.2A48 48 0 1 0 75 75a48 48 0 1 0 67.9 67.9zM369.1 437A48 48 0 1 0 437 369.1 48 48 0 1 0 369.1 437z"
+  />
+</svg>

demo/frontend/src/lib/index.ts

@@ -0,0 +1 @@
+// place files you want to import through the `$lib` alias in this folder.

demo/frontend/src/lib/lcmLive.ts

@@ -0,0 +1,99 @@
+import { writable } from 'svelte/store';
+
+
+export enum LCMLiveStatus {
+    CONNECTED = "connected",
+    DISCONNECTED = "disconnected",
+    WAIT = "wait",
+    SEND_FRAME = "send_frame",
+    TIMEOUT = "timeout",
+}
+
+const initStatus: LCMLiveStatus = LCMLiveStatus.DISCONNECTED;
+
+export const lcmLiveStatus = writable<LCMLiveStatus>(initStatus);
+export const streamId = writable<string | null>(null);
+
+let websocket: WebSocket | null = null;
+export const lcmLiveActions = {
+    async start(getSreamdata: () => any[]) {
+        return new Promise((resolve, reject) => {
+
+            try {
+                const userId = crypto.randomUUID();
+                const websocketURL = `${window.location.protocol === "https:" ? "wss" : "ws"
+                    }:${window.location.host}/api/ws/${userId}`;
+
+                websocket = new WebSocket(websocketURL);
+                websocket.onopen = () => {
+                    console.log("Connected to websocket");
+                };
+                websocket.onclose = () => {
+                    lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+                    console.log("Disconnected from websocket");
+                };
+                websocket.onerror = (err) => {
+                    console.error(err);
+                };
+                websocket.onmessage = (event) => {
+                    const data = JSON.parse(event.data);
+                    switch (data.status) {
+                        case "connected":
+                            lcmLiveStatus.set(LCMLiveStatus.CONNECTED);
+                            streamId.set(userId);
+                            resolve({ status: "connected", userId });
+                            break;
+                        case "send_frame":
+                            lcmLiveStatus.set(LCMLiveStatus.SEND_FRAME);
+                            const streamData = getSreamdata();
+                            websocket?.send(JSON.stringify({ status: "next_frame" }));
+                            for (const d of streamData) {
+                                this.send(d);
+                            }
+                            break;
+                        case "wait":
+                            lcmLiveStatus.set(LCMLiveStatus.WAIT);
+                            break;
+                        case "timeout":
+                            console.log("timeout");
+                            lcmLiveStatus.set(LCMLiveStatus.TIMEOUT);
+                            streamId.set(null);
+                            reject(new Error("timeout"));
+                            break;
+                        case "error":
+                            console.log(data.message);
+                            lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+                            streamId.set(null);
+                            reject(new Error(data.message));
+                            break;
+                    }
+                };
+
+            } catch (err) {
+                console.error(err);
+                lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+                streamId.set(null);
+                reject(err);
+            }
+        });
+    },
+    send(data: Blob | { [key: string]: any }) {
+        if (websocket && websocket.readyState === WebSocket.OPEN) {
+            if (data instanceof Blob) {
+                websocket.send(data);
+            } else {
+                websocket.send(JSON.stringify(data));
+            }
+        } else {
+            console.log("WebSocket not connected");
+        }
+    },
+    async stop() {
+        lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+        if (websocket) {
+            websocket.close();
+        }
+        websocket = null;
+        streamId.set(null);
+    },
+};

demo/frontend/src/lib/mediaStream.ts

@@ -0,0 +1,98 @@
+import { writable, type Writable, get } from 'svelte/store';
+
+export enum MediaStreamStatusEnum {
+    INIT = "init",
+    CONNECTED = "connected",
+    DISCONNECTED = "disconnected",
+}
+export const onFrameChangeStore: Writable<{ blob: Blob }> = writable({ blob: new Blob() });
+
+export const mediaDevices = writable<MediaDeviceInfo[]>([]);
+export const mediaStreamStatus = writable(MediaStreamStatusEnum.INIT);
+export const mediaStream = writable<MediaStream | null>(null);
+
+export const mediaStreamActions = {
+    async enumerateDevices() {
+        // console.log("Enumerating devices");
+        await navigator.mediaDevices.enumerateDevices()
+            .then(devices => {
+                const cameras = devices.filter(device => device.kind === 'videoinput');
+                mediaDevices.set(cameras);
+            })
+            .catch(err => {
+                console.error(err);
+            });
+    },
+    async start(mediaDevicedID?: string) {
+        const constraints = {
+            audio: false,
+            video: {
+                width: 1024, height: 1024, deviceId: mediaDevicedID
+            }
+        };
+
+        await navigator.mediaDevices
+            .getUserMedia(constraints)
+            .then((stream) => {
+                mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+                mediaStream.set(stream);
+            })
+            .catch((err) => {
+                console.error(`${err.name}: ${err.message}`);
+                mediaStreamStatus.set(MediaStreamStatusEnum.DISCONNECTED);
+                mediaStream.set(null);
+            });
+    },
+    async startScreenCapture() {
+        const displayMediaOptions = {
+            video: {
+                displaySurface: "window",
+            },
+            audio: false,
+            surfaceSwitching: "include"
+        };
+
+
+        let captureStream = null;
+
+        try {
+            captureStream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
+            const videoTrack = captureStream.getVideoTracks()[0];
+
+            console.log("Track settings:");
+            console.log(JSON.stringify(videoTrack.getSettings(), null, 2));
+            console.log("Track constraints:");
+            console.log(JSON.stringify(videoTrack.getConstraints(), null, 2));
+            mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+            mediaStream.set(captureStream)
+        } catch (err) {
+            console.error(err);
+        }
+
+    },
+    async switchCamera(mediaDevicedID: string) {
+        if (get(mediaStreamStatus) !== MediaStreamStatusEnum.CONNECTED) {
+            return;
+        }
+        const constraints = {
+            audio: false,
+            video: { width: 1024, height: 1024, deviceId: mediaDevicedID }
+        };
+        await navigator.mediaDevices
+            .getUserMedia(constraints)
+            .then((stream) => {
+                mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+                mediaStream.set(stream)
+            })
+            .catch((err) => {
+                console.error(`${err.name}: ${err.message}`);
+            });
+    },
+    async stop() {
+        navigator.mediaDevices.getUserMedia({ video: true }).then((stream) => {
+            stream.getTracks().forEach((track) => track.stop());
+        });
+        mediaStreamStatus.set(MediaStreamStatusEnum.DISCONNECTED);
+        mediaStream.set(null);
+    },
+};

demo/frontend/src/lib/store.ts

@@ -0,0 +1,15 @@
+
+import { derived, writable, get, type Writable, type Readable } from 'svelte/store';
+
+export const pipelineValues: Writable<Record<string, any>> = writable({});
+export const deboucedPipelineValues: Readable<Record<string, any>>
+    = derived(pipelineValues, ($pipelineValues, set) => {
+        const debounced = setTimeout(() => {
+            set($pipelineValues);
+        }, 100);
+        return () => clearTimeout(debounced);
+    });
+
+
+
+export const getPipelineValues = () => get(pipelineValues);

demo/frontend/src/lib/types.ts

@@ -0,0 +1,40 @@
+export const enum FieldType {
+    RANGE = "range",
+    SEED = "seed",
+    TEXTAREA = "textarea",
+    CHECKBOX = "checkbox",
+    SELECT = "select",
+}
+export const enum PipelineMode {
+    IMAGE = "image",
+    VIDEO = "video",
+    TEXT = "text",
+}
+
+
+export interface Fields {
+    [key: string]: FieldProps;
+}
+
+export interface FieldProps {
+    default: number | string;
+    max?: number;
+    min?: number;
+    title: string;
+    field: FieldType;
+    step?: number;
+    disabled?: boolean;
+    hide?: boolean;
+    id: string;
+    values?: string[];
+}
+export interface PipelineInfo {
+    title: {
+        default: string;
+    }
+    name: string;
+    description: string;
+    input_mode: {
+        default: PipelineMode;
+    }
+}

demo/frontend/src/lib/utils.ts

@@ -0,0 +1,38 @@
+import * as piexif from "piexifjs";
+
+interface IImageInfo {
+    prompt?: string;
+    negative_prompt?: string;
+    seed?: number;
+    guidance_scale?: number;
+}
+
+export function snapImage(imageEl: HTMLImageElement, info: IImageInfo) {
+    try {
+        const zeroth: { [key: string]: any } = {};
+        const exif: { [key: string]: any } = {};
+        const gps: { [key: string]: any } = {};
+        zeroth[piexif.ImageIFD.Make] = "LCM Image-to-Image ControNet";
+        zeroth[piexif.ImageIFD.ImageDescription] = `prompt: ${info?.prompt} | negative_prompt: ${info?.negative_prompt} | seed: ${info?.seed} | guidance_scale: ${info?.guidance_scale}`;
+        zeroth[piexif.ImageIFD.Software] = "https://github.com/radames/Real-Time-Latent-Consistency-Model";
+        exif[piexif.ExifIFD.DateTimeOriginal] = new Date().toISOString();
+
+        const exifObj = { "0th": zeroth, "Exif": exif, "GPS": gps };
+        const exifBytes = piexif.dump(exifObj);
+
+        const canvas = document.createElement("canvas");
+        canvas.width = imageEl.naturalWidth;
+        canvas.height = imageEl.naturalHeight;
+        const ctx = canvas.getContext("2d") as CanvasRenderingContext2D;
+        ctx.drawImage(imageEl, 0, 0);
+        const dataURL = canvas.toDataURL("image/jpeg");
+        const withExif = piexif.insert(exifBytes, dataURL);
+
+        const a = document.createElement("a");
+        a.href = withExif;
+        a.download = `lcm_txt_2_img${Date.now()}.png`;
+        a.click();
+    } catch (err) {
+        console.log(err);
+    }
+}

demo/frontend/src/routes/+layout.svelte

@@ -0,0 +1,5 @@
+<script>
+  import '../app.css';
+</script>
+
+<slot />

demo/frontend/src/routes/+page.svelte

@@ -0,0 +1,160 @@
+<script lang="ts">
+  import Button from '$lib/components/Button.svelte';
+  import ImagePlayer from '$lib/components/ImagePlayer.svelte';
+  import PipelineOptions from '$lib/components/PipelineOptions.svelte';
+  import VideoInput from '$lib/components/VideoInput.svelte';
+  import Warning from '$lib/components/Warning.svelte';
+  import Spinner from '$lib/icons/spinner.svelte';
+  import { lcmLiveActions, lcmLiveStatus, LCMLiveStatus } from '$lib/lcmLive';
+  import { mediaStreamActions, mediaStreamStatus, MediaStreamStatusEnum, onFrameChangeStore } from '$lib/mediaStream';
+  import { deboucedPipelineValues, getPipelineValues } from '$lib/store';
+  import type { Fields, PipelineInfo } from '$lib/types';
+  import { PipelineMode } from '$lib/types';
+  import { onMount } from 'svelte';
+
+  let pipelineParams: Fields;
+  let pipelineInfo: PipelineInfo;
+  let pageContent: string;
+  let isImageMode: boolean = false;
+  let maxQueueSize: number = 0;
+  let currentQueueSize: number = 0;
+  let queueCheckerRunning: boolean = false;
+  let warningMessage: string = '';
+  onMount(() => {
+    getSettings();
+  });
+
+  async function getSettings() {
+    const settings = await fetch('/api/settings').then((r) => r.json());
+    pipelineParams = settings.input_params.properties;
+    pipelineInfo = settings.info.properties;
+    isImageMode = pipelineInfo.input_mode.default === PipelineMode.IMAGE;
+    maxQueueSize = settings.max_queue_size;
+    pageContent = settings.page_content;
+    console.log(pipelineParams);
+    toggleQueueChecker(true);
+  }
+  function toggleQueueChecker(start: boolean) {
+    queueCheckerRunning = start && maxQueueSize > 0;
+    if (start) {
+      getQueueSize();
+    }
+  }
+  async function getQueueSize() {
+    if (!queueCheckerRunning) {
+      return;
+    }
+    const data = await fetch('/api/queue').then((r) => r.json());
+    currentQueueSize = data.queue_size;
+    setTimeout(getQueueSize, 10000);
+  }
+
+  function getSreamdata() {
+    if (isImageMode) {
+      return [getPipelineValues(), $onFrameChangeStore?.blob];
+    } else {
+      return [$deboucedPipelineValues];
+    }
+  }
+
+  $: isLCMRunning = $lcmLiveStatus !== LCMLiveStatus.DISCONNECTED;
+  $: if ($lcmLiveStatus === LCMLiveStatus.TIMEOUT) {
+    warningMessage = 'Session timed out. Please try again.';
+  }
+  let disabled = false;
+  async function toggleLcmLive() {
+    try {
+      if (!isLCMRunning) {
+        if (isImageMode) {
+            if ($mediaStreamStatus === MediaStreamStatusEnum.DISCONNECTED) {
+                warningMessage = 'Please allow camera access.';
+                return
+            }
+        //   await mediaStreamActions.enumerateDevices();
+        //   await mediaStreamActions.start();
+        }
+        disabled = true;
+        await lcmLiveActions.start(getSreamdata);
+        disabled = false;
+        toggleQueueChecker(false);
+      } else {
+        // if (isImageMode) {
+        //   mediaStreamActions.stop();
+        // }
+        lcmLiveActions.stop();
+        mediaStreamActions.stop()
+        toggleQueueChecker(true);
+      }
+    } catch (e) {
+      warningMessage = e instanceof Error ? e.message : '';
+      console.error(warningMessage)
+      disabled = false;
+      toggleQueueChecker(true);
+    }
+  }
+</script>
+
+<svelte:head>
+  <script
+    src="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.9/iframeResizer.contentWindow.min.js"
+  ></script>
+</svelte:head>
+
+{@debug isImageMode, pipelineParams}
+
+<main class="container mx-auto flex max-w-5xl flex-col gap-3 px-4 py-4">
+  <Warning bind:message={warningMessage}></Warning>
+  <article class="text-center">
+    {#if pageContent}
+      {@html pageContent}
+    {/if}
+    {#if maxQueueSize > 0}
+      <p class="text-sm">
+        There are <span id="queue_size" class="font-bold">{currentQueueSize}</span>
+        user(s) sharing the same GPU, affecting real-time performance. Maximum queue size is {maxQueueSize}.
+        <a
+          href="https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model?duplicate=true"
+          target="_blank"
+          class="text-blue-500 underline hover:no-underline">Duplicate</a
+        > and run it on your own GPU.
+      </p>
+    {/if}
+  </article>
+  {#if pipelineParams}
+    <article class="my-3 grid grid-cols-1 gap-3 sm:grid-cols-2">
+      {#if isImageMode}
+        <div class="sm:col-start-1">
+          <VideoInput
+            width={Number(pipelineParams.width.default)}
+            height={Number(pipelineParams.height.default)}
+          ></VideoInput>
+        </div>
+      {/if}
+      <div class={isImageMode ? 'sm:col-start-2' : 'col-span-2'}>
+        <ImagePlayer />
+      </div>
+      <div class="sm:col-span-2">
+        <Button on:click={toggleLcmLive} {disabled} classList={'text-lg my-1 p-2'}>
+          {#if isLCMRunning}
+            Stop
+          {:else}
+            Start
+          {/if}
+        </Button>
+        <PipelineOptions {pipelineParams} disabled={isLCMRunning}></PipelineOptions>
+      </div>
+    </article>
+  {:else}
+    <!-- loading -->
+    <div class="flex items-center justify-center gap-3 py-48 text-2xl">
+      <Spinner classList={'animate-spin opacity-50'}></Spinner>
+      <p>Loading...</p>
+    </div>
+  {/if}
+</main>
+
+<style lang="postcss">
+  :global(html) {
+    @apply text-black dark:bg-gray-900 dark:text-white;
+  }
+</style>

demo/frontend/src/routes/+page.ts

@@ -0,0 +1 @@
+export const prerender = true

demo/frontend/svelte.config.js

@@ -0,0 +1,17 @@
+import adapter from '@sveltejs/adapter-static';
+import { vitePreprocess } from '@sveltejs/kit/vite';
+/** @type {import('@sveltejs/kit').Config} */
+const config = {
+  preprocess: vitePreprocess({ postcss: true }),
+  kit: {
+    adapter: adapter({
+      pages: 'public',
+      assets: 'public',
+      fallback: undefined,
+      precompress: false,
+      strict: true
+    })
+  }
+};
+
+export default config;

demo/frontend/tailwind.config.js

@@ -0,0 +1,8 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: ['./src/**/*.{html,js,svelte,ts}', '../**/*.py'],
+  theme: {
+    extend: {}
+  },
+  plugins: []
+};

demo/frontend/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "./.svelte-kit/tsconfig.json",
+  "compilerOptions": {
+    "allowJs": true,
+    "checkJs": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "strict": true
+  }
+  // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias
+  //
+  // If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes
+  // from the referenced tsconfig.json - TypeScript does not merge them in
+}