Update space

model/matchmaker.py

@@ -0,0 +1,126 @@
+import numpy as np
+import json
+from trueskill import TrueSkill
+import paramiko
+import io, os
+import sys
+import random
+
+sys.path.append('../')
+from serve.constants import SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD, SSH_SKILL
+trueskill_env = TrueSkill()
+
+ssh_matchmaker_client = None
+sftp_matchmaker_client = None
+
+def create_ssh_matchmaker_client(server, port, user, password):
+    global ssh_matchmaker_client, sftp_matchmaker_client
+    ssh_matchmaker_client = paramiko.SSHClient()
+    ssh_matchmaker_client.load_system_host_keys()
+    ssh_matchmaker_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh_matchmaker_client.connect(server, port, user, password)
+
+    transport = ssh_matchmaker_client.get_transport()
+    transport.set_keepalive(60)
+
+    sftp_matchmaker_client = ssh_matchmaker_client.open_sftp()
+
+
+def is_connected():
+    global ssh_matchmaker_client, sftp_matchmaker_client
+    if ssh_matchmaker_client is None or sftp_matchmaker_client is None:
+        return False
+    if not ssh_matchmaker_client.get_transport().is_active():
+        return False
+    try:
+        sftp_matchmaker_client.listdir('.')
+    except Exception as e:
+        print(f"Error checking SFTP connection: {e}")
+        return False
+    return True
+
+
+def ucb_score(trueskill_diff, t, n):
+    exploration_term = np.sqrt((2 * np.log(t + 1e-5)) / (n + 1e-5))
+    ucb = -trueskill_diff + 1.0 * exploration_term
+    return ucb
+
+
+def update_trueskill(ratings, ranks):
+    new_ratings = trueskill_env.rate(ratings, ranks)
+    return new_ratings
+
+
+def serialize_rating(rating):
+    return {'mu': rating.mu, 'sigma': rating.sigma}
+
+
+def deserialize_rating(rating_dict):
+    return trueskill_env.Rating(mu=rating_dict['mu'], sigma=rating_dict['sigma'])
+
+
+def save_json_via_sftp(ratings, comparison_counts, total_comparisons):
+    global sftp_matchmaker_client
+    if not is_connected():
+        create_ssh_matchmaker_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+    data = {
+        'ratings': [serialize_rating(r) for r in ratings],
+        'comparison_counts': comparison_counts.tolist(),
+        'total_comparisons': total_comparisons
+    }  
+    json_data = json.dumps(data)
+    with sftp_matchmaker_client.open(SSH_SKILL, 'w') as f:
+        f.write(json_data)
+
+
+def load_json_via_sftp():
+    global sftp_matchmaker_client
+    if not is_connected():
+        create_ssh_matchmaker_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+    with sftp_matchmaker_client.open(SSH_SKILL, 'r') as f:
+        data = json.load(f)
+    ratings = [deserialize_rating(r) for r in data['ratings']]
+    comparison_counts = np.array(data['comparison_counts'])
+    total_comparisons = data['total_comparisons']
+    return ratings, comparison_counts, total_comparisons
+
+
+class RunningPivot(object):
+    running_pivot = []
+
+
+def matchmaker(num_players, k_group=4, not_run=[]):
+    trueskill_env = TrueSkill()
+    
+    ratings, comparison_counts, total_comparisons = load_json_via_sftp()
+
+    ratings = ratings[:num_players]
+    comparison_counts = comparison_counts[:num_players, :num_players]
+
+    # Randomly select a player
+    # selected_player = np.random.randint(0, num_players)
+    comparison_counts[RunningPivot.running_pivot, :] = float('inf')
+    comparison_counts[not_run, :] = float('inf')
+    selected_player = np.argmin(comparison_counts.sum(axis=1))
+
+    RunningPivot.running_pivot.append(selected_player)
+    RunningPivot.running_pivot = RunningPivot.running_pivot[-5:]
+    print(RunningPivot.running_pivot)
+
+    selected_trueskill_score = trueskill_env.expose(ratings[selected_player])
+    trueskill_scores = np.array([trueskill_env.expose(p) for p in ratings])
+    trueskill_diff = np.abs(trueskill_scores - selected_trueskill_score)
+    n = comparison_counts[selected_player]
+    ucb_scores = ucb_score(trueskill_diff, total_comparisons, n)
+
+    # Exclude self, select opponent with highest UCB score
+    ucb_scores[selected_player] = -float('inf')
+    ucb_scores[not_run] = -float('inf')
+    opponents = np.argsort(ucb_scores)[-k_group + 1:].tolist()
+
+    # Group players
+    model_ids = [selected_player] + opponents
+
+    random.shuffle(model_ids)
+        
+    return model_ids

model/matchmaker_video.py

@@ -0,0 +1,136 @@
+import numpy as np
+import json
+from trueskill import TrueSkill
+import paramiko
+import io, os
+import sys
+import random
+
+sys.path.append('../')
+from serve.constants import SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD, SSH_VIDEO_SKILL
+trueskill_env = TrueSkill()
+
+ssh_matchmaker_client = None
+sftp_matchmaker_client = None
+
+
+def create_ssh_matchmaker_client(server, port, user, password):
+    global ssh_matchmaker_client, sftp_matchmaker_client
+    ssh_matchmaker_client = paramiko.SSHClient()
+    ssh_matchmaker_client.load_system_host_keys()
+    ssh_matchmaker_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh_matchmaker_client.connect(server, port, user, password)
+
+    transport = ssh_matchmaker_client.get_transport()
+    transport.set_keepalive(60)
+
+    sftp_matchmaker_client = ssh_matchmaker_client.open_sftp()
+
+
+def is_connected():
+    global ssh_matchmaker_client, sftp_matchmaker_client
+    if ssh_matchmaker_client is None or sftp_matchmaker_client is None:
+        return False
+    if not ssh_matchmaker_client.get_transport().is_active():
+        return False
+    try:
+        sftp_matchmaker_client.listdir('.')
+    except Exception as e:
+        print(f"Error checking SFTP connection: {e}")
+        return False
+    return True
+
+
+def ucb_score(trueskill_diff, t, n):
+    exploration_term = np.sqrt((2 * np.log(t + 1e-5)) / (n + 1e-5))
+    ucb = -trueskill_diff + 1.0 * exploration_term
+    return ucb
+
+
+def update_trueskill(ratings, ranks):
+    new_ratings = trueskill_env.rate(ratings, ranks)
+    return new_ratings
+
+
+def serialize_rating(rating):
+    return {'mu': rating.mu, 'sigma': rating.sigma}
+
+
+def deserialize_rating(rating_dict):
+    return trueskill_env.Rating(mu=rating_dict['mu'], sigma=rating_dict['sigma'])
+
+
+def save_json_via_sftp(ratings, comparison_counts, total_comparisons):
+    global sftp_matchmaker_client
+    if not is_connected():
+        create_ssh_matchmaker_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+    data = {
+        'ratings': [serialize_rating(r) for r in ratings],
+        'comparison_counts': comparison_counts.tolist(),
+        'total_comparisons': total_comparisons
+    }  
+    json_data = json.dumps(data)
+    with sftp_matchmaker_client.open(SSH_VIDEO_SKILL, 'w') as f:
+        f.write(json_data)
+
+
+def load_json_via_sftp():
+    global sftp_matchmaker_client
+    if not is_connected():
+        create_ssh_matchmaker_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+    with sftp_matchmaker_client.open(SSH_VIDEO_SKILL, 'r') as f:
+        data = json.load(f)
+    ratings = [deserialize_rating(r) for r in data['ratings']]
+    comparison_counts = np.array(data['comparison_counts'])
+    total_comparisons = data['total_comparisons']
+    return ratings, comparison_counts, total_comparisons
+
+
+def matchmaker_video(num_players, k_group=4):
+    trueskill_env = TrueSkill()
+    
+    ratings, comparison_counts, total_comparisons = load_json_via_sftp()
+
+    ratings = ratings[:num_players]
+    comparison_counts = comparison_counts[:num_players, :num_players]
+
+    selected_player = np.argmin(comparison_counts.sum(axis=1))
+
+    selected_trueskill_score = trueskill_env.expose(ratings[selected_player])
+    trueskill_scores = np.array([trueskill_env.expose(p) for p in ratings])
+    trueskill_diff = np.abs(trueskill_scores - selected_trueskill_score)
+    n = comparison_counts[selected_player]
+    ucb_scores = ucb_score(trueskill_diff, total_comparisons, n)
+
+    # Exclude self, select opponent with highest UCB score
+    ucb_scores[selected_player] = -float('inf') 
+
+    excluded_players_1 = [7, 10]
+    excluded_players_2 = [6, 8, 9]
+    excluded_players = excluded_players_1 + excluded_players_2
+    if selected_player in excluded_players_1:
+        for player in excluded_players:
+            ucb_scores[player] = -float('inf')
+    if selected_player in excluded_players_2:
+        for player in excluded_players_1:
+            ucb_scores[player] = -float('inf')
+    else:
+        excluded_ucb_scores = {player: ucb_scores[player] for player in excluded_players}
+        max_player = max(excluded_ucb_scores, key=excluded_ucb_scores.get)
+        if max_player in excluded_players_1:
+            for player in excluded_players:
+                if player != max_player:
+                    ucb_scores[player] = -float('inf')
+        else:
+            for player in excluded_players_1:
+                ucb_scores[player] = -float('inf')
+
+
+    opponents = np.argsort(ucb_scores)[-k_group + 1:].tolist()
+
+    # Group players
+    model_ids = [selected_player] + opponents
+
+    random.shuffle(model_ids)
+        
+    return model_ids

model/model_manager.py

@@ -0,0 +1,187 @@
+import concurrent.futures 
+import random
+import gradio as gr
+import requests, os
+import io, base64, json
+import spaces
+import torch
+from PIL import Image
+from openai import OpenAI
+from .models import IMAGE_GENERATION_MODELS, VIDEO_GENERATION_MODELS, load_pipeline
+from serve.upload import get_random_mscoco_prompt, get_random_video_prompt, get_ssh_random_video_prompt, get_ssh_random_image_prompt
+from serve.constants import SSH_CACHE_OPENSOURCE, SSH_CACHE_ADVANCE, SSH_CACHE_PIKA, SSH_CACHE_SORA, SSH_CACHE_IMAGE
+
+
+class ModelManager:
+    def __init__(self):
+        self.model_ig_list = IMAGE_GENERATION_MODELS
+        self.model_ie_list = [] #IMAGE_EDITION_MODELS
+        self.model_vg_list = VIDEO_GENERATION_MODELS
+        self.model_b2i_list = []
+        self.loaded_models = {}
+
+    def load_model_pipe(self, model_name):
+        if not model_name in self.loaded_models:
+            pipe = load_pipeline(model_name)
+            self.loaded_models[model_name] = pipe
+        else:
+            pipe = self.loaded_models[model_name]
+        return pipe
+    
+    @spaces.GPU(duration=120)
+    def generate_image_ig(self, prompt, model_name):
+        pipe = self.load_model_pipe(model_name)
+        if 'Stable-cascade' not in model_name:
+            result = pipe(prompt=prompt).images[0]
+        else:
+            prior, decoder = pipe
+            prior.enable_model_cpu_offload()
+            prior_output = prior(
+                prompt=prompt,
+                height=512,
+                width=512,
+                negative_prompt='',
+                guidance_scale=4.0,
+                num_images_per_prompt=1,
+                num_inference_steps=20
+            )
+            decoder.enable_model_cpu_offload()
+            result = decoder(
+                image_embeddings=prior_output.image_embeddings.to(torch.float16),
+                prompt=prompt,
+                negative_prompt='',
+                guidance_scale=0.0,
+                output_type="pil",
+                num_inference_steps=10
+            ).images[0]
+        return result
+
+    def generate_image_ig_api(self, prompt, model_name):
+        pipe = self.load_model_pipe(model_name)
+        result = pipe(prompt=prompt)
+        return result
+
+    def generate_image_ig_parallel_anony(self, prompt, model_A, model_B, model_C, model_D):
+        if model_A == "" and model_B == "" and model_C == "" and model_D == "":
+            from .matchmaker import matchmaker
+            not_run = [20,21,22, 25,26, 30] #12,13,14,15,16,17,18,19,20,21,22, #23,24,
+            model_ids = matchmaker(num_players=len(self.model_ig_list), not_run=not_run)
+            print(model_ids)
+            model_names = [self.model_ig_list[i] for i in model_ids]
+            print(model_names)
+        else:
+            model_names = [model_A, model_B, model_C, model_D]
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [executor.submit(self.generate_image_ig, prompt, model) if model.startswith("huggingface")
+                    else executor.submit(self.generate_image_ig_api, prompt, model) for model in model_names]
+            results = [future.result() for future in futures]
+
+        return results[0], results[1], results[2], results[3], \
+            model_names[0], model_names[1], model_names[2], model_names[3]
+
+    def generate_image_ig_cache_anony(self, model_A, model_B, model_C, model_D):
+        if model_A == "" and model_B == "" and model_C == "" and model_D == "":
+            from .matchmaker import matchmaker
+            not_run = [20,21,22]
+            model_ids = matchmaker(num_players=len(self.model_ig_list), not_run=not_run)
+            print(model_ids)
+            model_names = [self.model_ig_list[i] for i in model_ids]
+            print(model_names)
+        else:
+            model_names = [model_A, model_B, model_C, model_D]
+
+        root_dir = SSH_CACHE_IMAGE
+        local_dir = "./cache_image"
+        if not os.path.exists(local_dir):
+            os.makedirs(local_dir)
+        prompt, results = get_ssh_random_image_prompt(root_dir, local_dir, model_names)
+
+        return results[0], results[1], results[2], results[3], \
+            model_names[0], model_names[1], model_names[2], model_names[3], prompt
+
+    def generate_video_vg_parallel_anony(self, model_A, model_B, model_C, model_D):
+        if model_A == "" and model_B == "" and model_C == "" and model_D == "":
+            # model_names = random.sample([model for model in self.model_vg_list], 4)
+
+            from .matchmaker_video import matchmaker_video
+            model_ids = matchmaker_video(num_players=len(self.model_vg_list))
+            print(model_ids)
+            model_names = [self.model_vg_list[i] for i in model_ids]
+            print(model_names)
+        else:
+            model_names = [model_A, model_B, model_C, model_D]
+        
+        root_dir = SSH_CACHE_OPENSOURCE
+        for name in model_names:
+            if "Runway-Gen3" in name or "Runway-Gen2" in name or "Pika-v1.0" in name:
+                root_dir = SSH_CACHE_ADVANCE
+            elif "Pika-beta" in name:
+                root_dir = SSH_CACHE_PIKA      
+            elif "Sora" in name and "OpenSora" not in name: 
+                root_dir = SSH_CACHE_SORA  
+        
+        local_dir = "./cache_video"
+        if not os.path.exists(local_dir):
+            os.makedirs(local_dir)
+        prompt, results = get_ssh_random_video_prompt(root_dir, local_dir, model_names)
+        cache_dir = local_dir
+
+        return results[0], results[1], results[2], results[3], \
+            model_names[0], model_names[1], model_names[2], model_names[3], prompt, cache_dir
+
+    def generate_image_ig_museum_parallel_anony(self, model_A, model_B, model_C, model_D):
+        if model_A == "" and model_B == "" and model_C == "" and model_D == "":
+            # model_names = random.sample([model for model in self.model_ig_list], 4)
+
+            from .matchmaker import matchmaker
+            model_ids = matchmaker(num_players=len(self.model_ig_list))
+            print(model_ids)
+            model_names = [self.model_ig_list[i] for i in model_ids]
+            print(model_names)
+        else:
+            model_names = [model_A, model_B, model_C, model_D]
+
+        prompt = get_random_mscoco_prompt()
+        print(prompt)
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [executor.submit(self.generate_image_ig, prompt, model) if model.startswith("huggingface")
+                       else executor.submit(self.generate_image_ig_api, prompt, model) for model in model_names]
+            results = [future.result() for future in futures]
+
+        return results[0], results[1], results[2], results[3], \
+            model_names[0], model_names[1], model_names[2], model_names[3], prompt
+
+    def generate_image_ig_parallel(self, prompt, model_A, model_B):
+        model_names = [model_A, model_B]
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [executor.submit(self.generate_image_ig, prompt, model) if model.startswith("imagenhub")
+                       else executor.submit(self.generate_image_ig_api, prompt, model) for model in model_names]
+            results = [future.result() for future in futures]
+        return results[0], results[1]
+
+    @spaces.GPU(duration=200)
+    def generate_image_ie(self, textbox_source, textbox_target, textbox_instruct, source_image, model_name):
+        pipe = self.load_model_pipe(model_name)
+        result = pipe(src_image = source_image, src_prompt = textbox_source, target_prompt = textbox_target, instruct_prompt = textbox_instruct)
+        return result
+
+    def generate_image_ie_parallel(self, textbox_source, textbox_target, textbox_instruct, source_image, model_A, model_B):
+        model_names = [model_A, model_B]
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self.generate_image_ie, textbox_source, textbox_target, textbox_instruct, source_image,
+                                model) for model in model_names]
+            results = [future.result() for future in futures]
+        return results[0], results[1]
+
+    def generate_image_ie_parallel_anony(self, textbox_source, textbox_target, textbox_instruct, source_image, model_A, model_B):
+        if model_A == "" and model_B == "":
+            model_names = random.sample([model for model in self.model_ie_list], 2)
+        else:
+            model_names = [model_A, model_B]
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [executor.submit(self.generate_image_ie, textbox_source, textbox_target, textbox_instruct, source_image, model) for model in model_names]
+            results = [future.result() for future in futures]
+        return results[0], results[1], model_names[0], model_names[1]

model/model_registry.py

@@ -0,0 +1,70 @@
+from collections import namedtuple
+from typing import List
+
+ModelInfo = namedtuple("ModelInfo", ["simple_name", "link", "description"])
+model_info = {}
+
+def register_model_info(
+    full_names: List[str], simple_name: str, link: str, description: str
+):
+    info = ModelInfo(simple_name, link, description)
+
+    for full_name in full_names:
+        model_info[full_name] = info
+
+def get_model_info(name: str) -> ModelInfo:
+    if name in model_info:
+        return model_info[name]
+    else:
+        # To fix this, please use `register_model_info` to register your model
+        return ModelInfo(
+            name, "", "Register the description at fastchat/model/model_registry.py"
+        )
+
+def get_model_description_md(model_list):
+    model_description_md = """
+| | | | | | | | | | | |
+| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+"""
+    ct = 0
+    visited = set()
+    for i, name in enumerate(model_list):
+        model_source, model_name, model_type = name.split("_")
+        minfo = get_model_info(model_name)
+        if minfo.simple_name in visited:
+            continue
+        visited.add(minfo.simple_name)
+        # one_model_md = f"[{minfo.simple_name}]({minfo.link}): {minfo.description}"
+        one_model_md = f"{minfo.simple_name}"
+
+        if ct % 11 == 0:
+            model_description_md += "|"
+        model_description_md += f" {one_model_md} |"
+        if ct % 11 == 10:
+            model_description_md += "\n"
+        ct += 1
+    return model_description_md
+
+def get_video_model_description_md(model_list):
+    model_description_md = """
+| | | | | | |
+| ---- | ---- | ---- | ---- | ---- | ---- |
+"""
+    ct = 0
+    visited = set()
+    for i, name in enumerate(model_list):
+        model_source, model_name, model_type = name.split("_")
+        minfo = get_model_info(model_name)
+        if minfo.simple_name in visited:
+            continue
+        visited.add(minfo.simple_name)
+        # one_model_md = f"[{minfo.simple_name}]({minfo.link}): {minfo.description}"
+        one_model_md = f"{minfo.simple_name}"
+
+        if ct % 7 == 0:
+            model_description_md += "|"
+        model_description_md += f" {one_model_md} |"
+        if ct % 7 == 6:
+            model_description_md += "\n"
+        ct += 1
+    return model_description_md

model/models/__init__.py

@@ -0,0 +1,78 @@
+from .huggingface_models import load_huggingface_model
+from .replicate_api_models import load_replicate_model
+from .openai_api_models import load_openai_model
+from .other_api_models import load_other_model
+
+
+IMAGE_GENERATION_MODELS = [ 
+                            'replicate_SDXL_text2image',
+                            'replicate_SD-v3.0_text2image',
+                            'replicate_SD-v2.1_text2image',
+                            'replicate_SD-v1.5_text2image',
+                            'replicate_SDXL-Lightning_text2image',
+                            'replicate_Kandinsky-v2.0_text2image',
+                            'replicate_Kandinsky-v2.2_text2image',
+                            'replicate_Proteus-v0.2_text2image',
+                            'replicate_Playground-v2.0_text2image',
+                            'replicate_Playground-v2.5_text2image',
+                            'replicate_Dreamshaper-xl-turbo_text2image',
+                            'replicate_SDXL-Deepcache_text2image',
+                            'replicate_Openjourney-v4_text2image',
+                            'replicate_LCM-v1.5_text2image',
+                            'replicate_Realvisxl-v3.0_text2image',
+                            'replicate_Realvisxl-v2.0_text2image',
+                            'replicate_Pixart-Sigma_text2image',
+                            'replicate_SSD-1b_text2image',
+                            'replicate_Open-Dalle-v1.1_text2image',
+                            'replicate_Deepfloyd-IF_text2image',
+                            'huggingface_SD-turbo_text2image',
+                            'huggingface_SDXL-turbo_text2image',
+                            'huggingface_Stable-cascade_text2image',
+                            'openai_Dalle-2_text2image',
+                            'openai_Dalle-3_text2image',
+                            'other_Midjourney-v6.0_text2image',
+                            'other_Midjourney-v5.0_text2image',
+                            "replicate_FLUX.1-schnell_text2image",
+                            "replicate_FLUX.1-pro_text2image",
+                            "replicate_FLUX.1-dev_text2image",
+                            'other_Meissonic_text2image',
+                            "replicate_FLUX-1.1-pro_text2image",
+                            'replicate_SD-v3.5-large_text2image',
+                            'replicate_SD-v3.5-large-turbo_text2image',
+                            ]
+
+VIDEO_GENERATION_MODELS = ['replicate_Zeroscope-v2-xl_text2video',
+                            'replicate_Animate-Diff_text2video',
+                            'replicate_OpenSora_text2video',
+                            'replicate_LaVie_text2video',
+                            'replicate_VideoCrafter2_text2video',
+                            'replicate_Stable-Video-Diffusion_text2video',
+                            'other_Runway-Gen3_text2video',
+                            'other_Pika-beta_text2video',
+                            'other_Pika-v1.0_text2video',
+                            'other_Runway-Gen2_text2video',
+                            'other_Sora_text2video',
+                            'replicate_Cogvideox-5b_text2video',
+                            'other_KLing-v1.0_text2video',
+                           ]
+
+
+def load_pipeline(model_name):
+    """
+    Load a model pipeline based on the model name
+    Args:
+        model_name (str): The name of the model to load, should be of the form {source}_{name}_{type}
+    """
+    model_source, model_name, model_type = model_name.split("_")
+
+    if model_source == "replicate":
+        pipe = load_replicate_model(model_name, model_type)
+    elif model_source == "huggingface":
+        pipe = load_huggingface_model(model_name, model_type)
+    elif model_source == "openai":
+        pipe = load_openai_model(model_name, model_type)
+    elif model_source == "other":
+        pipe = load_other_model(model_name, model_type)
+    else:
+        raise ValueError(f"Model source {model_source} not supported")
+    return pipe

model/models/huggingface_models.py

@@ -0,0 +1,59 @@
+from diffusers import DiffusionPipeline
+from diffusers import AutoPipelineForText2Image
+from diffusers import StableCascadeDecoderPipeline, StableCascadePriorPipeline
+import torch
+
+
+def load_huggingface_model(model_name, model_type):
+    if model_name == "SD-turbo":
+        pipe = AutoPipelineForText2Image.from_pretrained("stabilityai/sd-turbo", torch_dtype=torch.float16, variant="fp16")
+        pipe = pipe.to("cuda")
+    elif model_name == "SDXL-turbo":
+        pipe = AutoPipelineForText2Image.from_pretrained("stabilityai/sdxl-turbo", torch_dtype=torch.float16, variant="fp16")
+        pipe = pipe.to("cuda")
+    elif model_name == "Stable-cascade":
+        prior = StableCascadePriorPipeline.from_pretrained("stabilityai/stable-cascade-prior", variant="bf16", torch_dtype=torch.bfloat16)
+        decoder = StableCascadeDecoderPipeline.from_pretrained("stabilityai/stable-cascade", variant="bf16", torch_dtype=torch.float16)
+        pipe = [prior, decoder]
+    else:
+        raise NotImplementedError
+    # if model_name == "SD-turbo":
+    #     pipe = AutoPipelineForText2Image.from_pretrained("stabilityai/sd-turbo")
+    # elif model_name == "SDXL-turbo":
+    #     pipe = AutoPipelineForText2Image.from_pretrained("stabilityai/sdxl-turbo")
+    # else:
+    #     raise NotImplementedError
+    # pipe = pipe.to("cpu")
+    return pipe
+
+
+if __name__ == "__main__":
+    # for name in ["SD-turbo", "SDXL-turbo"]: #"SD-turbo", "SDXL-turbo"
+    #     pipe = load_huggingface_model(name, "text2image")
+
+    # for name in ["IF-I-XL-v1.0"]:
+    #     pipe = load_huggingface_model(name, 'text2image')
+    # pipe = DiffusionPipeline.from_pretrained("DeepFloyd/IF-I-XL-v1.0", variant="fp16", torch_dtype=torch.float16)
+
+    prompt = 'draw a tiger'
+    pipe = load_huggingface_model('Stable-cascade', "text2image")
+    prior, decoder = pipe
+    prior.enable_model_cpu_offload()
+    prior_output = prior(
+        prompt=prompt,
+        height=512,
+        width=512,
+        negative_prompt='',
+        guidance_scale=4.0,
+        num_images_per_prompt=1,
+        num_inference_steps=20
+    )
+    decoder.enable_model_cpu_offload()
+    result = decoder(
+        image_embeddings=prior_output.image_embeddings.to(torch.float16),
+        prompt=prompt,
+        negative_prompt='',
+        guidance_scale=0.0,
+        output_type="pil",
+        num_inference_steps=10
+    ).images[0]

model/models/openai_api_models.py

@@ -0,0 +1,57 @@
+from openai import OpenAI
+from PIL import Image
+import requests
+import io
+import os
+import base64
+
+
+class OpenaiModel():
+    def __init__(self, model_name, model_type):
+        self.model_name = model_name
+        self.model_type = model_type
+    
+    def __call__(self, *args, **kwargs):
+        if self.model_type == "text2image":
+            assert "prompt" in kwargs, "prompt is required for text2image model"
+
+            client = OpenAI()
+
+            if 'Dalle-3' in self.model_name:
+                client = OpenAI()
+                response = client.images.generate(
+                    model="dall-e-3",
+                    prompt=kwargs["prompt"],
+                    size="1024x1024",
+                    quality="standard",
+                    n=1,
+                    )
+            elif 'Dalle-2' in self.model_name:
+                client = OpenAI()
+                response = client.images.generate(
+                    model="dall-e-2",
+                    prompt=kwargs["prompt"],
+                    size="512x512",
+                    quality="standard",
+                    n=1,
+                    )
+            else:
+                raise NotImplementedError
+
+            result_url = response.data[0].url
+            response = requests.get(result_url)
+            result = Image.open(io.BytesIO(response.content))
+            return result
+        else:
+            raise ValueError("model_type must be text2image or image2image")
+
+
+def load_openai_model(model_name, model_type):
+    return OpenaiModel(model_name, model_type)
+
+
+if __name__ == "__main__":
+    pipe = load_openai_model('Dalle-3', 'text2image')
+    result = pipe(prompt='draw a tiger')
+    print(result)
+

model/models/other_api_models.py

@@ -0,0 +1,91 @@
+import requests
+import json
+import os
+from PIL import Image
+import io, time
+
+
+class OtherModel():
+    def __init__(self, model_name, model_type):
+        self.model_name = model_name
+        self.model_type = model_type
+        self.image_url = "https://www.xdai.online/mj/submit/imagine"
+        self.key = os.environ.get('MIDJOURNEY_KEY')
+        self.get_image_url = "https://www.xdai.online/mj/image/"
+        self.repeat_num = 5
+    
+    def __call__(self, *args, **kwargs):
+        if self.model_type == "text2image":
+            assert "prompt" in kwargs, "prompt is required for text2image model"
+            if self.model_name == "Midjourney-v6.0":
+                data = {
+                    "base64Array": [],
+                    "notifyHook": "",
+                    "prompt": "{} --v 6.0".format(kwargs["prompt"]),
+                    "state": "",
+                    "botType": "MID_JOURNEY",
+                }
+            elif self.model_name == "Midjourney-v5.0":
+                data = {
+                    "base64Array": [],
+                    "notifyHook": "",
+                    "prompt": "{} --v 5.0".format(kwargs["prompt"]),
+                    "state": "",
+                    "botType": "MID_JOURNEY",
+                }
+            else:
+                raise NotImplementedError
+            
+            headers = {
+                "Authorization": "Bearer {}".format(self.key),
+                "Content-Type": "application/json"
+            }
+            while 1:
+                response = requests.post(self.image_url, data=json.dumps(data), headers=headers)
+                if response.status_code == 200:
+                    print("Submit success!")
+                    response_json = json.loads(response.content.decode('utf-8'))
+                    img_id = response_json["result"]
+                    result_url = self.get_image_url + img_id
+                    print(result_url)
+                    self.repeat_num = 800
+                    while 1:
+                        time.sleep(1)
+                        img_response = requests.get(result_url)
+                        if img_response.status_code == 200:
+                            result = Image.open(io.BytesIO(img_response.content))
+                            width, height = result.size  
+                            new_width = width // 2  
+                            new_height = height // 2  
+                            result = result.crop((0, 0, new_width, new_height))  
+                            self.repeat_num = 5
+                            return result
+                        else:
+                            self.repeat_num = self.repeat_num - 1
+                            if self.repeat_num == 0:
+                                raise ValueError("Image request failed.")
+                            continue
+                        
+                else:
+                    self.repeat_num = self.repeat_num - 1
+                    if self.repeat_num == 0:
+                        raise ValueError("API request failed.")
+                    continue
+        if self.model_type == "text2video":
+            assert "prompt" in kwargs, "prompt is required for text2video model"
+
+        else:
+            raise ValueError("model_type must be text2image")
+
+
+def load_other_model(model_name, model_type):
+    return OtherModel(model_name, model_type)
+
+if __name__ == "__main__":
+    import http.client
+    import json
+
+    pipe = load_other_model("Midjourney-v5.0", "text2image")
+    result = pipe(prompt="An Impressionist illustration depicts a river winding through a meadow")
+    print(result)
+    exit()

model/models/replicate_api_models.py

@@ -0,0 +1,195 @@
+import replicate
+from PIL import Image
+import requests
+import io
+import os
+import base64
+
+Replicate_MODEl_NAME_MAP = {
+    "SDXL": "stability-ai/sdxl:7762fd07cf82c948538e41f63f77d685e02b063e37e496e96eefd46c929f9bdc",
+    "SD-v3.0": "stability-ai/stable-diffusion-3",
+    "SD-v2.1": "stability-ai/stable-diffusion:ac732df83cea7fff18b8472768c88ad041fa750ff7682a21affe81863cbe77e4",
+    "SD-v1.5": "stability-ai/stable-diffusion:b3d14e1cd1f9470bbb0bb68cac48e5f483e5be309551992cc33dc30654a82bb7",
+    "SDXL-Lightning": "bytedance/sdxl-lightning-4step:5f24084160c9089501c1b3545d9be3c27883ae2239b6f412990e82d4a6210f8f",
+    "Kandinsky-v2.0": "ai-forever/kandinsky-2:3c6374e7a9a17e01afe306a5218cc67de55b19ea536466d6ea2602cfecea40a9",
+    "Kandinsky-v2.2": "ai-forever/kandinsky-2.2:ad9d7879fbffa2874e1d909d1d37d9bc682889cc65b31f7bb00d2362619f194a",
+    "Proteus-v0.2": "lucataco/proteus-v0.2:06775cd262843edbde5abab958abdbb65a0a6b58ca301c9fd78fa55c775fc019",
+    "Playground-v2.0": "playgroundai/playground-v2-1024px-aesthetic:42fe626e41cc811eaf02c94b892774839268ce1994ea778eba97103fe1ef51b8",
+    "Playground-v2.5": "playgroundai/playground-v2.5-1024px-aesthetic:a45f82a1382bed5c7aeb861dac7c7d191b0fdf74d8d57c4a0e6ed7d4d0bf7d24",
+    "Dreamshaper-xl-turbo": "lucataco/dreamshaper-xl-turbo:0a1710e0187b01a255302738ca0158ff02a22f4638679533e111082f9dd1b615",
+    "SDXL-Deepcache": "lucataco/sdxl-deepcache:eaf678fb34006669e9a3c6dd5971e2279bf20ee0adeced464d7b6d95de16dc93",
+    "Openjourney-v4": "prompthero/openjourney:ad59ca21177f9e217b9075e7300cf6e14f7e5b4505b87b9689dbd866e9768969",
+    "LCM-v1.5": "fofr/latent-consistency-model:683d19dc312f7a9f0428b04429a9ccefd28dbf7785fef083ad5cf991b65f406f",
+    "Realvisxl-v3.0": "fofr/realvisxl-v3:33279060bbbb8858700eb2146350a98d96ef334fcf817f37eb05915e1534aa1c", 
+    
+    "Realvisxl-v2.0": "lucataco/realvisxl-v2.0:7d6a2f9c4754477b12c14ed2a58f89bb85128edcdd581d24ce58b6926029de08",
+    "Pixart-Sigma": "cjwbw/pixart-sigma:5a54352c99d9fef467986bc8f3a20205e8712cbd3df1cbae4975d6254c902de1",
+    "SSD-1b": "lucataco/ssd-1b:b19e3639452c59ce8295b82aba70a231404cb062f2eb580ea894b31e8ce5bbb6",
+    "Open-Dalle-v1.1": "lucataco/open-dalle-v1.1:1c7d4c8dec39c7306df7794b28419078cb9d18b9213ab1c21fdc46a1deca0144",
+    "Deepfloyd-IF": "andreasjansson/deepfloyd-if:fb84d659df149f4515c351e394d22222a94144aa1403870c36025c8b28846c8d",
+
+    "Zeroscope-v2-xl": "anotherjesse/zeroscope-v2-xl:9f747673945c62801b13b84701c783929c0ee784e4748ec062204894dda1a351",
+    # "Damo-Text-to-Video": "cjwbw/damo-text-to-video:1e205ea73084bd17a0a3b43396e49ba0d6bc2e754e9283b2df49fad2dcf95755",
+    "Animate-Diff": "lucataco/animate-diff:beecf59c4aee8d81bf04f0381033dfa10dc16e845b4ae00d281e2fa377e48a9f",
+    "OpenSora": "camenduru/open-sora:8099e5722ba3d5f408cd3e696e6df058137056268939337a3fbe3912e86e72ad",
+    "LaVie": "cjwbw/lavie:0bca850c4928b6c30052541fa002f24cbb4b677259c461dd041d271ba9d3c517",
+    "VideoCrafter2": "lucataco/video-crafter:7757c5775e962c618053e7df4343052a21075676d6234e8ede5fa67c9e43bce0",
+    "Stable-Video-Diffusion": "sunfjun/stable-video-diffusion:d68b6e09eedbac7a49e3d8644999d93579c386a083768235cabca88796d70d82",
+    "FLUX.1-schnell": "black-forest-labs/flux-schnell",
+    "FLUX.1-pro": "black-forest-labs/flux-pro",
+    "FLUX.1-dev": "black-forest-labs/flux-dev",
+    "FLUX-1.1-pro": "black-forest-labs/flux-1.1-pro",
+    "SD-v3.5-large":  "stability-ai/stable-diffusion-3.5-large",
+    "SD-v3.5-large-turbo": "stability-ai/stable-diffusion-3.5-large-turbo",
+    }
+
+
+class ReplicateModel():
+    def __init__(self, model_name, model_type):
+        self.model_name = model_name
+        self.model_type = model_type
+    
+    def __call__(self, *args, **kwargs):
+        if self.model_type == "text2image":
+            assert "prompt" in kwargs, "prompt is required for text2image model"
+            output  = replicate.run(
+                                    f"{Replicate_MODEl_NAME_MAP[self.model_name]}",
+                                    input={
+                                            "width": 512,
+                                            "height": 512,
+                                            "prompt": kwargs["prompt"]
+                                        },
+                                )
+            if 'Openjourney' in self.model_name:
+                for item in output:
+                    result_url = item
+                    break
+            elif isinstance(output, list):
+                result_url = output[0]
+            else:
+                result_url = output
+            print(self.model_name, result_url)
+            response = requests.get(result_url)
+            result = Image.open(io.BytesIO(response.content))
+            return result
+
+        elif self.model_type == "text2video":
+            assert "prompt" in kwargs, "prompt is required for text2image model"
+            if self.model_name == "Zeroscope-v2-xl":
+                input = {
+                            "fps": 24,
+                            "width": 512,
+                            "height": 512,
+                            "prompt": kwargs["prompt"],
+                            "guidance_scale": 17.5,
+                            # "negative_prompt": "very blue, dust, noisy, washed out, ugly, distorted, broken",
+                            "num_frames": 48,
+                        }
+            elif self.model_name == "Damo-Text-to-Video":
+                input={
+                    "fps": 8,
+                    "prompt": kwargs["prompt"],
+                    "num_frames": 16,
+                    "num_inference_steps": 50
+                }
+            elif self.model_name == "Animate-Diff":
+                input={
+                    "path": "toonyou_beta3.safetensors",
+                    "seed": 255224557,
+                    "steps": 25,
+                    "prompt": kwargs["prompt"],
+                    "n_prompt": "badhandv4, easynegative, ng_deepnegative_v1_75t, verybadimagenegative_v1.3, bad-artist, bad_prompt_version2-neg, teeth",
+                    "motion_module": "mm_sd_v14",
+                    "guidance_scale": 7.5
+                }
+            elif self.model_name == "OpenSora":
+                input={
+                    "seed": 1234,
+                    "prompt": kwargs["prompt"],
+                }
+            elif self.model_name == "LaVie":
+                input={
+                    "width": 512,
+                    "height": 512,
+                    "prompt": kwargs["prompt"],
+                    "quality": 9,
+                    "video_fps": 8,
+                    "interpolation": False,
+                    "sample_method": "ddpm",
+                    "guidance_scale": 7,
+                    "super_resolution": False,
+                    "num_inference_steps": 50
+                }
+            elif self.model_name == "VideoCrafter2":
+                input={
+                    "fps": 24,
+                    "seed": 64045,
+                    "steps": 40,
+                    "width": 512,
+                    "height": 512,
+                    "prompt": kwargs["prompt"],
+                }
+            elif self.model_name == "Stable-Video-Diffusion":
+                text2image_name = "SD-v2.1"
+                output = replicate.run(
+                                        f"{Replicate_MODEl_NAME_MAP[text2image_name]}",
+                                        input={
+                                                "width": 512,
+                                                "height": 512,
+                                                "prompt": kwargs["prompt"]
+                                            },
+                                    )
+                if isinstance(output, list):
+                    image_url = output[0]
+                else:
+                    image_url = output
+                print(image_url)
+
+                input={
+                    "cond_aug": 0.02,
+                    "decoding_t": 14,
+                    "input_image": "{}".format(image_url),
+                    "video_length": "14_frames_with_svd",
+                    "sizing_strategy": "maintain_aspect_ratio",
+                    "motion_bucket_id": 127,
+                    "frames_per_second": 6
+                }
+
+            output  = replicate.run(
+                                    f"{Replicate_MODEl_NAME_MAP[self.model_name]}",
+                                    input=input,
+                                )
+            if isinstance(output, list):
+                result_url = output[0]
+            else:
+                result_url = output
+            print(self.model_name)
+            print(result_url)
+            # response = requests.get(result_url)
+            # result = Image.open(io.BytesIO(response.content))
+
+        #     for event in handler.iter_events(with_logs=True):
+        #         if isinstance(event, fal_client.InProgress):
+        #             print('Request in progress')
+        #             print(event.logs)
+
+        #     result = handler.get()
+        #     print("result video: ====")
+        #     print(result)
+        #     result_url = result['video']['url']
+        #     return result_url
+            return result_url
+        else:
+            raise ValueError("model_type must be text2image or image2image")
+
+
+def load_replicate_model(model_name, model_type):
+    return ReplicateModel(model_name, model_type)
+
+
+if __name__ == "__main__":
+    model_name = 'replicate_zeroscope-v2-xl_text2video'
+    model_source, model_name, model_type = model_name.split("_")
+    pipe = load_replicate_model(model_name, model_type)
+    prompt = "Clown fish swimming in a coral reef, beautiful, 8k, perfect, award winning, national geographic"
+    result = pipe(prompt=prompt)

serve/Ksort.py

@@ -0,0 +1,411 @@
+import gradio as gr
+from PIL import Image, ImageDraw, ImageFont, ImageOps
+import os
+from .constants import KSORT_IMAGE_DIR
+from .constants import COLOR1, COLOR2, COLOR3, COLOR4
+from .vote_utils import save_any_image
+from .utils import disable_btn, enable_btn, invisible_btn
+from .upload import create_remote_directory, upload_ssh_all, upload_ssh_data
+import json
+
+
+def reset_level(Top_btn):
+    if Top_btn == "Top 1":
+        level = 0
+    elif Top_btn == "Top 2":
+        level = 1
+    elif Top_btn == "Top 3":
+        level = 2
+    elif Top_btn == "Top 4":
+        level = 3
+    return level
+
+
+def reset_rank(windows, rank, vote_level):
+    if windows == "Model A":
+        rank[0] = vote_level
+    elif windows == "Model B":
+        rank[1] = vote_level
+    elif windows == "Model C":
+        rank[2] = vote_level
+    elif windows == "Model D":
+        rank[3] = vote_level
+    return rank
+
+
+def reset_btn_rank(windows, rank, btn, vote_level):
+    if windows == "Model A" and btn == "1":
+        rank[0] = 0
+    elif windows == "Model A" and btn == "2":
+        rank[0] = 1
+    elif windows == "Model A" and btn == "3":
+        rank[0] = 2
+    elif windows == "Model A" and btn == "4":
+        rank[0] = 3
+    elif windows == "Model B" and btn == "1":
+        rank[1] = 0
+    elif windows == "Model B" and btn == "2":
+        rank[1] = 1
+    elif windows == "Model B" and btn == "3":
+        rank[1] = 2
+    elif windows == "Model B" and btn == "4":
+        rank[1] = 3
+    elif windows == "Model C" and btn == "1":
+        rank[2] = 0
+    elif windows == "Model C" and btn == "2":
+        rank[2] = 1
+    elif windows == "Model C" and btn == "3":
+        rank[2] = 2
+    elif windows == "Model C" and btn == "4":
+        rank[2] = 3
+    elif windows == "Model D" and btn == "1":
+        rank[3] = 0
+    elif windows == "Model D" and btn == "2":
+        rank[3] = 1
+    elif windows == "Model D" and btn == "3":
+        rank[3] = 2
+    elif windows == "Model D" and btn == "4":
+        rank[3] = 3
+    if btn == "1":
+        vote_level = 0
+    elif btn == "2":
+        vote_level = 1
+    elif btn == "3":
+        vote_level = 2
+    elif btn == "4":
+        vote_level = 3
+    return (rank, vote_level)
+
+
+def reset_vote_text(rank):
+    rank_str = ""
+    for i in range(len(rank)):
+        if rank[i] == None:
+            rank_str = rank_str + str(rank[i])
+        else:
+            rank_str = rank_str + str(rank[i]+1)
+        rank_str = rank_str + "  "
+    return rank_str
+
+
+def clear_rank(rank, vote_level):
+    for i in range(len(rank)):
+        rank[i] = None
+    vote_level = 0
+    return rank, vote_level
+
+
+def revote_windows(generate_ig0, generate_ig1, generate_ig2, generate_ig3, rank, vote_level):
+    for i in range(len(rank)):
+        rank[i] = None
+    vote_level = 0
+    return generate_ig0, generate_ig1, generate_ig2, generate_ig3, rank, vote_level
+
+
+def reset_submit(rank):
+    for i in range(len(rank)):
+        if rank[i] == None:
+            return disable_btn
+    return enable_btn
+
+
+def reset_mode(mode):
+
+    if mode == "Best":
+        return (gr.update(visible=False, interactive=False),) * 5 + \
+                (gr.update(visible=True, interactive=True),) * 16 + \
+                (gr.update(visible=True, interactive=True),) * 3 + \
+                (gr.Textbox(value="Rank", visible=False, interactive=False),)
+    elif mode == "Rank":
+        return (gr.update(visible=True, interactive=True),) * 5 + \
+                (gr.update(visible=False, interactive=False),) * 16 + \
+                (gr.update(visible=True, interactive=False),) * 2 + \
+                (gr.update(visible=True, interactive=True),) + \
+                (gr.Textbox(value="Best", visible=False, interactive=False),)
+    else:
+        raise ValueError("Undefined mode")
+
+
+def reset_chatbot(mode, generate_ig0, generate_ig1, generate_ig2, generate_ig3):
+    return generate_ig0, generate_ig1, generate_ig2, generate_ig3
+
+
+def get_json_filename(conv_id):
+    output_dir = f'{KSORT_IMAGE_DIR}/{conv_id}/json/'
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    output_file = os.path.join(output_dir, "information.json")
+    # name = os.path.join(KSORT_IMAGE_DIR, f"{conv_id}/json/information.json")
+    print(output_file)
+    return output_file
+
+
+def get_img_filename(conv_id, i):
+    output_dir = f'{KSORT_IMAGE_DIR}/{conv_id}/image/'
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    output_file = os.path.join(output_dir, f"{i}.jpg")
+    print(output_file)
+    return output_file
+
+
+def vote_submit(states, textbox, rank, request: gr.Request):
+    conv_id = states[0].conv_id
+
+    for i in range(len(states)):
+        output_file = get_img_filename(conv_id, i)
+        save_any_image(states[i].output, output_file)
+    with open(get_json_filename(conv_id), "a") as fout:
+        data = {
+            "models_name": [x.model_name for x in states],
+            "img_rank": [x for x in rank],
+            "prompt": [textbox],
+        }
+        fout.write(json.dumps(data) + "\n")
+
+
+def vote_ssh_submit(states, textbox, rank, user_name, user_institution):
+    conv_id = states[0].conv_id
+    output_dir = create_remote_directory(conv_id)
+    # upload_image(states, output_dir)
+
+    data = {
+        "models_name": [x.model_name for x in states],
+        "img_rank": [x for x in rank],
+        "prompt": [textbox],
+        "user_info": {"name": [user_name], "institution": [user_institution]}, 
+    }
+    output_file = os.path.join(output_dir, "result.json")
+    # upload_informance(data, output_file)
+    upload_ssh_all(states, output_dir, data, output_file)
+
+    from .update_skill import update_skill
+    update_skill(rank, [x.model_name for x in states])
+
+
+def vote_video_ssh_submit(states, textbox, prompt_path, rank, user_name, user_institution):
+    conv_id = states[0].conv_id
+    output_dir = create_remote_directory(conv_id, video=True)
+
+    data = {
+        "models_name": [x.model_name for x in states],
+        "video_rank": [x for x in rank],
+        "prompt": [textbox],
+        "prompt_path": [prompt_path],
+        "video_path": [x.output for x in states],
+        "user_info": {"name": [user_name], "institution": [user_institution]}, 
+    }
+    output_file = os.path.join(output_dir, "result.json")
+
+    upload_ssh_data(data, output_file)
+
+    from .update_skill_video import update_skill_video
+    update_skill_video(rank, [x.model_name for x in states])
+
+
+def submit_response_igm(
+    state0, state1, state2, state3, model_selector0, model_selector1, model_selector2, model_selector3, textbox, rank, user_name, user_institution, request: gr.Request
+):
+    # vote_submit([state0, state1, state2, state3], textbox, rank, request)
+    vote_ssh_submit([state0, state1, state2, state3], textbox, rank, user_name, user_institution)
+    if model_selector0 == "":
+        return (disable_btn,) * 6 + (
+            gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True),
+            gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True),
+            gr.Markdown(f"### Model C: {state2.model_name.split('_')[1]}", visible=True),
+            gr.Markdown(f"### Model D: {state3.model_name.split('_')[1]}", visible=True)
+            ) + (disable_btn,)
+    else:
+        return (disable_btn,) * 6 + (
+            gr.Markdown(state0.model_name, visible=True),
+            gr.Markdown(state1.model_name, visible=True),
+            gr.Markdown(state2.model_name, visible=True),
+            gr.Markdown(state3.model_name, visible=True)
+            ) + (disable_btn,)
+
+
+def submit_response_vg(
+    state0, state1, state2, state3, model_selector0, model_selector1, model_selector2, model_selector3, textbox, prompt_path, rank, user_name, user_institution, request: gr.Request
+):
+    vote_video_ssh_submit([state0, state1, state2, state3], textbox, prompt_path, rank, user_name, user_institution)
+    if model_selector0 == "":
+        return (disable_btn,) * 6 + (
+            gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True),
+            gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True),
+            gr.Markdown(f"### Model C: {state2.model_name.split('_')[1]}", visible=True),
+            gr.Markdown(f"### Model D: {state3.model_name.split('_')[1]}", visible=True)
+            ) + (disable_btn,)
+    else:
+        return (disable_btn,) * 6 + (
+            gr.Markdown(state0.model_name, visible=True),
+            gr.Markdown(state1.model_name, visible=True),
+            gr.Markdown(state2.model_name, visible=True),
+            gr.Markdown(state3.model_name, visible=True)
+            ) + (disable_btn,)
+
+
+def submit_response_rank_igm(
+    state0, state1, state2, state3, model_selector0, model_selector1, model_selector2, model_selector3, textbox, rank, right_vote_text, user_name, user_institution, request: gr.Request
+):
+    print(rank)
+    if right_vote_text == "right":
+        # vote_submit([state0, state1, state2, state3], textbox, rank, request)
+        vote_ssh_submit([state0, state1, state2, state3], textbox, rank, user_name, user_institution)
+        if model_selector0 == "":
+            return (disable_btn,) * 16 + (disable_btn,) * 3 + ("wrong",) + (
+                gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True),
+                gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True),
+                gr.Markdown(f"### Model C: {state2.model_name.split('_')[1]}", visible=True),
+                gr.Markdown(f"### Model D: {state3.model_name.split('_')[1]}", visible=True)
+                )
+        else:
+            return (disable_btn,) * 16 + (disable_btn,) * 3 + ("wrong",) + (
+                gr.Markdown(state0.model_name, visible=True),
+                gr.Markdown(state1.model_name, visible=True),
+                gr.Markdown(state2.model_name, visible=True),
+                gr.Markdown(state3.model_name, visible=True)
+                )
+    else:
+        return (enable_btn,) * 16 + (enable_btn,) * 3 + ("wrong",) + (gr.Markdown("", visible=False),) * 4
+
+
+def submit_response_rank_vg(
+    state0, state1, state2, state3, model_selector0, model_selector1, model_selector2, model_selector3, textbox, prompt_path, rank, right_vote_text, user_name, user_institution, request: gr.Request
+):
+    print(rank)
+    if right_vote_text == "right":
+        vote_video_ssh_submit([state0, state1, state2, state3], textbox, prompt_path, rank, user_name, user_institution)
+        if model_selector0 == "":
+            return (disable_btn,) * 16 + (disable_btn,) * 3 + ("wrong",) + (
+                gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True),
+                gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True),
+                gr.Markdown(f"### Model C: {state2.model_name.split('_')[1]}", visible=True),
+                gr.Markdown(f"### Model D: {state3.model_name.split('_')[1]}", visible=True)
+                )
+        else:
+            return (disable_btn,) * 16 + (disable_btn,) * 3 + ("wrong",) + (
+                gr.Markdown(state0.model_name, visible=True),
+                gr.Markdown(state1.model_name, visible=True),
+                gr.Markdown(state2.model_name, visible=True),
+                gr.Markdown(state3.model_name, visible=True)
+                )
+    else:
+        return (enable_btn,) * 16 + (enable_btn,) * 3 + ("wrong",) + (gr.Markdown("", visible=False),) * 4
+
+
+def text_response_rank_igm(generate_ig0, generate_ig1, generate_ig2, generate_ig3, Top1_text, Top2_text, Top3_text, Top4_text, vote_textbox):
+    rank_list = [char for char in vote_textbox if char.isdigit()]
+    generate_ig = [generate_ig0, generate_ig1, generate_ig2, generate_ig3]
+    chatbot = []
+    rank = [None, None, None, None]
+    if len(rank_list) != 4:
+        return generate_ig + ["error rank"] + ["wrong"] + [rank]
+    for num in range(len(rank_list)):
+        if rank_list[num] in ['1', '2', '3', '4']:
+            base_image = Image.fromarray(generate_ig[num]).convert("RGBA")
+            base_image = base_image.resize((512, 512), Image.ANTIALIAS)
+            if rank_list[num] == '1':
+                border_color = COLOR1  
+            elif rank_list[num] == '2':
+                border_color = COLOR2
+            elif rank_list[num] == '3':
+                border_color = COLOR3  
+            elif rank_list[num] == '4':
+                border_color = COLOR4 
+            border_size = 10  # Size of the border
+            base_image = ImageOps.expand(base_image, border=border_size, fill=border_color)
+
+            draw = ImageDraw.Draw(base_image)
+            font = ImageFont.truetype("./serve/Arial.ttf", 66)
+            text_position = (180, 25)
+            if rank_list[num] == '1':
+                text_color = COLOR1 
+                draw.text(text_position, Top1_text, font=font, fill=text_color)
+            elif rank_list[num] == '2':
+                text_color = COLOR2  
+                draw.text(text_position, Top2_text, font=font, fill=text_color)
+            elif rank_list[num] == '3':
+                text_color = COLOR3  
+                draw.text(text_position, Top3_text, font=font, fill=text_color)
+            elif rank_list[num] == '4':
+                text_color = COLOR4  
+                draw.text(text_position, Top4_text, font=font, fill=text_color)
+            base_image = base_image.convert("RGB")
+            chatbot.append(base_image.copy())
+        else:
+            return generate_ig + ["error rank"] + ["wrong"] + [rank]
+    rank_str = ""
+    for str_num in rank_list:
+        rank_str = rank_str + str_num
+        rank_str = rank_str + "  "
+    rank = [int(x) for x in rank_list]
+    
+    return chatbot + [rank_str] + ["right"] + [rank]
+
+
+def text_response_rank_vg(vote_textbox):
+    rank_list = [char for char in vote_textbox if char.isdigit()]
+    rank = [None, None, None, None]
+    if len(rank_list) != 4:
+        return ["error rank"] + ["wrong"] + [rank]
+    for num in range(len(rank_list)):
+        if rank_list[num] in ['1', '2', '3', '4']:
+            continue
+        else:
+            return ["error rank"] + ["wrong"] + [rank]
+    rank_str = ""
+    for str_num in rank_list:
+        rank_str = rank_str + str_num
+        rank_str = rank_str + "  "
+    rank = [int(x) for x in rank_list]
+    
+    return [rank_str] + ["right"] + [rank]
+
+
+def add_foreground(image, vote_level, Top1_text, Top2_text, Top3_text, Top4_text):
+    base_image = Image.fromarray(image).convert("RGBA")
+    base_image = base_image.resize((512, 512), Image.ANTIALIAS)
+    if vote_level == 0:
+        border_color = COLOR1  
+    elif vote_level == 1:
+        border_color = COLOR2
+    elif vote_level == 2:
+        border_color = COLOR3 
+    elif vote_level == 3:
+        border_color = COLOR4
+    border_size = 10  # Size of the border
+    base_image = ImageOps.expand(base_image, border=border_size, fill=border_color)
+
+    draw = ImageDraw.Draw(base_image)
+    font = ImageFont.truetype("./serve/Arial.ttf", 66)
+    
+    text_position = (180, 25)
+    if vote_level == 0:
+        text_color = COLOR1 
+        draw.text(text_position, Top1_text, font=font, fill=text_color)
+    elif vote_level == 1:
+        text_color = COLOR2  
+        draw.text(text_position, Top2_text, font=font, fill=text_color)
+    elif vote_level == 2:
+        text_color = COLOR3 
+        draw.text(text_position, Top3_text, font=font, fill=text_color)
+    elif vote_level == 3:
+        text_color = COLOR4  
+        draw.text(text_position, Top4_text, font=font, fill=text_color)
+
+    base_image = base_image.convert("RGB")
+    return base_image
+
+
+def add_green_border(image):
+    border_color = (0, 255, 0)  # RGB for green
+    border_size = 10  # Size of the border
+    img_with_border = ImageOps.expand(image, border=border_size, fill=border_color)
+    return img_with_border
+
+
+def check_textbox(textbox):
+    if textbox=="":
+        return False
+    else:
+        return True

serve/button.css

@@ -0,0 +1,24 @@
+/* style.css */
+.custom-button {
+    background-color: greenyellow; /* 背景颜色 */
+    color: white;            /* 文字颜色 */
+    border: none;            /* 无边框 */
+    padding: 10px 20px;      /* 内边距 */
+    text-align: center;      /* 文本居中 */
+    text-decoration: none;   /* 无下划线 */
+    display: inline-block;   /* 行内块 */
+    font-size: 16px;         /* 字体大小 */
+    margin: 4px 2px;         /* 外边距 */
+    cursor: pointer;         /* 鼠标指针 */
+    border-radius: 5px;      /* 圆角边框 */
+}
+
+.custom-button:hover {
+    background-color: darkgreen; /* 悬停时的背景颜色 */
+}
+/* css = """
+#warning {background: red;} 
+
+.feedback {font-size: 24px !important;}
+.feedback textarea {font-size: 24px !important;}
+""" */

serve/constants.py

@@ -0,0 +1,63 @@
+import os
+
+LOGDIR = os.getenv("LOGDIR", "./ksort-logs/vote_log")
+IMAGE_DIR = os.getenv("IMAGE_DIR", f"{LOGDIR}/images")
+KSORT_IMAGE_DIR = os.getenv("KSORT_IMAGE_DIR", f"{LOGDIR}/ksort_images")
+VIDEO_DIR = os.getenv("VIDEO_DIR", f"{LOGDIR}/videos")
+
+SERVER_PORT = os.getenv("SERVER_PORT", 7860)
+ROOT_PATH = os.getenv("ROOT_PATH", None)
+ELO_RESULTS_DIR = os.getenv("ELO_RESULTS_DIR", "./arena_elo/results/latest")
+
+LOG_SERVER = os.getenv("LOG_SERVER", "http://127.0.0.1:22005")
+LOG_SERVER_SUBDOMAIN = os.getenv("LOG_SERVER_SUBDOMAIN", "/ksort-logs")
+LOG_SERVER_ADDR = os.getenv("LOG_SERVER_ADDR", f"{LOG_SERVER}{LOG_SERVER_SUBDOMAIN}")
+# LOG SERVER API ENDPOINTS
+APPEND_JSON = "append_json"
+SAVE_IMAGE = "save_image"
+SAVE_VIDEO = "save_video"
+SAVE_LOG = "save_log"
+
+SSH_MSCOCO = os.getenv("SSH_MSCOCO", "/root/MSCOCO_prompt")
+
+
+SSH_SERVER = os.getenv("SSH_SERVER", "default_value")
+SSH_PASSWORD = os.getenv("SSH_PASSWORD", "default_value")
+SSH_PORT = os.getenv("SSH_PORT", "default_value")
+SSH_USER = os.getenv("SSH_USER", "default_value")
+SSH_LOG = os.getenv("SSH_LOG", "/home/zhendongucb/ksort/ksort_log/log")
+SSH_VIDEO_LOG = os.getenv("SSH_LOG", "/home/zhendongucb/ksort/ksort_log/video_log")
+SSH_SKILL = os.getenv("SSH_SKILL", "/home/zhendongucb/ksort/TrueSkill/trueskill_data.json")
+SSH_VIDEO_SKILL = os.getenv("SSH_VIDEO_SKILL", "/home/zhendongucb/ksort/TrueSkill/trueskill_video_data.json")
+
+SSH_CACHE_OPENSOURCE = os.getenv("SSH_CACHE_OPENSOURCE", "/home/zhendongucb/ksort/ksort_video_cache/Kaiyuan/")
+SSH_CACHE_ADVANCE = os.getenv("SSH_CACHE_ADVANCE", "/home/zhendongucb/ksort/ksort_video_cache/Advance/")
+SSH_CACHE_PIKA = os.getenv("SSH_CACHE_PIKA", "/home/zhendongucb/ksort/ksort_video_cache/Pika-Beta/")
+SSH_CACHE_SORA = os.getenv("SSH_CACHE_SORA", "/home/zhendongucb/ksort/ksort_video_cache/Sora/")
+
+SSH_CACHE_IMAGE = os.getenv("SSH_CACHE_IMAGE", "/home/zhendongucb/ksort/ksort_image_cache/")
+
+# COLOR1=(128, 214, 255)
+# COLOR2=(237, 247, 152)
+# COLOR3=(250, 181, 122)
+# COLOR4=(240, 104, 104)
+
+# COLOR1=(112, 161, 215)
+# COLOR2=(161, 222, 147)
+# COLOR3=(247, 244, 139)
+# COLOR4=(244, 124, 124)
+
+COLOR1=(168, 230, 207)
+COLOR2=(253, 255, 171)
+COLOR3=(255, 211, 182)
+COLOR4=(255, 170, 165)
+
+# COLOR1=(255, 212, 96)
+# COLOR2=(240, 123, 63)
+# COLOR3=(234, 84, 85)
+# COLOR4=(45, 64, 89)
+
+# COLOR1=(255, 189, 57)
+# COLOR2=(230, 28, 93)
+# COLOR3=(147, 0, 119)
+# COLOR4=(58, 0, 136)

serve/gradio_web.py

@@ -0,0 +1,789 @@
+from .utils import *
+from .vote_utils import (
+    upvote_last_response_ig as upvote_last_response,
+    downvote_last_response_ig as downvote_last_response,
+    flag_last_response_ig as flag_last_response,
+    leftvote_last_response_igm as leftvote_last_response,
+    left1vote_last_response_igm as left1vote_last_response,
+    rightvote_last_response_igm as rightvote_last_response,
+    right1vote_last_response_igm as right1vote_last_response,
+    tievote_last_response_igm as tievote_last_response,
+    bothbad_vote_last_response_igm as bothbad_vote_last_response,
+    share_click_igm as share_click,
+    generate_ig,
+    generate_ig_museum,
+    generate_igm,
+    generate_igm_museum,
+    generate_igm_annoy,
+    generate_igm_annoy_museum,
+    generate_igm_cache_annoy,
+    share_js
+)
+from .Ksort import (
+    add_foreground,
+    reset_level,
+    reset_rank,
+    revote_windows,
+    submit_response_igm,
+    submit_response_rank_igm,
+    reset_submit,
+    clear_rank,
+    reset_mode,
+    reset_chatbot,
+    reset_btn_rank,
+    reset_vote_text,
+    text_response_rank_igm,
+    check_textbox,
+)
+
+from functools import partial
+from .upload import get_random_mscoco_prompt
+from serve.constants import SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD
+from serve.upload import get_random_mscoco_prompt, create_ssh_client
+from serve.update_skill import create_ssh_skill_client
+from model.matchmaker import create_ssh_matchmaker_client
+def set_ssh():
+    create_ssh_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+    create_ssh_skill_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+    create_ssh_matchmaker_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+
+def build_side_by_side_ui_anony(models):
+    notice_markdown = """
+# ⚔️  K-Sort Arena (Text-to-Image Generation) ⚔️
+## 📜 Rules
+- Input a prompt for four anonymized models (e.g., SD, SDXL, OpenJourney for Text-guided Image Generation) and vote on their outputs.
+- Two voting modes available: Rank Mode and Best Mode. Switch freely between modes. Please note that ties are always allowed. In ranking mode, users can input rankings like 1 3 3 1. Any invalid rankings, such as 1 4 4 1, will be automatically corrected during post-processing.
+- Users are encouraged to make evaluations based on subjective preferences. Evaluation criteria: Alignment (50%) + Aesthetics (50%).
+    - Alignment includes: Entity Matching (30%) + Style Matching (20%);
+    - Aesthetics includes: Photorealism (30%) + Light and Shadow (10%) + Absence of Artifacts (10%).
+
+## 👇 Generating now!
+- Note: Due to the API's image safety checks, errors may occur. If this happens, please re-enter a different prompt.
+- At times, high API concurrency can cause congestion, potentially resulting in a generation time of up to 1.5 minutes per image. Thank you for your patience.
+"""
+
+    model_list = models.model_ig_list
+
+    state0 = gr.State()
+    state1 = gr.State()
+    state2 = gr.State()
+    state3 = gr.State()
+
+    gen_func = partial(generate_igm_annoy, models.generate_image_ig_parallel_anony)
+    gen_cache_func = partial(generate_igm_cache_annoy, models.generate_image_ig_cache_anony)
+    
+    # gen_func_random = partial(generate_igm_annoy_museum, models.generate_image_ig_museum_parallel_anony)
+    
+    gr.Markdown(notice_markdown, elem_id="notice_markdown")
+
+    with gr.Group(elem_id="share-region-anony"):
+        with gr.Accordion("🔍 Expand to see all Arena players", open=False):
+            model_description_md = get_model_description_md(model_list)
+            gr.Markdown(model_description_md, elem_id="model_description_markdown")
+        with gr.Row():
+            with gr.Column():
+                chatbot_left = gr.Image(width=512, label = "Model A")
+            with gr.Column():
+                chatbot_left1 = gr.Image(width=512, label = "Model B")
+            with gr.Column():
+                chatbot_right = gr.Image(width=512, label = "Model C")
+            with gr.Column():
+                chatbot_right1 = gr.Image(width=512, label = "Model D")
+
+        with gr.Row():
+            with gr.Column():
+                model_selector_left = gr.Markdown("", visible=False)
+            with gr.Column():
+                model_selector_left1 = gr.Markdown("", visible=False)
+            with gr.Column():
+                model_selector_right = gr.Markdown("", visible=False)
+            with gr.Column():
+                model_selector_right1 = gr.Markdown("", visible=False)
+        with gr.Row():
+            slow_warning = gr.Markdown("", elem_id="notice_markdown")
+
+        with gr.Row(elem_classes="row"):
+            with gr.Column(scale=1, min_width=10):
+                leftvote_btn = gr.Button(
+                    value="A is Best", visible=False, interactive=False, elem_id="btncolor1", elem_classes="best-button"
+                )
+            with gr.Column(scale=1, min_width=10):
+                left1vote_btn = gr.Button(
+                    value="B is Best", visible=False, interactive=False, elem_id="btncolor1", elem_classes="best-button"
+                )
+            with gr.Column(scale=1, min_width=10):
+                rightvote_btn = gr.Button(
+                    value="C is Best", visible=False, interactive=False, elem_id="btncolor1", elem_classes="best-button"
+                )
+            with gr.Column(scale=1, min_width=10):
+                right1vote_btn = gr.Button(
+                    value="D is Best", visible=False, interactive=False, elem_id="btncolor1", elem_classes="best-button"
+                )
+            with gr.Column(scale=1, min_width=10):
+                tie_btn = gr.Button(
+                    value="🤝  Tie", visible=False, interactive=False, elem_id="btncolor2", elem_classes="best-button"
+                )
+
+        with gr.Row():
+            with gr.Blocks():
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=10):
+                        A1_btn = gr.Button(
+                            value="1", visible=False, interactive=False, elem_id="btncolor1", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        A2_btn = gr.Button(
+                            value="2", visible=False, interactive=False, elem_id="btncolor2", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        A3_btn = gr.Button(
+                            value="3", visible=False, interactive=False, elem_id="btncolor3", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        A4_btn = gr.Button(
+                            value="4", visible=False, interactive=False, elem_id="btncolor4", elem_classes="custom-button"
+                        )
+            with gr.Blocks(): 
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=10):
+                        B1_btn = gr.Button(
+                            value="1", visible=False, interactive=False, elem_id="btncolor1", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        B2_btn = gr.Button(
+                            value="2", visible=False, interactive=False, elem_id="btncolor2", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        B3_btn = gr.Button(
+                            value="3", visible=False, interactive=False, elem_id="btncolor3", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        B4_btn = gr.Button(
+                            value="4", visible=False, interactive=False, elem_id="btncolor4", elem_classes="custom-button"
+                        )
+            with gr.Blocks(): 
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=10):
+                        C1_btn = gr.Button(
+                            value="1", visible=False, interactive=False, elem_id="btncolor1", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        C2_btn = gr.Button(
+                            value="2", visible=False, interactive=False, elem_id="btncolor2", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        C3_btn = gr.Button(
+                            value="3", visible=False, interactive=False, elem_id="btncolor3", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        C4_btn = gr.Button(
+                            value="4", visible=False, interactive=False, elem_id="btncolor4", elem_classes="custom-button"
+                        )
+            with gr.Blocks(): 
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=10):
+                        D1_btn = gr.Button(
+                            value="1", visible=False, interactive=False, elem_id="btncolor1", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        D2_btn = gr.Button(
+                            value="2", visible=False, interactive=False, elem_id="btncolor2", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        D3_btn = gr.Button(
+                            value="3", visible=False, interactive=False, elem_id="btncolor3", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        D4_btn = gr.Button(
+                            value="4", visible=False, interactive=False, elem_id="btncolor4", elem_classes="custom-button"
+                        )
+
+    with gr.Row():
+        vote_textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your rank (you can use buttons above, or directly type here, e.g. 1 2 3 4)",
+            container=True,
+            elem_id="input_box",
+            visible=False,
+        )
+        vote_submit_btn = gr.Button(value="Submit", visible=False, interactive=False, variant="primary", scale=0, elem_id="btnpink", elem_classes="submit-button")
+        vote_mode_btn = gr.Button(value="🔄 Mode", visible=False, interactive=False, variant="primary", scale=0, elem_id="btnpink", elem_classes="submit-button")
+
+    with gr.Row():
+        textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your prompt and press ENTER",
+            container=True,
+            elem_id="input_box",
+        )
+        # send_btn = gr.Button(value="Send", variant="primary", scale=0, elem_id="btnblue", elem_classes="send-button")
+        # draw_btn = gr.Button(value="🎲 Random sample", variant="primary", scale=0, elem_id="btnblue", elem_classes="send-button")
+        send_btn = gr.Button(value="Send", variant="primary", scale=0, elem_id="btnblue")
+        draw_btn = gr.Button(value="🎲 Random Prompt", variant="primary", scale=0, elem_id="btnblue")
+    with gr.Row():
+        cache_btn = gr.Button(value="🎲 Random Sample", interactive=True)
+    with gr.Row():
+        clear_btn = gr.Button(value="🎲 New Round", interactive=False)
+        # regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+        # share_btn = gr.Button(value="📷  Share")
+    with gr.Blocks(): 
+        with gr.Row(elem_id="centered-text"): # 
+            user_info = gr.Markdown("User information (to appear on the contributor leaderboard)", visible=True, elem_id="centered-text") #, elem_id="centered-text"
+            # with gr.Blocks(): 
+            # name = gr.Markdown("Name", visible=True)
+            user_name = gr.Textbox(show_label=False,placeholder="👉 Enter your name (optional)", elem_classes="custom-width")
+            # with gr.Blocks(): 
+            # institution = gr.Markdown("Institution", visible=True)
+            user_institution = gr.Textbox(show_label=False,placeholder="👉 Enter your affiliation (optional)", elem_classes="custom-width")
+    #gr.Markdown(acknowledgment_md, elem_id="ack_markdown")
+    dummy_img_output = gr.Image(width=512, visible=False)
+    gr.Examples(
+        examples=[["A train crossing a bridge that is going over a body of water.", os.path.join("./examples", "example1.jpg")],
+                  ["The man in the business suit wears a striped blue and white tie.", os.path.join("./examples", "example2.jpg")],
+                  ["A skier stands on a small ledge in the snow.",os.path.join("./examples", "example3.jpg")],
+                  ["The bathroom with green tile and a red shower curtain.", os.path.join("./examples", "example4.jpg")]],
+        inputs = [textbox, dummy_img_output])
+
+    order_btn_list = [textbox, send_btn, draw_btn, cache_btn, clear_btn]
+    vote_order_list = [leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
+                       A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn, \
+                       vote_textbox, vote_submit_btn, vote_mode_btn]
+
+    generate_ig0 = gr.Image(width=512, label = "generate A", visible=False, interactive=False)
+    generate_ig1 = gr.Image(width=512, label = "generate B", visible=False, interactive=False)
+    generate_ig2 = gr.Image(width=512, label = "generate C", visible=False, interactive=False)
+    generate_ig3 = gr.Image(width=512, label = "generate D", visible=False, interactive=False)
+    dummy_left_model = gr.State("")
+    dummy_left1_model = gr.State("")
+    dummy_right_model = gr.State("")
+    dummy_right1_model = gr.State("")
+
+    ig_rank = [None, None, None, None]
+    bastA_rank = [0, 3, 3, 3]
+    bastB_rank = [3, 0, 3, 3]
+    bastC_rank = [3, 3, 0, 3]
+    bastD_rank = [3, 3, 3, 0]
+    tie_rank = [0, 0, 0, 0]
+    bad_rank = [3, 3, 3, 3]
+    rank = gr.State(ig_rank)
+    rankA = gr.State(bastA_rank)
+    rankB = gr.State(bastB_rank)
+    rankC = gr.State(bastC_rank)
+    rankD = gr.State(bastD_rank)
+    rankTie = gr.State(tie_rank)
+    rankBad = gr.State(bad_rank)
+    Top1_text = gr.Textbox(value="Top 1", visible=False, interactive=False)
+    Top2_text = gr.Textbox(value="Top 2", visible=False, interactive=False)
+    Top3_text = gr.Textbox(value="Top 3", visible=False, interactive=False)
+    Top4_text = gr.Textbox(value="Top 4", visible=False, interactive=False)
+    window1_text = gr.Textbox(value="Model A", visible=False, interactive=False)
+    window2_text = gr.Textbox(value="Model B", visible=False, interactive=False)
+    window3_text = gr.Textbox(value="Model C", visible=False, interactive=False)
+    window4_text = gr.Textbox(value="Model D", visible=False, interactive=False)
+    vote_level = gr.Number(value=0, visible=False, interactive=False)
+    # Top1_btn.click(reset_level, inputs=[Top1_text], outputs=[vote_level])
+    # Top2_btn.click(reset_level, inputs=[Top2_text], outputs=[vote_level])
+    # Top3_btn.click(reset_level, inputs=[Top3_text], outputs=[vote_level])
+    # Top4_btn.click(reset_level, inputs=[Top4_text], outputs=[vote_level])
+    vote_mode = gr.Textbox(value="Rank", visible=False, interactive=False)
+    right_vote_text = gr.Textbox(value="wrong", visible=False, interactive=False)
+    cache_mode = gr.Textbox(value="True", visible=False, interactive=False)
+
+    textbox.submit(
+        disable_order_buttons,
+        inputs=[textbox],
+        outputs=order_btn_list
+    ).then(
+        gen_func,
+        inputs=[state0, state1, state2, state3, textbox, model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+        outputs=[state0, state1, state2, state3, generate_ig0, generate_ig1, generate_ig2, generate_ig3, chatbot_left, chatbot_left1, chatbot_right, chatbot_right1, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+        api_name="submit_btn_annony"
+    ).then(
+        enable_vote_mode_buttons,
+        inputs=[vote_mode, textbox],
+        outputs=vote_order_list
+    )   
+
+    send_btn.click(
+        disable_order_buttons,
+        inputs=[textbox],
+        outputs=order_btn_list
+    ).then(
+        gen_func,
+        inputs=[state0, state1, state2, state3, textbox, model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+        outputs=[state0, state1, state2, state3, generate_ig0, generate_ig1, generate_ig2, generate_ig3, chatbot_left, chatbot_left1, chatbot_right, chatbot_right1, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+        api_name="send_btn_annony"
+    ).then(
+        enable_vote_mode_buttons,
+        inputs=[vote_mode, textbox],
+        outputs=vote_order_list
+    )
+
+    cache_btn.click(
+        disable_order_buttons,
+        inputs=[textbox, cache_mode],
+        outputs=order_btn_list
+    ).then(
+        gen_cache_func,
+        inputs=[state0, state1, state2, state3, model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+        outputs=[state0, state1, state2, state3, generate_ig0, generate_ig1, generate_ig2, generate_ig3, chatbot_left, chatbot_left1, chatbot_right, chatbot_right1, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1, textbox],
+        api_name="send_btn_annony"
+    ).then(
+        enable_vote_mode_buttons,
+        inputs=[vote_mode, textbox],
+        outputs=vote_order_list
+    )
+
+    draw_btn.click(
+        get_random_mscoco_prompt,
+        inputs=None, 
+        outputs=[textbox],
+        api_name="draw_btn_annony"
+    )
+
+    clear_btn.click(
+        clear_history_side_by_side_anony, 
+        inputs=None, 
+        outputs=[state0, state1, state2, state3, textbox, vote_textbox, chatbot_left, chatbot_left1, chatbot_right, chatbot_right1, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+        api_name="clear_btn_annony"
+    ).then(
+        enable_order_buttons,
+        inputs=None,
+        outputs=order_btn_list
+    ).then(
+        clear_rank,
+        inputs=[rank, vote_level],
+        outputs=[rank, vote_level]
+    ).then(
+        disable_vote_mode_buttons,
+        inputs=None,
+        outputs=vote_order_list
+    )
+    
+    # regenerate_btn.click(
+    #     gen_func,
+    #     inputs=[state0, state1, state2, state3, textbox, model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+    #     outputs=[state0, state1, state2, state3, chatbot_left, chatbot_left1, chatbot_right, chatbot_right1, \
+    #              model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+    #     api_name="regenerate_btn_annony"
+    # ).then(
+    #     enable_best_buttons,
+    #     inputs=None,
+    #     outputs=btn_list
+    # )
+    vote_mode_btn.click(
+        reset_chatbot,
+        inputs=[vote_mode, generate_ig0, generate_ig1, generate_ig2, generate_ig3], 
+        outputs=[chatbot_left, chatbot_left1, chatbot_right, chatbot_right1]
+    ).then(
+        reset_mode, 
+        inputs=[vote_mode], 
+        outputs=[leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
+                    A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn, \
+                    vote_textbox, vote_submit_btn, vote_mode_btn, vote_mode]
+    )
+
+    vote_textbox.submit(
+        disable_vote,
+        inputs=None,
+        outputs=[vote_submit_btn, vote_mode_btn, \
+                 A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn]
+    ).then(
+        text_response_rank_igm,
+        inputs=[generate_ig0, generate_ig1, generate_ig2, generate_ig3, Top1_text, Top2_text, Top3_text, Top4_text, vote_textbox], 
+        outputs=[chatbot_left, chatbot_left1, chatbot_right, chatbot_right1, vote_textbox, right_vote_text, rank]
+    ).then(
+        submit_response_rank_igm,
+        inputs=[state0, state1, state2, state3, dummy_left_model, dummy_left1_model, dummy_right_model, dummy_right1_model, textbox, rank, right_vote_text, user_name, user_institution],
+        outputs=[A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn, \
+                    vote_textbox, vote_submit_btn, vote_mode_btn, right_vote_text, \
+                    model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+        api_name="submit_btn_annony"
+    ) 
+    vote_submit_btn.click(
+        disable_vote,
+        inputs=None,
+        outputs=[vote_submit_btn, vote_mode_btn, \
+                 A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn]
+    ).then(
+        text_response_rank_igm,
+        inputs=[generate_ig0, generate_ig1, generate_ig2, generate_ig3, Top1_text, Top2_text, Top3_text, Top4_text, vote_textbox], 
+        outputs=[chatbot_left, chatbot_left1, chatbot_right, chatbot_right1, vote_textbox, right_vote_text, rank]
+    ).then(
+        submit_response_rank_igm,
+        inputs=[state0, state1, state2, state3, dummy_left_model, dummy_left1_model, dummy_right_model, dummy_right1_model, textbox, rank, right_vote_text, user_name, user_institution],
+        outputs=[A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn, \
+                    vote_textbox, vote_submit_btn, vote_mode_btn, right_vote_text, \
+                    model_selector_left, model_selector_left1, model_selector_right, model_selector_right1],
+        api_name="submit_btn_annony"
+    ) 
+
+    # Revote_btn.click(
+    #     revote_windows, 
+    #     inputs=[generate_ig0, generate_ig1, generate_ig2, generate_ig3, rank, vote_level], 
+    #     outputs=[chatbot_left, chatbot_left1, chatbot_right, chatbot_right1, rank, vote_level]
+    # ).then(
+    #     reset_submit,
+    #     inputs = [rank],
+    #     outputs = [Submit_btn]
+    # )
+    # Submit_btn.click(
+    #     submit_response_igm,
+    #     inputs=[state0, state1, state2, state3, dummy_left_model, dummy_left1_model, dummy_right_model, dummy_right1_model, rank],
+    #     outputs=[textbox, Top1_btn, Top2_btn, Top3_btn, Top4_btn, Revote_btn, Submit_btn, \
+    #              model_selector_left, model_selector_left1, model_selector_right, model_selector_right1]
+    # )
+
+    # chatbot_left.select(add_foreground, inputs=[generate_ig0, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], outputs=[chatbot_left]).then(
+    #     reset_rank,
+    #     inputs = [window1_text, rank, vote_level],
+    #     outputs = [rank]
+    #     ).then(
+    #         reset_submit,
+    #         inputs = [rank],
+    #         outputs = [Submit_btn]
+    #     )
+    # chatbot_left1.select(add_foreground, inputs=[generate_ig1, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], outputs=[chatbot_left1]).then(
+    #     reset_rank,
+    #     inputs = [window2_text, rank, vote_level],
+    #     outputs = [rank]
+    #     ).then(
+    #         reset_submit,
+    #         inputs = [rank],
+    #         outputs = [Submit_btn]
+    #     )
+    # chatbot_right.select(add_foreground, inputs=[generate_ig2, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], outputs=[chatbot_right]).then(
+    #     reset_rank,
+    #     inputs = [window3_text, rank, vote_level],
+    #     outputs = [rank]
+    #     ).then(
+    #         reset_submit,
+    #         inputs = [rank],
+    #         outputs = [Submit_btn]
+    #     )
+    # chatbot_right1.select(add_foreground, inputs=[generate_ig3, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], outputs=[chatbot_right1]).then(
+    #     reset_rank,
+    #     inputs = [window4_text, rank, vote_level],
+    #     outputs = [rank]
+    #     ).then(
+    #         reset_submit,
+    #         inputs = [rank],
+    #         outputs = [Submit_btn]
+    #     )
+    
+
+    leftvote_btn.click(
+        submit_response_igm,
+        inputs=[state0, state1, state2, state3, dummy_left_model, dummy_left1_model, dummy_right_model, dummy_right1_model, textbox, rankA, user_name, user_institution],
+        outputs=[textbox, leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1, \
+                 vote_mode_btn]
+    )
+    left1vote_btn.click(
+        submit_response_igm,
+        inputs=[state0, state1, state2, state3, dummy_left_model, dummy_left1_model, dummy_right_model, dummy_right1_model, textbox, rankB, user_name, user_institution],
+        outputs=[textbox, leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1, \
+                 vote_mode_btn]
+    )
+    rightvote_btn.click(
+        submit_response_igm,
+        inputs=[state0, state1, state2, state3, dummy_left_model, dummy_left1_model, dummy_right_model, dummy_right1_model, textbox, rankC, user_name, user_institution],
+        outputs=[textbox, leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1, \
+                 vote_mode_btn]
+    )
+    right1vote_btn.click(
+        submit_response_igm,
+        inputs=[state0, state1, state2, state3, dummy_left_model, dummy_left1_model, dummy_right_model, dummy_right1_model, textbox, rankD, user_name, user_institution],
+        outputs=[textbox, leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1, \
+                 vote_mode_btn]
+    )
+    tie_btn.click(
+        submit_response_igm,
+        inputs=[state0, state1, state2, state3, dummy_left_model, dummy_left1_model, dummy_right_model, dummy_right1_model, textbox, rankTie, user_name, user_institution],
+        outputs=[textbox, leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
+                 model_selector_left, model_selector_left1, model_selector_right, model_selector_right1, \
+                 vote_mode_btn]
+    )
+
+    A1_btn.click(
+        reset_btn_rank, 
+        inputs=[window1_text, rank, A1_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig0, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_left]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    A2_btn.click(
+        reset_btn_rank, 
+        inputs=[window1_text, rank, A2_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig0, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_left]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    A3_btn.click(
+        reset_btn_rank, 
+        inputs=[window1_text, rank, A3_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig0, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_left]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    A4_btn.click(
+        reset_btn_rank, 
+        inputs=[window1_text, rank, A4_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig0, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_left]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    
+    B1_btn.click(
+        reset_btn_rank, 
+        inputs=[window2_text, rank, B1_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig1, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_left1]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    B2_btn.click(
+        reset_btn_rank, 
+        inputs=[window2_text, rank, B2_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig1, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_left1]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    B3_btn.click(
+        reset_btn_rank, 
+        inputs=[window2_text, rank, B3_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig1, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_left1]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    B4_btn.click(
+        reset_btn_rank, 
+        inputs=[window2_text, rank, B4_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig1, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_left1]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+
+    C1_btn.click(
+        reset_btn_rank, 
+        inputs=[window3_text, rank, C1_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig2, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_right]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    C2_btn.click(
+        reset_btn_rank, 
+        inputs=[window3_text, rank, C2_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig2, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_right]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    C3_btn.click(
+        reset_btn_rank, 
+        inputs=[window3_text, rank, C3_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig2, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_right]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    C4_btn.click(
+        reset_btn_rank, 
+        inputs=[window3_text, rank, C4_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig2, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_right]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+
+    D1_btn.click(
+        reset_btn_rank, 
+        inputs=[window4_text, rank, D1_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig3, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_right1]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    D2_btn.click(
+        reset_btn_rank, 
+        inputs=[window4_text, rank, D2_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig3, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_right1]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    D3_btn.click(
+        reset_btn_rank, 
+        inputs=[window4_text, rank, D3_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig3, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_right1]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )
+    D4_btn.click(
+        reset_btn_rank, 
+        inputs=[window4_text, rank, D4_btn, vote_level], 
+        outputs=[rank, vote_level]
+    ).then(
+        add_foreground, 
+        inputs=[generate_ig3, vote_level, Top1_text, Top2_text, Top3_text, Top4_text], 
+        outputs=[chatbot_right1]
+    ).then(
+        reset_submit,
+        inputs = [rank],
+        outputs = [vote_submit_btn]
+    ).then(
+        reset_vote_text,
+        inputs = [rank],
+        outputs = [vote_textbox]
+    )

serve/gradio_web_bbox.py

@@ -0,0 +1,492 @@
+import gradio as gr
+import time
+import numpy as np
+from gradio import processing_utils
+from PIL import Image, ImageDraw, ImageFont
+
+from serve.utils import *
+from serve.vote_utils import (
+    upvote_last_response_ig as upvote_last_response,
+    downvote_last_response_ig as downvote_last_response,
+    flag_last_response_ig as flag_last_response,
+    leftvote_last_response_igm as leftvote_last_response,
+    left1vote_last_response_igm as left1vote_last_response,
+    rightvote_last_response_igm as rightvote_last_response,
+    right1vote_last_response_igm as right1vote_last_response,
+    tievote_last_response_igm as tievote_last_response,
+    bothbad_vote_last_response_igm as bothbad_vote_last_response,
+    share_click_igm as share_click,
+    generate_ig,
+    generate_ig_museum,
+    generate_igm,
+    generate_igm_museum,
+    generate_igm_annoy,
+    generate_igm_annoy_museum,
+    generate_igm_cache_annoy,
+    share_js
+)
+from serve.Ksort import (
+    add_foreground,
+    reset_level,
+    reset_rank,
+    revote_windows,
+    submit_response_igm,
+    submit_response_rank_igm,
+    reset_submit,
+    clear_rank,
+    reset_mode,
+    reset_chatbot,
+    reset_btn_rank,
+    reset_vote_text,
+    text_response_rank_igm,
+    check_textbox,
+)
+
+from functools import partial
+from serve.upload import get_random_mscoco_prompt
+from serve.constants import SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD
+from serve.upload import get_random_mscoco_prompt, create_ssh_client
+from serve.update_skill import create_ssh_skill_client
+from model.matchmaker import create_ssh_matchmaker_client
+
+def set_ssh():
+    create_ssh_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+    create_ssh_skill_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+    create_ssh_matchmaker_client(SSH_SERVER, SSH_PORT, SSH_USER, SSH_PASSWORD)
+
+def binarize(x):
+    return (x != 0).astype('uint8') * 255
+
+def sized_center_crop(img, cropx, cropy):
+    y, x = img.shape[:2]
+    startx = x // 2 - (cropx // 2)
+    starty = y // 2 - (cropy // 2)    
+    return img[starty:starty+cropy, startx:startx+cropx]
+
+def sized_center_fill(img, fill, cropx, cropy):
+    y, x = img.shape[:2]
+    startx = x // 2 - (cropx // 2)
+    starty = y // 2 - (cropy // 2)    
+    img[starty:starty+cropy, startx:startx+cropx] = fill
+    return img
+
+def sized_center_mask(img, cropx, cropy):
+    y, x = img.shape[:2]
+    startx = x // 2 - (cropx // 2)
+    starty = y // 2 - (cropy // 2)    
+    center_region = img[starty:starty+cropy, startx:startx+cropx].copy()
+    img = (img * 0.2).astype('uint8')
+    img[starty:starty+cropy, startx:startx+cropx] = center_region
+    return img
+
+def center_crop(img, HW=None, tgt_size=(512, 512)):
+    if HW is None:
+        H, W = img.shape[:2]
+        HW = min(H, W)
+    img = sized_center_crop(img, HW, HW)
+    img = Image.fromarray(img)
+    img = img.resize(tgt_size)
+    return np.array(img)
+
+
+
+def draw_box(boxes=[], labels=[], img=None):
+    if len(boxes) == 0 and img is None:
+        return None
+    
+    # 确保输入的 img 是 PIL.Image 对象
+    if isinstance(img, np.ndarray):
+        img = Image.fromarray(img)
+
+    if img is None:
+        img = Image.new('RGB', (512, 512), (255, 255, 255))
+
+    colors = ["red", "olive", "blue", "green", "orange", "brown", "cyan", "purple"]
+
+    # 创建绘图对象
+    draw_obj = ImageDraw.Draw(img)
+    font = ImageFont.load_default()
+    # 获取字体大小
+    font_size = getattr(font, 'size_in_points', 10)  # 如果没有 size_in_points 属性，使用默认值 10
+
+    for bid, box in enumerate(boxes):
+        draw_obj.rectangle([box[0], box[1], box[2], box[3]], outline=colors[bid % len(colors)], width=4)
+        anno_text = labels[bid]
+        draw_obj.rectangle(
+            [
+                box[0], 
+                box[3] - int(font_size * 1.2), 
+                box[0] + int((len(anno_text) + 0.8) * font_size * 0.6), 
+                box[3]
+            ], 
+            outline=colors[bid % len(colors)], 
+            fill=colors[bid % len(colors)], 
+            width=4
+        )
+        draw_obj.text(
+            [box[0] + int(font_size * 0.2), box[3] - int(font_size * 1.2)], 
+            anno_text, 
+            font=font, 
+            fill=(255,255,255)
+        )
+    return img
+
+
+def draw(input, grounding_texts, new_image_trigger, state):
+    
+    # 确保输入数据中有必要的键
+    if isinstance(input, dict):
+        background = input.get('background', None)
+        print("background.shape", background.shape)
+        layers = input.get('layers', 1)
+        print("len(layers)", len(layers))
+        composite = input.get('composite', None)
+        print("composite.shape", composite.shape)
+    else:
+        # 如果 input 不是字典，直接使用其作为图像
+        background = input
+        layers = 1
+        composite = None
+
+    # 检查 background 是否有效
+    if background is None:
+        print("background is None")
+        return None
+    # background = np.ones((512, 512, 3), dtype='uint8') * 255
+
+    # 默认使用 composite 作为最终图像，如果没有 composite 则使用 background
+    if composite is None:
+        print("composite is None")
+        image = background
+    else:
+        image = composite
+
+    mask = binarize(image)
+
+    if type(mask) != np.ndarray:
+        mask = np.array(mask)
+
+    if mask.sum() == 0:
+        state = {}
+
+    # 更新状态，如果没有 boxes 和 masks，则初始化它们
+    if 'boxes' not in state:
+        state['boxes'] = []
+
+    if 'masks' not in state or len(state['masks']) == 0:
+        state['masks'] = []
+        last_mask = np.zeros_like(mask)
+    else:
+        last_mask = state['masks'][-1]
+
+    if type(mask) == np.ndarray and mask.size > 1:
+        diff_mask = mask - last_mask
+    else:
+        diff_mask = np.zeros([])
+
+    # 根据 mask 的变化来计算 box 的位置
+    if diff_mask.sum() > 0:
+        x1x2 = np.where(diff_mask.max(0) != 0)[0]
+        y1y2 = np.where(diff_mask.max(1) != 0)[0]
+        y1, y2 = y1y2.min(), y1y2.max()
+        x1, x2 = x1x2.min(), x1x2.max()
+
+        if (x2 - x1 > 5) and (y2 - y1 > 5):
+            state['masks'].append(mask.copy())
+            state['boxes'].append((x1, y1, x2, y2))
+
+    # 处理 grounding_texts
+    grounding_texts = [x.strip() for x in grounding_texts.split(';')]
+    grounding_texts = [x for x in grounding_texts if len(x) > 0]
+    if len(grounding_texts) < len(state['boxes']):
+        grounding_texts += [f'Obj. {bid+1}' for bid in range(len(grounding_texts), len(state['boxes']))]
+
+    # 绘制标注框
+    box_image = draw_box(state['boxes'], grounding_texts, background)
+
+    if box_image is not None and state.get('inpaint_hw', None):
+        inpaint_hw = state['inpaint_hw']
+        box_image_resize = np.array(box_image.resize((inpaint_hw, inpaint_hw)))
+        original_image = state['original_image'].copy()
+        box_image = sized_center_fill(original_image, box_image_resize, inpaint_hw, inpaint_hw)
+
+    return [box_image, new_image_trigger, 1.0, state]
+
+def build_side_by_side_bbox_ui_anony(models):
+    notice_markdown = """
+    # ⚔️  Control-Ability-Arena (Bbox-to-Image Generation) ⚔️
+    ## 📜 Rules
+    - Input a prompt for four anonymized models and vote on their outputs.
+    - Two voting modes available: Rank Mode and Best Mode. Switch freely between modes. Please note that ties are always allowed. In ranking mode, users can input rankings like 1 3 3 1. Any invalid rankings, such as 1 4 4 1, will be automatically corrected during post-processing.
+    - Users are encouraged to make evaluations based on subjective preferences. Evaluation criteria: Alignment (50%) + Aesthetics (50%).
+        - Alignment includes: Entity Matching (30%) + Style Matching (20%);
+        - Aesthetics includes: Photorealism (30%) + Light and Shadow (10%) + Absence of Artifacts (10%).
+
+    ## 👇 Generating now!
+    - Note: Due to the API's image safety checks, errors may occur. If this happens, please re-enter a different prompt.
+    - At times, high API concurrency can cause congestion, potentially resulting in a generation time of up to 1.5 minutes per image. Thank you for your patience.
+    """
+    model_list = models.model_b2i_list
+
+    state = gr.State({})
+    state0 = gr.State()
+    state1 = gr.State()
+    state2 = gr.State()
+    state3 = gr.State()
+
+    # gen_func = partial(generate_igm_annoy, models.generate_image_ig_parallel_anony)
+    # gen_cache_func = partial(generate_igm_cache_annoy, models.generate_image_ig_cache_anony)
+    
+   
+    gr.Markdown(notice_markdown, elem_id="notice_markdown")
+
+
+    with gr.Row():
+        sketch_pad_trigger = gr.Number(value=0, visible=False)
+        sketch_pad_resize_trigger = gr.Number(value=0, visible=False)
+        image_scale = gr.Number(value=0, elem_id="image_scale", visible=False)
+
+    with gr.Row():
+        sketch_pad = gr.ImageEditor(
+            label="Sketch Pad", 
+            type="numpy",
+            crop_size="1:1",
+            width=512,
+            height=512
+        )
+        out_imagebox = gr.Image(
+            type="pil",
+            label="Parsed Sketch Pad",
+            width=512,
+            height=512
+        )
+
+    with gr.Row():
+        textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your prompt and press ENTER",
+            container=True,
+            elem_id="input_box",
+        )
+        send_btn = gr.Button(value="Send", variant="primary", scale=0, elem_id="btnblue")
+
+    with gr.Row():
+        grounding_instruction = gr.Textbox(
+            label="Grounding instruction (Separated by semicolon)",
+            placeholder="👉 Enter your Grounding instruction (e.g. a cat; a dog; a bird; a fish)",
+        )
+
+    with gr.Group(elem_id="share-region-anony"):
+        with gr.Accordion("🔍 Expand to see all Arena players", open=False):
+            # model_description_md = get_model_description_md(model_list)
+            gr.Markdown("", elem_id="model_description_markdown")
+
+        
+        with gr.Row():
+            with gr.Column():
+                chatbot_left = gr.Image(width=512, label = "Model A")
+            with gr.Column():
+                chatbot_left1 = gr.Image(width=512, label = "Model B")
+            with gr.Column():
+                chatbot_right = gr.Image(width=512, label = "Model C")
+            with gr.Column():
+                chatbot_right1 = gr.Image(width=512, label = "Model D")
+
+        with gr.Row():
+            with gr.Column():
+                model_selector_left = gr.Markdown("", visible=False)
+            with gr.Column():
+                model_selector_left1 = gr.Markdown("", visible=False)
+            with gr.Column():
+                model_selector_right = gr.Markdown("", visible=False)
+            with gr.Column():
+                model_selector_right1 = gr.Markdown("", visible=False)
+        with gr.Row():
+            slow_warning = gr.Markdown("", elem_id="notice_markdown")
+
+        with gr.Row(elem_classes="row"):
+            with gr.Column(scale=1, min_width=10):
+                leftvote_btn = gr.Button(
+                    value="A is Best", visible=False, interactive=False, elem_id="btncolor1", elem_classes="best-button"
+                )
+            with gr.Column(scale=1, min_width=10):
+                left1vote_btn = gr.Button(
+                    value="B is Best", visible=False, interactive=False, elem_id="btncolor1", elem_classes="best-button"
+                )
+            with gr.Column(scale=1, min_width=10):
+                rightvote_btn = gr.Button(
+                    value="C is Best", visible=False, interactive=False, elem_id="btncolor1", elem_classes="best-button"
+                )
+            with gr.Column(scale=1, min_width=10):
+                right1vote_btn = gr.Button(
+                    value="D is Best", visible=False, interactive=False, elem_id="btncolor1", elem_classes="best-button"
+                )
+            with gr.Column(scale=1, min_width=10):
+                tie_btn = gr.Button(
+                    value="🤝  Tie", visible=False, interactive=False, elem_id="btncolor2", elem_classes="best-button"
+                )
+
+
+        with gr.Row():
+            with gr.Blocks():
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=10):
+                        A1_btn = gr.Button(
+                            value="1", visible=False, interactive=False, elem_id="btncolor1", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        A2_btn = gr.Button(
+                            value="2", visible=False, interactive=False, elem_id="btncolor2", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        A3_btn = gr.Button(
+                            value="3", visible=False, interactive=False, elem_id="btncolor3", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        A4_btn = gr.Button(
+                            value="4", visible=False, interactive=False, elem_id="btncolor4", elem_classes="custom-button"
+                        )
+            with gr.Blocks(): 
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=10):
+                        B1_btn = gr.Button(
+                            value="1", visible=False, interactive=False, elem_id="btncolor1", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        B2_btn = gr.Button(
+                            value="2", visible=False, interactive=False, elem_id="btncolor2", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        B3_btn = gr.Button(
+                            value="3", visible=False, interactive=False, elem_id="btncolor3", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        B4_btn = gr.Button(
+                            value="4", visible=False, interactive=False, elem_id="btncolor4", elem_classes="custom-button"
+                        )
+            with gr.Blocks(): 
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=10):
+                        C1_btn = gr.Button(
+                            value="1", visible=False, interactive=False, elem_id="btncolor1", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        C2_btn = gr.Button(
+                            value="2", visible=False, interactive=False, elem_id="btncolor2", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        C3_btn = gr.Button(
+                            value="3", visible=False, interactive=False, elem_id="btncolor3", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        C4_btn = gr.Button(
+                            value="4", visible=False, interactive=False, elem_id="btncolor4", elem_classes="custom-button"
+                        )
+            with gr.Blocks(): 
+                with gr.Row():
+                    with gr.Column(scale=1, min_width=10):
+                        D1_btn = gr.Button(
+                            value="1", visible=False, interactive=False, elem_id="btncolor1", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        D2_btn = gr.Button(
+                            value="2", visible=False, interactive=False, elem_id="btncolor2", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        D3_btn = gr.Button(
+                            value="3", visible=False, interactive=False, elem_id="btncolor3", elem_classes="custom-button"
+                        )
+                    with gr.Column(scale=1, min_width=10):
+                        D4_btn = gr.Button(
+                            value="4", visible=False, interactive=False, elem_id="btncolor4", elem_classes="custom-button"
+                        )
+    with gr.Row():
+        vote_textbox = gr.Textbox(
+            show_label=False,
+            placeholder="👉 Enter your rank (you can use buttons above, or directly type here, e.g. 1 2 3 4)",
+            container=True,
+            elem_id="input_box",
+            visible=False,
+        )
+        vote_submit_btn = gr.Button(value="Submit", visible=False, interactive=False, variant="primary", scale=0, elem_id="btnpink", elem_classes="submit-button")
+        vote_mode_btn = gr.Button(value="🔄 Mode", visible=False, interactive=False, variant="primary", scale=0, elem_id="btnpink", elem_classes="submit-button")
+
+    with gr.Row():
+        clear_btn = gr.Button(value="🎲 New Round", interactive=False)
+        # regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
+        # share_btn = gr.Button(value="📷  Share")
+    with gr.Blocks(): 
+        with gr.Row(elem_id="centered-text"): # 
+            user_info = gr.Markdown("User information (to appear on the contributor leaderboard)", visible=True, elem_id="centered-text") #, elem_id="centered-text"
+            # with gr.Blocks(): 
+            # name = gr.Markdown("Name", visible=True)
+            user_name = gr.Textbox(show_label=False,placeholder="👉 Enter your name (optional)", elem_classes="custom-width")
+            # with gr.Blocks(): 
+            # institution = gr.Markdown("Institution", visible=True)
+            user_institution = gr.Textbox(show_label=False,placeholder="👉 Enter your affiliation (optional)", elem_classes="custom-width")
+    
+    sketch_pad.change(
+        draw,
+        inputs=[sketch_pad, grounding_instruction, sketch_pad_resize_trigger, state],
+        outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, state],
+        queue=False,
+    )
+    grounding_instruction.change(
+        draw,
+        inputs=[sketch_pad, grounding_instruction, sketch_pad_resize_trigger, state],
+        outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, state],
+        queue=False,
+    )
+
+    order_btn_list = [textbox, send_btn, clear_btn]
+    vote_order_list = [leftvote_btn, left1vote_btn, rightvote_btn, right1vote_btn, tie_btn, \
+                       A1_btn, A2_btn, A3_btn, A4_btn, B1_btn, B2_btn, B3_btn, B4_btn, C1_btn, C2_btn, C3_btn, C4_btn, D1_btn, D2_btn, D3_btn, D4_btn, \
+                       vote_textbox, vote_submit_btn, vote_mode_btn]
+    
+    generate_ig0 = gr.Image(width=512, label = "generate A", visible=False, interactive=False)
+    generate_ig1 = gr.Image(width=512, label = "generate B", visible=False, interactive=False)
+    generate_ig2 = gr.Image(width=512, label = "generate C", visible=False, interactive=False)
+    generate_ig3 = gr.Image(width=512, label = "generate D", visible=False, interactive=False)
+    dummy_left_model = gr.State("")
+    dummy_left1_model = gr.State("")
+    dummy_right_model = gr.State("")
+    dummy_right1_model = gr.State("")
+
+    ig_rank = [None, None, None, None]
+    bastA_rank = [0, 3, 3, 3]
+    bastB_rank = [3, 0, 3, 3]
+    bastC_rank = [3, 3, 0, 3]
+    bastD_rank = [3, 3, 3, 0]
+    tie_rank = [0, 0, 0, 0]
+    bad_rank = [3, 3, 3, 3]
+    rank = gr.State(ig_rank)
+    rankA = gr.State(bastA_rank)
+    rankB = gr.State(bastB_rank)
+    rankC = gr.State(bastC_rank)
+    rankD = gr.State(bastD_rank)
+    rankTie = gr.State(tie_rank)
+    rankBad = gr.State(bad_rank)
+    Top1_text = gr.Textbox(value="Top 1", visible=False, interactive=False)
+    Top2_text = gr.Textbox(value="Top 2", visible=False, interactive=False)
+    Top3_text = gr.Textbox(value="Top 3", visible=False, interactive=False)
+    Top4_text = gr.Textbox(value="Top 4", visible=False, interactive=False)
+    window1_text = gr.Textbox(value="Model A", visible=False, interactive=False)
+    window2_text = gr.Textbox(value="Model B", visible=False, interactive=False)
+    window3_text = gr.Textbox(value="Model C", visible=False, interactive=False)
+    window4_text = gr.Textbox(value="Model D", visible=False, interactive=False)
+    vote_level = gr.Number(value=0, visible=False, interactive=False)
+    # Top1_btn.click(reset_level, inputs=[Top1_text], outputs=[vote_level])
+    # Top2_btn.click(reset_level, inputs=[Top2_text], outputs=[vote_level])
+    # Top3_btn.click(reset_level, inputs=[Top3_text], outputs=[vote_level])
+    # Top4_btn.click(reset_level, inputs=[Top4_text], outputs=[vote_level])
+    vote_mode = gr.Textbox(value="Rank", visible=False, interactive=False)
+    right_vote_text = gr.Textbox(value="wrong", visible=False, interactive=False)
+    cache_mode = gr.Textbox(value="True", visible=False, interactive=False)
+
+
+
+
+if __name__ == "__main__":
+    with gr.Blocks() as demo:
+        build_side_by_side_bbox_ui_anony()
+    demo.launch()

serve/leaderboard.py

@@ -0,0 +1,200 @@
+"""
+Live monitor of the website statistics and leaderboard.
+
+Dependency:
+sudo apt install pkg-config libicu-dev
+pip install pytz gradio gdown plotly polyglot pyicu pycld2 tabulate
+"""
+
+import argparse
+import ast
+import pickle
+import os
+import threading
+import time
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+import json
+from datetime import datetime
+
+
+# def make_leaderboard_md(elo_results):
+#     leaderboard_md = f"""
+# # 🏆 Chatbot Arena Leaderboard
+# | [Blog](https://lmsys.org/blog/2023-05-03-arena/) | [GitHub](https://github.com/lm-sys/FastChat) | [Paper](https://arxiv.org/abs/2306.05685) | [Dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md) | [Twitter](https://twitter.com/lmsysorg) | [Discord](https://discord.gg/HSWAKCrnFx) |
+
+# This leaderboard is based on the following three benchmarks.
+# - [Chatbot Arena](https://lmsys.org/blog/2023-05-03-arena/) - a crowdsourced, randomized battle platform. We use 100K+ user votes to compute Elo ratings.
+# - [MT-Bench](https://arxiv.org/abs/2306.05685) - a set of challenging multi-turn questions. We use GPT-4 to grade the model responses.
+# - [MMLU](https://arxiv.org/abs/2009.03300) (5-shot) - a test to measure a model's multitask accuracy on 57 tasks.
+
+# 💻 Code: The Arena Elo ratings are computed by this [notebook]({notebook_url}). The MT-bench scores (single-answer grading on a scale of 10) are computed by [fastchat.llm_judge](https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge). The MMLU scores are mostly computed by [InstructEval](https://github.com/declare-lab/instruct-eval). Higher values are better for all benchmarks. Empty cells mean not available. Last updated: November, 2023.
+# """
+#     return leaderboard_md
+
+def make_leaderboard_md():
+    leaderboard_md = f"""
+# 🏆 K-Sort Arena Leaderboard (Text-to-Image Generation)
+"""
+    return leaderboard_md
+
+
+def make_leaderboard_video_md():
+    leaderboard_md = f"""
+# 🏆 K-Sort Arena Leaderboard (Text-to-Video Generation)
+"""
+    return leaderboard_md
+
+
+def model_hyperlink(model_name, link):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_arena_leaderboard_md(total_models, total_votes, last_updated):
+    # last_updated = datetime.now()
+    # last_updated = last_updated.strftime("%Y-%m-%d")
+
+    leaderboard_md = f"""
+Total models: **{total_models}** (anonymized), Total votes: **{total_votes}** (equivalent to **{total_votes*6}** pairwise comparisons)
+\n Last updated: {last_updated}
+"""
+
+    return leaderboard_md
+
+
+def make_disclaimer_md():
+    disclaimer_md = '''
+        <div id="modal" style="display:none; position:fixed; top:50%; left:50%; transform:translate(-50%, -50%); padding:20px; background:white; box-shadow:0 0 10px rgba(0,0,0,0.5); z-index:1000;">
+            <p style="font-size:24px;"><strong>Disclaimer</strong></p>
+            <p style="font-size:18px;"><b>Purpose and Scope</b></b></p>
+            <p><b>This platform is designed for academic use, providing a space for evaluating and comparing Visual Generation Models. The information and services provided are intended for research and educational purposes only.</b></p>
+
+            <p style="font-size:18px;"><b>Privacy and Data Protection</b></p>
+            <p><b>While users may voluntarily submit their names and institutional affiliations, this information is not required and is collected solely for the purpose of academic recognition. Personal information submitted to this platform will be handled with care and used solely for the intended academic purposes. We are committed to protecting your privacy, and we will not share personal data with third parties without explicit consent.</b></p>
+
+            <p style="font-size:18px;"><b>Source of Models</b></p>
+            <p><b>All models evaluated and displayed on this platform are obtained from official sources, including but not limited to official repositories and Replicate.</b></p>
+
+            <p style="font-size:18px;"><b>Limitations of Liability</b></p>
+            <p><b>The platform and its administrators do not assume any legal liability for the use or interpretation of the information provided. The evaluations and comparisons are for academic purposes. Users should verify the information independently and must not use the platform for any illegal, harmful, violent, racist, or sexual purposes.</b></p>
+
+            <p style="font-size:18px;"><b>Modification of Terms</b></p>
+            <p><b>We reserve the right to modify these terms at any time. Users will be notified of significant changes through updates on the platform.</b></p>
+
+            <p style="font-size:18px;"><b>Contact Information</b></p>
+            <p><b>For any questions or to report issues, please contact us at [email protected].</b></p>
+        </div>
+        <div id="overlay" style="display:none; position:fixed; top:0; left:0; width:100%; height:100%; background:rgba(0,0,0,0.5); z-index:999;" onclick="document.getElementById('modal').style.display='none'; document.getElementById('overlay').style.display='none'"></div>
+        <p> This platform is designed for academic usage, for details please refer to <a href="#" id="open_link" onclick="document.getElementById('modal').style.display='block'; document.getElementById('overlay').style.display='block'">disclaimer</a>.</p>
+        '''
+    return disclaimer_md
+
+
+def make_arena_leaderboard_data(results):
+    import pandas as pd
+    df = pd.DataFrame(results)
+    return df
+
+
+def build_leaderboard_tab(score_result_file = 'sorted_score_list.json'):
+    with open(score_result_file, "r") as json_file:
+        data = json.load(json_file)
+    score_results = data["sorted_score_list"]
+    total_models = data["total_models"]
+    total_votes = data["total_votes"]
+    last_updated = data["last_updated"]
+
+    md = make_leaderboard_md()
+    md_1 = gr.Markdown(md, elem_id="leaderboard_markdown")
+
+    # with gr.Tab("Arena Score", id=0):
+    md = make_arena_leaderboard_md(total_models, total_votes, last_updated)
+    gr.Markdown(md, elem_id="leaderboard_markdown")
+    md = make_arena_leaderboard_data(score_results)
+    gr.Dataframe(md)
+
+    gr.Markdown(
+            """ 
+            - Note: When σ is large (we use the '*' labeling), it indicates that the model did not receive enough votes and its ranking is in the process of being updated.
+            """,
+                elem_id="sigma_note_markdown",
+            )
+    
+    gr.Markdown(
+                """ ### The leaderboard is regularly updated and continuously incorporates new models. 
+                """,
+                elem_id="leaderboard_markdown",
+            )
+    with gr.Blocks():
+        gr.HTML(make_disclaimer_md)
+    from .utils import acknowledgment_md, html_code
+    with gr.Blocks():
+        gr.Markdown(acknowledgment_md)
+
+
+def build_leaderboard_video_tab(score_result_file = 'sorted_score_list_video.json'):
+    with open(score_result_file, "r") as json_file:
+        data = json.load(json_file)
+    score_results = data["sorted_score_list"]
+    total_models = data["total_models"]
+    total_votes = data["total_votes"]
+    last_updated = data["last_updated"]
+
+    md = make_leaderboard_video_md()
+    md_1 = gr.Markdown(md, elem_id="leaderboard_markdown")
+    # with gr.Blocks():
+    #     gr.HTML(make_disclaimer_md)
+
+    # with gr.Tab("Arena Score", id=0):
+    md = make_arena_leaderboard_md(total_models, total_votes, last_updated)
+    gr.Markdown(md, elem_id="leaderboard_markdown")
+    md = make_arena_leaderboard_data(score_results)
+    gr.Dataframe(md)
+
+    notice_markdown_sora = """
+    - Note: When σ is large (we use the '*' labeling), it indicates that the model did not receive enough votes and its ranking is in the process of being updated.
+    - Note: As Sora's video generation function is not publicly available, we used sample videos from their official website. This may lead to a biased assessment of Sora's capabilities, as these samples likely represent Sora's best outputs. Therefore, Sora's position on our leaderboard should be considered as its upper bound. We are working on methods to conduct more comprehensive and fair comparisons in the future.
+    """
+
+    gr.Markdown(notice_markdown_sora, elem_id="notice_markdown_sora")
+    
+    gr.Markdown(
+                """ ### The leaderboard is regularly updated and continuously incorporates new models. 
+                """,
+                elem_id="leaderboard_markdown",
+            )
+    from .utils import acknowledgment_md, html_code
+    with gr.Blocks():
+        gr.Markdown(acknowledgment_md)
+
+
+def build_leaderboard_contributor(file = 'contributor.json'):
+
+    with open(file, "r") as json_file:
+        data = json.load(json_file)
+    score_results = data["contributor"]
+    last_updated = data["last_updated"]
+
+    md = f"""
+# 🏆 Contributor Leaderboard
+The submission of user information is entirely optional. This information is used solely for contribution statistics. We respect and safeguard users' privacy choices. 
+To maintain a clean and concise leaderboard, please ensure consistency in submitted names and affiliations. For example, use 'Berkeley' consistently rather than alternating with 'UC Berkeley'.
+- Votes*: Each image vote counts as one Vote*, while each video vote counts as two Votes* due to the increased effort involved.
+\n Last updated: {last_updated}
+"""
+
+    md_1 = gr.Markdown(md, elem_id="leaderboard_markdown")
+
+    # md = make_arena_leaderboard_md(total_models, total_votes, last_updated)
+    # gr.Markdown(md, elem_id="leaderboard_markdown")
+
+    md = make_arena_leaderboard_data(score_results)
+    gr.Dataframe(md)
+    
+    gr.Markdown(
+                """ ### The leaderboard is regularly updated. 
+                """,
+                elem_id="leaderboard_markdown",
+            )

serve/log_server.py

@@ -0,0 +1,86 @@
+from fastapi import FastAPI, File, UploadFile, Form, APIRouter
+from typing import Optional
+import json
+import os
+import aiofiles
+from .log_utils import build_logger
+from .constants import LOG_SERVER_SUBDOMAIN, APPEND_JSON, SAVE_IMAGE, SAVE_VIDEO, SAVE_LOG
+
+logger = build_logger("log_server", "log_server.log", add_remote_handler=False)
+
+app = APIRouter(prefix=LOG_SERVER_SUBDOMAIN)
+
+@app.post(f"/{APPEND_JSON}")
+async def append_json(json_str: str = Form(...), file_name: str = Form(...)):
+    """
+    Appends a JSON string to a specified file.
+    """
+    # Convert the string back to a JSON object (dict)
+    data = json.loads(json_str)
+    # Append the data to the specified file
+    if os.path.dirname(file_name):
+        os.makedirs(os.path.dirname(file_name), exist_ok=True)
+    async with aiofiles.open(file_name, mode='a') as f:
+        await f.write(json.dumps(data) + "\n")
+    
+    logger.info(f"Appended 1 JSON object to {file_name}")
+    return {"message": "JSON data appended successfully"}
+
+@app.post(f"/{SAVE_IMAGE}")
+async def save_image(image: UploadFile = File(...), image_path: str = Form(...)):
+    """
+    Saves an uploaded image to the specified path.
+    """
+    # Note: 'image_path' should include the file name and extension for the image to be saved.
+    if os.path.dirname(image_path):
+        os.makedirs(os.path.dirname(image_path), exist_ok=True)
+    async with aiofiles.open(image_path, mode='wb') as f:
+        content = await image.read()  # Read the content of the uploaded image
+        await f.write(content)  # Write the image content to a file
+    logger.info(f"Image saved successfully at {image_path}")
+    return {"message": f"Image saved successfully at {image_path}"}
+
+@app.post(f"/{SAVE_VIDEO}")
+async def save_video(video: UploadFile = File(...), video_path: str = Form(...)):
+    """
+    Saves an uploaded video to the specified path.
+    """
+    # Note: 'video_path' should include the file name and extension for the video to be saved.
+    if os.path.dirname(video_path):
+        os.makedirs(os.path.dirname(video_path), exist_ok=True)
+    async with aiofiles.open(video_path, mode='wb') as f:
+        content = await video.read()  # Read the content of the uploaded video
+        await f.write(content)  # Write the video content to a file
+    logger.info(f"Video saved successfully at {video_path}")
+    return {"message": f"Image saved successfully at {video_path}"}
+
+@app.post(f"/{SAVE_LOG}")
+async def save_log(message: str = Form(...), log_path: str = Form(...)):
+    """
+    Save a log message to a specified log file on the server.
+    """
+    # Ensure the directory for the log file exists
+    if os.path.dirname(log_path):
+        os.makedirs(os.path.dirname(log_path), exist_ok=True)
+    
+    # Append the log message to the specified log file
+    async with aiofiles.open(log_path, mode='a') as f:
+        await f.write(f"{message}\n")
+    
+    logger.info(f"Romote log message saved to {log_path}")
+    return {"message": f"Log message saved successfully to {log_path}"}
+
+
+@app.get(f"/read_file")
+async def read_file(file_name: str):
+    """
+    Reads the content of a specified file and returns it.
+    """
+    if not os.path.exists(file_name):
+        return {"message": f"File {file_name} does not exist."}
+
+    async with aiofiles.open(file_name, mode='r') as f:
+        content = await f.read()
+    
+    logger.info(f"Read file {file_name}")
+    return {"file_name": file_name, "content": content}

serve/log_utils.py

@@ -0,0 +1,142 @@
+"""
+Common utilities.
+"""
+from asyncio import AbstractEventLoop
+import json
+import logging
+import logging.handlers
+import os
+import platform
+import sys
+from typing import AsyncGenerator, Generator
+import warnings
+from pathlib import Path
+
+import requests
+
+from .constants import LOGDIR, LOG_SERVER_ADDR, SAVE_LOG
+from .utils import save_log_str_on_log_server
+
+
+handler = None
+visited_loggers = set()
+
+
+# Assuming LOGDIR and other necessary imports and global variables are defined
+
+class APIHandler(logging.Handler):
+    """Custom logging handler that sends logs to an API."""
+
+    def __init__(self, apiUrl, log_path, *args, **kwargs):
+        super(APIHandler, self).__init__(*args, **kwargs)
+        self.apiUrl = apiUrl
+        self.log_path = log_path
+
+    def emit(self, record):
+        log_entry = self.format(record)
+        try:
+            save_log_str_on_log_server(log_entry, self.log_path)
+        except requests.RequestException as e:
+            print(f"Error sending log to API: {e}", file=sys.stderr)
+
+def build_logger(logger_name, logger_filename, add_remote_handler=False):
+    global handler
+
+    formatter = logging.Formatter(
+        fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    # Set the format of root handlers
+    if not logging.getLogger().handlers:
+        if sys.version_info[1] >= 9:
+            # This is for windows
+            logging.basicConfig(level=logging.INFO, encoding="utf-8")
+        else:
+            if platform.system() == "Windows":
+                warnings.warn(
+                    "If you are running on Windows, "
+                    "we recommend you use Python >= 3.9 for UTF-8 encoding."
+                )
+            logging.basicConfig(level=logging.INFO)
+    logging.getLogger().handlers[0].setFormatter(formatter)
+
+    # Redirect stdout and stderr to loggers
+    stdout_logger = logging.getLogger("stdout")
+    stdout_logger.setLevel(logging.INFO)
+    sl = StreamToLogger(stdout_logger, logging.INFO)
+    sys.stdout = sl
+
+    stderr_logger = logging.getLogger("stderr")
+    stderr_logger.setLevel(logging.ERROR)
+    sl = StreamToLogger(stderr_logger, logging.ERROR)
+    sys.stderr = sl
+
+    # Get logger
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(logging.INFO)
+    
+    if add_remote_handler:
+        # Add APIHandler to send logs to your API
+        api_url = f"{LOG_SERVER_ADDR}/{SAVE_LOG}"
+        
+        remote_logger_filename = str(Path(logger_filename).stem + "_remote.log")
+        api_handler = APIHandler(apiUrl=api_url, log_path=f"{LOGDIR}/{remote_logger_filename}")
+        api_handler.setFormatter(formatter)
+        logger.addHandler(api_handler)
+        
+        stdout_logger.addHandler(api_handler)
+        stderr_logger.addHandler(api_handler)
+
+    # if LOGDIR is empty, then don't try output log to local file
+    if LOGDIR != "":
+        os.makedirs(LOGDIR, exist_ok=True)
+        filename = os.path.join(LOGDIR, logger_filename)
+        handler = logging.handlers.TimedRotatingFileHandler(
+            filename, when="D", utc=True, encoding="utf-8"
+        )
+        handler.setFormatter(formatter)
+
+        for l in [stdout_logger, stderr_logger, logger]:
+            if l in visited_loggers:
+                continue
+            visited_loggers.add(l)
+            l.addHandler(handler)
+
+    return logger
+
+
+class StreamToLogger(object):
+    """
+    Fake file-like stream object that redirects writes to a logger instance.
+    """
+
+    def __init__(self, logger, log_level=logging.INFO):
+        self.terminal = sys.stdout
+        self.logger = logger
+        self.log_level = log_level
+        self.linebuf = ""
+
+    def __getattr__(self, attr):
+        return getattr(self.terminal, attr)
+
+    def write(self, buf):
+        temp_linebuf = self.linebuf + buf
+        self.linebuf = ""
+        for line in temp_linebuf.splitlines(True):
+            # From the io.TextIOWrapper docs:
+            #   On output, if newline is None, any '\n' characters written
+            #   are translated to the system default line separator.
+            # By default sys.stdout.write() expects '\n' newlines and then
+            # translates them so this is still cross platform.
+            if line[-1] == "\n":
+                encoded_message = line.encode("utf-8", "ignore").decode("utf-8")
+                self.logger.log(self.log_level, encoded_message.rstrip())
+            else:
+                self.linebuf += line
+
+    def flush(self):
+        if self.linebuf != "":
+            encoded_message = self.linebuf.encode("utf-8", "ignore").decode("utf-8")
+            self.logger.log(self.log_level, encoded_message.rstrip())
+        self.linebuf = ""