import gradio as gr import os from serve.gradio_web import * from serve.gradio_web_video import build_side_by_side_video_ui_anony from serve.leaderboard import build_leaderboard_tab, build_leaderboard_video_tab, build_leaderboard_contributor from model.model_manager import ModelManager from pathlib import Path from serve.constants import SERVER_PORT, ROOT_PATH, ELO_RESULTS_DIR def make_default_md(): link_color = "#1976D2" # This color should be clear in both light and dark mode leaderboard_md = f""" # 🏅 K-Sort Arena: Efficient and Reliable Benchmarking for Generative Models via K-wise Human Preferences ### [Paper](https://arxiv.org/abs/2408.14468) | [Twitter](https://x.com/_akhaliq/status/1828280979242320014) - ⚡ For vision tasks, K-wise comparisons can provide much richer info but only take similar time as pairwise comparisons. - 🎯 Well designed matchmaking algorithm can further save human efforts than random match pairing in normal Arena. - 📈 Probabilistic modeling can obtain a faster and more stable convergence than Elo scoring system. """ return leaderboard_md def build_combine_demo(models, elo_results_file, leaderboard_table_file): with gr.Blocks( title="Play with Open Vision Models", theme=gr.themes.Default(), css=block_css, ) as demo: with gr.Blocks(): md = make_default_md() md_default = gr.Markdown(md, elem_id="default_leaderboard_markdown") with gr.Tabs() as tabs_combine: with gr.Tab("Image Generation", id=0): with gr.Tabs() as tabs_ig: with gr.Tab("Generation Leaderboard", id=0): build_leaderboard_tab() with gr.Tab("Generation Arena (battle)", id=1): build_side_by_side_ui_anony(models) with gr.Tab("Video Generation", id=1): with gr.Tabs() as tabs_ig: with gr.Tab("Generation Leaderboard", id=0): build_leaderboard_video_tab() with gr.Tab("Generation Arena (battle)", id=1): build_side_by_side_video_ui_anony(models) with gr.Tab("Contributor", id=2): build_leaderboard_contributor() return demo def load_elo_results(elo_results_dir): from collections import defaultdict elo_results_file = defaultdict(lambda: None) leaderboard_table_file = defaultdict(lambda: None) if elo_results_dir is not None: elo_results_dir = Path(elo_results_dir) elo_results_file = {} leaderboard_table_file = {} for file in elo_results_dir.glob('elo_results_*.pkl'): if 't2i_generation' in file.name: elo_results_file['t2i_generation'] = file # else: # raise ValueError(f"Unknown file name: {file.name}") for file in elo_results_dir.glob('*_leaderboard.csv'): if 't2i_generation' in file.name: leaderboard_table_file['t2i_generation'] = file # else: # raise ValueError(f"Unknown file name: {file.name}") return elo_results_file, leaderboard_table_file if __name__ == "__main__": server_port = int(SERVER_PORT) root_path = ROOT_PATH elo_results_dir = ELO_RESULTS_DIR models = ModelManager() elo_results_file, leaderboard_table_file = load_elo_results(elo_results_dir) demo = build_combine_demo(models, elo_results_file, leaderboard_table_file) demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, share=True) # demo.launch(server_name="0.0.0.0", server_port=7860, root_path=ROOT_PATH)