Implements leaderboard functionality and dataset download

Adds functions to download dataset snapshots and fetch leaderboard data.
Integrates leaderboard display with automatic refresh and user-friendly interface.
Enhances logging for better error tracking and debugging.

app.py

@@ -2,23 +2,29 @@ import datasets
 import gradio as gr
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
+from loguru import logger
 
+import populate
+from about import LEADERBOARD_INTRODUCTION_TEXT, LEADERBOARD_TITLE
 from app_configs import DEFAULT_SELECTIONS, THEME
 from components.quizbowl.bonus import BonusInterface
 from components.quizbowl.tossup import TossupInterface
+from display.css_html_js import fonts_header, js_head, leaderboard_css
 from display.custom_css import css_bonus, css_pipeline, css_tossup
 from display.guide import GUIDE_MARKDOWN
+from display.utils import AutoEvalColumn, fields
 
 # Constants
 from envs import (
     API,
     EVAL_REQUESTS_PATH,
     EVAL_RESULTS_PATH,
+    LEADERBOARD_REFRESH_INTERVAL,
     PLAYGROUND_DATASET_NAMES,
     QUEUE_REPO,
     REPO_ID,
     RESULTS_REPO,
-    TOKEN,
+    SERVER_REFRESH_INTERVAL,
 )
 from workflows import factory
 from workflows.configs import AVAILABLE_MODELS
@@ -28,107 +34,27 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-# Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO,
-        local_dir=EVAL_RESULTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
-except Exception:
-    restart_space()
-
-fonts_header = """
-<link rel="preconnect" href="https://fonts.googleapis.com">
-<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-<link href="https://fonts.googleapis.com/css2?family=Shantell+Sans:ital,wght@0,300..800;1,300..800&display=swap" rel="stylesheet">
-<link href="https://fonts.googleapis.com/css2?family=Space+Mono:ital,wght@0,400;0,700;1,400;1,700&display=swap" rel="stylesheet">
-<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;1,100;1,200;1,300;1,400;1,500;1,600;1,700&display=swap" rel="stylesheet">
-"""
-
-js_head = """
-<script>
-    const gradioApp = document.getElementsByTagName('gradio-app')[0];
-    console.log("Gradio app:", gradioApp);
-    console.log(gradioApp.querySelectorAll('.token'));
-    console.log(document.querySelectorAll('.token'));
-
-    // Function to trigger Python callback
-    const setHiddenIndex = (index) => {
-        console.log("Setting hidden index to:", index);
-        const hiddenIndex = gradioApp.querySelector("#hidden-index textarea");
-        if (hiddenIndex) {
-            hiddenIndex.value = index;
-            let event = new Event("input", { bubbles: true});
-            Object.defineProperty(event, "target", { value: hiddenIndex});
-            hiddenIndex.dispatchEvent(event);
-        }
-    };
-
-    // Add event listeners to all tokens
-    function setupTokenListeners() {
-        const tokens = gradioApp.querySelectorAll('.token');
-        console.log("Tokens:", tokens);
-        tokens.forEach(token => {
-            token.addEventListener('mouseover', function() {
-                const index = parseInt(this.getAttribute('data-index'));
-                console.log("Mouseover token index:", index);
-
-                // Reset all tokens
-                gradioApp.querySelectorAll('.token').forEach(el => {
-                    el.classList.remove('highlighted');
-                });
-
-                // Highlight this token
-                this.classList.add('highlighted');
-
-                // Update the hidden index to trigger the Python callback
-                setHiddenIndex(index);
-            });
-        });
-    }
-    console.log("Preamble complete");
-
-    document.addEventListener("DOMContentLoaded", function() {
-        // Setup initial listeners
-        console.log("DOM fully loaded and parsed");
-        setupTokenListeners();
-
-        // Setup a mutation observer to handle dynamically added tokens
-        const observer = new MutationObserver(function(mutations) {
-            mutations.forEach(function(mutation) {
-                if (mutation.addedNodes.length) {
-                    setupTokenListeners();
-                }
-            });
-        });
+def download_dataset_snapshot(repo_id, local_dir):
+    try:
+        logger.info(f"Downloading dataset snapshot from {repo_id} to {local_dir}")
+        snapshot_download(
+            repo_id=repo_id,
+            local_dir=local_dir,
+            repo_type="dataset",
+            tqdm_class=None,
+        )
+    except Exception as e:
+        logger.error(f"Error downloading dataset snapshot from {repo_id} to {local_dir}: {e}. Restarting space.")
+        restart_space()
+
+
+download_dataset_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH)
 
-        // Start observing the token container for changes
-        const tokenContainer = gradioApp.querySelector('.token-container');
-        console.log("Token container:", tokenContainer);
-        if (tokenContainer) {
-            observer.observe(tokenContainer.parentNode, { childList: true, subtree: true });
-        }
-        console.log("Listener setup complete");
-    });
-</script>
-"""
+
+def fetch_leaderboard_df():
+    logger.info("Leaderboard fetched...")
+    download_dataset_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH)
+    return populate.get_leaderboard_df(EVAL_RESULTS_PATH)
 
 
 def load_dataset(mode: str):
@@ -144,17 +70,24 @@ def load_dataset(mode: str):
     return ds
 
 
+def get_default_tab_id(request: gr.Request):
+    logger.info(f"Request: {request}")
+    tab_key_value = request.query_params.get("tab", "tossup")
+    return gr.update(selected=tab_key_value)
+
+
 if __name__ == "__main__":
     scheduler = BackgroundScheduler()
-    scheduler.add_job(restart_space, "interval", seconds=1800)
+    scheduler.add_job(restart_space, "interval", seconds=SERVER_REFRESH_INTERVAL)
     scheduler.start()
 
-    full_css = css_pipeline + css_tossup + css_bonus
+    css = css_pipeline + css_tossup + css_bonus + leaderboard_css
+    head = fonts_header + js_head
     tossup_ds = load_dataset("tossup")
     bonus_ds = load_dataset("bonus")
     with gr.Blocks(
-        css=full_css,
-        head=fonts_header + js_head,
+        css=css,
+        head=head,
         theme=THEME,
         title="Quizbowl Bot",
     ) as demo:
@@ -162,16 +95,31 @@ if __name__ == "__main__":
             gr.Markdown(GUIDE_MARKDOWN)
         with gr.Row():
             gr.Markdown("## Welcome to Quizbowl Bot! This is a tool for creating and testing quizbowl agents.")
-        with gr.Tabs():
-            with gr.Tab("Tossup Agents"):
+        with gr.Tabs() as gtab:
+            with gr.Tab("🛎️ Tossup Agents", id="tossup"):
                 defaults = DEFAULT_SELECTIONS["tossup"] | {
                     "init_workflow": factory.create_simple_qb_tossup_workflow(),
                 }
                 tossup_interface = TossupInterface(demo, tossup_ds, AVAILABLE_MODELS, defaults)
-            with gr.Tab("Bonus Round Agents"):
+            with gr.Tab("🙋🏻‍♂️ Bonus Round Agents", id="bonus"):
                 defaults = DEFAULT_SELECTIONS["bonus"] | {
                     "init_workflow": factory.create_simple_qb_bonus_workflow(),
                 }
                 bonus_interface = BonusInterface(demo, bonus_ds, AVAILABLE_MODELS, defaults)
+            with gr.Tab("🏅 Leaderboard", elem_id="llm-benchmark-tab-table", id="leaderboard"):
+                leaderboard_timer = gr.Timer(LEADERBOARD_REFRESH_INTERVAL)
+                gr.Markdown("<a id='leaderboard' href='#leaderboard'>QANTA Leaderboard</a>")
+                gr.Markdown(LEADERBOARD_INTRODUCTION_TEXT)
+                refresh_btn = gr.Button("🔄 Refresh")
+                leaderboard_table = gr.Dataframe(
+                    value=fetch_leaderboard_df,
+                    every=leaderboard_timer,
+                    headers=[c.name for c in fields(AutoEvalColumn)],
+                    datatype=[c.type for c in fields(AutoEvalColumn)],
+                    elem_id="leaderboard-table",
+                    interactive=False,
+                    visible=True,
+                )
+                refresh_btn.click(fn=fetch_leaderboard_df, inputs=[], outputs=leaderboard_table)
 
     demo.queue(default_concurrency_limit=40).launch()

src/about.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    task0 = Task("anli_r1", "acc", "ANLI")
+    task1 = Task("logiqa", "acc_norm", "LogiQA")
+
+
+NUM_FEWSHOT = 0  # Change with your few shot
+# ---------------------------------------------------
+
+
+# Your leaderboard name
+LEADERBOARD_TITLE = """<h1 align="center" id="space-title">QANTA Leaderboard</h1>"""
+
+# What does your leaderboard evaluate?
+LEADERBOARD_INTRODUCTION_TEXT = """
+Build an open-domain QA system that can answer any question posed by humans! For more: https://sites.google.com/view/qanta/home
+"""
+
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = """
+## How it works
+
+## Reproducibility
+To reproduce our results, here is the commands you can run:
+
+"""

src/components/quizbowl/populate.py

@@ -20,7 +20,7 @@ def get_pipeline_names(competition_type: str, profile: gr.OAuthProfile | None) -
     demo_example_names = submit.get_demo_example_submissions(competition_type)
     user_model_names = submit.get_user_submission_names(competition_type, profile)
     all_names = demo_example_names + user_model_names
-    logger.info("Loaded model names: {all_names}")
+    logger.info(f"Loaded model names: {all_names}")
     return all_names
 
 

src/display/css_html_js.py

@@ -1,4 +1,4 @@
-custom_css = """
+leaderboard_css = """
 
 .markdown-text {
     font-size: 16px !important;
@@ -46,27 +46,6 @@ custom_css = """
     white-space: nowrap;
 }
 
-/* Workflow JSON styling */
-.workflow-json-container {
-    margin-top: 20px;
-    margin-bottom: 30px;
-}
-
-.workflow-json {
-    border: 1px solid #ddd;
-    border-radius: 8px;
-    box-shadow: 0 2px 5px rgba(0,0,0,0.1);
-}
-
-.workflow-json pre {
-    max-height: 500px;
-    overflow-y: auto;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
 #scale-logo {
     border-style: none !important;
     box-shadow: none;
@@ -113,6 +92,30 @@ custom_css = """
 }
 """
 
+
+workflow_json_css = """
+/* Workflow JSON styling */
+.workflow-json-container {
+    margin-top: 20px;
+    margin-bottom: 30px;
+}
+
+.workflow-json {
+    border: 1px solid #ddd;
+    border-radius: 8px;
+    box-shadow: 0 2px 5px rgba(0,0,0,0.1);
+}
+
+.workflow-json pre {
+    max-height: 500px;
+    overflow-y: auto;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+"""
+
 get_window_url_params = """
     function(url_params) {
         const params = new URLSearchParams(window.location.search);
@@ -120,3 +123,80 @@ get_window_url_params = """
         return url_params;
     }
     """
+
+
+fonts_header = """
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Shantell+Sans:ital,wght@0,300..800;1,300..800&display=swap" rel="stylesheet">
+<link href="https://fonts.googleapis.com/css2?family=Space+Mono:ital,wght@0,400;0,700;1,400;1,700&display=swap" rel="stylesheet">
+<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;1,100;1,200;1,300;1,400;1,500;1,600;1,700&display=swap" rel="stylesheet">
+"""
+
+js_head = """
+<script>
+    const gradioApp = document.getElementsByTagName('gradio-app')[0];
+    console.log("Gradio app:", gradioApp);
+    console.log(gradioApp.querySelectorAll('.token'));
+    console.log(document.querySelectorAll('.token'));
+
+    // Function to trigger Python callback
+    const setHiddenIndex = (index) => {
+        console.log("Setting hidden index to:", index);
+        const hiddenIndex = gradioApp.querySelector("#hidden-index textarea");
+        if (hiddenIndex) {
+            hiddenIndex.value = index;
+            let event = new Event("input", { bubbles: true});
+            Object.defineProperty(event, "target", { value: hiddenIndex});
+            hiddenIndex.dispatchEvent(event);
+        }
+    };
+
+    // Add event listeners to all tokens
+    function setupTokenListeners() {
+        const tokens = gradioApp.querySelectorAll('.token');
+        console.log("Tokens:", tokens);
+        tokens.forEach(token => {
+            token.addEventListener('mouseover', function() {
+                const index = parseInt(this.getAttribute('data-index'));
+                console.log("Mouseover token index:", index);
+
+                // Reset all tokens
+                gradioApp.querySelectorAll('.token').forEach(el => {
+                    el.classList.remove('highlighted');
+                });
+
+                // Highlight this token
+                this.classList.add('highlighted');
+
+                // Update the hidden index to trigger the Python callback
+                setHiddenIndex(index);
+            });
+        });
+    }
+    console.log("Preamble complete");
+
+    document.addEventListener("DOMContentLoaded", function() {
+        // Setup initial listeners
+        console.log("DOM fully loaded and parsed");
+        setupTokenListeners();
+
+        // Setup a mutation observer to handle dynamically added tokens
+        const observer = new MutationObserver(function(mutations) {
+            mutations.forEach(function(mutation) {
+                if (mutation.addedNodes.length) {
+                    setupTokenListeners();
+                }
+            });
+        });
+
+        // Start observing the token container for changes
+        const tokenContainer = gradioApp.querySelector('.token-container');
+        console.log("Token container:", tokenContainer);
+        if (tokenContainer) {
+            observer.observe(tokenContainer.parentNode, { childList: true, subtree: true });
+        }
+        console.log("Listener setup complete");
+    });
+</script>
+"""

src/display/utils.py

@@ -1,110 +1,20 @@
 from dataclasses import dataclass, make_dataclass
-from enum import Enum
 
-import pandas as pd
-
-from src.about import Tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
 @dataclass
 class ColumnContent:
     name: str
     type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
 
-## Leaderboard columns
+
 auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown")])
+auto_eval_column_dict.append(["buzz_accuracy", ColumnContent, ColumnContent("Buzz Accuracy ⬆️", "number")])
+auto_eval_column_dict.append(["win_rate_human", ColumnContent, ColumnContent("Win Rate (Human Teams)", "number")])
+auto_eval_column_dict.append(["win_rate_model", ColumnContent, ColumnContent("Win Rate (Model Teams)", "number")])
 
-# We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/display/utils_old.py

@@ -0,0 +1,116 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+import pandas as pd
+
+from about import Tasks
+
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = ""  # emoji
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        return Precision.Unknown
+
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]

src/envs.py

@@ -9,22 +9,22 @@ OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
 ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY")
 COHERE_API_KEY = os.environ.get("COHERE_API_KEY")
 
-OWNER = (
-    "umdclip"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
-)
-# ----------------------------------
+# Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "umdclip"
 
 REPO_ID = f"{OWNER}/quizbowl-submission"
 QUEUE_REPO = f"{OWNER}/advcal-requests"
-RESULTS_REPO = f"{OWNER}/advcal-results"
+RESULTS_REPO = f"{OWNER}/model-results"  # TODO: change to advcal-results after testing is done
 
 EXAMPLES_PATH = "examples"
 
 PLAYGROUND_DATASET_NAMES = {
-    "tossup": "umdclip/acf-co24-tossups",
-    "bonus": "umdclip/acf-co24-bonuses",
+    "tossup": f"{OWNER}/acf-co24-tossups",
+    "bonus": f"{OWNER}/acf-co24-bonuses",
 }
 
+# ----------------------------------
+
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH = os.getenv("HF_HOME", ".")
 
@@ -35,4 +35,7 @@ EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
 
+SERVER_REFRESH_INTERVAL = 86400  # seconds (one day)
+LEADERBOARD_REFRESH_INTERVAL = 600  # seconds (10 minutes)
+
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -0,0 +1,195 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from display.formatting import make_clickable_model
+from display.utils_old import AutoEvalColumn, ModelType, Precision, Tasks, WeightType
+from submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
+
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
+    org: str
+    model: str
+    revision: str  # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original  # Original or Adapter
+    architecture: str = "Unknown"
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = ""  # submission date of request file
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision.value.name}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision.value.name}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,
+            revision=config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture,
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(
+                f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}"
+            )
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if req_content["status"] in ["FINISHED"] and req_content["precision"] == precision.split(".")[-1]:
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict()  # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/populate.py

@@ -0,0 +1,69 @@
+import json
+import os
+
+import pandas as pd
+
+from display.formatting import make_clickable_model
+from display.utils_old import EvalQueueColumn
+
+
+def get_leaderboard_df(results_path: str) -> pd.DataFrame:
+    model_result_filepaths = []
+    for root, _, files in os.walk(results_path):
+        if len(files) == 0 or not all(f.endswith(".json") for f in files):
+            continue
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {"model": [], "buzz_accuracy": [], "win_rate_human": [], "win_rate_model": []}
+    for model_result_filepath in model_result_filepaths:
+        with open(model_result_filepath, "r") as fin:
+            model_result = json.load(fin)
+            model_id = model_result["model_id"]
+            buzz_accuracy = model_result["buzz_accuracy"]
+            win_rate_human = model_result["win_rate_human"]
+            win_rate_model = model_result["win_rate_model"]
+            eval_results["model"].append(model_id)
+            eval_results["buzz_accuracy"].append(buzz_accuracy)
+            eval_results["win_rate_human"].append(win_rate_human)
+            eval_results["win_rate_model"].append(win_rate_model)
+    return pd.DataFrame(eval_results)
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    # TODO: This function is stale, but might be a good reference point for new implementation
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [
+                e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")
+            ]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]