Clone Leaderboard

app.py

@@ -9,8 +9,10 @@ from src.about import (
     CITATION_BUTTON_TEXT,
     EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
+    ABOUT_TEXT,
     TITLE,
+    Training_Dataset,
+    Testing_Type
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
@@ -21,7 +23,6 @@ from src.display.utils import (
     AutoEvalColumn,
     ModelType,
     fields,
-    WeightType,
     Precision
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
@@ -34,14 +35,12 @@ def restart_space():
 
 ### Space initialisation
 try:
-    print(EVAL_REQUESTS_PATH)
     snapshot_download(
         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
     )
 except Exception:
     restart_space()
 try:
-    print(EVAL_RESULTS_PATH)
     snapshot_download(
         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
     )
@@ -53,140 +52,160 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
 
 (
     finished_eval_queue_df,
-    running_eval_queue_df,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
+
+    with gr.Tabs(elem_classes="leaderboard-tabs") as leaderboard_tabs:
+        for testing_type in Testing_Type:
+            with gr.TabItem("Average Scores" if testing_type.value == "avg" else testing_type.name, elem_id=f"{testing_type.value}_Leaderboard"):
+                if testing_type.value == "avg":
+                    gr.Markdown("The scores presented in this tab are averaged scores across all datasets.")
+
+                try:
+                    leaderboard = Leaderboard(
+                        value=dataframe[dataframe["Testing Type"] == testing_type.name],
+                        datatype=[c.type for c in fields(AutoEvalColumn)],
+                        select_columns=SelectColumns(
+                            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+                            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+                            label="Select Columns to Display:",
+                        ),
+                        search_columns=[AutoEvalColumn.model_name.name],
+                        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+                        filter_columns=[
+                            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+                            ColumnFilter(AutoEvalColumn.training_dataset_type.name, type="checkboxgroup", label="Training Dataset"),
+                            ColumnFilter(
+                                AutoEvalColumn.model_parameters.name,
+                                type="slider",
+                                min=0,
+                                max=10000,
+                                default=["0", "100"],
+                                label="Select the number of parameters (M)",
+                            ),
+                        ],
+                        bool_checkboxgroup_label="Hide Models",
+                        interactive=False,
+                    )
+                except:
+                   gr.Markdown("There are no submissions for this testing type yet.")
+
+def init_submissions():
+    with gr.Column():
+        with gr.Row():
+            gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+        with gr.Column():
+            with gr.Accordion(
+                f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                open=False,
+            ):
+                with gr.Row():
+                    finished_eval_table = gr.components.Dataframe(
+                        value=finished_eval_queue_df,
+                        headers=EVAL_COLS,
+                        datatype=EVAL_TYPES,
+                        row_count=5,
+                    )
+
+            with gr.Accordion(
+                f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                open=False,
+            ):
+                with gr.Row():
+                    pending_eval_table = gr.components.Dataframe(
+                        value=pending_eval_queue_df,
+                        headers=EVAL_COLS,
+                        datatype=EVAL_TYPES,
+                        row_count=5,
+                    )
+
+    with gr.Row():
+        gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+    with gr.Row():
+        with gr.Column():
+            model_name_textbox = gr.Textbox(label="Model name")
+            model_link_textbox = gr.Textbox(label="Link to Model")
+            model_backbone_textbox = gr.Dropdown(
+                choices=["Original"],
+                label="Model Backbone",
+                value="Original",
+                allow_custom_value=True,
+            )
+
+            model_parameter_number = gr.Number(label="Model Parameter Count (M)", precision=1, minimum=0)
+
+            precision = gr.Dropdown(
+                choices=[i.name for i in Precision],
+                label="Precision",
+                multiselect=False,
+                value="float32",
+                interactive=True,
+            )
+            paper_name_textbox = gr.Textbox(label="Paper Name")
+            paper_link_textbox = gr.Textbox(label="Link To Paper")
+            
+
+        with gr.Column():
+            training_dataset = gr.Dropdown(
+                choices=[i.value for i in Training_Dataset if i.value != Training_Dataset.Other.value],
+                label="Training Dataset",
+                multiselect=False,
+                value=Training_Dataset.XCL.value,
+                interactive=True,
+                allow_custom_value=True,
+            )
+            testing_type = gr.Dropdown(
+                choices=[i.name for i in Testing_Type],
+                label="Tested on",
+                multiselect=False,
+                value=Testing_Type.AVG.name,
+                interactive=True,
+            )
+            cmap_value = gr.Number(label="cmAP Performance", precision=2, minimum=0.00, maximum=1.00, step=0.01)
+            auroc_value = gr.Number(label="AUROC Performance", precision=2, minimum=0.00, maximum=1.00, step=0.01)
+            t1acc_value = gr.Number(label="T1-Acc Performance", precision=2, minimum=0.00, maximum=1.00, step=0.01)
+
+    submit_button = gr.Button("Submit Eval")
+    submission_result = gr.Markdown()
+    submit_button.click(
+        fn=add_new_eval,
+        inputs=[
+            model_name_textbox,
+            model_link_textbox,
+            model_backbone_textbox,
+            precision,
+            model_parameter_number,
+            paper_name_textbox,
+            paper_link_textbox,
+            training_dataset,
+            testing_type,
+            cmap_value,
+            auroc_value,
+            t1acc_value,
         ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
+        outputs=submission_result,
     )
 
-
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+        with gr.TabItem("🏅 Leaderboard", elem_id="leaderboard-tab-table", id=0):
+            init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+        with gr.TabItem("📝 About", elem_id="leaderboard-tab-table", id=2):
+            gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+        with gr.TabItem("🚀 Submit here! ", elem_id="leaderboard-tab-table", id=3):
+            init_submissions()
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
@@ -201,4 +220,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.launch()

src/about.py

@@ -1,9 +1,51 @@
 from dataclasses import dataclass
 from enum import Enum
 
+class Model_Backbone(Enum):
+    Original  = "Original"
+    Other = "Other"
+
+    def from_str(model_backbone: str):
+        if model_backbone == Model_Backbone.Original.value:
+            return Model_Backbone.Original
+        return Model_Backbone.Other
+    
+    @classmethod
+    def format_for_leaderboard(cls, model_backbone: str):
+        return (cls.from_str(model_backbone), model_backbone)
+
+class Training_Dataset(Enum):
+    XCL = "BirdSet (XCL)"
+    XCM = "BirdSet (XCM)"
+    Dedicated = "BirdSet (Dedicated)"
+    Other = "other"
+
+    def from_str(training_dataset: str):
+        if training_dataset in [Training_Dataset.Dedicated.value, Training_Dataset.Dedicated.name, "BirdSet - Dedicated", "dt", "DT"]:
+            return Training_Dataset.Dedicated
+        if training_dataset in [Training_Dataset.XCM.value, Training_Dataset.XCM.name, "BirdSet - XCM", "mt", "MT"]:
+            return Training_Dataset.XCM
+        if training_dataset in [Training_Dataset.XCL.value, Training_Dataset.XCL.name, "BirdSet - XCL", "lt", "LT"]:
+            return Training_Dataset.XCL
+        return Training_Dataset.Other
+    
+    @classmethod
+    def format_for_leaderboard(cls, training_dataset: str):
+        return (cls.from_str(training_dataset), training_dataset)
+
+class Testing_Type(Enum):
+    AVG = "avg"
+    PER = "per"
+    NES = "nes"
+    UHH = "uhh"
+    HSN = "hsn"
+    NBP = "nbp"
+    SSW = "ssw"
+    SNE = "sne"
+
+
 @dataclass
 class Task:
-    benchmark: str
     metric: str
     col_name: str
 
@@ -11,62 +53,45 @@ class Task:
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    # metric_key in the json file, name to display in the leaderboard 
+    cmap = Task("cmap", "cmAP")
+    auroc = Task("auroc", "AUROC")
+    t1acc = Task("t1-acc", "T1-Acc")
 
-NUM_FEWSHOT = 0 # Change with your few shot
+NUM_FEWSHOT = 0
 # ---------------------------------------------------
 
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">BirdSet Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+This leaderboard accompanies the [BirdSet Dataset Collection](https://huggingface.co/datasets/DBD-research-group/BirdSet). You can find out more about BirdSet in the \"About\" Tab.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
+ABOUT_TEXT = f"""
+## What is BirdSet
+Deep learning models have emerged as a powerful tool in avian bioacoustics to assess environmental health.
+To maximize the potential of cost-effective and minimal-invasive passive acoustic monitoring (PAM), models must analyze bird vocalizations across a wide range of species and environmental conditions.
+However, data fragmentation challenges a evaluation of generalization performance.
+Therefore, we introduce the BirdSet dataset, comprising approximately 520,000 global bird recordings for training and over 400 hours PAM recordings for testing in a multi-label classification setting.  
+
+You can find the datasets on [Huggingface](https://huggingface.co/datasets/DBD-research-group/BirdSet) and the code on [Github](https://github.com/DBD-research-group/BirdSet).
 """
 
 EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+## How to Submit a Model
+First you need to evaluate your model on the BirdSet dataset.  
+Then you can enter your evaluation information and submit a request.  
+We will then check your request and approve it if everything is alright.
 
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+Please make sure that you model is publicly available so that we can check you results.
 
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+If you want to submit an average over all datasets then choose \"AVG\" as \"Tested on\".
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""
+CITATION_BUTTON_TEXT = r""""""

src/display/formatting.py

@@ -1,10 +1,10 @@
-def model_hyperlink(link, model_name):
+def make_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
 
 def make_clickable_model(model_name):
     link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
+    return make_hyperlink(link, model_name)
 
 
 def styled_error(error):

src/display/utils.py

@@ -23,22 +23,23 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["model_name", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["paper", ColumnContent, ColumnContent("Paper", "markdown", False)])
+auto_eval_column_dict.append(["training_dataset_type", ColumnContent, ColumnContent("Training Dataset Type", "markdown", False, hidden=True)])
+auto_eval_column_dict.append(["training_dataset", ColumnContent, ColumnContent("Training Dataset", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+#auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "markdown", False)])
+auto_eval_column_dict.append(["model_backbone_type", ColumnContent, ColumnContent("Model Backbone Type", "markdown", False, hidden=True)])
+auto_eval_column_dict.append(["model_backbone", ColumnContent, ColumnContent("Model Backbone", "str", True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "markdown", False)])
+auto_eval_column_dict.append(["model_parameters", ColumnContent, ColumnContent("Parameter Count", "markdown", False)])
+auto_eval_column_dict.append(["model_link", ColumnContent, ColumnContent("Link To Model", "markdown", True)])
+auto_eval_column_dict.append(["testing_type", ColumnContent, ColumnContent("Testing Type", "str", False, hidden=True)])
+
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -46,11 +47,10 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
+    model = ColumnContent("model", "str", True)
     precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
+    training_dataset = ColumnContent("training_dataset", "str", True)
+    testing_type = ColumnContent("testing_type", "str", True)
     status = ColumnContent("status", "str", True)
 
 ## All the model information that we might need
@@ -66,7 +66,7 @@ class ModelType(Enum):
     FT = ModelDetails(name="fine-tuned", symbol="🔶")
     IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
     RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
+    Other = ModelDetails(name="Other", symbol="?")
 
     def to_str(self, separator=" "):
         return f"{self.value.symbol}{separator}{self.value.name}"
@@ -81,27 +81,19 @@ class ModelType(Enum):
             return ModelType.RL
         if "instruction-tuned" in type or "⭕" in type:
             return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
+        return ModelType.Other
 
 class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
+    float32 =  "float32"
+    Other = "Other"
 
     def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
+        if precision in ["torch.float32", "float32"]:
+            return Precision.float32
+        return Precision.Other
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+COLS = [c.name for c in fields(AutoEvalColumn)]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]

src/envs.py

@@ -6,12 +6,12 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "DBD-research-group" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+REPO_ID = f"{OWNER}/BirdSet-Leaderboard"
+QUEUE_REPO = f"{OWNER}/Leaderboard-Requests"
+RESULTS_REPO = f"{OWNER}/Leaderboard-Results"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")

src/leaderboard/read_evals.py

@@ -7,30 +7,31 @@ from dataclasses import dataclass
 import dateutil
 import numpy as np
 
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
+from src.display.formatting import make_hyperlink
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision
+from src.about import Model_Backbone, Training_Dataset, Testing_Type
 
 
 @dataclass
 class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
     """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
+    Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # model_training_testing_precision (identifier for evaluations)
+    model_name: str
+    training_dataset_type: Training_Dataset
+    training_dataset: str
+    testing_type: Testing_Type
     results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
+    paper_name: str = ""
+    model_link: str = ""
+    paper_link: str = ""
+    model_backbone_type: Model_Backbone = Model_Backbone.Other
+    model_backbone: str = ""
+    precision: Precision = Precision.Other
+    model_parameters: float = 0
+    model_type: ModelType = ModelType.Other # Pretrained, fine tuned, ...
     date: str = "" # submission date of request file
-    still_on_hub: bool = False
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -40,118 +41,96 @@ class EvalResult:
 
         config = data.get("config")
 
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
+        # Extract evaluation config
+        model_name = config["model_name"]
+        training_dataset_type = Training_Dataset.from_str(config["training_dataset"])
+        if training_dataset_type.name != Training_Dataset.Other.name:
+            training_dataset = training_dataset_type.value
         else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+            training_dataset = config["training_dataset"]
+        testing_type = Testing_Type(config["testing_type"])
+        precision = Precision.from_str(config.get("model_dtype"))
+        eval_name = model_name + precision.value + training_dataset + testing_type.value
 
-        # Extract results available in this file (some results are split in several files)
+        # Extract results
         results = {}
         for task in Tasks:
             task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
+            results[task.metric] = data["results"].get(task.metric, -1)
 
         return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
+            eval_name=eval_name,
+            model_name=model_name,
+            training_dataset_type=training_dataset_type,
+            training_dataset=training_dataset,
+            testing_type=testing_type,
+            precision=precision,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
         )
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+        if self.training_dataset_type.name != Training_Dataset.Other.name:
+            training_dataset_request = self.training_dataset_type.name
+        else:
+            training_dataset_request = self.training_dataset
+        training_dataset_request = "_".join(training_dataset_request.split())
+        request_file = get_request_file_for_model(requests_path, self.model_name, self.precision.value, training_dataset_request, self.testing_type.value)
 
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
+            self.model_parameters = request.get("model_parameters", 0)
+            self.model_link = request.get("model_link", "None")
+            self.model_backbone = request.get("model_backbone", "Unknown")
+            self.model_backbone_type = Model_Backbone.from_str(self.model_backbone)
+            self.paper_name = request.get("paper_name", "None")
+            self.paper_link = request.get("paper_link", "None")
             self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            print(f"Could not find request file for {self.model_name} with precision {self.precision.value}, training dataset {self.training_dataset} and testing type {self.testing_type.value}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumn.precision.name: self.precision.value,
+            AutoEvalColumn.model_parameters.name: self.model_parameters,
+            AutoEvalColumn.model_name.name: self.model_name,
+            AutoEvalColumn.paper.name: make_hyperlink(self.paper_link, self.paper_name) if self.paper_link.startswith("http") else self.paper_name,
+            AutoEvalColumn.model_backbone_type.name: self.model_backbone_type.value,
+            AutoEvalColumn.model_backbone.name: self.model_backbone,
+            AutoEvalColumn.training_dataset_type.name: self.training_dataset_type.value,
+            AutoEvalColumn.training_dataset.name: self.training_dataset,
+            AutoEvalColumn.testing_type.name: self.testing_type.name,
+            AutoEvalColumn.model_link.name: self.model_link
         }
 
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            data_dict[task.value.col_name] = self.results[task.value.metric]
 
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
+def get_request_file_for_model(requests_path, model_name, precision, training_dataset, testing_type):
+    """Selects the correct request file for a given model if it's marked as FINISHED"""
+    request_filename = os.path.join(
         requests_path,
-        f"{model_name}_eval_request_*.json",
+        model_name,
+        f"{model_name}_eval_request_{precision}_{training_dataset}_{testing_type}.json",
     )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
+
+    # check for request file
+    try:
+        with open(request_filename, "r") as file:
+            req_content = json.load(file)
+            if req_content["status"] not in ["FINISHED"]:
+                return None
+    except OSError:
+        return None
+
+    return request_filename
 
 
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
@@ -163,12 +142,6 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
 
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
@@ -178,12 +151,8 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
         eval_result.update_with_request_file(requests_path)
 
-        # Store results of same eval together
         eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
+        eval_results[eval_name] = eval_result
 
     results = []
     for v in eval_results.values():
@@ -192,5 +161,5 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             results.append(v)
         except KeyError:  # not all eval values present
             continue
-
+    
     return results

src/populate.py

@@ -5,6 +5,7 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.about import Tasks, Training_Dataset
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
@@ -14,11 +15,11 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
+    df = df.sort_values(by=[Tasks.auroc.value.col_name, Tasks.cmap.value.col_name, Tasks.t1acc.value.col_name], ascending=False)
     return df
 
 
@@ -33,27 +34,28 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             with open(file_path) as fp:
                 data = json.load(fp)
 
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            data[EvalQueueColumn.model.name] = data["model_name"]
+            data[EvalQueueColumn.precision.name] = data.get("precision", "other")
+            data[EvalQueueColumn.training_dataset.name] = Training_Dataset.from_str(data.get("training_dataset", "other")).value
+            data[EvalQueueColumn.testing_type.name] = data["testing_type"]
 
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
-            #sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.isfile(e) and not e.startswith(".")]
-            sub_entries = []
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
             for sub_entry in sub_entries:
                 file_path = os.path.join(save_path, entry, sub_entry)
                 with open(file_path) as fp:
                     data = json.load(fp)
 
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                data[EvalQueueColumn.model.name] = data["model_name"]
+                data[EvalQueueColumn.precision.name] = data.get("precision", "other")
+                data[EvalQueueColumn.training_dataset.name] = Training_Dataset.from_str(data.get("training_dataset", "None"))
+                data[EvalQueueColumn.testing_type.name] = data["testing_type"]
                 all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+    return df_finished[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,84 +1,10 @@
 import json
 import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
 
 def already_submitted_models(requested_models_dir: str) -> set[str]:
     """Gather a list of already submitted models to avoid duplicates"""
     depth = 1
     file_names = []
-    users_to_submission_dates = defaultdict(list)
 
     for root, _, files in os.walk(requested_models_dir):
         current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
@@ -88,12 +14,6 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
+                    file_names.append(f"{info['model_name']}_{info['training_dataset']}_{info['testing_type']}_{info['precision']}")
 
-    return set(file_names), users_to_submission_dates
+    return set(file_names)

src/submission/submit.py

@@ -4,112 +4,91 @@ from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
 from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
+from src.submission.check_validity import already_submitted_models
+from src.about import Training_Dataset
+
 
 REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
 
 def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
+    model_name : str = None,
+    model_link : str = None,
+    model_backbone : str = "Unknown",
+    precision : str = None,
+    model_parameters: float = 0,
+    paper_name: str = "None",
+    paper_link: str = "None",
+    training_dataset: str = "",
+    testing_type : str = None,
+    cmap_value : float = 0,
+    auroc_value : float = 0,
+    t1acc_value : float = 0,
 ):
+
     global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+        REQUESTED_MODELS = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
+    if model_name is None or model_name == "":
+        return styled_error("Please enter a model name")
+    
+    if model_link is None or model_link == "":
+        return styled_error("Please provide a link to your model")
 
-    precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
+    training_dataset_type = Training_Dataset.from_str(training_dataset)
+    if training_dataset_type.name != Training_Dataset.Other.name:
+        training_dataset = training_dataset_type.name
 
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
+    training_dataset = "_".join(training_dataset.split())
+    model_name = "_".join(model_name.split())
+    testing_type = testing_type.lower()
 
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
     print("Adding new eval")
 
     eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
+        "model_name": model_name,
+        "model_link": model_link,
+        "model_backbone": model_backbone,
         "precision": precision,
-        "weight_type": weight_type,
+        "model_parameters": model_parameters,
+        "paper_name": paper_name,
+        "paper_link": paper_link,
         "status": "PENDING",
         "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
+        "training_dataset": training_dataset,
+        "testing_type": testing_type,
+        "claimed_cmap": cmap_value,
+        "claimed_auroc": auroc_value,
+        "claimed_t1acc": t1acc_value
     }
 
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+    if f"{model_name}_{training_dataset}_{testing_type}_{precision}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    try:
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model_name}"
+        os.makedirs(OUT_DIR, exist_ok=True)
+        out_path = f"{OUT_DIR}/{model_name}_eval_request_{precision}_{training_dataset}_{testing_type}.json"
 
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
+        with open(out_path, "w") as f:
+            f.write(json.dumps(eval_entry))
+    except:
+        return styled_error("There was an error while creating your request. Make sure there are no \"/\" in your model name.")
 
     print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
+    try:
+        API.upload_file(
+            path_or_fileobj=out_path,
+            path_in_repo=out_path.split("eval-queue/")[1],
+            repo_id=QUEUE_REPO,
+            repo_type="dataset",
+            commit_message=f"Add {model_name}_{training_dataset}_{testing_type} to eval queue",
+        )
+    except:
+        return styled_error("There was an error while uploading your request.")
 
     # Remove the local file
     os.remove(out_path)
@@ -117,3 +96,5 @@ def add_new_eval(
     return styled_message(
         "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
     )
+
+