update leaderboard logic

app.py

@@ -1,5 +1,6 @@
 import json
 import gzip
+import shutils
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
@@ -81,7 +82,6 @@ def init_leaderboard(dataframe, track):
         interactive=False,
     )
 
-submitted_predictions = {}
 def process_json(temp_file):
     if temp_file is None:
         return {}
@@ -92,11 +92,9 @@ def process_json(temp_file):
         if file_path.endswith('.gz'):
             with gzip.open(file_path, 'rt') as f:
                 data = json.load(f)
-                submitted_predictions.update(data)
         else:
             with open(file_path, 'r') as f:
                 data = json.load(f)
-                submitted_predictions.update(data)
     except Exception as e:
         raise gr.Error(f"Error processing file: {str(e)}")
 
@@ -164,7 +162,8 @@ with demo:
 
             with gr.Row():
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
+                    model_name_textbox = gr.Textbox(label="Model name. This will be displayed on the leaderboard.")
+                    model_id_textbox = gr.Textbox(label="Huggingface model ID (if applicable). This looks like `owner/repo_id`, not like a URL.", placeholder="")
                     revision_name_textbox = gr.Textbox(label="Model revision commit", placeholder="main")
                     track_name = gr.Dropdown(
                         choices = ["strict", "strict-small", "multimodal"],
@@ -174,12 +173,12 @@ with demo:
                         interactive=True
                     )
 
+                    predictions_data = gr.State()
                     upload_button = gr.UploadButton(label="Upload predictions", file_types=[".json", ".gz"], file_count="single")
-                    output_json = gr.JSON(label="Processed JSON")
                     upload_button.upload(
                         fn=process_json,
                         inputs=upload_button,
-                        outputs=output_json,
+                        outputs=predictions_data,
                         api_name="upload_json"
                     )
                 
@@ -189,9 +188,10 @@ with demo:
                 add_new_eval,
                 [
                     model_name_textbox,
+                    model_id_textbox,
                     revision_name_textbox,
                     track_name,
-                    upload_button,
+                    predictions_data,
                 ],
                 submission_result,
             )

src/about.py

@@ -61,4 +61,5 @@ Make sure your model has an open license! This is a leaderboard that is meant to
 
 CITATION_BUTTON_LABEL = "If you would like to cite these results, please cite the 2024 BabyLM Findings paper, as well as the authors of the model(s) whose results you cite!"
 CITATION_BUTTON_TEXT = r"""
+Stay tuned!
 """

src/display/formatting.py

@@ -2,8 +2,8 @@ def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
 
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
+def make_clickable_model(model_repo, model_name):
+    link = f"https://huggingface.co/{model_repo}"
     return model_hyperlink(link, model_name)
 
 

src/display/utils.py

@@ -25,6 +25,7 @@ auto_eval_column_dict = []
 auto_eval_column_dict_multimodal = []
 # Init
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["hf_repo", ColumnContent, ColumnContent("HF Repo", "str", False)])
 auto_eval_column_dict.append(["track", ColumnContent, ColumnContent("Track", "markdown", False)])
 #Scores
 for task in Tasks:
@@ -35,6 +36,7 @@ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Avai
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 auto_eval_column_dict_multimodal.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict_multimodal.append(["hf_repo", ColumnContent, ColumnContent("HF Repo", "str", False)])
 auto_eval_column_dict_multimodal.append(["track", ColumnContent, ColumnContent("Track", "markdown", False)])
 for task in TasksMultimodal:
     auto_eval_column_dict_multimodal.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])

src/leaderboard/read_evals.py

@@ -17,7 +17,8 @@ class EvalResult:
     """Represents one full evaluation. Built from a combination of the result and request file for a given run.
     """
     eval_name: str # org_model_track (uid)
-    full_model: str # org/model (path on hub)
+    full_model: str # org/model (name of model)
+    repo_id: str # org/model (path to model on HF)
     track: str
     org: str 
     model: str
@@ -37,6 +38,7 @@ class EvalResult:
 
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
+        repo_id = config.get("hf_repo", config.get("hf_repo", None))
         org_and_model = org_and_model.split("/", 1)
 
         if len(org_and_model) == 1:
@@ -49,7 +51,7 @@ class EvalResult:
         eval_name = "_".join(org_and_model) + f"_{track}"
 
         still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+            repo_id, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
         )
 
         def _get_task_results(task):
@@ -80,6 +82,7 @@ class EvalResult:
         return self(
             eval_name=eval_name,
             full_model=full_model,
+            repo_id=repo_id,
             track=track,
             org=org,
             model=model,
@@ -103,10 +106,16 @@ class EvalResult:
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         eval_column = AutoEvalColumnMultimodal if self.track.lower() == "multimodal" else AutoEvalColumn
         vision_tasks = ("VQA", "Winoground", "DevBench", "vqa", "winoground", "devbench")
-        text_average = sum([v for k, v in self.results.items() if v is not None and k not in vision_tasks]) / len(Tasks)
+        num_text_tasks = len(Tasks)
+        text_average = sum([v for k, v in self.results.items() if v is not None and k not in vision_tasks]) / num_text_tasks
+        if self.still_on_hub:
+            model_display_name = make_clickable_model(self.full_model)
+        else:
+            model_display_name = self.full_model
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            eval_column.model.name: make_clickable_model(self.full_model),
+            eval_column.model.name: model_display_name,
+            eval_column.hf_repo.name: self.repo_id,
             eval_column.revision.name: self.revision,
             eval_column.text_average.name: text_average,
             eval_column.still_on_hub.name: self.still_on_hub,
@@ -114,7 +123,8 @@ class EvalResult:
 
         if self.track.lower() == "multimodal":
             taskset = TasksMultimodal
-            vision_average = sum([v for k, v in self.results.items() if v is not None and k in vision_tasks]) / len(Tasks)
+            num_vision_tasks = len(TasksMultimodal) - len(Tasks)
+            vision_average = sum([v for k, v in self.results.items() if v is not None and k in vision_tasks]) / num_vision_tasks
             data_dict[eval_column.vision_average.name] = vision_average
         else:
             taskset = Tasks

src/populate.py

@@ -41,8 +41,12 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             with open(file_path) as fp:
                 data = json.load(fp)
 
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            if data["still_on_hub"]:
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["hf_repo"], data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            else:
+                data[EvalQueueColumn.model.name] = data["model"]
+                data[EvalQueueColumn.revision.name] = "N/A"
 
             all_evals.append(data)
         elif ".md" not in entry:

src/submission/submit.py

@@ -16,6 +16,7 @@ USERS_TO_SUBMISSION_DATES = None
 
 def add_new_eval(
     model_name: str,
+    model_id: str,
     revision: str,
     track: str,
     predictions: dict,
@@ -25,6 +26,8 @@ def add_new_eval(
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
+    out_message = ""
+
     user_name = ""
     model_path = model_name
     if "/" in model_name:
@@ -42,19 +45,20 @@ def add_new_eval(
 
     # Is the model info correctly filled?
     try:
-        model_info = API.model_info(repo_id=model_name, revision=revision)
+        model_info = API.model_info(repo_id=model_id, revision=revision)
     except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
+        return styled_warning("Could not get your model information. Please fill it up properly.")
 
     modelcard_OK, error_msg = check_model_card(model_name)
     if not modelcard_OK:
-        return styled_error(error_msg)
+        return styled_warning(error_msg)
 
     # Seems good, creating the eval
     print("Adding new eval")
 
     eval_entry = {
         "model_name": model_name,
+        "hf_repo": model_id,
         "revision": revision,
         "track": track,
         "predictions": predictions,