4

app.py

@@ -70,23 +70,9 @@ def init_leaderboard(dataframe):
     return Leaderboard(
         value=dataframe,
         datatype=[c.type for c in fields(AutoEvalColumn)],
-        # select_columns=SelectColumns(
-        #     default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-        #     cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-        #     label="Select Columns to Display:",
-        # ),
         search_columns=[AutoEvalColumn.model.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        hide_columns=["Available on the hub"],
         filter_columns=[
-            # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            # ColumnFilter(
-            #     AutoEvalColumn.params.name,
-            #     type="slider",
-            #     min=0.01,
-            #     max=150,
-            #     label="Select the number of parameters (B)",
-            # ),
             ColumnFilter(
                 AutoEvalColumn.still_on_hub.name, type="boolean", label="🔑 Show Open Models Only", default=False
             ),

src/display/formatting.py

@@ -2,9 +2,9 @@ def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
 
-def make_clickable_model(model_name):
+def make_clickable_model(model_show , model_name):
     link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
+    return model_hyperlink(link, model_show)
 
 
 def styled_error(error):

src/display/utils.py

@@ -24,22 +24,12 @@ class ColumnContent:
 auto_eval_column_dict = []
 #TODO
 # Init
-#auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-# #Scores
-# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "float", True , never_hidden= True)])
-# Model information
-#auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-#auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-#auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-#auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-#auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-#auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "float", False)])
-#auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-#auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)

src/leaderboard/read_evals.py

@@ -11,6 +11,15 @@ from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
 from src.submission.check_validity import is_model_on_hub
 
+from huggingface_hub import model_info, HfApi
+
+def is_model_open_source(org_model: str) -> bool:
+    api = HfApi()
+    try:
+        info = model_info(org_model) 
+        return True 
+    except Exception:
+        return False 
 
 @dataclass
 class EvalResult:
@@ -18,8 +27,7 @@ class EvalResult:
     """
     eval_name: str # org_model_precision (uid)
     full_model: str # org/model (path on hub)
-    org: str 
-    model: str
+    model_show : str # model name to display
     revision: str # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
@@ -39,8 +47,12 @@ class EvalResult:
             data = json.load(fp)
         config = data.get("config")
 
-
-
+        is_open = is_model_open_source(config.get("model_name", ""))
+        model_show = config.get("model_show", "")
+        if is_open:
+            model_to_show = f"🔑 {model_show}"
+        else:
+            model_to_show = f"🔒 {model_show}"
         # Extract results available in this file (some results are split in several files)
         results = {}
         for task in Tasks:
@@ -55,11 +67,12 @@ class EvalResult:
             results[task.benchmark] = mean_acc
 
         return self(
-            eval_name= config.get("model_show", ""),
-            full_model= config.get("model_name", ""),
-            results=results, 
-            revision= config.get("model_sha", ""),
-            still_on_hub=config.get("open_source", False)
+            eval_name = model_show,
+            full_model = config.get("model_name", ""),
+            model_show = model_to_show,
+            results = results, 
+            revision = config.get("model_sha", ""),
+            still_on_hub = is_open,
         )
 
     def update_with_request_file(self, requests_path):
@@ -72,26 +85,15 @@ class EvalResult:
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            print(f"Could not find request file for {self.full_model} ")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
 
-        # The first one is the average
-        #average = next(iter(self.results.values()))
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            #AutoEvalColumn.precision.name: self.precision.value.name,   #results/precision
-            #AutoEvalColumn.model_type.name: self.model_type.value.name,  #requests/model_type
-            #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            #AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            #AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            #AutoEvalColumn.revision.name: self.revision,
-            #AutoEvalColumn.average.name: average,
-            #AutoEvalColumn.license.name: self.license,
-            #AutoEvalColumn.likes.name: self.likes,
-            #AutoEvalColumn.params.name: self.num_params,
+
+            AutoEvalColumn.model.name: make_clickable_model(self.model_show , self.full_model),
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
@@ -108,19 +110,15 @@ def get_request_file_for_model(requests_path, model_name, precision):
         requests_path,
         f"{model_name}_eval_request_*.json",
     )
-    print(request_files)
     request_files = glob.glob(request_files)
-    print(request_files)
     # Select correct request file (precision)
     request_file = ""
     request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
-        print(tmp_request_file)
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
             if (
                 req_content["status"] in ["FINISHED"]
-                #and req_content["precision"] == precision.split(".")[-1]
             ):
                 request_file = tmp_request_file
     return request_file

src/populate.py

@@ -15,7 +15,6 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-
     df = df.sort_values(by=[AutoEvalColumn.task0.name], ascending=False)
     df = df[cols].round(decimals=2)