Fixed Average Error

app.py

@@ -265,7 +265,7 @@ with demo:
             )
             # Check query parameter once at startup and update search bar + hidden component
             demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
-            
+
             for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
                 selector.change(
                     update_table,

eval-queue/.gitattributes

@@ -53,3 +53,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

eval-results/.gitattributes

@@ -53,3 +53,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

eval-results/01-ai/Yi-1.5-9B-32K/result_2024_07_30 20:36:30.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.015945068581236614,
             "mc2": 0.4670848140389129,
             "mc2_stderr": 0.01585178282587417
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.47107438016528924,
-            "acc_stderr": 0.017161563949916348,
-            "acc_norm": 0.5171192443919717,
-            "acc_norm_stderr": 0.017180275246085626
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "01-ai/Yi-1.5-9B-32K",

eval-results/BioMistral/BioMistral-7B/BioMistral_BioMistral-7B_result_2024-05-30 01_33_58.json

@@ -1,10 +1,10 @@
 {
     "results": {
         "daily": {
-          "daily": 10
+          "daily": 9
         },
         "quarterly": {
-          "quarterly": 10
+          "quarterly": 9
         },
         "harness|arc_challenge|25": {
             "acc": 0.257679180887372,
@@ -365,12 +365,6 @@
             "mc1_stderr": 0.016150201321323002,
             "mc2": 0.4721418472000992,
             "mc2_stderr": 0.01626625866283201
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.27863046044864226,
-            "acc_stderr": 0.01541373949434568,
-            "acc_norm": 0.3825265643447462,
-            "acc_norm_stderr": 0.016709165387228803
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "BioMistral/BioMistral-7B",

eval-results/EleutherAI/polyglot-ko-1.3b/EleutherAI_polyglot-ko-1.3b_result_2023-09-24 15_21_38.json

@@ -1,10 +1,10 @@
 {
     "results": {
         "daily": {
-          "daily": 11
+          "daily": 10
         },
         "quarterly": {
-          "quarterly": 11
+          "quarterly": 10
         },
         "harness|arc_challenge|25": {
             "acc": 0.2235494880546075,
@@ -365,12 +365,6 @@
             "mc1_stderr": 0.015176985027707682,
             "mc2": 0.4116568832959107,
             "mc2_stderr": 0.015044504977529799
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.27744982290436837,
-            "acc_stderr": 0.015393630236605975,
-            "acc_norm": 0.3400236127508855,
-            "acc_norm_stderr": 0.016286717220737674
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "EleutherAI/polyglot-ko-1.3b",

eval-results/HuggingFaceH4/zephyr-7b-beta/result.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.01648214881024147,
             "mc2": 0.5171680571717291,
             "mc2_stderr": 0.01606077987901482
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.39787485242030696,
-            "acc_stderr": 0.01682795905473339,
-            "acc_norm": 0.4014167650531287,
-            "acc_norm_stderr": 0.01685290785872906
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "HuggingFaceH4/zephyr-7b-beta",

eval-results/nlpai-lab/KULLM3/result.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.016542412809494877,
             "mc2": 0.49995145184296846,
             "mc2_stderr": 0.015887726098900913
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.564344746162928,
-            "acc_stderr": 0.017047415229476316,
-            "acc_norm": 0.6068476977567887,
-            "acc_norm_stderr": 0.016793262801287068
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "nlpai-lab/KULLM3",

eval-results/x2bee/POLAR-14B-DPO-v1.3/result.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.01687480500145318,
             "mc2": 0.7522925779273922,
             "mc2_stderr": 0.014568927682929578
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.45218417945690675,
-            "acc_stderr": 0.017111567130916785,
-            "acc_norm": 0.45454545454545453,
-            "acc_norm_stderr": 0.017119172208061504
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "x2bee/POLAR-14B-DPO-v1.3",

eval-results/x2bee/POLAR-14B-DPO-v1.4/result.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.01746379386716811,
             "mc2": NaN,
             "mc2_stderr": NaN
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.44037780401416765,
-            "acc_stderr": 0.01706769977431298,
-            "acc_norm": 0.44510035419126326,
-            "acc_norm_stderr": 0.01708641743100547
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "x2bee/POLAR-14B-DPO-v1.4",

eval-results/x2bee/POLAR-14B-HES-DPO-v1.5/result.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.0165424128094949,
             "mc2": 0.7515104740134964,
             "mc2_stderr": 0.014200593490054807
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.5147579693034239,
-            "acc_stderr": 0.01718286443499856,
-            "acc_norm": 0.526564344746163,
-            "acc_norm_stderr": 0.017166075717577747
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "x2bee/POLAR-14B-HES-DPO-v1.5",

eval-results/x2bee/POLAR-14B-SON-SFT-v0.1/result.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.017106588140700332,
             "mc2": 0.7254831072808595,
             "mc2_stderr": 0.014162522228042162
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.5926800472255017,
-            "acc_stderr": 0.01689245669519127,
-            "acc_norm": 0.6269185360094451,
-            "acc_norm_stderr": 0.016627318275137453
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "x2bee/POLAR-14B-SON-SFT-v0.1",

eval-results/x2bee/POLAR-14B-v0.2/result.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.01563813566777552,
             "mc2": 0.8107575910195236,
             "mc2_stderr": 0.013335029489665237
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.525383707201889,
-            "acc_stderr": 0.017168187201429253,
-            "acc_norm": 0.5442739079102715,
-            "acc_norm_stderr": 0.017122829143292655
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "x2bee/POLAR-14B-v0.2",

eval-results/x2bee/POLAR-14B-v0.5/result.json

@@ -365,12 +365,6 @@
             "mc1_stderr": 0.014421468452506978,
             "mc2": 0.8572574997405501,
             "mc2_stderr": 0.01200311225898601
-        },
-        "harness|commongen_v2|2": {
-            "acc": 0.5159386068476978,
-            "acc_stderr": 0.017181617837190195,
-            "acc_norm": 0.5301062573789846,
-            "acc_norm_stderr": 0.01715916359017022
         }
     },
     "versions": {
@@ -434,8 +428,7 @@
         "harness|mmlu_professional_law|5": 1,
         "harness|mmlu_high_school_us_history|5": 1,
         "harness|mmlu_high_school_european_history|5": 1,
-        "harness|truthfulqa_mc|0": 0,
-        "harness|commongen_v2|2": 1
+        "harness|truthfulqa_mc|0": 0
     },
     "config_general": {
         "model_name": "x2bee/POLAR-14B-v0.5",

src/display/utils.py

@@ -3,8 +3,11 @@ from enum import Enum
 
 import pandas as pd
 
+
 def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+    return [
+        v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
+    ]
 
 
 @dataclass
@@ -13,6 +16,7 @@ class Task:
     metric: str
     col_name: str
 
+
 class Tasks(Enum):
     arc = Task("arc_challenge", "acc_norm", "ARC")
     hellaswag = Task("hellaswag", "acc_norm", "HellaSwag")
@@ -20,12 +24,13 @@ class Tasks(Enum):
     truthfulqa = Task("truthfulqa_mc", "mc2", "TruthfulQA")
     # winogrande = Task("winogrande", "acc_norm", "Winogrande")
     # gsm8k = Task("gsm8k", "acc_norm", "GSM8k")
-    commongen_v2 = Task("commongen_v2", "acc_norm", "CommonGen V2")
+    # commongen_v2 = Task("commongen_v2", "acc_norm", "CommonGen V2")
     # eqBench = Task("eq_bench", "acc_norm", "EQ Bench")
     # instFollow = Task("inst_follow", "acc_norm", "InstFollow")
     # harmlessness = Task("harmlessness", "acc_norm", "Harmlessness")
     # helpfulness = Task("helpfulness", "acc_norm", "Helpfulness")
 
+
 class Ranks(Enum):
     daily = Task("daily", "daily", "Daily Rank")
     quarterly = Task("quarterly", "quarterly", "Quarterly Rank")
@@ -43,31 +48,84 @@ class ColumnContent:
     never_hidden: bool = False
     dummy: bool = False
 
+
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(
+    [
+        "model_type_symbol",
+        ColumnContent,
+        ColumnContent("T", "str", True, never_hidden=True),
+    ]
+)
+auto_eval_column_dict.append(
+    [
+        "model",
+        ColumnContent,
+        ColumnContent("Model", "markdown", True, never_hidden=True),
+    ]
+)
 # Ranks
-auto_eval_column_dict.append(["daily", ColumnContent, ColumnContent("Daily Rank", "number", True)])
-auto_eval_column_dict.append(["quarterly", ColumnContent, ColumnContent("Quarterly Rank", "number", True)])
+auto_eval_column_dict.append(
+    ["daily", ColumnContent, ColumnContent("Daily Rank", "number", True)]
+)
+auto_eval_column_dict.append(
+    ["quarterly", ColumnContent, ColumnContent("Quarterly Rank", "number", True)]
+)
 # Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(
+    ["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]
+)
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append(
+        [task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)]
+    )
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["merged", ColumnContent, ColumnContent("Merged", "bool", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, False)])
+auto_eval_column_dict.append(
+    ["model_type", ColumnContent, ColumnContent("Type", "str", False)]
+)
+auto_eval_column_dict.append(
+    ["architecture", ColumnContent, ColumnContent("Architecture", "str", False)]
+)
+auto_eval_column_dict.append(
+    ["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)]
+)
+auto_eval_column_dict.append(
+    ["precision", ColumnContent, ColumnContent("Precision", "str", False)]
+)
+auto_eval_column_dict.append(
+    ["merged", ColumnContent, ColumnContent("Merged", "bool", False)]
+)
+auto_eval_column_dict.append(
+    ["license", ColumnContent, ColumnContent("Hub License", "str", False)]
+)
+auto_eval_column_dict.append(
+    ["params", ColumnContent, ColumnContent("#Params (B)", "number", False)]
+)
+auto_eval_column_dict.append(
+    ["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)]
+)
+auto_eval_column_dict.append(
+    [
+        "still_on_hub",
+        ColumnContent,
+        ColumnContent("Available on the hub", "bool", False),
+    ]
+)
+auto_eval_column_dict.append(
+    ["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]
+)
+auto_eval_column_dict.append(
+    ["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, False)]
+)
 # Dummy column for the search bar (hidden by the custom CSS)
-auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+auto_eval_column_dict.append(
+    [
+        "dummy",
+        ColumnContent,
+        ColumnContent("model_name_for_query", "str", False, dummy=True),
+    ]
+)
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -82,15 +140,17 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 # Define the human baselines
 human_baseline_row = {
     AutoEvalColumn.model.name: "<p>Human performance</p>",
 }
 
+
 @dataclass
 class ModelDetails:
     name: str
-    symbol: str = "" # emoji, only for the model type
+    symbol: str = ""  # emoji, only for the model type
 
 
 class ModelType(Enum):
@@ -115,11 +175,13 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     # bfloat16 = ModelDetails("bfloat16")
@@ -138,15 +200,17 @@ class Precision(Enum):
         if precision in ["GPTQ", "None"]:
             return Precision.qt_GPTQ
         return Precision.Unknown
-        
-
 
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+COLS_LITE = [
+    c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
+]
+TYPES_LITE = [
+    c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
+]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]

src/populate.py

@@ -9,7 +9,9 @@ from src.leaderboard.filter_models import filter_models
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df(
+    results_path: str, requests_path: str, cols: list, benchmark_cols: list
+) -> pd.DataFrame:
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
     # all_data_json.append(baseline_row)
@@ -49,7 +51,9 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            sub_entries = [
+                e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")
+            ]
             for sub_entry in sub_entries:
                 file_path = os.path.join(save_path, entry, sub_entry)
                 with open(file_path) as fp:
@@ -61,7 +65,11 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    finished_list = [
+        e
+        for e in all_evals
+        if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"
+    ]
     failed_list = [e for e in all_evals if e["status"] == "FAILED"]
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
     df_running = pd.DataFrame.from_records(running_list, columns=cols)