update task

src/about.py

@@ -12,16 +12,16 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    cmmmu = Task("cmmmu", "acc", "CMMMU")
-    cmmu = Task("cmmu", "acc", "CMMU")
-    cv_bench = Task("cv_bench", "acc", "CV_Bench")
-    hallusion_bench = Task("hallusion_bench", "acc", "Hallusion_Bench")
-    mmmu = Task("mmmu", "acc", "MMMU")
-    mmmu_pro_standard = Task("mmmu_pro_standard", "acc", "MMMU_Pro_Standard")
-    mmmu_pro_vision = Task("mmmu_pro_vision", "acc", "MMMU_Pro_Vision")
-    ocrbench = Task("ocrbench", "acc", "OCRBench")
-    math_vision = Task("math_vision", "acc", "Math_Vision")
-    ciibench = Task("ciibench", "acc", "CIIBench")
+    CMMMU = Task("CMMMU", "acc", "CMMMU")
+    CMMU = Task("CMMU", "acc", "CMMU")
+    ChartQA = Task('ChartQA',"acc", "ChartQA")
+    MMMU = Task("MMMU", "acc", "MMMU")
+    OCRBench = Task("OCRBench", "acc", "OCRBench")
+    MMMU_Pro_standard = Task("MMMU_Pro_standard", "acc", "MMMU_Pro_standard")
+    MMMU_Pro_vision = Task("MMMU_Pro_vision", "acc", "MMMU_Pro_vision")
+    MathVision = Task("MathVision", "acc", "MathVision")
+    CII_Bench = Task("CII-Bench", "acc", "CII-Bench")
+    Blink = Task("Blink", "acc", "Blink")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -144,16 +144,16 @@ baseline_row = {
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 92.75,
     AutoEvalColumn.merged.name: False,
-    AutoEvalColumn.cmmmu.name: 100,
-    AutoEvalColumn.cmmu.name: 100,
-    AutoEvalColumn.cv_bench.name: 100,
-    AutoEvalColumn.hallusion_bench.name: 100,
-    AutoEvalColumn.mmmu.name: 100,
-    AutoEvalColumn.mmmu_pro_standard.name: 100,
-    AutoEvalColumn.mmmu_pro_vision.name: 100,
-    AutoEvalColumn.ocrbench.name: 100,
-    AutoEvalColumn.math_vision.name: 100,
-    AutoEvalColumn.ciibench.name: 100,
+    AutoEvalColumn.CMMMU.name: 100,
+    AutoEvalColumn.CMMU.name: 100,
+    AutoEvalColumn.ChartQA.name: 100,
+    AutoEvalColumn.MMMU.name: 100,
+    AutoEvalColumn.MMMU_Pro_standard.name: 100,
+    AutoEvalColumn.MMMU_Pro_vision.name: 100,
+    AutoEvalColumn.OCRBench.name: 100,
+    AutoEvalColumn.MathVision.name: 100,
+    AutoEvalColumn.CII_Bench.name: 100,
+    AutoEvalColumn.Blink.name: 100,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
     AutoEvalColumn.flagged.name: False,
@@ -166,16 +166,16 @@ human_baseline_row = {
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 92.75,
     AutoEvalColumn.merged.name: False,
-    AutoEvalColumn.cmmmu.name: 100,
-    AutoEvalColumn.cmmu.name: 100,
-    AutoEvalColumn.cv_bench.name: 100,
-    AutoEvalColumn.hallusion_bench.name: 100,
-    AutoEvalColumn.mmmu.name: 100,
-    AutoEvalColumn.mmmu_pro_standard.name: 100,
-    AutoEvalColumn.mmmu_pro_vision.name: 100,
-    AutoEvalColumn.ocrbench.name: 100,
-    AutoEvalColumn.math_vision.name: 100,
-    AutoEvalColumn.ciibench.name: 100,
+    AutoEvalColumn.CMMMU.name: 100,
+    AutoEvalColumn.CMMU.name: 100,
+    AutoEvalColumn.ChartQA.name: 100,
+    AutoEvalColumn.MMMU.name: 100,
+    AutoEvalColumn.MMMU_Pro_standard.name: 100,
+    AutoEvalColumn.MMMU_Pro_vision.name: 100,
+    AutoEvalColumn.OCRBench.name: 100,
+    AutoEvalColumn.MathVision.name: 100,
+    AutoEvalColumn.CII_Bench.name: 100,
+    AutoEvalColumn.Blink.name: 100,
     AutoEvalColumn.dummy.name: "human_baseline",
     AutoEvalColumn.model_type.name: "",
     AutoEvalColumn.flagged.name: False,

src/leaderboard/read_evals.py

@@ -114,6 +114,13 @@ class EvalResult:
             self.status = "FAILED"
             print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
 
+    def update_with_dynamic_file_dict(self, file_dict):
+        self.license = file_dict.get("license", "?")
+        self.likes = file_dict.get("likes", 0)
+        self.still_on_hub = file_dict["still_on_hub"]
+        self.flagged = any("flagged" in tag for tag in file_dict["tags"])
+        self.tags = file_dict["tags"]
+
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = 0