refactor: rename the benchmarks enum

app.py

@@ -6,8 +6,8 @@ from src.about import (
     TITLE
 )
 from src.benchmarks import (
-    BenchmarksQA,
-    BenchmarksLongDoc
+    QABenchmarks,
+    LongDocBenchmarks
 )
 from src.display.css_html_js import custom_css
 from src.envs import (
@@ -76,11 +76,11 @@ def update_metric_long_doc(
     return update_metric(data["AIR-Bench_24.04"].raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
 
 
-DOMAIN_COLS_QA = list(frozenset([c.value.domain for c in list(BenchmarksQA)]))
-LANG_COLS_QA = list(frozenset([c.value.lang for c in list(BenchmarksQA)]))
+DOMAIN_COLS_QA = list(frozenset([c.value.domain for c in list(QABenchmarks)]))
+LANG_COLS_QA = list(frozenset([c.value.lang for c in list(QABenchmarks)]))
 
-DOMAIN_COLS_LONG_DOC = list(frozenset([c.value.domain for c in list(BenchmarksLongDoc)]))
-LANG_COLS_LONG_DOC = list(frozenset([c.value.lang for c in list(BenchmarksLongDoc)]))
+DOMAIN_COLS_LONG_DOC = list(frozenset([c.value.domain for c in list(LongDocBenchmarks)]))
+LANG_COLS_LONG_DOC = list(frozenset([c.value.lang for c in list(LongDocBenchmarks)]))
 
 demo = gr.Blocks(css=custom_css)
 

src/benchmarks.py

@@ -54,5 +54,5 @@ def get_benchmarks_enum(benchmark_version):
 
 _qa_benchmark_dict, _long_doc_benchmark_dict = get_benchmarks_enum('AIR-Bench_24.04')
 
-BenchmarksQA = Enum('BenchmarksQA', _qa_benchmark_dict)
-BenchmarksLongDoc = Enum('BenchmarksLongDoc', _long_doc_benchmark_dict)
+QABenchmarks = Enum('QABenchmarks', _qa_benchmark_dict)
+LongDocBenchmarks = Enum('LongDocBenchmarks', _long_doc_benchmark_dict)

src/display/columns.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, make_dataclass
 
-from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
+from src.benchmarks import QABenchmarks, LongDocBenchmarks
 
 COL_NAME_AVG = "Average ⬆️"
 COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
@@ -66,7 +66,7 @@ def get_default_auto_eval_column_dict():
     return auto_eval_column_dict
 
 
-def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
+def make_autoevalcolumn(cls_name="QABenchmarks", benchmarks=QABenchmarks):
     auto_eval_column_dict = get_default_auto_eval_column_dict()
     # Leaderboard columns
     for benchmark in benchmarks:
@@ -79,9 +79,9 @@ def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
 
 
 AutoEvalColumnQA = make_autoevalcolumn(
-    "AutoEvalColumnQA", BenchmarksQA)
+    "AutoEvalColumnQA", QABenchmarks)
 AutoEvalColumnLongDoc = make_autoevalcolumn(
-    "AutoEvalColumnLongDoc", BenchmarksLongDoc)
+    "AutoEvalColumnLongDoc", LongDocBenchmarks)
 
 fixed_cols = get_default_auto_eval_column_dict()[:-3]
 

src/utils.py

@@ -6,7 +6,7 @@ from typing import List
 
 import pandas as pd
 
-from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
+from src.benchmarks import QABenchmarks, LongDocBenchmarks
 from src.display.formatting import styled_message, styled_error
 from src.display.columns import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RANK, \
     COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS, COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, \
@@ -68,11 +68,11 @@ def get_default_cols(task: str, columns: list=[], add_fix_cols: bool=True) -> li
     if task == "qa":
         cols_list = COLS_QA
         types_list = TYPES_QA
-        benchmark_list = [c.value.col_name for c in list(BenchmarksQA)]
+        benchmark_list = [c.value.col_name for c in list(QABenchmarks)]
     elif task == "long-doc":
         cols_list = COLS_LONG_DOC
         types_list = TYPES_LONG_DOC
-        benchmark_list = [c.value.col_name for c in list(BenchmarksLongDoc)]
+        benchmark_list = [c.value.col_name for c in list(LongDocBenchmarks)]
     else:
         raise NotImplemented
     for col_name, col_type in zip(cols_list, types_list):
@@ -110,9 +110,9 @@ def select_columns(
     selected_cols = []
     for c in cols:
         if task == "qa":
-            eval_col = BenchmarksQA[c].value
+            eval_col = QABenchmarks[c].value
         elif task == "long-doc":
-            eval_col = BenchmarksLongDoc[c].value
+            eval_col = LongDocBenchmarks[c].value
         if eval_col.domain not in domain_query:
             continue
         if eval_col.lang not in language_query:
@@ -329,10 +329,10 @@ def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -
     cols = [COL_NAME_IS_ANONYMOUS, ]
     if task == "qa":
         cols += COLS_QA
-        benchmark_cols = [t.value.col_name for t in BenchmarksQA]
+        benchmark_cols = [t.value.col_name for t in QABenchmarks]
     elif task == "long-doc":
         cols += COLS_LONG_DOC
-        benchmark_cols = [t.value.col_name for t in BenchmarksLongDoc]
+        benchmark_cols = [t.value.col_name for t in LongDocBenchmarks]
     else:
         raise NotImplemented
     all_data_json = []

tests/src/test_benchmarks.py

@@ -1,11 +1,11 @@
-from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
+from src.benchmarks import QABenchmarks, LongDocBenchmarks
 
 
 def test_qabenchmarks():
-    print(list(BenchmarksQA))
-    for benchmark in list(BenchmarksQA):
+    print(list(QABenchmarks))
+    for benchmark in list(QABenchmarks):
         print(benchmark.name, benchmark.metric, benchmark.col_name, benchmark.domain, benchmark.lang, benchmark.task)
 
 
 def test_longdocbenchmarks():
-    print(list(BenchmarksLongDoc))
+    print(list(LongDocBenchmarks))