feat: add versioning for the long-doc

app.py

@@ -69,16 +69,16 @@ def update_metric_qa(
     return update_metric(datastore, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
 
 
-# def update_metric_long_doc(
-#         metric: str,
-#         domains: list,
-#         langs: list,
-#         reranking_model: list,
-#         query: str,
-#         show_anonymous: bool,
-#         show_revision_and_timestamp,
-# ):
-#     return update_metric(datastore.raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+def update_metric_long_doc(
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+        show_anonymous: bool,
+        show_revision_and_timestamp,
+):
+    return update_metric(datastore, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
 
 
 def update_datastore(version):
@@ -274,18 +274,15 @@ with demo:
                             lb_table_reranker,
                             queue=True
                         )
-"""
             with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
                 with gr.Row():
                     with gr.Column(min_width=320):
                         # select domain
                         with gr.Row():
-                            selected_domains = get_domain_dropdown(DOMAIN_COLS_LONG_DOC, DOMAIN_COLS_LONG_DOC)
+                            selected_domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
                         # select language
                         with gr.Row():
-                            selected_langs = get_language_dropdown(
-                                LANG_COLS_LONG_DOC, LANG_COLS_LONG_DOC
-                            )
+                            selected_langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
                     with gr.Column():
                         # select the metric
                         with gr.Row():
@@ -301,22 +298,29 @@ with demo:
                                 search_bar = get_search_bar()
                             # select reranking model
                             with gr.Column():
-                                selected_rerankings = get_reranking_dropdown(data["AIR-Bench_24.04"].reranking_models)
+                                selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
 
                         lb_table = get_leaderboard_table(
-                            data["AIR-Bench_24.04"].leaderboard_df_long_doc, data["AIR-Bench_24.04"].types_long_doc
+                            datastore.leaderboard_df_long_doc, datastore.types_long_doc
                         )
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_table_for_search = get_leaderboard_table(
-                            data["AIR-Bench_24.04"].raw_df_long_doc, data["AIR-Bench_24.04"].types_long_doc, visible=False
+                        hidden_lb_table = get_leaderboard_table(
+                            datastore.raw_df_long_doc, datastore.types_long_doc, visible=False
+                        )
+
+                        selected_version.change(
+                            update_datastore,
+                            [selected_version,],
+                            [selected_domains, selected_langs, selected_rerankings, lb_table, hidden_lb_table]
                         )
 
                         set_listeners(
                             "long-doc",
                             lb_table,
-                            hidden_lb_table_for_search,
+                            hidden_lb_table,
                             search_bar,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_rerankings,
@@ -339,6 +343,7 @@ with demo:
                             lb_table,
                             queue=True
                         )
+"""
                     with gr.TabItem("Retrieval Only", id=21):
                         with gr.Row():
                             with gr.Column(scale=1):

src/benchmarks.py

@@ -51,6 +51,8 @@ def get_benchmarks_enum(benchmark_version, task_type):
                         benchmark_name = f"{domain}_{lang}_{dataset}"
                         benchmark_name = get_safe_name(benchmark_name)
                         col_name = benchmark_name
+                        if "test" not in dataset_list[dataset]["splits"]:
+                            continue
                         for metric in METRIC_LIST:
                             benchmark_dict[benchmark_name] = \
                                 Benchmark(benchmark_name, metric, col_name, domain, lang, task)

src/loaders.py

@@ -76,17 +76,17 @@ def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
         lb_data_store.leaderboard_df_qa[~lb_data_store.leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
     lb_data_store.leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
-    # lb_data_store.raw_df_long_doc = get_leaderboard_df(
-    #     lb_data_store, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
-    # print(f'Long-Doc data loaded: {len(lb_data_store.raw_df_long_doc)}')
-    # lb_data_store.leaderboard_df_long_doc = lb_data_store.raw_df_long_doc.copy()
-    # shown_columns_long_doc, types_long_doc = get_default_cols(
-    #     'long-doc', lb_data_store.leaderboard_df_long_doc.columns, add_fix_cols=True)
-    # lb_data_store.types_long_doc = types_long_doc
-    # lb_data_store.leaderboard_df_long_doc = \
-    #     lb_data_store.leaderboard_df_long_doc[
-    #         ~lb_data_store.leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
-    # lb_data_store.leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+    lb_data_store.raw_df_long_doc = get_leaderboard_df(
+        lb_data_store, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
+    print(f'Long-Doc data loaded: {len(lb_data_store.raw_df_long_doc)}')
+    lb_data_store.leaderboard_df_long_doc = lb_data_store.raw_df_long_doc.copy()
+    shown_columns_long_doc, types_long_doc = get_default_cols(
+        'long-doc', lb_data_store.slug, add_fix_cols=True)
+    lb_data_store.types_long_doc = types_long_doc
+    lb_data_store.leaderboard_df_long_doc = \
+        lb_data_store.leaderboard_df_long_doc[
+            ~lb_data_store.leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
+    lb_data_store.leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
     lb_data_store.reranking_models = sorted(
         list(frozenset([eval_result.reranking_model for eval_result in lb_data_store.raw_data])))

src/utils.py

@@ -64,12 +64,12 @@ def get_default_cols(task: str, version_slug, add_fix_cols: bool=True) -> tuple:
     types = []
     if task == "qa":
         benchmarks = QABenchmarks[version_slug]
-        cols_list, types_list = get_default_col_names_and_types(benchmarks)
-        # cols_list = COLS_QA
-        # types_list = TYPES_QA
-        benchmark_list = [c.value.col_name for c in list(benchmarks.value)]
+    elif task == "long-doc":
+        benchmarks = LongDocBenchmarks[version_slug]
     else:
         raise NotImplemented
+    cols_list, types_list = get_default_col_names_and_types(benchmarks)
+    benchmark_list = [c.value.col_name for c in list(benchmarks.value)]
     for col_name, col_type in zip(cols_list, types_list):
         if col_name not in benchmark_list:
             continue
@@ -90,40 +90,6 @@ def get_default_cols(task: str, version_slug, add_fix_cols: bool=True) -> tuple:
     return cols, types
 
 
-# def get_default_cols(task: str, columns: list, add_fix_cols: bool=True) -> list:
-#     cols = []
-#     types = []
-#     if task == "qa":
-#         cols_list = COLS_QA
-#         types_list = TYPES_QA
-#         benchmark_list = [c.value.col_name for c in list(QABenchmarks)]
-#     elif task == "long-doc":
-#         cols_list = COLS_LONG_DOC
-#         types_list = TYPES_LONG_DOC
-#         benchmark_list = [c.value.col_name for c in list(LongDocBenchmarks)]
-#     else:
-#         raise NotImplemented
-#     for col_name, col_type in zip(cols_list, types_list):
-#         if col_name not in benchmark_list:
-#             continue
-#         if len(columns) > 0 and col_name not in columns:
-#             continue
-#         cols.append(col_name)
-#         types.append(col_type)
-#
-#     if add_fix_cols:
-#         _cols = []
-#         _types = []
-#         for col_name, col_type in zip(cols, types):
-#             if col_name in FIXED_COLS:
-#                 continue
-#             _cols.append(col_name)
-#             _types.append(col_type)
-#         cols = FIXED_COLS + _cols
-#         types = FIXED_COLS_TYPES + _types
-#     return cols, types
-
-
 def select_columns(
         df: pd.DataFrame,
         domain_query: list,
@@ -360,14 +326,13 @@ def get_leaderboard_df(datastore, task: str, metric: str) -> pd.DataFrame:
     cols = [COL_NAME_IS_ANONYMOUS, ]
     if task == "qa":
         benchmarks = QABenchmarks[datastore.slug]
-        cols_qa, _ = get_default_col_names_and_types(benchmarks)
-        cols += cols_qa
-        benchmark_cols = [t.value.col_name for t in list(benchmarks.value)]
-    # elif task == "long-doc":
-    #     cols += COLS_LONG_DOC
-    #     benchmark_cols = [t.value.col_name for t in LongDocBenchmarks]
+    elif task == "long-doc":
+        benchmarks = LongDocBenchmarks[datastore.slug]
     else:
         raise NotImplemented
+    cols_qa, _ = get_default_col_names_and_types(benchmarks)
+    cols += cols_qa
+    benchmark_cols = [t.value.col_name for t in list(benchmarks.value)]
     all_data_json = []
     for v in raw_data:
         all_data_json += v.to_dict(task=task, metric=metric)