Add reranking functionality and update data loading methods for improved leaderboard display

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from utils import submit_gradio_module, load_retrieval_results
+from utils import submit_gradio_module, load_retrieval_results, load_reranking_results
 from fuzzywuzzy import fuzz
 
 HEADER = """<div style="text-align: center; margin-bottom: 20px;">
@@ -41,42 +41,42 @@ CITATION_BUTTON_TEXT = """
 """
 
 retrieval_df = None
-original_columns_order = None
+reranking_df = None
 
-def search_leaderboard(model_name, columns_to_show):
+def search_leaderboard(df, model_name, columns_to_show, threshold=95):
     if len(model_name.strip()) == 0:
-        return retrieval_df.loc[:, columns_to_show]
+        return df.loc[:, columns_to_show]
     
-    threshold = 95  # You can adjust this value to make the search more or less strict
-    filtered_df = retrieval_df.copy()
-
     def calculate_similarity(row):
         similarity = fuzz.partial_ratio(model_name.lower(), row["Model"].lower())
         return similarity if similarity >= threshold else 0
 
-    # Add a new column for similarity scores
+    filtered_df = df.copy()
     filtered_df["similarity"] = filtered_df.apply(calculate_similarity, axis=1)
-
-    # Filter and sort the dataframe
     filtered_df = filtered_df[filtered_df["similarity"] > 0].sort_values('similarity', ascending=False)
-
-    # Remove the similarity column before returning
     filtered_df = filtered_df.drop('similarity', axis=1)
-
-    # Filter the columns to show
     filtered_df = filtered_df.loc[:, columns_to_show]
 
     return filtered_df
 
+def retrieval_search_leaderboard(model_name, columns_to_show):
+    return search_leaderboard(retrieval_df, model_name, columns_to_show)
+
+def reranking_search_leaderboard(model_name, columns_to_show):
+    return search_leaderboard(reranking_df, model_name, columns_to_show)
+
 
 def main():
-    global retrieval_df, original_columns_order
-    retrieval_df = load_retrieval_results()
-    retrieval_df[["Model"]] = retrieval_df[["Model"]].map(lambda x: f'<a href="https://huggingface.co/{x}" target="_blank">{x}</a>')
-    retrieval_df.drop(columns=["Revision", "Precision", "Task"], inplace=True)
-    retrieval_df.sort_values("Web Search Dataset (Overall Score)", ascending=False, inplace=True)
+    global retrieval_df, reranking_df
+
+    # Prepare retrieval dataframe
+    retrieval_df = load_retrieval_results(prepare_for_display=True)
+    retrieval_columns_to_show = ["Model", "Web Search Dataset (Overall Score)", "Model Size (in Millions)", "Embedding Dimension", "Max Tokens", "Num Likes"]
+
+    # Prepare reranking dataframe
+    reranking_df = load_reranking_results(prepare_for_display=True)
+    reranking_columns_to_show = reranking_df.columns.tolist()
 
-    columns_to_show = ["Model", "Web Search Dataset (Overall Score)", "Model Size (in Millions)", "Embedding Dimension", "Max Tokens", "Num Likes"]
     with gr.Blocks() as demo:
         gr.HTML(HEADER)
         
@@ -90,15 +90,15 @@ def main():
                                 label="Search", 
                                 scale=5
                             )
-                            columns_to_show_input = gr.CheckboxGroup(
+                            retrieval_columns_to_show_input = gr.CheckboxGroup(
                                 label="Columns to Show",
                                 choices=retrieval_df.columns.tolist(),
-                                value=columns_to_show,
+                                value=retrieval_columns_to_show,
                                 scale=4
                             )
 
                         retrieval_leaderboard = gr.Dataframe(
-                            value=retrieval_df[columns_to_show],
+                            value=retrieval_df[retrieval_columns_to_show],
                             datatype="markdown",
                             wrap=True,
                             show_fullscreen_button=True,
@@ -107,13 +107,13 @@ def main():
                         
                         # Submit the search box and the leaderboard
                         search_box_retrieval.input(
-                            search_leaderboard,
-                            inputs=[search_box_retrieval, columns_to_show_input],
+                            retrieval_search_leaderboard,
+                            inputs=[search_box_retrieval, retrieval_columns_to_show_input],
                             outputs=retrieval_leaderboard
                         )
-                        columns_to_show_input.select(
+                        retrieval_columns_to_show_input.select(
                             lambda columns: retrieval_df.loc[:, columns],
-                            inputs=columns_to_show_input,
+                            inputs=retrieval_columns_to_show_input,
                             outputs=retrieval_leaderboard
                         )
                         
@@ -126,23 +126,36 @@ def main():
             with gr.Tab("📊 Reranking"):
                 with gr.Tabs():
                     with gr.Tab("👑 Leaderboard"):
-                        search_box_reranker = gr.Textbox(
+                        with gr.Row():
+                            search_box_reranker = gr.Textbox(
                                 placeholder="Search for models...", 
                                 label="Search", 
-                                interactive=True
+                                scale=5
+                            )
+                            reranking_columns_to_show_input = gr.CheckboxGroup(
+                                label="Columns to Show",
+                                choices=reranking_df.columns.tolist(),
+                                value=reranking_columns_to_show,
+                                scale=4
                             )
                         
                         reranker_leaderboard = gr.Dataframe(
-                            value=retrieval_df[columns_to_show],
+                            value=reranking_df[reranking_columns_to_show],
                             datatype="markdown",
                             wrap=True,
                             show_fullscreen_button=True,
                             interactive=False
                         )
 
-                        search_box_reranker.submit(
-                            search_leaderboard,
-                            inputs=search_box_reranker,
+                        # Submit the search box and the leaderboard
+                        search_box_reranker.input(
+                            reranking_search_leaderboard,
+                            inputs=[search_box_reranker, reranking_columns_to_show_input],
+                            outputs=reranker_leaderboard
+                        )
+                        reranking_columns_to_show_input.select(
+                            lambda columns: reranking_df.loc[:, columns],
+                            inputs=reranking_columns_to_show_input,
                             outputs=reranker_leaderboard
                         )
 

results/reranking_results.json

@@ -0,0 +1,110 @@
+[
+    {
+        "Model":"BAAI\/bge-reranker-v2-m3",
+        "Overall Score":85.4,
+        "Model Size (in Millions)":568.0,
+        "Downloads Last Month (Jan-Feb)":889663,
+        "MRR@10":79.41,
+        "NDCG@10":95.1,
+        "MAP":81.69
+    },
+    {
+        "Model":"NAMAA-Space\/Namaa-ARA-Reranker-V1",
+        "Overall Score":84.1,
+        "Model Size (in Millions)":568.0,
+        "Downloads Last Month (Jan-Feb)":118,
+        "MRR@10":76.48,
+        "NDCG@10":93.14,
+        "MAP":82.67
+    },
+    {
+        "Model":"NAMAA-Space\/GATE-Reranker-V1",
+        "Overall Score":76.81,
+        "Model Size (in Millions)":135.0,
+        "Downloads Last Month (Jan-Feb)":714,
+        "MRR@10":62.61,
+        "NDCG@10":89.7,
+        "MAP":78.11
+    },
+    {
+        "Model":"cross-encoder\/ms-marco-MiniLM-L-12-v2",
+        "Overall Score":60.54,
+        "Model Size (in Millions)":33.4,
+        "Downloads Last Month (Jan-Feb)":765472,
+        "MRR@10":41.69,
+        "NDCG@10":67.62,
+        "MAP":72.32
+    },
+    {
+        "Model":"Omartificial-Intelligence-Space\/Arabic-MiniLM-L12-v2-all-nli-triplet",
+        "Overall Score":58.17,
+        "Model Size (in Millions)":118.0,
+        "Downloads Last Month (Jan-Feb)":475,
+        "MRR@10":42.28,
+        "NDCG@10":75.63,
+        "MAP":56.59
+    },
+    {
+        "Model":"aubmindlab\/bert-base-arabert",
+        "Overall Score":56.69,
+        "Model Size (in Millions)":136.0,
+        "Downloads Last Month (Jan-Feb)":74132,
+        "MRR@10":40.16,
+        "NDCG@10":71.14,
+        "MAP":58.77
+    },
+    {
+        "Model":"OrdalieTech\/Solon-embeddings-large-0.1",
+        "Overall Score":54.73,
+        "Model Size (in Millions)":560.0,
+        "Downloads Last Month (Jan-Feb)":29994,
+        "MRR@10":32.59,
+        "NDCG@10":60.18,
+        "MAP":71.41
+    },
+    {
+        "Model":"intfloat\/multilingual-e5-large-instruct",
+        "Overall Score":53.39,
+        "Model Size (in Millions)":560.0,
+        "Downloads Last Month (Jan-Feb)":371648,
+        "MRR@10":48.35,
+        "NDCG@10":56.06,
+        "MAP":55.78
+    },
+    {
+        "Model":"Omartificial-Intelligence-Space\/Arabic-all-nli-triplet-Matryoshka",
+        "Overall Score":52.16,
+        "Model Size (in Millions)":135.0,
+        "Downloads Last Month (Jan-Feb)":1784,
+        "MRR@10":40.58,
+        "NDCG@10":54.7,
+        "MAP":61.2
+    },
+    {
+        "Model":"Snowflake\/snowflake-arctic-embed-l-v2.0",
+        "Overall Score":51.61,
+        "Model Size (in Millions)":568.0,
+        "Downloads Last Month (Jan-Feb)":83755,
+        "MRR@10":32.16,
+        "NDCG@10":58.63,
+        "MAP":64.05
+    },
+    {
+        "Model":"aubmindlab\/bert-large-arabertv2",
+        "Overall Score":50.84,
+        "Model Size (in Millions)":371.0,
+        "Downloads Last Month (Jan-Feb)":8939,
+        "MRR@10":27.84,
+        "NDCG@10":53.42,
+        "MAP":71.27
+    },
+    {
+        "Model":"Omartificial-Intelligence-Space\/Arabic-labse-Matryoshka",
+        "Overall Score":48.76,
+        "Model Size (in Millions)":471.0,
+        "Downloads Last Month (Jan-Feb)":460,
+        "MRR@10":32.71,
+        "NDCG@10":56.6,
+        "MAP":56.97
+    }
+]

utils.py

@@ -10,17 +10,38 @@ api = HfApi()
 OWNER = "Navid-AI"
 DATASET_REPO_ID = f"{OWNER}/requests-dataset"
 
-def load_retrieval_results():
-    base_path = Path(__file__).parent
-    results_dir = base_path / "results"
+results_dir = Path(__file__).parent / "results"
 
-    retrieval_dataframe_path = results_dir / "retrieval_results.json"
-    if not retrieval_dataframe_path.exists():
-        df = pd.DataFrame(columns=["Model", "Revision", "Precision", "Task", "Model Size (in Millions)", "Embedding Dimension", "Max Tokens", "Num Likes", "Web Search Dataset (Overall Score)"])
+def load_retrieval_results(prepare_for_display=False):
+    # Load the retrieval results
+    dataframe_path = results_dir / "retrieval_results.json"
+    if dataframe_path.exists():
+        df = pd.read_json(dataframe_path)
     else:
-        df = pd.read_json(retrieval_dataframe_path)
+        raise FileNotFoundError(f"File '{dataframe_path}' not found.")
+
+    if prepare_for_display:
+        df[["Model"]] = df[["Model"]].map(lambda x: f'<a href="https://huggingface.co/{x}" target="_blank">{x}</a>')
+        df.drop(columns=["Revision", "Precision", "Task"], inplace=True)
+        df.sort_values("Web Search Dataset (Overall Score)", ascending=False, inplace=True)
+
     return df
 
+def load_reranking_results(prepare_for_display=False):
+    # Load the reranking results
+    dataframe_path = results_dir / "reranking_results.json"
+    if dataframe_path.exists():
+        df = pd.read_json(dataframe_path)
+    else:
+        raise FileNotFoundError(f"File '{dataframe_path}' not found.")
+
+    if prepare_for_display:
+        df[["Model"]] = df[["Model"]].map(lambda x: f'<a href="https://huggingface.co/{x}" target="_blank">{x}</a>')
+        df.sort_values("Overall Score", ascending=False, inplace=True)
+
+    return df
+
+
 def get_model_info(model_id, verbose=False):
     model_info = api.model_info(model_id)
     num_downloads = model_info.downloads
@@ -49,7 +70,7 @@ def submit_model(model_name, revision, precision, params, license, task):
     if task == "Retriever":
         df = load_retrieval_results()
     elif task == "Reranker":
-        df = load_retrieval_results()
+        df = load_reranking_results()
     else:
         return gr.Error(f"Error: Task '{task}' is not supported.")