Add utility functions for model submission and retrieval in Gradio interface

app.py

@@ -4,6 +4,8 @@ import numpy as np
 import pandas as pd
 import gradio as gr
 from huggingface_hub import HfApi, hf_hub_download
+from pathlib import Path
+from utils import submit_gradio_module, load_retrieval_results
 
 
 OWNER = "Navid-AI"
@@ -19,19 +21,19 @@ HEADER = """<div style="text-align: center; margin-bottom: 20px;">
 ABOUT_SECTION = """
 ## About
 
-The AraGen Leaderboard is designed to evaluate and compare the performance of Chat Arabic Large Language Models (LLMs) on a set of generative tasks. By leveraging the new **3C3H** evaluation measure which evaluate the model's output across six dimensions —Correctness, Completeness, Conciseness, Helpfulness, Honesty, and Harmlessness— the leaderboard provides a comprehensive and holistic evaluation of a model's performance in generating human-like and ethically responsible content.
+The Arabic RAG Leaderboard is designed to evaluate and compare the performance of Retrieval-Augmented Generation (RAG) models on a set of retrieval and generative tasks. By leveraging a comprehensive evaluation framework, the leaderboard provides a detailed assessment of a model's ability to retrieve relevant information and generate accurate, coherent, and contextually appropriate responses.
 
-### Why Focus on Chat Models?
+### Why Focus on RAG Models?
 
-AraGen Leaderboard —And 3C3H in general— is specifically designed to assess **chat models**, which interact in conversational settings, intended for end user interaction and require a blend of factual accuracy and user-centric dialogue capabilities. While it is technically possible to submit foundational models, we kindly ask users to refrain from doing so. For evaluations of foundational models using likelihood accuracy based benchmarks, please refer to the [Open Arabic LLM Leaderboard (OALL)](https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard).
+The Arabic RAG Leaderboard is specifically designed to assess **RAG models**, which combine retrieval mechanisms with generative capabilities to enhance the quality and relevance of generated content. These models are particularly useful in scenarios where access to up-to-date and contextually relevant information is crucial. While foundational models can be evaluated, the primary focus is on RAG models that excel in both retrieval and generation tasks.
 
 ### How to Submit Your Model?
 
-Navigate to the submission section below to submit your open chat model from the HuggingFace Hub for evaluation. Ensure that your model is public and the submmited metadata (precision, revision, #params) is accurate.
+Navigate to the submission section below to submit your RAG model from the HuggingFace Hub for evaluation. Ensure that your model is public and the submitted metadata (precision, revision, #params) is accurate.
 
 ### Contact
 
-For any inquiries or assistance, feel free to reach out through the community tab at [Inception AraGen Community](https://huggingface.co/spaces/inceptionai/AraGen-Leaderboard/discussions) or via [email](mailto:ali.filali@inceptionai.ai).
+For any inquiries or assistance, feel free to reach out through the community tab at [Navid-AI Community](https://huggingface.co/spaces/Navid-AI/The-Arabic-Rag-Leaderboard/discussions) or via [email](mailto:support@navid-ai.com).
 """
 
 CITATION_BUTTON_LABEL = """
@@ -40,286 +42,18 @@ Copy the following snippet to cite these results
 
 CITATION_BUTTON_TEXT = """
 @misc{AraGen,
-  author = {El Filali, Ali and Sengupta, Neha and Abouelseoud, Arwa and Nakov, Preslav and Fourrier, Clémentine},
-  title = {Rethinking LLM Evaluation with 3C3H: AraGen Benchmark and Leaderboard},
-  year = {2024},
-  publisher = {Inception},
-  howpublished = "url{https://huggingface.co/spaces/inceptionai/AraGen-Leaderboard}"
+  author = {Mohaned A. Rashad, Hamza Shahid},
+  title = {The Arabic RAG Leaderboard},
+  year = {2025},
+  publisher = {Navid-AI},
+  howpublished = "url{https://huggingface.co/spaces/Navid-AI/The-Arabic-Rag-Leaderboard}"
 }
 """
 
-
-def load_results():
-    # Get the current directory of the script and construct the path to results.json
-    current_dir = os.path.dirname(os.path.abspath(__file__))
-    results_file = os.path.join(current_dir, "assets", "results", "results.json")
-    
-    # Load the JSON data from the specified file
-    with open(results_file, 'r') as f:
-        data = json.load(f)
-    
-    # Filter out any entries that only contain '_last_sync_timestamp'
-    filtered_data = []
-    for entry in data:
-        # If '_last_sync_timestamp' is the only key, skip it
-        if len(entry.keys()) == 1 and "_last_sync_timestamp" in entry:
-            continue
-        filtered_data.append(entry)
-    
-    data = filtered_data
-    
-    # Lists to collect data
-    data_3c3h = []
-    data_tasks = []
-    
-    for model_data in data:
-        # Extract model meta data
-        meta = model_data.get('Meta', {})
-        model_name = meta.get('Model Name', 'UNK')
-        revision = meta.get('Revision', 'UNK')
-        precision = meta.get('Precision', 'UNK')
-        params = meta.get('Params', 'UNK')
-        license = meta.get('License', 'UNK')
-        
-        # Convert "Model Size" to numeric, treating "UNK" as infinity
-        try:
-            model_size_numeric = float(params)
-        except (ValueError, TypeError):
-            model_size_numeric = np.inf
-        
-        # 3C3H Scores
-        scores_data = model_data.get('claude-3.5-sonnet Scores', {})
-        scores_3c3h = scores_data.get('3C3H Scores', {})
-        scores_tasks = scores_data.get('Tasks Scores', {})
-        
-        # Multiply scores by 100 to get percentages (keep them as numeric values)
-        formatted_scores_3c3h = {k: v*100 for k, v in scores_3c3h.items()}
-        formatted_scores_tasks = {k: v*100 for k, v in scores_tasks.items()}
-        
-        # For 3C3H Scores DataFrame
-        data_entry_3c3h = {
-            'Model Name': model_name,
-            'Revision': revision,
-            'License': license,
-            'Precision': precision,
-            'Model Size': model_size_numeric,  # Numeric value for sorting
-            '3C3H Score': formatted_scores_3c3h.get("3C3H Score", np.nan),
-            'Correctness': formatted_scores_3c3h.get("Correctness", np.nan),
-            'Completeness': formatted_scores_3c3h.get("Completeness", np.nan),
-            'Conciseness': formatted_scores_3c3h.get("Conciseness", np.nan),
-            'Helpfulness': formatted_scores_3c3h.get("Helpfulness", np.nan),
-            'Honesty': formatted_scores_3c3h.get("Honesty", np.nan),
-            'Harmlessness': formatted_scores_3c3h.get("Harmlessness", np.nan),
-        }
-        data_3c3h.append(data_entry_3c3h)
-        
-        # For Tasks Scores DataFrame
-        data_entry_tasks = {
-            'Model Name': model_name,
-            'Revision': revision,
-            'License': license,
-            'Precision': precision,
-            'Model Size': model_size_numeric,  # Numeric value for sorting
-            **formatted_scores_tasks
-        }
-        data_tasks.append(data_entry_tasks)
-    
-    df_3c3h = pd.DataFrame(data_3c3h)
-    df_tasks = pd.DataFrame(data_tasks)
-    
-    # Round the numeric score columns to 4 decimal places
-    score_columns_3c3h = ['3C3H Score', 'Correctness', 'Completeness', 'Conciseness', 'Helpfulness', 'Honesty', 'Harmlessness']
-    df_3c3h[score_columns_3c3h] = df_3c3h[score_columns_3c3h].round(4)
-    
-    # Replace np.inf with a large number in 'Model Size Filter' for filtering
-    max_model_size_value = 1000  # Define a maximum value
-    df_3c3h['Model Size Filter'] = df_3c3h['Model Size'].replace(np.inf, max_model_size_value)
-    
-    # Sort df_3c3h by '3C3H Score' descending if column exists
-    if '3C3H Score' in df_3c3h.columns:
-        df_3c3h = df_3c3h.sort_values(by='3C3H Score', ascending=False)
-        df_3c3h.insert(0, 'Rank', range(1, len(df_3c3h) + 1))  # Add Rank column starting from 1
-    else:
-        df_3c3h.insert(0, 'Rank', range(1, len(df_3c3h) + 1))
-    
-    # Extract task columns
-    task_columns = [col for col in df_tasks.columns if col not in ['Model Name', 'Revision', 'License', 'Precision', 'Model Size', 'Model Size Filter']]
-    
-    # Round the task score columns to 4 decimal places
-    if task_columns:
-        df_tasks[task_columns] = df_tasks[task_columns].round(4)
-    
-    # Replace np.inf with a large number in 'Model Size Filter' for filtering
-    df_tasks['Model Size Filter'] = df_tasks['Model Size'].replace(np.inf, max_model_size_value)
-    
-    # Sort df_tasks by the first task column if it exists
-    if task_columns:
-        first_task = task_columns[0]
-        df_tasks = df_tasks.sort_values(by=first_task, ascending=False)
-        df_tasks.insert(0, 'Rank', range(1, len(df_tasks) + 1))  # Add Rank column starting from 1
-    else:
-        df_tasks = df_tasks.sort_values(by='Model Name', ascending=True)
-        df_tasks.insert(0, 'Rank', range(1, len(df_tasks) + 1))
-    
-    return df_3c3h, df_tasks, task_columns
-
-def load_requests(status_folder):
-    api = HfApi()
-    requests_data = []
-    folder_path_in_repo = status_folder  # 'pending', 'finished', or 'failed'
-
-    hf_api_token = os.environ.get('HF_API_TOKEN', None)
-
-    try:
-        # List files in the dataset repository
-        files_info = api.list_repo_files(
-            repo_id=DATASET_REPO_ID,
-            repo_type="dataset",
-            token=hf_api_token
-        )
-    except Exception as e:
-        print(f"Error accessing dataset repository: {e}")
-        return pd.DataFrame()  # Return empty DataFrame if repository not found or inaccessible
-
-    # Filter files in the desired folder
-    files_in_folder = [f for f in files_info if f.startswith(f"{folder_path_in_repo}/") and f.endswith('.json')]
-
-    for file_path in files_in_folder:
-        try:
-            # Download the JSON file
-            local_file_path = hf_hub_download(
-                repo_id=DATASET_REPO_ID,
-                filename=file_path,
-                repo_type="dataset",
-                token=hf_api_token
-            )
-            # Load JSON data
-            with open(local_file_path, 'r') as f:
-                request = json.load(f)
-            requests_data.append(request)
-        except Exception as e:
-            print(f"Error loading file {file_path}: {e}")
-            continue  # Skip files that can't be loaded
-
-    df = pd.DataFrame(requests_data)
-    return df
-
-def submit_model(model_name, revision, precision, params, license):
-    # Load existing evaluations
-    df_3c3h, df_tasks, _ = load_results()
-    existing_models_results = df_3c3h[['Model Name', 'Revision', 'Precision']]
-
-    # Handle 'Missing' precision
-    if precision == 'Missing':
-        precision = None
-    else:
-        precision = precision.strip().lower()
-
-    # Load pending and finished requests from the dataset repository
-    df_pending = load_requests('pending')
-    df_finished = load_requests('finished')
-
-    # Check if model is already evaluated
-    model_exists_in_results = ((existing_models_results['Model Name'] == model_name) &
-                               (existing_models_results['Revision'] == revision) &
-                               (existing_models_results['Precision'] == precision)).any()
-    if model_exists_in_results:
-        return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' has already been evaluated.**"
-
-    # Check if model is in pending requests
-    if not df_pending.empty:
-        existing_models_pending = df_pending[['model_name', 'revision', 'precision']]
-        model_exists_in_pending = ((existing_models_pending['model_name'] == model_name) &
-                                   (existing_models_pending['revision'] == revision) &
-                                   (existing_models_pending['precision'] == precision)).any()
-        if model_exists_in_pending:
-            return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' is already in the pending evaluations.**"
-
-    # Check if model is in finished requests
-    if not df_finished.empty:
-        existing_models_finished = df_finished[['model_name', 'revision', 'precision']]
-        model_exists_in_finished = ((existing_models_finished['model_name'] == model_name) &
-                                    (existing_models_finished['revision'] == revision) &
-                                    (existing_models_finished['precision'] == precision)).any()
-        if model_exists_in_finished:
-            return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' has already been evaluated.**"
-
-    # Check if model exists on HuggingFace Hub
-    api = HfApi()
-    try:
-        model_info = api.model_info(model_name)
-    except Exception as e:
-        return f"**Error: Could not find model '{model_name}' on HuggingFace Hub. Please ensure the model name is correct and the model is public.**"
-
-    # Proceed with submission
-    status = "PENDING"
-
-    # Prepare the submission data
-    submission = {
-        "model_name": model_name,
-        "license": license,
-        "revision": revision,
-        "precision": precision,
-        "status": status,
-        "params": params
-    }
-
-    # Serialize the submission to JSON
-    submission_json = json.dumps(submission, indent=2)
-
-    # Define the file path in the repository
-    org_model = model_name.split('/')
-    if len(org_model) != 2:
-        return "**Please enter the full model name including the organization or username, e.g., 'inceptionai/jais-family-30b-8k'**"
-    org, model_id = org_model
-    precision_str = precision if precision else 'Missing'
-    file_path_in_repo = f"pending/{org}/{model_id}_eval_request_{revision}_{precision_str}.json"
-
-    # Upload the submission to the dataset repository
-    try:
-        hf_api_token = os.environ.get('HF_API_TOKEN', None)
-        api.upload_file(
-            path_or_fileobj=submission_json.encode('utf-8'),
-            path_in_repo=file_path_in_repo,
-            repo_id=DATASET_REPO_ID,
-            repo_type="dataset",
-            token=hf_api_token
-        )
-    except Exception as e:
-        return f"**Error: Could not submit the model. {str(e)}**"
-
-    return f"**Model '{model_name}' has been submitted for evaluation.**"
+df = load_retrieval_results()
+print(df)
 
 def main():
-    df_3c3h, df_tasks, task_columns = load_results()
-
-    # Extract unique Precision and License values for filters
-    precision_options_3c3h = sorted(df_3c3h['Precision'].dropna().unique().tolist())
-    precision_options_3c3h = [p for p in precision_options_3c3h if p != 'UNK']
-    precision_options_3c3h.append('Missing')
-
-    license_options_3c3h = sorted(df_3c3h['License'].dropna().unique().tolist())
-    license_options_3c3h = [l for l in license_options_3c3h if l != 'UNK']
-    license_options_3c3h.append('Missing')
-
-    precision_options_tasks = sorted(df_tasks['Precision'].dropna().unique().tolist())
-    precision_options_tasks = [p for p in precision_options_tasks if p != 'UNK']
-    precision_options_tasks.append('Missing')
-
-    license_options_tasks = sorted(df_tasks['License'].dropna().unique().tolist())
-    license_options_tasks = [l for l in license_options_tasks if l != 'UNK']
-    license_options_tasks.append('Missing')
-
-    # Get min and max model sizes for sliders, handling 'inf' values
-    min_model_size_3c3h = int(df_3c3h['Model Size Filter'].min())
-    max_model_size_3c3h = int(df_3c3h['Model Size Filter'].max())
-
-    min_model_size_tasks = int(df_tasks['Model Size Filter'].min())
-    max_model_size_tasks = int(df_tasks['Model Size Filter'].max())
-
-    # Exclude 'Model Size Filter' from column selectors
-    column_choices_3c3h = [col for col in df_3c3h.columns if col != 'Model Size Filter']
-    column_choices_tasks = [col for col in df_tasks.columns if col != 'Model Size Filter']
 
     with gr.Blocks() as demo:
         gr.Markdown(HEADER)
@@ -335,176 +69,107 @@ def main():
                                 interactive=True
                             )
 
+                        with gr.Row():
+                            column_selector_tasks = gr.CheckboxGroup(
+                                choices=[],
+                                value=['Rank', 'Model Name'],
+                                label="Select columns to display",
+                            )
+                        
                         with gr.Row():
                             license_filter_retrieval = gr.CheckboxGroup(
-                                choices=license_options_3c3h,
-                                value=license_options_3c3h.copy(),  # Default all selected
+                                choices=[],
+                                value=[],  # Default all selected
                                 label="Filter by License",
                             )
                             precision_filter_retrieval = gr.CheckboxGroup(
-                                choices=precision_options_3c3h,
-                                value=precision_options_3c3h.copy(),  # Default all selected
+                                choices=[],
+                                value=[],  # Default all selected
                                 label="Filter by Precision",
                             )
-                        with gr.Row():
-                            model_size_min_filter_3c3h = gr.Slider(
-                                minimum=min_model_size_3c3h,
-                                maximum=max_model_size_3c3h,
-                                value=min_model_size_3c3h,
-                                step=1,
-                                label="Minimum Model Size",
-                                interactive=True
-                            )
-                            model_size_max_filter_3c3h = gr.Slider(
-                                minimum=min_model_size_3c3h,
-                                maximum=max_model_size_3c3h,
-                                value=max_model_size_3c3h,
-                                step=1,
-                                label="Maximum Model Size",
-                                interactive=True
-                            )
                         
-                        leaderboard_3c3h = gr.Dataframe(
-                            df_3c3h[['Rank', 'Model Name', '3C3H Score', 'Correctness', 'Completeness',
-                                   'Conciseness', 'Helpfulness', 'Honesty', 'Harmlessness']],
+                        retrieval_leaderboard = gr.Dataframe(
+                            df,
                             interactive=False
                         )
                         
-                        def filter_df_3c3h(search_query, selected_cols, precision_filters, license_filters, min_size, max_size):
-                            filtered_df = df_3c3h.copy()
+                        # def filter_df_3c3h(search_query, selected_cols, precision_filters, license_filters):
+                        #     filtered_df = df_3c3h.copy()
                             
-                            # Ensure min_size <= max_size
-                            if min_size > max_size:
-                                min_size, max_size = max_size, min_size
+                        #     # Ensure min_size <= max_size
+                        #     if min_size > max_size:
+                        #         min_size, max_size = max_size, min_size
                             
-                            # Apply search filter
-                            if search_query:
-                                filtered_df = filtered_df[filtered_df['Model Name'].str.contains(search_query, case=False, na=False)]
+                        #     # Apply search filter
+                        #     if search_query:
+                        #         filtered_df = filtered_df[filtered_df['Model Name'].str.contains(search_query, case=False, na=False)]
                             
-                            # Apply Precision filter
-                            if precision_filters:
-                                include_missing = 'Missing' in precision_filters
-                                selected_precisions = [p for p in precision_filters if p != 'Missing']
-                                if include_missing:
-                                    filtered_df = filtered_df[
-                                        (filtered_df['Precision'].isin(selected_precisions)) |
-                                        (filtered_df['Precision'] == 'UNK') |
-                                        (filtered_df['Precision'].isna())
-                                    ]
-                                else:
-                                    filtered_df = filtered_df[filtered_df['Precision'].isin(selected_precisions)]
+                        #     # Apply Precision filter
+                        #     if precision_filters:
+                        #         include_missing = 'Missing' in precision_filters
+                        #         selected_precisions = [p for p in precision_filters if p != 'Missing']
+                        #         if include_missing:
+                        #             filtered_df = filtered_df[
+                        #                 (filtered_df['Precision'].isin(selected_precisions)) |
+                        #                 (filtered_df['Precision'] == 'UNK') |
+                        #                 (filtered_df['Precision'].isna())
+                        #             ]
+                        #         else:
+                        #             filtered_df = filtered_df[filtered_df['Precision'].isin(selected_precisions)]
                             
-                            # Apply License filter
-                            if license_filters:
-                                include_missing = 'Missing' in license_filters
-                                selected_licenses = [l for l in license_filters if l != 'Missing']
-                                if include_missing:
-                                    filtered_df = filtered_df[
-                                        (filtered_df['License'].isin(selected_licenses)) |
-                                        (filtered_df['License'] == 'UNK') |
-                                        (filtered_df['License'].isna())
-                                    ]
-                                else:
-                                    filtered_df = filtered_df[filtered_df['License'].isin(selected_licenses)]
+                        #     # Apply License filter
+                        #     if license_filters:
+                        #         include_missing = 'Missing' in license_filters
+                        #         selected_licenses = [l for l in license_filters if l != 'Missing']
+                        #         if include_missing:
+                        #             filtered_df = filtered_df[
+                        #                 (filtered_df['License'].isin(selected_licenses)) |
+                        #                 (filtered_df['License'] == 'UNK') |
+                        #                 (filtered_df['License'].isna())
+                        #             ]
+                        #         else:
+                        #             filtered_df = filtered_df[filtered_df['License'].isin(selected_licenses)]
                             
-                            # Apply Model Size filter
-                            filtered_df = filtered_df[
-                                (filtered_df['Model Size Filter'] >= min_size) &
-                                (filtered_df['Model Size Filter'] <= max_size)
-                            ]
+                        #     # Apply Model Size filter
+                        #     filtered_df = filtered_df[
+                        #         (filtered_df['Model Size Filter'] >= min_size) &
+                        #         (filtered_df['Model Size Filter'] <= max_size)
+                        #     ]
                             
-                            # Remove existing 'Rank' column if present
-                            if 'Rank' in filtered_df.columns:
-                                filtered_df = filtered_df.drop(columns=['Rank'])
+                        #     # Remove existing 'Rank' column if present
+                        #     if 'Rank' in filtered_df.columns:
+                        #         filtered_df = filtered_df.drop(columns=['Rank'])
                             
-                            # Recalculate Rank after filtering
-                            filtered_df = filtered_df.reset_index(drop=True)
-                            filtered_df.insert(0, 'Rank', range(1, len(filtered_df) + 1))
+                        #     # Recalculate Rank after filtering
+                        #     filtered_df = filtered_df.reset_index(drop=True)
+                        #     filtered_df.insert(0, 'Rank', range(1, len(filtered_df) + 1))
                             
-                            # Ensure selected columns are present
-                            selected_cols = [col for col in selected_cols if col in filtered_df.columns]
+                        #     # Ensure selected columns are present
+                        #     selected_cols = [col for col in selected_cols if col in filtered_df.columns]
                             
-                            return filtered_df[selected_cols]
+                        #     return filtered_df[selected_cols]
                         
-                        # Bind the filter function to the appropriate events
-                        filter_inputs_3c3h = [
-                            search_box_retrieval,
-                            precision_filter_retrieval,
-                            license_filter_retrieval,
-                            model_size_min_filter_3c3h,
-                            model_size_max_filter_3c3h
-                        ]
-                        search_box_retrieval.submit(
-                            filter_df_3c3h,
-                            inputs=filter_inputs_3c3h,
-                            outputs=leaderboard_3c3h
-                        )
+                        # # Bind the filter function to the appropriate events
+                        # filter_inputs_3c3h = [
+                        #     search_box_retrieval,
+                        #     precision_filter_retrieval,
+                        #     license_filter_retrieval,
+                        # ]
+                        # search_box_retrieval.submit(
+                        #     filter_df_3c3h,
+                        #     inputs=filter_inputs_3c3h,
+                        #     outputs=leaderboard_3c3h
+                        # )
                         
-                        # Bind change events for CheckboxGroups and sliders
-                        for component in filter_inputs_3c3h:
-                            component.change(
-                                filter_df_3c3h,
-                                inputs=filter_inputs_3c3h,
-                                outputs=leaderboard_3c3h
-                            )
+                        # # Bind change events for CheckboxGroups and sliders
+                        # for component in filter_inputs_3c3h:
+                        #     component.change(
+                        #         filter_df_3c3h,
+                        #         inputs=filter_inputs_3c3h,
+                        #         outputs=leaderboard_3c3h
+                        #     )
                     
-                    with gr.Tab("Submit Retriever"): 
-
-                        model_name_input = gr.Textbox(
-                            label="Model", 
-                            placeholder="Enter the full model name from HuggingFace Hub (e.g., inceptionai/jais-family-30b-8k)"
-                        )
-                        revision_input = gr.Textbox(
-                            label="Revision", 
-                            placeholder="main", 
-                            value="main"
-                        )
-                        precision_input = gr.Dropdown(
-                            choices=["float16", "float32", "bfloat16", "8bit", "4bit"], 
-                            label="Precision",
-                            value="float16"
-                        )
-                        params_input = gr.Textbox(
-                            label="Params", 
-                            placeholder="Enter the approximate number of parameters as Integer (e.g., 7, 13, 30, 70 ...)"
-                        )
-                        # Changed from Dropdown to Textbox with default value "Open"
-                        license_input = gr.Textbox(
-                            label="License", 
-                            placeholder="Enter the license type (Generic one is 'Open' in case no License is provided)", 
-                            value="Open"
-                        )
-                        submit_button = gr.Button("Submit Model")
-                        submission_result = gr.Markdown()
-
-                        submit_button.click(
-                            submit_model,
-                            inputs=[model_name_input, revision_input, precision_input, params_input, license_input],
-                            outputs=submission_result
-                        )
-                        
-                        # Load pending, finished, and failed requests
-                        df_pending = load_requests('pending')
-                        df_finished = load_requests('finished')
-                        df_failed = load_requests('failed')
-
-                        # Display the tables
-                        gr.Markdown("## Evaluation Status")
-                        with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=False):
-                            if not df_pending.empty:
-                                gr.Dataframe(df_pending)
-                            else:
-                                gr.Markdown("No pending evaluations.")
-                        with gr.Accordion(f"Finished Evaluations ({len(df_finished)})", open=False):
-                            if not df_finished.empty:
-                                gr.Dataframe(df_finished)
-                            else:
-                                gr.Markdown("No finished evaluations.")
-                        with gr.Accordion(f"Failed Evaluations ({len(df_failed)})", open=False):
-                            if not df_failed.empty:
-                                gr.Dataframe(df_failed)
-                            else:
-                                gr.Markdown("No failed evaluations.")
+                    submit_gradio_module()
 
             with gr.Tab("Reranking"):
                 with gr.Tabs():
@@ -518,133 +183,134 @@ def main():
                             )
                         with gr.Row():
                             column_selector_tasks = gr.CheckboxGroup(
-                                choices=column_choices_tasks,
-                                value=['Rank', 'Model Name'] + task_columns,
+                                choices=[],
+                                value=['Rank', 'Model Name'],
                                 label="Select columns to display",
                             )
                         with gr.Row():
                             license_filter_tasks = gr.CheckboxGroup(
-                                choices=license_options_tasks,
-                                value=license_options_tasks.copy(),  # Default all selected
+                                choices=[],
+                                value=[],  # Default all selected
                                 label="Filter by License",
                             )
                             precision_filter_tasks = gr.CheckboxGroup(
-                                choices=precision_options_tasks,
-                                value=precision_options_tasks.copy(),  # Default all selected
+                                choices=[],
+                                value=[],  # Default all selected
                                 label="Filter by Precision",
                             )
-                        with gr.Row():
-                            model_size_min_filter_tasks = gr.Slider(
-                                minimum=min_model_size_tasks,
-                                maximum=max_model_size_tasks,
-                                value=min_model_size_tasks,
-                                step=1,
-                                label="Minimum Model Size",
-                                interactive=True
-                            )
-                            model_size_max_filter_tasks = gr.Slider(
-                                minimum=min_model_size_tasks,
-                                maximum=max_model_size_tasks,
-                                value=max_model_size_tasks,
-                                step=1,
-                                label="Maximum Model Size",
-                                interactive=True
-                            )
+                        # with gr.Row():
+                        #     model_size_min_filter_tasks = gr.Slider(
+                        #         minimum=min_model_size_tasks,
+                        #         maximum=max_model_size_tasks,
+                        #         value=min_model_size_tasks,
+                        #         step=1,
+                        #         label="Minimum Model Size",
+                        #         interactive=True
+                        #     )
+                        #     model_size_max_filter_tasks = gr.Slider(
+                        #         minimum=min_model_size_tasks,
+                        #         maximum=max_model_size_tasks,
+                        #         value=max_model_size_tasks,
+                        #         step=1,
+                        #         label="Maximum Model Size",
+                        #         interactive=True
+                        #     )
                         
                         leaderboard_tasks = gr.Dataframe(
-                            df_tasks[['Rank', 'Model Name'] + task_columns],
-                            interactive=False
+                            df,
+                            # headers="auto",  # Automatically use DataFrame's headers
+                            # label="MultiIndex DataFrame",
+                            # interactive=False,
                         )
                         
-                        def filter_df_tasks(search_query, selected_cols, precision_filters, license_filters, min_size, max_size):
-                            filtered_df = df_tasks.copy()
+                        # def filter_df_tasks(search_query, selected_cols, precision_filters, license_filters, min_size, max_size):
+                        #     filtered_df = df_tasks.copy()
                             
-                            # Ensure min_size <= max_size
-                            if min_size > max_size:
-                                min_size, max_size = max_size, min_size
+                        #     # Ensure min_size <= max_size
+                        #     if min_size > max_size:
+                        #         min_size, max_size = max_size, min_size
                             
-                            # Apply search filter
-                            if search_query:
-                                filtered_df = filtered_df[filtered_df['Model Name'].str.contains(search_query, case=False, na=False)]
+                        #     # Apply search filter
+                        #     if search_query:
+                        #         filtered_df = filtered_df[filtered_df['Model Name'].str.contains(search_query, case=False, na=False)]
                             
-                            # Apply Precision filter
-                            if precision_filters:
-                                include_missing = 'Missing' in precision_filters
-                                selected_precisions = [p for p in precision_filters if p != 'Missing']
-                                if include_missing:
-                                    filtered_df = filtered_df[
-                                        (filtered_df['Precision'].isin(selected_precisions)) |
-                                        (filtered_df['Precision'] == 'UNK') |
-                                        (filtered_df['Precision'].isna())
-                                    ]
-                                else:
-                                    filtered_df = filtered_df[filtered_df['Precision'].isin(selected_precisions)]
+                        #     # Apply Precision filter
+                        #     if precision_filters:
+                        #         include_missing = 'Missing' in precision_filters
+                        #         selected_precisions = [p for p in precision_filters if p != 'Missing']
+                        #         if include_missing:
+                        #             filtered_df = filtered_df[
+                        #                 (filtered_df['Precision'].isin(selected_precisions)) |
+                        #                 (filtered_df['Precision'] == 'UNK') |
+                        #                 (filtered_df['Precision'].isna())
+                        #             ]
+                        #         else:
+                        #             filtered_df = filtered_df[filtered_df['Precision'].isin(selected_precisions)]
                             
-                            # Apply License filter
-                            if license_filters:
-                                include_missing = 'Missing' in license_filters
-                                selected_licenses = [l for l in license_filters if l != 'Missing']
-                                if include_missing:
-                                    filtered_df = filtered_df[
-                                        (filtered_df['License'].isin(selected_licenses)) |
-                                        (filtered_df['License'] == 'UNK') |
-                                        (filtered_df['License'].isna())
-                                    ]
-                                else:
-                                    filtered_df = filtered_df[filtered_df['License'].isin(selected_licenses)]
+                        #     # Apply License filter
+                        #     if license_filters:
+                        #         include_missing = 'Missing' in license_filters
+                        #         selected_licenses = [l for l in license_filters if l != 'Missing']
+                        #         if include_missing:
+                        #             filtered_df = filtered_df[
+                        #                 (filtered_df['License'].isin(selected_licenses)) |
+                        #                 (filtered_df['License'] == 'UNK') |
+                        #                 (filtered_df['License'].isna())
+                        #             ]
+                        #         else:
+                        #             filtered_df = filtered_df[filtered_df['License'].isin(selected_licenses)]
                             
-                            # Apply Model Size filter
-                            filtered_df = filtered_df[
-                                (filtered_df['Model Size Filter'] >= min_size) &
-                                (filtered_df['Model Size Filter'] <= max_size)
-                            ]
+                        #     # Apply Model Size filter
+                        #     filtered_df = filtered_df[
+                        #         (filtered_df['Model Size Filter'] >= min_size) &
+                        #         (filtered_df['Model Size Filter'] <= max_size)
+                        #     ]
                             
-                            # Remove existing 'Rank' column if present
-                            if 'Rank' in filtered_df.columns:
-                                filtered_df = filtered_df.drop(columns=['Rank'])
+                        #     # Remove existing 'Rank' column if present
+                        #     if 'Rank' in filtered_df.columns:
+                        #         filtered_df = filtered_df.drop(columns=['Rank'])
                             
-                            # Sort by the first task column if it exists
-                            if task_columns:
-                                first_task = task_columns[0]
-                                filtered_df = filtered_df.sort_values(by=first_task, ascending=False)
-                            else:
-                                filtered_df = filtered_df.sort_values(by='Model Name', ascending=True)
+                        #     # Sort by the first task column if it exists
+                        #     if task_columns:
+                        #         first_task = task_columns[0]
+                        #         filtered_df = filtered_df.sort_values(by=first_task, ascending=False)
+                        #     else:
+                        #         filtered_df = filtered_df.sort_values(by='Model Name', ascending=True)
                             
-                            # Recalculate Rank after filtering
-                            filtered_df = filtered_df.reset_index(drop=True)
-                            filtered_df.insert(0, 'Rank', range(1, len(filtered_df) + 1))
+                        #     # Recalculate Rank after filtering
+                        #     filtered_df = filtered_df.reset_index(drop=True)
+                        #     filtered_df.insert(0, 'Rank', range(1, len(filtered_df) + 1))
                             
-                            # Ensure selected columns are present
-                            selected_cols = [col for col in selected_cols if col in filtered_df.columns]
+                        #     # Ensure selected columns are present
+                        #     selected_cols = [col for col in selected_cols if col in filtered_df.columns]
                             
-                            return filtered_df[selected_cols]
+                        #     return filtered_df[selected_cols]
                         
-                        # Bind the filter function to the appropriate events
-                        filter_inputs_tasks = [
-                            search_box_tasks,
-                            column_selector_tasks,
-                            precision_filter_tasks,
-                            license_filter_tasks,
-                            model_size_min_filter_tasks,
-                            model_size_max_filter_tasks
-                        ]
-                        search_box_tasks.submit(
-                            filter_df_tasks,
-                            inputs=filter_inputs_tasks,
-                            outputs=leaderboard_tasks
-                        )
+                        # # Bind the filter function to the appropriate events
+                        # filter_inputs_tasks = [
+                        #     search_box_tasks,
+                        #     column_selector_tasks,
+                        #     precision_filter_tasks,
+                        #     license_filter_tasks,
+                        #     model_size_min_filter_tasks,
+                        #     model_size_max_filter_tasks
+                        # ]
+                        # search_box_tasks.submit(
+                        #     filter_df_tasks,
+                        #     inputs=filter_inputs_tasks,
+                        #     outputs=leaderboard_tasks
+                        # )
                         
-                        # Bind change events for CheckboxGroups and sliders
-                        for component in filter_inputs_tasks:
-                            component.change(
-                                filter_df_tasks,
-                                inputs=filter_inputs_tasks,
-                                outputs=leaderboard_tasks
-                            )
-
-                    with gr.Tab("Submit Reranker"):
-                        pass
-
+                        # # Bind change events for CheckboxGroups and sliders
+                        # for component in filter_inputs_tasks:
+                        #     component.change(
+                        #         filter_df_tasks,
+                        #         inputs=filter_inputs_tasks,
+                        #         outputs=leaderboard_tasks
+                        #     )
+
+                    submit_gradio_module()
+                    
             with gr.Tab("LLM Context Answering"):
                 with gr.Tabs():
                     with gr.Tab("Leaderboard"):
@@ -654,7 +320,7 @@ def main():
 
             with gr.Row():
                 with gr.Accordion("📙 Citation", open=False):
-                    citation_button = gr.Textbox(
+                    gr.Textbox(
                         value=CITATION_BUTTON_TEXT,
                         label=CITATION_BUTTON_LABEL,
                         lines=20,

utils.py

@@ -0,0 +1,251 @@
+import gradio as gr
+import pandas as pd
+import json
+import os
+from pathlib import Path
+from huggingface_hub import HfApi
+
+api = HfApi()
+
+def load_retrieval_results():
+    base_path = Path(__file__).parent
+    results_dir = base_path / "assets" / "results"
+
+    retrieval_dataframe_path = results_dir / "retrieval_results.json"
+    if not retrieval_dataframe_path.exists():
+        df = pd.DataFrame(columns=["Model", "Model Size", "Embedding Dimension", "Max Tokens", "Web Search Dataset (MRR)", "Web Search Dataset (nDCG@k=None)"])
+    else:
+        df = pd.read_json(retrieval_dataframe_path)
+    return df
+
+def get_model_info(model_id, verbose=False):
+    model_info = api.model_info(model_id)
+    num_downloads = model_info.downloads
+    num_likes = model_info.likes
+    license = model_info.card_data["license"]
+    num_parameters = round(model_info.safetensors.total / 1e6)
+    supported_precisions = list(model_info.safetensors.parameters.keys())
+    if verbose:
+        print(f"Model '{model_id}' has {num_downloads} downloads, {num_likes} likes, and is licensed under {license}.")
+        print(f"The model has approximately {num_parameters:.2f} billion parameters.")
+        print(f"The model supports the following precisions: {supported_precisions}")
+    return num_downloads, num_likes, license, num_parameters, supported_precisions
+
+def fetch_model_information(model_name):
+    try:
+        _, _, license, num_parameters, supported_precisions = get_model_info(model_name)
+        if len(supported_precisions) == 0:
+            supported_precisions = [None]
+    except Exception as e:
+        gr.Error(f"**Error: Could not fetch model information. {str(e)}**")
+        return
+    return gr.update(choices=supported_precisions, value=supported_precisions[0]), num_parameters, license
+
+def submit_model(model_name, revision, precision, params, license):
+    # Load existing evaluations
+    df_retrieval = load_retrieval_results()
+    if df_retrieval.empty:
+        return "**Error: Could not load the retrieval results.**"
+    
+    existing_models_results = df_retrieval[['Model', 'Revision', 'Precision']]
+
+    # Handle 'Missing' precision
+    if precision == 'Missing':
+        precision = None
+    else:
+        precision = precision.strip().lower()
+
+    # Load pending and finished requests from the dataset repository
+    df_pending = load_requests('pending')
+    df_finished = load_requests('finished')
+
+    # Check if model is already evaluated
+    model_exists_in_results = ((existing_models_results['Model Name'] == model_name) &
+                               (existing_models_results['Revision'] == revision) &
+                               (existing_models_results['Precision'] == precision)).any()
+    if model_exists_in_results:
+        return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' has already been evaluated.**"
+
+    # Check if model is in pending requests
+    if not df_pending.empty:
+        existing_models_pending = df_pending[['model_name', 'revision', 'precision']]
+        model_exists_in_pending = ((existing_models_pending['model_name'] == model_name) &
+                                   (existing_models_pending['revision'] == revision) &
+                                   (existing_models_pending['precision'] == precision)).any()
+        if model_exists_in_pending:
+            return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' is already in the pending evaluations.**"
+
+    # Check if model is in finished requests
+    if not df_finished.empty:
+        existing_models_finished = df_finished[['model_name', 'revision', 'precision']]
+        model_exists_in_finished = ((existing_models_finished['model_name'] == model_name) &
+                                    (existing_models_finished['revision'] == revision) &
+                                    (existing_models_finished['precision'] == precision)).any()
+        if model_exists_in_finished:
+            return f"**Model '{model_name}' with revision '{revision}' and precision '{precision}' has already been evaluated.**"
+
+    # Check if model exists on HuggingFace Hub
+    try:
+        api.model_info(model_name)
+    except Exception as e:
+        return f"**Error: Could not find model '{model_name}' on HuggingFace Hub. Please ensure the model name is correct and the model is public.**"
+
+    # Proceed with submission
+    status = "PENDING"
+
+    # Prepare the submission data
+    submission = {
+        "model_name": model_name,
+        "license": license,
+        "revision": revision,
+        "precision": precision,
+        "status": status,
+        "params": params
+    }
+
+    # Serialize the submission to JSON
+    submission_json = json.dumps(submission, indent=2)
+
+    # Define the file path in the repository
+    org_model = model_name.split('/')
+    if len(org_model) != 2:
+        return "**Please enter the full model name including the organization or username, e.g., 'intfloat/multilingual-e5-large-instruct'**"
+    org, model_id = org_model
+    precision_str = precision if precision else 'Missing'
+    file_path_in_repo = f"pending/{org}/{model_id}_eval_request_{revision}_{precision_str}.json"
+
+    # Upload the submission to the dataset repository
+    try:
+        hf_api_token = os.environ.get('HF_API_TOKEN', None)
+        api.upload_file(
+            path_or_fileobj=submission_json.encode('utf-8'),
+            path_in_repo=file_path_in_repo,
+            repo_id=DATASET_REPO_ID,
+            repo_type="dataset",
+            token=hf_api_token
+        )
+    except Exception as e:
+        return f"**Error: Could not submit the model. {str(e)}**"
+
+    return f"**Model '{model_name}' has been submitted for evaluation.**"
+
+def load_requests(status_folder):
+    api = HfApi()
+    requests_data = []
+    folder_path_in_repo = status_folder  # 'pending', 'finished', or 'failed'
+
+    hf_api_token = os.environ.get('HF_API_TOKEN', None)
+
+    try:
+        # List files in the dataset repository
+        files_info = api.list_repo_files(
+            repo_id=DATASET_REPO_ID,
+            repo_type="dataset",
+            token=hf_api_token
+        )
+    except Exception as e:
+        print(f"Error accessing dataset repository: {e}")
+        return pd.DataFrame()  # Return empty DataFrame if repository not found or inaccessible
+
+    # Filter files in the desired folder
+    files_in_folder = [f for f in files_info if f.startswith(f"{folder_path_in_repo}/") and f.endswith('.json')]
+
+    for file_path in files_in_folder:
+        try:
+            # Download the JSON file
+            local_file_path = hf_hub_download(
+                repo_id=DATASET_REPO_ID,
+                filename=file_path,
+                repo_type="dataset",
+                token=hf_api_token
+            )
+            # Load JSON data
+            with open(local_file_path, 'r') as f:
+                request = json.load(f)
+            requests_data.append(request)
+        except Exception as e:
+            print(f"Error loading file {file_path}: {e}")
+            continue  # Skip files that can't be loaded
+
+    df = pd.DataFrame(requests_data)
+    return df
+
+
+def submit_gradio_module():
+    with gr.Tab("Submit Model") as submitter_tab: 
+
+        with gr.Row(equal_height=True):
+            model_name_input = gr.Textbox(
+                label="Model", 
+                placeholder="Enter the full model name from HuggingFace Hub (e.g., intfloat/multilingual-e5-large-instruct)",
+                scale=4,
+            )
+            fetch_data_button = gr.Button(value="Auto Fetch Model Info", variant="secondary")
+        
+        with gr.Row():
+            precision_input = gr.Dropdown(
+                choices=["F16", "F32", "BF16", "I8", "U8", "I16"], 
+                label="Precision",
+                value="F16"
+            )
+            params_input = gr.Textbox(
+                label="Params (in Millions)",
+                placeholder="Enter the approximate number of parameters as Integer (e.g., 7, 13, 30, 70 ...)"
+            )
+        
+        with gr.Row():
+            license_input = gr.Textbox(
+                label="License", 
+                placeholder="Enter the license type (Generic one is 'Open' in case no License is provided)", 
+                value="Open"
+            )
+            revision_input = gr.Textbox(
+                label="Revision", 
+                placeholder="main", 
+                value="main"
+            )
+        
+        submit_button = gr.Button("Submit Model", variant="primary")
+        submission_result = gr.Markdown()
+        fetch_outputs = [precision_input, params_input, license_input]
+
+        fetch_data_button.click(
+            fetch_model_information,
+            inputs=[model_name_input],
+            outputs=fetch_outputs
+        )
+        model_name_input.submit(
+            fetch_model_information,
+            inputs=[model_name_input],
+            outputs=fetch_outputs
+        )
+        submit_button.click(
+            submit_model,
+            inputs=[model_name_input, revision_input, precision_input, params_input, license_input],
+            outputs=submission_result
+        )
+        
+        # Load pending, finished, and failed requests
+        df_pending = load_requests('pending')
+        df_finished = load_requests('finished')
+        df_failed = load_requests('failed')
+
+        # Display the tables
+        gr.Markdown("## Evaluation Status")
+        with gr.Accordion(f"Pending Evaluations ({len(df_pending)})", open=False):
+            if not df_pending.empty:
+                gr.Dataframe(df_pending)
+            else:
+                gr.Markdown("No pending evaluations.")
+        with gr.Accordion(f"Finished Evaluations ({len(df_finished)})", open=False):
+            if not df_finished.empty:
+                gr.Dataframe(df_finished)
+            else:
+                gr.Markdown("No finished evaluations.")
+        with gr.Accordion(f"Failed Evaluations ({len(df_failed)})", open=False):
+            if not df_failed.empty:
+                gr.Dataframe(df_failed)
+            else:
+                gr.Markdown("No failed evaluations.")
+
+    return submitter_tab