updated scripts

app.py

@@ -9,10 +9,11 @@ import src.display.utils as utils
 import src.envs as envs
 import src.populate as populate
 import src.submission.submit as submit
-
-
+import os
+TOKEN = os.environ.get("HF_TOKEN", None)
+print("TOKEN", TOKEN)
 def restart_space():
-    envs.API.restart_space(repo_id=envs.REPO_ID, token=envs.TOKEN)
+    envs.API.restart_space(repo_id=envs.REPO_ID, token=TOKEN)
 
 try:
     print(envs.EVAL_REQUESTS_PATH)

main_backend.py

@@ -111,7 +111,7 @@ def main():
     parser = argparse.ArgumentParser(description="Run auto evaluation with optional reproducibility feature")
 
     # Optional arguments
-    parser.add_argument("--reproduce", type=bool, default=True, help="Reproduce the evaluation results")
+    parser.add_argument("--reproduce", type=bool, default=False, help="Reproduce the evaluation results")
     parser.add_argument("--model", type=str, default=None, help="Your Model ID")
     parser.add_argument("--precision", type=str, default="float16", help="Precision of your model")
     parser.add_argument("--publish", type=bool, default=False, help="whether directly publish the evaluation results on HF")

src/Makefile

@@ -1,13 +0,0 @@
-.PHONY: style format
-
-
-style:
-	python -m black --line-length 119 .
-	python -m isort .
-	ruff check --fix .
-
-
-quality:
-	python -m black --check --line-length 119 .
-	python -m isort --check-only .
-	ruff check .

src/README.md

@@ -1,47 +0,0 @@
----
-title: Humanlike Evaluation Leaderboard
-emoji: 🥇
-colorFrom: blue
-colorTo: indigo
-sdk: gradio
-sdk_version: 4.37.1
-app_file: app.py
-pinned: true
-license: apache-2.0
-tags:
-- leaderboard
-models:
-- google/gemma-2-9b
----
-
-
-python>3.10
-pip spacy
-python -m spacy download en_core_web_sm
-pip install google.generativeai
-python -m spacy download en_core_web_trf
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
-
-Most of the variables to change for a default leaderboard are in env (replace the path for your leaderboard) and src/display/about.
-
-Results files should have the following format:
-```
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
-```
-
-Request files are created automatically by this tool.

src/app.py

@@ -1,329 +0,0 @@
-import gradio as gr
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-import src.display.about as about
-from src.display.css_html_js import custom_css
-import src.display.utils as utils
-import src.envs as envs
-import src.populate as populate
-import src.submission.submit as submit
-import os
-TOKEN = os.environ.get("HF_TOKEN", None)
-print("TOKEN", TOKEN)
-def restart_space():
-    envs.API.restart_space(repo_id=envs.REPO_ID, token=TOKEN)
-
-try:
-    print(envs.EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=envs.QUEUE_REPO, local_dir=envs.EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-    )
-except Exception:
-    restart_space()
-try:
-    print(envs.EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=envs.RESULTS_REPO, local_dir=envs.EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-    )
-except Exception:
-    restart_space()
-
-raw_data, original_df = populate.get_leaderboard_df(envs.EVAL_RESULTS_PATH, envs.EVAL_REQUESTS_PATH, utils.COLS, utils.BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = populate.get_evaluation_queue_df(envs.EVAL_REQUESTS_PATH, utils.EVAL_COLS)
-
-
-# Searching and filtering
-def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
-    query: str,
-):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
-    return df
-
-
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[utils.AutoEvalColumn.dummy.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [
-        utils.AutoEvalColumn.model_type_symbol.name,
-        utils.AutoEvalColumn.model.name,
-    ]
-    # We use COLS to maintain sorting
-    filtered_df = df[
-        always_here_cols + [c for c in utils.COLS if c in df.columns and c in columns] + [utils.AutoEvalColumn.dummy.name]
-    ]
-    return filtered_df
-
-
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-    final_df = []
-    if query != "":
-        queries = [q.strip() for q in query.split(";")]
-        for _q in queries:
-            _q = _q.strip()
-            if _q != "":
-                temp_filtered_df = search_table(filtered_df, _q)
-                if len(temp_filtered_df) > 0:
-                    final_df.append(temp_filtered_df)
-        if len(final_df) > 0:
-            filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(
-                subset=[utils.AutoEvalColumn.model.name, utils.AutoEvalColumn.precision.name, utils.AutoEvalColumn.revision.name]
-            )
-
-    return filtered_df
-
-
-def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
-    # Show all models
-    # if show_deleted:
-    #   filtered_df = df
-    # else:  # Show only still on the hub models
-        # filtered_df = df[df[utils.AutoEvalColumn.still_on_hub.name]]
-
-    filtered_df = df
-    
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[utils.AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[utils.AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    numeric_interval = pd.IntervalIndex(sorted([utils.NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[utils.AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
-
-    return filtered_df
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(about.TITLE)
-    gr.Markdown(about.INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in utils.fields(utils.AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden and not c.dummy
-                            ],
-                            value=[
-                                c.name
-                                for c in utils.fields(utils.AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in utils.ModelType],
-                        value=[t.to_str() for t in utils.ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in utils.Precision],
-                        value=[i.value.name for i in utils.Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(utils.NUMERIC_INTERVALS.keys()),
-                        value=list(utils.NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-            leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[
-                    [c.name for c in utils.fields(utils.AutoEvalColumn) if c.never_hidden]
-                    + shown_columns.value
-                    + [utils.AutoEvalColumn.dummy.name]
-                ],
-                headers=[c.name for c in utils.fields(utils.AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=utils.TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-                column_widths=["2%", "33%"]
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[utils.COLS],
-                headers=utils.COLS,
-                datatype=utils.TYPES,
-                visible=False,
-            )
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    deleted_models_visibility,
-                    search_bar,
-                ],
-                leaderboard_table,
-            )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        deleted_models_visibility,
-                        search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(about.LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(about.EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=utils.EVAL_COLS,
-                                datatype=utils.EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=utils.EVAL_COLS,
-                                datatype=utils.EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=utils.EVAL_COLS,
-                                datatype=utils.EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in utils.ModelType if t != utils.ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in utils.Precision if i != utils.Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in utils.WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                submit.add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=about.CITATION_BUTTON_TEXT,
-                label=about.CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()

src/backend/model_operations.py

@@ -166,14 +166,14 @@ class SummaryGenerator:
                     Stimuli_2_column = df_sheet["Stimuli-2"]
                 
                 # 遍历Prompt0列的值  
-                for j, prompt_value in enumerate(tqdm(prompt_column[0:2], desc=f"Processing {sheet_name}"), start=0):  
+                for j, prompt_value in enumerate(tqdm(prompt_column, desc=f"Processing {sheet_name}"), start=0):  
                     ID = 'E' + str(i)  
                     # q_ID = ID + '_' + str(j)  
                       
                     # print(ID, q_ID, prompt_value)
                     system_prompt = envs.SYSTEM_PROMPT 
                     _user_prompt = prompt_value
-                    for ii in range(2):
+                    for ii in range(10):
                     # user_prompt = f"{envs.USER_PROMPT}\nPassage:\n{_source}"
                         while True:
                             try:

src/main_backend.py

@@ -1,126 +0,0 @@
-import argparse
-import logging
-import pprint
-import os
-
-from huggingface_hub import snapshot_download
-
-import src.backend.run_eval_suite as run_eval_suite
-import src.backend.manage_requests as manage_requests
-import src.backend.sort_queue as sort_queue
-import src.envs as envs
-
-os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
-
-logging.basicConfig(level=logging.ERROR)
-pp = pprint.PrettyPrinter(width=80)
-
-PENDING_STATUS = "PENDING"
-RUNNING_STATUS = "RUNNING"
-FINISHED_STATUS = "FINISHED"
-FAILED_STATUS = "FAILED"
-# import os
-snapshot_download(repo_id=envs.RESULTS_REPO, revision="main",
-                local_dir=envs.EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-
-snapshot_download(repo_id=envs.QUEUE_REPO, revision="main",
-                local_dir=envs.EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-# exit()
-
-def run_auto_eval(args):
-    if not args.reproduce:
-        current_pending_status = [PENDING_STATUS]
-        print('_________________')
-        manage_requests.check_completed_evals(
-            api=envs.API,
-            checked_status=RUNNING_STATUS,
-            completed_status=FINISHED_STATUS,
-            failed_status=FAILED_STATUS,
-            hf_repo=envs.QUEUE_REPO,
-            local_dir=envs.EVAL_REQUESTS_PATH_BACKEND,
-            hf_repo_results=envs.RESULTS_REPO,
-            local_dir_results=envs.EVAL_RESULTS_PATH_BACKEND
-        )
-        logging.info("Checked completed evals")
-        eval_requests = manage_requests.get_eval_requests(job_status=current_pending_status,
-                                                        hf_repo=envs.QUEUE_REPO,
-                                                        local_dir=envs.EVAL_REQUESTS_PATH_BACKEND)
-        logging.info("Got eval requests")
-        eval_requests = sort_queue.sort_models_by_priority(api=envs.API, models=eval_requests)
-        logging.info("Sorted eval requests")
-
-        print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
-        print(eval_requests)
-        if len(eval_requests) == 0:
-            print("No eval requests found. Exiting.")
-            return
-
-        if args.model is not None:
-            eval_request = manage_requests.EvalRequest(
-                model=args.model,
-                status=PENDING_STATUS,
-                precision=args.precision
-            )
-            pp.pprint(eval_request)
-        else:
-            eval_request = eval_requests[0]
-            pp.pprint(eval_request)
-
-        # manage_requests.set_eval_request(
-        #     api=envs.API,
-        #     eval_request=eval_request,
-        #     new_status=RUNNING_STATUS,
-        #     hf_repo=envs.QUEUE_REPO,
-        #     local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
-        # )
-        # logging.info("Set eval request to running, now running eval")
-
-        run_eval_suite.run_evaluation(
-            eval_request=eval_request,
-            local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
-            results_repo=envs.RESULTS_REPO,
-            batch_size=1,
-            device=envs.DEVICE,
-            no_cache=True,
-            need_check=not args.publish,
-            write_results=args.update
-        )
-        logging.info("Eval finished, now setting status to finished")
-    else:
-        eval_request = manage_requests.EvalRequest(
-            model=args.model,
-            status=PENDING_STATUS,
-            precision=args.precision
-        )
-        pp.pprint(eval_request)
-        logging.info("Running reproducibility eval")
-
-        run_eval_suite.run_evaluation(
-            eval_request=eval_request,
-            local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
-            results_repo=envs.RESULTS_REPO,
-            batch_size=1,
-            device=envs.DEVICE,
-            need_check=not args.publish,
-            write_results=args.update
-        )
-        logging.info("Reproducibility eval finished")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Run auto evaluation with optional reproducibility feature")
-
-    # Optional arguments
-    parser.add_argument("--reproduce", type=bool, default=False, help="Reproduce the evaluation results")
-    parser.add_argument("--model", type=str, default=None, help="Your Model ID")
-    parser.add_argument("--precision", type=str, default="float16", help="Precision of your model")
-    parser.add_argument("--publish", type=bool, default=False, help="whether directly publish the evaluation results on HF")
-    parser.add_argument("--update", type=bool, default=False, help="whether to update google drive files")
-
-    args = parser.parse_args()
-
-    run_auto_eval(args)
-
-
-if __name__ == "__main__":
-    main()

src/pyproject.toml

@@ -1,13 +0,0 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119

src/requirements.txt

@@ -1,17 +0,0 @@
-APScheduler==3.10.1
-black==23.11.0
-click==8.1.3
-datasets==2.14.5
-gradio==4.4.0
-gradio_client==0.7.0
-huggingface-hub>=0.18.0
-litellm==1.15.1
-matplotlib==3.7.1
-numpy==1.24.2
-pandas==2.0.0
-python-dateutil==2.8.2
-requests==2.28.2
-tqdm==4.65.0
-transformers==4.35.2
-tokenizers>=0.15.0
-sentence-transformers==2.2.2