feat: modify leaderboard

.pre-commit-config.yaml

@@ -1,53 +0,0 @@
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-default_language_version:
- python: python3
-
-ci:
- autofix_prs: true
- autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
- autoupdate_schedule: quarterly
-
-repos:
- - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.3.0
- hooks:
- - id: check-yaml
- - id: check-case-conflict
- - id: detect-private-key
- - id: check-added-large-files
- args: ['--maxkb=1000']
- - id: requirements-txt-fixer
- - id: end-of-file-fixer
- - id: trailing-whitespace
-
- - repo: https://github.com/PyCQA/isort
- rev: 5.12.0
- hooks:
- - id: isort
- name: Format imports
-
- - repo: https://github.com/psf/black
- rev: 22.12.0
- hooks:
- - id: black
- name: Format code
- additional_dependencies: ['click==8.0.2']
-
- - repo: https://github.com/charliermarsh/ruff-pre-commit
- # Ruff version.
- rev: 'v0.0.267'
- hooks:
- - id: ruff

app.py

@@ -1,345 +1,151 @@
-import subprocess
+# matplotlib.use('macosx')
 import gradio as gr
+import matplotlib
+import numpy as np
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
- CITATION_BUTTON_LABEL,
- CITATION_BUTTON_TEXT,
- EVALUATION_QUEUE_TEXT,
- INTRODUCTION_TEXT,
- LLM_BENCHMARKS_TEXT,
- TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
- BENCHMARK_COLS,
- COLS,
- EVAL_COLS,
- EVAL_TYPES,
- NUMERIC_INTERVALS,
- TYPES,
- AutoEvalColumn,
- ModelType,
- fields,
- WeightType,
- Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
- API.restart_space(repo_id=REPO_ID)
-
-try:
- print(EVAL_REQUESTS_PATH)
- snapshot_download(
- repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
- )
-except Exception:
- restart_space()
-try:
- print(EVAL_RESULTS_PATH)
- snapshot_download(
- repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+
+TASK1_COLS = [
+ ("Model", "str"),
+ ("Acc", "number"),
+ ("F1", "number"),
+ ("MCC", "number"),
+]
+
+TASK2_COLS = [
+ ("Model", "str"),
+ ("Rouge-1", "number"),
+ ("Rouge-2", "number"),
+ ("Rouge-L", "number"),
+ ("BertScore", "number"),
+ ("BartScore", "number"),
+]
+
+TASK3_COLS = [
+ ("Model", "str"),
+ ("Sharpe Ratio", "number"),
+ ("Sharpe Ratio - DRIV", "number"),
+ ("Sharpe Ratio - FORM", "number"),
+ ("Sharpe Ratio - JNJ", "number"),
+ ("Sharpe Ratio - MSFT", "number"),
+]
+
+
+# Extract column names
+task1_cols = [col_name for col_name, _ in TASK1_COLS]
+task2_cols = [col_name for col_name, _ in TASK2_COLS]
+task3_cols = [col_name for col_name, _ in TASK3_COLS]
+
+
+def create_df_dict(lang, lang_cols):
+ # Load leaderboard data with column names
+ leaderboard_df = pd.read_csv(f"{lang}_result.csv", names=lang_cols)
+ leaderboard_df = leaderboard_df.sort_index(axis=1)
+ # Move 'key' column to the front
+ leaderboard_df = leaderboard_df[["Model"] + [col for col in leaderboard_df.columns if col != "Model"]]
+ cols = leaderboard_df.columns
+ types = ["str"] + ["number"] * (len(lang_cols) - 1)
+
+ # Split merged_df into subtask dataframes
+ df_dict = {"overall": leaderboard_df}
+ return df_dict
+
+
+df_lang = {
+ "Task 1": create_df_dict("task1", task1_cols),
+ "Task 2": create_df_dict("task2", task2_cols),
+ "Task 3": create_df_dict("task3", task3_cols),
+}
+
+
+# Constants
+TITLE = '<h1 align="center" id="space-title">🐲 IJCAI 2024 FinLLM Challenge Leaderboard</h1>'
+INTRODUCTION_TEXT = """📊 Introduction
+
+The FinLLM Challenge rigorously evaluates state-of-the-art models in financial text analysis, generation, and decision-making tasks. These tasks include financial classification, financial text summarization, and single stock trading.
+
+📈 Unique Evaluation Metrics
+
+Our leaderboard incorporates a comprehensive evaluation using diverse metrics like Accuracy, F1 Score, ROUGE, BERTScore, and Sharpe Ratio to assess the models' capabilities in real-world financial applications.
+
+📚 Task Details
+
+**Task 1: Financial Classification**
+
+- **Objective:** Classify sentences as claims or premises.
+- **Dataset:** 7.75k training data, 969 test data.
+- **Evaluation Metrics:** F1 Score (final ranking metric) and Accuracy.
+
+**Task 2: Financial Text Summarization**
+
+- **Objective:** Summarize financial news articles into concise texts.
+- **Dataset:** 8k training data, 2k test data.
+- **Evaluation Metrics:** ROUGE (1, 2, L) and BERTScore (ROUGE-1 as the final ranking metric).
+
+**Task 3: Single Stock Trading**
+
+- **Objective:** Make stock trading decisions (buy, sell, hold) with reasonings.
+- **Dataset:** 291 data points.
+- **Evaluation Metrics:** Sharpe Ratio (final ranking metric), Cumulative Return, Daily and Annualized Volatility, Maximum Drawdown.
+
+For more details, refer to our [Challenge page](https://sites.google.com/nlg.csie.ntu.edu.tw/finnlp-agentscen/shared-task-finllm?authuser=0).
+"""
+
+
+def create_data_interface(df):
+ headers = df.columns
+ types = ["str"] + ["number"] * (len(headers) - 1)
+
+ return gr.components.Dataframe(
+ value=df.values.tolist(),
+ headers=[col_name for col_name in headers],
+ datatype=types,
+ max_rows=10,
  )
-except Exception:
- restart_space()
-
-
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
-
-(
- finished_eval_queue_df,
- running_eval_queue_df,
- pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-
-# Searching and filtering
-def update_table(
- hidden_df: pd.DataFrame,
- columns: list,
- type_query: list,
- precision_query: str,
- size_query: list,
- show_deleted: bool,
- query: str,
-):
- filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
- filtered_df = filter_queries(query, filtered_df)
- df = select_columns(filtered_df, columns)
- return df
-
-
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
- return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
- always_here_cols = [
- AutoEvalColumn.model_type_symbol.name,
- AutoEvalColumn.model.name,
- ]
- # We use COLS to maintain sorting
- filtered_df = df[
- always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
- ]
- return filtered_df
-
-
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
- final_df = []
- if query != "":
- queries = [q.strip() for q in query.split(";")]
- for _q in queries:
- _q = _q.strip()
- if _q != "":
- temp_filtered_df = search_table(filtered_df, _q)
- if len(temp_filtered_df) > 0:
- final_df.append(temp_filtered_df)
- if len(final_df) > 0:
- filtered_df = pd.concat(final_df)
- filtered_df = filtered_df.drop_duplicates(
- subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
- )
-
- return filtered_df
-
-
-def filter_models(
- df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
- # Show all models
- if show_deleted:
- filtered_df = df
- else: # Show only still on the hub models
- filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
- type_emoji = [t[0] for t in type_query]
- filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
- filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
- numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
- params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
- mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
- filtered_df = filtered_df.loc[mask]
-
- return filtered_df
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
- gr.HTML(TITLE)
- gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
- with gr.Tabs(elem_classes="tab-buttons") as tabs:
- with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
- with gr.Row():
- with gr.Column():
- with gr.Row():
- search_bar = gr.Textbox(
- placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
- show_label=False,
- elem_id="search-bar",
- )
- with gr.Row():
- shown_columns = gr.CheckboxGroup(
- choices=[
- c.name
- for c in fields(AutoEvalColumn)
- if not c.hidden and not c.never_hidden
- ],
- value=[
- c.name
- for c in fields(AutoEvalColumn)
- if c.displayed_by_default and not c.hidden and not c.never_hidden
- ],
- label="Select columns to show",
- elem_id="column-select",
- interactive=True,
- )
- with gr.Row():
- deleted_models_visibility = gr.Checkbox(
- value=False, label="Show gated/private/deleted models", interactive=True
- )
- with gr.Column(min_width=320):
- #with gr.Box(elem_id="box-filter"):
- filter_columns_type = gr.CheckboxGroup(
- label="Model types",
- choices=[t.to_str() for t in ModelType],
- value=[t.to_str() for t in ModelType],
- interactive=True,
- elem_id="filter-columns-type",
- )
- filter_columns_precision = gr.CheckboxGroup(
- label="Precision",
- choices=[i.value.name for i in Precision],
- value=[i.value.name for i in Precision],
- interactive=True,
- elem_id="filter-columns-precision",
- )
- filter_columns_size = gr.CheckboxGroup(
- label="Model sizes (in billions of parameters)",
- choices=list(NUMERIC_INTERVALS.keys()),
- value=list(NUMERIC_INTERVALS.keys()),
- interactive=True,
- elem_id="filter-columns-size",
- )
-
- leaderboard_table = gr.components.Dataframe(
- value=leaderboard_df[
- [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
- + shown_columns.value
- ],
- headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
- datatype=TYPES,
- elem_id="leaderboard-table",
- interactive=False,
- visible=True,
- )
-
- # Dummy leaderboard for handling the case when the user uses backspace key
- hidden_leaderboard_table_for_search = gr.components.Dataframe(
- value=original_df[COLS],
- headers=COLS,
- datatype=TYPES,
- visible=False,
- )
- search_bar.submit(
- update_table,
- [
- hidden_leaderboard_table_for_search,
- shown_columns,
- filter_columns_type,
- filter_columns_precision,
- filter_columns_size,
- deleted_models_visibility,
- search_bar,
- ],
- leaderboard_table,
- )
- for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
- selector.change(
- update_table,
- [
- hidden_leaderboard_table_for_search,
- shown_columns,
- filter_columns_type,
- filter_columns_precision,
- filter_columns_size,
- deleted_models_visibility,
- search_bar,
- ],
- leaderboard_table,
- queue=True,
- )
-
- with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
- gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
- with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
- with gr.Column():
- with gr.Row():
- gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
- with gr.Column():
- with gr.Accordion(
- f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- finished_eval_table = gr.components.Dataframe(
- value=finished_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
- with gr.Accordion(
- f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- running_eval_table = gr.components.Dataframe(
- value=running_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
-
- with gr.Accordion(
- f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- pending_eval_table = gr.components.Dataframe(
- value=pending_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
- with gr.Row():
- gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
- with gr.Row():
- with gr.Column():
- model_name_textbox = gr.Textbox(label="Model name")
- revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
- model_type = gr.Dropdown(
- choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
- label="Model type",
- multiselect=False,
- value=None,
- interactive=True,
- )
-
- with gr.Column():
- precision = gr.Dropdown(
- choices=[i.value.name for i in Precision if i != Precision.Unknown],
- label="Precision",
- multiselect=False,
- value="float16",
- interactive=True,
- )
- weight_type = gr.Dropdown(
- choices=[i.value.name for i in WeightType],
- label="Weights type",
- multiselect=False,
- value="Original",
- interactive=True,
- )
- base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
- submit_button = gr.Button("Submit Eval")
- submission_result = gr.Markdown()
- submit_button.click(
- add_new_eval,
- [
- model_name_textbox,
- base_model_name_textbox,
- revision_name_textbox,
- precision,
- weight_type,
- model_type,
- ],
- submission_result,
- )
-
- with gr.Row():
- with gr.Accordion("📙 Citation", open=False):
- citation_button = gr.Textbox(
- value=CITATION_BUTTON_TEXT,
- label=CITATION_BUTTON_LABEL,
- lines=20,
- elem_id="citation-button",
- show_copy_button=True,
- )
+
+
+def plot_radar_chart(df, attributes, category_name):
+ fig = go.Figure()
+
+ for index, row in df.iterrows():
+ model = row["Model"]
+ values = row[attributes].tolist()
+ fig.add_trace(go.Scatterpolar(r=values, theta=attributes, fill="toself", name=model))
+
+ fig.update_layout(title="FLARE", polar=dict(radialaxis=dict(visible=True, range=[0, 0.9])), showlegend=True)
+
+ return fig
+
+
+def create_data_interface_for_aggregated(df, category_name):
+ attributes = df.columns[1:]
+ print(attributes)
+ plt = plot_radar_chart(df, attributes, category_name)
+ return plt
+
+
+def create_lang_leaderboard(df_dict):
+ for key, df in df_dict.items():
+ with gr.Tab(key):
+ create_data_interface(df)
+
+
+def launch_gradio():
+ demo = gr.Blocks()
+
+ with demo:
+ gr.HTML(TITLE)
+ gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+ for key, df_dict in df_lang.items():
+ with gr.Tab(key):
+ create_lang_leaderboard(df_dict)
+
+ demo.launch()
+
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(launch_gradio, "interval", seconds=3600)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+
+# Launch immediately
+launch_gradio()