Upload 19 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,13 +1 @@
-auto_evals/
-venv/
-__pycache__/
-.env
-.ipynb_checkpoints
-*ipynb
-.vscode/
-
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
-logs/
+hf_token

README.md

@@ -1,44 +1,16 @@
----
-title: LIBRA Leaderboard
-emoji: 🥇
-colorFrom: green
-colorTo: indigo
-sdk: gradio
-app_file: app.py
-pinned: true
-license: mit
----
-
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
- "config": {
- "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
- "model_name": "path of the model on the hub: org/model",
- "model_sha": "revision on the hub",
- },
- "results": {
- "task_name": {
- "metric_name": score,
- },
- "task_name2": {
- "metric_name": score,
- }
- }
-}
-```
-
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+---
+title: LIBRA Leaderboard
+emoji: 🏆
+colorFrom: indigo
+colorTo: green
+sdk: gradio
+sdk_version: 4.36.1
+app_file: app.py
+pinned: false
+license: mit
+tags:
+- leaderboard
+short_description: LLM extra long context benchmark
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,204 +1,225 @@
-import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
- CITATION_BUTTON_LABEL,
- CITATION_BUTTON_TEXT,
- EVALUATION_QUEUE_TEXT,
- INTRODUCTION_TEXT,
- LLM_BENCHMARKS_TEXT,
- TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
- BENCHMARK_COLS,
- COLS,
- EVAL_COLS,
- EVAL_TYPES,
- AutoEvalColumn,
- ModelType,
- fields,
- WeightType,
- Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
- API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
- print(EVAL_REQUESTS_PATH)
- snapshot_download(
- repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
- )
-except Exception:
- restart_space()
-try:
- print(EVAL_RESULTS_PATH)
- snapshot_download(
- repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
- )
-except Exception:
- restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
- finished_eval_queue_df,
- running_eval_queue_df,
- pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
- if dataframe is None or dataframe.empty:
- raise ValueError("Leaderboard DataFrame is empty or None.")
- return Leaderboard(
- value=dataframe,
- datatype=[c.type for c in fields(AutoEvalColumn)],
- select_columns=SelectColumns(
- default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
- cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
- label="Select Columns to Display:",
- ),
- search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
- hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
- filter_columns=[
- ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
- ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
- ColumnFilter(
- AutoEvalColumn.params.name,
- type="slider",
- min=0.01,
- max=150,
- label="Select the number of parameters (B)",
- ),
- ColumnFilter(
- AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
- ),
- ],
- bool_checkboxgroup_label="Hide models",
- interactive=False,
- )
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
- gr.HTML(TITLE)
- gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
- with gr.Tabs(elem_classes="tab-buttons") as tabs:
- with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
- leaderboard = init_leaderboard(LEADERBOARD_DF)
-
- with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
- gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
- with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
- with gr.Column():
- with gr.Row():
- gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
- with gr.Column():
- with gr.Accordion(
- f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- finished_eval_table = gr.components.Dataframe(
- value=finished_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
- with gr.Accordion(
- f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- running_eval_table = gr.components.Dataframe(
- value=running_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
-
- with gr.Accordion(
- f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- pending_eval_table = gr.components.Dataframe(
- value=pending_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
- with gr.Row():
- gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
- with gr.Row():
- with gr.Column():
- model_name_textbox = gr.Textbox(label="Model name")
- revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
- model_type = gr.Dropdown(
- choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
- label="Model type",
- multiselect=False,
- value=None,
- interactive=True,
- )
-
- with gr.Column():
- precision = gr.Dropdown(
- choices=[i.value.name for i in Precision if i != Precision.Unknown],
- label="Precision",
- multiselect=False,
- value="float16",
- interactive=True,
- )
- weight_type = gr.Dropdown(
- choices=[i.value.name for i in WeightType],
- label="Weights type",
- multiselect=False,
- value="Original",
- interactive=True,
- )
- base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
- submit_button = gr.Button("Submit Eval")
- submission_result = gr.Markdown()
- submit_button.click(
- add_new_eval,
- [
- model_name_textbox,
- base_model_name_textbox,
- revision_name_textbox,
- precision,
- weight_type,
- model_type,
- ],
- submission_result,
- )
-
- with gr.Row():
- with gr.Accordion("📙 Citation", open=False):
- citation_button = gr.Textbox(
- value=CITATION_BUTTON_TEXT,
- label=CITATION_BUTTON_LABEL,
- lines=20,
- elem_id="citation-button",
- show_copy_button=True,
- )
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+import os
+import json
+import gradio as gr
+import pandas as pd
+import numpy as np
+
+from collections import defaultdict
+
+
+LENGTHS = ["dataset_total_score", "4k", "8k", "16k", "32k", "64k", "128k"]
+datasets_params = json.load(open("datasets_config.json", "r"))
+TASKS = datasets_params.keys()
+
+
+def make_default_md():
+ leaderboard_md = "LeaderBoard"
+ return leaderboard_md
+
+
+def make_model_desc_md():
+ with open("docs/description.md", "r") as f:
+ description = f.read()
+ return description
+
+
+def make_overall_table_by_tasks(files):
+ results = defaultdict(list)
+
+ result_dct = {}
+ for file in files:
+ if not file.endswith("json"): continue
+ path = "results/" + file
+ data = json.load(open(path))
+ model_name = file.split('/')[-1].split(".json")[0]
+ result_dct[model_name] = {}
+ for dataset in data.keys():
+ if dataset == "total_score":
+ result_dct[model_name][dataset] = round(data[dataset] * 100, 1)
+ continue
+ result_dct[model_name][dataset] = round(data[dataset]["dataset_total_score"] * 100, 1)
+
+ for file in files:
+ if not file.endswith("json"): continue
+ model_name = file.split('/')[-1].split(".json")[0]
+ results['Model'].append(model_name)
+ for key in result_dct[model_name].keys():
+ if key == "total_score":
+ results["Total Score"].append(result_dct[model_name][key])
+ else:
+ results[datasets_params[key]["name"]].append(result_dct[model_name][key])
+
+ table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False)
+ cols = table.columns.tolist()
+ cols = [cols[0]] + [cols[22]] + cols[1:22]
+ return table[cols]
+
+
+def make_overall_table_by_lengths(files):
+ results = defaultdict(list)
+
+ result_dct = {}
+ for file in files:
+ if not file.endswith("json"): continue
+ path = "results/" + file
+ data = json.load(open(path))
+ model_name = file.split('/')[-1].split(".json")[0]
+ result_dct[model_name] = {}
+ for dataset in data.keys():
+ if dataset == "total_score":
+ result_dct[model_name][dataset] = data[dataset]
+ continue
+ for length in data[dataset].keys():
+ if length == "dataset_total_score": continue
+ if length not in result_dct[model_name]:
+ result_dct[model_name][length] = []
+ result_dct[model_name][length].append(data[dataset][length])
+
+ for model_name in result_dct.keys():
+ for length in result_dct[model_name].keys():
+ result_dct[model_name][length] = round(np.mean(result_dct[model_name][length]) * 100, 1)
+
+ for file in files:
+ if not file.endswith("json"): continue
+ model_name = file.split('/')[-1].split(".json")[0]
+ results['Model'].append(model_name)
+ for key in result_dct[model_name].keys():
+ if key == "total_score":
+ results["Total Score"].append(result_dct[model_name][key])
+ else:
+ results[key].append(result_dct[model_name][key])
+
+ table = pd.DataFrame(results).sort_values(['Total Score'], ascending=False)
+ cols = table.columns.tolist()
+ cols = [cols[0]] + [cols[7]] + cols[1:7]
+ return table[cols]
+
+
+def load_model(files, tab_name):
+ results = defaultdict(list)
+
+ for file in files:
+ if not file.endswith("json"): continue
+ model_name = file.split('/')[-1].split(".json")[0]
+ results['Model'].append(model_name)
+ result = json.load(open("results/" + file, "r"))
+ for length in LENGTHS:
+ if length in result[tab_name].keys():
+ if length == "dataset_total_score":
+ results["Dataset Total Score"].append(round(result[tab_name][length] * 100, 1))
+ continue
+ results[length].append(round(result[tab_name][length] * 100, 1))
+ else:
+ results[length].append("-")
+
+ return pd.DataFrame(results).sort_values(['Dataset Total Score'], ascending=False)
+
+
+def build_leaderboard_tab(files):
+ default_md = make_default_md()
+ md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
+
+ with gr.Tabs() as tabs:
+
+ with gr.Tab("Results by Lengths", id=0):
+ df = make_overall_table_by_lengths(files)
+ gr.Dataframe(
+ headers=[
+ "Model",
+ ] + LENGTHS,
+ datatype=[
+ "markdown",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ ],
+ value=df,
+ elem_id="arena_leaderboard_dataframe",
+ height=700,
+ wrap=True,
+ )
+
+ with gr.Tab("Results by Tasks", id=1):
+ df = make_overall_table_by_tasks(files)
+ gr.Dataframe(
+ headers=[
+ "Model",
+ ] + LENGTHS,
+ datatype=[
+ "markdown",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str"
+ ],
+ value=df,
+ elem_id="arena_leaderboard_dataframe",
+ height=700,
+ wrap=False,
+ )
+
+ for tab_id, tab_name in enumerate(TASKS):
+ df = load_model(files, tab_name)
+ with gr.Tab(datasets_params[tab_name]["name"], id=tab_id+2):
+ gr.Dataframe(
+ headers=[
+ "Model",
+ ] + LENGTHS,
+ datatype=[
+ "markdown",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ "str",
+ ],
+ value=df,
+ elem_id="arena_leaderboard_dataframe",
+ height=700,
+ wrap=True,
+ )
+
+ with gr.Tab("Description", id=tab_id + 3):
+ desc_md = make_model_desc_md()
+ gr.Markdown(desc_md, elem_id="leaderboard_markdown")
+
+ return [md_1]
+
+
+def build_demo(files):
+ text_size = gr.themes.sizes.text_lg
+
+ with gr.Blocks(title="LIBRA leaderboard",
+ theme=gr.themes.Base(text_size=text_size)) as demo:
+ build_leaderboard_tab(files)
+ return demo
+
+
+if __name__ == "__main__":
+ files = os.listdir("results")
+ demo = build_demo(files)
+ demo.launch(share=False)

datasets_config.json

@@ -0,0 +1 @@
+{"passkey": {"name": "Passkey", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "matreshka_yes_no": {"name": "MatreshkaYesNo", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "matreshka_names": {"name": "MatreshkaNames", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "passkey_with_librusec": {"name": "PasskeyWithLibrusec", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "librusec_history": {"name": "LibrusecHistory", "lengths": ["8k", "16k", "32k", "64k"]}, "ru_gsm100": {"name": "ruGSM100", "lengths": ["16k"]}, "ru_sci_passage_count": {"name": "ruSciPassageCount", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_2wikimultihopqa": {"name": "ru2WikiMultihopQA", "lengths": ["8k", "16k", "32k"]}, "long_context_multiq": {"name": "LongContextMultiQ", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_sci_abstract_retrieval": {"name": "ruSciAbstractRetrieval", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_trec": {"name": "ruTREC", "lengths": ["4k", "8k", "16k", "32k"]}, "ru_sci_fi": {"name": "ruSciFi", "lengths": ["32k", "64k"]}, "librusec_mhqa": {"name": "LibrusecMHQA", "lengths": ["8k"]}, "ru_babilong_qa1": {"name": "ruBABILongQA1", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_babilong_qa2": {"name": "ruBABILongQA2", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_babilong_qa3": {"name": "ruBABILongQA3", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_babilong_qa4": {"name": "ruBABILongQA4", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_babilong_qa5": {"name": "ruBABILongQA5", "lengths": ["4k", "8k", "16k", "32k", "64k", "128k"]}, "ru_quality": {"name": "ruQuALITY", "lengths": ["8k", "16k"]}, "ru_tpo": {"name": "ruTPO", "lengths": ["8k"]}, "ru_qasper": {"name": "ruQasper", "lengths": ["8k", "16k", "32k"]}}

docs/description.md

@@ -0,0 +1,63 @@
+# LIBRA: Long Input Benchmark for Russian Analysis
+
+<img src="https://i.imgur.com/BNleRrG.png" width="800" />
+
+## Dataset Summary
+
+LIBRA (Long Input Benchmark for Russian Analysis) is designed to evaluate the capabilities of large language models (LLMs) in understanding and processing long texts in Russian. This benchmark includes 21 datasets adapted for different tasks and complexities. The tasks are divided into four complexity groups and allow evaluation across various context lengths ranging from 4,000 up to 128,000 tokens.
+
+## Tasks and Complexity Groups
+
+### Group I: Simple Information Retrieval
+- **Passkey**: Extract a relevant piece of code number from a long text fragment.
+- **PasskeyWithLibrusec**: Similar to Passkey but with added noise from Librusec texts.
+
+### Group II: Question Answering and Multiple Choice
+- **MatreshkaNames**: Identify the person in dialogues based on the discussed topic.
+- **MatreshkaYesNo**: Indicate whether a specific topic was mentioned in the dialog.
+- **LibrusecHistory**: Answer questions based on historical texts.
+- **ruTREC**: Few-shot in-context learning for topic classification. Created by translating the TREC dataset from LongBench.
+- **ruSciFi**: Answer true/false based on context and general world knowledge. Translation of SciFi dataset from L-Eval.
+- **ruSciAbstractRetrieval**: Retrieve relevant paragraphs from scientific abstracts.
+- **ruTPO**: Multiple-choice questions similar to TOEFL exams. Translation of the TPO dataset from L-Eval.
+- **ruQuALITY**: Multiple-choice QA tasks based on detailed texts. Created by translating the QuALITY dataset from L-Eval.
+
+### Group III: Multi-hop Question Answering
+- **ruBABILongQA**: 5 long-context reasoning tasks for QA using facts hidden among irrelevant information.
+- **LongContextMultiQ**: Multi-hop QA based on Wikidata and Wikipedia.
+- **LibrusecMHQA**: Multi-hop QA requiring information distributed across several text parts.
+- **ru2WikiMultihopQA**: Translation of the 2WikiMultihopQA dataset from LongBench.
+
+### Group IV: Complex Reasoning and Mathematical Problems
+- **ruSciPassageCount**: Count unique paragraphs in a long text.
+- **ruQasper**: Question Answering over academic research papers. Created by translating the Qasper dataset from LongBench.
+- **ruGSM100**: Solve math problems using Chain-of-Thought reasoning.
+
+## Dataset Structure
+
+The datasets are divided into subsets based on context lengths: 4k, 8k, 16k, 32k, 64k, and 128k tokens. Each subset contains a different number of samples depending on the task complexity.
+
+## Usage
+
+The LIBRA benchmark is available under the MIT license. Researchers and developers can use these datasets to evaluate the long-context understanding abilities of various LLMs. The datasets, codebase, and public leaderboard are open-source to guide forthcoming research in this area.
+
+## Citation
+
+_TODO_
+
+@article{LIBRA2024,
+title={Long Input Benchmark for Russian Analysis},
+author={Anonymous},
+journal={ACL},
+year={2024}
+}
+
+## License
+
+The datasets are published under the MIT license.
+
+## Acknowledgments
+
+_TODO_
+
+For more details and code, please visit our [GitHub repository](#).

requirements.txt

@@ -1,16 +1,4 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.9
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
+plotly
+gradio
+numpy 
+pandas

results/ChatGLM2-6B-32K.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 0.82, "64k": 0, "128k": 0, "dataset_total_score": 0.6366666666666666}, "matreshka_yes_no": {"4k": 0.5016722408026756, "8k": 0.5, "16k": 0.5, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.33361204013377926}, "matreshka_names": {"4k": 0.04666666666666667, "8k": 0.02666666666666667, "16k": 0.006666666666666667, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.013333333333333334}, "passkey_with_librusec": {"4k": 0.99, "8k": 0.995, "16k": 0.985, "32k": 0.93, "64k": 0, "128k": 0, "dataset_total_score": 0.65}, "librusec_history": {"8k": 0.21875, "16k": 0.09375, "32k": 0.03125, "64k": 0, "dataset_total_score": 0.0859375}, "ru_gsm100": {"16k": 0.05, "dataset_total_score": 0.05}, "ru_sci_passage_count": {"4k": 0.09, "8k": 0.06, "16k": 0.07, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.03666666666666667}, "ru_2wikimultihopqa": {"8k": 0.1836734693877551, "16k": 0.21875, "32k": 0.12195121951219512, "dataset_total_score": 0.17479156296665008}, "long_context_multiq": {"8k": 0.05, "16k": 0.015, "4k": 0.005, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.011666666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.4166666666666667, "8k": 0.21887180280037422, "16k": 0.1810637996020305, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.13610037817817858}, "ru_trec": {"4k": 0.05405405405405406, "8k": 0.04, "16k": 0.04395604395604396, "32k": 0.040983606557377046, "dataset_total_score": 0.04474842614186877}, "ru_sci_fi": {"32k": 0.0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.06770833333333333, "dataset_total_score": 0.06770833333333333}, "ru_babilong_qa1": {"4k": 0.27, "8k": 0.23, "16k": 0.23, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.12166666666666666}, "ru_babilong_qa2": {"4k": 0.05, "8k": 0.04, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.015}, "ru_babilong_qa3": {"4k": 0.07, "8k": 0.03, "16k": 0.04, "32k": 0.01, "64k": 0, "128k": 0, "dataset_total_score": 0.025000000000000005}, "ru_babilong_qa4": {"4k": 0.02, "8k": 0.018000000000000002, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.006333333333333334}, "ru_babilong_qa5": {"4k": 0.2, "8k": 0.18, "16k": 0.15, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.08833333333333333}, "ru_quality": {"16k": 0.43892339544513453, "8k": 0.5447154471544716, "dataset_total_score": 0.49181942129980305}, "ru_tpo": {"8k": 0.28950863213811423, "dataset_total_score": 0.28950863213811423}, "ru_qasper": {"16k": 0.03254246612062729, "8k": 0.03544011573751087, "32k": 0.009411737687136327, "dataset_total_score": 0.025798106515091495}, "total_score": 0.15736624130349933}

results/GLM4-9B-Chat.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "matreshka_yes_no": {"4k": 0.7926421404682275, "8k": 0.75, "16k": 0.7133333333333334, "32k": 0.67, "64k": 0.5966666666666667, "128k": 0.56, "dataset_total_score": 0.6804403567447045}, "matreshka_names": {"4k": 0.6466666666666666, "8k": 0.5066666666666667, "16k": 0.52, "32k": 0.47333333333333333, "64k": 0.37333333333333335, "128k": 0.32, "dataset_total_score": 0.47333333333333333}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "librusec_history": {"8k": 0.84375, "16k": 0.84375, "32k": 0.84375, "64k": 0.75, "dataset_total_score": 0.8203125}, "ru_gsm100": {"16k": 0.08, "dataset_total_score": 0.08}, "ru_sci_passage_count": {"4k": 0.27, "8k": 0.08, "16k": 0.09, "32k": 0.0, "64k": 0.01, "128k": 0.0, "dataset_total_score": 0.07500000000000001}, "ru_2wikimultihopqa": {"8k": 0.5510204081632653, "16k": 0.5546875, "32k": 0.35772357723577236, "dataset_total_score": 0.4878104951330125}, "long_context_multiq": {"8k": 0.265, "16k": 0.035, "4k": 0.055, "64k": 0.005, "32k": 0.005, "128k": 0.1, "dataset_total_score": 0.0775}, "ru_sci_abstract_retrieval": {"4k": 0.9819047619047618, "8k": 0.923411865911866, "16k": 0.9122101461259002, "32k": 0.8189266620312142, "64k": 0.6411071734029656, "128k": 0.3908439729202464, "dataset_total_score": 0.7780674303828258}, "ru_trec": {"4k": 0.5675675675675675, "8k": 0.7, "16k": 0.7582417582417582, "32k": 0.7704918032786885, "dataset_total_score": 0.6990752822720037}, "ru_sci_fi": {"32k": 0.3888888888888889, "64k": 0.42857142857142855, "dataset_total_score": 0.4087301587301587}, "librusec_mhqa": {"8k": 0.4453125, "dataset_total_score": 0.4453125}, "ru_babilong_qa1": {"4k": 0.699375, "8k": 0.59, "16k": 0.6, "32k": 0.508125, "64k": 0.429375, "128k": 0.42, "dataset_total_score": 0.5411458333333333}, "ru_babilong_qa2": {"4k": 0.389375, "8k": 0.33, "16k": 0.299375, "32k": 0.2693333333333333, "64k": 0.2675, "128k": 0.23491666666666666, "dataset_total_score": 0.29841666666666666}, "ru_babilong_qa3": {"4k": 0.24598809523809526, "8k": 0.2792380952380953, "16k": 0.21408333333333335, "32k": 0.2264761904761905, "64k": 0.18666666666666668, "128k": 0.18545833333333334, "dataset_total_score": 0.22298511904761908}, "ru_babilong_qa4": {"4k": 0.6207142857142857, "8k": 0.5964285714285714, "16k": 0.5657142857142857, "32k": 0.58, "64k": 0.43, "128k": 0.37714285714285717, "dataset_total_score": 0.5283333333333334}, "ru_babilong_qa5": {"4k": 0.7300000000000001, "8k": 0.7350000000000001, "16k": 0.7200000000000002, "32k": 0.6683333333333334, "64k": 0.6966666666666668, "128k": 0.6700000000000002, "dataset_total_score": 0.7033333333333335}, "ru_quality": {"16k": 0.6521739130434783, "8k": 0.8292682926829268, "dataset_total_score": 0.7407211028632026}, "ru_tpo": {"8k": 0.8685258964143426, "dataset_total_score": 0.8685258964143426}, "ru_qasper": {"16k": 0.05927748156784547, "8k": 0.06532155413695329, "32k": 0.025813608477215297, "dataset_total_score": 0.050137548060671354}, "total_score": 0.5228181376023114}

results/GPT-4o.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "matreshka_yes_no": {"4k": 0.8, "8k": 0.6, "16k": 1.0, "32k": 0.8, "64k": 0.7, "128k": 0.9, "dataset_total_score": 0.8000000000000002}, "matreshka_names": {"4k": 0.6, "8k": 0.6, "16k": 0.5, "32k": 0.4, "64k": 0.5, "128k": 0.5, "dataset_total_score": 0.5166666666666667}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 1.0, "128k": 1.0, "dataset_total_score": 1.0}, "librusec_history": {"8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0.9, "dataset_total_score": 0.975}, "ru_gsm100": {"16k": 1.0, "dataset_total_score": 1.0}, "ru_sci_passage_count": {"4k": 1.0, "8k": 0.5, "16k": 0.3, "32k": 0.0, "64k": 0.2, "128k": 0.1, "dataset_total_score": 0.35000000000000003}, "ru_2wikimultihopqa": {"8k": 0.8, "16k": 0.8, "32k": 0.7, "dataset_total_score": 0.7666666666666666}, "long_context_multiq": {"4k": 0.3, "8k": 1.0, "16k": 0.7, "32k": 0.0, "64k": 0.1, "128k": 0.1, "dataset_total_score": 0.3666666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.99, "8k": 0.9541666666666668, "16k": 0.9254479578392623, "32k": 0.9562564463343153, "64k": 0.590978869808793, "128k": 0.19764315322255238, "dataset_total_score": 0.7690821823119315}, "ru_trec": {"4k": 0.6, "8k": 0.8, "16k": 0.9, "32k": 0.7, "dataset_total_score": 0.75}, "ru_sci_fi": {"32k": 0.6, "64k": 0.9, "dataset_total_score": 0.75}, "librusec_mhqa": {"8k": 0.5, "dataset_total_score": 0.5}, "ru_babilong_qa1": {"4k": 0.9, "8k": 0.8, "16k": 0.7, "32k": 0.9, "64k": 0.8, "128k": 0.6, "dataset_total_score": 0.7833333333333333}, "ru_babilong_qa2": {"4k": 0.4, "8k": 0.3, "16k": 0.4, "32k": 0.4, "64k": 0.5, "128k": 0.2, "dataset_total_score": 0.3666666666666667}, "ru_babilong_qa3": {"4k": 0.2, "8k": 0.3, "16k": 0.1, "32k": 0.2, "64k": 0.2, "128k": 0.2866666666666667, "dataset_total_score": 0.21444444444444444}, "ru_babilong_qa4": {"4k": 0.8800000000000001, "8k": 0.8, "16k": 0.8, "32k": 0.5714285714285715, "64k": 0.8857142857142858, "128k": 0.8, "dataset_total_score": 0.7895238095238096}, "ru_babilong_qa5": {"4k": 0.8666666666666666, "8k": 0.8666666666666666, "16k": 0.9333333333333333, "32k": 0.9666666666666666, "64k": 0.8666666666666668, "128k": 0.9, "dataset_total_score": 0.9}, "ru_quality": {"8k": 0.8, "16k": 0.8666666666666668, "dataset_total_score": 0.8333333333333335}, "ru_tpo": {"8k": 1.0, "dataset_total_score": 1.0}, "ru_qasper": {"8k": 0.2865100250626566, "16k": 0.3184757236227824, "32k": 0.3465384615384616, "dataset_total_score": 0.3171747367413002}, "total_score": 0.70231230982642}

results/LLaMA-2-7B-32k.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0, "128k": 0, "dataset_total_score": 0.6666666666666666}, "matreshka_yes_no": {"4k": 0.5016722408026756, "8k": 0.5, "16k": 0.5, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.33361204013377926}, "matreshka_names": {"4k": 0.08, "8k": 0.06666666666666667, "16k": 0.02, "32k": 0.04, "64k": 0, "128k": 0, "dataset_total_score": 0.034444444444444444}, "passkey_with_librusec": {"4k": 1.0, "8k": 0.975, "16k": 0.985, "32k": 0.97, "64k": 0, "128k": 0, "dataset_total_score": 0.6549999999999999}, "librusec_history": {"8k": 0.6875, "16k": 0.5, "32k": 0.4375, "64k": 0, "dataset_total_score": 0.40625}, "ru_gsm100": {"16k": 0.07, "dataset_total_score": 0.07}, "ru_sci_passage_count": {"4k": 0.18, "8k": 0.05, "16k": 0.05, "32k": 0.005, "64k": 0, "128k": 0, "dataset_total_score": 0.047499999999999994}, "ru_2wikimultihopqa": {"8k": 0.4489795918367347, "16k": 0.3984375, "32k": 0.2682926829268293, "dataset_total_score": 0.37190325825452136}, "long_context_multiq": {"8k": 0.33, "16k": 0.1, "4k": 0.045, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.07916666666666666}, "ru_sci_abstract_retrieval": {"4k": 0.8519047619047619, "8k": 0.7612582259010829, "16k": 0.4675868475624726, "32k": 0.2665782908189807, "64k": 0, "128k": 0, "dataset_total_score": 0.39122135436454974}, "ru_trec": {"4k": 0.24324324324324326, "8k": 0.18, "16k": 0.24175824175824176, "32k": 0.28688524590163933, "dataset_total_score": 0.2379716827257811}, "ru_sci_fi": {"32k": 0.1111111111111111, "64k": 0, "dataset_total_score": 0.05555555555555555}, "librusec_mhqa": {"8k": 0.2760416666666667, "dataset_total_score": 0.2760416666666667}, "ru_babilong_qa1": {"4k": 0.6, "8k": 0.66, "16k": 0.66, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.4033333333333333}, "ru_babilong_qa2": {"4k": 0.25, "8k": 0.3, "16k": 0.25875, "32k": 0.19, "64k": 0, "128k": 0, "dataset_total_score": 0.16645833333333335}, "ru_babilong_qa3": {"4k": 0.22933333333333333, "8k": 0.28933333333333333, "16k": 0.26, "32k": 0.2, "64k": 0, "128k": 0, "dataset_total_score": 0.1631111111111111}, "ru_babilong_qa4": {"4k": 0.31, "8k": 0.34, "16k": 0.23, "32k": 0.12, "64k": 0, "128k": 0, "dataset_total_score": 0.16666666666666666}, "ru_babilong_qa5": {"4k": 0.59, "8k": 0.66, "16k": 0.64, "32k": 0.69, "64k": 0, "128k": 0, "dataset_total_score": 0.43}, "ru_quality": {"16k": 0.13871635610766048, "8k": 0.17073170731707318, "dataset_total_score": 0.15472403171236682}, "ru_tpo": {"8k": 0.5431606905710492, "dataset_total_score": 0.5431606905710492}, "ru_qasper": {"16k": 0.05999038960490889, "8k": 0.0580459343880765, "32k": 0.022401950361811175, "dataset_total_score": 0.04681275811826552}, "total_score": 0.2714095362059408}

results/LLaMA-3-8B-Instruct.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "matreshka_yes_no": {"4k": 0.8394648829431438, "8k": 0.8, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.27324414715719064}, "matreshka_names": {"4k": 0.5333333333333333, "8k": 0.46, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.16555555555555557}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "librusec_history": {"8k": 0.90625, "16k": 0, "32k": 0, "64k": 0, "dataset_total_score": 0.2265625}, "ru_gsm100": {"16k": 0, "dataset_total_score": 0.0}, "ru_sci_passage_count": {"4k": 0.31, "8k": 0.08, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.065}, "ru_2wikimultihopqa": {"8k": 0.5306122448979592, "16k": 0, "32k": 0, "dataset_total_score": 0.17687074829931973}, "long_context_multiq": {"8k": 0.245, "4k": 0.05, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.049166666666666664}, "ru_sci_abstract_retrieval": {"4k": 0.9663095238095238, "8k": 0.9151886869744013, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.31358303513065416}, "ru_trec": {"4k": 0.5945945945945946, "8k": 0.5, "16k": 0, "32k": 0, "dataset_total_score": 0.2736486486486487}, "ru_sci_fi": {"32k": 0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.4609375, "dataset_total_score": 0.4609375}, "ru_babilong_qa1": {"4k": 0.6862083333333334, "8k": 0.7335416666666665, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.236625}, "ru_babilong_qa2": {"4k": 0.14, "8k": 0.10866666666666668, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.04144444444444445}, "ru_babilong_qa3": {"4k": 0.09, "8k": 0.1798095238095238, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.04496825396825397}, "ru_babilong_qa4": {"4k": 0.5725714285714285, "8k": 0.6034285714285714, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.19599999999999998}, "ru_babilong_qa5": {"4k": 0.7666666666666667, "8k": 0.7516666666666667, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2530555555555556}, "ru_quality": {"8k": 0.6910569105691058, "16k": 0, "dataset_total_score": 0.3455284552845529}, "ru_tpo": {"8k": 0.7808764940239044, "dataset_total_score": 0.7808764940239044}, "ru_qasper": {"8k": 0.06525418503550595, "16k": 0, "32k": 0, "dataset_total_score": 0.021751395011835317}, "total_score": 0.21864214601967852}

results/LLaMA-3-8B.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "matreshka_yes_no": {"4k": 0.6220735785953178, "8k": 0.59, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2020122630992196}, "matreshka_names": {"4k": 0.4, "8k": 0.2, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.10000000000000002}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "librusec_history": {"8k": 0.90625, "16k": 0, "32k": 0, "64k": 0, "dataset_total_score": 0.2265625}, "ru_gsm100": {"16k": 0, "dataset_total_score": 0.0}, "ru_sci_passage_count": {"4k": 0.15, "8k": 0.05, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.03333333333333333}, "ru_2wikimultihopqa": {"8k": 0.5510204081632653, "16k": 0, "32k": 0, "dataset_total_score": 0.18367346938775508}, "long_context_multiq": {"8k": 0.325, "4k": 0.095, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.07}, "ru_sci_abstract_retrieval": {"4k": 0.9711111111111111, "8k": 0.8806912531912532, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3086337273837274}, "ru_trec": {"4k": 0.3783783783783784, "8k": 0.38, "16k": 0, "32k": 0, "dataset_total_score": 0.1895945945945946}, "ru_sci_fi": {"32k": 0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.4140625, "dataset_total_score": 0.4140625}, "ru_babilong_qa1": {"4k": 0.68, "8k": 0.57, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.20833333333333334}, "ru_babilong_qa2": {"4k": 0.27, "8k": 0.19, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.07666666666666667}, "ru_babilong_qa3": {"4k": 0.28470833333333334, "8k": 0.25866666666666666, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.09056249999999999}, "ru_babilong_qa4": {"4k": 0.5838571428571429, "8k": 0.5642857142857143, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.19135714285714286}, "ru_babilong_qa5": {"4k": 0.6716666666666667, "8k": 0.6866666666666668, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2263888888888889}, "ru_quality": {"8k": 0.17073170731707318, "16k": 0, "dataset_total_score": 0.08536585365853659}, "ru_tpo": {"8k": 0.5816733067729084, "dataset_total_score": 0.5816733067729084}, "ru_qasper": {"8k": 0.06526998039125843, "16k": 0, "32k": 0, "dataset_total_score": 0.021756660130419478}, "total_score": 0.18460206698919968}

results/LongAlpaca.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 0.775, "8k": 0.825, "16k": 0.575, "32k": 0.37, "64k": 0, "128k": 0, "dataset_total_score": 0.42416666666666664}, "matreshka_yes_no": {"4k": 0.4782608695652174, "8k": 0.3933333333333333, "16k": 0.48, "32k": 0.4766666666666667, "64k": 0, "128k": 0, "dataset_total_score": 0.30471014492753623}, "matreshka_names": {"4k": 0.013333333333333334, "8k": 0.006666666666666667, "16k": 0.006666666666666667, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0044444444444444444}, "passkey_with_librusec": {"4k": 0.71, "8k": 0.7, "16k": 0.56, "32k": 0.465, "64k": 0, "128k": 0, "dataset_total_score": 0.4058333333333333}, "librusec_history": {"8k": 0.1875, "16k": 0.15625, "32k": 0.1875, "64k": 0, "dataset_total_score": 0.1328125}, "ru_gsm100": {"16k": 0.02, "dataset_total_score": 0.02}, "ru_sci_passage_count": {"4k": 0.13, "8k": 0.05, "16k": 0.02, "32k": 0.03, "64k": 0, "128k": 0, "dataset_total_score": 0.03833333333333333}, "ru_2wikimultihopqa": {"8k": 0.40816326530612246, "16k": 0.2890625, "32k": 0.21138211382113822, "dataset_total_score": 0.3028692930424202}, "long_context_multiq": {"8k": 0.015, "16k": 0.0, "4k": 0.03, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0075}, "ru_sci_abstract_retrieval": {"4k": 0.6496428571428572, "8k": 0.44704965669251384, "16k": 0.2043098561917367, "32k": 0.11199263409419845, "64k": 0, "128k": 0, "dataset_total_score": 0.23549916735355103}, "ru_trec": {"4k": 0.0, "8k": 0.02, "16k": 0.0, "32k": 0.0, "dataset_total_score": 0.005}, "ru_sci_fi": {"32k": 0.027777777777777776, "64k": 0, "dataset_total_score": 0.013888888888888888}, "librusec_mhqa": {"8k": 0.078125, "dataset_total_score": 0.078125}, "ru_babilong_qa1": {"4k": 0.09, "8k": 0.06, "16k": 0.06, "32k": 0.02, "64k": 0, "128k": 0, "dataset_total_score": 0.03833333333333333}, "ru_babilong_qa2": {"4k": 0.01, "8k": 0.01, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0033333333333333335}, "ru_babilong_qa3": {"4k": 0.05, "8k": 0.09, "16k": 0.04, "32k": 0.029333333333333336, "64k": 0, "128k": 0, "dataset_total_score": 0.03488888888888889}, "ru_babilong_qa4": {"4k": 0.0, "8k": 0.01, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0016666666666666668}, "ru_babilong_qa5": {"4k": 0.4416666666666667, "8k": 0.44, "16k": 0.475, "32k": 0.40666666666666673, "64k": 0, "128k": 0, "dataset_total_score": 0.2938888888888889}, "ru_quality": {"16k": 0.4824016563146999, "8k": 0.39837398373983746, "dataset_total_score": 0.4403878200272687}, "ru_tpo": {"8k": 0.06772908366533864, "dataset_total_score": 0.06772908366533864}, "ru_qasper": {"16k": 0.02179956634280568, "8k": 0.02260857442083522, "32k": 0.015666306859159988, "dataset_total_score": 0.020024815874266962}, "total_score": 0.13683026679372187}

results/LongChat-7B-v1.5-32k.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 0.995, "8k": 1.0, "16k": 1.0, "32k": 0.995, "64k": 0, "128k": 0, "dataset_total_score": 0.665}, "matreshka_yes_no": {"4k": 0.5016722408026756, "8k": 0.5, "16k": 0.5, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.33361204013377926}, "matreshka_names": {"4k": 0.17333333333333334, "8k": 0.06666666666666667, "16k": 0.08, "32k": 0.03333333333333333, "64k": 0, "128k": 0, "dataset_total_score": 0.058888888888888886}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0.985, "32k": 0.975, "64k": 0, "128k": 0, "dataset_total_score": 0.66}, "librusec_history": {"8k": 0.5625, "16k": 0.34375, "32k": 0.15625, "64k": 0, "dataset_total_score": 0.265625}, "ru_gsm100": {"16k": 0.05, "dataset_total_score": 0.05}, "ru_sci_passage_count": {"4k": 0.18, "8k": 0.08, "16k": 0.01, "32k": 0.02, "64k": 0, "128k": 0, "dataset_total_score": 0.04833333333333334}, "ru_2wikimultihopqa": {"8k": 0.42857142857142855, "16k": 0.3984375, "32k": 0.22764227642276422, "dataset_total_score": 0.3515504016647309}, "long_context_multiq": {"8k": 0.14, "16k": 0.025, "4k": 0.025, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.03166666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.8742063492063491, "8k": 0.7618964368964368, "16k": 0.6055101098106439, "32k": 0.22205033016504383, "64k": 0, "128k": 0, "dataset_total_score": 0.4106105376797456}, "ru_trec": {"4k": 0.05405405405405406, "8k": 0.1, "16k": 0.07692307692307693, "32k": 0.06557377049180328, "dataset_total_score": 0.07413772536723356}, "ru_sci_fi": {"32k": 0.05555555555555555, "64k": 0, "dataset_total_score": 0.027777777777777776}, "librusec_mhqa": {"8k": 0.24739583333333334, "dataset_total_score": 0.24739583333333334}, "ru_babilong_qa1": {"4k": 0.26, "8k": 0.29, "16k": 0.31, "32k": 0.19, "64k": 0, "128k": 0, "dataset_total_score": 0.17500000000000002}, "ru_babilong_qa2": {"4k": 0.11, "8k": 0.08, "16k": 0.16, "32k": 0.08, "64k": 0, "128k": 0, "dataset_total_score": 0.07166666666666667}, "ru_babilong_qa3": {"4k": 0.09, "8k": 0.05, "16k": 0.04, "32k": 0.06, "64k": 0, "128k": 0, "dataset_total_score": 0.04}, "ru_babilong_qa4": {"4k": 0.25214285714285717, "8k": 0.2921428571428571, "16k": 0.15642857142857142, "32k": 0.05928571428571429, "64k": 0, "128k": 0, "dataset_total_score": 0.12666666666666665}, "ru_babilong_qa5": {"4k": 0.5133333333333333, "8k": 0.5, "16k": 0.48333333333333334, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.3327777777777778}, "ru_quality": {"16k": 0.1780538302277433, "8k": 0.28455284552845533, "dataset_total_score": 0.2313033378780993}, "ru_tpo": {"8k": 0.3957503320053121, "dataset_total_score": 0.3957503320053121}, "ru_qasper": {"16k": 0.06475047107877817, "8k": 0.060803436943138944, "32k": 0.023538832049622788, "dataset_total_score": 0.04969758002384664}, "total_score": 0.22130764599351707}

results/Mistral-7B-Instruct-v0.3.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0, "128k": 0, "dataset_total_score": 0.6666666666666666}, "matreshka_yes_no": {"4k": 0.5652173913043478, "8k": 0.5066666666666667, "16k": 0.5466666666666666, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.35309178743961356}, "matreshka_names": {"4k": 0.38, "8k": 0.32, "16k": 0.16666666666666666, "32k": 0.11333333333333333, "64k": 0, "128k": 0, "dataset_total_score": 0.1633333333333333}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 0.995, "64k": 0, "128k": 0, "dataset_total_score": 0.6658333333333334}, "librusec_history": {"8k": 0.71875, "16k": 0.625, "32k": 0.6875, "64k": 0, "dataset_total_score": 0.5078125}, "ru_gsm100": {"16k": 0.11, "dataset_total_score": 0.11}, "ru_sci_passage_count": {"4k": 0.26, "8k": 0.14, "16k": 0.07, "32k": 0.02, "64k": 0, "128k": 0, "dataset_total_score": 0.08166666666666668}, "ru_2wikimultihopqa": {"8k": 0.5510204081632653, "16k": 0.46875, "32k": 0.2764227642276423, "dataset_total_score": 0.4320643907969692}, "long_context_multiq": {"8k": 0.22, "16k": 0.035, "4k": 0.035, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.04833333333333334}, "ru_sci_abstract_retrieval": {"4k": 0.9824603174603175, "8k": 0.8690578865578865, "16k": 0.7111051699917106, "32k": 0.05148203921970241, "64k": 0, "128k": 0, "dataset_total_score": 0.43568423553826957}, "ru_trec": {"4k": 0.5675675675675675, "8k": 0.38, "16k": 0.4065934065934066, "32k": 0.3442622950819672, "dataset_total_score": 0.4246058173107353}, "ru_sci_fi": {"32k": 0.3055555555555556, "64k": 0, "dataset_total_score": 0.1527777777777778}, "librusec_mhqa": {"8k": 0.3359375, "dataset_total_score": 0.3359375}, "ru_babilong_qa1": {"4k": 0.25, "8k": 0.15, "16k": 0.22, "32k": 0.24, "64k": 0, "128k": 0, "dataset_total_score": 0.14333333333333334}, "ru_babilong_qa2": {"4k": 0.08, "8k": 0.05, "16k": 0.02, "32k": 0.02, "64k": 0, "128k": 0, "dataset_total_score": 0.028333333333333332}, "ru_babilong_qa3": {"4k": 0.1, "8k": 0.08, "16k": 0.1, "32k": 0.08, "64k": 0, "128k": 0, "dataset_total_score": 0.060000000000000005}, "ru_babilong_qa4": {"4k": 0.5178571428571428, "8k": 0.4428571428571429, "16k": 0.39285714285714285, "32k": 0.3028571428571429, "64k": 0, "128k": 0, "dataset_total_score": 0.2760714285714286}, "ru_babilong_qa5": {"4k": 0.5466666666666667, "8k": 0.62, "16k": 0.5533333333333333, "32k": 0.5333333333333333, "64k": 0, "128k": 0, "dataset_total_score": 0.3755555555555556}, "ru_quality": {"16k": 0.22981366459627334, "8k": 0.3821138211382114, "dataset_total_score": 0.30596374286724237}, "ru_tpo": {"8k": 0.6640106241699868, "dataset_total_score": 0.6640106241699868}, "ru_qasper": {"16k": 0.06576022201401649, "8k": 0.06619134922698901, "32k": 0.02936483912182809, "dataset_total_score": 0.05377213678761119}, "total_score": 0.2992784522292948}

results/Mistral-7B-v0.1.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 0.975, "16k": 0.125, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.35000000000000003}, "matreshka_yes_no": {"4k": 0.5016722408026756, "8k": 0.5, "16k": 0.0033333333333333335, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.1675009290226682}, "matreshka_names": {"4k": 0.32666666666666666, "8k": 0.16, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.08111111111111112}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 0.3, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.3833333333333333}, "librusec_history": {"8k": 0.78125, "16k": 0.15625, "32k": 0.0, "64k": 0, "dataset_total_score": 0.234375}, "ru_gsm100": {"16k": 0.13, "dataset_total_score": 0.13}, "ru_sci_passage_count": {"4k": 0.04, "8k": 0.04, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.013333333333333334}, "ru_2wikimultihopqa": {"8k": 0.42857142857142855, "16k": 0.1796875, "32k": 0.08130081300813008, "dataset_total_score": 0.22985324719318623}, "long_context_multiq": {"8k": 0.22, "16k": 0.005, "4k": 0.04, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.04416666666666667}, "ru_sci_abstract_retrieval": {"4k": 0.9484126984126985, "8k": 0.7607802118516404, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.28486548504405645}, "ru_trec": {"4k": 0.02702702702702703, "8k": 0.1, "16k": 0.03296703296703297, "32k": 0.0, "dataset_total_score": 0.039998514998515}, "ru_sci_fi": {"32k": 0.027777777777777776, "64k": 0, "dataset_total_score": 0.013888888888888888}, "librusec_mhqa": {"8k": 0.3411458333333333, "dataset_total_score": 0.3411458333333333}, "ru_babilong_qa1": {"4k": 0.63, "8k": 0.63, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.21}, "ru_babilong_qa2": {"4k": 0.21, "8k": 0.25, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.07666666666666666}, "ru_babilong_qa3": {"4k": 0.29, "8k": 0.25, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.09000000000000001}, "ru_babilong_qa4": {"4k": 0.4292857142857143, "8k": 0.3157142857142857, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.12416666666666669}, "ru_babilong_qa5": {"4k": 0.7, "8k": 0.6933333333333335, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.23222222222222225}, "ru_quality": {"16k": 0.11801242236024845, "8k": 0.22764227642276424, "dataset_total_score": 0.17282734939150635}, "ru_tpo": {"8k": 0.3957503320053121, "dataset_total_score": 0.3957503320053121}, "ru_qasper": {"16k": 0.011042882576489372, "8k": 0.0625419691096683, "32k": 0.0008322260797508323, "dataset_total_score": 0.024805692588636172}, "total_score": 0.17333387011743345}

results/Mistral-7B-v0.3.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0, "128k": 0, "dataset_total_score": 0.6666666666666666}, "matreshka_yes_no": {"4k": 0.44816053511705684, "8k": 0.47, "16k": 0.5, "32k": 0.5, "64k": 0, "128k": 0, "dataset_total_score": 0.31969342251950944}, "matreshka_names": {"4k": 0.2866666666666667, "8k": 0.16, "16k": 0.10666666666666667, "32k": 0.04666666666666667, "64k": 0, "128k": 0, "dataset_total_score": 0.09999999999999999}, "passkey_with_librusec": {"4k": 1.0, "8k": 1.0, "16k": 1.0, "32k": 1.0, "64k": 0, "128k": 0, "dataset_total_score": 0.6666666666666666}, "librusec_history": {"8k": 0.9375, "16k": 0.9375, "32k": 0.84375, "64k": 0, "dataset_total_score": 0.6796875}, "ru_gsm100": {"16k": 0.09, "dataset_total_score": 0.09}, "ru_sci_passage_count": {"4k": 0.0, "8k": 0.0, "16k": 0.0, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.0}, "ru_2wikimultihopqa": {"8k": 0.46938775510204084, "16k": 0.4921875, "32k": 0.2682926829268293, "dataset_total_score": 0.40995597934295674}, "long_context_multiq": {"8k": 0.24, "16k": 0.035, "4k": 0.04, "32k": 0.0, "64k": 0, "128k": 0, "dataset_total_score": 0.05249999999999999}, "ru_sci_abstract_retrieval": {"4k": 0.8735714285714286, "8k": 0.5658155733155733, "16k": 0.3690303371278, "32k": 0.01873920061760887, "64k": 0, "128k": 0, "dataset_total_score": 0.30452608993873514}, "ru_trec": {"4k": 0.0, "8k": 0.08, "16k": 0.04395604395604396, "32k": 0.09016393442622951, "dataset_total_score": 0.05352999459556837}, "ru_sci_fi": {"32k": 0.0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.390625, "dataset_total_score": 0.390625}, "ru_babilong_qa1": {"4k": 0.6, "8k": 0.63, "16k": 0.58, "32k": 0.43, "64k": 0, "128k": 0, "dataset_total_score": 0.37333333333333335}, "ru_babilong_qa2": {"4k": 0.35, "8k": 0.23, "16k": 0.18, "32k": 0.24, "64k": 0, "128k": 0, "dataset_total_score": 0.16666666666666666}, "ru_babilong_qa3": {"4k": 0.29, "8k": 0.23, "16k": 0.23, "32k": 0.19, "64k": 0, "128k": 0, "dataset_total_score": 0.15666666666666665}, "ru_babilong_qa4": {"4k": 0.4628571428571429, "8k": 0.3442857142857143, "16k": 0.36214285714285716, "32k": 0.24857142857142858, "64k": 0, "128k": 0, "dataset_total_score": 0.23630952380952383}, "ru_babilong_qa5": {"4k": 0.7033333333333335, "8k": 0.6866666666666668, "16k": 0.7533333333333334, "32k": 0.6833333333333335, "64k": 0, "128k": 0, "dataset_total_score": 0.4711111111111112}, "ru_quality": {"16k": 0.06832298136645963, "8k": 0.23577235772357727, "dataset_total_score": 0.15204766954501844}, "ru_tpo": {"8k": 0.3970783532536521, "dataset_total_score": 0.3970783532536521}, "ru_qasper": {"16k": 0.0648885639017482, "8k": 0.08898968989798027, "32k": 0.01870864419884028, "dataset_total_score": 0.05752896599952292}, "total_score": 0.27355207667217124}

results/Saiga-LLaMA-3-8B.json

@@ -0,0 +1 @@
+{"passkey": {"4k": 1.0, "8k": 1.0, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3333333333333333}, "matreshka_yes_no": {"4k": 0.8729096989966555, "8k": 0.81, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.28048494983277594}, "matreshka_names": {"4k": 0.5333333333333333, "8k": 0.4, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.15555555555555556}, "passkey_with_librusec": {"4k": 1.0, "8k": 0.995, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.3325}, "librusec_history": {"8k": 0.96875, "16k": 0, "32k": 0, "64k": 0, "dataset_total_score": 0.2421875}, "ru_gsm100": {"16k": 0, "dataset_total_score": 0.0}, "ru_sci_passage_count": {"4k": 0.195, "8k": 0.035, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.03833333333333334}, "ru_2wikimultihopqa": {"8k": 0.5306122448979592, "16k": 0, "32k": 0, "dataset_total_score": 0.17687074829931973}, "long_context_multiq": {"8k": 0.235, "4k": 0.055, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.04833333333333333}, "ru_sci_abstract_retrieval": {"4k": 0.9772222222222221, "8k": 0.9258564054992626, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.31717977128691416}, "ru_trec": {"4k": 0.5135135135135135, "8k": 0.54, "16k": 0, "32k": 0, "dataset_total_score": 0.2633783783783784}, "ru_sci_fi": {"32k": 0, "64k": 0, "dataset_total_score": 0.0}, "librusec_mhqa": {"8k": 0.4505208333333333, "dataset_total_score": 0.4505208333333333}, "ru_babilong_qa1": {"4k": 0.7629583333333333, "8k": 0.758125, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.25351388888888887}, "ru_babilong_qa2": {"4k": 0.195625, "8k": 0.06875, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.044062500000000004}, "ru_babilong_qa3": {"4k": 0.14733333333333334, "8k": 0.21585714285714286, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.06053174603174603}, "ru_babilong_qa4": {"4k": 0.6347142857142858, "8k": 0.5821428571428572, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.20280952380952386}, "ru_babilong_qa5": {"4k": 0.7466666666666667, "8k": 0.7633333333333334, "16k": 0, "32k": 0, "64k": 0, "128k": 0, "dataset_total_score": 0.2516666666666667}, "ru_quality": {"8k": 0.35772357723577236, "16k": 0, "dataset_total_score": 0.17886178861788618}, "ru_tpo": {"8k": 0.7569721115537849, "dataset_total_score": 0.7569721115537849}, "ru_qasper": {"8k": 0.07413702213069599, "16k": 0, "32k": 0, "dataset_total_score": 0.024712340710231998}, "total_score": 0.21008610966500027}