from dataclasses import dataclass
from enum import Enum


@dataclass
class Task:
    benchmark: str
    metric: str
    col_name: str


# Init: to update with your specific keys
class Tasks(Enum):
    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
    task0 = Task("agree_cs", "accuracy", "agree_cs")
    task1 = Task("anli_cs", "accuracy", "anli_cs")
    task2 = Task("arc_challenge_cs", "accuracy", "arc_challenge_cs")
    task3 = Task("arc_easy_cs", "accuracy", "arc_easy_cs")
    task4 = Task("belebele_cs", "accuracy", "belebele_cs")
    task5 = Task("ctkfacts_cs", "accuracy", "ctkfacts_cs")
    task6 = Task("czechnews_cs", "accuracy", "czechnews_cs")
    task7 = Task("fb_comments_cs", "accuracy", "fb_comments_cs")
    task8 = Task("gsm8k_cs", "accuracy", "gsm8k_cs")
    task9 = Task("klokanek_cs", "accuracy", "klokanek_cs")
    task10 = Task("mall_reviews_cs", "accuracy", "mall_reviews_cs")
    task11 = Task("mmlu_cs", "accuracy", "mmlu_cs")
    task12 = Task("sqad_cs", "accuracy", "sqad_cs")
    task13 = Task("subjectivity_cs", "accuracy", "subjectivity_cs")
    task14 = Task("truthfulqa_cs", "accuracy", "truthfulqa_cs")

# Your leaderboard name
TITLE = """<h1 align="center" id="space-title">🇨🇿 CzechBench Leaderboard</h1>"""

# What does your leaderboard evaluate?
INTRODUCTION_TEXT = """
Czech-Bench is a collection of LLM benchmarks available for the Czech language. It currently consists of 15 Czech benchmarks, including new machine translations of the popular ARC, GSM8K, MMLU, and TruthfulQA datasets.

Czech-Bench is developed by <a href="https://huggingface.co/CIIRC-NLP">CIIRC-NLP</a>.
"""

TABLE_DESC = "The values presented in the table represent the accuracy metric."

# Which evaluations are you running? how can people reproduce what you have?
LLM_BENCHMARKS_TEXT = f"""
## Basic Information
The goal of this project is to provide a comprehensive and practical benchmark for evaluating Czech language models. This benchmark consists of 15 selected test tasks containing test data in the Czech language. It includes both original Czech datasets and machine translations of popular datasets such as ARC, GSM8K, MMLU, and TruthfulQA. A list of all datasets can be found at [GitHub](https://github.com/jirkoada/czechbench_eval_harness/tree/main/lm_eval/tasks/czechbench#readme)

Key Features and Benefits:
- **Tailored for the Czech Language:** The benchmark includes both original Czech datasets and adapted versions of international datasets, ensuring relevant evaluation of model performance in the Czech context.
- **Wide Range of Tasks:** It contains 15 different tasks that cover various aspects of language understanding and text generation, enabling a comprehensive assessment of the model's capabilities.
- **Ease of Use:** The benchmark is designed to be easily integrated into your development process, saving time and resources during model testing and improvement.
- **Up-to-date and Relevant:** We regularly update our datasets to reflect the latest findings and trends in language model development.

By using this benchmark, you will gain deep insights into the strengths and weaknesses of your models, allowing you to better focus on key areas for optimization. This will not only improve the performance of your models but also enhance their real-world deployment in various Czech contexts.

## Evaluation Process

### 1. Install CzechBench:
```
git clone https://github.com/jirkoada/czechbench_eval_harness.git
cd czechbench_eval_harness
pip install -e “.[api]” 
```

### 2. Run evaluation
* `export MODEL=your_model_name` where your_model_name is HF path for public model. For example: `export MODEL=meta-llama/Meta-Llama-3.1-8B-Instruct`
* `export OUTPUT_PATH=my_output_path` where my_output_path is directory for evaluation reports 


Run following command (you can adjust parameters like batch_size or device):
```
lm_eval --model hf \\
    --model_args pretrained=$MODEL \\
    --tasks czechbench_tasks \\
    --device cuda:0 \\
    --batch_size 1 \\
    --write_out \\
    --log_samples \\
    --output_path $OUTPUT_PATH \\
    --apply_chat_template \\
```
	

### 3. Upload results to Leaderboard
in `$OUTPUT_PATH` directory you can find file `results.json` upload `result.json` to [CzechBench Leaderboard](https://huggingface.co/spaces/CIIRC-NLP/czechbench_leaderboard) on **Submit Here!** tab. 

"""

EVALUATION_QUEUE_TEXT = """

"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""
"""