Add files using upload-large-folder tool

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Arabic Language (General)"
+"tag": "arabicmmlu_language_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_arabic_language_(general)"
+"task_alias": "Arabic Language (General)"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_high_economics.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "High Economics"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_high_economics"
+"task_alias": "High Economics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_civics.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Middle Civics"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_middle_civics"
+"task_alias": "Middle Civics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_history.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Middle History"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_middle_history"
+"task_alias": "Middle History"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_islamic_studies.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Middle Islamic Studies"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_middle_islamic_studies"
+"task_alias": "Middle Islamic Studies"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Primary Arabic Language"
+"tag": "arabicmmlu_language_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_arabic_language"
+"task_alias": "Primary Arabic Language"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_general_knowledge.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Primary General Knowledge"
+"tag": "arabicmmlu_other_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_general_knowledge"
+"task_alias": "Primary General Knowledge"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_geography.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Primary Geography"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_geography"
+"task_alias": "Primary Geography"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_islamic_studies.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Primary Islamic Studies"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_islamic_studies"
+"task_alias": "Primary Islamic Studies"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_social_science.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Primary Social Science"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_primary_social_science"
+"task_alias": "Primary Social Science"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_prof_law.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Prof Law"
+"tag": "arabicmmlu_humanities_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_prof_law"
+"task_alias": "Prof Law"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Univ Management"
+"tag": "arabicmmlu_other_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_univ_management"
+"task_alias": "Univ Management"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml

@@ -0,0 +1,5 @@
+"dataset_name": "Univ Political Science"
+"tag": "arabicmmlu_social_science_tasks"
+"include": "_default_arabicmmlu_template_yaml"
+"task": "arabicmmlu_univ_political_science"
+"task_alias": "Univ Political Science"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/arabicmmlu/utils.py

@@ -0,0 +1,44 @@
+PROMPT = "This is a {}. Select the correct answer!\n\nQuestion: {}\n{}\n\nAnswer:"
+
+level_en = {
+    "Primary": "primary school",
+    "Middle": "middle school",
+    "High": "high school",
+    "Univ": "university",
+    "Prof": "professional",
+}
+
+alpa = ["A.", "B.", "C.", "D.", "E."]
+
+
+def doc_to_text(doc):
+    """
+    Refactoring `prepare_data_en` to fit with the lm harness framework.
+    https://github.com/mbzuai-nlp/ArabicMMLU/blob/main/util_prompt.py
+    """
+
+    level = "" if not doc["Level"] else " for " + level_en[doc["Level"]]
+    country = "" if not doc["Country"] else " in " + doc["Country"]
+    main_meta_data = f"{doc['Subject']} question{level}{country}"
+
+    question = (
+        doc["Question"]
+        if doc["Context"] == ""
+        else f"{doc['Context']}\n\n{doc['Question']}"
+    )
+
+    options = []
+    for i, opt in enumerate(
+        ["Option 1", "Option 2", "Option 3", "Option 4", "Option 5"]
+    ):
+        if not doc[opt]:
+            break
+        options.append(f"{alpa[i]} {doc[opt]}")
+
+    doc_text = PROMPT.format(main_meta_data, question, "\n".join(options))
+
+    return doc_text
+
+
+def doc_to_choice(doc):
+    return [alpa[i][0] for i in range(5) if doc[f"Option {i+1}"]]

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/README.md

@@ -0,0 +1,47 @@
+# TMMLU+
+
+### Paper
+
+Title: `An Improved Traditional Chinese Evaluation Suite for Foundation Model`
+
+Abstract: `We present TMMLU+, a comprehensive dataset designed for the Traditional Chinese massive multitask language understanding dataset. TMMLU+ is a multiple-choice question-answering dataset with 66 subjects from elementary to professional level. Compared to its predecessor, TMMLU, TMMLU+ is six times larger and boasts a more balanced subject distribution. We included benchmark results in TMMLU+ from closed-source models and 24 open-weight Chinese large language models of parameters ranging from 1.8B to 72B. Our findings reveal that Traditional Chinese models still trail behind their Simplified Chinese counterparts. Additionally, current large language models have yet to outperform human performance in average scores. We publicly release our dataset and the corresponding benchmark source code.`
+
+
+Homepage: [https://huggingface.co/datasets/ikala/tmmluplus](https://huggingface.co/datasets/ikala/tmmluplus)
+
+
+### Citation
+
+```
+@article{ikala2024improved,
+  title={An Improved Traditional Chinese Evaluation Suite for Foundation Model},
+  author={Tam, Zhi-Rui and Pai, Ya-Ting and Lee, Yen-Wei and Cheng, Sega and Shuai, Hong-Han},
+  journal={arXiv preprint arXiv:2403.01858},
+  year={2024}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* `tmmluplus`: `The dataset comprises 22,690 multiple-choice questions from 66 subjects ranging from primary to professional level. `
+
+#### Tasks
+
+The following tasks evaluate subjects in the TMMLU+ dataset using loglikelihood-based multiple-choice scoring:
+
+* `tmmluplus_{subject_english}`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/_generate_configs.py

@@ -0,0 +1,211 @@
+"""
+Take in a YAML, and output all "other" splits with this YAML
+"""
+
+import argparse
+import os
+
+import pandas as pd
+import yaml
+from tqdm import tqdm
+
+
+# Copy from https://github.com/iKala/ievals/blob/main/ievals/settings.py
+# from TMMLU+ offical example
+categories = {
+    "STEM": [
+        "physics",
+        "chemistry",
+        "biology",
+        "computer science",
+        "math",
+        "engineering",
+    ],
+    "humanities": ["history", "philosophy", "law"],
+    "social_sciences": [
+        "politics",
+        "culture",
+        "economics",
+        "geography",
+        "psychology",
+        "education",
+    ],
+    "other": ["other", "business", "health"],  # (business, health, misc.)
+}
+
+task_list = [
+    "engineering_math",
+    "dentistry",
+    "traditional_chinese_medicine_clinical_medicine",
+    "clinical_psychology",
+    "technical",
+    "culinary_skills",
+    "mechanical",
+    "logic_reasoning",
+    "real_estate",
+    "general_principles_of_law",
+    "finance_banking",
+    "anti_money_laundering",
+    "ttqav2",
+    "marketing_management",
+    "business_management",
+    "organic_chemistry",
+    "advance_chemistry",
+    "physics",
+    "secondary_physics",
+    "human_behavior",
+    "national_protection",
+    "jce_humanities",
+    "politic_science",
+    "agriculture",
+    "official_document_management",
+    "financial_analysis",
+    "pharmacy",
+    "educational_psychology",
+    "statistics_and_machine_learning",
+    "management_accounting",
+    "introduction_to_law",
+    "computer_science",
+    "veterinary_pathology",
+    "accounting",
+    "fire_science",
+    "optometry",
+    "insurance_studies",
+    "pharmacology",
+    "taxation",
+    "education_(profession_level)",
+    "economics",
+    "veterinary_pharmacology",
+    "nautical_science",
+    "occupational_therapy_for_psychological_disorders",
+    "trust_practice",
+    "geography_of_taiwan",
+    "physical_education",
+    "auditing",
+    "administrative_law",
+    "basic_medical_science",
+    "macroeconomics",
+    "trade",
+    "chinese_language_and_literature",
+    "tve_design",
+    "junior_science_exam",
+    "junior_math_exam",
+    "junior_chinese_exam",
+    "junior_social_studies",
+    "tve_mathematics",
+    "tve_chinese_language",
+    "tve_natural_sciences",
+    "junior_chemistry",
+    "music",
+    "education",
+    "three_principles_of_people",
+    "taiwanese_hokkien",
+]
+subject2name = {}
+# subject2category = {}
+SUBJECTS = {}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base_yaml_path", required=True)
+    parser.add_argument("--save_prefix_path", default="tmmluplus")
+    parser.add_argument("--cot_prompt_path", default=None)
+    parser.add_argument("--task_prefix", default="")
+    parser.add_argument("--group_prefix", default="")
+    parser.add_argument("--subject_file", default="subject.tsv")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    from pathlib import Path
+
+    # Initialization
+    SUBJECT_FILE = Path(__file__).parent / Path(args.subject_file)
+
+    df = pd.read_csv(SUBJECT_FILE, delimiter="\t")
+
+    for _, row in df.iterrows():
+        for _c in categories:
+            if row["subject"] in SUBJECTS:
+                raise ValueError("Duplicate tasks.")
+            if row["category"] in categories[_c]:  # append new item into SUBJECTS
+                SUBJECTS[row["subject"]] = _c
+                subject2name[row["subject"]] = row["name"]
+                break
+    # End of SUBJECTS initialization
+
+    # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs.
+    base_yaml_name = os.path.split(args.base_yaml_path)[-1]
+    with open(args.base_yaml_path) as f:
+        base_yaml = yaml.full_load(f)
+
+    if args.cot_prompt_path is not None:
+        import json
+
+        with open(args.cot_prompt_path) as f:
+            cot_file = json.load(f)
+
+    ALL_CATEGORIES = []
+    for subject, category in tqdm(SUBJECTS.items()):
+        if category not in ALL_CATEGORIES:
+            ALL_CATEGORIES.append(category)
+
+        if args.cot_prompt_path is not None:
+            description = cot_file[subject]
+        else:
+            name_of_subject = subject2name[subject].replace("＿", " ")
+            description = f"以下為{name_of_subject}的單選題，請提供正確答案的選項。\n\n"
+            # description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n"
+
+        yaml_dict = {
+            "include": base_yaml_name,
+            "group": f"tmmluplus_{args.task_prefix}_{category}"
+            if args.task_prefix != ""
+            else f"tmmluplus_{category}",
+            "group_alias": category.replace("_", " "),
+            "task": f"tmmluplus_{args.task_prefix}_{subject}"
+            if args.task_prefix != ""
+            else f"tmmluplus_{subject}",
+            "task_alias": subject.replace("_", " "),
+            "dataset_name": subject,
+            "description": description,
+        }
+
+        file_save_path = args.save_prefix_path + f"_{subject}.yaml"
+        # eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}")
+        with open(file_save_path, "w") as yaml_file:
+            yaml.dump(
+                yaml_dict,
+                yaml_file,
+                # width=float("inf"),
+                allow_unicode=True,
+                default_style='"',
+            )
+
+    if args.task_prefix != "":
+        mmlu_subcategories = [
+            f"tmmluplus_{args.task_prefix}_{category}" for category in ALL_CATEGORIES
+        ]
+    else:
+        mmlu_subcategories = [f"tmmluplus_{category}" for category in ALL_CATEGORIES]
+
+    if args.group_prefix != "":
+        file_save_path = args.group_prefix + ".yaml"
+    else:
+        file_save_path = args.save_prefix_path + ".yaml"
+
+    # eval_logger.info(f"Saving benchmark config to {file_save_path}")
+    with open(file_save_path, "w") as yaml_file:
+        yaml.dump(
+            {
+                "group": f"tmmluplus_{args.task_prefix}"
+                if args.task_prefix != ""
+                else "tmmluplus",
+                "task": mmlu_subcategories,
+            },
+            yaml_file,
+            indent=4,
+            default_flow_style=False,
+        )

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "accounting"
+"description": "以下為會計學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_accounting"
+"task_alias": "accounting"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "administrative_law"
+"description": "以下為行政法的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_humanities"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "tmmluplus_administrative_law"
+"task_alias": "administrative law"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "agriculture"
+"description": "以下為農業的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_agriculture"
+"task_alias": "agriculture"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "anti_money_laundering"
+"description": "以下為洗錢防制的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_humanities"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "tmmluplus_anti_money_laundering"
+"task_alias": "anti money laundering"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "auditing"
+"description": "以下為審計學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_auditing"
+"task_alias": "auditing"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "basic_medical_science"
+"description": "以下為基礎醫學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_basic_medical_science"
+"task_alias": "basic medical science"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "business_management"
+"description": "以下為企業管理的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_business_management"
+"task_alias": "business management"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "chinese_language_and_literature"
+"description": "以下為國文的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_chinese_language_and_literature"
+"task_alias": "chinese language and literature"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "clinical_psychology"
+"description": "以下為臨床心理學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_clinical_psychology"
+"task_alias": "clinical psychology"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "computer_science"
+"description": "以下為資訊工程的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_computer_science"
+"task_alias": "computer science"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "dentistry"
+"description": "以下為牙醫學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_dentistry"
+"task_alias": "dentistry"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "economics"
+"description": "以下為經濟學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_economics"
+"task_alias": "economics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "education"
+"description": "以下為教育常識的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_education"
+"task_alias": "education"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml

@@ -0,0 +1,7 @@
+"dataset_name": "education_(profession_level)"
+"description": "以下為教育專業的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_education_(profession_level)"
+"task_alias": "education (profession level)"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "educational_psychology"
+"description": "以下為教育心理的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_educational_psychology"
+"task_alias": "educational psychology"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "engineering_math"
+"description": "以下為工程數學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_engineering_math"
+"task_alias": "engineering math"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "finance_banking"
+"description": "以下為金融與法規的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_finance_banking"
+"task_alias": "finance banking"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "financial_analysis"
+"description": "以下為財務分析的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_financial_analysis"
+"task_alias": "financial analysis"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "fire_science"
+"description": "以下為火災學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_fire_science"
+"task_alias": "fire science"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "human_behavior"
+"description": "以下為人類行為與社會的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_human_behavior"
+"task_alias": "human behavior"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "insurance_studies"
+"description": "以下為保險學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_insurance_studies"
+"task_alias": "insurance studies"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "introduction_to_law"
+"description": "以下為法律概論的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_humanities"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "tmmluplus_introduction_to_law"
+"task_alias": "introduction to law"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "junior_chemistry"
+"description": "以下為國中理化的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_junior_chemistry"
+"task_alias": "junior chemistry"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "junior_chinese_exam"
+"description": "以下為國中會考基測國文的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_junior_chinese_exam"
+"task_alias": "junior chinese exam"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "junior_math_exam"
+"description": "以下為國中會考基測數學科的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_junior_math_exam"
+"task_alias": "junior math exam"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "junior_science_exam"
+"description": "以下為國中會考基測自然科的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_junior_science_exam"
+"task_alias": "junior science exam"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "junior_social_studies"
+"description": "以下為國中會考基測社會科的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_junior_social_studies"
+"task_alias": "junior social studies"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "linear_algebra"
+"description": "以下為線代的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_linear_algebra"
+"task_alias": "linear algebra"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "logic_reasoning"
+"description": "以下為邏輯思維的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_logic_reasoning"
+"task_alias": "logic reasoning"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "macroeconomics"
+"description": "以下為總經的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_macroeconomics"
+"task_alias": "macroeconomics"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "management_accounting"
+"description": "以下為管理會計的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_management_accounting"
+"task_alias": "management accounting"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "marketing_management"
+"description": "以下為行銷管理的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_marketing_management"
+"task_alias": "marketing management"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "music"
+"description": "以下為音樂科的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_music"
+"task_alias": "music"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "national_protection"
+"description": "以下為軍事的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_national_protection"
+"task_alias": "national protection"