Added specified metric for every task in "tasks_metadata.json"

tasks_metadata.json

@@ -4,377 +4,431 @@
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_argumentace",
     "short_name": "P-Argumentace",
     "category": "NLI",
-    "abbreviation": "P-ARG"
+    "abbreviation": "P-ARG",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_fabulace": {
     "name": "Propaganda – Fabulace",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_fabulace",
     "short_name": "P-Fabulace",
     "category": "NLI",
-    "abbreviation": "P-FAB"
+    "abbreviation": "P-FAB",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_nazor": {
     "name": "Propaganda – Názor",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_nazor",
     "short_name": "P-Názor",
     "category": "NLI",
-    "abbreviation": "P-NAZOR"
+    "abbreviation": "P-NAZOR",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_strach": {
     "name": "Propaganda – Strach",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_strach",
     "short_name": "P-Strach",
     "category": "NLI",
-    "abbreviation": "P-STCH"
+    "abbreviation": "P-STCH",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_zamereni": {
     "name": "Propaganda – Zaměření",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_zamereni",
     "short_name": "P-Zaměření",
     "category": "NLI",
-    "abbreviation": "P-MER"
+    "abbreviation": "P-MER",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_demonizace": {
     "name": "Propaganda – Démonizace",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_demonizace",
     "short_name": "P-Démonizace",
     "category": "NLI",
-    "abbreviation": "P-DEMON"
+    "abbreviation": "P-DEMON",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_lokace": {
     "name": "Propaganda – Lokace",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_lokace",
     "short_name": "P-Lokace",
     "category": "NLI",
-    "abbreviation": "P-LOK"
+    "abbreviation": "P-LOK",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_relativizace": {
     "name": "Propaganda – Relativizace",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_relativizace",
     "short_name": "P-Relativizace",
     "category": "NLI",
-    "abbreviation": "P-REL"
+    "abbreviation": "P-REL",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_vina": {
     "name": "Propaganda – Vina",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_vina",
     "short_name": "P-Vina",
     "category": "NLI",
-    "abbreviation": "P-VINA"
+    "abbreviation": "P-VINA",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_zanr": {
     "name": "Propaganda – Žánr",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_zanr",
     "short_name": "P-Žánr",
     "category": "NLI",
-    "abbreviation": "P-ZANR"
+    "abbreviation": "P-ZANR",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_emoce": {
     "name": "Propaganda – Emoce",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_emoce",
     "short_name": "P-Emoce",
     "category": "NLI",
-    "abbreviation": "P-EMOCE"
+    "abbreviation": "P-EMOCE",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_nalepkovani": {
     "name": "Propaganda – Nálepkování",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_nalepkovani",
     "short_name": "P-Nálepkování",
     "category": "NLI",
-    "abbreviation": "P-LEP"
+    "abbreviation": "P-LEP",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_propaganda_rusko": {
     "name": "Propaganda – Rusko",
     "source_url": "https://huggingface.co/datasets/CZLC/propaganda_rusko",
     "short_name": "P-Rusko",
     "category": "NLI",
-    "abbreviation": "P-RUS"
+    "abbreviation": "P-RUS",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_sentiment_mall": {
     "name": "CzechSentiment MALL",
     "source_url": "https://huggingface.co/datasets/CZLC/mall_sentiment_balanced",
     "short_name": "S-Mall",
     "category": "Sentiment",
-    "abbreviation": "S-MALL"
+    "abbreviation": "S-MALL",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_sentiment_fb": {
     "name": "CzechSentiment FB",
     "source_url": "https://huggingface.co/datasets/CZLC/fb_sentiment_balanced",
     "short_name": "S-FB",
     "category": "Sentiment",
-    "abbreviation": "S-FB"
+    "abbreviation": "S-FB",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_sentiment_csfd": {
     "name": "CzechSentiment CSFD",
     "source_url": "https://huggingface.co/datasets/CZLC/csfd_sentiment_balanced",
     "short_name": "S-CSFD",
     "category": "Sentiment",
-    "abbreviation": "S-CSFD"
+    "abbreviation": "S-CSFD",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_summarization": {
     "name": "SUMECZECH",
     "source_url": "https://huggingface.co/datasets/CZLC/sumeczech_downsampled",
     "short_name": "Summarization",
     "category": "Summarization",
-    "abbreviation": "SUM"
+    "abbreviation": "SUM",
+    "metric": "rouge_raw_r2_mid_f"
   },
   "benczechmark_grammarerrorcorrection": {
     "name": "GrammarErrorCorrection",
     "source_url": "https://huggingface.co/datasets/CZLC/cs_gec",
     "short_name": "Grammar Error Correction",
     "category": "Syntactical Reasoning",
-    "abbreviation": "GEC"
+    "abbreviation": "GEC",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_cs_naturalquestions": {
     "name": "NaturalQuestions-CZ",
     "source_url": "https://huggingface.co/datasets/CZLC/cs_naturalquestions",
     "short_name": "CS Natural Questions",
     "category": "Knowledge",
-    "abbreviation": "NQ"
+    "abbreviation": "NQ",
+    "metric": "exact_match"
   },
   "benczechmark_cs_sqad32": {
     "name": "SQAD3.2",
     "source_url": "https://huggingface.co/datasets/CZLC/SQAD_3.2",
     "short_name": "CS SQAD 3.2",
     "category": "Knowledge",
-    "abbreviation": "SQAD32"
+    "abbreviation": "SQAD32",
+    "metric": "exact_match"
   },
   "benczechmark_cs_triviaQA": {
     "name": "TriviaQA-CZ",
     "source_url": "https://huggingface.co/datasets/CZLC/cs_triviaqa",
     "short_name": "CS TriviaQA",
     "category": "Knowledge",
-    "abbreviation": "TQA"
+    "abbreviation": "TQA",
+    "metric": "exact_match"
   },
   "benczechmark_csfever_nli": {
     "name": "CSFEVER",
     "source_url": "https://huggingface.co/datasets/CZLC/ctu-aic/csfever_nli",
     "short_name": "CSFever NLI",
     "category": "NLI",
-    "abbreviation": "CFR"
+    "abbreviation": "CFR",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_ctkfacts_nli": {
     "name": "CTKFACTS",
     "source_url": "https://huggingface.co/datasets/CZLC/ctu-aic/ctkfacts_nli",
     "short_name": "CTKFacts NLI",
     "category": "NLI",
-    "abbreviation": "CTK"
+    "abbreviation": "CTK",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_cs_ner": {
     "name": "Czech Named Entity Corpus 2.0\n",
     "source_url": "https://huggingface.co/datasets/CZLC/cnec_2.0",
     "short_name": "CNEC2.0",
     "category": "NER",
-    "abbreviation": "CNEC"
+    "abbreviation": "CNEC",
+    "metric": "exact_match"
   },
   "benczechmark_hellaswag": {
     "name": "HellaSwag-CZ",
     "source_url": "https://huggingface.co/datasets/CZLC/cs_hellaswag",
     "short_name": "HellaSwag",
     "category": "Language Modeling",
-    "abbreviation": "HASG"
+    "abbreviation": "HASG",
+    "metric": "acc"
   },
   "benczechmark_histcorpus": {
     "name": "Historical Corpus",
     "source_url": "https://huggingface.co/datasets/CZLC/benczechmark_histcorpus",
     "short_name": "HistCorpus",
     "category": "Language Modeling",
-    "abbreviation": "HIST"
+    "abbreviation": "HIST",
+    "metric": "word_perplexity"
   },
   "benczechmark_klokan_qa": {
     "name": "Klokan QA",
     "source_url": "https://huggingface.co/datasets/hynky/klokan-qa",
     "short_name": "Klokan QA",
     "category": "Czech Math Reasoning",
-    "abbreviation": "KQA"
+    "abbreviation": "KQA",
+    "metric": "acc"
   },
   "benczechmark_cs_court_decisions_ner": {
     "name": "Czech Court Decisions",
     "source_url": "https://huggingface.co/datasets/CZLC/ner_court_decisions",
     "short_name": "CS Court Decisions NER",
     "category": "NER",
-    "abbreviation": "CCDNER"
+    "abbreviation": "CCDNER",
+    "metric": "exact_match"
   },
   "benczechmark_umimeto_biology": {
     "name": "Umimeto.cz – Biology",
     "source_url": "https://huggingface.co/datasets/CZLC/umimeto-qa/viewer/biology",
     "short_name": "Umimeto.cz – Biology",
     "category": "General Reasoning",
-    "abbreviation": "UT-BIO"
+    "abbreviation": "UT-BIO",
+    "metric": "acc"
   },
   "benczechmark_umimeto_chemistry": {
     "name": "Umimeto.cz – Chemistry",
     "source_url": "https://huggingface.co/datasets/CZLC/umimeto-qa/viewer/chemistry",
     "short_name": "Umimeto.cz – Chemistry",
     "category": "General Reasoning",
-    "abbreviation": "UT-CHEM"
+    "abbreviation": "UT-CHEM",
+    "metric": "acc"
   },
   "benczechmark_umimeto_czech": {
     "name": "Umimeto.cz – Czech Language",
     "source_url": "https://huggingface.co/datasets/CZLC/umimeto-qa/viewer/czech",
     "short_name": "Umimeto.cz – Czech",
     "category": "General Reasoning",
-    "abbreviation": "UT-CZEL"
+    "abbreviation": "UT-CZEL",
+    "metric": "acc"
   },
   "benczechmark_umimeto_history": {
     "name": "Umimeto.cz – History",
     "source_url": "https://huggingface.co/datasets/CZLC/umimeto-qa/viewer/history",
     "short_name": "Umimeto.cz – History",
     "category": "General Reasoning",
-    "abbreviation": "UT-HIST"
+    "abbreviation": "UT-HIST",
+    "metric": "acc"
   },
   "benczechmark_umimeto_informatics": {
     "name": "Umimeto.cz – Informatics",
     "source_url": "https://huggingface.co/datasets/CZLC/umimeto-qa/viewer/informatics",
     "short_name": "Umimeto.cz – Informatics",
     "category": "General Reasoning",
-    "abbreviation": "UT-IT"
+    "abbreviation": "UT-IT",
+    "metric": "acc"
   },
   "benczechmark_umimeto_math": {
     "name": "Umimeto.cz ��� Math",
     "source_url": "https://huggingface.co/datasets/CZLC/umimeto-qa/viewer/math",
     "short_name": "Umimeto.cz – Math",
     "category": "Czech Math Reasoning",
-    "abbreviation": "UT-MATH"
+    "abbreviation": "UT-MATH",
+    "metric": "acc"
   },
   "benczechmark_umimeto_physics": {
     "name": "Umimeto.cz – Physics",
     "source_url": "https://huggingface.co/datasets/CZLC/umimeto-qa/viewer/physics",
     "short_name": "Umimeto.cz – Physics",
     "category": "General Reasoning",
-    "abbreviation": "UT-PHYS"
+    "abbreviation": "UT-PHYS",
+    "metric": "acc"
   },
   "benczechmark_cermat_czmath_mc": {
     "name": "CERMAT – Czech Math – MC",
     "source_url": "https://huggingface.co/datasets/CZLC/cermat_math_mc",
     "short_name": "Cermat Czech Math MC",
     "category": "Czech Math Reasoning",
-    "abbreviation": "CCM-MC"
+    "abbreviation": "CCM-MC",
+    "metric": "acc"
   },
   "benczechmark_cermat_czmath_open": {
     "name": "CERMAT – Czech Math – OPEN",
     "source_url": "https://huggingface.co/datasets/CZLC/cermat_math_open",
     "short_name": "Cermat Czech Math Open",
     "category": "Czech Math Reasoning",
-    "abbreviation": "CCM-OPEN"
+    "abbreviation": "CCM-OPEN",
+    "metric": "exact_match"
   },
   "benczechmark_cermat_czech_tf": {
     "name": "CERMAT – Czech Language – TF",
     "source_url": "https://huggingface.co/datasets/CZLC/cermat_czech_tf",
     "short_name": "Cermat Czech Language TF",
     "category": "General Reasoning",
-    "abbreviation": "CCL-TF"
+    "abbreviation": "CCL-TF",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_cermat_czech_mc": {
     "name": "CERMAT – Czech Language – MC",
     "source_url": "https://huggingface.co/datasets/CZLC/cermat_czech_mc",
     "short_name": "Cermat Czech Language MC",
     "category": "General Reasoning",
-    "abbreviation": "CCL-MC"
+    "abbreviation": "CCL-MC",
+    "metric": "acc"
   },
   "benczechmark_cermat_czech_open": {
     "name": "CERMAT – Czech Language – OPEN",
     "source_url": "https://huggingface.co/datasets/CZLC/cermat_czech_open",
     "short_name": "Cermat Czech Language Open",
     "category": "General Reasoning",
-    "abbreviation": "CCL-OPEN"
+    "abbreviation": "CCL-OPEN",
+    "metric": "exact_match"
   },
   "benczechmark_history_ir": {
     "name": "Historical Relevance Grading",
     "source_url": "https://huggingface.co/datasets/CZLC/history_retrieval",
     "abbreviation": "HIST-IR",
     "category": "Historical",
-    "short_name": "Czech History IR"
+    "short_name": "Czech History IR",
+    "metric": "acc"
   },
   "benczechmark_agree": {
     "name": "Agree",
     "source_url": "https://huggingface.co/datasets/davidadamczyk/czechbench_agree",
     "abbreviation": "Agree",
     "category": "Syntactical Reasoning",
-    "short_name": "Agree"
+    "short_name": "Agree",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_belebele": {
     "name": "Belebele",
     "source_url": "https://huggingface.co/datasets/davidadamczyk/czechbench_belebele",
     "abbreviation": "BB",
     "category": "Knowledge",
-    "short_name": "Belebele"
+    "short_name": "Belebele",
+    "metric": "acc"
   },
   "benczechmark_czechnews": {
     "name": "CZ-NEWS",
     "source_url": "https://huggingface.co/datasets/davidadamczyk/czechbench_czech_news",
     "abbreviation": "CZ-NEWS",
     "category": "Topic Classification",
-    "short_name": "Czech News"
+    "short_name": "Czech News",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_subjectivity": {
     "name": "Subjectivity",
     "source_url": "https://huggingface.co/datasets/davidadamczyk/czechbench_subjectivity",
     "abbreviation": "SUBJ",
     "category": "Sentiment",
-    "short_name": "Subjectivity"
+    "short_name": "Subjectivity",
+    "metric": "avg_mcauroc"
   },
   "benczechmark_essay": {
     "name": "Czech National Corpus – Essays",
     "source_url": "https://huggingface.co/datasets/CZLC/CNC_skript12",
     "abbreviation": "CNC-E",
     "category": "Language Modeling",
-    "short_name": "CNC – Essays"
+    "short_name": "CNC – Essays",
+    "metric": "word_perplexity"
   },
   "benczechmark_fiction": {
     "name": "Czech National Corpus – Fiction",
     "source_url": "https://huggingface.co/datasets/CZLC/CNC_fictree",
     "abbreviation": "CNC-F",
     "category": "Language Modeling",
-    "short_name": "CNC – Fiction"
+    "short_name": "CNC – Fiction",
+    "metric": "word_perplexity"
   },
   "benczechmark_capek": {
     "name": "Czech National Corpus – Karel Čapek",
     "source_url": "https://huggingface.co/datasets/CZLC/CNC_Capek",
     "abbreviation": "CNC-CAP",
     "category": "Language Modeling",
-    "short_name": "CNC – Čapek"
+    "short_name": "CNC – Čapek",
+    "metric": "word_perplexity"
   },
   "benczechmark_correspondence": {
     "name": "Czech National Corpus – Correspondence",
     "source_url": "https://huggingface.co/datasets/CZLC/CNC_KSK",
     "abbreviation": "CNC-KSK",
     "category": "Language Modeling",
-    "short_name": "CNC – Correspondence"
+    "short_name": "CNC – Correspondence",
+    "metric": "word_perplexity"
   },
   "benczechmark_havlicek": {
     "name": "Czech National Corpus – Karel Havlíček – Noviny",
     "source_url": "https://huggingface.co/datasets/CZLC/CNC_KHavlicek_HistNews",
     "abbreviation": "CNC-KH",
     "category": "Language Modeling",
-    "short_name": "CNC – KHavlicek – HistNews"
+    "short_name": "CNC – KHavlicek – HistNews",
+    "metric": "word_perplexity"
   },
   "benczechmark_speeches": {
     "name": "Czech National Corpus – Speeches",
     "source_url": "https://huggingface.co/datasets/CZLC/CNC_PrezPrejavy",
     "abbreviation": "CNC-SPCH",
     "category": "Language Modeling",
-    "short_name": "CNC – Speeches"
+    "short_name": "CNC – Speeches",
+    "metric": "word_perplexity"
   },
   "benczechmark_spoken": {
     "name": "Czech National Corpus – Spoken",
     "source_url": "https://huggingface.co/datasets/CZLC/CNC_oral_ortofon",
     "abbreviation": "CNC-SPK",
     "category": "Language Modeling",
-    "short_name": "CNC – Spoken"
+    "short_name": "CNC – Spoken",
+    "metric": "word_perplexity"
   },
   "benczechmark_dialect": {
     "name": "Czech National Corpus – Dialect",
     "source_url": "https://huggingface.co/datasets/CZLC/CNC_Dialekt",
     "abbreviation": "CNC-DIAL",
     "category": "Language Modeling",
-    "short_name": "CNC – DIALEKT"
+    "short_name": "CNC – DIALEKT",
+    "metric": "word_perplexity"
   },
   "benczechmark_snli": {
     "name": "Czech SNLI",
     "source_url": "https://huggingface.co/datasets/CZLC/snli",
     "abbreviation": "SNLI",
     "category": "NLI",
-    "short_name": "Czech SNLI"
+    "short_name": "Czech SNLI",
+    "metric": "avg_mcauroc"
   }
 }