config.yaml

config:
  REPO_ID: "mteb/leaderboard"
  RESULTS_REPO: mteb/results
  LEADERBOARD_NAME: "MTEB Leaderboard"
tasks:
  BitextMining:
    icon: "🎌"
    metric: f1
    metric_description: "[F1](https://huggingface.co/spaces/evaluate-metric/f1)"
    task_description: "Bitext mining is the task of finding parallel sentences in two languages."
  Classification:
    icon: "❤️"
    metric: accuracy
    metric_description: "[Accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy)"
    task_description: "Classification is the task of assigning a label to a text."
  Clustering:
    icon: "✨"
    metric: v_measure
    metric_description: "Validity Measure (V-measure)"
    task_description: "Clustering is the task of grouping similar documents together."
  PairClassification:
    icon: "🎭"
    metric: ap
    metric_description: "Average Precision (AP) based on the models similarity metric (usually cosine)"
    task_description: "Pair classification is the task of determining whether two texts are similar."
  Reranking: 
    icon: "🥈"
    metric: map
    metric_description: "Mean Average Precision (MAP)"
    task_description: "Reranking is the task of reordering a list of documents to improve relevance."
  Retrieval:
    icon: "🔎"
    metric: ndcg_at_10
    metric_description: "Normalized Discounted Cumulative Gain @ 10 (nDCG@10)"
    task_description: "Retrieval is the task of finding relevant documents for a query."
  STS:
    icon: "☘️"
    metric: spearman
    metric_description: "Spearman correlation based on the model's similarity metric (usually cosine)"
    task_description: "Semantic Textual Similarity is the task of determining how similar two texts are."
  Summarization:
    icon: "📜"
    metric: spearman
    metric_description: "Spearman correlation based on the model's similarity metric (usually cosine)"
    task_description: "Summarization is the task of generating a summary of a text."
  InstructionRetrieval:
    icon: "🔎📋"
    metric: "p-MRR"
    metric_description: "paired mean reciprocal rank (p-MRR)"
    task_description: "Retrieval w/Instructions is the task of finding relevant documents for a query that has detailed instructions."
boards:
  en:
    title: English
    language_long: "English"
    has_overall: true
    acronym: null
    icon: null
    special_icons: null
    credits: null
    tasks:
      Classification:
        - AmazonCounterfactualClassification (en)
        - AmazonPolarityClassification
        - AmazonReviewsClassification (en)
        - Banking77Classification
        - EmotionClassification
        - ImdbClassification
        - MassiveIntentClassification (en)
        - MassiveScenarioClassification (en)
        - MTOPDomainClassification (en)
        - MTOPIntentClassification (en)
        - ToxicConversationsClassification
        - TweetSentimentExtractionClassification
      Clustering:
        - ArxivClusteringP2P
        - ArxivClusteringS2S
        - BiorxivClusteringP2P
        - BiorxivClusteringS2S
        - MedrxivClusteringP2P
        - MedrxivClusteringS2S
        - RedditClustering
        - RedditClusteringP2P
        - StackExchangeClustering
        - StackExchangeClusteringP2P
        - TwentyNewsgroupsClustering
      PairClassification:
        - SprintDuplicateQuestions
        - TwitterSemEval2015
        - TwitterURLCorpus
      Reranking:
        - AskUbuntuDupQuestions
        - MindSmallReranking
        - SciDocsRR
        - StackOverflowDupQuestions
      Retrieval:
        - ArguAna
        - ClimateFEVER
        - CQADupstackRetrieval
        - DBPedia
        - FEVER
        - FiQA2018
        - HotpotQA
        - MSMARCO
        - NFCorpus
        - NQ
        - QuoraRetrieval
        - SCIDOCS
        - SciFact
        - Touche2020
        - TRECCOVID
      STS:
        - BIOSSES
        - SICK-R
        - STS12
        - STS13
        - STS14
        - STS15
        - STS16
        - STS17 (en-en)
        - STS22 (en)
        - STSBenchmark
      Summarization:
        - SummEval
  en-x:
    title: "English-X"
    language_long: "117 (Pairs of: English & other language)"
    has_overall: false
    acronym: null
    icon: null
    special_icons: null
    credits: null
    tasks:
      BitextMining: ['BUCC (de-en)', 'BUCC (fr-en)', 'BUCC (ru-en)', 'BUCC (zh-en)', 'Tatoeba (afr-eng)', 'Tatoeba (amh-eng)', 'Tatoeba (ang-eng)', 'Tatoeba (ara-eng)', 'Tatoeba (arq-eng)', 'Tatoeba (arz-eng)', 'Tatoeba (ast-eng)', 'Tatoeba (awa-eng)', 'Tatoeba (aze-eng)', 'Tatoeba (bel-eng)', 'Tatoeba (ben-eng)', 'Tatoeba (ber-eng)', 'Tatoeba (bos-eng)', 'Tatoeba (bre-eng)', 'Tatoeba (bul-eng)', 'Tatoeba (cat-eng)', 'Tatoeba (cbk-eng)', 'Tatoeba (ceb-eng)', 'Tatoeba (ces-eng)', 'Tatoeba (cha-eng)', 'Tatoeba (cmn-eng)', 'Tatoeba (cor-eng)', 'Tatoeba (csb-eng)', 'Tatoeba (cym-eng)', 'Tatoeba (dan-eng)', 'Tatoeba (deu-eng)', 'Tatoeba (dsb-eng)', 'Tatoeba (dtp-eng)', 'Tatoeba (ell-eng)', 'Tatoeba (epo-eng)', 'Tatoeba (est-eng)', 'Tatoeba (eus-eng)', 'Tatoeba (fao-eng)', 'Tatoeba (fin-eng)', 'Tatoeba (fra-eng)', 'Tatoeba (fry-eng)', 'Tatoeba (gla-eng)', 'Tatoeba (gle-eng)', 'Tatoeba (glg-eng)', 'Tatoeba (gsw-eng)', 'Tatoeba (heb-eng)', 'Tatoeba (hin-eng)', 'Tatoeba (hrv-eng)', 'Tatoeba (hsb-eng)', 'Tatoeba (hun-eng)', 'Tatoeba (hye-eng)', 'Tatoeba (ido-eng)', 'Tatoeba (ile-eng)', 'Tatoeba (ina-eng)', 'Tatoeba (ind-eng)', 'Tatoeba (isl-eng)', 'Tatoeba (ita-eng)', 'Tatoeba (jav-eng)', 'Tatoeba (jpn-eng)', 'Tatoeba (kab-eng)', 'Tatoeba (kat-eng)', 'Tatoeba (kaz-eng)', 'Tatoeba (khm-eng)', 'Tatoeba (kor-eng)', 'Tatoeba (kur-eng)', 'Tatoeba (kzj-eng)', 'Tatoeba (lat-eng)', 'Tatoeba (lfn-eng)', 'Tatoeba (lit-eng)', 'Tatoeba (lvs-eng)', 'Tatoeba (mal-eng)', 'Tatoeba (mar-eng)', 'Tatoeba (max-eng)', 'Tatoeba (mhr-eng)', 'Tatoeba (mkd-eng)', 'Tatoeba (mon-eng)', 'Tatoeba (nds-eng)', 'Tatoeba (nld-eng)', 'Tatoeba (nno-eng)', 'Tatoeba (nob-eng)', 'Tatoeba (nov-eng)', 'Tatoeba (oci-eng)', 'Tatoeba (orv-eng)', 'Tatoeba (pam-eng)', 'Tatoeba (pes-eng)', 'Tatoeba (pms-eng)', 'Tatoeba (pol-eng)', 'Tatoeba (por-eng)', 'Tatoeba (ron-eng)', 'Tatoeba (rus-eng)', 'Tatoeba (slk-eng)', 'Tatoeba (slv-eng)', 'Tatoeba (spa-eng)', 'Tatoeba (sqi-eng)', 'Tatoeba (srp-eng)', 'Tatoeba (swe-eng)', 'Tatoeba (swg-eng)', 'Tatoeba (swh-eng)', 'Tatoeba (tam-eng)', 'Tatoeba (tat-eng)', 'Tatoeba (tel-eng)', 'Tatoeba (tgl-eng)', 'Tatoeba (tha-eng)', 'Tatoeba (tuk-eng)', 'Tatoeba (tur-eng)', 'Tatoeba (tzl-eng)', 'Tatoeba (uig-eng)', 'Tatoeba (ukr-eng)', 'Tatoeba (urd-eng)', 'Tatoeba (uzb-eng)', 'Tatoeba (vie-eng)', 'Tatoeba (war-eng)', 'Tatoeba (wuu-eng)', 'Tatoeba (xho-eng)', 'Tatoeba (yid-eng)', 'Tatoeba (yue-eng)', 'Tatoeba (zsm-eng)']
  zh:
    title: Chinese
    language_long: Chinese
    has_overall: true
    acronym: C-MTEB
    icon: "🇨🇳"
    special_icons:
      Classification: "🧡"
    credits: "[FlagEmbedding](https://github.com/FlagOpen/FlagEmbedding)"
    tasks:
      Classification:
        - AmazonReviewsClassification (zh)
        - IFlyTek
        - JDReview
        - MassiveIntentClassification (zh-CN)
        - MassiveScenarioClassification (zh-CN)
        - MultilingualSentiment
        - OnlineShopping
        - TNews
        - Waimai
      Clustering:
        - CLSClusteringP2P
        - CLSClusteringS2S
        - ThuNewsClusteringP2P
        - ThuNewsClusteringS2S
      PairClassification:
        - Cmnli
        - Ocnli
      Reranking:
        - CMedQAv1
        - CMedQAv2
        - MMarcoReranking
        - T2Reranking
      Retrieval:
        - CmedqaRetrieval
        - CovidRetrieval
        - DuRetrieval
        - EcomRetrieval
        - MedicalRetrieval
        - MMarcoRetrieval
        - T2Retrieval
        - VideoRetrieval
      STS:
        - AFQMC
        - ATEC
        - BQ
        - LCQMC
        - PAWSX
        - QBQTC
        - STS22 (zh)
        - STSB
  da:
    title: Danish
    language_long: Danish
    has_overall: false
    acronym: null
    icon: "🇩🇰"
    special_icons:
      Classification: "🤍"
    credits: "[Kenneth Enevoldsen](https://github.com/KennethEnevoldsen), [scandinavian-embedding-benchmark](https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/)"
    tasks:
      BitextMining:
        - BornholmBitextMining
      Classification:
        - AngryTweetsClassification
        - DanishPoliticalCommentsClassification
        - DKHateClassification
        - LccSentimentClassification
        - MassiveIntentClassification (da)
        - MassiveScenarioClassification (da)
        - NordicLangClassification
        - ScalaDaClassification
  fr:
    title: French
    language_long: "French"
    has_overall: true
    acronym: "F-MTEB"
    icon: "🇫🇷"
    special_icons:
      Classification: "💙"
    credits: "[Lyon-NLP](https://github.com/Lyon-NLP): [Gabriel Sequeira](https://github.com/GabrielSequeira), [Imene Kerboua](https://github.com/imenelydiaker), [Wissam Siblini](https://github.com/wissam-sib), [Mathieu Ciancone](https://github.com/MathieuCiancone), [Marion Schaeffer](https://github.com/schmarion)"
    tasks:
      Classification:
        - AmazonReviewsClassification (fr)
        - MasakhaNEWSClassification (fra)
        - MassiveIntentClassification (fr)
        - MassiveScenarioClassification (fr)
        - MTOPDomainClassification (fr)
        - MTOPIntentClassification (fr)
      Clustering:
        - AlloProfClusteringP2P
        - AlloProfClusteringS2S
        - HALClusteringS2S
        - MLSUMClusteringP2P (fr)
        - MLSUMClusteringS2S (fr)
        - MasakhaNEWSClusteringP2P (fra)
        - MasakhaNEWSClusteringS2S (fra)
      PairClassification:
        - OpusparcusPC (fr)
        - PawsXPairClassification (fr)
      Reranking:
        - AlloprofReranking
        - SyntecReranking
      Retrieval:
        - AlloprofRetrieval
        - BSARDRetrieval
        - MintakaRetrieval (fr)
        - SyntecRetrieval
        - XPQARetrieval (fr)
      STS:
        - STS22 (fr)
        - STSBenchmarkMultilingualSTS (fr)
        - SICKFr
      Summarization:
        - SummEvalFr
  'no':
    title: Norwegian
    language_long: "Norwegian Bokmål"
    has_overall: false
    acronym: null
    icon: "🇳🇴"
    special_icons:
      Classification: "💙"
    credits: "[Kenneth Enevoldsen](https://github.com/KennethEnevoldsen), [scandinavian-embedding-benchmark](https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/)"
    tasks:
      Classification: &id001
        - NoRecClassification
        - NordicLangClassification
        - NorwegianParliament
        - MassiveIntentClassification (nb)
        - MassiveScenarioClassification (nb)
        - ScalaNbClassification
  instructions:
    title: English
    language_long: "English"
    has_overall: false
    acronym: null
    icon: null
    credits: "[Orion Weller, FollowIR](https://arxiv.org/abs/2403.15246)"
    tasks:
      InstructionRetrieval:
      - Robust04InstructionRetrieval
      - News21InstructionRetrieval
      - Core17InstructionRetrieval
  law:
    title: Law
    language_long: "English, German, Chinese"
    has_overall: false
    acronym: null
    icon: "⚖️"
    special_icons: null
    credits: "[Voyage AI](https://www.voyageai.com/)"
    tasks:
      Retrieval:
        - AILACasedocs
        - AILAStatutes
        - GerDaLIRSmall
        - LeCaRDv2
        - LegalBenchConsumerContractsQA
        - LegalBenchCorporateLobbying
        - LegalQuAD
        - LegalSummarization
  longembed:
    title: LongEmbed
    language_long: "English"
    has_overall: false
    acronym: null
    icon: "📚"
    special_icons: null
    credits: "[LongEmbed](https://arxiv.org/abs/2404.12096v2)"
    metric: nDCG@10 (for NarrativeQA, QMSum, SummScreenFD, WikimQA) & nDCG@1 (for passkey and needle)
    tasks:
      Retrieval:
        - LEMBNarrativeQARetrieval
        - LEMBNeedleRetrieval
        - LEMBPasskeyRetrieval
        - LEMBQMSumRetrieval
        - LEMBSummScreenFDRetrieval
        - LEMBWikimQARetrieval
  de:
    title: German
    language_long: "German"
    has_overall: false
    acronym: null
    icon: "🇩🇪"
    special_icons: null
    credits: "[Silvan](https://github.com/slvnwhrl)"
    tasks:
      Clustering:
      - BlurbsClusteringP2P
      - BlurbsClusteringS2S
      - TenKGnadClusteringP2P
      - TenKGnadClusteringS2S
  pl:
    title: Polish
    language_long: Polish
    has_overall: true
    acronym: null
    icon: "🇵🇱"
    special_icons:
      Classification: "🤍"
    credits: "[Rafał Poświata](https://github.com/rafalposwiata)"
    tasks:
      Classification:
        - AllegroReviews
        - CBD
        - MassiveIntentClassification (pl)
        - MassiveScenarioClassification (pl)
        - PAC
        - PolEmo2.0-IN
        - PolEmo2.0-OUT
      Clustering:
        - 8TagsClustering
      PairClassification:
        - CDSC-E
        - PPC
        - PSC
        - SICK-E-PL
      Retrieval:
        - ArguAna-PL
        - DBPedia-PL
        - FiQA-PL
        - HotpotQA-PL
        - MSMARCO-PL
        - NFCorpus-PL
        - NQ-PL
        - Quora-PL
        - SCIDOCS-PL
        - SciFact-PL
        - TRECCOVID-PL
      STS:
        - CDSC-R
        - SICK-R-PL
        - STS22 (pl)
  se:
    title: Swedish
    language_long: Swedish
    has_overall: false
    acronym: null
    icon: "🇸🇪"
    special_icons:
      Classification: "💛"
    credits: "[Kenneth Enevoldsen](https://github.com/KennethEnevoldsen), [scandinavian-embedding-benchmark](https://kennethenevoldsen.github.io/scandinavian-embedding-benchmark/)"
    tasks:
      Classification:
        - NoRecClassification
        - NordicLangClassification
        - NorwegianParliament
        - MassiveIntentClassification (nb)
        - MassiveScenarioClassification (nb)
        - ScalaNbClassification
  other-cls:
    title: "Other Languages"
    language_long: "47 (Only languages not included in the other tabs)"
    has_overall: false
    acronym: null
    icon: null
    special_icons:
      Classification: "💜💚💙"
    credits: null
    tasks:
      Classification: ['AmazonCounterfactualClassification (de)', 'AmazonCounterfactualClassification (ja)', 'AmazonReviewsClassification (de)', 'AmazonReviewsClassification (es)', 'AmazonReviewsClassification (fr)', 'AmazonReviewsClassification (ja)', 'AmazonReviewsClassification (zh)', 'MTOPDomainClassification (de)', 'MTOPDomainClassification (es)', 'MTOPDomainClassification (fr)', 'MTOPDomainClassification (hi)', 'MTOPDomainClassification (th)', 'MTOPIntentClassification (de)', 'MTOPIntentClassification (es)', 'MTOPIntentClassification (fr)', 'MTOPIntentClassification (hi)', 'MTOPIntentClassification (th)', 'MassiveIntentClassification (af)', 'MassiveIntentClassification (am)', 'MassiveIntentClassification (ar)', 'MassiveIntentClassification (az)', 'MassiveIntentClassification (bn)', 'MassiveIntentClassification (cy)', 'MassiveIntentClassification (de)', 'MassiveIntentClassification (el)', 'MassiveIntentClassification (es)', 'MassiveIntentClassification (fa)', 'MassiveIntentClassification (fi)', 'MassiveIntentClassification (fr)', 'MassiveIntentClassification (he)', 'MassiveIntentClassification (hi)', 'MassiveIntentClassification (hu)', 'MassiveIntentClassification (hy)', 'MassiveIntentClassification (id)', 'MassiveIntentClassification (is)', 'MassiveIntentClassification (it)', 'MassiveIntentClassification (ja)', 'MassiveIntentClassification (jv)', 'MassiveIntentClassification (ka)', 'MassiveIntentClassification (km)', 'MassiveIntentClassification (kn)', 'MassiveIntentClassification (ko)', 'MassiveIntentClassification (lv)', 'MassiveIntentClassification (ml)', 'MassiveIntentClassification (mn)', 'MassiveIntentClassification (ms)', 'MassiveIntentClassification (my)', 'MassiveIntentClassification (nl)', 'MassiveIntentClassification (pt)', 'MassiveIntentClassification (ro)', 'MassiveIntentClassification (ru)', 'MassiveIntentClassification (sl)', 'MassiveIntentClassification (sq)', 'MassiveIntentClassification (sw)', 'MassiveIntentClassification (ta)', 'MassiveIntentClassification (te)', 'MassiveIntentClassification (th)', 'MassiveIntentClassification (tl)', 'MassiveIntentClassification (tr)', 'MassiveIntentClassification (ur)', 'MassiveIntentClassification (vi)', 'MassiveIntentClassification (zh-TW)', 'MassiveScenarioClassification (af)', 'MassiveScenarioClassification (am)', 'MassiveScenarioClassification (ar)', 'MassiveScenarioClassification (az)', 'MassiveScenarioClassification (bn)', 'MassiveScenarioClassification (cy)', 'MassiveScenarioClassification (de)', 'MassiveScenarioClassification (el)', 'MassiveScenarioClassification (es)', 'MassiveScenarioClassification (fa)', 'MassiveScenarioClassification (fi)', 'MassiveScenarioClassification (fr)', 'MassiveScenarioClassification (he)', 'MassiveScenarioClassification (hi)', 'MassiveScenarioClassification (hu)', 'MassiveScenarioClassification (hy)', 'MassiveScenarioClassification (id)', 'MassiveScenarioClassification (is)', 'MassiveScenarioClassification (it)', 'MassiveScenarioClassification (ja)', 'MassiveScenarioClassification (jv)', 'MassiveScenarioClassification (ka)', 'MassiveScenarioClassification (km)', 'MassiveScenarioClassification (kn)', 'MassiveScenarioClassification (ko)', 'MassiveScenarioClassification (lv)', 'MassiveScenarioClassification (ml)', 'MassiveScenarioClassification (mn)', 'MassiveScenarioClassification (ms)', 'MassiveScenarioClassification (my)', 'MassiveScenarioClassification (nl)', 'MassiveScenarioClassification (pt)', 'MassiveScenarioClassification (ro)', 'MassiveScenarioClassification (ru)', 'MassiveScenarioClassification (sl)', 'MassiveScenarioClassification (sq)', 'MassiveScenarioClassification (sw)', 'MassiveScenarioClassification (ta)', 'MassiveScenarioClassification (te)', 'MassiveScenarioClassification (th)', 'MassiveScenarioClassification (tl)', 'MassiveScenarioClassification (tr)', 'MassiveScenarioClassification (ur)', 'MassiveScenarioClassification (vi)', 'MassiveScenarioClassification (zh-TW)']
  other-sts:
    title: Other
    language_long: "Arabic, Chinese, Dutch, English, French, German, Italian, Korean, Polish, Russian, Spanish (Only language combos not included in the other tabs)"
    has_overall: false
    acronym: null
    icon: null
    special_icons: null
    credits: null
    tasks:
      STS: ["STS17 (ar-ar)", "STS17 (en-ar)", "STS17 (en-de)", "STS17 (en-tr)", "STS17 (es-en)", "STS17 (es-es)", "STS17 (fr-en)", "STS17 (it-en)", "STS17 (ko-ko)", "STS17 (nl-en)", "STS22 (ar)", "STS22 (de)", "STS22 (de-en)", "STS22 (de-fr)", "STS22 (de-pl)", "STS22 (es)", "STS22 (es-en)", "STS22 (es-it)", "STS22 (fr)", "STS22 (fr-pl)", "STS22 (it)", "STS22 (pl)", "STS22 (pl-en)", "STS22 (ru)", "STS22 (tr)", "STS22 (zh-en)", "STSBenchmark"]
  rar-b:
    title: RAR-b
    language_long: "English"
    has_overall: false
    acronym: null
    icon: "📚"
    special_icons: null
    credits: "[RAR-b (Xiao et al.)](https://arxiv.org/abs/2404.06347/)"
    metric: nDCG@10
    tasks:
      Retrieval:
        - ARCChallenge
        - AlphaNLI
        - HellaSwag
        - PIQA
        - Quail
        - RARbCode
        - RARbMath
        - SIQA
        - SpartQA
        - TempReasonL1
        - TempReasonL2Fact
        - TempReasonL2Pure
        - TempReasonL3Fact
        - TempReasonL3Pure
        - WinoGrande
  bright:
    title: BRIGHT
    language_long: "English"
    has_overall: false
    acronym: null
    icon: "🌟"
    special_icons: null
    credits: "[BRIGHT (Hongjin Su, Howard Yen, Mengzhou Xia et al.)](https://brightbenchmark.github.io/)"
    metric: nDCG@10
    tasks:
      Retrieval:
        - BrightRetrieval (biology)
        - BrightRetrieval (earth_science)
        - BrightRetrieval (economics)
        - BrightRetrieval (psychology)
        - BrightRetrieval (robotics)
        - BrightRetrieval (stackoverflow)
        - BrightRetrieval (sustainable_living)
        - BrightRetrieval (pony)
        - BrightRetrieval (leetcode)
        - BrightRetrieval (aops)
        - BrightRetrieval (theoremqa_theorems)
        - BrightRetrieval (theoremqa_questions)