Adding the Open Portuguese LLM Leaderboard Evaluation Results

README.md

@@ -1,16 +1,163 @@
 ---
+language:
+- pt
+license: apache-2.0
 library_name: transformers
 tags:
 - mixtral
 - portuguese
 - portugues
-license: apache-2.0
+base_model: mistralai/Mixtral-8x7B-Instruct-v0.1
 datasets:
 - rhaymison/superset
-language:
-- pt
 pipeline_tag: text-generation
-base_model: mistralai/Mixtral-8x7B-Instruct-v0.1
+model-index:
+- name: Mistral-8x7b-portuguese-luana
+ results:
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: ENEM Challenge (No Images)
+ type: eduagarcia/enem_challenge
+ split: train
+ args:
+ num_few_shot: 3
+ metrics:
+ - type: acc
+ value: 69.63
+ name: accuracy
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: BLUEX (No Images)
+ type: eduagarcia-temp/BLUEX_without_images
+ split: train
+ args:
+ num_few_shot: 3
+ metrics:
+ - type: acc
+ value: 59.11
+ name: accuracy
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: OAB Exams
+ type: eduagarcia/oab_exams
+ split: train
+ args:
+ num_few_shot: 3
+ metrics:
+ - type: acc
+ value: 49.61
+ name: accuracy
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: Assin2 RTE
+ type: assin2
+ split: test
+ args:
+ num_few_shot: 15
+ metrics:
+ - type: f1_macro
+ value: 61.21
+ name: f1-macro
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: Assin2 STS
+ type: eduagarcia/portuguese_benchmark
+ split: test
+ args:
+ num_few_shot: 15
+ metrics:
+ - type: pearson
+ value: 79.95
+ name: pearson
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: FaQuAD NLI
+ type: ruanchaves/faquad-nli
+ split: test
+ args:
+ num_few_shot: 15
+ metrics:
+ - type: f1_macro
+ value: 78.6
+ name: f1-macro
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: HateBR Binary
+ type: ruanchaves/hatebr
+ split: test
+ args:
+ num_few_shot: 25
+ metrics:
+ - type: f1_macro
+ value: 72.42
+ name: f1-macro
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: PT Hate Speech Binary
+ type: hate_speech_portuguese
+ split: test
+ args:
+ num_few_shot: 25
+ metrics:
+ - type: f1_macro
+ value: 73.01
+ name: f1-macro
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
+ - task:
+ type: text-generation
+ name: Text Generation
+ dataset:
+ name: tweetSentBR
+ type: eduagarcia/tweetsentbr_fewshot
+ split: test
+ args:
+ num_few_shot: 25
+ metrics:
+ - type: f1_macro
+ value: 50.9
+ name: f1-macro
+ source:
+ url: https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query=rhaymison/Mistral-8x7b-portuguese-luana
+ name: Open Portuguese LLM Leaderboard
 ---
 
 # Mistral-8x7b-portuguese-luana
@@ -118,3 +265,22 @@ email: [email protected]
  <img src="https://img.shields.io/badge/GitHub-100000?style=for-the-badge&logo=github&logoColor=white">
  </a>
  </div>
+
+
+# Open Portuguese LLM Leaderboard Evaluation Results 
+
+Detailed results can be found [here](https://huggingface.co/datasets/eduagarcia-temp/llm_pt_leaderboard_raw_results/tree/main/rhaymison/Mistral-8x7b-portuguese-luana) and on the [🚀 Open Portuguese LLM Leaderboard](https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard)
+
+| Metric | Value |
+|--------------------------|---------|
+|Average |**66.05**|
+|ENEM Challenge (No Images)| 69.63|
+|BLUEX (No Images) | 59.11|
+|OAB Exams | 49.61|
+|Assin2 RTE | 61.21|
+|Assin2 STS | 79.95|
+|FaQuAD NLI | 78.60|
+|HateBR Binary | 72.42|
+|PT Hate Speech Binary | 73.01|
+|tweetSentBR | 50.90|
+