Update README.md

README.md

@@ -2,8 +2,6 @@
 license: apache-2.0
 tags:
 - merge
-- OpenPipe/mistral-ft-optimized-1218
-- mlabonne/NeuralHermes-2.5-Mistral-7B
 ---
 
 # mistral-7b-merged-slerp
@@ -34,391 +32,39 @@ dtype: bfloat16
 
 ```
 
-## Evaluation
-https://huggingface.co/datasets/open-llm-leaderboard/details_mychen76__mistral-7b-merged-slerp
+## 💻 Usage
+```python
+!pip install -qU transformers bitsandbytes accelerate
 
-Latest Result:
-https://huggingface.co/datasets/open-llm-leaderboard/details_mychen76__mistral-7b-merged-slerp/blob/main/results_2024-03-10T11-04-57.263703.json
+from transformers import AutoTokenizer
+import transformers
+import torch
 
+model = "mychen76/mistral-7b-merged-slerp"
+
+tokenizer = AutoTokenizer.from_pretrained(model)
+pipeline = transformers.pipeline(
+ "text-generation",
+ model=model,
+ model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
+)
+
+messages = [{"role": "user", "content": "why the sky is blue."}]
+prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
 ```
-{
- "all": {
- "acc": 0.6444688446653744,
- "acc_stderr": 0.03217564834975917,
- "acc_norm": 0.6448609553287138,
- "acc_norm_stderr": 0.032833467276313325,
- "mc1": 0.4283965728274174,
- "mc1_stderr": 0.017323088597314754,
- "mc2": 0.5985018412437423,
- "mc2_stderr": 0.01514980059720055
- },
- "harness|arc:challenge|25": {
- "acc": 0.6476109215017065,
- "acc_stderr": 0.013960142600598675,
- "acc_norm": 0.6774744027303754,
- "acc_norm_stderr": 0.013659980894277364
- },
- "harness|hellaswag|10": {
- "acc": 0.6700856403106951,
- "acc_stderr": 0.004692208279690595,
- "acc_norm": 0.8616809400517825,
- "acc_norm_stderr": 0.0034452899250117337
- },
- "harness|hendrycksTest-abstract_algebra|5": {
- "acc": 0.3,
- "acc_stderr": 0.046056618647183814,
- "acc_norm": 0.3,
- "acc_norm_stderr": 0.046056618647183814
- },
- "harness|hendrycksTest-anatomy|5": {
- "acc": 0.6074074074074074,
- "acc_stderr": 0.0421850621536888,
- "acc_norm": 0.6074074074074074,
- "acc_norm_stderr": 0.0421850621536888
- },
- "harness|hendrycksTest-astronomy|5": {
- "acc": 0.7105263157894737,
- "acc_stderr": 0.03690677986137283,
- "acc_norm": 0.7105263157894737,
- "acc_norm_stderr": 0.03690677986137283
- },
- "harness|hendrycksTest-business_ethics|5": {
- "acc": 0.61,
- "acc_stderr": 0.04902071300001975,
- "acc_norm": 0.61,
- "acc_norm_stderr": 0.04902071300001975
- },
- "harness|hendrycksTest-clinical_knowledge|5": {
- "acc": 0.6830188679245283,
- "acc_stderr": 0.02863723563980089,
- "acc_norm": 0.6830188679245283,
- "acc_norm_stderr": 0.02863723563980089
- },
- "harness|hendrycksTest-college_biology|5": {
- "acc": 0.7777777777777778,
- "acc_stderr": 0.03476590104304134,
- "acc_norm": 0.7777777777777778,
- "acc_norm_stderr": 0.03476590104304134
- },
- "harness|hendrycksTest-college_chemistry|5": {
- "acc": 0.45,
- "acc_stderr": 0.05,
- "acc_norm": 0.45,
- "acc_norm_stderr": 0.05
- },
- "harness|hendrycksTest-college_computer_science|5": {
- "acc": 0.49,
- "acc_stderr": 0.05024183937956912,
- "acc_norm": 0.49,
- "acc_norm_stderr": 0.05024183937956912
- },
- "harness|hendrycksTest-college_mathematics|5": {
- "acc": 0.29,
- "acc_stderr": 0.045604802157206845,
- "acc_norm": 0.29,
- "acc_norm_stderr": 0.045604802157206845
- },
- "harness|hendrycksTest-college_medicine|5": {
- "acc": 0.6473988439306358,
- "acc_stderr": 0.036430371689585475,
- "acc_norm": 0.6473988439306358,
- "acc_norm_stderr": 0.036430371689585475
- },
- "harness|hendrycksTest-college_physics|5": {
- "acc": 0.39215686274509803,
- "acc_stderr": 0.04858083574266345,
- "acc_norm": 0.39215686274509803,
- "acc_norm_stderr": 0.04858083574266345
- },
- "harness|hendrycksTest-computer_security|5": {
- "acc": 0.74,
- "acc_stderr": 0.04408440022768078,
- "acc_norm": 0.74,
- "acc_norm_stderr": 0.04408440022768078
- },
- "harness|hendrycksTest-conceptual_physics|5": {
- "acc": 0.5829787234042553,
- "acc_stderr": 0.03223276266711712,
- "acc_norm": 0.5829787234042553,
- "acc_norm_stderr": 0.03223276266711712
- },
- "harness|hendrycksTest-econometrics|5": {
- "acc": 0.5,
- "acc_stderr": 0.047036043419179864,
- "acc_norm": 0.5,
- "acc_norm_stderr": 0.047036043419179864
- },
- "harness|hendrycksTest-electrical_engineering|5": {
- "acc": 0.5448275862068965,
- "acc_stderr": 0.04149886942192117,
- "acc_norm": 0.5448275862068965,
- "acc_norm_stderr": 0.04149886942192117
- },
- "harness|hendrycksTest-elementary_mathematics|5": {
- "acc": 0.41534391534391535,
- "acc_stderr": 0.025379524910778405,
- "acc_norm": 0.41534391534391535,
- "acc_norm_stderr": 0.025379524910778405
- },
- "harness|hendrycksTest-formal_logic|5": {
- "acc": 0.4603174603174603,
- "acc_stderr": 0.04458029125470973,
- "acc_norm": 0.4603174603174603,
- "acc_norm_stderr": 0.04458029125470973
- },
- "harness|hendrycksTest-global_facts|5": {
- "acc": 0.38,
- "acc_stderr": 0.048783173121456316,
- "acc_norm": 0.38,
- "acc_norm_stderr": 0.048783173121456316
- },
- "harness|hendrycksTest-high_school_biology|5": {
- "acc": 0.7774193548387097,
- "acc_stderr": 0.023664216671642518,
- "acc_norm": 0.7774193548387097,
- "acc_norm_stderr": 0.023664216671642518
- },
- "harness|hendrycksTest-high_school_chemistry|5": {
- "acc": 0.5073891625615764,
- "acc_stderr": 0.035176035403610105,
- "acc_norm": 0.5073891625615764,
- "acc_norm_stderr": 0.035176035403610105
- },
- "harness|hendrycksTest-high_school_computer_science|5": {
- "acc": 0.69,
- "acc_stderr": 0.04648231987117316,
- "acc_norm": 0.69,
- "acc_norm_stderr": 0.04648231987117316
- },
- "harness|hendrycksTest-high_school_european_history|5": {
- "acc": 0.7696969696969697,
- "acc_stderr": 0.0328766675860349,
- "acc_norm": 0.7696969696969697,
- "acc_norm_stderr": 0.0328766675860349
- },
- "harness|hendrycksTest-high_school_geography|5": {
- "acc": 0.7878787878787878,
- "acc_stderr": 0.029126522834586818,
- "acc_norm": 0.7878787878787878,
- "acc_norm_stderr": 0.029126522834586818
- },
- "harness|hendrycksTest-high_school_government_and_politics|5": {
- "acc": 0.9015544041450777,
- "acc_stderr": 0.02150024957603346,
- "acc_norm": 0.9015544041450777,
- "acc_norm_stderr": 0.02150024957603346
- },
- "harness|hendrycksTest-high_school_macroeconomics|5": {
- "acc": 0.6538461538461539,
- "acc_stderr": 0.02412112541694119,
- "acc_norm": 0.6538461538461539,
- "acc_norm_stderr": 0.02412112541694119
- },
- "harness|hendrycksTest-high_school_mathematics|5": {
- "acc": 0.32222222222222224,
- "acc_stderr": 0.028493465091028593,
- "acc_norm": 0.32222222222222224,
- "acc_norm_stderr": 0.028493465091028593
- },
- "harness|hendrycksTest-high_school_microeconomics|5": {
- "acc": 0.6890756302521008,
- "acc_stderr": 0.03006676158297793,
- "acc_norm": 0.6890756302521008,
- "acc_norm_stderr": 0.03006676158297793
- },
- "harness|hendrycksTest-high_school_physics|5": {
- "acc": 0.32450331125827814,
- "acc_stderr": 0.03822746937658752,
- "acc_norm": 0.32450331125827814,
- "acc_norm_stderr": 0.03822746937658752
- },
- "harness|hendrycksTest-high_school_psychology|5": {
- "acc": 0.8550458715596331,
- "acc_stderr": 0.01509421569970048,
- "acc_norm": 0.8550458715596331,
- "acc_norm_stderr": 0.01509421569970048
- },
- "harness|hendrycksTest-high_school_statistics|5": {
- "acc": 0.5185185185185185,
- "acc_stderr": 0.034076320938540516,
- "acc_norm": 0.5185185185185185,
- "acc_norm_stderr": 0.034076320938540516
- },
- "harness|hendrycksTest-high_school_us_history|5": {
- "acc": 0.8186274509803921,
- "acc_stderr": 0.027044621719474082,
- "acc_norm": 0.8186274509803921,
- "acc_norm_stderr": 0.027044621719474082
- },
- "harness|hendrycksTest-high_school_world_history|5": {
- "acc": 0.8059071729957806,
- "acc_stderr": 0.0257449025322909,
- "acc_norm": 0.8059071729957806,
- "acc_norm_stderr": 0.0257449025322909
- },
- "harness|hendrycksTest-human_aging|5": {
- "acc": 0.6905829596412556,
- "acc_stderr": 0.03102441174057221,
- "acc_norm": 0.6905829596412556,
- "acc_norm_stderr": 0.03102441174057221
- },
- "harness|hendrycksTest-human_sexuality|5": {
- "acc": 0.7786259541984732,
- "acc_stderr": 0.03641297081313729,
- "acc_norm": 0.7786259541984732,
- "acc_norm_stderr": 0.03641297081313729
- },
- "harness|hendrycksTest-international_law|5": {
- "acc": 0.8099173553719008,
- "acc_stderr": 0.03581796951709282,
- "acc_norm": 0.8099173553719008,
- "acc_norm_stderr": 0.03581796951709282
- },
- "harness|hendrycksTest-jurisprudence|5": {
- "acc": 0.7685185185185185,
- "acc_stderr": 0.04077494709252626,
- "acc_norm": 0.7685185185185185,
- "acc_norm_stderr": 0.04077494709252626
- },
- "harness|hendrycksTest-logical_fallacies|5": {
- "acc": 0.7730061349693251,
- "acc_stderr": 0.03291099578615769,
- "acc_norm": 0.7730061349693251,
- "acc_norm_stderr": 0.03291099578615769
- },
- "harness|hendrycksTest-machine_learning|5": {
- "acc": 0.4642857142857143,
- "acc_stderr": 0.04733667890053756,
- "acc_norm": 0.4642857142857143,
- "acc_norm_stderr": 0.04733667890053756
- },
- "harness|hendrycksTest-management|5": {
- "acc": 0.7572815533980582,
- "acc_stderr": 0.04245022486384495,
- "acc_norm": 0.7572815533980582,
- "acc_norm_stderr": 0.04245022486384495
- },
- "harness|hendrycksTest-marketing|5": {
- "acc": 0.8547008547008547,
- "acc_stderr": 0.023086635086841407,
- "acc_norm": 0.8547008547008547,
- "acc_norm_stderr": 0.023086635086841407
- },
- "harness|hendrycksTest-medical_genetics|5": {
- "acc": 0.7,
- "acc_stderr": 0.046056618647183814,
- "acc_norm": 0.7,
- "acc_norm_stderr": 0.046056618647183814
- },
- "harness|hendrycksTest-miscellaneous|5": {
- "acc": 0.8352490421455939,
- "acc_stderr": 0.013265346261323793,
- "acc_norm": 0.8352490421455939,
- "acc_norm_stderr": 0.013265346261323793
- },
- "harness|hendrycksTest-moral_disputes|5": {
- "acc": 0.7283236994219653,
- "acc_stderr": 0.023948512905468365,
- "acc_norm": 0.7283236994219653,
- "acc_norm_stderr": 0.023948512905468365
- },
- "harness|hendrycksTest-moral_scenarios|5": {
- "acc": 0.36312849162011174,
- "acc_stderr": 0.016083749986853697,
- "acc_norm": 0.36312849162011174,
- "acc_norm_stderr": 0.016083749986853697
- },
- "harness|hendrycksTest-nutrition|5": {
- "acc": 0.7450980392156863,
- "acc_stderr": 0.02495418432487991,
- "acc_norm": 0.7450980392156863,
- "acc_norm_stderr": 0.02495418432487991
- },
- "harness|hendrycksTest-philosophy|5": {
- "acc": 0.7106109324758842,
- "acc_stderr": 0.025755865922632945,
- "acc_norm": 0.7106109324758842,
- "acc_norm_stderr": 0.025755865922632945
- },
- "harness|hendrycksTest-prehistory|5": {
- "acc": 0.7469135802469136,
- "acc_stderr": 0.024191808600712995,
- "acc_norm": 0.7469135802469136,
- "acc_norm_stderr": 0.024191808600712995
- },
- "harness|hendrycksTest-professional_accounting|5": {
- "acc": 0.4787234042553192,
- "acc_stderr": 0.029800481645628693,
- "acc_norm": 0.4787234042553192,
- "acc_norm_stderr": 0.029800481645628693
- },
- "harness|hendrycksTest-professional_law|5": {
- "acc": 0.4726205997392438,
- "acc_stderr": 0.012751075788015058,
- "acc_norm": 0.4726205997392438,
- "acc_norm_stderr": 0.012751075788015058
- },
- "harness|hendrycksTest-professional_medicine|5": {
- "acc": 0.6948529411764706,
- "acc_stderr": 0.027971541370170598,
- "acc_norm": 0.6948529411764706,
- "acc_norm_stderr": 0.027971541370170598
- },
- "harness|hendrycksTest-professional_psychology|5": {
- "acc": 0.6748366013071896,
- "acc_stderr": 0.01895088677080631,
- "acc_norm": 0.6748366013071896,
- "acc_norm_stderr": 0.01895088677080631
- },
- "harness|hendrycksTest-public_relations|5": {
- "acc": 0.6636363636363637,
- "acc_stderr": 0.04525393596302506,
- "acc_norm": 0.6636363636363637,
- "acc_norm_stderr": 0.04525393596302506
- },
- "harness|hendrycksTest-security_studies|5": {
- "acc": 0.746938775510204,
- "acc_stderr": 0.027833023871399673,
- "acc_norm": 0.746938775510204,
- "acc_norm_stderr": 0.027833023871399673
- },
- "harness|hendrycksTest-sociology|5": {
- "acc": 0.835820895522388,
- "acc_stderr": 0.026193923544454115,
- "acc_norm": 0.835820895522388,
- "acc_norm_stderr": 0.026193923544454115
- },
- "harness|hendrycksTest-us_foreign_policy|5": {
- "acc": 0.85,
- "acc_stderr": 0.0358870281282637,
- "acc_norm": 0.85,
- "acc_norm_stderr": 0.0358870281282637
- },
- "harness|hendrycksTest-virology|5": {
- "acc": 0.5301204819277109,
- "acc_stderr": 0.03885425420866767,
- "acc_norm": 0.5301204819277109,
- "acc_norm_stderr": 0.03885425420866767
- },
- "harness|hendrycksTest-world_religions|5": {
- "acc": 0.8304093567251462,
- "acc_stderr": 0.02878210810540171,
- "acc_norm": 0.8304093567251462,
- "acc_norm_stderr": 0.02878210810540171
- },
- "harness|truthfulqa:mc|0": {
- "mc1": 0.4283965728274174,
- "mc1_stderr": 0.017323088597314754,
- "mc2": 0.5985018412437423,
- "mc2_stderr": 0.01514980059720055
- },
- "harness|winogrande|5": {
- "acc": 0.8018942383583267,
- "acc_stderr": 0.01120186274448705
- },
- "harness|gsm8k|5": {
- "acc": 0.6853677028051555,
- "acc_stderr": 0.01279103722733604
- }
-}
-```
+
+# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
+Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_mychen76__mistral-7b-merged-slerp)
+
+
+| Metric |Value|
+|---------------------------------|----:|
+|Avg. |71.09|
+|AI2 Reasoning Challenge (25-Shot)|67.75|
+|HellaSwag (10-Shot) |86.17|
+|MMLU (5-Shot) |64.05|
+|TruthfulQA (0-shot) |59.85|
+|Winogrande (5-shot) |80.19|
+|GSM8k (5-shot) |68.54|