Spaces:
Runtime error
Runtime error
| { | |
| "config_general": { | |
| "lighteval_sha": "167773f1d5d1647c60dadc31c9e731ab7dbcbbad", | |
| "num_few_shot_default": 0, | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": 1, | |
| "max_samples": null, | |
| "job_id": "", | |
| "model_name": "Undi95/MM-ReMM-L2-20B", | |
| "model_sha": "37869800c15fb37d017ea83bb50fec6d6141f6ba", | |
| "model_dtype": "torch.float16", | |
| "model_size": "37.36 GB" | |
| }, | |
| "results": { | |
| "harness|drop|3": { | |
| "em": 0.10843120805369127, | |
| "em_stderr": 0.0031841552068159317, | |
| "f1": 0.18655725671140816, | |
| "f1_stderr": 0.003391655697712374 | |
| }, | |
| "harness|gsm8k|5": { | |
| "acc": 0.07733131159969674, | |
| "acc_stderr": 0.00735771352322235 | |
| }, | |
| "harness|winogrande|5": { | |
| "acc": 0.7576953433307024, | |
| "acc_stderr": 0.012042352526174785 | |
| }, | |
| "all": { | |
| "em": 0.10843120805369127, | |
| "em_stderr": 0.0031841552068159317, | |
| "f1": 0.18655725671140816, | |
| "f1_stderr": 0.003391655697712374, | |
| "acc": 0.4175133274651996, | |
| "acc_stderr": 0.009700033024698568 | |
| } | |
| }, | |
| "versions": { | |
| "all": 0, | |
| "harness|drop|3": 1, | |
| "harness|gsm8k|5": 0, | |
| "harness|winogrande|5": 0 | |
| }, | |
| "config_tasks": { | |
| "harness|drop": "LM Harness task", | |
| "harness|gsm8k": "LM Harness task", | |
| "harness|winogrande": "LM Harness task" | |
| }, | |
| "summary_tasks": { | |
| "harness|drop|3": { | |
| "hashes": { | |
| "hash_examples": "1d27416e8324e9a3", | |
| "hash_full_prompts": "a5513ff9a741b385", | |
| "hash_input_tokens": "42076f0efbb50aa6", | |
| "hash_cont_tokens": "d772779ca662f4c1" | |
| }, | |
| "truncated": 3, | |
| "non_truncated": 9533, | |
| "padded": 0, | |
| "non_padded": 9536, | |
| "effective_few_shots": 3.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|gsm8k|5": { | |
| "hashes": { | |
| "hash_examples": "4c0843a5d99bcfdc", | |
| "hash_full_prompts": "41d55e83abc0e02d", | |
| "hash_input_tokens": "bda342e47b5099b2", | |
| "hash_cont_tokens": "5ad68bb8d2bab381" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1319, | |
| "padded": 0, | |
| "non_padded": 1319, | |
| "effective_few_shots": 5.0, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "harness|winogrande|5": { | |
| "hashes": { | |
| "hash_examples": "aada0a176fd81218", | |
| "hash_full_prompts": "c8655cbd12de8409", | |
| "hash_input_tokens": "c0bedf98cb040854", | |
| "hash_cont_tokens": "f08975ad6f2d5864" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 1267, | |
| "padded": 2432, | |
| "non_padded": 102, | |
| "effective_few_shots": 5.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "9b4d8993161e637d", | |
| "hash_full_prompts": "08215e527b7e60a5", | |
| "hash_input_tokens": "a12f3e3c934bd78b", | |
| "hash_cont_tokens": "8d80264197b98513" | |
| }, | |
| "truncated": 3, | |
| "non_truncated": 12119, | |
| "padded": 2432, | |
| "non_padded": 10957, | |
| "num_truncated_few_shots": 0, | |
| "total_evaluation_time_secondes": 0 | |
| } | |
| } |