| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "override_batch_size": -1, | |
| "max_samples": null, | |
| "job_id": "", | |
| "start_time": 1541.919794194, | |
| "end_time": 2653.777581735, | |
| "total_evaluation_time_secondes": "1111.8577875409999", | |
| "model_name": "tinycompany/BiBo-Mini-v0.99", | |
| "model_sha": "9e66d1135c27c915ecf382453909e5bc1f6a650d", | |
| "model_dtype": "torch.bfloat16", | |
| "model_size": "3.31 GB", | |
| "config": null | |
| }, | |
| "results": { | |
| "indiceval|Boolq:hindi|5": { | |
| "acc": 0.6678899082568808, | |
| "acc_stderr": 0.00823732150008232 | |
| }, | |
| "all": { | |
| "acc": 0.6678899082568808, | |
| "acc_stderr": 0.00823732150008232 | |
| } | |
| }, | |
| "versions": { | |
| "indiceval|Boolq:hindi|5": 0 | |
| }, | |
| "config_tasks": { | |
| "indiceval|Boolq:hindi": { | |
| "name": "Boolq:hindi", | |
| "prompt_function": "boolq_harness_indic", | |
| "hf_repo": "Cognitive-Lab/Indic-BoolQ", | |
| "hf_subset": "hi", | |
| "metric": [ | |
| "loglikelihood_acc" | |
| ], | |
| "hf_avail_splits": [ | |
| "train", | |
| "validation" | |
| ], | |
| "evaluation_splits": [ | |
| "validation" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": null, | |
| "generation_size": -1, | |
| "stop_sequence": [ | |
| "\n" | |
| ], | |
| "output_regex": null, | |
| "frozen": false, | |
| "suite": [ | |
| "indiceval", | |
| "leaderboard", | |
| "superglue" | |
| ], | |
| "original_num_docs": 3270, | |
| "effective_num_docs": 3270, | |
| "trust_dataset": true, | |
| "must_remove_duplicate_docs": null | |
| } | |
| }, | |
| "summary_tasks": { | |
| "indiceval|Boolq:hindi|5": { | |
| "hashes": { | |
| "hash_examples": "1a2035c16478d97f", | |
| "hash_full_prompts": "6a700dcb36e00984", | |
| "hash_input_tokens": "fb4c967f34bf3bc1", | |
| "hash_cont_tokens": "a329388f38464b5f" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 3270, | |
| "padded": 6516, | |
| "non_padded": 24, | |
| "effective_few_shots": 5.0, | |
| "num_truncated_few_shots": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "fa93795de2d55c79", | |
| "hash_full_prompts": "ba6a7917f7ec4138", | |
| "hash_input_tokens": "acc16b51fd4266e9", | |
| "hash_cont_tokens": "3aa1443bfadab8ee" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 3270, | |
| "padded": 6516, | |
| "non_padded": 24, | |
| "num_truncated_few_shots": 0 | |
| }, | |
| "email": "[email protected]", | |
| "language": "hindi" | |
| } |