Nathan Habib
commited on
Commit
•
4aff44e
1
Parent(s):
2f6ebf5
fix new config name
Browse files
src/auto_leaderboard/load_results.py
CHANGED
@@ -2,6 +2,7 @@ from dataclasses import dataclass
|
|
2 |
|
3 |
import glob
|
4 |
import json
|
|
|
5 |
from typing import Dict, List, Tuple
|
6 |
|
7 |
from src.utils_display import AutoEvalColumn, make_clickable_model
|
@@ -55,12 +56,16 @@ class EvalResult:
|
|
55 |
def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
|
56 |
with open(json_filepath) as fp:
|
57 |
data = json.load(fp)
|
|
|
58 |
|
59 |
for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
|
60 |
if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
|
61 |
return None, [] # we skip models with the wrong version
|
62 |
|
63 |
-
|
|
|
|
|
|
|
64 |
model = config.get("model_name", None)
|
65 |
if model is None:
|
66 |
model = config.get("model_args", None)
|
|
|
2 |
|
3 |
import glob
|
4 |
import json
|
5 |
+
import os
|
6 |
from typing import Dict, List, Tuple
|
7 |
|
8 |
from src.utils_display import AutoEvalColumn, make_clickable_model
|
|
|
56 |
def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
|
57 |
with open(json_filepath) as fp:
|
58 |
data = json.load(fp)
|
59 |
+
|
60 |
|
61 |
for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
|
62 |
if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
|
63 |
return None, [] # we skip models with the wrong version
|
64 |
|
65 |
+
try:
|
66 |
+
config = data["config"]
|
67 |
+
except KeyError:
|
68 |
+
config = data["config_general"]
|
69 |
model = config.get("model_name", None)
|
70 |
if model is None:
|
71 |
model = config.get("model_args", None)
|