Spaces:
Runtime error
Runtime error
File size: 1,940 Bytes
00a16d9 f8b127b 00a16d9 f8b127b 00a16d9 f8b127b 00a16d9 f8b127b 00a16d9 f8b127b 00a16d9 b40b011 4d33ca7 00a16d9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
import json
import math
import pandas as pd
from src.assets.symbols import UP_ARROW, DOWN_ARROW
from src.tasks import TASKS
def load_from_hub(fs, repo_path, is_private=False):
files = fs.glob(f"{repo_path}/**/*.json")
set_organization_models = {}
tasks = {}
for file in files:
organization, model, task = file.split("/")[-3:]
organization_model = f"{organization}/{model}"
task_code = task.replace(".json", "")
if task_code not in map(lambda task: task.code, TASKS):
continue
if is_private != list(filter(lambda task: task.code == task_code, TASKS))[0].private_test:
continue
set_organization_models[organization_model] = 1
tasks[task_code] = 1
table = pd.DataFrame(
index=list(set_organization_models.keys()),
columns=["Organization", "Model"] + list(tasks.keys()),
data=None,
)
for file in files:
organization, model, task = file.split("/")[-3:]
organization_model = f"{organization}/{model}"
task_code = task.replace(".json", "")
if task_code not in map(lambda task: task.code, TASKS):
continue
if is_private != list(filter(lambda task: task.code == task_code, TASKS))[0].private_test:
continue
data = json.loads(fs.open(file, "r").read())
metric = list(filter(lambda task: task.code == task_code, TASKS))[0].metric
result = round(data["results"][task_code][metric], 4)
table.loc[organization_model, task_code] = result
table.loc[organization_model, "Organization"] = organization
table.loc[organization_model, "Model"] = model
table.rename(columns={
task.code: f"{task.name} {UP_ARROW if task.higher_is_better else DOWN_ARROW}"
for task in TASKS}, inplace=True)
table = table[~table["Organization"].str.contains("Viet-Mistral")]
return table |