Clémentine
commited on
Commit
•
a40c960
1
Parent(s):
699e8ff
removed need for tokens in the leaderboard + removed skull in flagged models
Browse files- src/auto_leaderboard/get_model_metadata.py +0 -3
- src/init.py +17 -24
src/auto_leaderboard/get_model_metadata.py
CHANGED
@@ -100,13 +100,10 @@ def get_model_type(leaderboard_data: List[dict]):
|
|
100 |
model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
|
101 |
|
102 |
def flag_models(leaderboard_data:List[dict]):
|
103 |
-
flag_symbol = "💀"
|
104 |
for model_data in leaderboard_data:
|
105 |
if model_data["model_name_for_query"] in FLAGGED_MODELS:
|
106 |
issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
|
107 |
issue_link = model_hyperlink(FLAGGED_MODELS[model_data["model_name_for_query"]], f"See discussion #{issue_num}")
|
108 |
-
|
109 |
-
model_data[AutoEvalColumn.model_type_symbol.name] = flag_symbol
|
110 |
model_data[AutoEvalColumn.model.name] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
|
111 |
|
112 |
def apply_metadata(leaderboard_data: List[dict]):
|
|
|
100 |
model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
|
101 |
|
102 |
def flag_models(leaderboard_data:List[dict]):
|
|
|
103 |
for model_data in leaderboard_data:
|
104 |
if model_data["model_name_for_query"] in FLAGGED_MODELS:
|
105 |
issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
|
106 |
issue_link = model_hyperlink(FLAGGED_MODELS[model_data["model_name_for_query"]], f"See discussion #{issue_num}")
|
|
|
|
|
107 |
model_data[AutoEvalColumn.model.name] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
|
108 |
|
109 |
def apply_metadata(leaderboard_data: List[dict]):
|
src/init.py
CHANGED
@@ -1,8 +1,6 @@
|
|
1 |
import os
|
2 |
from huggingface_hub import Repository
|
3 |
|
4 |
-
H4_TOKEN = os.environ.get("H4_TOKEN", None)
|
5 |
-
|
6 |
|
7 |
def get_all_requested_models(requested_models_dir):
|
8 |
depth = 1
|
@@ -20,28 +18,23 @@ def load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, QUEUE_PATH, RESULTS_PATH):
|
|
20 |
eval_results_repo = None
|
21 |
requested_models = None
|
22 |
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
eval_results_repo.git_pull()
|
41 |
-
|
42 |
-
requested_models = get_all_requested_models("eval-queue")
|
43 |
-
else:
|
44 |
-
print("No HuggingFace token provided. Skipping evaluation requests and results.")
|
45 |
|
46 |
return eval_queue_repo, requested_models, eval_results_repo
|
47 |
|
|
|
1 |
import os
|
2 |
from huggingface_hub import Repository
|
3 |
|
|
|
|
|
4 |
|
5 |
def get_all_requested_models(requested_models_dir):
|
6 |
depth = 1
|
|
|
18 |
eval_results_repo = None
|
19 |
requested_models = None
|
20 |
|
21 |
+
print("Pulling evaluation requests and results.")
|
22 |
+
|
23 |
+
eval_queue_repo = Repository(
|
24 |
+
local_dir=QUEUE_PATH,
|
25 |
+
clone_from=QUEUE_REPO,
|
26 |
+
repo_type="dataset",
|
27 |
+
)
|
28 |
+
eval_queue_repo.git_pull()
|
29 |
+
|
30 |
+
eval_results_repo = Repository(
|
31 |
+
local_dir=RESULTS_PATH,
|
32 |
+
clone_from=RESULTS_REPO,
|
33 |
+
repo_type="dataset",
|
34 |
+
)
|
35 |
+
eval_results_repo.git_pull()
|
36 |
+
|
37 |
+
requested_models = get_all_requested_models("eval-queue")
|
|
|
|
|
|
|
|
|
|
|
38 |
|
39 |
return eval_queue_repo, requested_models, eval_results_repo
|
40 |
|