Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Added support for re-submitting rejected
Browse files- app.py +0 -4
- main_backend.py +0 -78
- src/submission/check_validity.py +3 -3
- src/submission/submit.py +8 -1
app.py
CHANGED
@@ -31,9 +31,6 @@ from src.submission.submit import add_new_eval
|
|
31 |
def restart_space():
|
32 |
API.restart_space(repo_id=REPO_ID)
|
33 |
|
34 |
-
# def launch_backend():
|
35 |
-
# _ = subprocess.run(["python", "main_backend.py"])
|
36 |
-
|
37 |
try:
|
38 |
print(EVAL_REQUESTS_PATH)
|
39 |
snapshot_download(
|
@@ -353,6 +350,5 @@ with demo:
|
|
353 |
|
354 |
scheduler = BackgroundScheduler()
|
355 |
scheduler.add_job(restart_space, "interval", seconds=1800)
|
356 |
-
# scheduler.add_job(launch_backend, "interval", seconds=100) # will only allow one job to be run at the same time
|
357 |
scheduler.start()
|
358 |
demo.queue(default_concurrency_limit=40).launch(allowed_paths=["logos/"])
|
|
|
31 |
def restart_space():
|
32 |
API.restart_space(repo_id=REPO_ID)
|
33 |
|
|
|
|
|
|
|
34 |
try:
|
35 |
print(EVAL_REQUESTS_PATH)
|
36 |
snapshot_download(
|
|
|
350 |
|
351 |
scheduler = BackgroundScheduler()
|
352 |
scheduler.add_job(restart_space, "interval", seconds=1800)
|
|
|
353 |
scheduler.start()
|
354 |
demo.queue(default_concurrency_limit=40).launch(allowed_paths=["logos/"])
|
main_backend.py
DELETED
@@ -1,78 +0,0 @@
|
|
1 |
-
import logging
|
2 |
-
import pprint
|
3 |
-
|
4 |
-
from huggingface_hub import snapshot_download
|
5 |
-
|
6 |
-
logging.getLogger("openai").setLevel(logging.WARNING)
|
7 |
-
|
8 |
-
from src.backend.run_eval_suite import run_evaluation
|
9 |
-
from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
|
10 |
-
from src.backend.sort_queue import sort_models_by_priority
|
11 |
-
|
12 |
-
from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, DEVICE, API, LIMIT, TOKEN
|
13 |
-
from src.about import Tasks, NUM_FEWSHOT
|
14 |
-
TASKS_HARNESS = [task.value.benchmark for task in Tasks]
|
15 |
-
|
16 |
-
logging.basicConfig(level=logging.ERROR)
|
17 |
-
pp = pprint.PrettyPrinter(width=80)
|
18 |
-
|
19 |
-
PENDING_STATUS = "PENDING"
|
20 |
-
RUNNING_STATUS = "RUNNING"
|
21 |
-
FINISHED_STATUS = "FINISHED"
|
22 |
-
FAILED_STATUS = "FAILED"
|
23 |
-
|
24 |
-
snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
|
25 |
-
snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
|
26 |
-
|
27 |
-
def run_auto_eval():
|
28 |
-
current_pending_status = [PENDING_STATUS]
|
29 |
-
|
30 |
-
# pull the eval dataset from the hub and parse any eval requests
|
31 |
-
# check completed evals and set them to finished
|
32 |
-
check_completed_evals(
|
33 |
-
api=API,
|
34 |
-
checked_status=RUNNING_STATUS,
|
35 |
-
completed_status=FINISHED_STATUS,
|
36 |
-
failed_status=FAILED_STATUS,
|
37 |
-
hf_repo=QUEUE_REPO,
|
38 |
-
local_dir=EVAL_REQUESTS_PATH_BACKEND,
|
39 |
-
hf_repo_results=RESULTS_REPO,
|
40 |
-
local_dir_results=EVAL_RESULTS_PATH_BACKEND
|
41 |
-
)
|
42 |
-
|
43 |
-
# Get all eval request that are PENDING, if you want to run other evals, change this parameter
|
44 |
-
eval_requests = get_eval_requests(job_status=current_pending_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
|
45 |
-
# Sort the evals by priority (first submitted first run)
|
46 |
-
eval_requests = sort_models_by_priority(api=API, models=eval_requests)
|
47 |
-
|
48 |
-
print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
|
49 |
-
|
50 |
-
if len(eval_requests) == 0:
|
51 |
-
return
|
52 |
-
|
53 |
-
eval_request = eval_requests[0]
|
54 |
-
pp.pprint(eval_request)
|
55 |
-
|
56 |
-
set_eval_request(
|
57 |
-
api=API,
|
58 |
-
eval_request=eval_request,
|
59 |
-
set_to_status=RUNNING_STATUS,
|
60 |
-
hf_repo=QUEUE_REPO,
|
61 |
-
local_dir=EVAL_REQUESTS_PATH_BACKEND,
|
62 |
-
)
|
63 |
-
|
64 |
-
run_evaluation(
|
65 |
-
eval_request=eval_request,
|
66 |
-
task_names=TASKS_HARNESS,
|
67 |
-
num_fewshot=NUM_FEWSHOT,
|
68 |
-
local_dir=EVAL_RESULTS_PATH_BACKEND,
|
69 |
-
results_repo=RESULTS_REPO,
|
70 |
-
batch_size=1,
|
71 |
-
device=DEVICE,
|
72 |
-
no_cache=True,
|
73 |
-
limit=LIMIT
|
74 |
-
)
|
75 |
-
|
76 |
-
|
77 |
-
if __name__ == "__main__":
|
78 |
-
run_auto_eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/submission/check_validity.py
CHANGED
@@ -79,7 +79,7 @@ def get_model_arch(model_info: ModelInfo):
|
|
79 |
|
80 |
def already_submitted_models(requested_models_dir: str) -> set[str]:
|
81 |
depth = 1
|
82 |
-
|
83 |
users_to_submission_dates = defaultdict(list)
|
84 |
|
85 |
for root, _, files in os.walk(requested_models_dir):
|
@@ -92,7 +92,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
|
|
92 |
continue
|
93 |
with open(file_path, "r") as f:
|
94 |
info = json.load(f)
|
95 |
-
|
96 |
|
97 |
# Select organisation
|
98 |
if info["model"].count("/") == 0 or "submitted_time" not in info:
|
@@ -100,4 +100,4 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
|
|
100 |
organisation, _ = info["model"].split("/")
|
101 |
users_to_submission_dates[organisation].append(info["submitted_time"])
|
102 |
|
103 |
-
return set(
|
|
|
79 |
|
80 |
def already_submitted_models(requested_models_dir: str) -> set[str]:
|
81 |
depth = 1
|
82 |
+
requested_models = {}
|
83 |
users_to_submission_dates = defaultdict(list)
|
84 |
|
85 |
for root, _, files in os.walk(requested_models_dir):
|
|
|
92 |
continue
|
93 |
with open(file_path, "r") as f:
|
94 |
info = json.load(f)
|
95 |
+
requested_models[f"{info['model']}_{info['revision']}_{info['precision']}"] = info["status"]
|
96 |
|
97 |
# Select organisation
|
98 |
if info["model"].count("/") == 0 or "submitted_time" not in info:
|
|
|
100 |
organisation, _ = info["model"].split("/")
|
101 |
users_to_submission_dates[organisation].append(info["submitted_time"])
|
102 |
|
103 |
+
return set(requested_models), users_to_submission_dates
|
src/submission/submit.py
CHANGED
@@ -107,7 +107,14 @@ def add_new_eval(
|
|
107 |
}
|
108 |
|
109 |
# Check for duplicate submission
|
110 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
111 |
return styled_warning("This model has been already submitted.")
|
112 |
|
113 |
print("Creating eval file")
|
|
|
107 |
}
|
108 |
|
109 |
# Check for duplicate submission
|
110 |
+
request_keys = [f"{model}_{revision}_{precision}", f"{model}_{actual_revision}_{precision}"]
|
111 |
+
requested_status = next((REQUESTED_MODELS[key] for key in request_keys if key in REQUESTED_MODELS), 'NONE')
|
112 |
+
|
113 |
+
# If it's None or Rejected - let it through. Otherwise - inform the user
|
114 |
+
if requested_status not in ['NONE', 'REJECTED']:
|
115 |
+
# if it failed - spell that out and tell him to contact if we wants to resubmit
|
116 |
+
if requested_status == 'FAILED':
|
117 |
+
return styled_warning("This model has been already submitted and failed to run - please open a discussion or contact the support email.")
|
118 |
return styled_warning("This model has been already submitted.")
|
119 |
|
120 |
print("Creating eval file")
|