Shaltiel commited on
Commit
93a260a
1 Parent(s): 4a85986

Added support for re-submitting rejected

Browse files
app.py CHANGED
@@ -31,9 +31,6 @@ from src.submission.submit import add_new_eval
31
  def restart_space():
32
  API.restart_space(repo_id=REPO_ID)
33
 
34
- # def launch_backend():
35
- # _ = subprocess.run(["python", "main_backend.py"])
36
-
37
  try:
38
  print(EVAL_REQUESTS_PATH)
39
  snapshot_download(
@@ -353,6 +350,5 @@ with demo:
353
 
354
  scheduler = BackgroundScheduler()
355
  scheduler.add_job(restart_space, "interval", seconds=1800)
356
- # scheduler.add_job(launch_backend, "interval", seconds=100) # will only allow one job to be run at the same time
357
  scheduler.start()
358
  demo.queue(default_concurrency_limit=40).launch(allowed_paths=["logos/"])
 
31
  def restart_space():
32
  API.restart_space(repo_id=REPO_ID)
33
 
 
 
 
34
  try:
35
  print(EVAL_REQUESTS_PATH)
36
  snapshot_download(
 
350
 
351
  scheduler = BackgroundScheduler()
352
  scheduler.add_job(restart_space, "interval", seconds=1800)
 
353
  scheduler.start()
354
  demo.queue(default_concurrency_limit=40).launch(allowed_paths=["logos/"])
main_backend.py DELETED
@@ -1,78 +0,0 @@
1
- import logging
2
- import pprint
3
-
4
- from huggingface_hub import snapshot_download
5
-
6
- logging.getLogger("openai").setLevel(logging.WARNING)
7
-
8
- from src.backend.run_eval_suite import run_evaluation
9
- from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
10
- from src.backend.sort_queue import sort_models_by_priority
11
-
12
- from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, DEVICE, API, LIMIT, TOKEN
13
- from src.about import Tasks, NUM_FEWSHOT
14
- TASKS_HARNESS = [task.value.benchmark for task in Tasks]
15
-
16
- logging.basicConfig(level=logging.ERROR)
17
- pp = pprint.PrettyPrinter(width=80)
18
-
19
- PENDING_STATUS = "PENDING"
20
- RUNNING_STATUS = "RUNNING"
21
- FINISHED_STATUS = "FINISHED"
22
- FAILED_STATUS = "FAILED"
23
-
24
- snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
25
- snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
26
-
27
- def run_auto_eval():
28
- current_pending_status = [PENDING_STATUS]
29
-
30
- # pull the eval dataset from the hub and parse any eval requests
31
- # check completed evals and set them to finished
32
- check_completed_evals(
33
- api=API,
34
- checked_status=RUNNING_STATUS,
35
- completed_status=FINISHED_STATUS,
36
- failed_status=FAILED_STATUS,
37
- hf_repo=QUEUE_REPO,
38
- local_dir=EVAL_REQUESTS_PATH_BACKEND,
39
- hf_repo_results=RESULTS_REPO,
40
- local_dir_results=EVAL_RESULTS_PATH_BACKEND
41
- )
42
-
43
- # Get all eval request that are PENDING, if you want to run other evals, change this parameter
44
- eval_requests = get_eval_requests(job_status=current_pending_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
45
- # Sort the evals by priority (first submitted first run)
46
- eval_requests = sort_models_by_priority(api=API, models=eval_requests)
47
-
48
- print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
49
-
50
- if len(eval_requests) == 0:
51
- return
52
-
53
- eval_request = eval_requests[0]
54
- pp.pprint(eval_request)
55
-
56
- set_eval_request(
57
- api=API,
58
- eval_request=eval_request,
59
- set_to_status=RUNNING_STATUS,
60
- hf_repo=QUEUE_REPO,
61
- local_dir=EVAL_REQUESTS_PATH_BACKEND,
62
- )
63
-
64
- run_evaluation(
65
- eval_request=eval_request,
66
- task_names=TASKS_HARNESS,
67
- num_fewshot=NUM_FEWSHOT,
68
- local_dir=EVAL_RESULTS_PATH_BACKEND,
69
- results_repo=RESULTS_REPO,
70
- batch_size=1,
71
- device=DEVICE,
72
- no_cache=True,
73
- limit=LIMIT
74
- )
75
-
76
-
77
- if __name__ == "__main__":
78
- run_auto_eval()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/submission/check_validity.py CHANGED
@@ -79,7 +79,7 @@ def get_model_arch(model_info: ModelInfo):
79
 
80
  def already_submitted_models(requested_models_dir: str) -> set[str]:
81
  depth = 1
82
- file_names = []
83
  users_to_submission_dates = defaultdict(list)
84
 
85
  for root, _, files in os.walk(requested_models_dir):
@@ -92,7 +92,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
92
  continue
93
  with open(file_path, "r") as f:
94
  info = json.load(f)
95
- file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
96
 
97
  # Select organisation
98
  if info["model"].count("/") == 0 or "submitted_time" not in info:
@@ -100,4 +100,4 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
100
  organisation, _ = info["model"].split("/")
101
  users_to_submission_dates[organisation].append(info["submitted_time"])
102
 
103
- return set(file_names), users_to_submission_dates
 
79
 
80
  def already_submitted_models(requested_models_dir: str) -> set[str]:
81
  depth = 1
82
+ requested_models = {}
83
  users_to_submission_dates = defaultdict(list)
84
 
85
  for root, _, files in os.walk(requested_models_dir):
 
92
  continue
93
  with open(file_path, "r") as f:
94
  info = json.load(f)
95
+ requested_models[f"{info['model']}_{info['revision']}_{info['precision']}"] = info["status"]
96
 
97
  # Select organisation
98
  if info["model"].count("/") == 0 or "submitted_time" not in info:
 
100
  organisation, _ = info["model"].split("/")
101
  users_to_submission_dates[organisation].append(info["submitted_time"])
102
 
103
+ return set(requested_models), users_to_submission_dates
src/submission/submit.py CHANGED
@@ -107,7 +107,14 @@ def add_new_eval(
107
  }
108
 
109
  # Check for duplicate submission
110
- if f"{model}_{revision}_{precision}" in REQUESTED_MODELS or f"{model}_{actual_revision}_{precision}" in REQUESTED_MODELS:
 
 
 
 
 
 
 
111
  return styled_warning("This model has been already submitted.")
112
 
113
  print("Creating eval file")
 
107
  }
108
 
109
  # Check for duplicate submission
110
+ request_keys = [f"{model}_{revision}_{precision}", f"{model}_{actual_revision}_{precision}"]
111
+ requested_status = next((REQUESTED_MODELS[key] for key in request_keys if key in REQUESTED_MODELS), 'NONE')
112
+
113
+ # If it's None or Rejected - let it through. Otherwise - inform the user
114
+ if requested_status not in ['NONE', 'REJECTED']:
115
+ # if it failed - spell that out and tell him to contact if we wants to resubmit
116
+ if requested_status == 'FAILED':
117
+ return styled_warning("This model has been already submitted and failed to run - please open a discussion or contact the support email.")
118
  return styled_warning("This model has been already submitted.")
119
 
120
  print("Creating eval file")