add report submited model

app.py

@@ -14,7 +14,7 @@ from src.about import (
 )
 from src.display.css_html_js import custom_css
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-
+from watch_leaderboard import watch_submit_queue
 
 def restart_space():
  API.restart_space(repo_id=REPO_ID)
@@ -78,6 +78,7 @@ with demo:
 
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(watch_submit_queue, "interval", seconds=3500)
+scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()
 demo.queue(default_concurrency_limit=40).launch()

watch_leaderboard.py

@@ -0,0 +1,64 @@
+import json
+import os
+from typing import List, List, Tuple, Any, Dict
+from huggingface_hub import snapshot_download
+from collections import defaultdict
+import requests
+
+from src.envs import EVAL_REQUESTS_PATH, QUEUE_REPO
+
+def webhook_bot(msg: str):
+ bot_url = os.environ.get("DISCORD_WEBHOOK_URL", "")
+ if bot_url != "":
+ requests.post(bot_url, json={"content": msg})
+
+def read_all_pending_model(EVAL_REQUESTS_PATH: str) -> Dict[str, List[Tuple[Any, str]]]:
+ depth = 1
+ alls = defaultdict(list)
+ for root, _, files in os.walk(EVAL_REQUESTS_PATH):
+ current_depth = root.count(os.sep) - EVAL_REQUESTS_PATH.count(os.sep)
+ if current_depth == depth:
+ for file in files:
+ if not file.endswith(".json"):
+ continue
+ file_abs_path = os.path.join(root, file)
+ with open(file_abs_path, "r") as f:
+ info = json.load(f)
+ alls[info['model']].append((info, file_abs_path))
+
+ pendings = {}
+ for k in alls.keys():
+ is_pending = False
+ for stat in alls[k]:
+ info_dict = stat[0]
+ if info_dict['status'] == "PENDING":
+ is_pending = True
+ if is_pending:
+ pendings[k] = alls[k]
+ return pendings
+
+def watch_submit_queue():
+ try:
+ snapshot_download(
+ repo_id=QUEUE_REPO,
+ local_dir=EVAL_REQUESTS_PATH,
+ repo_type="dataset",
+ tqdm_class=None,
+ etag_timeout=30,
+ )
+ 
+ alls = read_all_pending_model(EVAL_REQUESTS_PATH)
+ pending_model = []
+ for model_name in alls.keys():
+ for request_row in alls[model_name]:
+ info, filepath = request_row
+ status = info["status"]
+ model_name = info["model"]
+ if status == "PENDING":
+ pending_model.append(model_name)
+ pending_model = list(set(pending_model))
+ pending_model_str = '\n'.join(pending_model)
+ webhook_bot(f'Leaderboard model pending: {len(pending_model)}\n### Models\n{pending_model_str}')
+ except Exception as e:
+ print(f'Watch submit queue error: {e}')
+ pass