File size: 3,370 Bytes
d06d966 0eac6a7 d06d966 5bb9974 0eac6a7 d06d966 0eac6a7 d06d966 a2cb75e d06d966 a2cb75e d06d966 a2cb75e d06d966 a2cb75e d06d966 0eac6a7 d06d966 25306b9 d06d966 a2cb75e d06d966 5bb9974 d06d966 aa04fbd d06d966 aa04fbd d06d966 aa04fbd d06d966 aa04fbd b065235 0eac6a7 b065235 d06d966 0eac6a7 d06d966 5bb9974 0eac6a7 d06d966 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
import os
import time
from dataclasses import dataclass
import gradio as gr
import schedule
import wandb
from substrateinterface import Keypair
from wandb.apis.public import Run, Runs
wandb_api = wandb.Api()
demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}")
SOURCE_VALIDATOR_UID = int(os.environ["SOURCE_VALIDATOR_UID"])
WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]
@dataclass
class LeaderboardEntry:
uid: int
model: str
score: float
hotkey: str
previous_day_winner: bool
rank: int
def is_valid_run(run: Run):
required_config_keys = ["hotkey", "uid", "contest", "signature"]
for key in required_config_keys:
if key not in run.config:
return False
uid = run.config["uid"]
validator_hotkey = run.config["hotkey"]
contest_name = run.config["contest"]
signing_message = f"{uid}:{validator_hotkey}:{contest_name}"
try:
return Keypair(validator_hotkey).verify(signing_message, run.config["signature"])
except Exception:
return False
def refresh_leaderboard():
print("Refreshing Leaderboard")
demo.clear()
with demo:
with gr.Accordion("Contest #1 Submission Leader: New Dream SDXL on NVIDIA RTX 4090s"):
runs: Runs = wandb_api.runs(
WANDB_RUN_PATH,
filters={"config.type": "validator", "config.uid": SOURCE_VALIDATOR_UID},
order="-created_at",
)
entries: dict[int, LeaderboardEntry] = {}
for run in runs:
if not is_valid_run(run):
continue
has_data = False
for key, value in run.summary.items():
if key.startswith("_"):
continue
has_data = True
try:
uid = int(key)
if uid not in entries:
entries[uid] = LeaderboardEntry(
uid=uid,
rank=value["rank"],
model=value["model"],
score=value["score"],
hotkey=value["hotkey"],
previous_day_winner=value["multiday_winner"],
)
except Exception:
continue
if has_data:
break
leaderboard: list[tuple] = [
(entry.rank + 1, entry.uid, entry.model, entry.score, entry.hotkey, entry.previous_day_winner)
for entry in sorted(entries.values(), key=lambda entry: entry.rank)
]
gr.components.Dataframe(
value=leaderboard,
headers=["Rank", "Uid", "Model", "Score", "Hotkey", "Previous day winner"],
datatype=["number", "number", "markdown", "number", "markdown", "bool"],
elem_id="leaderboard-table",
interactive=False,
visible=True,
)
def main():
refresh_leaderboard()
schedule.every().day.at("09:30:00", "America/New_York").do(refresh_leaderboard)
demo.launch(prevent_thread_lock=True)
while True:
schedule.run_pending()
time.sleep(1)
main()
|