AlexNijjar
commited on
Commit
β’
92f1024
1
Parent(s):
d06906e
Better state reporting, show winner
Browse files
app.py
CHANGED
@@ -31,9 +31,11 @@ class Status(Enum):
|
|
31 |
IN_PROGRESS = ("In Progress", "orange")
|
32 |
DONE = ("Done", "springgreen")
|
33 |
BROKEN = ("Broken", "red")
|
|
|
|
|
34 |
|
35 |
def get_alt_time_text(self) -> str:
|
36 |
-
return "β" if self
|
37 |
|
38 |
def name(self):
|
39 |
return self.value[0]
|
@@ -41,12 +43,16 @@ class Status(Enum):
|
|
41 |
def color(self):
|
42 |
return self.value[1]
|
43 |
|
|
|
|
|
|
|
44 |
|
45 |
@dataclass
|
46 |
class State:
|
47 |
status: Status
|
48 |
hotkey: str
|
49 |
version: str
|
|
|
50 |
submissions: int
|
51 |
benchmarks: int
|
52 |
invalid: int
|
@@ -77,11 +83,11 @@ def is_valid_run(run: Run):
|
|
77 |
|
78 |
|
79 |
def fetch_wandb_data():
|
80 |
-
|
81 |
global data
|
82 |
wandb_runs = wandb_api.runs(
|
83 |
WANDB_RUN_PATH,
|
84 |
-
filters={"config.type": "validator", "created_at": {'$gt': str(
|
85 |
order="-created_at",
|
86 |
)
|
87 |
|
@@ -93,6 +99,7 @@ def fetch_wandb_data():
|
|
93 |
if not metagraph.validator_permit[uid]:
|
94 |
continue
|
95 |
|
|
|
96 |
submissions: set[int] = set()
|
97 |
benchmarks: set[int] = set()
|
98 |
invalid: list[int] = []
|
@@ -111,14 +118,22 @@ def fetch_wandb_data():
|
|
111 |
submissions.add(int(submission_key))
|
112 |
|
113 |
elif key == "benchmarks":
|
114 |
-
|
115 |
-
|
|
|
|
|
|
|
116 |
|
117 |
for benchmark_key, benchmark_value in value.items():
|
118 |
benchmarks.add(int(benchmark_key))
|
119 |
|
120 |
status = Status.IN_PROGRESS
|
121 |
-
|
|
|
|
|
|
|
|
|
|
|
122 |
status = Status.DONE
|
123 |
elif not submissions or (not average_benchmark_time and benchmarks):
|
124 |
status = Status.BROKEN
|
@@ -127,6 +142,7 @@ def fetch_wandb_data():
|
|
127 |
status=status,
|
128 |
hotkey=run.config["hotkey"],
|
129 |
version=run.tags[1][8:],
|
|
|
130 |
submissions=len(submissions),
|
131 |
benchmarks=len(benchmarks),
|
132 |
invalid=len(invalid),
|
@@ -185,6 +201,7 @@ def refresh():
|
|
185 |
get_validator_name(uid),
|
186 |
f"<span style='color: {'springgreen' if state.version == latest_version else 'red'}'>{state.version}</span>",
|
187 |
f"<span style='color: {state.status.color()}'>{state.status.name()}</span>",
|
|
|
188 |
state.benchmarks + state.invalid,
|
189 |
state.submissions,
|
190 |
state.invalid,
|
@@ -195,8 +212,8 @@ def refresh():
|
|
195 |
|
196 |
gr.components.Dataframe(
|
197 |
elements,
|
198 |
-
headers=["UID", "Name", "Version", "Status", "Tested", "Submissions", "Invalid", "Avg. Benchmark Time", "ETA (Eastern Time)", "ETA Remaining"],
|
199 |
-
datatype=["number", "markdown", "markdown", "markdown", "number", "number", "number", "markdown", "markdown", "markdown"],
|
200 |
elem_id="state-table",
|
201 |
)
|
202 |
|
|
|
31 |
IN_PROGRESS = ("In Progress", "orange")
|
32 |
DONE = ("Done", "springgreen")
|
33 |
BROKEN = ("Broken", "red")
|
34 |
+
STOPPED = ("Stopped", "red")
|
35 |
+
CRASHED = ("Crashed", "red")
|
36 |
|
37 |
def get_alt_time_text(self) -> str:
|
38 |
+
return "β" if self.is_broken() else "N/A"
|
39 |
|
40 |
def name(self):
|
41 |
return self.value[0]
|
|
|
43 |
def color(self):
|
44 |
return self.value[1]
|
45 |
|
46 |
+
def is_broken(self):
|
47 |
+
return self == Status.BROKEN or self == Status.STOPPED or self == Status.CRASHED
|
48 |
+
|
49 |
|
50 |
@dataclass
|
51 |
class State:
|
52 |
status: Status
|
53 |
hotkey: str
|
54 |
version: str
|
55 |
+
winner: int | None
|
56 |
submissions: int
|
57 |
benchmarks: int
|
58 |
invalid: int
|
|
|
83 |
|
84 |
|
85 |
def fetch_wandb_data():
|
86 |
+
noon = datetime.now(tz=ZoneInfo("America/New_York")).replace(hour=12, minute=0, second=0, microsecond=0)
|
87 |
global data
|
88 |
wandb_runs = wandb_api.runs(
|
89 |
WANDB_RUN_PATH,
|
90 |
+
filters={"config.type": "validator", "created_at": {'$gt': str(noon)}},
|
91 |
order="-created_at",
|
92 |
)
|
93 |
|
|
|
99 |
if not metagraph.validator_permit[uid]:
|
100 |
continue
|
101 |
|
102 |
+
winner = None
|
103 |
submissions: set[int] = set()
|
104 |
benchmarks: set[int] = set()
|
105 |
invalid: list[int] = []
|
|
|
118 |
submissions.add(int(submission_key))
|
119 |
|
120 |
elif key == "benchmarks":
|
121 |
+
for benchmark_uid, benchmark in value.items():
|
122 |
+
if "winner" in benchmark:
|
123 |
+
winner = benchmark_uid
|
124 |
+
completed = True
|
125 |
+
break
|
126 |
|
127 |
for benchmark_key, benchmark_value in value.items():
|
128 |
benchmarks.add(int(benchmark_key))
|
129 |
|
130 |
status = Status.IN_PROGRESS
|
131 |
+
run_state = run.state
|
132 |
+
if run_state == "finished":
|
133 |
+
status = Status.STOPPED
|
134 |
+
elif run_state == "crashed":
|
135 |
+
status = Status.CRASHED
|
136 |
+
elif completed:
|
137 |
status = Status.DONE
|
138 |
elif not submissions or (not average_benchmark_time and benchmarks):
|
139 |
status = Status.BROKEN
|
|
|
142 |
status=status,
|
143 |
hotkey=run.config["hotkey"],
|
144 |
version=run.tags[1][8:],
|
145 |
+
winner=winner,
|
146 |
submissions=len(submissions),
|
147 |
benchmarks=len(benchmarks),
|
148 |
invalid=len(invalid),
|
|
|
201 |
get_validator_name(uid),
|
202 |
f"<span style='color: {'springgreen' if state.version == latest_version else 'red'}'>{state.version}</span>",
|
203 |
f"<span style='color: {state.status.color()}'>{state.status.name()}</span>",
|
204 |
+
f"<span style='color: {'springgreen' if state.winner else 'orange'}'>{state.winner if state.winner else 'N/A'}</span>",
|
205 |
state.benchmarks + state.invalid,
|
206 |
state.submissions,
|
207 |
state.invalid,
|
|
|
212 |
|
213 |
gr.components.Dataframe(
|
214 |
elements,
|
215 |
+
headers=["UID", "Name", "Version", "Status", "Winner", "Tested", "Submissions", "Invalid", "Avg. Benchmark Time", "ETA (Eastern Time)", "ETA Remaining"],
|
216 |
+
datatype=["number", "markdown", "markdown", "markdown", "markdown", "number", "number", "number", "markdown", "markdown", "markdown"],
|
217 |
elem_id="state-table",
|
218 |
)
|
219 |
|