AlexNijjar
commited on
Commit
β’
20cc3c8
1
Parent(s):
ecf13ce
Use gradio's built-in refresh
Browse files
app.py
CHANGED
@@ -2,7 +2,6 @@ import os
|
|
2 |
from dataclasses import dataclass
|
3 |
from datetime import datetime, timedelta
|
4 |
from enum import Enum
|
5 |
-
from time import sleep
|
6 |
from zoneinfo import ZoneInfo
|
7 |
|
8 |
import bittensor as bt
|
@@ -51,18 +50,19 @@ class Status(Enum):
|
|
51 |
class State:
|
52 |
status: Status
|
53 |
hotkey: str
|
|
|
54 |
version: str
|
55 |
winner: int | None
|
56 |
submissions: int
|
57 |
benchmarks: int
|
58 |
invalid: int
|
59 |
average_benchmark_time: float
|
|
|
|
|
60 |
|
61 |
|
62 |
data: dict[int, State] = {}
|
63 |
-
|
64 |
-
validator_vtrust: dict[int, float] = {}
|
65 |
-
validator_updated: dict[int, int] = {}
|
66 |
|
67 |
|
68 |
def is_valid_run(run: Run):
|
@@ -84,12 +84,19 @@ def is_valid_run(run: Run):
|
|
84 |
return False
|
85 |
|
86 |
|
87 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
now = datetime.now(tz=ZoneInfo("America/New_York"))
|
89 |
noon = now.replace(hour=12, minute=0, second=0, microsecond=0)
|
90 |
if now.hour < 12:
|
91 |
noon -= timedelta(days=1)
|
92 |
-
|
93 |
wandb_runs = wandb_api.runs(
|
94 |
WANDB_RUN_PATH,
|
95 |
filters={"config.type": "validator", "created_at": {'$gt': str(noon)}},
|
@@ -143,45 +150,25 @@ def fetch_wandb_data():
|
|
143 |
elif not submissions or (not average_benchmark_time and benchmarks):
|
144 |
status = Status.BROKEN
|
145 |
|
|
|
146 |
data[uid] = State(
|
147 |
status=status,
|
148 |
hotkey=run.config["hotkey"],
|
|
|
149 |
version=run.tags[1][8:],
|
150 |
winner=winner,
|
151 |
submissions=len(submissions),
|
152 |
benchmarks=len(benchmarks),
|
153 |
invalid=len(invalid),
|
154 |
-
average_benchmark_time=average_benchmark_time
|
|
|
|
|
155 |
)
|
156 |
|
157 |
-
|
158 |
-
|
159 |
-
|
160 |
-
def fetch_identities():
|
161 |
-
validator_identities.clear()
|
162 |
-
for uid in data.keys():
|
163 |
-
identity = subtensor.substrate.query('SubtensorModule', 'Identities', [metagraph.coldkeys[uid]])
|
164 |
-
if identity != None:
|
165 |
-
validator_identities[uid] = identity.value["name"]
|
166 |
-
|
167 |
|
168 |
-
def fetch_metagraph_data():
|
169 |
-
validator_vtrust.clear()
|
170 |
-
validator_updated.clear()
|
171 |
-
block = subtensor.get_current_block()
|
172 |
-
for uid in data.keys():
|
173 |
-
validator_vtrust[uid] = metagraph.validator_trust[uid]
|
174 |
-
validator_updated[uid] = block - metagraph.last_update[uid]
|
175 |
|
176 |
-
|
177 |
-
def get_validator_name(validator_uid: int) -> str:
|
178 |
-
if validator_uid in validator_identities:
|
179 |
-
return validator_identities[validator_uid]
|
180 |
-
else:
|
181 |
-
return metagraph.hotkeys[validator_uid]
|
182 |
-
|
183 |
-
|
184 |
-
def get_latest_version() -> str:
|
185 |
latest_version = version.parse("0.0.0")
|
186 |
for source_validator_uid, state in data.items():
|
187 |
current_version = version.parse(state.version)
|
@@ -190,62 +177,58 @@ def get_latest_version() -> str:
|
|
190 |
return str(latest_version)
|
191 |
|
192 |
|
193 |
-
def
|
194 |
-
|
195 |
-
|
196 |
-
fetch_identities()
|
197 |
-
fetch_metagraph_data()
|
198 |
-
demo.clear()
|
199 |
now = datetime.now(tz=ZoneInfo("America/New_York"))
|
200 |
|
201 |
-
|
202 |
-
|
203 |
-
|
204 |
-
|
205 |
-
|
206 |
-
for uid, state in data.items():
|
207 |
-
eta = int(state.average_benchmark_time * (state.submissions - (state.benchmarks + state.invalid)))
|
208 |
-
time_left = timedelta(seconds=eta)
|
209 |
-
eta_date = now + time_left
|
210 |
-
eta_time = eta_date.strftime("%Y-%m-%d %I:%M:%S %p") if eta > 0 and state.status == Status.IN_PROGRESS else state.status.get_alt_time_text()
|
211 |
-
|
212 |
-
average_time_text = f"{timedelta(seconds=int(state.average_benchmark_time))}" if state.average_benchmark_time else state.status.get_alt_time_text(),
|
213 |
-
|
214 |
-
elements.append((
|
215 |
-
uid,
|
216 |
-
get_validator_name(uid),
|
217 |
-
f"<span style='color: {'springgreen' if state.version == latest_version else 'red'}'>{state.version}</span>",
|
218 |
-
f"<span style='color: {state.status.color()}'>{state.status.name()}</span>",
|
219 |
-
f"<span style='color: {'springgreen' if state.winner else 'orange'}'>{state.winner if state.winner else 'N/A'}</span>",
|
220 |
-
state.benchmarks + state.invalid,
|
221 |
-
state.submissions,
|
222 |
-
state.invalid,
|
223 |
-
f"<span style='color: {'orange' if state.average_benchmark_time > AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD else 'springgreen'}'>{average_time_text[0]}</span>",
|
224 |
-
f"<span style='color: {'orange' if eta > ETA_WARNING_THRESHOLD else 'springgreen'}'>{eta_time}</span>",
|
225 |
-
f"<span style='color: {'orange' if eta > ETA_WARNING_THRESHOLD else 'springgreen'}'>{time_left if eta > 0 and state.status == Status.IN_PROGRESS else state.status.get_alt_time_text()}</span>",
|
226 |
-
f"<span style='color: {'springgreen' if validator_vtrust[uid] > 0.75 else 'red'}'>{validator_vtrust[uid]:.4f}</span>",
|
227 |
-
f"<span style='color: {'springgreen' if validator_updated[uid] < 1000 else 'red'}'>{validator_updated[uid]}</span>",
|
228 |
-
))
|
229 |
|
230 |
-
|
231 |
-
|
232 |
-
headers=["UID", "Name", "Version", "Status", "Winner UID", "Tested", "Submissions", "Invalid", "Avg. Benchmark Time", "ETA (Eastern Time)", "ETA Remaining", "VTrust", "Updated"],
|
233 |
-
datatype=["number", "markdown", "markdown", "markdown", "markdown", "number", "number", "number", "markdown", "markdown", "markdown", "markdown", "markdown"],
|
234 |
-
elem_id="state-table",
|
235 |
-
)
|
236 |
|
|
|
|
|
|
|
|
|
|
|
237 |
|
238 |
-
|
239 |
-
refresh()
|
240 |
-
demo.launch(prevent_thread_lock=True)
|
241 |
|
242 |
-
|
243 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
244 |
|
245 |
-
now = datetime.now(tz=ZoneInfo("America/New_York"))
|
246 |
-
print(f"Refreshing States at {now.strftime('%Y-%m-%d %H:%M:%S')}")
|
247 |
|
248 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
249 |
|
250 |
|
251 |
if __name__ == '__main__':
|
|
|
2 |
from dataclasses import dataclass
|
3 |
from datetime import datetime, timedelta
|
4 |
from enum import Enum
|
|
|
5 |
from zoneinfo import ZoneInfo
|
6 |
|
7 |
import bittensor as bt
|
|
|
50 |
class State:
|
51 |
status: Status
|
52 |
hotkey: str
|
53 |
+
name: str
|
54 |
version: str
|
55 |
winner: int | None
|
56 |
submissions: int
|
57 |
benchmarks: int
|
58 |
invalid: int
|
59 |
average_benchmark_time: float
|
60 |
+
vtrust: float
|
61 |
+
updated: int
|
62 |
|
63 |
|
64 |
data: dict[int, State] = {}
|
65 |
+
last_refresh: datetime = datetime.fromtimestamp(0, tz=ZoneInfo("America/New_York"))
|
|
|
|
|
66 |
|
67 |
|
68 |
def is_valid_run(run: Run):
|
|
|
84 |
return False
|
85 |
|
86 |
|
87 |
+
def get_identity(uid: int) -> str | None:
|
88 |
+
identity = subtensor.substrate.query('SubtensorModule', 'Identities', [metagraph.coldkeys[uid]])
|
89 |
+
return identity.value["name"] if identity != None else None
|
90 |
+
|
91 |
+
|
92 |
+
def fetch_wandb_data() -> dict[int, State]:
|
93 |
+
data: dict[int, State] = {}
|
94 |
+
|
95 |
now = datetime.now(tz=ZoneInfo("America/New_York"))
|
96 |
noon = now.replace(hour=12, minute=0, second=0, microsecond=0)
|
97 |
if now.hour < 12:
|
98 |
noon -= timedelta(days=1)
|
99 |
+
|
100 |
wandb_runs = wandb_api.runs(
|
101 |
WANDB_RUN_PATH,
|
102 |
filters={"config.type": "validator", "created_at": {'$gt': str(noon)}},
|
|
|
150 |
elif not submissions or (not average_benchmark_time and benchmarks):
|
151 |
status = Status.BROKEN
|
152 |
|
153 |
+
block = subtensor.get_current_block()
|
154 |
data[uid] = State(
|
155 |
status=status,
|
156 |
hotkey=run.config["hotkey"],
|
157 |
+
name=get_identity(uid) or run.config["hotkey"],
|
158 |
version=run.tags[1][8:],
|
159 |
winner=winner,
|
160 |
submissions=len(submissions),
|
161 |
benchmarks=len(benchmarks),
|
162 |
invalid=len(invalid),
|
163 |
+
average_benchmark_time=average_benchmark_time,
|
164 |
+
vtrust=metagraph.validator_trust[uid],
|
165 |
+
updated=block - metagraph.last_update[uid],
|
166 |
)
|
167 |
|
168 |
+
return dict(sorted(data.items()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
170 |
|
171 |
+
def get_latest_version(data: dict[int, State]) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
172 |
latest_version = version.parse("0.0.0")
|
173 |
for source_validator_uid, state in data.items():
|
174 |
current_version = version.parse(state.version)
|
|
|
177 |
return str(latest_version)
|
178 |
|
179 |
|
180 |
+
def get_data() -> list[tuple]:
|
181 |
+
global data
|
182 |
+
global last_refresh
|
|
|
|
|
|
|
183 |
now = datetime.now(tz=ZoneInfo("America/New_York"))
|
184 |
|
185 |
+
if (now - last_refresh).total_seconds() > REFRESH_RATE:
|
186 |
+
metagraph.sync(subtensor=subtensor)
|
187 |
+
data = fetch_wandb_data()
|
188 |
+
last_refresh = now
|
189 |
+
print(f"Refreshing States at {now.strftime('%Y-%m-%d %H:%M:%S')}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
190 |
|
191 |
+
elements: list[tuple] = []
|
192 |
+
latest_version = get_latest_version(data)
|
|
|
|
|
|
|
|
|
193 |
|
194 |
+
for uid, state in data.items():
|
195 |
+
eta = int(state.average_benchmark_time * (state.submissions - (state.benchmarks + state.invalid)))
|
196 |
+
time_left = timedelta(seconds=eta)
|
197 |
+
eta_date = now + time_left
|
198 |
+
eta_time = eta_date.strftime("%Y-%m-%d %I:%M:%S %p") if eta > 0 and state.status == Status.IN_PROGRESS else state.status.get_alt_time_text()
|
199 |
|
200 |
+
average_time_text = f"{timedelta(seconds=int(state.average_benchmark_time))}" if state.average_benchmark_time else state.status.get_alt_time_text(),
|
|
|
|
|
201 |
|
202 |
+
elements.append((
|
203 |
+
uid,
|
204 |
+
state.name,
|
205 |
+
f"<span style='color: {'springgreen' if state.version == latest_version else 'red'}'>{state.version}</span>",
|
206 |
+
f"<span style='color: {state.status.color()}'>{state.status.name()}</span>",
|
207 |
+
f"<span style='color: {'springgreen' if state.winner else 'orange'}'>{state.winner if state.winner else 'N/A'}</span>",
|
208 |
+
state.benchmarks + state.invalid,
|
209 |
+
state.submissions,
|
210 |
+
state.invalid,
|
211 |
+
f"<span style='color: {'orange' if state.average_benchmark_time > AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD else 'springgreen'}'>{average_time_text[0]}</span>",
|
212 |
+
f"<span style='color: {'orange' if eta > ETA_WARNING_THRESHOLD else 'springgreen'}'>{eta_time}</span>",
|
213 |
+
f"<span style='color: {'orange' if eta > ETA_WARNING_THRESHOLD else 'springgreen'}'>{time_left if eta > 0 and state.status == Status.IN_PROGRESS else state.status.get_alt_time_text()}</span>",
|
214 |
+
f"<span style='color: {'springgreen' if state.vtrust > 0.75 else 'red'}'>{state.vtrust:.4f}</span>",
|
215 |
+
f"<span style='color: {'springgreen' if state.updated < 1000 else 'red'}'>{state.updated}</span>",
|
216 |
+
))
|
217 |
+
|
218 |
+
return elements
|
219 |
|
|
|
|
|
220 |
|
221 |
+
def main():
|
222 |
+
with demo:
|
223 |
+
with gr.Accordion(f"Validator States"):
|
224 |
+
gr.components.Dataframe(
|
225 |
+
get_data,
|
226 |
+
every=REFRESH_RATE,
|
227 |
+
headers=["UID", "Name", "Version", "Status", "Winner UID", "Tested", "Submissions", "Invalid", "Avg. Benchmark Time", "ETA (Eastern Time)", "ETA Remaining", "VTrust", "Updated"],
|
228 |
+
datatype=["number", "markdown", "markdown", "markdown", "markdown", "number", "number", "number", "markdown", "markdown", "markdown", "markdown", "markdown"],
|
229 |
+
elem_id="state-table",
|
230 |
+
)
|
231 |
+
demo.queue().launch()
|
232 |
|
233 |
|
234 |
if __name__ == '__main__':
|