Update app.py

app.py

@@ -14,22 +14,23 @@ WANDB_RUN_PATH = os.environ["WANDB_RUN_PATH"]
 
 REFRESH_RATE = 30
 NET_UID = 39
-AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD = 600 # 10 minutes
+AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD = 300 # 5 minutes
 ETA_WARNING_THRESHOLD = 43200 # 12 hours
+UPDATED_WARNING_THRESHOLD = 1000
+VTRUST_WARNING_THRESHOLD = 0.75
 
 subtensor = bt.subtensor()
 metagraph = bt.metagraph(netuid=NET_UID)
 
-demo = gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}", fill_height=True, fill_width=True)
 
-
-class Status(Enum):
- BENCHMARKING = ("Benchmarking", "orange")
- DONE = ("Done", "springgreen")
- BROKEN = ("Broken", "red")
+class BenchmarkState(Enum):
+ NOT_STARTED = ("Not Started", "orange")
+ IN_PROGRESS = ("In Progress", "orange")
+ FINISHED = ("Finished", "springgreen")
+ OUT_OF_DATE = ("Out of Date", "red")
  STOPPED = ("Stopped", "red")
  CRASHED = ("Crashed", "red")
- INITIALIZING = ("Initializing", "orange")
+ FAILED = ("Failed", "red")
 
  def get_alt_time_text(self) -> str:
  return "∞" if self.is_broken() else "N/A"
@@ -41,12 +42,12 @@ class Status(Enum):
  return self.value[1]
 
  def is_broken(self):
- return self == Status.BROKEN or self == Status.STOPPED or self == Status.CRASHED or self == Status.INITIALIZING
+ return self == BenchmarkState.NOT_STARTED or self == BenchmarkState.STOPPED or self == BenchmarkState.CRASHED or self == BenchmarkState.FAILED
 
 
 @dataclass
 class State:
- status: Status
+ benchmarking_state: BenchmarkState
  hotkey: str
  name: str
  version: str
@@ -123,11 +124,7 @@ def fetch_wandb_data():
  average_benchmark_time = float(value)
 
  elif key == "invalid":
- try:
- invalid = dict(value)
- except TypeError:
- invalid = {uid: "Unknown" for uid in value}
-
+ invalid = dict(value)
 
  elif key == "submissions":
  submissions = {int(uid) for uid in value.keys()}
@@ -135,25 +132,27 @@ def fetch_wandb_data():
  elif key == "benchmarks":
  benchmarks = {int(uid) for uid in value.keys()}
 
- benchmarks = set(benchmarks) - set(invalid.keys())
 
- status = Status.BENCHMARKING
+ benchmarking_state: BenchmarkState
  run_state = run.state
  if run_state == "finished":
- status = Status.STOPPED
+ benchmarking_state = BenchmarkState.STOPPED
  elif run_state == "crashed":
- status = Status.CRASHED
- elif len(submissions) == len(benchmarks) + len(invalid):
- status = Status.DONE
- elif not submissions or (not average_benchmark_time and benchmarks):
- if 12 <= now.hour < 13:
- status = Status.INITIALIZING
- else:
- status = Status.BROKEN
+ benchmarking_state = BenchmarkState.CRASHED
+ elif run_state == "failed":
+ benchmarking_state = BenchmarkState.FAILED
+ elif "benchmarking_state" in run.summary:
+ benchmarking_state = BenchmarkState[run.summary.get("benchmarking_state")]
+ else:
+ benchmarking_state = BenchmarkState.OUT_OF_DATE
+
+ for invalid_uid, reason in invalid.items():
+ if not reason.startswith("Duplicate of UID"):
+ benchmarks.add(invalid_uid)
 
  block = subtensor.get_current_block()
  data[uid] = State(
- status=status,
+ benchmarking_state=benchmarking_state,
  hotkey=run.config["hotkey"],
  name=get_identity(uid) or run.config["hotkey"],
  version=run.tags[1][8:],
@@ -196,26 +195,26 @@ def get_data() -> gr.Dataframe:
  now = datetime.now(tz=ZoneInfo("US/Pacific"))
 
  for uid, state in data.items():
- eta = int(state.average_benchmark_time * (state.submissions - (state.benchmarks + state.invalid)))
+ eta = int(state.average_benchmark_time * (state.submissions - state.benchmarks))
  time_left = timedelta(seconds=eta)
  eta_date = now + time_left
- eta_time = eta_date.strftime("%Y-%m-%d %I:%M:%S %p") if eta > 0 and state.status == Status.BENCHMARKING else state.status.get_alt_time_text()
+ eta_time = eta_date.strftime("%Y-%m-%d %I:%M:%S %p") if eta > 0 and state.benchmarking_state == BenchmarkState.IN_PROGRESS else state.benchmarking_state.get_alt_time_text()
 
- average_time_text = f"{timedelta(seconds=int(state.average_benchmark_time))}" if state.average_benchmark_time else state.status.get_alt_time_text(),
+ average_time_text = f"{timedelta(seconds=int(state.average_benchmark_time))}" if state.average_benchmark_time else state.benchmarking_state.get_alt_time_text(),
 
  elements.append((
  uid,
  state.name,
  f"<span style='color: {'springgreen' if state.version == latest_version else 'red'}'>{state.version}</span>",
- f"<span style='color: {state.status.color()}'>{state.status.name()}</span>",
- state.benchmarks + state.invalid,
+ f"<span style='color: {state.benchmarking_state.color()}'>{state.benchmarking_state.name()}</span>",
+ state.benchmarks,
  state.submissions,
  state.invalid,
  f"<span style='color: {'orange' if state.average_benchmark_time > AVERAGE_BENCHMARK_TIME_WARNING_THRESHOLD else 'springgreen'}'>{average_time_text[0]}</span>",
  f"<span style='color: {'orange' if eta > ETA_WARNING_THRESHOLD else 'springgreen'}'>{eta_time}</span>",
- f"<span style='color: {'orange' if eta > ETA_WARNING_THRESHOLD else 'springgreen'}'>{time_left if eta > 0 and state.status == Status.BENCHMARKING else state.status.get_alt_time_text()}</span>",
- f"<span style='color: {'springgreen' if state.vtrust > 0.75 else 'red'}'>{state.vtrust:.4f}</span>",
- f"<span style='color: {'springgreen' if state.updated < 1000 else 'red'}'>{state.updated}</span>",
+ f"<span style='color: {'orange' if eta > ETA_WARNING_THRESHOLD else 'springgreen'}'>{time_left if eta > 0 and state.benchmarking_state == BenchmarkState.IN_PROGRESS else state.benchmarking_state.get_alt_time_text()}</span>",
+ f"<span style='color: {'springgreen' if state.vtrust > VTRUST_WARNING_THRESHOLD else 'red'}'>{state.vtrust:.4f}</span>",
+ f"<span style='color: {'springgreen' if state.updated < UPDATED_WARNING_THRESHOLD else 'red'}'>{state.updated}</span>",
  ))
 
  return gr.Dataframe(
@@ -228,7 +227,7 @@ def get_data() -> gr.Dataframe:
 
 
 def main():
- with demo:
+ with gr.Blocks(css=".typewriter {font-family: 'JMH Typewriter', sans-serif;}", fill_height=True, fill_width=True) as demo:
  table = gr.Dataframe()
  table.attach_load_event(lambda: get_data(), None)
  demo.launch()