import streamlit as st import pandas as pd import wandb import datetime def get_all_competition_summary(api, projects): data = { "Number of competitions": 0, "Number of model runs": 0, "Last updated": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } for project in projects: entity = projects[project]["entity"] project_name = projects[project]["project"] runs = api.runs(f"{entity}/{project_name}") data["Number of competitions"] += 1 data["Number of model runs"] += len(runs) return data def fetch_competition_summary(api, projects, selected_project): data = [] st.write(f"Fetching competition summary for {selected_project}") entity = projects[selected_project]["entity"] project = projects[selected_project]["project"] runs = api.runs(f"{entity}/{project}") st.write(f"Number of runs to process: {len(runs)}") for run in runs: try: summary = run.summary if summary.get("validator_id") and summary.get("winning_hotkey"): data.append({ "ID": run.id, "Validator ID": summary.get("validator_id"), "Winning Hotkey": summary.get("winning_hotkey"), "Run Time (s)": summary.get("run_time_s"), "Created At": run.created_at, }) except Exception as e: st.write(f"Error processing run {run.id}: {str(e)}") df = pd.DataFrame(data) if not df.empty: df['Created At'] = pd.to_datetime(df['Created At']) df = df.sort_values(by="Created At", ascending=False) st.write(f"Competition summary data: {len(df)} rows") return df def fetch_models_evaluation(api, projects, selected_project): data = [] st.write(f"Fetching models evaluation for {selected_project}") entity = projects[selected_project]["entity"] project = projects[selected_project]["project"] runs = api.runs(f"{entity}/{project}") st.write(f"Number of runs to process: {len(runs)}") for run in runs: try: summary = run.summary if summary.get("accuracy") is not None: # Assuming runs with accuracy are model evaluations data.append({ "ID": run.id, "Username": run.user.username, "Model Name": summary.get("model_name", "N/A"), "Hotkey": summary.get("hotkey", "N/A"), "Score": summary.get("score"), "F1-beta": summary.get("f1-beta"), "Accuracy": summary.get("accuracy"), "Recall": summary.get("recall"), "ROC AUC": summary.get("roc_auc"), "Created At": run.created_at, }) except Exception as e: st.write(f"Error processing run {run.id}: {str(e)}") df = pd.DataFrame(data) if not df.empty: df['Created At'] = pd.to_datetime(df['Created At']) df = df.sort_values(by="Created At", ascending=False) st.write(f"Models evaluation data: {len(df)} rows") return df def update_leader_info(leader_info, competition, best_model): current_date = datetime.now().strftime("%Y-%m-%d") if leader_info.get(competition) is None: leader_info[competition] = { "Username": best_model["Username"], "Model Name": best_model["Model Name"], "Hotkey": best_model["Hotkey"], "Date": current_date, "UID": best_model["ID"], "Days on Top": 1 } else: if leader_info[competition]["UID"] == best_model["ID"]: leader_info[competition]["Days on Top"] += 1 else: leader_info[competition] = { "Username": best_model["Username"], "Model Name": best_model["Model Name"], "Hotkey": best_model["Hotkey"], "Date": current_date, "UID": best_model["ID"], "Days on Top": 1 } return leader_info[competition]