File size: 5,161 Bytes
f7832ad
 
6322afe
fe0c719
f8ebe3d
cc1b673
 
 
2136d22
cc1b673
dbccbb1
cc1b673
dbccbb1
cc1b673
 
 
dbccbb1
 
 
 
 
 
cc1b673
dbccbb1
 
 
bc0d85d
 
f8ebe3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2136d22
da95145
37341c1
2136d22
 
da95145
 
 
 
 
 
a73977e
f7832ad
27a3e14
a73977e
da95145
 
f7832ad
da95145
 
f7832ad
da95145
 
 
 
 
f7832ad
6322afe
2136d22
1638e8f
da95145
2136d22
 
da95145
 
 
 
1638e8f
da95145
a73977e
8171f9f
858de24
 
 
 
 
 
da95145
919a76a
 
 
a73977e
 
da95145
 
a73977e
 
 
 
da95145
a73977e
bca906c
858de24
bca906c
a73977e
da95145
 
 
 
 
 
 
 
 
 
1638e8f
a73977e
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
import streamlit as st
import pandas as pd
import wandb
import time
from datetime import datetime, timedelta
import requests


def get_competitions(CONFIG_URL):
    """
    Get competition names and their evaluation times from the config URL.
    """
    competitions = []
    try:
        config = requests.get(CONFIG_URL).json()
        for competition in config:
            # Extract competition name and evaluation times
            competition_name = competition["competition_id"]
            evaluation_times = competition.get("evaluation_times", [])
            
            # Store competition name and evaluation times as a tuple
            competitions.append((competition_name, evaluation_times))
    except Exception as e:
        print(f"Error loading competition data: {str(e)}")
    
    return competitions


def get_latest_evaluation_time(evaluation_times):
    """
    Get the latest UTC evaluation time as a datetime object (either today or yesterday).
    
    Args:
        evaluation_times (list): List of evaluation times as strings (in HH:MM format).
        
    Returns:
        datetime: Latest evaluation time as a datetime object.
    """
    
    # Get the current UTC date and time
    current_utc_datetime = datetime.utcnow()
    current_utc_time = current_utc_datetime.time()
    
    # Convert evaluation times to datetime.time objects
    eval_times = [datetime.strptime(time, "%H:%M").time() for time in evaluation_times]
    
    # Sort the evaluation times to ensure they are in chronological order
    eval_times.sort()
    
    # Loop through the evaluation times in reverse to find the latest time that has passed
    for eval_time in reversed(eval_times):
        if current_utc_time >= eval_time:
            # If the time has passed today, return today's date with that time
            return current_utc_datetime.replace(hour=eval_time.hour, minute=eval_time.minute, second=0, microsecond=0)
    
    # If none of the evaluation times have passed, return the latest one from the previous day
    yesterday = current_utc_datetime - timedelta(days=1)
    latest_eval_time = eval_times[-1]
    
    return yesterday.replace(hour=latest_eval_time.hour, minute=latest_eval_time.minute, second=0, microsecond=0)


def fetch_competition_summary(api, entity, project):
    data = []
        
    # entity = projects[selected_project]["entity"]
    # project = projects[selected_project]["project"]
    
    runs = api.runs(f"{entity}/{project}")
    
    for run in runs:
        try:
            summary = run.summary
            if summary.get("validator_hotkey") and summary.get("winning_hotkey"):
                data.append({
                    "Created At": run.created_at,
                    "Validator ID": summary.get("validator_hotkey"),
                    "Winning Hotkey": summary.get("winning_hotkey"),
                    "Run Time (s)": summary.get("run_time_s"),
                })
        except Exception as e:
            st.write(f"Error processing run {run.id}: {str(e)}")
    
    df = pd.DataFrame(data)
    if not df.empty:
        df['Created At'] = pd.to_datetime(df['Created At'])
        df = df.sort_values(by="Created At", ascending=False)
    
    return df

def fetch_models_evaluation(api, entity, project):
    data = []
    
    # entity = projects[selected_project]["entity"]
    # project = projects[selected_project]["project"]
    
    runs = api.runs(f"{entity}/{project}")
    
    for run in runs:
        try:
            summary = run.summary
            if summary.get("score") is not None:  # Assuming runs with score are model evaluations
                model_link = summary.get("hf_model_link", "")
                if model_link:
                    # Create clickable link
                    model_link_html = f'<a href="{model_link}" target="_blank">Model Link</a>'
                else:
                    model_link_html = "N/A"

                data.append({
                    "Created At": run.created_at,

                    "Miner hotkey": summary.get("miner_hotkey", "N/A"),

                    "F1-beta": summary.get("fbeta"),
                    "Accuracy": summary.get("accuracy"),
                    "Recall": summary.get("recall"),
                    "Precision": summary.get("precision"),

                    "Tested entries": summary.get("tested_entries"),

                    "ROC AUC": summary.get("roc_auc"),
                    "Confusion Matrix": summary.get("confusion_matrix"),

                    "Model link": model_link_html,
                    "Score": summary.get("score"),
                    #TODO link to huggingface model
                })
        except Exception as e:
            st.write(f"Error processing run {run.id}: {str(e)}")
    
    df = pd.DataFrame(data)
    if not df.empty:
        df['Created At'] = pd.to_datetime(df['Created At'])
        df = df.sort_values(by="Created At", ascending=False)
    
    return df

def highlight_score_column(s):
    """
    Highlight the 'Score' column with a custom background color.
    """
    return ['background-color: yellow' if s.name == 'Score' else '' for _ in s]