{text}"
def add_column(self, column_name: str, formula: str):
"""Create and add a new column with the given formula."""
# If the user did not provide the name of the new column,
# generate a unique name for them.
if not column_name:
counter = 1
while (column_name := f"custom{counter}") in self.full_df.columns:
counter += 1
# If the user did not provide a formula, return an error message.
if not formula:
return self.cur_df, self._format_msg("Please enter a formula.")
# If there is an equal sign in the formula, `df.eval` will
# return an entire DataFrame with the new column, instead of
# just the new column. This is not what we want, so we check
# for this case and return an error message.
if "=" in formula:
return self.cur_df, self._format_msg("Invalid formula: expr cannot contain '='.")
# The user may want to update an existing column.
verb = "Updated" if column_name in self.full_df.columns else "Added"
# Evaluate the formula and catch any error.
try:
# Give the users some helper functions that can be used in the formula
# like "@sum(response_length)". Also wipe out some global variables.
col = self.full_df.eval(
formula,
local_dict={"sum": sum, "len": len, "max": max, "min": min},
global_dict={"global_tbm": None},
)
except Exception as exc:
return self.cur_df, self._format_msg(f"Invalid formula: {exc}")
# If the result is a numeric scalar, make it a Series.
# We may have deleted some models (rows) form the full dataframe when we
# called dropna, so we need to query the maximum index instead of taking len.
if isinstance(col, (int, float)):
col = pd.Series([col] * (self.full_df.index.max() + 1))
# We only accept numeric columns.
if not is_numeric_dtype(col):
return self.cur_df, self._format_msg("Invalid formula: result must be numeric.")
# Round if it's floating point.
if is_float_dtype(col):
col = col.round(2)
# If the column already exists, update it.
if column_name in self.full_df.columns:
self.full_df[column_name] = col
else:
self.full_df.insert(len(self.schema) + 1, column_name, col)
# If adding a column succeeded, `self.cur_df` should also be updated.
self.cur_df = self.full_df.loc[self.cur_index]
return self.cur_df, self._format_msg(f"{verb} column '{column_name}'.")
def get_dropdown(self):
columns = self.full_df.columns.tolist()[1:]
return [
gr.Dropdown(choices=columns, value="parameters", label="X"),
gr.Dropdown(choices=columns, value="energy", label="Y"),
gr.Dropdown(choices=["None", *columns], label="Z (optional)"),
]
def update_dropdown(self):
columns = self.full_df.columns.tolist()[1:]
return [
gr.Dropdown.update(choices=columns),
gr.Dropdown.update(choices=columns),
gr.Dropdown.update(choices=["None", *columns]),
]
def set_filter_get_df(self, *filters) -> pd.DataFrame:
"""Set the current set of filters and return the filtered DataFrame."""
# If the filter is empty, we default to the first choice for each key.
if not filters:
filters = [choices[:1] for choices in self.schema.values()]
index = np.full(len(self.full_df), True)
for setup, choice in zip(self.schema, filters):
index = index & self.full_df[setup].isin(choice)
self.cur_df = self.full_df.loc[index]
self.cur_index = index
return self.cur_df
def plot_scatter(self, width, height, x, y, z):
# The user did not select either x or y.
if not x or not y:
return None, width, height, self._format_msg("Please select both X and Y.")
# Width and height may be an empty string. Then we set them to 600.
if not width and not height:
width, height = "600", "600"
elif not width:
width = height
elif not height:
height = width
try:
width, height = int(width), int(height)
except ValueError:
return None, width, height, self._format_msg("Width and height should be positive integers.")
# Strip the tag from model names.
text = self.cur_df["model"].apply(lambda x: x.split(">")[1].split("<")[0])
# Hide model names since they clutter the plots, and only show them on hover.
if z is None or z == "None" or z == "":
fig = px.scatter(self.cur_df, x=x, y=y, hover_name=text)
else:
fig = px.scatter_3d(self.cur_df, x=x, y=y, z=z, hover_name=text)
fig.update_traces(marker=dict(size=12, line=dict(width=2, color="DarkSlateGrey")))
fig.update_layout(width=width, height=height)
return fig, width, height, ""
# The global instance of the TableManager should only be used when
# initializing components in the Gradio interface. If the global instance
# is mutated while handling user sessions, the change will be reflected
# in every user session. Instead, the instance provided by gr.State should
# be used.
global_tbm = TableManager("data")
# Fetch the latest update date of the leaderboard repository.
resp = requests.get("https://api.github.com/repos/ml-energy/leaderboard/commits/master")
if resp.status_code != 200:
current_date = "[Failed to fetch]"
print("Failed to fetch the latest release date of the leaderboard repository.")
print(resp.json())
else:
current_datetime = parser.parse(resp.json()["commit"]["author"]["date"])
current_date = current_datetime.astimezone(tz.gettz("US/Eastern")).strftime("%Y-%m-%d")
# Custom JS.
# XXX: This is a hack to make the model names clickable.
# Ideally, we should set `datatype` in the constructor of `gr.DataFrame` to
# `["markdown"] + ["number"] * (len(df.columns) - 1)` and format models names
# as an HTML tag. However, because we also want to dynamically add new
# columns to the table and Gradio < 4.0 does not support updating `datatype` with
# `gr.DataFrame.update` yet, we need to manually walk into the DOM and replace
# the innerHTML of the model name cells with dynamically interpreted HTML.
# Desired feature tracked at https://github.com/gradio-app/gradio/issues/3732
dataframe_update_js = f"""
function format_model_link() {{
// Iterate over the cells of the first column of the leaderboard table.
for (let index = 1; index <= {len(global_tbm.full_df)}; index++) {{
// Get the cell.
var cell = document.querySelector(
`#tab-leaderboard > div > div > div > table > tbody > tr:nth-child(${{index}}) > td:nth-child(1) > div > span`
);
// If nothing was found, it likely means that now the visible table has less rows
// than the full table. This happens when the user filters the table. In this case,
// we should just return.
if (cell == null) break;
// This check exists to make this function idempotent.
// Multiple changes to the Dataframe component may invoke this function,
// multiple times to the same HTML table (e.g., adding and sorting cols).
// Thus, we check whether we already formatted the model names by seeing
// whether the child of the cell is a text node. If it is not,
// it means we already parsed it into HTML, so we should just return.
if (cell.firstChild.nodeType != 3) break;
// Decode and interpret the innerHTML of the cell as HTML.
var decoded_string = new DOMParser().parseFromString(cell.innerHTML, "text/html").documentElement.textContent;
var temp = document.createElement("template");
temp.innerHTML = decoded_string;
var model_anchor = temp.content.firstChild;
// Replace the innerHTML of the cell with the interpreted HTML.
cell.replaceChildren(model_anchor);
}}
// Return all arguments as is.
return arguments
}}
"""
# Custom CSS.
custom_css = """
/* Make ML.ENERGY look like a clickable logo. */
.text-logo {
color: #23d175 !important;
text-decoration: none !important;
}
/* Make the submit button the same color as the logo. */
.btn-submit {
background: #23d175 !important;
color: white !important;
border: 0 !important;
}
/* Center the plotly plot inside its container. */
.plotly > div {
margin: auto !important;
}
/* Limit the width of the first column to 300 px. */
table td:first-child,
table th:first-child {
max-width: 300px;
overflow: auto;
white-space: nowrap;
}
/* Make tab buttons larger */
.tab-nav > button {
font-size: 18px !important;
}
/* Color texts. */
.green-text {
color: #23d175 !important;
}
.red-text {
color: #ff3860 !important;
}
/* Flashing model name borders. */
@keyframes blink {
0%, 33%, 67%, 100% {
border-color: transparent;
}
17%, 50%, 83% {
border-color: #23d175;
}
}
.model-name-text {
border: 2px solid transparent; /* Transparent border initially */
animation: blink 3s ease-in-out 1; /* One complete cycle of animation, lasting 3 seconds */
}
"""
intro_text = """
We used Zeus to benchmark various open source LLMs in terms of how much time and energy they consume for inference. Time and energy are of course not the only things we care about -- so we also benchmarked all of the models on a variety of NLP datasets, including the ARC Challenge (reasoning), HellaSwag (common sense), and TruthfulQA (truthfulness).
For more detailed information, please take a look at the About tab. Every benchmark is limited in some sense -- Before you interpret the results, please take a look at the Limitations section there, too.
""" # The app will not start without a controller address set. controller_addr = os.environ["COLOSSEUM_CONTROLLER_ADDR"] global_controller_client = ControllerClient(controller_addr=controller_addr, timeout=15) # Colosseum helper functions. def enable_interact(): return [gr.update(interactive=True)] * 2 def disable_interact(): return [gr.update(interactive=False)] * 2 def consumed_less_energy_message(energy_a, energy_b): """Return a message that indicates that the user chose the model that consumed less energy. By default report in "%f %" but if the difference is larger than 2 times, report in "%f X". """ less_energy = min(energy_a, energy_b) more_energy = max(energy_a, energy_b) factor = less_energy / more_energy if factor <= 0.5: message = f"