intel results accesible in the leaderboard

app.py

@@ -4,6 +4,7 @@ from src.assets import custom_css
 
 # from src.attention import create_attn_plots
 from src.content import ABOUT, CITATION_BUTTON, CITATION_BUTTON_LABEL, LOGO, TITLE
+from src.hardware import load_hardware_configs
 from src.leaderboard import create_leaderboard_table
 from src.llm_perf import get_llm_perf_df
 from src.map import create_lat_score_mem_plot
@@ -13,15 +14,7 @@ from src.panel import (
     create_select_callback,
 )
 
-# from custom_kernels import create_quant_krnl_plots
-
-MACHINE_TO_HARDWARE = {
-    "1xA10": "A10-24GB-150W 🖥️",
-    "1xA100": "A100-80GB-275W 🖥️",
-    "1xT4": "T4-16GB-70W 🖥️",
-    "intel": "4th-Gen-Intel-Xeon-385W 🖥️",
-    # "1xH100": "H100-80GB-700W 🖥️",
-}
+configs = load_hardware_configs("hardware.yml")
 
 
 demo = gr.Blocks(css=custom_css)
@@ -30,12 +23,13 @@ with demo:
     gr.HTML(TITLE, elem_classes="title")
     ####################### HARDWARE TABS #######################
     with gr.Tabs(elem_classes="tabs"):
-        for id, (machine, hardware) in enumerate(MACHINE_TO_HARDWARE.items()):
-            with gr.TabItem(hardware, id=id):
-                ####################### CONTROL PANEL #######################
+        for id, config in enumerate(configs):
+            with gr.TabItem(config.description, id=id):
+                # ####################### CONTROL PANEL #######################
                 (
                     filter_button,
                     machine_textbox,
+                    subsets_values,
                     score_slider,
                     memory_slider,
                     backend_checkboxes,
@@ -43,17 +37,18 @@ with demo:
                     optimization_checkboxes,
                     quantization_checkboxes,
                     kernels_checkboxes,
-                ) = create_control_panel(machine=machine)
+                ) = create_control_panel(machine=config.machine, subsets=config.subsets)
                 ####################### HARDWARE SUBTABS #######################
                 with gr.Tabs(elem_classes="subtabs"):
-                    open_llm_perf_df = get_llm_perf_df(machine=machine)
+                    open_llm_perf_df = get_llm_perf_df(machine=config.machine, subsets=config.subsets)
                     ####################### LEADERBOARD TAB #######################
                     with gr.TabItem("Leaderboard 🏅", id=0):
                         search_bar, columns_checkboxes, leaderboard_table = (
                             create_leaderboard_table(open_llm_perf_df)
                         )
-                    with gr.TabItem("Find Your Best Model 🧭", id=1):
-                        lat_score_mem_plot = create_lat_score_mem_plot(open_llm_perf_df)
+                    if config.hardware_type != "intel": # TODO intel CPU does not measure the memory requirements correctly, so disable the graph feature until we fix the underlying issue
+                        with gr.TabItem("Find Your Best Model 🧭", id=1):
+                            lat_score_mem_plot = create_lat_score_mem_plot(open_llm_perf_df)
                     ###################### ATTENTIONS SPEEDUP TAB #######################
                     # with gr.TabItem("Attention 📈", id=2):
                     #     attn_prefill_plot, attn_decode_plot = create_attn_plots(
@@ -70,6 +65,7 @@ with demo:
                     filter_button,
                     # inputs
                     machine_textbox,
+                    subsets_values,
                     score_slider,
                     memory_slider,
                     backend_checkboxes,
@@ -92,6 +88,7 @@ with demo:
                 create_select_callback(
                     # inputs
                     machine_textbox,
+                    subsets_values,
                     # interactive
                     columns_checkboxes,
                     search_bar,
@@ -100,7 +97,7 @@ with demo:
                 )
 
         ####################### ABOUT TAB #######################
-        with gr.TabItem("About 📖", id=3):
+        with gr.TabItem("About 📖", id=len(configs)):
             gr.Markdown(ABOUT, elem_classes="descriptive-text")
     ####################### CITATION
     with gr.Row():

src/llm_perf.py

@@ -1,7 +1,10 @@
 import os
+from typing import List
 
 import pandas as pd
 
+from src.hardware import HardwareConfig
+
 from .utils import process_kernels, process_quantizations
 
 DATASET_DIRECTORY = "dataset"
@@ -28,13 +31,12 @@ COLUMNS_MAPPING = {
     "#Params (B)": "Params (B)",
 }
 SORTING_COLUMNS = ["Open LLM Score (%)", "Decode (tokens/s)", "Prefill (s)"]
-SUBSETS = ["unquantized", "awq", "bnb", "gptq"]
 SORTING_ASCENDING = [False, True, False]
 
 
-def get_raw_llm_perf_df(machine: str = "1xA10"):
+def get_raw_llm_perf_df(machine: str, subsets: List[str]):
     dfs = []
-    for subset in SUBSETS:
+    for subset in subsets:
         try:
             dfs.append(
                 pd.read_csv(
@@ -110,14 +112,14 @@ def processed_llm_perf_df(llm_perf_df):
     return llm_perf_df
 
 
-def get_llm_perf_df(machine: str = "1xA10"):
+def get_llm_perf_df(machine: str, subsets: List[str]):
     if not os.path.exists(DATASET_DIRECTORY):
         os.makedirs(DATASET_DIRECTORY)
 
     if os.path.exists(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv"):
         llm_perf_df = pd.read_csv(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv")
     else:
-        llm_perf_df = get_raw_llm_perf_df(machine)
+        llm_perf_df = get_raw_llm_perf_df(machine, subsets)
         llm_perf_df = processed_llm_perf_df(llm_perf_df)
         llm_perf_df.to_csv(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv", index=False)
 

src/panel.py

@@ -1,3 +1,5 @@
+from typing import List
+
 import gradio as gr
 
 from src.leaderboard import get_leaderboard_df
@@ -8,9 +10,10 @@ from src.llm_perf import get_llm_perf_df
 from src.map import get_lat_score_mem_fig
 
 
-def create_control_panel(machine: str):
+def create_control_panel(machine: str, subsets: List[str]):
     # controls
     machine_textbox = gr.Textbox(value=machine, visible=False)
+    subsets_values = gr.State(value=subsets)
     with gr.Accordion("Control Panel 🎛️", open=False, elem_id="control-panel"):
         with gr.Row():
             with gr.Column(scale=2, variant="panel"):
@@ -107,6 +110,7 @@ def create_control_panel(machine: str):
     return (
         filter_button,
         machine_textbox,
+        subsets_values,
         score_slider,
         memory_slider,
         backend_checkboxes,
@@ -119,6 +123,7 @@ def create_control_panel(machine: str):
 
 def filter_rows_fn(
     machine,
+    subsets,
     # inputs
     score,
     memory,
@@ -131,7 +136,7 @@ def filter_rows_fn(
     columns,
     search,
 ):
-    llm_perf_df = get_llm_perf_df(machine=machine)
+    llm_perf_df = get_llm_perf_df(machine=machine, subsets=subsets)
     # print(attentions)
     # print(llm_perf_df["Attention 👁️"].unique())
     filtered_llm_perf_df = llm_perf_df[
@@ -145,7 +150,7 @@ def filter_rows_fn(
         & (llm_perf_df["Memory (MB)"] <= memory)
     ]
     selected_filtered_llm_perf_df = select_columns_fn(
-        machine, columns, search, filtered_llm_perf_df
+        machine, subsets, columns, search, filtered_llm_perf_df
     )
     selected_filtered_lat_score_mem_fig = get_lat_score_mem_fig(filtered_llm_perf_df)
     # filtered_bt_prefill_fig = get_bt_prefill_fig(filtered_df)
@@ -172,6 +177,7 @@ def create_control_callback(
     filter_button,
     # fixed
     machine_textbox,
+    subsets_textbox,
     # inputs
     score_slider,
     memory_slider,
@@ -198,6 +204,7 @@ def create_control_callback(
         inputs=[
             # fixed
             machine_textbox,
+            subsets_textbox,
             # inputs
             score_slider,
             memory_slider,
@@ -223,9 +230,9 @@ def create_control_callback(
     )
 
 
-def select_columns_fn(machine, columns, search, llm_perf_df=None):
+def select_columns_fn(machine, subsets, columns, search, llm_perf_df=None):
     if llm_perf_df is None:
-        llm_perf_df = get_llm_perf_df(machine=machine)
+        llm_perf_df = get_llm_perf_df(machine=machine, subsets=subsets)
 
     selected_leaderboard_df = get_leaderboard_df(llm_perf_df)
     selected_leaderboard_df = selected_leaderboard_df[
@@ -239,6 +246,7 @@ def select_columns_fn(machine, columns, search, llm_perf_df=None):
 def create_select_callback(
     # fixed
     machine_textbox,
+    subsets_values,
     # interactive
     columns_checkboxes,
     search_bar,
@@ -247,11 +255,11 @@ def create_select_callback(
 ):
     columns_checkboxes.change(
         fn=select_columns_fn,
-        inputs=[machine_textbox, columns_checkboxes, search_bar],
+        inputs=[machine_textbox, subsets_values, columns_checkboxes, search_bar],
         outputs=[leaderboard_table],
     )
     search_bar.change(
         fn=select_columns_fn,
-        inputs=[machine_textbox, columns_checkboxes, search_bar],
+        inputs=[machine_textbox, subsets_values, columns_checkboxes, search_bar],
         outputs=[leaderboard_table],
     )