vllm

app.py

@@ -11,7 +11,6 @@ import time
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from huggingface_hub import snapshot_download
-from pytz import utc
 
 from src.display.about import (
     CITATION_BUTTON_LABEL,
@@ -160,7 +159,6 @@ def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precisio
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.inference_framework.name].isin(size_query)]
 
     # numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
     # params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
@@ -259,7 +257,7 @@ with demo:
                                 for c in fields(AutoEvalColumn)
                                 if c.displayed_by_default and not c.hidden and not c.never_hidden
                             ],
-                            label="Tasks",
+                            label="Select columns to show",
                             elem_id="column-select",
                             interactive=True,
                         )
@@ -479,7 +477,7 @@ with demo:
                 show_copy_button=True,
             )
 
-scheduler = BackgroundScheduler(timezone=utc)
+scheduler = BackgroundScheduler()
 
 scheduler.add_job(restart_space, "interval", hours=6)
 

backend-cli.py

@@ -458,7 +458,6 @@ def get_args():
     parser.add_argument("--gpu-type", type=str, default="NVIDIA-A100-PCIe-80GB", 
                         help="GPU type. NVIDIA-A100-PCIe-80GB; NVIDIA-RTX-A5000-24GB; NVIDIA-H100-PCIe-80GB")
     parser.add_argument("--debug_repo", action="store_true", help="Use debug repo")
-    parser.add_argument("--model_type", type=str, default="chat", help="Model type")
     return parser.parse_args()
 
 
@@ -489,8 +488,7 @@ if __name__ == "__main__":
                         json_filepath="",
                         precision=precision,  # Use precision from arguments
                         inference_framework=args.inference_framework,  # Use inference framework from arguments
-                        gpu_type=args.gpu_type,
-                        model_type=args.model_type,
+                        gpu_type=args.gpu_type
                     )
                     curr_gpu_type = get_gpu_details()
                     if eval_request.gpu_type != curr_gpu_type:

src/backend/tasks/arena_hard/task.py

@@ -72,7 +72,7 @@ class ArenaHard(ConfigurableTask):
         super().__init__(config={"metadata": {"version": self.VERSION}})
         # these end tokens are hard coded because of the current limitaion of the llm-eval.
         # self.generation_kwargs = {"until": ["\n\n", "<unk>", "<|im_end|>", "</s>", "<|endoftext|>"], "max_length": 512}
-        self.generation_kwargs = {"until": ["</s>", "<|im_end|>"], "max_gen_toks": 4096}
+        self.generation_kwargs = {"until": ["</s>", "<|im_end|>"], "max_length": 4096}
         # self.generation_kwargs_sampling_number = 5  # the number of sampling for self-consistence
         # self.generation_kwargs_sampling = {
         #     "temperature": 0.99,

src/display/about.py

@@ -19,8 +19,8 @@ Columns and Metrics:
 - E2E(s): Average End to End generation time in seconds.
 - PRE(s): Prefilling Time of input prompt in seconds.
 - T/s: Tokens throughout per second.
-- S-MBU(%): Sparse Model Bandwidth Utilization.
-- S-MFU(%): Sparse Model FLOPs Utilization.
+- MBU(%): Model Bandwidth Utilization.
+- MFU(%): Model FLOPs Utilization.
 - Precision: The precison of used model.
 
 """

src/display/utils.py

@@ -18,8 +18,8 @@ GPU_Power = 'Power(W)'
 GPU_Mem = 'Mem(G)'
 GPU_Name = "GPU"
 GPU_Util = 'Util(%)'
-MFU = 'S-MFU(%)'
-MBU = 'S-MBU(%)'
+MFU = 'MFU(%)'
+MBU = 'MBU(%)'
 BATCH_SIZE = 'bs'
 PRECISION = "Precision"
 system_metrics_to_name_map = {
@@ -106,7 +106,7 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 # # auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg", "number", True)])
 
 # Inference framework
-auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnContent(f"{InFrame}", "str", True, dummy=True)])
+auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnContent(f"{InFrame}", "str", True)])
 
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
@@ -126,15 +126,15 @@ for task in Tasks:
 
 
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False, dummy=True)])
-# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", True, dummy=True)])
-# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", True)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 
@@ -160,10 +160,10 @@ class ModelDetails:
 
 
 class ModelType(Enum):
-    # PT = ModelDetails(name="pretrained", symbol="🟢")
-    # FT = ModelDetails(name="fine-tuned on domain-specific datasets", symbol="🔶")
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned on domain-specific datasets", symbol="🔶")
     chat = ModelDetails(name="chat models (RLHF, DPO, IFT, ...)", symbol="💬")
-    # merges = ModelDetails(name="base merges and moerges", symbol="🤝")
+    merges = ModelDetails(name="base merges and moerges", symbol="🤝")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -171,24 +171,22 @@ class ModelType(Enum):
 
     @staticmethod
     def from_str(type):
-        # if "fine-tuned" in type or "🔶" in type:
-        #     return ModelType.FT
-        # if "pretrained" in type or "🟢" in type:
-        #     return ModelType.PT
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
         if any([k in type for k in ["instruction-tuned", "RL-tuned", "chat", "🟦", "⭕", "💬"]]):
             return ModelType.chat
-        # if "merge" in type or "🤝" in type:
-        #     return ModelType.merges
+        if "merge" in type or "🤝" in type:
+            return ModelType.merges
         return ModelType.Unknown
 
 
 class InferenceFramework(Enum):
     # "moe-infinity", hf-chat
-    # MoE_Infinity = ModelDetails("moe-infinity")
+    MoE_Infinity = ModelDetails("moe-infinity")
     HF_Chat = ModelDetails("hf-chat")
     VLLM = ModelDetails("vllm_moe")
-    TRTLLM = ModelDetails("tensorrt_llm")
-    VLLM_FIX = ModelDetails("vllm_moe_fixbs")
     Unknown = ModelDetails("?")
 
     def to_str(self):
@@ -196,16 +194,12 @@ class InferenceFramework(Enum):
 
     @staticmethod
     def from_str(inference_framework: str):
-        # if inference_framework in ["moe-infinity"]:
-        #     return InferenceFramework.MoE_Infinity
-        if inference_framework in ["tensorrt_llm"]:
-            return InferenceFramework.TRTLLM
+        if inference_framework in ["moe-infinity"]:
+            return InferenceFramework.MoE_Infinity
         if inference_framework in ["hf-chat"]:
             return InferenceFramework.HF_Chat
         if inference_framework in ["vllm_moe"]:
             return InferenceFramework.VLLM
-        if inference_framework in ["vllm_moe_fixbs"]:
-            return InferenceFramework.VLLM_FIX
         return InferenceFramework.Unknown
 
 class GPUType(Enum):
@@ -231,28 +225,28 @@ class WeightType(Enum):
 
 
 class Precision(Enum):
-    # float32 = ModelDetails("float32")
-    # float16 = ModelDetails("float16")
+    float32 = ModelDetails("float32")
+    float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
     qt_8bit = ModelDetails("8bit")
     qt_4bit = ModelDetails("4bit")
-    # qt_GPTQ = ModelDetails("GPTQ")
+    qt_GPTQ = ModelDetails("GPTQ")
     Unknown = ModelDetails("?")
 
     @staticmethod
     def from_str(precision: str):
-        # if precision in ["torch.float32", "float32"]:
-        #     return Precision.float32
-        # if precision in ["torch.float16", "float16"]:
-        #     return Precision.float16
+        if precision in ["torch.float32", "float32"]:
+            return Precision.float32
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
         if precision in ["torch.bfloat16", "bfloat16"]:
             return Precision.bfloat16
         if precision in ["8bit"]:
             return Precision.qt_8bit
         if precision in ["4bit"]:
             return Precision.qt_4bit
-        # if precision in ["GPTQ", "None"]:
-        #     return Precision.qt_GPTQ
+        if precision in ["GPTQ", "None"]:
+            return Precision.qt_GPTQ
         return Precision.Unknown
 
 

src/leaderboard/read_evals.py

@@ -140,7 +140,6 @@ class EvalResult:
             revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture,
-            model_type=ModelType.from_str(config.get("model_type", "")),
             inference_framework=inference_framework,
         )
 
@@ -175,22 +174,22 @@ class EvalResult:
 
         # breakpoint()
         # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        
+
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
-            # AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
             AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            # AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.dummy.name: self.full_model,
-            # AutoEvalColumn.revision.name: self.revision,
-            # # AutoEvalColumn.average.name: average,
-            # AutoEvalColumn.license.name: self.license,
-            # AutoEvalColumn.likes.name: self.likes,
-            # AutoEvalColumn.params.name: self.num_params,
-            # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumn.revision.name: self.revision,
+            # AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
             AutoEvalColumn.inference_framework.name: self.inference_framework,
         }