Various leaderboard fixes

src/leaderboard/read_evals.py

@@ -24,7 +24,7 @@ class EvalResult:
     results: dict
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
+    weight_type: WeightType = WeightType.PyTorch # Default to PyTorch if not specified
     architecture: str = "Unknown" 
     license: str = "?"
     likes: int = 0
@@ -86,7 +86,7 @@ class EvalResult:
             model=model,
             results=results,
             precision=precision,  
-            revision= config.get("model_sha", ""),
+            revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture
         )
@@ -99,7 +99,15 @@ class EvalResult:
             with open(request_file, "r") as f:
                 request = json.load(f)
             self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            
+            # Handle weight type conversion from old format
+            weight_type = request.get("weight_type", "PyTorch")
+            if weight_type == "Original":
+                weight_type = "PyTorch"
+            elif weight_type == "Adapter":
+                weight_type = "Other"
+            self.weight_type = WeightType[weight_type]
+            
             self.license = request.get("license", "?")
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)

src/submission/submit.py

@@ -9,6 +9,7 @@ from src.submission.check_validity import (
     check_model_card,
     get_model_size,
     is_model_on_hub,
+    check_safetensors_format,
 )
 
 REQUESTED_MODELS = None
@@ -44,15 +45,14 @@ def add_new_eval(
         revision = "main"
 
     # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
+    model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+    if not model_on_hub:
+        return styled_error(f'Model "{model}" {error}')
 
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
+    # Check if model uses safetensors format
+    safetensors_ok, error_msg = check_safetensors_format(model, revision, TOKEN)
+    if not safetensors_ok:
+        return styled_error(error_msg)
 
     # Is the model info correctly filled?
     try:
@@ -72,6 +72,23 @@ def add_new_eval(
     if not modelcard_OK:
         return styled_error(error_msg)
 
+    # Validate weight type
+    if weight_type not in ["Safetensors", "PyTorch", "Other"]:
+        return styled_error("Invalid weight type. Must be one of: Safetensors, PyTorch, Other")
+
+    # Force Safetensors for new submissions
+    if weight_type != "Safetensors":
+        return styled_error(
+            "Only Safetensors format is accepted for new submissions. Please convert your model using:\n"
+            "```python\n"
+            "from transformers import AutoModelForCausalLM\n"
+            "from safetensors.torch import save_file\n\n"
+            "model = AutoModelForCausalLM.from_pretrained('your-model')\n"
+            "state_dict = model.state_dict()\n"
+            "save_file(state_dict, 'model.safetensors')\n"
+            "```"
+        )
+
     # Seems good, creating the eval
     print("Adding new eval")
 
@@ -115,5 +132,9 @@ def add_new_eval(
     os.remove(out_path)
 
     return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+        "Your request has been submitted to the evaluation queue!\n"
+        "The model will be evaluated for:\n"
+        "1. Safetensors compliance\n"
+        "2. Security awareness using the stacklok/insecure-code dataset\n"
+        "Please wait for up to an hour for the model to show in the PENDING list."
     )