CPU, TOKEN, env variables (#4)

- Suggesting change for accidentally hard-coded environment variable (1190865ec3a7b3e156035aa2c793ba4ecd862235)
- Adds CPU/Float32 support; adds user access Token passing; fixes hard-coded environment variable. (24b20ff4285a655000d36f4525e97131930257e8)
- Just trying to open a PR without access to the org through the command line. (75324f0a44165244cedff94702842545307482b8)
- Creating branch of changes through command line. (b8617767cc29e44c44d696c108152d8c0e9da21f)
- Test (4e21c860af2562fa5ba7980b8b8a5684a5832387)
- Remove tmp file (7599d4bb85377600cc1b63549a92669e53e9a20f)
- Undo 1 change; keeping 'eval-queue' hard-coded. (cd4f3f159e844de76f32a79d75406a5ab18bfa30)
- Undo 1 change; keeping 'eval-queue' hard-coded. (c0ed1f14cdcf8b3c7e5fa32b64818dfc83bf9bfd)


Co-authored-by: Margaret Mitchell <[email protected]>

app.py

@@ -26,7 +26,7 @@ from src.display.utils import (
     WeightType,
     Precision
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, DEVICE, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
@@ -309,7 +309,7 @@ with demo:
                         choices=[i.value.name for i in Precision if i != Precision.Unknown],
                         label="Precision",
                         multiselect=False,
-                        value="float16",
+                        value="float16" if DEVICE != "cpu" else "float32",
                         interactive=True,
                     )
                     weight_type = gr.Dropdown(

main_backend.py

@@ -9,7 +9,7 @@ from src.backend.run_eval_suite import run_evaluation
 from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
 from src.backend.sort_queue import sort_models_by_priority
 
-from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, DEVICE, API, LIMIT
+from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, DEVICE, API, LIMIT, TOKEN
 from src.about import Tasks, NUM_FEWSHOT
 TASKS_HARNESS = [task.value.benchmark for task in Tasks]
 
@@ -21,8 +21,8 @@ RUNNING_STATUS = "RUNNING"
 FINISHED_STATUS = "FINISHED"
 FAILED_STATUS = "FAILED"
 
-snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
+snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60, token=TOKEN)
 
 def run_auto_eval():
     current_pending_status = [PENDING_STATUS]

scripts/create_request_file.py

@@ -7,11 +7,9 @@ from datetime import datetime, timezone
 import click
 from colorama import Fore
 from huggingface_hub import HfApi, snapshot_download
+from src.envs import TOKEN, EVAL_REQUESTS_PATH, QUEUE_REPO
 
-EVAL_REQUESTS_PATH = "eval-queue"
-QUEUE_REPO = "open-llm-leaderboard/requests"
-
-precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
+precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ", "float32")
 model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")
 weight_types = ("Original", "Delta", "Adapter")
 
@@ -36,7 +34,7 @@ def get_model_size(model_info, precision: str):
 def main():
     api = HfApi()
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset")
+    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", token=TOKEN)
 
     model_name = click.prompt("Enter model name")
     revision = click.prompt("Enter revision", default="main")

src/backend/manage_requests.py

@@ -26,7 +26,7 @@ class EvalRequest:
     def get_model_args(self):
         model_args = f"pretrained={self.model},revision={self.revision}"
 
-        if self.precision in ["float16", "bfloat16"]:
+        if self.precision in ["float16", "bfloat16", "float32"]:
             model_args += f",dtype={self.precision}"
         # Quantized models need some added config, the install of bits and bytes, etc
         #elif self.precision == "8bit":
@@ -71,7 +71,7 @@ def get_eval_requests(job_status: list, local_dir: str, hf_repo: str) -> list[Ev
     Returns:
         `list[EvalRequest]`: a list of model info dicts.
     """
-    snapshot_download(repo_id=hf_repo, revision="main", local_dir=local_dir, repo_type="dataset", max_workers=60)
+    snapshot_download(repo_id=hf_repo, revision="main", local_dir=local_dir, repo_type="dataset", max_workers=60, token=TOKEN)
     json_files = glob.glob(f"{local_dir}/**/*.json", recursive=True)
 
     eval_requests = []
@@ -97,7 +97,7 @@ def check_completed_evals(
     local_dir_results: str,
 ):
     """Checks if the currently running evals are completed, if yes, update their status on the hub."""
-    snapshot_download(repo_id=hf_repo_results, revision="main", local_dir=local_dir_results, repo_type="dataset", max_workers=60)
+    snapshot_download(repo_id=hf_repo_results, revision="main", local_dir=local_dir_results, repo_type="dataset", max_workers=60, token=TOKEN)
 
     running_evals = get_eval_requests(checked_status, hf_repo=hf_repo, local_dir=local_dir)
 

src/display/utils.py

@@ -94,6 +94,7 @@ class WeightType(Enum):
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
+    float32 = ModelDetails("float32")
     #qt_8bit = ModelDetails("8bit")
     #qt_4bit = ModelDetails("4bit")
     #qt_GPTQ = ModelDetails("GPTQ")
@@ -104,6 +105,8 @@ class Precision(Enum):
             return Precision.float16
         if precision in ["torch.bfloat16", "bfloat16"]:
             return Precision.bfloat16
+        if precision in ["float32"]:
+            return Precision.float32
         #if precision in ["8bit"]:
         #    return Precision.qt_8bit
         #if precision in ["4bit"]:

src/envs.py

@@ -7,7 +7,7 @@ from huggingface_hub import HfApi
 TOKEN = os.environ.get("TOKEN") # A read/write token for your org
 
 OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request file
-DEVICE = "cpu" # cuda:0 if you add compute
+DEVICE = "cpu" # "cuda:0" if you add compute
 LIMIT = 20 # !!!! Should be None for actual evaluations!!!
 # ----------------------------------
 

src/submission/check_validity.py

@@ -8,7 +8,7 @@ import huggingface_hub
 from huggingface_hub import ModelCard
 from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import tokenizer_class_from_name, get_tokenizer_config
+from transformers.models.auto.tokenization_auto import AutoTokenizer
 
 def check_model_card(repo_id: str) -> tuple[bool, str]:
     """Checks if the model card and license exist and have been filled"""

src/submission/submit.py

@@ -50,7 +50,7 @@ def add_new_eval(
             return styled_error(f'Base model "{base_model}" {error}')
 
     if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, test_tokenizer=True)
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')