|
import json |
|
import os |
|
import glob |
|
import yaml |
|
|
|
import click |
|
from colorama import Fore |
|
from huggingface_hub import HfApi, snapshot_download |
|
|
|
EVAL_REQUESTS_PATH = "eval-queue" |
|
QUEUE_REPO = "cot-leaderboard/cot-leaderboard-requests" |
|
READMES = ["README.md", "../open_cot_dashboard/README.md"] |
|
|
|
precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ") |
|
model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned") |
|
weight_types = ("Original", "Delta", "Adapter") |
|
|
|
|
|
def update_readme(readme_path: str, models: list[str]): |
|
|
|
with open(readme_path, "r") as f: |
|
lines = f.readlines() |
|
lines = [line.rstrip() for line in lines] |
|
if not lines: |
|
raise ValueError(f"Readme file {readme_path} is empty") |
|
if not lines[0].startswith("---"): |
|
raise ValueError(f"Readme file {readme_path} does not start with a metadata block") |
|
lines = lines[1:] |
|
if "---" not in lines: |
|
raise ValueError(f"Readme file {readme_path} does not close the metadata block") |
|
lines = lines[:lines.index("---")] |
|
models_idx = lines.index("models:") |
|
if any(not line.startswith(" - ") for line in lines[models_idx + 1:]): |
|
raise ValueError(f"Readme file {readme_path} does not have a valid list of models") |
|
lines = lines[:models_idx + 1] |
|
for model in models: |
|
lines.append(f" - {model}") |
|
lines = ["---"] + lines + ["---"] |
|
|
|
with open(readme_path, "w") as f: |
|
f.write("\n".join(lines)+"\n") |
|
|
|
|
|
def main(): |
|
api = HfApi() |
|
snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset") |
|
|
|
eval_requests = [] |
|
|
|
for file in glob.glob(f"{EVAL_REQUESTS_PATH}/**/*.json", recursive=True): |
|
with open(file, "r") as f: |
|
eval_requests.append(json.load(f)) |
|
|
|
models_evaluated = [ |
|
eval_request['model'] for eval_request in eval_requests |
|
if eval_request["status"] == "FINISHED" |
|
] |
|
models_evaluated.sort() |
|
|
|
print(models_evaluated) |
|
|
|
for readme in READMES: |
|
update_readme(readme, models_evaluated) |
|
|
|
|
|
if __name__ == "__main__": |
|
main() |
|
|