File size: 2,196 Bytes
b7dd05f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
import json
import os
import glob
import yaml
import click
from colorama import Fore
from huggingface_hub import HfApi, snapshot_download
EVAL_REQUESTS_PATH = "eval-queue"
QUEUE_REPO = "cot-leaderboard/cot-leaderboard-requests"
READMES = ["README.md", "../open_cot_dashboard/README.md"]
precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")
weight_types = ("Original", "Delta", "Adapter")
def update_readme(readme_path: str, models: list[str]):
# read lines from the readme
with open(readme_path, "r") as f:
lines = f.readlines()
lines = [line.rstrip() for line in lines]
if not lines:
raise ValueError(f"Readme file {readme_path} is empty")
if not lines[0].startswith("---"):
raise ValueError(f"Readme file {readme_path} does not start with a metadata block")
lines = lines[1:]
if "---" not in lines:
raise ValueError(f"Readme file {readme_path} does not close the metadata block")
lines = lines[:lines.index("---")]
models_idx = lines.index("models:")
if any(not line.startswith(" - ") for line in lines[models_idx + 1:]):
raise ValueError(f"Readme file {readme_path} does not have a valid list of models")
lines = lines[:models_idx + 1]
for model in models:
lines.append(f" - {model}")
lines = ["---"] + lines + ["---"]
# write lines back to the readme
with open(readme_path, "w") as f:
f.write("\n".join(lines)+"\n")
def main():
api = HfApi()
snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset")
eval_requests = []
for file in glob.glob(f"{EVAL_REQUESTS_PATH}/**/*.json", recursive=True):
with open(file, "r") as f:
eval_requests.append(json.load(f))
models_evaluated = [
eval_request['model'] for eval_request in eval_requests
if eval_request["status"] == "FINISHED"
]
models_evaluated.sort()
print(models_evaluated)
for readme in READMES:
update_readme(readme, models_evaluated)
if __name__ == "__main__":
main()
|