missing

src/leaderboard/read_evals.py

@@ -447,6 +447,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
     all_models = []
     missing_results_for_task = {}
     missing_metadata = []
+    for_run=[]
     for v in eval_results.values():
         r = v.to_dict()
         for task in Tasks:
@@ -454,8 +455,12 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
                 task_name = f"{r['n_shot']}|{task.value.benchmark}"
                 if task_name in missing_results_for_task:
                     missing_results_for_task[task_name].append(f"{v.full_model}|{v.org_and_model}")
+                    if v.still_on_hub and task.value.benchmark in all_tasks:
+                        print(f'batch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
                 else:
                     missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
+                    if v.still_on_hub and task.value.benchmark in all_tasks:
+                        print(f'batch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
         if r[AutoEvalColumn.lang.name] is None or r[AutoEvalColumn.lang.name] == "?":
             missing_metadata.append(f"{v.full_model}")
         all_models.append((v.full_model, v.num_params, v.still_on_hub))