fix: show partial results even if some evaluations haven't finished

app.py

@@ -65,7 +65,7 @@ def update_table(
 
 def apply_format_styling(df: pd.DataFrame) -> style.Styler:
  df = df.style.format(
- {c: "{:.1f}" for c in BENCHMARK_COLS} | {AutoEvalColumn.average.name: "{:.2f}"}
+ {c: "{:.1f}" for c in BENCHMARK_COLS} | {AutoEvalColumn.average.name: "{:.2f}"}, na_rep="-"
  )
  return df
 

src/leaderboard/read_evals.py

@@ -124,7 +124,10 @@ class EvalResult:
  }
 
  for task in Tasks:
- data_dict[task.value.col_name] = self.results[task.value.benchmark]
+ if task.value.benchmark in self.results.keys():
+ data_dict[task.value.col_name] = self.results[task.value.benchmark]
+ else:
+ data_dict[task.value.col_name] = None
 
  return data_dict
 

src/populate.py

@@ -1,11 +1,12 @@
 import pandas as pd
 
-from src.display.formatting import has_no_nan_values
 from src.display.utils import AutoEvalColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+from src.leaderboard.read_evals import get_raw_eval_results, EvalResult
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df(
+ results_path: str, requests_path: str, cols: list, benchmark_cols: list
+) -> tuple[list[EvalResult], pd.DataFrame]:
  """Creates a dataframe from all the individual experiment results"""
  raw_data = get_raw_eval_results(results_path, requests_path)
  all_data_json = [v.to_dict() for v in raw_data]
@@ -14,6 +15,4 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
  df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
  df = df[cols].round(decimals=2)
 
- # filter out if any of the benchmarks have not been produced
- df = df[has_no_nan_values(df, benchmark_cols)]
  return raw_data, df