Point to the QUEUE_REPO, not eval

app.py

@@ -105,7 +105,7 @@ LEADERBOARD_DF = get_leaderboard_df(COLS, BENCHMARK_COLS)
     finished_eval_queue_df,
     running_eval_queue_df,
     pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+) = get_evaluation_queue_df(EVAL_COLS)
 
 # Function to update the leaderboard
 def update_leaderboard():
@@ -367,7 +367,7 @@ with demo:
                     logger.info(f"Added request for {model} to {QUEUE_REPO}")
 
                     # Get updated pending evaluations
-                    _, _, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+                    _, _, pending_eval_queue_df = get_evaluation_queue_df(EVAL_COLS)
 
                     # Start processing queue in background
                     scheduler.add_job(process_evaluation_queue, id='process_queue_job', replace_existing=True)
@@ -402,7 +402,7 @@ with demo:
 
 # Update evaluation tables periodically
 def update_evaluation_tables():
-    finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+    finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_COLS)
     return finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
 # Setup schedulers

src/populate.py

@@ -10,7 +10,7 @@ from src.leaderboard.read_evals import get_raw_eval_results
 logger = logging.getLogger(__name__)
 
 from huggingface_hub import HfApi
-from src.config import RESULTS_REPO
+from src.config import RESULTS_REPO, QUEUE_REPO
 
 def get_leaderboard_df(cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
@@ -86,18 +86,26 @@ def get_leaderboard_df(cols: list, benchmark_cols: list) -> pd.DataFrame:
     return df
 
 
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    print(f"Looking for eval requests in {save_path}")
+def get_evaluation_queue_df(cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requests"""
+    logger.info(f"Looking for eval requests in {QUEUE_REPO}")
     all_evals = []
 
-    # Walk through all directories recursively
-    for root, _, files in os.walk(save_path):
-        for file in files:
-            if file.endswith('.json'):
-                file_path = os.path.join(root, file)
-                print(f"Reading JSON file: {file_path}")
-                with open(file_path) as fp:
+    api = HfApi()
+
+    try:
+        # List all files in the repository
+        files = api.list_repo_files(repo_id=QUEUE_REPO, repo_type="dataset")
+        
+        # Filter for JSON files
+        json_files = [f for f in files if f.endswith('.json')]
+        
+        for file in json_files:
+            try:
+                # Download and read each JSON file
+                content = api.hf_hub_download(repo_id=QUEUE_REPO, filename=file, repo_type="dataset")
+                logger.info(f"Reading JSON file: {file}")
+                with open(content, 'r') as fp:
                     data = json.load(fp)
 
                 # Check if data is a list (multiple requests in one file)
@@ -109,6 +117,11 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                     # Single request in the file
                     formatted_data = format_eval_data(data)
                     all_evals.append(formatted_data)
+            except Exception as e:
+                logger.error(f"Error processing file {file}: {str(e)}", exc_info=True)
+
+    except Exception as e:
+        logger.error(f"Error fetching requests from {QUEUE_REPO}: {str(e)}", exc_info=True)
 
     print(f"Found {len(all_evals)} total eval requests")
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]