Change to dataset, remove upload_file

app.py

@@ -7,6 +7,7 @@ import os
 import logging
 import json
 from datetime import datetime
+from datasets import Dataset
 
 from src.core.evaluation import EvaluationManager, EvaluationRequest
 from src.logging_config import setup_logging
@@ -111,15 +112,15 @@ def update_leaderboard():
 def process_evaluation_queue():
     """Process pending evaluation requests."""
     logger.info("Processing evaluation queue")
-    
+
     # Fetch pending requests from Hugging Face repository
     _, _, pending_requests = get_evaluation_queue_df(EVAL_COLS + ['model_raw'])
-    
+
     for _, request in pending_requests.iterrows():
         try:
             model_name = request['model_raw']
             logger.info(f"Processing request for model: {model_name}")
-            
+
             # Convert queue request to evaluation request
             eval_request = EvaluationRequest(
                 model=model_name,
@@ -153,23 +154,21 @@ def update_request_status(model_name, status):
     try:
         api = HfApi()
         filename = f"{model_name.replace('/', '_')}_request.json"
-        
+
         # Fetch the current request data
         file_content = api.hf_hub_download(repo_id=QUEUE_REPO, filename=filename, repo_type="dataset")
         with open(file_content, 'r') as f:
             request_data = json.load(f)
-        
+
         # Update the status
         request_data['status'] = status
-        
-        # Upload the updated file
-        api.upload_file(
-            path_or_fileobj=json.dumps(request_data).encode(),
-            path_in_repo=filename,
-            repo_id=QUEUE_REPO,
-            repo_type="dataset",
-            token=TOKEN
-        )
+
+        # Create a Dataset object from the updated request data
+        dataset = Dataset.from_dict(request_data)
+
+        # Push the updated dataset to the Hugging Face Hub
+        dataset.push_to_hub(QUEUE_REPO)
+
         logger.info(f"Updated status for {model_name} to {status}")
     except Exception as e:
         logger.error(f"Failed to update status for {model_name}: {str(e)}", exc_info=True)
@@ -181,25 +180,17 @@ from huggingface_hub import HfApi
 def save_results_to_repo(results, repo):
     """Save evaluation results to the specified repository."""
     try:
-        api = HfApi()
         model_id = results.get('model', '').replace('/', '_')
         if not model_id:
             raise ValueError("Model ID not found in results")
-        filename = f"{model_id}_results.json"
 
-        # Convert results to JSON string
-        json_results = json.dumps(results, indent=2)
+        # Create a Dataset object from the results
+        dataset = Dataset.from_dict(results)
 
-        # Save results to the repository
-        api.upload_file(
-            path_or_fileobj=json_results.encode(),
-            path_in_repo=filename,
-            repo_id=repo,
-            repo_type="dataset",
-            token=TOKEN
-        )
+        # Push the dataset to the Hugging Face Hub
+        dataset.push_to_hub(repo)
 
-        logger.info(f"Saved results for {model_id} to {repo}/{filename}")
+        logger.info(f"Saved results for {model_id} to {repo}")
     except Exception as e:
         logger.error(f"Failed to save results to {repo}: {str(e)}", exc_info=True)
 
@@ -230,7 +221,7 @@ def init_leaderboard(df):
         "safetensors_compliant": "Safetensors",
         "precision": "Precision"
     }
-    
+
     for src, dst in column_mapping.items():
         if src in df.columns:
             df[dst] = df[src]
@@ -376,7 +367,7 @@ with demo:
                 """Handle new model submission."""
                 try:
                     logger.info(f"New submission received for {model}")
-            
+
                     # Prepare request data
                     request_data = {
                         "model": model,
@@ -388,31 +379,26 @@ with demo:
                         "status": "PENDING",
                         "timestamp": datetime.now().isoformat()
                     }
-            
-                    # Add to queue in QUEUE_REPO
-                    api = HfApi()
-                    filename = f"{model.replace('/', '_')}_request.json"
-                    api.upload_file(
-                        path_or_fileobj=json.dumps(request_data).encode(),
-                        path_in_repo=filename,
-                        repo_id=QUEUE_REPO,
-                        repo_type="dataset",
-                        token=TOKEN
-                    )
-            
+
+                    # Convert request data to a Dataset
+                    dataset = Dataset.from_dict(request_data)
+
+                    # Push the dataset to the Hugging Face Hub
+                    dataset.push_to_hub(QUEUE_REPO)
+
                     logger.info(f"Added request for {model} to {QUEUE_REPO}")
-            
+
                     # Get updated pending evaluations
                     _, _, pending_eval_queue_df = get_evaluation_queue_df(EVAL_COLS)
-            
+
                     # Start processing queue in background
                     scheduler.add_job(process_evaluation_queue, id='process_queue_job', replace_existing=True)
-            
+
                     return gr.Markdown("Submission successful! Your model has been added to the evaluation queue. Please check the 'Pending Evaluation Queue' for status updates."), pending_eval_queue_df
                 except Exception as e:
                     logger.error(f"Submission failed: {str(e)}", exc_info=True)
                     return gr.Markdown(f"Error: {str(e)}"), None
-            
+
             # Remove the queue_manager initialization
             # queue_manager = QueueManager(queue_dir=os.path.join(CACHE_PATH, "eval-queue"))
 

src/submission/submit.py

@@ -22,6 +22,7 @@ from src.config import (
     LOG_LEVEL,
     EVALUATION_WAIT_TIME
 )
+from datasets import Dataset
 
 REQUESTED_MODELS: Optional[Dict[str, Any]] = None
 USERS_TO_SUBMISSION_DATES: Optional[Dict[str, Any]] = None
@@ -158,45 +159,23 @@ def add_new_eval(
 
     model_size = get_model_size(model_info=model_info, precision=precision)
 
-    eval_entry = create_eval_entry(model, base_model, revision, precision, weight_type, model_type, model_info, model_size)
-
     # Check for duplicate submission
     if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
 
-    logger.info("Creating eval file")
-    OUT_DIR = os.path.join(EVAL_REQUESTS_PATH, user_name)
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = os.path.join(OUT_DIR, f"{model_path}_eval_request_False_{precision}_{weight_type}.json")
+    logger.info("Creating eval entry")
+    eval_entry = create_eval_entry(model, base_model, revision, precision, weight_type, model_type, model_info, model_size)
 
-    try:
-        with open(out_path, "w") as f:
-            json.dump(eval_entry, f)
-    except IOError as e:
-        logger.error(f"Failed to write eval file: {e}")
-        return styled_error(f"Failed to create eval file: {e}")
+    # Convert eval entry to a Dataset
+    dataset = Dataset.from_dict(eval_entry)
 
-    logger.info("Uploading eval file")
+    logger.info("Uploading eval entry")
     try:
-        # Get the relative path from EVAL_REQUESTS_PATH
-        rel_path = os.path.relpath(out_path, EVAL_REQUESTS_PATH)
-        
-        API.upload_file(
-            path_or_fileobj=out_path,
-            path_in_repo=rel_path,
-            repo_id=QUEUE_REPO,
-            repo_type="dataset",
-            commit_message=f"Add {model} to eval queue",
-        )
+        # Push the dataset to the Hugging Face Hub
+        dataset.push_to_hub(QUEUE_REPO, commit_message=f"Add {model} to eval queue")
     except Exception as e:
-        logger.error(f"Failed to upload eval file: {e}")
-        return styled_error(f"Failed to upload eval file: {e}")
-
-    # Remove the local file
-    try:
-        os.remove(out_path)
-    except OSError as e:
-        logger.warning(f"Failed to remove local eval file: {e}")
+        logger.error(f"Failed to upload eval entry: {e}")
+        return styled_error(f"Failed to upload eval entry: {e}")
 
     return styled_message(
         f"Your request has been submitted to the evaluation queue!\n"