HF pipelines, Model submission msg bugfix,

README.md

@@ -1,14 +1,13 @@
 ---
-title: QANTA 2025 Quizbowl Submission and Leaderboard
+title: QANTA 2025 Agent Creator and Submission
 emoji: 🥇
 colorFrom: green
 colorTo: indigo
 sdk: gradio
+sdk_version: 5.31.0
 app_file: app.py
 pinned: true
 license: gpl-3.0
 hf_oauth: true
 short_description: Quizbowl Agent Creator and Submission platform.
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -4,7 +4,6 @@ import sys
 import datasets
 import gradio as gr
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
 from loguru import logger
 
 from app_configs import DEFAULT_SELECTIONS, THEME
@@ -23,20 +22,18 @@ from envs import (
     CONTACT_EMAIL,
     DISCORD_URL,
     DOCS_REPO_URL,
-    EVAL_REQUESTS_PATH,
-    EVAL_RESULTS_PATH,
     GITHUB_ISSUES_URL,
-    LEADERBOARD_REFRESH_INTERVAL,
     LEADERBOARD_URL,
+    LOCAL_REQUESTS_PATH,
     LOG_LEVEL,
     PLAYGROUND_DATASET,
-    QUEUE_REPO,
+    QUEUE_SYNC_INTERVAL,
     REGISTRATION_URL,
     REPO_ID,
-    RESULTS_REPO,
+    REQUESTS_REPO,
     SERVER_RESTART_INTERVAL,
 )
-from hf_datasets_utils import download_dataset_snapshot
+from hf_datasets_utils import check_and_create_dataset_repo, download_dataset_snapshot
 from shared.workflows import factory
 from shared.workflows.configs import AVAILABLE_MODELS
 from shared.workflows.llms import llm_cache
@@ -51,8 +48,6 @@ def restart_space():
 logger.remove()
 logger.add(sys.stdout, level=LOG_LEVEL, diagnose=False)
 
-download_dataset_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH)
-
 
 def filter_qids(qid: str, packet_ids: list[int]) -> bool:
     packet_id = int(qid.split("-")[-2])
@@ -92,6 +87,8 @@ def presave_pipeline_state(
 
 
 if __name__ == "__main__":
+    check_and_create_dataset_repo(REQUESTS_REPO)
+    download_dataset_snapshot(REQUESTS_REPO, LOCAL_REQUESTS_PATH)
     scheduler = BackgroundScheduler()
     scheduler.add_job(restart_space, "interval", seconds=SERVER_RESTART_INTERVAL)
     scheduler.start()
@@ -145,8 +142,8 @@ if __name__ == "__main__":
                     **DEFAULT_SELECTIONS["bonus"], init_workflow=factory.create_simple_qb_bonus_workflow()
                 )
                 bonus_interface = BonusInterface(demo, browser_state, bonus_ds, AVAILABLE_MODELS, defaults)
-            # with gr.Tab("🤗 HuggingFace Pipelines", elem_id="hf-pipeline-tab", id="hf-pipeline-tab"):
-            #     hf_pipeline_interface = create_hf_pipeline_submission_interface(demo)
+            with gr.Tab("🤗 HuggingFace Pipelines", elem_id="hf-pipeline-tab", id="hf-pipeline-tab"):
+                hf_pipeline_interface = create_hf_pipeline_submission_interface(demo)
             with gr.Tab("❓ Help", id="help"):
                 with gr.Row():
                     with gr.Column():

shared/workflows

@@ -1 +1 @@
-Subproject commit 8517e9923b959264b1380df1930a246951f0ba07
+Subproject commit 9e8f88dd705d8a9d0a05bef4018782131606461d

src/components/hf_pipelines.py

@@ -1,64 +1,196 @@
 import gradio as gr
+from huggingface_hub import HfApi
+from loguru import logger
 
-from submission.submit import create_hf_submission
+from display.formatting import styled_error
+from submission.submit import submit_hf_pipeline_agent, validate_model_name
+
+
+def error_html(msg: str, model_id) -> str:
+    error_msg = msg.format(
+        model_id=f'<code style="color:#222;background:#f2f2f2;padding:2px 4px;border-radius:4px;">{model_id}</code>'
+    )
+    return (
+        f"<div style='background-color:#ffeaea; color:#b30000; border:1px solid #ffcccc; "
+        f"padding:10px; border-radius:6px; font-size:16px; text-align:left;'>"
+        f"{error_msg}"
+        f"</div>"
+    )
+
+
+def verify_pipeline(model_id: str) -> tuple[bool, str]:
+    """
+    Check if model_id is a public, non-gated model repo on Hugging Face Hub.
+    Returns (success: bool, error_message: str)
+    """
+
+    try:
+        api = HfApi()
+        # Get model info
+        info = api.model_info(model_id)
+        # Check if model is public and not gated
+        is_public = info.private is False
+        is_gated = getattr(info, "gated", False)
+        if not is_public or is_gated:
+            if not is_public:
+                return False, error_html("Model {model_id} is private. Please make your model public.", model_id)
+            if is_gated:
+                return False, error_html("Model {model_id} is gated. Please use a non-gated model.", model_id)
+        return True, ""
+    except Exception as e:
+        logger.exception(e)
+        return False, error_html(
+            "Could not verify model {model_id}. Please check if the model is public and not gated.", model_id
+        )
+
+
+def attempt_submission(model_id: str, description: str, competition_type: str, profile: gr.OAuthProfile | None):
+    if profile is None:
+        return styled_error("Authentication required. Please log in first to submit your model.")
+
+    if "/" not in model_id:
+        full_model_id = f"{profile.username}/{model_id}"
+
+    elif model_id.split("/")[0] != profile.username:
+        return error_html(f"Model {{model_id}} is not owned by you (username: {profile.username}).", model_id)
+    else:
+        full_model_id = model_id
+        model_id = model_id.split("/", 1)[1]
+        valid, msg = validate_model_name(model_id)
+        if not valid:
+            return error_html(msg, model_id)
+
+    success, msg = verify_pipeline(full_model_id)
+    if not success:
+        return msg
+    try:
+        return submit_hf_pipeline_agent(model_id, description, competition_type, profile)
+    except Exception as e:
+        return styled_error(f"Error: Could not submit model '{model_id}': {e}")
+
+
+def create_model_submission_panel(app: gr.Blocks, competition_type: str):
+    with gr.Column():
+        model_name_input = gr.Textbox(
+            label="Hugging Face Model ID",
+            placeholder="<yourname/my-qbt-model> OR <my-qbt-model>",
+        )
+        description_input = gr.Textbox(
+            label="Pipeline Description",
+            placeholder="e.g. My QBT model is a simple model that uses a pipeline to predict the answer to a question.",
+        )
+
+        submit_btn = gr.Button("📤 Submit", interactive=False)
+
+    submit_status = gr.HTML(label="Submission Status", visible=False)
+
+    def check_user_login(profile: gr.OAuthProfile | None):
+        if profile is not None:
+            return gr.update(interactive=True, value="📤 Submit")
+        return gr.update(interactive=False, value="🔒 Login to submit")
+
+    gr.on(triggers=app.load, fn=check_user_login, inputs=[], outputs=[submit_btn])
+    submit_btn.click(
+        attempt_submission,
+        inputs=[model_name_input, description_input, gr.State(competition_type)],
+        outputs=[submit_status],
+        concurrency_limit=1,
+    )
+    return model_name_input, description_input, submit_btn, submit_status
 
 
 def create_hf_pipeline_submission_interface(demo: gr.Blocks):
-    with gr.Tabs():
-        with gr.TabItem("Tossup"):
+    gr.Markdown(
+        """
+        # Submit Your Hugging Face Pipeline Model
+
+        Welcome to the Hugging Face pipeline submission interface for the QANTA 2025 competition!
+        This page allows you to submit your models for both Tossup and Bonus tasks.
+
+        **General Requirements:**
+        - Your model must be a public, non-gated repository on the Hugging Face Hub.
+        - Ensure your model can be loaded using the `pipeline()` function from the `transformers` library.
+        - Adhere to the specified input/output formats for each task.
+
+        For help getting started, check out our [Starter Code](https://github.com/qanta-challenge/qanta25-starter).
+
+        You can also refer to Hugging Face's [custom pipeline creation guide](https://huggingface.co/docs/transformers/en/add_new_pipeline) for more information on how to create a custom pipeline.
+
+        Select the appropriate tab below based on the type of question your model is designed for.
+        """
+    )
+    with gr.Row():
+        with gr.Column():
             gr.Markdown(
                 """
-                # 📝 QuizBowl Tossup (QBT) – Submit your model
+                ## 🛎️ QuizBowl Tossup – Submit your model
+
+                Tossup questions are individual questions progressively revealed where you need to provide an answer, a confidence score, and decide whether to buzz.
+
+                **Pipeline Loading:**
+                Your model repository **must** be loadable with:
+                ```python
+                from transformers import pipeline
+                model = pipeline(task="quizbowl-tossup", model="<your-username/your-repo-name>")
+                ```
 
-                1. Your repo **must load with**
-                `pipeline(task=\"quizbowl-tossup\", model=\"<repo-id>\")`.
-                2. The pipeline must return
-                `{"answer": str, "confidence": float, "buzz": bool}`.
-                3. Runtime limit: **15 min / 1k examples** (T4 GPU).
+                **Input:**
+                The pipeline will receive a dictionary with the key `question_text` (string) which contains the progressively revealed question so far.
+                ```python
+                {
+                    "question_text": "In 1900, this city hosted a world's fair that introduced the public to the first escalator. Its famous tower, designed by Gustave Eiffel, was initially criticized by artists but is now a global icon. Name this European capital."
+                }
+                ```
 
-                Enter the <username>/<repo-name> below and hit **Evaluate**.
+                **Output:**
+                Similar to our agents, the pipeline **must** return a dictionary with the following keys:
+                ```python
+                {
+                    "answer": <str: concise answer>,
+                    "confidence": <float: confidence score between 0.0 and 1.0>,
+                    "buzz": <bool: True if your model decides to buzz in with the answer, False otherwise>
+                }
+                ```
+                Enter your Hugging Face model repository ID (`<your-username/your-repo-name>`) and a brief description below, then click "Submit".
                 """
             )
-            with gr.Row():
-                tossup_model_box = gr.Textbox(
-                    label="Hugging Face repo or model ID", placeholder="e.g. yourname/my-qbt-model"
-                )
-                tossup_submit_btn = gr.Button("Evaluate")
-            tossup_output_json = gr.JSON(label="Tossup Metrics")
-            tossup_submit_btn.click(
-                lambda model_id: create_hf_submission(model_id, model_id, "", "tossup"),
-                inputs=tossup_model_box,
-                outputs=[tossup_output_json],
-                concurrency_limit=1,
-            )
+            create_model_submission_panel(demo, "tossup")
 
-        with gr.TabItem("Bonus"):
+        with gr.Column():
             gr.Markdown(
                 """
-                # 🎁 QuizBowl Bonus (QBB) – Submit your model
+                ## 🧐 QuizBowl Bonus – Submit your model
+
+                Bonus questions consist of a lead-in paragraph followed by multiple parts. Your model will be called for each part.
 
-                1. Your repo **must load with**
-                `pipeline(task=\"quizbowl-bonus\", model=\"<repo-id>\")`.
-                2. The pipeline must return
-                `{"answer": str, "confidence": float, "explanation": str}`.
-                3. Runtime limit: **15 min / 1k examples** (T4 GPU).
+                **Pipeline Loading:**
+                Your model repository **must** be loadable with:
+                ```python
+                from transformers import pipeline
+                model = pipeline(task="quizbowl-bonus", model="<your-username/your-repo-name>")
+                ```
 
-                - Each bonus question has a `leadin` and a `part`.
-                - Your model will be called for each part with the same leadin and a different part.
-                - The output should include a brief explanation for human collaboration.
+                **Input:**
+                The pipeline will receive a dictionary with two keys:
+                - `leadin` (str): The introductory paragraph for the bonus question.
+                - `part` (str): The specific part of the bonus question to answer.
+                ```python
+                {
+                    "leadin": "This author wrote about a young wizard attending a magical school.",
+                    "part": "For 10 points, name this author."
+                }
+                ```
 
-                Enter the <username>/<repo-name> below and hit **Evaluate**.
+                **Output:**
+                Similar to our agents, the pipeline **must** return a dictionary with the following keys for each part:
+                ```python
+                {
+                    "answer": <str: concise answer to the part>,
+                    "confidence": <float: confidence score between 0.0 and 1.0>,
+                    "explanation": <str: brief explanation (<=30 words) for your answer (useful for human collaboration)>
+                }
+                ```
+                Enter your Hugging Face model repository ID (`<your-username/your-repo-name>`) and a brief description below, then click "Submit".
                 """
             )
-            with gr.Row():
-                bonus_model_box = gr.Textbox(
-                    label="Hugging Face repo or model ID", placeholder="e.g. yourname/my-qbb-model"
-                )
-                bonus_submit_btn = gr.Button("Evaluate")
-            bonus_output_json = gr.JSON(label="Bonus Metrics")
-            bonus_submit_btn.click(
-                lambda model_id: create_hf_submission(model_id, model_id, "", "bonus"),
-                inputs=bonus_model_box,
-                outputs=[bonus_output_json],
-                concurrency_limit=1,
-            )
+            create_model_submission_panel(demo, "bonus")

src/components/quizbowl/bonus.py

@@ -291,7 +291,8 @@ class BonusInterface:
         """Submit the model output."""
         try:
             workflow = self.validator.validate_state_dict(state_dict, playground=False)
-            return submit.submit_model(model_name, description, workflow, "bonus", profile)
+            msg = submit.submit_workflow_agent(model_name, description, "bonus", workflow, profile)
+            return gr.update(visible=True, value=msg)
         except Exception as e:
             logger.exception(f"Error submitting model: {e.args}")
             return gr.update(visible=True, value=styled_error(f"Error: {str(e)}"))

src/components/quizbowl/tossup.py

@@ -332,7 +332,8 @@ class TossupInterface:
         """Submit the model output."""
         try:
             workflow = self.validator.validate_state_dict(state_dict, playground=False)
-            return submit.submit_model(model_name, description, workflow, "tossup", profile)
+            msg = submit.submit_workflow_agent(model_name, description, "tossup", workflow, profile)
+            return gr.update(visible=True, value=msg)
         except Exception as e:
             logger.exception(f"Error submitting model: {e.args}")
             return gr.update(visible=True, value=styled_error(f"Error: {str(e)}"))

src/envs.py

@@ -2,54 +2,60 @@ import os
 
 from huggingface_hub import HfApi
 
-# Info to change for your repository
-# ----------------------------------
+# --------------* Secrets *----------------------------------------
 TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
+ENV_NAME = os.getenv("ENV_NAME", "test")  # Use advcal for production, test for testing
 OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
 ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY")
 COHERE_API_KEY = os.environ.get("COHERE_API_KEY")
 
+# ------------------------ ENV VARS --------------------------------
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+HF_HOME = os.getenv("HF_HOME", ".hf")
+
 # Change to your org - don't forget to create a results and request dataset, with the correct format!
 OWNER = "qanta-challenge"
 
+# --------------* READ-ONLY REPOS *---------------------------------
 REPO_ID = f"{OWNER}/quizbowl-submission"
-QUEUE_REPO = f"{OWNER}/advcal-requests"
 RESULTS_REPO = f"{OWNER}/advcal-results"
-LLM_CACHE_REPO = f"{OWNER}/advcal-llm-cache"
 USERS_REPO = f"{OWNER}/registered-users"
-EVAL_SPLITS = ["tiny_eval"]
 
-# Important Links
-QANTA_WEBSITE_URL = "https://sites.google.com/view/qanta/home"
-COMPETITION_URL = "https://sites.google.com/view/qanta/2025-competition"
-DOCS_REPO_URL = "https://github.com/qanta-challenge/QANTA25"
-DOCS_URL = DOCS_REPO_URL + "/tree/main"
-GITHUB_ISSUES_URL = DOCS_REPO_URL + "/issues"
+# --------------* READ-WRITE REPOS *--------------------------------
+REQUESTS_REPO = f"{OWNER}/{ENV_NAME}-requests"
+LLM_CACHE_REPO = f"{OWNER}/{ENV_NAME}-llm-cache"
 
-CONTACT_EMAIL = "[email protected]"
-DISCORD_URL = "https://discord.gg/ChmDVatJ6Y"
-REGISTRATION_URL = "https://huggingface.co/spaces/qanta-challenge/register"
-LEADERBOARD_URL = "https://huggingface.co/spaces/qanta-challenge/leaderboard"
-EXAMPLES_PATH = "examples"
 
 PLAYGROUND_DATASET = f"{OWNER}/acf-co24"
 
 # ----------------------------------
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH = os.getenv("HF_HOME", ".")
-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+CACHE_PATH = ".cache"
+EXAMPLES_PATH = "examples"
 
 # Local caches
-LLM_CACHE_PATH = os.path.join(CACHE_PATH, "llm-cache")
-USERS_PATH = os.path.join(CACHE_PATH, "registered-users")
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+LLM_CACHE_PATH = os.path.join(HF_HOME, "llm-cache")
+USERS_PATH = os.path.join(HF_HOME, "registered-users")
+LOCAL_REQUESTS_PATH = os.path.join(HF_HOME, "eval-queue")
+LOCAL_RESULTS_PATH = os.path.join(HF_HOME, "eval-results")
+LITELLM_CACHE_DIR = os.getenv("LITELLM_CACHE_DIR", f"{CACHE_PATH}/litellm-cache")
 
 
+# --------------* IMPORTANT LINKS *--------------------------------
+QANTA_WEBSITE_URL = "https://sites.google.com/view/qanta/home"
+COMPETITION_URL = "https://sites.google.com/view/qanta/2025-competition"
+DOCS_REPO_URL = "https://github.com/qanta-challenge/QANTA25"
+DOCS_URL = DOCS_REPO_URL + "/tree/main"
+GITHUB_ISSUES_URL = DOCS_REPO_URL + "/issues"
+
+CONTACT_EMAIL = "[email protected]"
+DISCORD_URL = "https://discord.gg/ChmDVatJ6Y"
+REGISTRATION_URL = "https://huggingface.co/spaces/qanta-challenge/register"
+LEADERBOARD_URL = "https://huggingface.co/spaces/qanta-challenge/leaderboard"
+
 LLM_CACHE_REFRESH_INTERVAL = 600  # seconds (30 minutes)
+QUEUE_SYNC_INTERVAL = 600  # seconds (10 minutes)
 SERVER_RESTART_INTERVAL = 2 * 24 * 60 * 60  # seconds (2 days)
 LEADERBOARD_REFRESH_INTERVAL = 600  # seconds (10 minutes)
 

src/hf_datasets_utils.py

@@ -4,6 +4,15 @@ from loguru import logger
 api = HfApi()
 
 
+def check_and_create_dataset_repo(repo_id: str):
+    try:
+        api.repo_info(repo_id=repo_id, repo_type="dataset")
+        print(f"{repo_id} exists")
+    except Exception:
+        print(f"Creating {repo_id}")
+        api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True, private=True)
+
+
 def download_dataset_snapshot(repo_id, local_dir):
     try:
         logger.info(f"Downloading dataset snapshot from {repo_id} to {local_dir}")

src/submission/_submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )

src/submission/submit.py

@@ -12,14 +12,14 @@ from loguru import logger
 
 from app_configs import DAILY_SUBMISSION_LIMIT_PER_USER
 from display.formatting import styled_error, styled_message
-from envs import API, EVAL_REQUESTS_PATH, EXAMPLES_PATH, OWNER, QUEUE_REPO
+from envs import API, EXAMPLES_PATH, LOCAL_REQUESTS_PATH, OWNER, REQUESTS_REPO
 from shared.workflows.structs import TossupWorkflow, Workflow
-from submission.structs import CompetitionType, Submission, SubmissionStatus
+from submission.structs import CompetitionType, Submission, SubmissionStatus, SubmissionType
 
 
 def get_user_submissions(username: str, competition_type: str, pattern: str = None) -> list[Submission]:
     """Get all submissions for a user."""
-    out_dir = f"{EVAL_REQUESTS_PATH}/{username}"
+    out_dir = f"{LOCAL_REQUESTS_PATH}/{username}"
     submissions = []
     if not os.path.exists(out_dir):
         return submissions
@@ -57,7 +57,7 @@ def get_user_submissions_by_date(
 ) -> dict[str, list[Submission]]:
     """Get all submissions for a user for a given competition type."""
     date_str = datetime(year, month, day).strftime("%Y%m%d")
-    out_dir = f"{EVAL_REQUESTS_PATH}/{username}"
+    out_dir = f"{LOCAL_REQUESTS_PATH}/{username}"
     if not os.path.exists(out_dir):
         return {}
     submissions = []
@@ -171,11 +171,12 @@ def validate_model_name(model_name: str):
     return True, ""
 
 
-def submit_model(
+def submit_agent(
     model_name: str,
     description: str,
-    workflow: Workflow,
     competition_type: CompetitionType,
+    submission_type: SubmissionType,
+    workflow: Workflow | None,
     profile: gr.OAuthProfile | None,
 ) -> str:
     """
@@ -212,33 +213,49 @@ def submit_model(
         return styled_error(f"Submission Error! Invalid model name '{model_name}'.<br>{error_msg}")
 
     try:
-        submission = create_workflow_submission(
-            username=username,
-            model_name=model_name,
-            description=description,
-            workflow=workflow,
-            competition_type=competition_type,
-        )
+        if submission_type == "simple_workflow":
+            submission = create_workflow_submission(
+                username=username,
+                model_name=model_name,
+                description=description,
+                workflow=workflow,
+                competition_type=competition_type,
+            )
+        elif submission_type == "hf_pipeline":
+            submission = create_hf_submission(
+                username=username,
+                model_name=model_name,
+                description=description,
+                competition_type=competition_type,
+            )
+        else:
+            return styled_error(f"Unsupported submission type: {submission_type}")
+
         # Convert to dictionary format
         submission_dict = submission.to_dict()
 
         # Create output directory path
-        out_dir = f"{EVAL_REQUESTS_PATH}/{username}"
-        out_path = f"{out_dir}/{submission.id}.json"
+
+        rel_filepath = f"{username}/{submission.id}.json"
+        filepath = f"{LOCAL_REQUESTS_PATH}/{rel_filepath}"
+
+        os.makedirs(os.path.dirname(filepath), exist_ok=True)
+        with open(filepath, "w") as f:
+            json.dump(submission_dict, f, indent=2)
 
         # Upload to HuggingFace dataset
         API.upload_file(
-            path_or_fileobj=json.dumps(submission_dict, indent=2).encode(),
-            path_in_repo=out_path.split("eval-queue/")[1],
-            repo_id=QUEUE_REPO,
+            path_or_fileobj=filepath,
+            path_in_repo=rel_filepath,
+            repo_id=REQUESTS_REPO,
             repo_type="dataset",
-            commit_message=f"Add tossup submission {submission.id}",
+            commit_message=f"Add {competition_type} {submission_type} submission {submission.id}",
         )
 
         return styled_message(
-            f"Successfully submitted tossup model!<br>"
+            f"Successfully submitted {competition_type} {submission_type} model!<br>"
             f"Submission name: {username}/{model_name}<br>"
-            f"Please wait for up to an hour for the model to show in the PENDING list."
+            f"Please wait for up to one day for the model to show up on the leaderboard."
         )
 
     except Exception as e:
@@ -246,6 +263,27 @@ def submit_model(
         return styled_error(f"Error submitting model: {str(e)}")
 
 
+def submit_hf_pipeline_agent(
+    model_name: str,
+    description: str,
+    competition_type: CompetitionType,
+    profile: gr.OAuthProfile | None,
+) -> str:
+    return submit_agent(model_name, description, competition_type, "hf_pipeline", workflow=None, profile=profile)
+
+
+def submit_workflow_agent(
+    model_name: str,
+    description: str,
+    competition_type: CompetitionType,
+    workflow: Workflow,
+    profile: gr.OAuthProfile | None,
+) -> str:
+    return submit_agent(
+        model_name, description, competition_type, "simple_workflow", workflow=workflow, profile=profile
+    )
+
+
 def load_demo_example(model_name: str, competition_type: CompetitionType) -> Workflow | TossupWorkflow:
     """Load a demo example submission."""
     examples_dir = f"{EXAMPLES_PATH}/{competition_type}"
@@ -288,12 +326,10 @@ if __name__ == "__main__":
     )
 
     # Submit model
-    result = submit_model(
+    result = submit_agent(
         model_name="GPT-4 Tossup",
         description="A simple GPT-4 model for tossup questions",
         workflow=workflow,
         competition_type="tossup",
     )
     print(result)
-
-# %%