added inference api

app.py

@@ -1,15 +1,20 @@
 import gradio as gr
 from evaluation_logic import run_evaluation, AVAILABLE_PROMPT_FORMATS
 
-def gradio_run_evaluation(model_name, prompt_format):
+def gradio_run_evaluation(inference_api, model_name, prompt_format):
     output = []
-    for result in run_evaluation(str(model_name).strip(), prompt_format):
+    for result in run_evaluation(inference_api, str(model_name).strip(), prompt_format):
         output.append(result)
         yield "\n".join(output)
 
 with gr.Blocks() as demo:
     gr.Markdown("# DuckDB SQL Evaluation App")
 
+    inference_api = gr.Dropdown(
+        label="Inference API",
+        choices=['openrouter', 'hf_inference_api'], #AVAILABLE_PROMPT_FORMATS,
+        value="openrouter"
+    )
     model_name = gr.Textbox(label="Model Name (e.g., qwen/qwen-2.5-72b-instruct)")
     prompt_format = gr.Dropdown(
         label="Prompt Format",
@@ -19,6 +24,6 @@ with gr.Blocks() as demo:
     start_btn = gr.Button("Start Evaluation")
     output = gr.Textbox(label="Output", lines=20)
 
-    start_btn.click(fn=gradio_run_evaluation, inputs=[model_name, prompt_format], outputs=output)
+    start_btn.click(fn=gradio_run_evaluation, inputs=[inference_api, model_name, prompt_format], outputs=output)
 
 demo.queue().launch()

evaluation_logic.py

@@ -19,14 +19,14 @@ from eval.schema import TextToSQLParams, Table
 
 AVAILABLE_PROMPT_FORMATS = list(PROMPT_FORMATTERS.keys())
 
-def run_prediction(model_name, prompt_format, output_file):
+def run_prediction(inference_api, model_name, prompt_format, output_file):
     dataset_path = str(eval_dir / "data/dev.json")
     table_meta_path = str(eval_dir / "data/tables.json")
     stop_tokens = [';']
     max_tokens = 30000
     temperature = 0.1
     num_beams = -1
-    manifest_client = "openrouter"
+    manifest_client = inference_api
     manifest_engine = model_name
     manifest_connection = "http://localhost:5000"
     overwrite_manifest = True
@@ -95,10 +95,13 @@ def run_prediction(model_name, prompt_format, output_file):
         yield f"Prediction failed with error: {str(e)}"
         yield f"Error traceback: {traceback.format_exc()}"
 
-def run_evaluation(model_name, prompt_format="duckdbinstgraniteshort"):
+def run_evaluation(inference_api, model_name, prompt_format="duckdbinstgraniteshort"):
     if "OPENROUTER_API_KEY" not in os.environ:
         yield "Error: OPENROUTER_API_KEY not found in environment variables."
         return
+    if "HF_TOKEN" not in os.environ:
+        yield "Error: HF_TOKEN not found in environment variables."
+        return
 
     try:
         # Set up the arguments
@@ -119,7 +122,7 @@ def run_evaluation(model_name, prompt_format="duckdbinstgraniteshort"):
             yield "Skipping prediction step and proceeding to evaluation."
         else:
             # Run prediction
-            for output in run_prediction(model_name, prompt_format, output_file):
+            for output in run_prediction(inference_api, model_name, prompt_format, output_file):
                 yield output
 
         # Run evaluation