Only HF

app.py

@@ -26,20 +26,15 @@ CLIENT_MODEL_KWARGS = {
 }
 
 GUIDE_KWARGS = {
-    "expert_model": "accounts/fireworks/models/llama-v3p1-70b-instruct",
+    "expert_model": "HuggingFaceH4/zephyr-7b-beta",
                   # "meta-llama/Meta-Llama-3.1-70B-Instruct",
-                  # "accounts/fireworks/models/nous-hermes-2-mixtral-8x7b-dpo-fp8",
-                  # "accounts/fireworks/models/llama-v3-8b-instruct-hf",
-                  # "accounts/fireworks/models/nous-hermes-2-mixtral-8x7b-dpo-fp8",
-    "inference_server_url": "https://api.fireworks.ai/inference/v1",
+    "inference_server_url": "https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta",
                   # "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3.1-70B-Instruct",
-                  # "https://api.fireworks.ai/inference/v1",
-    "llm_backend": "Fireworks",
+    "llm_backend": "HFChat",
     "classifier_kwargs": {
         "model_id": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
-        "inference_server_url": "https://sa710i91bnjvbhir.us-east-1.aws.endpoints.huggingface.cloud",
-        # "inference_server_url": "https://api-inference.huggingface.co/models/MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
-        "batch_size": 128,
+        "inference_server_url": "https://api-inference.huggingface.co/models/MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli",
+        "batch_size": 8,
     },
 }
 
@@ -190,7 +185,7 @@ async def bot(
     if len(history_langchain_format) <= 1:
 
         guide_kwargs = copy.deepcopy(GUIDE_KWARGS)
-        guide_kwargs["api_key"] = os.getenv("FW_TOKEN")  # expert model api key
+        guide_kwargs["api_key"] = os.getenv("HF_TOKEN")  # expert model api key
         guide_kwargs["classifier_kwargs"]["api_key"] = os.getenv("HF_TOKEN")  # classifier api key
 
         guide_config = RecursiveBalancingGuideConfig(**guide_kwargs)