Update app.py

app.py

@@ -20,6 +20,7 @@ from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.feature_extraction.text import TfidfVectorizer
 import nltk
 from nltk.corpus import stopwords
+from huggingface_hub import download_url
 
 nltk.download('stopwords')
 
@@ -234,7 +235,7 @@ global_data = {
 model_configs = [
     {"repo_id": "Hjgugugjhuhjggg/testing_semifinal-Q2_K-GGUF", "filename": "testing_semifinal-q2_k.gguf", "name": "testing"},
     {"repo_id": "bartowski/Llama-3.2-3B-Instruct-uncensored-GGUF", "filename": "Llama-3.2-3B-Instruct-uncensored-Q2_K.gguf", "name": "Llama-3.2-3B-Instruct"},
-    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-70B-Q2_K-GGUF", "filename": "meta-llama-3.1-70b-q2_k.gguf", "name": "Meta-Llama-3.1-70B"},
+    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-70B", "filename": "meta-llama-3.1-70B", "name": "Meta-Llama-3.1-70B"},
     {"repo_id": "Hhhbvvkgh/Heidi-Llama-v4-Q2_K-GGUF", "filename": "heidi-llama-v4-q2_k.gguf", "name": "Heidi-Llama-V4"}
 ]
 
@@ -244,14 +245,24 @@ def normalize_input(input_text):
     filtered_words = [word for word in words if word.lower() not in stop_words]
     return " ".join(filtered_words)
 
+async def load_models():
+    for model in model_configs:
+        model_path = os.path.join("models", model["filename"])
+        if not os.path.exists(model_path):
+            url = f"https://huggingface.co/{model['repo_id']}/resolve/main/{model['filename']}"
+            download_url(url, model_path)
+        global_data['models'][model['name']] = Llama(model_path)
+
 async def generate_model_response(model, inputs):
     try:
         response = await model.generate(inputs)
         return response
     except Exception as e:
-        return ""
+        return {"error": str(e)}
 
 def get_best_response(responses):
+    if not responses:
+        return {"error": "No valid responses from models."}
     return max(responses, key=lambda x: x['score'])
 
 async def process_message(message):
@@ -287,6 +298,7 @@ def run_gradio():
     iface.launch(server_port=7862, prevent_thread_lock=True)
 
 if __name__ == "__main__":
+    asyncio.run(load_models())
     Thread(target=run_uvicorn).start()
     Thread(target=run_gradio).start()
     asyncio.get_event_loop().run_forever()