q8 model

app.py

@@ -2,9 +2,14 @@ import gradio as gr
 
 from llama_cpp import Llama, LlamaTokenizer
 
+# llm = Llama.from_pretrained(
+#    repo_id="ID2223JR/gguf_model",
+#    filename="unsloth.Q4_K_M.gguf",
+# )
+
 llm = Llama.from_pretrained(
-    repo_id="ID2223JR/gguf_model",
-    filename="unsloth.Q4_K_M.gguf",
+    repo_id="ID2223JR/gguf_model_q8",
+    filename="unsloth.Q8_0.gguf",
 )
 
 
@@ -56,21 +61,11 @@ def submit_to_model():
         content = ""
 
         for partial_response in response:
-            if "choices" in partial_response:
-                if "delta" in partial_response["choices"][0]:
-                    content += partial_response["choices"][0]["delta"].get(
-                        "content", ""
-                    )
-                elif "message" in partial_response["choices"][0]:
-                    content += partial_response["choices"][0]["message"].get(
-                        "content", ""
-                    )
-                else:
-                    content += partial_response["choices"][0].get("text", "")
-                if content:
-                    yield content
-            else:
-                yield "Unexpected response structure."
+
+            content += partial_response["choices"][0]["delta"].get("content", "")
+
+            if content:
+                yield content
 
         ingredients_list.clear()  # Reset list after generation