Update app.py

app.py

@@ -11,12 +11,24 @@ from huggingface_hub import hf_hub_download
 
 hf_hub_download(
     repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF",
-    filename="dorna-llama3-8b-instruct.Q2_K.gguf",
+    filename="dorna-llama3-8b-instruct.Q4_0.gguf",
     local_dir = "."
 )
 hf_hub_download(
     repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF",
-    filename="dorna-llama3-8b-instruct.Q4_0.gguf",
+    filename="dorna-llama3-8b-instruct.Q8_0.gguf",
+    local_dir = "."
+)
+
+hf_hub_download(
+    repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF",
+    filename="dorna-llama3-8b-instruct.Q5_0.gguf",
+    local_dir = "."
+)
+
+hf_hub_download(
+    repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF",
+    filename="dorna-llama3-8b-instruct.bf16.gguf",
     local_dir = "."
 )
 
@@ -96,7 +108,7 @@ def respond(
         messages.add_message(assistant)
     
     stream = agent.get_chat_response(
-        message,
+        message[-2:],
         llm_sampling_settings=settings,
         chat_history=messages,
         returns_streaming_generator=True,
@@ -120,7 +132,17 @@ PLACEHOLDER = """
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
-        gr.Textbox(value="You are a helpful assistant.", label="System message"),
+        gr.Textbox(value="You are a helpful Persian assistant. Please answer questions in the asked language.", label="System message"),
+        gr.Dropdown([
+                'dorna-llama3-8b-instruct.Q8_0.gguf',
+                'dorna-llama3-8b-instruct.Q4_0.gguf',
+                'dorna-llama3-8b-instruct.Q5_0.gguf',
+                'dorna-llama3-8b-instruct.bf16.gguf',
+            
+            ],
+            value="dorna-llama3-8b-instruct.Q8_0.gguf",
+            label="Model"
+        ),
         gr.Slider(minimum=1, maximum=8192, value=2048, step=1, label="Max tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(
@@ -144,13 +166,7 @@ demo = gr.ChatInterface(
             step=0.1,
             label="Repetition penalty",
         ),
-        gr.Dropdown([
-                'dorna-llama3-8b-instruct.Q2_K.gguf',
-                'dorna-llama3-8b-instruct.Q4_0.gguf',
-            ],
-            value="dorna-llama3-8b-instruct.Q2_K.gguf",
-            label="Model"
-        ),
+
     ],
     theme=gr.themes.Soft(primary_hue="violet", secondary_hue="violet", neutral_hue="gray",font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set(
         body_background_fill_dark="#16141c",
@@ -168,7 +184,7 @@ demo = gr.ChatInterface(
     undo_btn="Undo",
     clear_btn="Clear",
     submit_btn="Send",
-    description="Chat with Dorna-Llama3 8B (2-bit GGUF)",
+    description="Chat with Dorna-Llama3 8B GGUF",
     chatbot=gr.Chatbot(scale=1, placeholder=PLACEHOLDER)
 )