.

app.py

@@ -42,13 +42,23 @@ def respond(
     temperature,
     top_p,
 ):
-    # Construct the input prompt
-    prompt = f"{system_message}\n\n"
+    # Construct the chat list
+    chat_list = [{"role": "system", "content": system_message}]
     for user, assistant in history:
-        prompt += f"Human: {user}\nAssistant: {assistant}\n"
-    prompt += f"Human: {message}\nAssistant:"
-
-    # Generate the response
+        chat_list.extend(
+            [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": assistant},
+            ]
+        )
+    chat_list.append({"role": "user", "content": message})
+
+    # Apply chat template
+    prompt = pipe.tokenizer.apply_chat_template(
+        chat_list, tokenize=False, add_generation_prompt=True
+    )
+
+    # Generate response with streaming
     response = ""
     for output in pipe(
         prompt,
@@ -56,9 +66,11 @@ def respond(
         do_sample=True,
         temperature=temperature,
         top_p=top_p,
-        streamer=None,  # Use default streamer
+        num_beams=1,
+        top_k=50,
+        streamer=gr.utils.Iteratorize(),
     ):
-        new_text = output[0]["generated_text"][len(prompt) :]
+        new_text = output[0]['generated_text'][len(prompt) + len(response):]
         response += new_text
         yield response.strip()
 
@@ -81,7 +93,7 @@ demo = gr.ChatInterface(
             label="System message",
         ),
         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(
             minimum=0.1,
             maximum=1.0,
@@ -90,6 +102,8 @@ demo = gr.ChatInterface(
             label="Top-p (nucleus sampling)",
         ),
     ],
+    title="General Knowledge Assistant",
+    description="Ask me anything about general knowledge. I'll try to answer succinctly using first principles.",
 )