use streaming

app.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel
 from typing import List
 import ast
 import os
+import sseclient
 
 
 QUEUE_MAX_SIZE = int(os.getenv("QUEUE_MAX_SIZE", 20))
@@ -21,6 +22,7 @@ class LearningBotRequest(BaseModel):
     session_id: str
     context: dict
     user_serial: str
+    stream: bool
 
 
 def generate_uuid():
@@ -41,15 +43,13 @@ def construct_message(list_message):
 
 
 def send_message(url, request):
-    response = requests.post(url, data=request.json())
-    if response.status_code != 200:
-        raise gr.Error(response.text)
-    else:
-        result = response.json()["data"]["reply"]
-        return result
+    return requests.post(url, stream=True, data=request.json())
 
 
-def respond(chat_history, message, session_id, user_serial, persona, context, endpoint):
+def respond(chat_history, message, history, session_id, user_serial, persona, context, endpoint):
+    if history is None:
+        history = []
+    history.append(message)
     if session_id is None:
         session_id = generate_uuid()
 
@@ -64,13 +64,30 @@ def respond(chat_history, message, session_id, user_serial, persona, context, en
         persona=persona,
         session_id=session_id,
         context=context,
-        user_serial=user_serial
+        user_serial=user_serial,
+        stream=True,
+        product="learning_companion"
     )
 
     response = send_message(endpoint, request)
 
+    token_counter = 0
+    partial_reply = ""
+    client = sseclient.SSEClient(response)
+    for event in client.events():
+        data = json.loads(event.data)["data"]
+        partial_reply = partial_reply + data["data"]["reply"]
 
-    return chat_history + [[message, response]] , "Success", session_id, session_id
+        if token_counter == 0:
+            history.append(" " + partial_reply)
+        else:
+            history[-1] = partial_reply
+
+        chat = [(history[i], history[i + 1]) for i in range(0, len(history) - 1, 2)]  # convert to tuples of list
+
+        token_counter += 1
+
+        yield chat, history, "Success", session_id, session_id
 
 
 def reset_textbox():
@@ -88,10 +105,11 @@ with gr.Blocks() as demo:
             with gr.Accordion("Parameters", open=False):
                 user_serial = gr.Textbox(label="User serial")
                 context = gr.Textbox(label="context", value={})
-                persona = gr.Textbox(label="persona")
+                persona = gr.Textbox(label="persona", value="a493700848d84d0dab8d0095c2477c1e")
 
     chatbot = gr.Chatbot()
     message = gr.Textbox(placeholder="Halo kak, aku mau bertanya", label="Chat Here")
+    state = gr.State([])
     with gr.Row():
         with gr.Column(scale=5):
             send = gr.Button("Send")
@@ -99,12 +117,12 @@ with gr.Blocks() as demo:
             status_box = gr.Textbox(label="Status code from OpenAI server")
             session = gr.Textbox(label="session_id")
 
-    message.submit(respond, [chatbot, message, session_id, user_serial, persona, context, endpoint], [chatbot, status_box, session, session_id])
+    message.submit(respond, [chatbot, message, state, session_id, user_serial, persona, context, endpoint], [chatbot, state, status_box, session, session_id])
 
     clear.click(lambda: None, None, chatbot, queue=False)
     clear.click(lambda: None, None, session_id, queue=False)
 
-    send.click(respond, [chatbot, message, session_id, user_serial, persona, context, endpoint], [chatbot, status_box, session, session_id])
+    send.click(respond, [chatbot, message, state, session_id, user_serial, persona, context, endpoint], [chatbot, state, status_box, session, session_id])
     send.click(reset_textbox, [], [message])
     message.submit(reset_textbox, [], [message])
 
@@ -112,4 +130,4 @@ with gr.Blocks() as demo:
         demo
         .queue(max_size=QUEUE_MAX_SIZE, concurrency_count=QUEUE_CONCURENCY_COUNT)
         .launch(auth=(USERNAME, PASSWORD), debug=True)
-    )
+    )

requirements.txt

@@ -1,2 +1,2 @@
 openai==0.27.2
-pydantic==1.10.6
+sseclient-py==1.7.2