Update app.py

app.py

@@ -3,13 +3,39 @@ from llama_cpp import Llama
 import os
 from groq import Groq
 
+#tts
+from balacoon_tts import TTS
+from threading import Lock
+from io import BytesIO
+from huggingface_hub import hf_hub_download, list_repo_files
+
+#tts cpu model
+tts_model_str = "en_us_hifi_jets_cpu.addon"
+
+model_repo_dir = "/data"
+for name in list_repo_files(repo_id="balacoon/tts"):
+    if name == tts_model_str:
+        if not os.path.isfile(os.path.join(model_repo_dir, name)):
+            hf_hub_download(
+                repo_id="balacoon/tts",
+                filename=name,
+                local_dir=model_repo_dir,
+            )
+
+#tts speaker
+speaker_str = "92"
+
+# locker that disallow access to the tts object from more then one thread
+locker = Lock()
+
+#client
 client = Groq(
     api_key=os.getenv("GROQ_API_KEY"),
 )
 
 llm = Llama.from_pretrained(
-    repo_id="amir22010/fine_tuned_product_marketing_email_gemma_2_9b_q4_k_m",
-    filename="unsloth.Q4_K_M.gguf",
+    repo_id="amir22010/fine_tuned_product_marketing_email_gemma_2_9b_q4_k_m", #custom fine tuned model
+    filename="unsloth.Q4_K_M.gguf", #model file name
     cache_dir=os.path.abspath(os.getcwd()),
     n_ctx=2048, 
     n_batch=126,
@@ -17,7 +43,6 @@ llm = Llama.from_pretrained(
 )
 
 #guardrail model
-#guard_llm = "llama-guard-3-8b"
 guard_llm = "llama-3.1-8b-instant"
 
 #marketing prompt
@@ -71,19 +96,25 @@ async def greet(product,description):
     ]
     response = client.chat.completions.create(model=guard_llm, messages=messages, temperature=0)
     if response.choices[0].message.content != "not moderated":
-        a_list = ["Sorry can't proceed for generate marketing email!. Your content needs to be moderated first."]
-        for i in a_list:
-            yield i
+        audio_stream = BytesIO()
+        tts = TTS(os.path.join(model_repo_dir, tts_model_str))
+        a_list = ["Sorry, I can't proceed for generating marketing email. Your content needs to be moderated first. Thank you!"]
+        with locker:
+            audio_stream.write(tts.synthesize(a_list[0], speaker_str)) 
+        audio_stream.seek(0)
+        yield audio_stream
     else:
+        audio_stream = BytesIO()
+        tts = TTS(os.path.join(model_repo_dir, tts_model_str))
         output = llm.create_chat_completion(
         messages=[
             {
                 "role": "system",
-                "content": "Your go-to Email Marketing Guru - I'm here to help you craft compelling campaigns, boost conversions, and take your business to the next level.",
+                "content": "Your go-to Email Marketing Guru - I'm here to help you craft short and concise compelling campaigns, boost conversions, and take your business to the next level.",
             },
             {"role": "user", "content":  user_reques},
         ],
-        max_tokens=4096,
+        max_tokens=2048,
         temperature=0.7,
         stream=True
         )
@@ -91,8 +122,12 @@ async def greet(product,description):
         for chunk in output:
             delta = chunk['choices'][0]['delta']
             if 'content' in delta:
-                partial_message = partial_message + delta.get('content', '')
-                yield partial_message
+                with locker:
+                    audio_stream.write(tts.synthesize(delta.get('content', ''), speaker_str)) 
+                # partial_message = partial_message + delta.get('content', '')
+                # yield partial_message
+        audio_stream.seek(0)
+        yield audio_stream
 
-demo = gr.Interface(fn=greet, inputs=["text","text"], outputs="text", concurrency_limit=10)
+demo = gr.Interface(fn=greet, inputs=["text","text"], outputs=gr.Audio(), concurrency_limit=10)
 demo.launch()