Merge changes from main

app.py

@@ -7,7 +7,6 @@ from chainlit.element import ElementBased
 from dotenv import load_dotenv
 
 # modules for audio processing
-import httpx
 from langchain.schema.runnable.config import RunnableConfig
 from langchain_openai.chat_models import ChatOpenAI
 from openai import AsyncOpenAI
@@ -19,11 +18,6 @@ client = AsyncOpenAI()
 # ---- ENV VARIABLES ---- #
 load_dotenv()
 OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
-#QDRANT_CLOUD_KEY = os.environ.get("QDRANT_CLOUD_KEY")
-#QDRANT_CLOUD_URL = "https://30591e3d-7092-41c4-95e1-4d3c7ef6e894.us-east4-0.gcp.cloud.qdrant.io"
-#ELEVENLABS_API_KEY = os.environ.get("ELEVENLABS_API_KEY")
-#ELEVENLABS_VOICE_ID = os.environ.get("ELEVENLABS_VOICE_ID")
-
 
 # -- AUGMENTED -- #
 
@@ -112,35 +106,6 @@ async def generate_text_answer(transcription):
     return msg.content
 
 
-# Text-to-Speech Function: Take the text answer generated and convert it to an audio file
-# @cl.step(type="tool")
-# async def text_to_speech(text: str, mime_type: str):
-#     CHUNK_SIZE = 2048  # try 4096 or 8192 if getting read timeout error. the bigger the chunk size, the fewer API calls but longer wait time
-#     url = f"https://api.elevenlabs.io/v1/text-to-speech/{ELEVENLABS_VOICE_ID}"
-#     headers = {"Accept": mime_type, "Content-Type": "application/json", "xi-api-key": ELEVENLABS_API_KEY}
-#     data = {
-#         "text": text,
-#         "model_id": "eleven_monolingual_v1",
-#         "voice_settings": {"stability": 0.5, "similarity_boost": 0.5},
-#     }
-
-#     # make an async HTTP POST request to the ElevenLabs API to convert text to speech and return an audio file
-#     async with httpx.AsyncClient(timeout=60.0) as client:
-#         response = await client.post(url, json=data, headers=headers)
-#         response.raise_for_status()  # Ensure we notice bad responses
-#         buffer = BytesIO()
-#         buffer.name = f"output_audio.{mime_type.split('/')[1]}"
-#         async for chunk in response.aiter_bytes(chunk_size=CHUNK_SIZE):
-#             if chunk:
-#                 buffer.write(chunk)
-
-#         buffer.seek(0)
-#         return buffer.name, buffer.read()
-
-
-# ---- AUDIO PROCESSING ---- #
-
-
 # Audio Chunk Function: Process audio chunks as they arrive from the user's microphone
 @cl.on_audio_chunk
 async def on_audio_chunk(chunk: cl.AudioChunk):
@@ -182,19 +147,3 @@ async def on_audio_end(elements: list[ElementBased]):
             pprint(f"================== Node: '{key}':")
 
     await msg.send()
-
-    # text_answer = await generate_text_answer(
-    #     transcription
-    # )  # need to change this to generate answer based on base_rag_chain
-
-    # output_name, output_audio = await text_to_speech(text_answer, audio_mime_type)
-
-    # output_audio_el = cl.Audio(
-    #     name=output_name,
-    #     auto_play=True,
-    #     mime=audio_mime_type,
-    #     content=output_audio,
-    # )
-    # answer_message = await cl.Message(content="").send()
-    # answer_message.elements = [output_audio_el]
-    # await answer_message.update()

chainlit.md

@@ -1,9 +1,19 @@
+---
+title: RAGalicious App (AIE3)
+emoji: 👩‍🍳
+colorFrom: purple
+colorTo: blue
+sdk: docker
+pinned: false
+license: mit
+---
+
 # 😋 Welcome to RAGalicious!
 
 RAGalicious is a chatbot designed to help users discover delectable recipes from the NYTimes Cooking section. It simplifies the recipe discovery process, providing users inspiration based on ingredients, diet, occasion or equipment you have.
 
 ## Example prompts
 
-- **Plan your daily meals:** E.g, "Give me ideas for making an easy weeknight dinner."
+- **Plan your daily meals:** E.g. "Give me ideas for making an easy weeknight dinner."
 - **Get ready to host occasions:** E.g. "What are good dishes to make for Rosh Hashanah?"
 - **Get scrappy with ingredients you already have:** E.g. "What can I make with pasta, lemon and chickpeas?"

utils/graph.py

@@ -186,6 +186,7 @@ def generate_workflow(base_llm, power_llm):
         shortlisted_recipes = state["shortlisted_recipes"]
         messages = state["messages"]
         last_message = messages[-1] if messages else ""
+        question_type = state["question_type"]
 
         # LLM with tool and validation
         base_rag_prompt_template = """\
@@ -207,7 +208,20 @@ def generate_workflow(base_llm, power_llm):
         chain = base_rag_prompt | power_llm
 
         full_response = ""
+        thumbnail_url = ""
         cl_msg = config["configurable"]["cl_msg"]
+        if state["question_type"] == "show_specific_recipe":
+            selected_recipe = state.get("selected_recipe")
+            if selected_recipe and selected_recipe.get("thumbnail"):
+                thumbnail_url = selected_recipe["thumbnail"]
+                image = cl.Image(url=thumbnail_url, name="thumbnail", display="inline", size="large")
+
+                # Attach the image to the message
+                await cl.Message(
+                    content="",
+                    elements=[image],
+                ).send()
+
         async for chunk in chain.astream(
             {
                 "question": question,
@@ -220,9 +234,9 @@ def generate_workflow(base_llm, power_llm):
                 await cl_msg.stream_token(chunk.content)
                 full_response += chunk.content
 
-        selected_recipe = get_selected_recipe(base_llm, question, shortlisted_recipes, messages)
-
-        return {"messages": [full_response], "selected_recipe": selected_recipe}
+        return {
+            "messages": [full_response],
+        }
 
     async def _node_single_recipe_qa(state: AgentState, config):
         print("--- Q&A with SINGLE RECIPE ---")
@@ -271,7 +285,7 @@ def generate_workflow(base_llm, power_llm):
         selected_recipe = state.get("selected_recipe")
         messages = state["messages"]
         last_message = messages[-1] if messages else ""
-
+        cl_msg = config["configurable"]["cl_msg"]
         # LLM with tool and validation
         base_rag_prompt_template = """\
             You are a friendly AI assistant.
@@ -303,15 +317,18 @@ def generate_workflow(base_llm, power_llm):
         )
 
         print("message", message)
-
+        tool_arguments = json.loads(message.additional_kwargs["function_call"]["arguments"])
         action = ToolInvocation(
             tool=message.additional_kwargs["function_call"]["name"],
-            tool_input=json.loads(message.additional_kwargs["function_call"]["arguments"]),
+            tool_input=tool_arguments,
         )
 
         response = tool_executor.invoke(action)
 
         function_message = FunctionMessage(content=str(response), name=action.tool)
+        await cl_msg.stream_token(
+            f"""Sure! I've sent a text to {tool_arguments['number']} with the following: \n\n{tool_arguments['text']}"""
+        )
 
         return {"messages": [function_message]}
 
@@ -335,9 +352,6 @@ def generate_workflow(base_llm, power_llm):
         shortlisted_recipes = state.get("shortlisted_recipes")
         selected_recipe = state.get("selected_recipe")
 
-        # if not shortlisted_recipes or len(shortlisted_recipes) == 0:
-        #     print("going to retrieve since no shortlisted_recipes")
-        #     return "retrieve"
         if question_type == "asking_for_recipe_suggestions":
             return "retrieve"
         if question_type in ["referring_to_shortlisted_recipes", "show_specific_recipe"]:

utils/graph_chains.py

@@ -125,14 +125,14 @@ def get_question_type_chain(llm_model):
             description="The likelihood / chance that the User Question is asking for recipe suggestions based on some criteria, integers from 1 to 100"
         )
         referring_to_specific_recipe: int = Field(
-            description="The likelihood / chance that the User Question is asking questions about one specific full recipe, integers from 1 to 100"
+            description="The likelihood / chance that the User Question is asking specific questions about a single specific recipe, integers from 1 to 100"
         )
         referring_to_shortlisted_recipes: int = Field(
-            description="The likelihood / chance that the User Question is asking generally about shortlisted recipes provided in the last message, integers from 1 to 100"
+            description="The likelihood / chance that the User Question is asking generally about more than one recipe provided in the last message, integers from 1 to 100"
         )
 
         show_specific_recipe: int = Field(
-            description="The likelihood / chance that the User Question is asking to show a specific recipe, integers from 1 to 100"
+            description="The likelihood / chance that the User Question is asking to show the full recipe for a specific recipe, integers from 1 to 100"
         )
         send_text: int = Field(
             description="The likelihood / chance that the User Question is to send a SMS or text, integers from 1 to 100"
@@ -155,6 +155,8 @@ def get_question_type_chain(llm_model):
         it is highly likely that the user is asking questions referring to shortlisted recipes.
         If the last message was a full single recipe, it is generally likely that the user 
         is asking questions referring to specific recipe.
+        If the user is asking to show the full recipe, it is highly likely that they are asking 
+        to show a specific recipe and less likely that they are asking for anything else.
 
         {format_instructions}