Fridays updates

Dockerfile

@@ -8,4 +8,4 @@ COPY --chown=user . $HOME/app
 COPY ./requirements.txt ~/app/requirements.txt
 RUN pip install -r requirements.txt
 COPY . .
-CMD ["chainlit", "run", "app_main.py", "--port", "7860"]
+CMD ["chainlit", "run", "app.py", "--port", "7860"]

app_main.py → app.py

@@ -31,7 +31,22 @@ whisper_model = whisper.load_model("base")
 # Action callbacks
 #############################################
 
-@cl.action_callback("HSBC: Lending - Loan Origination System (Qualification)")
+@cl.action_callback("Deal Analysis")
+async def on_action_anayze_deal(action):
+    session_state = cl.user_session.get("session_state", None)
+    await prep_opportunities(session_state)
+
+@cl.action_callback("Customer Research")
+async def on_action_anayze_deal(action):
+    session_state = cl.user_session.get("session_state", None)
+    await get_latest_news("HSBC")
+
+@cl.action_callback("Sales Simulation")
+async def on_action_sales_simulation(action):
+    session_state = cl.user_session.get("session_state", None)
+    await callback_run_scenario(action)
+
+@cl.action_callback("HSBC: Lending - Loan Origination System (Proposal)")
 async def on_action_anayze_opportunity(action):
     await prep_opportunity_analysis()
 
@@ -57,6 +72,7 @@ async def on_action_display_queries_responses(action):
     await callback_display_queries_responses()
 
 
+
 #############################################
 ### On Chat Start (Session Start) Section ###
 #############################################
@@ -74,9 +90,7 @@ async def on_chat_start():
 
     await prep_start(session_state)
 
-    await prep_opportunities(session_state)
-
-    
+    # await prep_opportunities(session_state)
 
     # await prep_opportunity_analysis(session_state)
 

app_am.py

@@ -18,7 +18,7 @@ from langchain_core.runnables.passthrough import RunnablePassthrough
 from langchain.schema.runnable.config import RunnableConfig
 from langsmith.evaluation import LangChainStringEvaluator, evaluate
 from datetime import datetime
-from objection_eval import generate_response_to_objection
+from utils_evaluate_objections import generate_response_to_objection
 import pandas as pd 
 import uuid
 import chainlit as cl

classes.py

@@ -4,8 +4,9 @@ class SessionState:
     do_opportunity_analysis = False
     do_customer_research = False
     do_objections = False
-    add_objections_to_analysis = False   
+    add_objections_to_analysis = True   
     ask_objections = True
+    use_objection_cache = True
     do_ragas_evaluation = False
     customer_research_report_md = "HSBC Quarterly Report 2024-10-16.md"
     customer_research_report_pdf = "HSBC Quarterly Report 2024-10-16.pdf"
@@ -20,7 +21,7 @@ class SessionState:
     duration_minutes = None
     attitude = "Happy"
     mood_score = 5
-    num_questions = 4
+    num_questions = 2
     current_question_index = 0
     previous_answer = None
     question = ""
@@ -40,6 +41,7 @@ class SessionState:
         self.do_objections = False
         self.add_objections_to_analysis = False
         self.ask_objections = True
+        self.use_objection_cache = True
         self.do_ragas_evaluation = False
         self.customer_research_report_md = "HSBC Quarterly Report 2024-10-16.md"
         self.customer_research_report_pdf = "HSBC Quarterly Report 2024-10-16.pdf"
@@ -54,7 +56,7 @@ class SessionState:
         self.duration_minutes = None
         self.attitude = "Happy"
         self.mood_score = 5
-        self.num_questions = 4
+        self.num_questions = 2
         self.current_question_index = 0
         self.previous_answer = None
         self.question = ""

data/Opportunity_Information.csv

@@ -1,3 +1,3 @@
 Opportunity ID,Customer Name,Opportunity Name,Opportunity Stage,Opportunity Description,Opportunity Value,Close Date,Customer Contact,Customer Contact Role,Activity,Next Steps
-XFR0001,HSBC,Lending - Loan Origination System,Qualification,Developing analytic capabilities for the loan origination system,"$250,000",12/30/2024,John Smith,"VP, Information Technology",Had first meeting with John on 9/16. Identified that they are unhappy with the analytic capability of their current lending solution. Next step is to meet with them to understand their business drivers and pain points better. Will also use the next meeting to explain our company's value proposition.,Meet with John on 10/12 to qualify the opportunity better and share our value proposition
+XFR0001,HSBC,Lending - Loan Origination System,Proposal,Developing analytic capabilities for the loan origination system,"$250,000",11/30/2024,John Smith,"VP, Information Technology",We have had several meetings with HSBC's lending team regarding replacing the analytics engine for their lending solution. The current system is slow and inflexible. They have the renewal coming up with the existing vendor next year so there is urgency regarding the decision process. ,"Next Steps: Meet with John on 10/26 discuss next steps in the decision-making process, potentially moving towards a pilot program or final negotiations."
 XFR0002,Citi,Competitive Analysis SaaS,Negotiation,Develop SaaS for analysis of Citi's competitors in foreign markets,"$100,000",11/15/2024,Peter Branson,"CEO",Contract finally ironed out by both parties' lawyers. Have verbal agreement that this will be moving forwards.,Finalize contract with Peter and lawyers on October 25th

utils_actions.py

@@ -1,10 +1,21 @@
 import chainlit as cl
 
+async def offer_initial_actions():
+    actions = [
+        cl.Action(name="Deal Analysis", value="deal-analysis", description="Deal Analysis"),
+        cl.Action(name="Customer Research", value="customer-research", description="Customer Research"),
+        cl.Action(name="Sales Simulation", value="sales-simulation", description="Sales Simulation"),
+    ]
+    await cl.Message(content=" ", actions=actions).send()
+    await cl.Message(content="\n\n").send()
+
 async def offer_actions():
+    await cl.Message(content="\n\n").send()
     actions = [
         cl.Action(name="Get Latest News on this Customer", value="HSBC", description="Get Latest News"),
         cl.Action(name="Enter Meeting Simulation", value="enter-meeting-simulation", description="Enter Meeting Simulation"),
         cl.Action(name="Review Another Opportunity", value="review-another-opportunity", description="Review Another Opportunity"),
     ]
     await cl.Message(content="Select an action", actions=actions).send()
+    await cl.Message(content="\n\n").send()
 

utils_callbacks.py

@@ -66,15 +66,25 @@ async def callback_run_scenario(action):
     await cl.Message(content="Click to start simulation", actions=start_actions).send()
 
 
+
 async def callback_start_scenario():
     print("callback_start_scenario()")
     session_state = cl.user_session.get("session_state", None)
+    await cl.Message(content="3...").send()
+    await asyncio.sleep(1)
+    await cl.Message(content="2...").send()
+    await asyncio.sleep(1)
+    await cl.Message(content="1...").send()
+    await asyncio.sleep(1)
+    await cl.Message(content="**Simulation Starting**").send()
+    await cl.Message(content="\n\n").send()
     start_time = datetime.now()
     print("setting start time")
     session_state.start_time = start_time
     output = f"{session_state.customer.contact_name} joins the zoom call"
     print(output)
     await cl.Message(content=output).send()
+    await cl.Message(content="\n\n").send()
 
 async def callback_evaluate_performance():
     session_state = cl.user_session.get("session_state", None)

utils_chain_parameters.py

@@ -5,27 +5,32 @@ def prepare_chain_parameters(session_state, message, history):
     next_question = ""  
     ground_truth = ""
     command = ""
+    all_questions_answers = ""
     print(f"Index: {session_state.current_question_index}")
     if session_state.current_question_index == 0:
         previous_question = ""
         rep_answer = ""
         ground_truth = ""
         next_question = session_state.questions[session_state.current_question_index]["question"]
-        command = "You should greet the rep"
+        all_questions_answers = ""
+        command = "You should greet the salesrep"
     elif session_state.current_question_index >= len(session_state.questions):
         next_question = "" 
         previous_question = session_state.questions[session_state.current_question_index - 1]["question"]
         rep_answer = session_state.previous_answer
         ground_truth = session_state.questions[session_state.current_question_index - 1]["ground_truth"]
+        for response in session_state.responses:
+            all_questions_answers += f"Question: {response['question']}\nAnswer: {response['response']}\n\n"
         command = """Thank the customer, offer a comment on the answer and overall performance.
-            Conclude the conversation with a summary and give a farewell. 
-            You can add additional comments as needed.
+            Conclude the conversation with a conclusion based on all of the questions and answers.
+            Give a polite farewell. 
         """
     else:
         previous_question = session_state.questions[session_state.current_question_index - 1]["question"]
         rep_answer = session_state.previous_answer
         next_question = session_state.questions[session_state.current_question_index]["question"]
         ground_truth = session_state.questions[session_state.current_question_index]["ground_truth"]
+        all_questions_answers = ""
         command = "You should respond to the answer based on how well the rep answered the previous question."
     session_state.ground_truth = ground_truth
     session_state.question = previous_question
@@ -72,6 +77,7 @@ def prepare_chain_parameters(session_state, message, history):
             "rep_answer": rep_answer,   
             "conversation_history": history,   
             "command": command,
+            "all_questions_answers": all_questions_answers
             }
     return parm
 

utils_evaluate.py

@@ -10,6 +10,25 @@ from ragas.metrics import (
 from rouge_score import rouge_scorer
 from sentence_transformers import SentenceTransformer, util
 
+from utils_evaluate_objections import generate_objection_scores
+
+
+def evaluate_objections(session):
+
+    for response in session.responses:
+        question = response.get("question", "")
+        answer = response.get("response", "")
+        print(f"Question: {question}")
+        print(f"Answer: {answer}")
+        
+        q_and_a = {
+            "objection": question,
+            "answer": answer
+        }         
+        score = generate_objection_scores(q_and_a)
+        response["evaluation_score"] = score
+
+
 def evaluate_answers(session):
     ragas_results = evaluate_with_ragas(session)
     session.ragas_results = ragas_results
@@ -27,8 +46,6 @@ def evaluate_answers(session):
     session.scores = scores
     return scores
 
-
-
 def evaluate_with_ragas(session):
     questions = [] 
     answers = []

objection_eval.py → utils_evaluate_objections.py

@@ -92,6 +92,24 @@ class SatisfyRate(MetricWithLLM, SingleTurnMetric):
         )
         return int(prompt_response.satisfy)
     
+async def generate_objection_scores(question_answer):
+    from langchain_openai import ChatOpenAI
+    from ragas.llms.base import LangchainLLMWrapper
+    import pandas as pd
+    # user_response= pd.read_csv(file_path)
+    openai_model = LangchainLLMWrapper(ChatOpenAI(model_name="gpt-4o", api_key=OPENAI_API_KEY))
+    scorer = SatisfyRate(llm=openai_model)
+    
+    sample = SingleTurnSample(user_input=question_answer['objection'], response=question_answer['answer'])
+    
+    #(user_response['objection'][num], user_response['response'][num])
+    satisfy_0_1 = await scorer.single_turn_ascore(sample)
+    
+    print (question_answer['objection'], question_answer['answer'], satisfy_0_1)
+    # Implement your logic to generate a response based on the user's input
+    return satisfy_0_1 #f"Response to your objection: {user_response['objection'][num]}, {user_response['response'][num]}, {satisfy_0_1}" 
+
+    
 async def generate_response_to_objection(file_path, num):
     from langchain_openai import ChatOpenAI
     from ragas.llms.base import LangchainLLMWrapper
@@ -124,5 +142,4 @@ if __name__ == "__main__":
     file_path = sys.argv[1]
 
     # Run the main async function
-    asyncio.run(main(file_path))
-
+    asyncio.run(main(file_path))

utils_objections.py

@@ -18,11 +18,20 @@ from qdrant_client.http.models import Distance, VectorParams
 
 
 async def create_objections(session_state):
-    customer_document_file = session_state.customer_research_report_pdf
-    customer_file_path = "reports/" + customer_document_file
-    bettertech_document_file = session_state.bettetech_value_proposition_pdf
-    bettertech_file_path = "data/" + bettertech_document_file
-    objections = await process_files(customer_file_path, bettertech_file_path)
+    if session_state.use_objection_cache:
+
+        objections = [
+            "1. Can you provide customer references in our industry?",
+            "2. Second question, what training options are available for our team?",
+            "3. Last but not least, your pricing seems high compared to some other solutions we've seen. Is there any flexibility??",
+        ]
+
+    else:
+        customer_document_file = session_state.customer_research_report_pdf
+        customer_file_path = "reports/" + customer_document_file
+        bettertech_document_file = session_state.bettetech_value_proposition_pdf
+        bettertech_file_path = "data/" + bettertech_document_file
+        objections = await process_files(customer_file_path, bettertech_file_path)
     return objections
 
 

utils_output.py

@@ -3,8 +3,8 @@ import json
 import re
 from datetime import datetime
 
-from utils_evaluate import evaluate_answers
-
+from utils_evaluate import evaluate_answers, evaluate_objections
+from utils_prep import offer_initial_actions
 async def display_llm_responses(cl, session_state):
     output = f"**Responses**"
     await cl.Message(content=output).send()
@@ -61,7 +61,10 @@ async def display_evaluation_results(cl, session_state):
     out_text = "*Preparing evaluation results ...*"
     await cl.Message(content=out_text).send()
 
-    evaluate_answers(session_state) 
+    if session_state.do_evaluation:
+        evaluate_answers(session_state) 
+    elif session_state.add_objections_to_analysis:
+        evaluate_objections(session_state)
     await asyncio.sleep(1)
 
     output = f"**Session Summary**"
@@ -73,15 +76,15 @@ async def display_evaluation_results(cl, session_state):
     output = output + f"**Total Questions Answered:** {len(session_state.responses)} \n"
     await cl.Message(content=output).send()
 
-    results_df = session_state.ragas_results.to_pandas()
-    columns_to_average = ['answer_relevancy', 'answer_correctness']
-    averages = results_df[columns_to_average].mean()
+    if session_state.do_ragas_evaluation:
+        results_df = session_state.ragas_results.to_pandas()
+        columns_to_average = ['answer_relevancy', 'answer_correctness']
+        averages = results_df[columns_to_average].mean()
 
     await cl.Message(content="**Overall Summary (By SalesBuddy)**").send()
-    output = f"**Overall Score:** {session_state.responses[-1]['overall_score']} \n"
-    output = output + f"**Overall Evaluation:** {session_state.responses[-1]['overall_evaluation']} \n"
-    output = output + f"**Final Mood Score:** {session_state.responses[-1]['mood_score']} \n"
-    output = output + f"**Customer Next Steps:** {session_state.llm_next_steps} \n"
+    output = f"**SalesBuddy Score:** {session_state.responses[-1]['overall_score']} \n"
+    output = output + f"**SalesBuddy Evaluation:** {session_state.responses[-1]['overall_evaluation']} \n"
+    output = output + f"**SalesBuddy Final Mood Score:** {session_state.responses[-1]['mood_score']} \n"
     await cl.Message(content=output).send()
 
     if session_state.do_ragas_evaluation:
@@ -93,21 +96,23 @@ async def display_evaluation_results(cl, session_state):
     await cl.Message(content="**Individual Question Scores**").send()
 
     for index, resp in enumerate(session_state.responses):
-        scores = session_state.scores[index]
-        relevancy = results_df.iloc[index].get('answer_relevancy', 'N/A')
-        correctness = results_df.iloc[index].get('answer_correctness', 'N/A')
-        bleu_score = scores.get('bleu_score', 'N/A')
-        rouge1_score = scores.get('rouge_score', {}).get('rouge1', 'N/A')
-        rouge1_output = format_rogue_score(rouge1_score)
-        rougeL_score = scores.get('rouge_score', {}).get('rougeL', 'N/A')
-        rougeL_output = format_rogue_score(rougeL_score)
-        semantic_similarity_score = scores.get('semantic_similarity_score', 'N/A')
+        
         output = f"""
             **Question:** {resp.get('question', 'N/A')}
-            **Answer:** {resp.get('response', 'N/A')}
-            **Ground Truth:** {resp.get('ground_truth', 'N/A')}  
+            **Answer:** {resp.get('response', 'N/A')} 
+            **SalesBuddy Evaluation:** {resp.get('response_evaluation', 'N/A')}
+            **Evaluation Score:** {resp.get('response_score', 'N/A')}
         """
         if session_state.do_ragas_evaluation:
+            scores = session_state.scores[index]
+            relevancy = scores.get('answer_relevancy', 'N/A')
+            correctness = scores.get('answer_correctness', 'N/A')
+            bleu_score = scores.get('bleu_score', 'N/A')
+            rouge1_score = scores.get('rouge_score', {}).get('rouge1', 'N/A')
+            rouge1_output = format_rogue_score(rouge1_score)
+            rougeL_score = scores.get('rouge_score', {}).get('rougeL', 'N/A')
+            rougeL_output = format_rogue_score(rougeL_score)
+            semantic_similarity_score = scores.get('semantic_similarity_score', 'N/A')
             numbers = f"""   
                 **Answer Relevancy:** {format_score(relevancy)}
                 **Answer Correctness:** {format_score(correctness)}
@@ -120,3 +125,5 @@ async def display_evaluation_results(cl, session_state):
             await cl.Message(content=numbers).send()
         else:
             await cl.Message(content=output).send()
+
+    await offer_initial_actions()

utils_prep.py

@@ -2,7 +2,7 @@ import asyncio
 import chainlit as cl
 from langchain_openai import ChatOpenAI
 
-from utils_actions import offer_actions
+from utils_actions import offer_actions,offer_initial_actions
 from utils_data import get_company_data, get_opportunities
 from utils_prompt import get_chat_prompt
 from utils_objections import create_objections
@@ -16,9 +16,18 @@ async def prep_start(session_state):
     simple_chain = chat_prompt | chat_model
     cl.user_session.set("chain", simple_chain)
 
-    welcome_message = f"**Welcome to {session_state.company.name} SalesBuddy**\n*Home of {session_state.company.product}*"
+    welcome_message = f"**Welcome to {session_state.company.name} SalesBuddy**\n*Your AI assistant for sales and sales management*"
     await cl.Message(content=welcome_message).send()
-    await cl.Message(content=session_state.company.product_summary).send()
+    # await cl.Message(content=session_state.company.product_summary).send()
+
+    image = cl.Image(path="images/salesbuddy_logo.jpg", name="salesbuddy_logo", display="inline")
+    await cl.Message(
+        content=" ",
+        elements=[image],
+    ).send()
+
+    await offer_initial_actions()
+
 
     opportunities = get_opportunities()
     cl.user_session.set("opportunities", opportunities)
@@ -93,6 +102,7 @@ async def prep_opportunity_analysis():
         for output_message in output_messages:  
             await cl.Message(content=output_message).send()
             await cl.Message(content="").send() 
+        await cl.Message(content="\n\n").send()
 
     await offer_actions()
 
@@ -104,7 +114,7 @@ async def prep_research(session_state):
 
 
 def get_opportunity_analysis():
-    output_1 = "**Summary:** The HSBC opportunity involves replacing the existing analytics engine for their loan origination system, valued at $250,000. The current system is slow and lacks flexibility, creating urgency due to an impending renewal with the existing vendor. Multiple meetings have been conducted, culminating in a proposal review. The decision process is progressing, with a meeting scheduled to discuss the next steps on October 18, 2024."
+    output_1 = "**Summary:** The HSBC opportunity involves replacing the existing analytics engine for their loan origination system, valued at $250,000. The current system is slow and lacks flexibility, creating urgency due to an impending renewal with the existing vendor. Multiple meetings have been conducted, culminating in a proposal review. The decision process is progressing, with a meeting scheduled on October 26 with John Smith to discuss the next steps. Potential for pilot program or final negotiations."
     output_2 = "**Score: 75**"
     output_3 = "**MEDDIC Evaluation:**" 
     output_4 = "**Metrics: 70** - The proposal discussed expected performance improvements and ROI, but specific quantitative metrics driving the decision were not detailed."

utils_prompt.py

@@ -30,6 +30,9 @@ def get_user_template_openai_short():
 
         Rep Answer:
         {rep_answer}
+
+        All Questions and Answers:
+        {all_questions_answers}
         
     """
     return user_template
@@ -81,6 +84,9 @@ def get_user_template_openai_long():
         Rep Answer:
         {rep_answer}
         
+        All Questions and Answers:
+        {all_questions_answers}
+        
         Conversation History:
         {conversation_history}
     """
@@ -110,6 +116,8 @@ def get_system_template_openai_short():
         You can make conversation but you must follow the command.
         If a previous question and answer are provided, you must evaluate the rep's answer.
         You will perform evaluation based on how well and thoroughly the rep answered the previous question.
+        If asked to provide a conclusion, you must consider all of the rep's answers to your questions.
+        These are provided in the 'All questions and answers:' section.
         You will ALWAYS provide your response in valid JSON format
         Remember all string values must be enclosed in double quotes.
         You will include with the following fields in JSON format:

utils_simulation.py

@@ -51,7 +51,7 @@ async def do_simulation(client, session_state, message):
             if session_state.do_voice:
                 await reply_with_voice(cl, client, message_to_rep)
             else:
-                await cl.Message(message_to_rep).send()
+                await cl.Message(content=message_to_rep, author="John Smith").send()
             # await cl.Message(this_response).send()
             history.append({"role": "assistant", "content": response_content})
             cl.user_session.set("history", history)
@@ -74,8 +74,10 @@ async def do_simulation(client, session_state, message):
             if session_state.do_evaluation:
                 await display_evaluation_results(cl, session_state)
             else:
+                await cl.Message(content="**Simulation Complete**").send()
                 evaluate_actions = [
                     cl.Action(name="Evaluate Performance", value="evaluate", description="Evaluate Performance"),
                     cl.Action(name="Display Queries and Responses", value="display_llm_responses", description="Display LLM Responses")
                 ]
-                await cl.Message(content="Click to evaluate", actions=evaluate_actions).send()
+                await cl.Message(content="Click to evaluate performance", actions=evaluate_actions).send()
+                await cl.Message(content="\n\n").send()

utils_voice.py

@@ -17,7 +17,7 @@ async def reply_with_voice(cl,client, assistant_message):
         elements = [
             cl.Audio(name="Voice", path=speech_file_path, display="inline")
         ]
-        await cl.Message(content=assistant_message, elements=elements).send()
+        await cl.Message(content=assistant_message, elements=elements, author="John Smith").send()
     except Exception as e:
         await cl.Message(content=f"Error generating or sending audio: {e}").send()
     finally: