training adherence scoring features

app_config.py

@@ -57,6 +57,7 @@ DB_BATTLES = 'battles'
 DB_ERRORS = 'completion_errors'
 DB_CPC = "cpc_comparison"
 DB_BP = "bad_practices_comparison"
+DB_TA = "convo_scoring_comparison"
 
 MAX_MSG_COUNT = 60
 WARN_MSG_COUT = int(MAX_MSG_COUNT*0.8)

models/ta_models/config.py

@@ -23,4 +23,152 @@ BP_THRESHOLD = 0.7
 BP_LAB2STR = {
     "is_advice": "Advice",
     "is_personal_info": "Personal Info Sharing",
-}
+}
+
+QUESTION2PHASE = {
+    "question_1": ["0_ActiveEngagement","1_Explore"],
+    "question_4": ["1_Explore"],
+    "question_5": ["0_ActiveEngagement", "1_Explore"],
+    # "question_7": ["1_Explore"],
+    # "question_9": ["4_SP&NS"],
+    "question_10": ["4_SP&NS"],
+    # "question_11": ["4_SP&NS"],
+    "question_14": ["6_WrappingUp"],
+    # "question_15": ["ALL"],
+    "question_19": ["ALL"],
+    # "question_21": ["ALL"],
+    # "question_22": ["ALL"],
+    "question_23": ["2_IRA", "3_SafetyAssessment"],
+}
+
+QUESTION2FILTERARGS = {
+    "question_1": {
+        "phases": QUESTION2PHASE["question_1"],
+        "pre_n": 2,
+        "post_n": 8,
+        "ignore": ["7_Other"],
+    },
+    "question_4": {
+        "phases": QUESTION2PHASE["question_4"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+    "question_5": {
+        "phases": QUESTION2PHASE["question_5"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+    # "question_7": {
+    #     "phases": QUESTION2PHASE["question_7"],
+    #     "pre_n": 5,
+    #     "post_n": 15,
+    #     "ignore": ["7_Other"],
+    # },
+    # "question_9": {
+    #     "phases": QUESTION2PHASE["question_9"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    "question_10": {
+        "phases": QUESTION2PHASE["question_10"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+    # "question_11": {
+    #     "phases": QUESTION2PHASE["question_11"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    "question_14": {
+        "phases": QUESTION2PHASE["question_14"],
+        "pre_n": 10,
+        "post_n": 0,
+        "ignore": ["7_Other"],
+    },
+    # "question_15": {
+    #     "phases": QUESTION2PHASE["question_15"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    "question_19": {
+        "phases": QUESTION2PHASE["question_19"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+    # "question_21": {
+    #     "phases": QUESTION2PHASE["question_21"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    # "question_22": {
+    #     "phases": QUESTION2PHASE["question_22"],
+    #     "pre_n": 5,
+    #     "post_n": 5,
+    #     "ignore": ["7_Other"],
+    # },
+    "question_23": {
+        "phases": QUESTION2PHASE["question_23"],
+        "pre_n": 5,
+        "post_n": 5,
+        "ignore": ["7_Other"],
+    },
+}
+
+
+START_INST = "<|user|>"
+END_INST = "<|end|>\n<|assistant|>"
+
+NAME2QUESTION = {
+    "question_1": "Did the helper introduce themself in the opening message? Answer only Yes or No.",
+    "question_4": "Did the helper actively listened to the texter's crisis? Answer only Yes or No.",
+    "question_5": "Did the helper reflect on the main issue that led the texter reach out? Answer only Yes or No.",
+    # "question_7": "Did the helper collaborated with the texter to identify the goal of the conversation? Answer only Yes or No.",
+    # "question_9": "Did the helper collaborated with the texter to create next steps? Answer only Yes or No.",
+    "question_10": "Did the helper explored texter's existing coping skills? Answer only Yes or No.",
+    # "question_11": "Did the helper explored texter’s social support? Answer only Yes or No.",
+    "question_14": "Did helper reflected the texter’s plan, reiterate coping skills, and end in a supportive way? Answer only Yes or No.",
+    # "question_15": "Did the helper consistently used Good Contact Techniques? Answer only Yes or No.",
+    "question_19": "Did the helper consistently reflected empathy through the conversation? Answer only Yes or No.",
+    # "question_21": "Did the helper shared personal information? Answer only Yes or No.",
+    # "question_22": "Did the helper gave advice? Answer only Yes or No.",
+    "question_23": "Did the helper explicitely initiated imminent risk assessment? Answer only Yes or No.",
+}
+
+NAME2PROMPT = {
+    k: "--------Conversation:\n{convo}\n{start_inst}" + v + "\n{end_inst}"
+    for k, v in NAME2QUESTION.items()
+}
+
+NAME2PROMPT_EXPL = {
+    k: v.split("Answer only Yes or No.")[0] + "Answer Yes or No, and give an explanation in a new line.\n{end_inst}"
+    for k, v in NAME2PROMPT.items()
+}
+
+QUESTIONDEFAULTS = {
+    "question_1": {True: "No, There was no evidence of Active Engagement", False: "No"},
+    "question_4": {True: "No, There was no evidence of Exploration Phase", False: "No"},
+    "question_5": {True: "No, There was no evidence of Exploration Phase", False: "No"},
+    # "question_7": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    # "question_9": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    "question_10": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    # "question_11": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    "question_14": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    # "question_15": "Did the helper consistently used Good Contact Techniques? Answer only Yes or No.",
+    "question_19": {True: "N/A Texter disengaged, Not Applicable", False: "N/A"},
+    # "question_21": "Did the helper shared personal information? Answer only Yes or No.",
+    # "question_22": "Did the helper gave advice? Answer only Yes or No.",
+    "question_23": {True: "No, There was no evidence of Imminent Risk Assessment", False: "No"},
+}
+
+TEXTER_PREFIX = "texter"
+HELPER_PREFIX = "helper"
+
+TA_OPTIONS = ["N/A", "No", "Yes"]

models/ta_models/ta_filter_utils.py

@@ -0,0 +1,150 @@
+from itertools import chain
+from typing import List, Tuple
+
+import numpy as np
+import pandas as pd
+
+possible_movements = [-1, 1]
+
+
+def dfs(indexes: List[int], x0: int, i: int, cur_island: List[int], d=2):
+    """Deep First Search Implementation for 2D movement.
+    To consider an Island only move one step left or right
+    See possible movements
+
+    Args:
+        indexes (List[int]): Indexes of positive examples. i.e [20,21,23,50,51]
+        x0 (int): Initial island anchor
+        i (int): Current index to test against anchor
+        cur_island (List[int]): Current Island from anchor
+        d (int, optional): Bounding distance to consider an island. Defaults to 2. For example
+            the list [20,21,23,50,51] has two islands with d=2: (20,21,23), and (50,51) but it has
+            three islands with d=: (20,21), (23), and (50,51)
+    """
+    rows = len(indexes)
+    if i < 0 or i >= rows:
+        return
+    if indexes[i] in cur_island:
+        return
+    if abs(indexes[x0] - indexes[i]) > d:
+        return
+    # computing coordinates with x0 as base
+    cur_island.append(indexes[i])
+
+    # repeat dfs for neighbors
+    for movement in possible_movements:
+        dfs(indexes, i, i + movement, cur_island, d)
+
+
+def get_list_islands(indexes: List[int], **kwargs) -> List[List[int]]:
+    """Wrapper over DFS method to obtain islands from list of indexes of positive examples
+
+    Args:
+        indexes (List[int]): Indexes of positive examples. i.e [20,21,23,50,51]
+
+    Returns:
+        List[List[int]]: List of islands (each being a list)
+    """
+    islands = []
+    rows = len(indexes)
+    if rows == 0:
+        return islands
+
+    for i, valuei in enumerate(indexes):
+        # If already visited index in another dfs continue
+        if valuei in list(chain.from_iterable(islands)):
+            continue
+        # to hold coordinates of new island
+        cur_island = []
+        dfs(indexes, i, i, cur_island, **kwargs)
+
+        islands.append(cur_island)
+
+    return islands
+
+
+def get_phases_islands_minmax(
+    convo: pd.DataFrame,
+    phases: List[str],
+    column: str = "convo_part",
+    ignore: List[str] = [],
+    **kwargs,
+) -> List[Tuple[int]]:
+    """Given a conversation with predicted Phases (or Parts), get minimum and maximum index of calculated islands.
+
+    Args:
+        convo (pd.DataFrame): Conversation with predicted phases stored in `column`
+        phases (List[str]): Phases to filter in
+        column (str, optional): Column where predicted phases information is stored. Defaults to "convo_part".
+        ignore (List[str], optional): Ignore phases list. Defaults to [].
+
+    Returns:
+        List[Tuple[int]]: Minimum and Maximum values of calulated islands. i.e [(20,30), (40,60)]
+    """
+
+    reset = convo.query(f"{column}=={column} and {column} not in @ignore").reset_index()
+    sub_ = reset.query(f"{column} in @phases").copy()
+    indexes = sub_.index.tolist()
+    islands = get_list_islands(indexes, **kwargs)
+    if len(islands) > 1:
+        # If there is more than one island we want to make sure to root out comparable small islands
+        # I.e. if there is an island with 10 messages, and island of 1 messages is not useful in that context.
+        max_len = np.max([len(x) for x in islands])
+        len_cut = 3 if max_len > 9 else 2 if max_len > 3 else 1
+        islands = [x for x in islands if len(x) > len_cut]
+
+    islands = [reset.iloc[x] for x in islands]
+    minmax_islands = [(x["index"].min(), x["index"].max()) for x in islands]
+
+    return minmax_islands
+
+
+def filter_convo(
+    convo: pd.DataFrame,
+    phases: List[str],
+    column: str = "convo_part",
+    strategy: str = "islands",
+    pre_n: int = 5,
+    post_n: int = 5,
+    return_all_on_empty: bool = False,
+    **kwargs,
+) -> pd.DataFrame:
+    """Filter convo to include only specified phases. Take into account that sometimes predicted phases
+    can be messy. i.e. a prediciton of explore, explore, explore, safety_planning, explore; should return all
+    these messages as explore (probably safety_planning message has a low probability here.)
+
+    Args:
+        convo (pd.DataFrame): Conversation with predicted phases stored in `column`
+        phases (List[str]): Phases to filter in
+        column (str, optional): Column where predicted phases information is stored. Defaults to "convo_part".
+        strategy (str, optional): Strategy to use, can be minmax or islands. Defaults to "islands".
+        pre_n (int, optional): How many messages pre-phase to include. Defaults to 5.
+        post_n (int, optional): How many messages post-phase to include. Defaults to 5.
+        return_all_on_empty (bool, optional): Whether to return all messages when specified phases is not found. Defaults to False.
+
+    Returns:
+        pd.DataFrame: Filtered messages from the convo
+    """
+    if phases == ["ALL"]:
+        minidx = convo.index.min()
+        maxidx = convo.index.max()
+        minmax = [(minidx, maxidx)]
+    elif strategy == "minmax":
+        minidx = convo.query(f"{column} in @phases").index.min()
+        maxidx = convo.query(f"{column} in @phases").index.max() + 1
+        minmax = [(minidx, maxidx)]
+    elif strategy == "islands":
+        minmax = get_phases_islands_minmax(convo, phases, column, **kwargs)
+    parts = []
+    for minidx, maxidx in minmax:
+        minidx = max(convo.index.min(), minidx - pre_n)
+        maxidx = min(convo.index.max(), maxidx + post_n)
+        parts.append(convo.loc[minidx:maxidx])
+    if len(parts) == 0:
+        if return_all_on_empty:
+            return convo
+        else:
+            return pd.DataFrame(columns=convo.columns)
+    filtered = pd.concat(parts)
+    filtered = filtered[~filtered.index.duplicated(keep="first")]
+    return filtered

models/ta_models/ta_prompt_utils.py

@@ -0,0 +1,128 @@
+import inspect
+
+import pandas as pd
+
+from .config import QUESTION2FILTERARGS, TEXTER_PREFIX, HELPER_PREFIX
+
+# Utils to filter convo according to a phase
+from .ta_filter_utils import filter_convo
+
+
+def join_messages(
+    grp: pd.DataFrame, texter_prefix: str = "texter", helper_prefix: str = "helper"
+) -> str:
+    """join messages from dataframe using texter an helper prefixes
+
+    Args:
+        grp (pd.DataFrame): conversation in DataFrame with each row corresponding to each **message**.
+            Must have the following columns:
+            - actor_role
+            - message
+
+        texter_prefix (str, optional): prefix to use as the texter. Defaults to "texter".
+        helper_prefix (str, optional): prefix to use as the counselor (helper). Defaults to "helper".
+
+    Returns:
+        str: joined messages string separated by prefixes
+    """
+
+    if "actor_role" not in grp:
+        raise Exception("Column 'actor_role' not in DataFrame")
+    if "message" not in grp:
+        raise Exception("Column 'message' not in DataFrame")
+
+    roles = grp.actor_role.replace(
+        {"texter": texter_prefix, "counselor": helper_prefix, "helper": helper_prefix}
+    )
+    messages = roles.str.strip() + ": " + grp.message.str.strip()
+    return "\n".join(messages)
+
+
+def _get_context(grp: pd.DataFrame, **kwargs) -> str:
+    """Get context as a str taking into account message to delete, context marker
+    and the type of question to use. This allows for better truncation later
+
+    Args:
+        grp (pd.DataFrame): conversation in DataFrame with each row corresponding to each **message**.
+            Must have the following columns:
+            - actor_role
+            - message
+            - `column`
+        column (str): column name in which the marker of the problem is
+
+    Returns:
+        pd.DataFrame: joined messages string separated by prefixes
+    """
+
+    if "actor_role" not in grp:
+        raise Exception("Column 'actor_role' not in DataFrame")
+    if "message" not in grp:
+        raise Exception("Column 'message' not in DataFrame")
+
+    join_args = list(inspect.signature(join_messages).parameters)
+    join_kwargs = {k: kwargs.pop(k) for k in dict(kwargs) if k in join_args}
+
+    ## DEPRECATED
+    # context_args = list(inspect.signature(get_context_on_marker).parameters)
+    # context_kwargs = {k: kwargs.pop(k) for k in dict(kwargs) if k in context_args}
+
+    return join_messages(grp, **join_kwargs)
+
+
+def load_context(
+    messages: pd.DataFrame,
+    question: str,
+    message_col: str,
+    col_type: str,
+    inference: bool = False,
+    **kwargs,
+) -> pd.DataFrame:
+    """Load and filter conversation from messages given a question (with configured parameters of what phase that question is answered)
+
+    Args:
+        messages (pd.DataFrame): Messages dataframe with conversation_id, actor_role, `message_col` and phase prediction
+        question (str): Question to get context to
+        message_col (str): Column where messages are
+        col_type (str): type of message_col, can be "individual" or "joined"
+        base_dir (str, optional): Base directory to find model base args. Defaults to "../../".
+
+    Raises:
+        Exception: If question is not supported
+
+    Returns:
+        pd.DataFrame: filtered messages according to question configuration
+    """
+
+    if question not in QUESTION2FILTERARGS:
+        raise Exception(f"Question {question} not supported")
+
+    texter_prefix = TEXTER_PREFIX
+    helper_prefix = HELPER_PREFIX
+    context_data = messages.copy()
+
+    def convo_cpc_get_context(grp, **kwargs):
+        """Filter convo according to Convo Phase Classifier (CPC) predictions"""
+        context_ = filter_convo(grp, **QUESTION2FILTERARGS[question])
+        return _get_context(context_, **kwargs)
+
+    if col_type == "individual":
+        if "actor_role" in context_data:
+            context_data.dropna(subset=["actor_role"], inplace=True)
+        if "delete_message" in context_data:
+            context_data.delete_message.replace({1: True}, inplace=True)
+            context_data.delete_message.fillna(False, inplace=True)
+
+        context_data = (
+            context_data.groupby("conversation_id")
+            .apply(
+                convo_cpc_get_context,
+                helper_prefix=helper_prefix,
+                texter_prefix=texter_prefix,
+            )
+            .rename("q_context")
+        )
+    elif col_type == "joined":
+        context_data = context_data.groupby("conversation_id")[[message_col]].max()
+        context_data.rename(columns={message_col: "q_context"}, inplace=True)
+
+    return context_data

models/ta_models/ta_utils.py

@@ -0,0 +1,113 @@
+import os
+import re
+import requests
+import string
+import streamlit as st
+from streamlit.logger import get_logger
+from app_config import ENDPOINT_NAMES
+from models.ta_models.config import NAME2PROMPT, NAME2PROMPT_EXPL, START_INST, END_INST, QUESTIONDEFAULTS, TA_OPTIONS, NAME2QUESTION
+import pandas as pd
+from langchain_core.messages import AIMessage, HumanMessage
+from models.ta_models.ta_prompt_utils import load_context
+from utils.mongo_utils import new_convo_scoring_comparison
+
+logger = get_logger(__name__)
+TA_URL = os.environ["DATABRICKS_URL"].format(endpoint_name=ENDPOINT_NAMES["training_adherence"]['name'])
+HEADERS = {
+    "Authorization": f"Bearer {os.environ['DATABRICKS_TOKEN']}",
+    "Content-Type": "application/json",
+}
+
+def memory2df(memory, conversation_id="convo1234"):
+    df = []
+    for i, msg in enumerate(memory.buffer_as_messages):
+        actor_role = "texter" if type(msg) == AIMessage else "helper" if type(msg) == HumanMessage else None
+        if actor_role:
+            convo_part = msg.response_metadata.get("phase",None)
+            row = {"conversation_id":conversation_id, "message_number":i+1, "actor_role":actor_role, "message":msg.content, "convo_part":convo_part}
+            df.append(row)
+
+    return pd.DataFrame(df)
+
+def get_default(question, make_explanation=False):
+    return QUESTIONDEFAULTS[question][make_explanation]
+
+def get_context(memory, question, make_explanation=False, **kwargs):
+    df = memory2df(memory, **kwargs)
+    contexti = load_context(df, question, "messages", "individual").iloc[0]
+    if contexti == "":
+        return ""
+    
+    if make_explanation:
+        return NAME2PROMPT_EXPL[question].format(convo=contexti, start_inst=START_INST, end_inst=END_INST)
+    else:
+        return NAME2PROMPT[question].format(convo=contexti, start_inst=START_INST, end_inst=END_INST)
+
+def post_process_response(full_response, delimiter="\n\n", n=2):
+    parts = full_response.split(delimiter)[:n]
+    response = extract_response(parts[0])
+    logger.debug(f"Response extracted is {response}")
+    if len(parts) > 1:
+        if len(parts[0]) < len(parts[1]):
+            full_response = parts[1]
+        else: full_response = parts[0]
+    else:
+        full_response = parts[0]
+    explanation = full_response.lstrip(response).lstrip(string.punctuation)
+    explanation = explanation.strip()
+    logger.debug(f"Explanation extracted is {explanation}")
+    return response, explanation
+
+def TA_predict_convo(memory, question, make_explanation=False, **kwargs):
+    full_convo = memory.load_memory_variables({})[memory.memory_key]
+    PROMPT = get_context(memory, question, make_explanation=make_explanation, **kwargs)
+    logger.debug(f"Raw TA prompt is {PROMPT}")
+    if PROMPT == "":
+        full_response = get_default(question, make_explanation)
+        # response, explanation = post_process_response(full_response)
+        return full_convo, PROMPT, full_response
+    
+    max_tokens = 128 if make_explanation else 3
+    body_request = {
+        "prompt": PROMPT,
+        "temperature": 0,
+        "max_tokens": max_tokens,
+    }
+
+    try:
+        # Send request to Serving
+        response = requests.post(url=TA_URL, headers=HEADERS, json=body_request)
+        if response.status_code == 200:
+            response = response.json()
+        full_response = response[0]['choices'][0]['text']
+        logger.debug(f"Raw TA response is {full_response}")
+        # response, explanation = post_process_response(full_response)
+        return full_convo, PROMPT, full_response
+    except:
+        pass
+
+def extract_response(x: str, default: str = TA_OPTIONS[0]) -> str:
+    """Extract Response from generated answer
+    Extract only search strings
+
+    Args:
+        x (str): prediction
+        default (str, optional): default in case no response founds. Defaults to "N/A".
+
+    Returns:
+        str: _description_
+    """
+
+    try:
+        return re.findall("|".join(TA_OPTIONS), x)[0]
+    except Exception:
+        return default
+    
+def ta_push_convo_comparison(ytrue, ypred):
+    new_convo_scoring_comparison(**{
+        "client": st.session_state['db_client'],
+        "convo_id": st.session_state['convo_id'],
+        "context": st.session_state["context"] + "\nhelper:" + st.session_state["last_message"],
+        "ytrue": ytrue,
+        "ypred": ypred,
+    })

pages/convosim.py

@@ -49,7 +49,7 @@ with st.sidebar:
     if 'counselor_name' not in st.session_state:
         st.session_state["counselor_name"] = username #get_random_name(names_df=DEFAULT_NAMES_DF)
     # temperature = st.slider("Temperature", 0., 1., value=0.8, step=0.1)
-    issue = st.selectbox("Select a Scenario", ISSUES, index=0, format_func=issue2label,
+    issue = st.selectbox("Select a Scenario", ISSUES, index=ISSUES.index(st.session_state['issue']), format_func=issue2label,
                             on_change=clear_memory, kwargs={"memories":memories, "username":username, "language":"English"}
                         )
     supported_languages = ['en', "es"] if issue == "Anxiety" else ['en']
@@ -135,8 +135,9 @@ if prompt := st.chat_input(disabled=st.session_state['total_messages'] > MAX_MSG
     if any([x['score'] for x in st.session_state['bp_prediction']]):
         for bp in st.session_state['bp_prediction']:
             if bp["score"]:
-                st.error(f"Detected {BP_LAB2STR[bp['label']]} in the last message!")
+                st.toast(f"Detected {BP_LAB2STR[bp['label']]} in the last message!", icon=":material/warning:")
         st.session_state.changed_bp = True
+        sent_request_llm(llm_chain, prompt)
     else:
         sent_request_llm(llm_chain, prompt)
 
@@ -171,6 +172,9 @@ with st.sidebar:
             key="sel_bp"
         )
 
+        if st.button("Score Conversation"):
+            st.switch_page("pages/training_adherence.py")
+
 st.session_state['total_messages'] = len(memoryA.chat_memory.messages)
 if st.session_state['total_messages'] >= MAX_MSG_COUNT:
     st.toast(f"Total of {MAX_MSG_COUNT} Messages reached. Conversation Ended", icon=":material/verified:")

pages/training_adherence.py

@@ -0,0 +1,74 @@
+import streamlit as st
+import numpy as np
+from collections import defaultdict
+from langchain_core.messages import HumanMessage
+from utils.app_utils import are_models_alive
+from models.ta_models.ta_utils import TA_predict_convo, ta_push_convo_comparison, post_process_response
+from models.ta_models.config import QUESTION2PHASE, NAME2QUESTION, TA_OPTIONS
+
+if "memory" not in st.session_state:
+    st.switch_page("pages/convosim.py")
+
+if not are_models_alive():
+    st.switch_page("pages/model_loader.py")
+
+memory = st.session_state['memory']
+@st.cache_data(show_spinner="Retrieving responses from the server ...")
+def get_ta_responses():
+    data = defaultdict(defaultdict)
+    # with st.spinner("Retrieving responses from the server ...")
+    for question in QUESTION2PHASE.keys():
+        # responses = ["Yes, The helper showed some respect.",
+        #             "Yes. The helper is good! No doubt",
+        #             "N/A, Texter disengaged.",
+        #             "No. While texter is trying is lacking.",
+        #             "No \n\n This is an explanation."]
+        # full_response = np.random.choice(responses)
+        full_convo, prompt, full_response = TA_predict_convo(memory, question, make_explanation=True, conversation_id=st.session_state['convo_id'])
+        response, explanation = post_process_response(full_response)
+        data[question]["response"] = response
+        data[question]["explanation"] = explanation
+    return data
+
+with st.container():
+    col1, col2 = st.columns(2)
+    if col1.button("Go Back"):
+        get_ta_responses.clear()
+        st.switch_page("pages/convosim.py")
+    expl = col2.checkbox("Show Scoring Explanations")
+
+tab1, tab2 = st.tabs(["Scoring", "Conversation"])
+data = get_ta_responses()
+
+with tab2:
+    for msg in memory.buffer_as_messages:
+        role = "user" if type(msg) == HumanMessage else "assistant"
+        st.chat_message(role).write(msg.content)
+
+with tab1:
+    for question in QUESTION2PHASE.keys():
+        with st.container(border=True):
+            question_str = NAME2QUESTION[question].split(' Answer')[0]
+            st.radio(
+                f"**{question_str}**", options=TA_OPTIONS, 
+                index=TA_OPTIONS.index(data[question]['response']), horizontal=True,
+                key=f"{question}_manual"
+            )
+            if expl:
+                st.write(data[question]["explanation"])
+
+    with st.container():
+        col1, col2 = st.columns(2)
+        if col1.button("Go Back", key="goback2"):
+            get_ta_responses.clear()
+            st.switch_page("pages/convosim.py")
+        if col2.button("Submit Scoring", type="primary"):
+            ytrue = {
+                question: {
+                    "response":st.session_state[f"{question}_manual"]
+                } 
+                for question in QUESTION2PHASE.keys()
+            }
+            ta_push_convo_comparison(ytrue, data)
+            get_ta_responses.clear()
+            st.switch_page("pages/convosim.py")

utils/app_utils.py

@@ -101,6 +101,7 @@ def is_model_alive(name, timeout=2, model_type="classificator"):
         except:
             return "404" 
 
+@st.cache_data(ttl=300, show_spinner=False)
 def are_models_alive():
     models_alive = []
     for config in ENDPOINT_NAMES.values():

utils/mongo_utils.py

@@ -4,7 +4,7 @@ import streamlit as st
 from streamlit.logger import get_logger
 from pymongo.mongo_client import MongoClient
 from pymongo.server_api import ServerApi
-from app_config import DB_SCHEMA, DB_COMPLETIONS, DB_CONVOS, DB_BATTLES, DB_ERRORS, DB_CPC, DB_BP
+from app_config import DB_SCHEMA, DB_COMPLETIONS, DB_CONVOS, DB_BATTLES, DB_ERRORS, DB_CPC, DB_BP, DB_TA
 
 DB_URL = os.environ['MONGO_URL']
 DB_USR = os.environ['MONGO_USR']
@@ -19,7 +19,7 @@ def get_db_client():
     # Send a ping to confirm a successful connection
     try:
         client.admin.command('ping')
-        logger.info(f"DBUTILS: Pinged your deployment. You successfully connected to MongoDB!")
+        logger.debug(f"DBUTILS: Pinged your deployment. You successfully connected to MongoDB!")
         return client
     except Exception as e:
         logger.error(e)
@@ -38,7 +38,7 @@ def new_convo(client, issue, language, username, is_comparison, model_one, model
     db = client[DB_SCHEMA]
     convos = db[DB_CONVOS]
     convo_id = convos.insert_one(convo).inserted_id
-    logger.info(f"DBUTILS: new convo id is {convo_id}")
+    logger.debug(f"DBUTILS: new convo id is {convo_id}")
     st.session_state['convo_id'] = convo_id
 
 def new_comparison(client, prompt_timestamp, completion_timestamp, 
@@ -66,7 +66,7 @@ def new_comparison(client, prompt_timestamp, completion_timestamp,
     db = client[DB_SCHEMA]
     comparisons = db[DB_COMPLETIONS]
     comparison_id = comparisons.insert_one(comparison).inserted_id
-    logger.info(f"DBUTILS: new comparison id is {comparison_id}")
+    logger.debug(f"DBUTILS: new comparison id is {comparison_id}")
     st.session_state['comparison_id'] = comparison_id
 
 def new_battle_result(client, comparison_id, convo_id, username, model_one, model_two, winner):
@@ -84,7 +84,7 @@ def new_battle_result(client, comparison_id, convo_id, username, model_one, mode
     db = client[DB_SCHEMA]
     battles = db[DB_BATTLES]
     battle_id = battles.insert_one(battle).inserted_id
-    logger.info(f"DBUTILS: new battle id is {battle_id}")
+    logger.debug(f"DBUTILS: new battle id is {battle_id}")
 
 def new_completion_error(client, comparison_id, username, model):
     error = {
@@ -97,12 +97,12 @@ def new_completion_error(client, comparison_id, username, model):
     db = client[DB_SCHEMA]
     errors = db[DB_ERRORS]
     error_id = errors.insert_one(error).inserted_id
-    logger.info(f"DBUTILS: new error id is {error_id}")
+    logger.debug(f"DBUTILS: new error id is {error_id}")
 
 def new_cpc_comparison(client, convo_id, model, context, last_message, ytrue, ypred):
     # context = memory.load_memory_variables({})[memory.memory_key]
     comp = {
-        "error_timestamp": dt.datetime.now(tz=dt.timezone.utc),
+        "CPC_timestamp": dt.datetime.now(tz=dt.timezone.utc),
         "conversation_id": convo_id,
         "model": model,    
         "context": context,
@@ -114,12 +114,12 @@ def new_cpc_comparison(client, convo_id, model, context, last_message, ytrue, yp
     db = client[DB_SCHEMA]
     cpc_comps = db[DB_CPC]
     comarison_id = cpc_comps.insert_one(comp).inserted_id
-    # logger.info(f"DBUTILS: new error id is {error_id}")
+    logger.debug(f"DBUTILS: new error id is {comarison_id}")
 
 def new_bp_comparison(client, convo_id, model, context, last_message, ytrue, ypred):
     # context = memory.load_memory_variables({})[memory.memory_key]
     comp = {
-        "error_timestamp": dt.datetime.now(tz=dt.timezone.utc),
+        "BP_timestamp": dt.datetime.now(tz=dt.timezone.utc),
         "conversation_id": convo_id,
         "model": model,    
         "context": context,
@@ -133,7 +133,22 @@ def new_bp_comparison(client, convo_id, model, context, last_message, ytrue, ypr
     db = client[DB_SCHEMA]
     bp_comps = db[DB_BP]
     comarison_id = bp_comps.insert_one(comp).inserted_id
-    logger.info(f"DBUTILS: new BP id is {comarison_id}")
+    logger.debug(f"DBUTILS: new BP id is {comarison_id}")
+
+def new_convo_scoring_comparison(client, convo_id, context, ytrue, ypred):
+    # context = memory.load_memory_variables({})[memory.memory_key]
+    comp = {
+        "scoring_timestamp": dt.datetime.now(tz=dt.timezone.utc),
+        "conversation_id": convo_id,   
+        "context": context,
+        "manual_scoring": ytrue,
+        "model_scoring": ypred,
+    }
+
+    db = client[DB_SCHEMA]
+    ta_comps = db[DB_TA]
+    comarison_id = ta_comps.insert_one(comp).inserted_id
+    logger.debug(f"DBUTILS: new TA convo comparison id is {comarison_id}")
 
 def get_non_assesed_comparison(client, username):
     from bson.son import SON