|
import os |
|
from typing import Dict, List, Union |
|
|
|
import numpy as np |
|
import openai |
|
import pandas as pd |
|
import streamlit as st |
|
from langchain.document_loaders import TextLoader |
|
from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings |
|
from langchain.text_splitter import CharacterTextSplitter |
|
from langchain.vectorstores import Chroma |
|
from scipy.spatial.distance import cosine |
|
|
|
openai.api_key = os.environ["OPENAI_API_KEY"] |
|
|
|
|
|
def merge_dataframes(dataframes: List[pd.DataFrame]) -> pd.DataFrame: |
|
"""Merges a list of DataFrames, keeping only specific columns.""" |
|
|
|
combined_dataframe = pd.concat( |
|
dataframes, ignore_index=True |
|
) |
|
|
|
|
|
combined_dataframe = combined_dataframe[ |
|
["context", "questions", "answers"] |
|
] |
|
|
|
return combined_dataframe |
|
|
|
|
|
def call_chatgpt(prompt: str) -> str: |
|
""" |
|
Uses the OpenAI API to generate an AI response to a prompt. |
|
|
|
Args: |
|
prompt: A string representing the prompt to send to the OpenAI API. |
|
|
|
Returns: |
|
A string representing the AI's generated response. |
|
|
|
""" |
|
|
|
|
|
response = openai.Completion.create( |
|
model="gpt-3.5-turbo-instruct", |
|
prompt=prompt, |
|
temperature=0.5, |
|
max_tokens=500, |
|
top_p=1, |
|
frequency_penalty=0, |
|
presence_penalty=0, |
|
) |
|
|
|
|
|
ans = response.choices[0]["text"] |
|
|
|
|
|
return ans |
|
|
|
|
|
def openai_text_embedding(prompt: str) -> str: |
|
return openai.Embedding.create(input=prompt, model="text-embedding-ada-002")[ |
|
"data" |
|
][0]["embedding"] |
|
|
|
|
|
def calculate_sts_openai_score(sentence1: str, sentence2: str) -> float: |
|
|
|
embedding1 = openai_text_embedding(sentence1) |
|
embedding2 = openai_text_embedding(sentence2) |
|
|
|
|
|
embedding1 = np.asarray(embedding1) |
|
embedding2 = np.asarray(embedding2) |
|
|
|
|
|
similarity_score = 1 - cosine(embedding1, embedding2) |
|
|
|
return similarity_score |
|
|
|
|
|
def add_dist_score_column( |
|
dataframe: pd.DataFrame, |
|
sentence: str, |
|
) -> pd.DataFrame: |
|
dataframe["stsopenai"] = dataframe["questions"].apply( |
|
lambda x: calculate_sts_openai_score(str(x), sentence) |
|
) |
|
|
|
sorted_dataframe = dataframe.sort_values(by="stsopenai", ascending=False) |
|
return sorted_dataframe.iloc[:5, :] |
|
|
|
|
|
def convert_to_list_of_dict(df: pd.DataFrame) -> List[Dict[str, str]]: |
|
""" |
|
Reads in a pandas DataFrame and produces a list of dictionaries with two keys each, 'question' and 'answer.' |
|
|
|
Args: |
|
df: A pandas DataFrame with columns named 'questions' and 'answers'. |
|
|
|
Returns: |
|
A list of dictionaries, with each dictionary containing a 'question' and 'answer' key-value pair. |
|
""" |
|
|
|
|
|
result = [] |
|
|
|
|
|
for index, row in df.iterrows(): |
|
|
|
qa_dict_quest = {"role": "user", "content": row["questions"]} |
|
qa_dict_ans = {"role": "assistant", "content": row["answers"]} |
|
|
|
|
|
result.append(qa_dict_quest) |
|
result.append(qa_dict_ans) |
|
|
|
|
|
return result |
|
|
|
|
|
file_names = [f"output_files/file_{i}.txt" for i in range(131)] |
|
|
|
|
|
|
|
|
|
all_documents = [] |
|
|
|
|
|
for file_name in file_names: |
|
loader = TextLoader(file_name) |
|
documents = loader.load() |
|
all_documents.extend(documents) |
|
|
|
|
|
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0) |
|
docs = text_splitter.split_documents(all_documents) |
|
|
|
|
|
embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") |
|
|
|
|
|
|
|
|
|
db = Chroma.from_documents(docs, embedding_function) |
|
|
|
|
|
st.title("Youth Homelessness Chatbot") |
|
|
|
|
|
if "messages" not in st.session_state: |
|
st.session_state.messages = [] |
|
|
|
|
|
for message in st.session_state.messages: |
|
with st.chat_message(message["role"]): |
|
st.markdown(message["content"]) |
|
|
|
st.sidebar.markdown("""This is an app to help you navigate the website of YSA""") |
|
|
|
clear_button = st.sidebar.button("Clear Conversation", key="clear") |
|
|
|
if clear_button: |
|
st.session_state.messages = [] |
|
|
|
|
|
if prompt := st.chat_input("Tell me about YSA"): |
|
|
|
st.chat_message("user").markdown(prompt) |
|
|
|
st.session_state.messages.append({"role": "user", "content": prompt}) |
|
|
|
question = prompt |
|
|
|
docs = db.similarity_search(question) |
|
docs_2 = db.similarity_search_with_score(question) |
|
docs_2_table = pd.DataFrame( |
|
{ |
|
"source": [docs_2[i][0].metadata["source"] for i in range(len(docs))], |
|
"content": [docs_2[i][0].page_content for i in range(len(docs))], |
|
"distances": [docs_2[i][1] for i in range(len(docs))], |
|
} |
|
) |
|
ref_from_db_search = docs_2_table["content"] |
|
|
|
engineered_prompt = f""" |
|
Based on the context: {ref_from_db_search}, |
|
answer the user question: {question}. |
|
Answer the question directly (don't say "based on the context, ...") |
|
""" |
|
|
|
answer = call_chatgpt(engineered_prompt) |
|
response = answer |
|
|
|
|
|
with st.chat_message("assistant"): |
|
with st.spinner("Wait for it..."): |
|
st.markdown(response) |
|
with st.expander("See reference:"): |
|
st.table(docs_2_table) |
|
|
|
st.session_state.messages.append({"role": "assistant", "content": response}) |
|
st.session_state.messages.append( |
|
{"role": "assistant", "content": docs_2_table.to_json()} |
|
) |
|
|