Update app.py

app.py

@@ -1,157 +1,291 @@
-import os  
-import streamlit as st  
-import google.generativeai as genai  
-from langchain_google_genai import GoogleGenerativeAIEmbeddings  
-from langchain_google_genai import ChatGoogleGenerativeAI  
-from langchain_community.document_loaders import PyPDFLoader   
-from langchain.text_splitter import RecursiveCharacterTextSplitter  
-from langchain_community.vectorstores import Chroma  
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder  
-from langchain_core.messages import HumanMessage, SystemMessage  
-from langchain.chains import create_history_aware_retriever, create_retrieval_chain 
-from langchain.chains.combine_documents import create_stuff_documents_chain 
-from dotenv import load_dotenv  
-from langchain.embeddings import HuggingFaceEmbeddings
+# import os  
+# import streamlit as st  
+# import google.generativeai as genai  
+# from langchain_google_genai import GoogleGenerativeAIEmbeddings  
+# from langchain_google_genai import ChatGoogleGenerativeAI  
+# from langchain_community.document_loaders import PyPDFLoader   
+# from langchain.text_splitter import RecursiveCharacterTextSplitter  
+# from langchain_community.vectorstores import Chroma  
+# from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder  
+# from langchain_core.messages import HumanMessage, SystemMessage  
+# from langchain.chains import create_history_aware_retriever, create_retrieval_chain 
+# from langchain.chains.combine_documents import create_stuff_documents_chain 
+# from dotenv import load_dotenv  
+# from langchain.embeddings import HuggingFaceEmbeddings
+
+# from sentence_transformers import SentenceTransformer
+
+# import pysqlite3
+# import sys
+# sys.modules['sqlite3'] = pysqlite3
+
+# import os
+
+# # Retrieve Google API key
+# GOOGLE_API_KEY = str(os.getenv('GOOGLE_API_KEY'))
+# HF_TOKEN = str(os.getenv("HF_TOKEN"))
+
+# if not GOOGLE_API_KEY:
+#     raise ValueError("Gemini API key not found. Please set it in the .env file.")
+
+# os.environ["GOOGLE_API_KEY"] = GOOGLE_API_KEY
+# os.environ["HF_TOKEN"] = HF_TOKEN
+# # Streamlit app configuration
+# st.set_page_config(page_title="English Chatbot", layout="centered")
+# st.title("English Tutor Bot")
+
+# # Initialize Google Generative AI LLM 
+# llm = ChatGoogleGenerativeAI(
+#     model="gemini-1.5-pro-latest",
+#     temperature=0.2,
+#     max_tokens=500,
+#     timeout=None,
+#     max_retries=2,
+# )
+
+# # Initialize embeddings using HuggingFace
+# embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+
+# def load_preprocessed_vectorstore():
+#     try:
+#         loader = PyPDFLoader("sound.pdf")
+#         documents = loader.load()
+
+#         text_splitter = RecursiveCharacterTextSplitter(
+#             separators=["\n\n", "\n", ". ", " ", ""],
+#             chunk_size=500, 
+#             chunk_overlap=150
+#         )
+
+#         document_chunks = text_splitter.split_documents(documents)
+
+#         vector_store = Chroma.from_documents(
+#             embedding=embeddings,
+#             documents=document_chunks,
+#             persist_directory="./data32"
+#         )
+#         return vector_store
+#     except Exception as e:
+#         st.error(f"Error creating vector store: {e}")
+#         return None
+
+# def get_context_retriever_chain(vector_store):
+#     retriever = vector_store.as_retriever()
+
+#     prompt = ChatPromptTemplate.from_messages([
+#         MessagesPlaceholder(variable_name="chat_history"),
+#         ("human", "{input}"),
+#         ("system", """You are an expert english tutor, your task is to help users to learn english. Given the chat history and the latest user question, which might reference context in the chat history, Answer the question
+#         by taking reference from the document.
+# If the question is directly addressed within the provided document, provide a relevant answer. 
+# If the question is not explicitly addressed in the document, return the following message: 
+# 'This question is beyond the scope of the available information. Please contact your mentor for further assistance.'
+# Do NOT answer the question directly, just reformulate it if needed and otherwise return it as is.""")
+#     ])
+
+#     retriever_chain = create_history_aware_retriever(llm, retriever, prompt)
+#     return retriever_chain
+
+# def get_conversational_chain(retriever_chain):
+#     prompt = ChatPromptTemplate.from_messages([
+#         ("system", """Hello! I'm your English Tutor, I am here to help you with learning english and can also take quiz to test your skills.
+# Note: I will only provide information that is available within our database to ensure accuracy. Let's get started!
+# """
+#          "\n\n"
+#          "{context}"),
+#         MessagesPlaceholder(variable_name="chat_history"),
+#         ("human", "{input}")
+#     ])
+
+#     stuff_documents_chain = create_stuff_documents_chain(llm, prompt)
+#     return create_retrieval_chain(retriever_chain, stuff_documents_chain)
+
+# def get_response(user_query):
+#     retriever_chain = get_context_retriever_chain(st.session_state.vector_store)
+#     conversation_rag_chain = get_conversational_chain(retriever_chain)
+
+#     formatted_chat_history = []
+#     for message in st.session_state.chat_history:
+#         if isinstance(message, HumanMessage):
+#             formatted_chat_history.append({"author": "user", "content": message.content})
+#         elif isinstance(message, SystemMessage):
+#             formatted_chat_history.append({"author": "assistant", "content": message.content})
+
+#     response = conversation_rag_chain.invoke({
+#         "chat_history": formatted_chat_history,
+#         "input": user_query
+#     })
+
+#     return response['answer']
+
+# # Load the preprocessed vector store from the local directory
+# st.session_state.vector_store = load_preprocessed_vectorstore()
+
+# # Initialize chat history if not present
+# if "chat_history" not in st.session_state:
+#     st.session_state.chat_history = [
+#         {"author": "assistant", "content": "Hello, I am a English Tutor Bot. How can I help you?"}
+#     ]
+
+# # Main app logic
+# if st.session_state.get("vector_store") is None:
+#     st.error("Failed to load preprocessed data. Please ensure the data exists in './data' directory.")
+# else:
+#     # Display chat history
+#     with st.container():
+#         for message in st.session_state.chat_history:
+#             if message["author"] == "assistant":
+#                 with st.chat_message("system"):
+#                     st.write(message["content"])
+#             elif message["author"] == "user":
+#                 with st.chat_message("human"):
+#                     st.write(message["content"])
 
-from sentence_transformers import SentenceTransformer
+#     # Add user input box below the chat
+#     with st.container():
+#         with st.form(key="chat_form", clear_on_submit=True):
+#             user_query = st.text_input("Type your message here...", key="user_input")
+#             submit_button = st.form_submit_button("Send")
+
+#         if submit_button and user_query:
+#             # Get bot response
+#             response = get_response(user_query)
+#             st.session_state.chat_history.append({"author": "user", "content": user_query})
+#             st.session_state.chat_history.append({"author": "assistant", "content": response})
+
+#             # Rerun the app to refresh the chat display
+#             st.rerun()
 
-import pysqlite3
-import sys
-sys.modules['sqlite3'] = pysqlite3
 
 import os
+import logging
+import pathlib
+from telegram import Update
+from telegram.ext import Updater, CommandHandler, MessageHandler, CallbackContext, Filters
+from langchain.prompts import PromptTemplate
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import LLMChain, create_retrieval_chain
+from langchain_google_genai import GoogleGenerativeAI
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from langchain_community.document_loaders import Docx2txtLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceEmbeddings
 
-# Retrieve Google API key
-GOOGLE_API_KEY = str(os.getenv('GOOGLE_API_KEY'))
-HF_TOKEN = str(os.getenv("HF_TOKEN"))
+# Enable logging
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
+)
+logger = logging.getLogger(__name__)
 
-if not GOOGLE_API_KEY:
-    raise ValueError("Gemini API key not found. Please set it in the .env file.")
+GOOGLE_API_KEY = "AIzaSyAytkzRS0Xp0pCyo6WqKJ4m1o330bF-gPk"
+OPENAI_API_KEY = "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"
+TOKEN = '8126949340:AAGmr4ByOLlYXtEQuleOsinS2w_wUogldj0'
 
+# Set up OpenAI API key
 os.environ["GOOGLE_API_KEY"] = GOOGLE_API_KEY
-os.environ["HF_TOKEN"] = HF_TOKEN
-# Streamlit app configuration
-st.set_page_config(page_title="English Chatbot", layout="centered")
-st.title("English Tutor Bot")
-
-# Initialize Google Generative AI LLM 
-llm = ChatGoogleGenerativeAI(
-    model="gemini-1.5-pro-latest",
-    temperature=0.2,
-    max_tokens=500,
-    timeout=None,
-    max_retries=2,
-)
 
 # Initialize embeddings using HuggingFace
 embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
 
 def load_preprocessed_vectorstore():
+    """Load documents and create a vector store."""
     try:
-        loader = PyPDFLoader("sound.pdf")
+        loader = Docx2txtLoader("./Pre.docx")
         documents = loader.load()
-
+        
         text_splitter = RecursiveCharacterTextSplitter(
             separators=["\n\n", "\n", ". ", " ", ""],
-            chunk_size=500, 
-            chunk_overlap=150
-        )
-
+            chunk_size=3000,
+            chunk_overlap=1000)
+        
         document_chunks = text_splitter.split_documents(documents)
 
         vector_store = Chroma.from_documents(
             embedding=embeddings,
             documents=document_chunks,
-            persist_directory="./data32"
+            persist_directory="./data"
         )
         return vector_store
     except Exception as e:
-        st.error(f"Error creating vector store: {e}")
+        logger.error(f"Error creating vector store: {e}")
         return None
 
-def get_context_retriever_chain(vector_store):
-    retriever = vector_store.as_retriever()
-
-    prompt = ChatPromptTemplate.from_messages([
-        MessagesPlaceholder(variable_name="chat_history"),
-        ("human", "{input}"),
-        ("system", """You are an expert english tutor, your task is to help users to learn english. Given the chat history and the latest user question, which might reference context in the chat history, Answer the question
-        by taking reference from the document.
-If the question is directly addressed within the provided document, provide a relevant answer. 
-If the question is not explicitly addressed in the document, return the following message: 
-'This question is beyond the scope of the available information. Please contact your mentor for further assistance.'
-Do NOT answer the question directly, just reformulate it if needed and otherwise return it as is.""")
-    ])
-
-    retriever_chain = create_history_aware_retriever(llm, retriever, prompt)
-    return retriever_chain
-
-def get_conversational_chain(retriever_chain):
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", """Hello! I'm your English Tutor, I am here to help you with learning english and can also take quiz to test your skills.
-Note: I will only provide information that is available within our database to ensure accuracy. Let's get started!
-"""
-         "\n\n"
-         "{context}"),
-        MessagesPlaceholder(variable_name="chat_history"),
-        ("human", "{input}")
-    ])
-
-    stuff_documents_chain = create_stuff_documents_chain(llm, prompt)
-    return create_retrieval_chain(retriever_chain, stuff_documents_chain)
-
-def get_response(user_query):
-    retriever_chain = get_context_retriever_chain(st.session_state.vector_store)
-    conversation_rag_chain = get_conversational_chain(retriever_chain)
+# Initialize the vector store
+vector_store = load_preprocessed_vectorstore()
 
+# Define the Langchain chain with a retrieval mechanism
+def get_response(user_message, context):
+    retriever_chain = vector_store.as_retriever()
+    prompt_template = PromptTemplate(input_variables=["chat_history", "input"], template="{chat_history}\nUser: {input}\nAssistant:")
+    
+    conversation_chain = create_retrieval_chain(retriever_chain, prompt_template)
+    
     formatted_chat_history = []
-    for message in st.session_state.chat_history:
-        if isinstance(message, HumanMessage):
-            formatted_chat_history.append({"author": "user", "content": message.content})
-        elif isinstance(message, SystemMessage):
-            formatted_chat_history.append({"author": "assistant", "content": message.content})
-
-    response = conversation_rag_chain.invoke({
+    if 'chat_history' in context.user_data:
+        formatted_chat_history.extend(context.user_data['chat_history'])
+    
+    response = conversation_chain.invoke({
         "chat_history": formatted_chat_history,
-        "input": user_query
+        "input": user_message
     })
-
+    
+    # Update chat history in user data
+    context.user_data['chat_history'] = formatted_chat_history + [{"author": "user", "content": user_message}, {"author": "assistant", "content": response['answer']}]
+    
     return response['answer']
 
-# Load the preprocessed vector store from the local directory
-st.session_state.vector_store = load_preprocessed_vectorstore()
-
-# Initialize chat history if not present
-if "chat_history" not in st.session_state:
-    st.session_state.chat_history = [
-        {"author": "assistant", "content": "Hello, I am a English Tutor Bot. How can I help you?"}
-    ]
-
-# Main app logic
-if st.session_state.get("vector_store") is None:
-    st.error("Failed to load preprocessed data. Please ensure the data exists in './data' directory.")
-else:
-    # Display chat history
-    with st.container():
-        for message in st.session_state.chat_history:
-            if message["author"] == "assistant":
-                with st.chat_message("system"):
-                    st.write(message["content"])
-            elif message["author"] == "user":
-                with st.chat_message("human"):
-                    st.write(message["content"])
-
-    # Add user input box below the chat
-    with st.container():
-        with st.form(key="chat_form", clear_on_submit=True):
-            user_query = st.text_input("Type your message here...", key="user_input")
-            submit_button = st.form_submit_button("Send")
-
-        if submit_button and user_query:
-            # Get bot response
-            response = get_response(user_query)
-            st.session_state.chat_history.append({"author": "user", "content": user_query})
-            st.session_state.chat_history.append({"author": "assistant", "content": response})
-
-            # Rerun the app to refresh the chat display
-            st.rerun()
+# Start the bot
+def start(update: Update, context: CallbackContext) -> None:
+    """Send a message when the command /start is issued."""
+    user = update.effective_user
+    update.message.reply_text(f'Hi {user.first_name}! I\'m a bot powered by OpenAI. Ask me anything.')
+
+# Help command
+def help_command(update: Update, context: CallbackContext) -> None:
+    """Send a message when the command /help is issued."""
+    update.message.reply_text('Ask me any question, and I\'ll try to answer using my knowledge!')
+
+# Handle messages
+def handle_message(update: Update, context: CallbackContext) -> None:
+    """Handle user messages and generate responses using Langchain."""
+    user_message = update.message.text
+
+    try:
+        # Generate a response using Langchain and OpenAI
+        response = get_response(user_message, context)
+        update.message.reply_text(response)
+    except Exception as e:
+        update.message.reply_text("Sorry, I couldn't process your request at the moment.")
+        logger.error(f"Error: {e}")
+
+# Error handler
+def error_handler(update: Update, context: CallbackContext) -> None:
+    """Log Errors caused by Updates."""
+    logger.warning(f'Update "{update}" caused error "{context.error}"')
+
+def main() -> None:
+    """Start the bot."""
+    updater = Updater(TOKEN)
+
+    # Get the dispatcher to register handlers
+    dispatcher = updater.dispatcher
+
+    # On different commands - answer in Telegram
+    dispatcher.add_handler(CommandHandler("start", start))
+    dispatcher.add_handler(CommandHandler("help", help_command))
+
+    # On non-command i.e. message - handle the message
+    dispatcher.add_handler(MessageHandler(Filters.text & ~Filters.command, handle_message))
+
+    # Log all errors
+    dispatcher.add_error_handler(error_handler)
+
+    # Start the Bot
+    updater.start_polling()
+
+    # Run the bot until you press Ctrl-C or the process receives SIGINT, SIGTERM, or SIGABRT
+    updater.idle()
+
+if __name__ == '__main__':
+    main()