Upload 7 files

.env

@@ -0,0 +1 @@
+GOOGLE_API_KEY = "AIzaSyCOVPvbV9NEg2dYAsP5i98bQnsGQW_qWMc"

Precollege.py

@@ -0,0 +1,153 @@
+import os
+import tempfile
+import pathlib
+import getpass
+import streamlit as st
+import google.generativeai as genai
+from langchain_community.vectorstores import Chroma 
+from langchain_community.document_loaders import Docx2txtLoader 
+from langchain.text_splitter import RecursiveCharacterTextSplitter 
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder 
+from langchain_core.messages import HumanMessage, SystemMessage 
+from langchain_google_genai import GoogleGenerativeAIEmbeddings 
+from langchain.chains import create_history_aware_retriever, create_retrieval_chain 
+from langchain.chains.combine_documents import create_stuff_documents_chain 
+
+os.environ["GOOGLE_API_KEY"] = "AIzaSyCOVPvbV9NEg2dYAsP5i98bQnsGQW_qWMc" 
+
+from langchain_google_genai import ChatGoogleGenerativeAI 
+
+llm = ChatGoogleGenerativeAI( 
+    model="gemini-1.5-pro-latest", 
+    temperature=0, 
+    max_tokens=None, 
+    timeout=None, 
+    max_retries=2, 
+)
+
+embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+
+def get_vectorstore_from_docx(docx_file):
+    try:
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.docx') as temp_file:
+            temp_file.write(docx_file.read())
+            temp_file_path = temp_file.name
+
+        loader = Docx2txtLoader(temp_file_path)
+        documents = loader.load()
+
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=5000, chunk_overlap=0)
+        document_chunks = text_splitter.split_documents(documents)
+
+        vector_store = Chroma.from_documents(
+            embedding=embeddings,
+            documents=document_chunks,
+            persist_directory="./data"
+        )
+        os.remove(temp_file_path)
+        return vector_store
+    except Exception as e:
+        st.error(f"Error creating vector store: {e}")
+        return None
+
+def get_context_retriever_chain(vector_store):
+    retriever = vector_store.as_retriever()
+    
+    prompt = ChatPromptTemplate.from_messages([
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("user", "{input}"),
+
+        ("system", """Act as a PreCollege AI assistant dedicated to guiding students through their JEE Mains journey. Your goal is to provide personalized, accurate, and interactive advice for students seeking college admissions guidance. Tailor your responses to address students' individual needs, including:
+
+1. College Selection and Counseling: Help students identify colleges they qualify for based on their JEE Mains rank and preferences, including NITs, IIITs, GFTIs, and private institutions. Consider factors like location, course offerings, placement records, and fees.
+
+2. Admission Process Guidance: Clarify the college admission procedures, including JoSAA counseling, spot rounds, document verification, and category-specific quotas (if applicable).
+
+3. Career and Branch Selection Advice: Assist students in making informed decisions about their preferred engineering branches based on interest, market trends, and scope of opportunities.
+
+Interactive Sessions: Engage students in Q&A sessions to answer their doubts related to preparation, counseling, and career choices.
+
+Maintain a professional and friendly tone. Use your expertise to ensure students receive relevant and clear information. Provide examples, stats, and other insights to support your advice wherever needed""")
+    ])
+    
+    retriever_chain = create_history_aware_retriever(llm, retriever, prompt)
+    return retriever_chain
+
+def get_conversational_chain(retriever_chain):
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", "Answer the user's questions based on the context below:\n\n{context}"),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("user", "{input}")
+    ])
+    
+    stuff_documents_chain = create_stuff_documents_chain(llm, prompt)
+    return create_retrieval_chain(retriever_chain, stuff_documents_chain)
+
+def get_response(user_query):
+    retriever_chain = get_context_retriever_chain(st.session_state.vector_store)
+    conversation_rag_chain = get_conversational_chain(retriever_chain)
+    
+    formatted_chat_history = []
+    for message in st.session_state.chat_history:
+        if isinstance(message, HumanMessage):
+            formatted_chat_history.append({"author": "user", "content": message.content})
+        elif isinstance(message, SystemMessage):
+            formatted_chat_history.append({"author": "assistant", "content": message.content})
+    
+    response = conversation_rag_chain.invoke({
+        "chat_history": formatted_chat_history,
+        "input": user_query
+    })
+    
+    return response['answer']
+
+st.set_page_config(page_title="College Data Chatbot")
+st.title("College Data Chatbot")
+
+with st.sidebar:
+    st.header("Settings")
+    docx_files = st.file_uploader("Upload College Data Document", accept_multiple_files=True)
+
+    if not docx_files:
+        st.info("Please upload a .docx file")
+    else:
+        docx_file = docx_files[0]
+
+        if "docx_name" in st.session_state and st.session_state.docx_name != docx_file.name:
+            st.session_state.pop("vector_store", None)
+            st.session_state.pop("chat_history", None)
+        
+        if st.button("Preprocess"):
+            st.session_state.vector_store = get_vectorstore_from_docx(docx_file)
+            if st.session_state.vector_store:
+                st.session_state.docx_name = docx_file.name
+                st.success("Document processed successfully!")
+
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = [
+        {"author": "assistant", "content": "Hello, I am a bot. How can I help you?"}
+    ]
+
+if st.session_state.get("vector_store") is None:
+    st.info("Please preprocess the document by clicking the 'Preprocess' button in the sidebar.")
+else:
+    for message in st.session_state.chat_history:
+        if message["author"] == "assistant":
+            with st.chat_message("system"):
+                st.write(message["content"])
+        elif message["author"] == "user":
+            with st.chat_message("human"):
+                st.write(message["content"])
+
+    with st.form(key="chat_form", clear_on_submit=True):
+            user_query = st.text_input("Type your message here...", key="user_input")
+            submit_button = st.form_submit_button("Send")
+
+    if submit_button and user_query:
+            # Get bot response
+            response = get_response(user_query)
+            st.session_state.chat_history.append({"author": "user", "content": user_query})
+            st.session_state.chat_history.append({"author": "assistant", "content": response})
+
+            # Rerun the app to refresh the chat display
+            st.rerun()

chat_1.py

@@ -0,0 +1,336 @@
+# import os  
+# import streamlit as st  
+# import google.generativeai as genai  
+# from langchain_google_genai import GoogleGenerativeAIEmbeddings, ChatGoogleGenerativeAI  
+# from langchain_community.document_loaders import Docx2txtLoader   
+# from langchain.text_splitter import RecursiveCharacterTextSplitter  
+# from langchain_community.vectorstores import Chroma  
+# from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder  
+# from langchain_core.messages import HumanMessage, SystemMessage  
+# from langchain.chains import create_history_aware_retriever, create_retrieval_chain 
+# from langchain.chains.combine_documents import create_stuff_documents_chain 
+# from langchain.embeddings import HuggingFaceEmbeddings
+# from bert_score import score
+# from sklearn.metrics import f1_score
+# import pysqlite3
+# import sys
+# sys.modules['sqlite3'] = pysqlite3
+
+# # Retrieve Google API Key
+# GOOGLE_API_KEY = "AIzaSyAytkzRS0Xp0pCyo6WqKJ4m1o330bF-gPk"
+# if not GOOGLE_API_KEY:
+#     raise ValueError("Gemini API key not found. Please set it in the .env file.")
+# os.environ["GOOGLE_API_KEY"] = GOOGLE_API_KEY
+
+# # Streamlit configuration
+# st.set_page_config(page_title="College Data Chatbot", layout="centered")
+# st.title("PreCollege Chatbot GEMINI+ HuggingFace Embeddings")
+
+# # Initialize LLM and embeddings
+# llm = ChatGoogleGenerativeAI(
+#     model="gemini-1.5-pro-latest",
+#     temperature=0.2,
+#     max_tokens=None,
+#     timeout=None,
+#     max_retries=2,
+# )
+# embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')
+
+# # Load vector store
+# def load_preprocessed_vectorstore():
+#     try:
+#         loader = Docx2txtLoader("./Updated_structred_aman.docx")
+#         documents = loader.load()
+
+#         text_splitter = RecursiveCharacterTextSplitter(
+#             separators=["\n\n", "\n", ". ", " ", ""],
+#             chunk_size=3000, 
+#             chunk_overlap=1000
+#         )
+#         document_chunks = text_splitter.split_documents(documents)
+
+#         vector_store = Chroma.from_documents(
+#             embedding=embeddings,
+#             documents=document_chunks,
+#             persist_directory="./data32"
+#         )
+#         return vector_store
+#     except Exception as e:
+#         st.error(f"Error creating vector store: {e}")
+#         return None
+
+# # Evaluation Metrics
+# def calculate_recall_at_k(retrieved_docs, relevant_docs, k=5):
+#     retrieved_top_k = retrieved_docs[:k]
+#     relevant_in_top_k = len(set(retrieved_top_k).intersection(set(relevant_docs)))
+#     total_relevant = len(relevant_docs)
+#     return relevant_in_top_k / total_relevant if total_relevant > 0 else 0.0
+
+# def calculate_bertscore(generated_responses, reference_responses):
+#     P, R, F1 = score(generated_responses, reference_responses, lang="en", rescale_with_baseline=True)
+#     return {"precision": P.mean().item(), "recall": R.mean().item(), "f1": F1.mean().item()}
+
+# def calculate_f1_score(generated_response, relevant_text):
+#     generated_tokens = set(generated_response.split())
+#     relevant_tokens = set(relevant_text.split())
+#     intersection = generated_tokens.intersection(relevant_tokens)
+    
+#     precision = len(intersection) / len(generated_tokens) if len(generated_tokens) > 0 else 0
+#     recall = len(intersection) / len(relevant_tokens) if len(relevant_tokens) > 0 else 0
+    
+#     if precision + recall > 0:
+#         f1 = 2 * (precision * recall) / (precision + recall)
+#     else:
+#         f1 = 0.0
+#     return f1
+
+# # Context Retriever Chain
+# def get_context_retriever_chain(vector_store):
+#     retriever = vector_store.as_retriever()
+#     prompt = ChatPromptTemplate.from_messages([
+#         MessagesPlaceholder(variable_name="chat_history"),
+#         ("human", "{input}"),
+#         ("system", """Given a chat history and the latest user question, 
+#         reformulate it as a standalone question without using chat history. 
+#         Do NOT answer it, just reformulate.""")
+#     ])
+#     return create_history_aware_retriever(llm, retriever, prompt)
+
+# def get_conversational_chain(retriever_chain):
+#     prompt = ChatPromptTemplate.from_messages([
+#         ("system", """Hello! I'm your PreCollege AI assistant. I'll guide you through your JEE Mains journey.
+#         To get started, share your JEE Mains rank and preferred engineering branches or colleges."""),
+#         MessagesPlaceholder(variable_name="chat_history"),
+#         ("human", "{input}")
+#     ])
+#     stuff_documents_chain = create_stuff_documents_chain(llm, prompt)
+#     return create_retrieval_chain(retriever_chain, stuff_documents_chain)
+
+# def get_response(user_query):
+#     retriever_chain = get_context_retriever_chain(st.session_state.vector_store)
+#     conversation_rag_chain = get_conversational_chain(retriever_chain)
+    
+#     formatted_chat_history = []
+#     for message in st.session_state.chat_history:
+#         if isinstance(message, HumanMessage):
+#             formatted_chat_history.append({"author": "user", "content": message.content})
+#         elif isinstance(message, SystemMessage):
+#             formatted_chat_history.append({"author": "assistant", "content": message.content})
+    
+#     response = conversation_rag_chain.invoke({
+#         "chat_history": formatted_chat_history,
+#         "input": user_query
+#     })
+    
+#     return response['answer']
+
+# # Initialize vector store and metrics
+# st.session_state.vector_store = load_preprocessed_vectorstore()
+# if "metrics" not in st.session_state:
+#     st.session_state.metrics = {"recall_at_5": [], "bert_scores": [], "f1_scores": []}
+
+# # Initialize chat history
+# if "chat_history" not in st.session_state:
+#     st.session_state.chat_history = [
+#         {"author": "assistant", "content": "Hello, I am Precollege. How can I help you?"}
+#     ]
+
+# # Main app logic
+# if st.session_state.get("vector_store") is None:
+#     st.error("Failed to load preprocessed data. Ensure the data exists in './data' directory.")
+# else:
+#     with st.container():
+#         for message in st.session_state.chat_history:
+#             if message["author"] == "assistant":
+#                 with st.chat_message("system"):
+#                     st.write(message["content"])
+#             elif message["author"] == "user":
+#                 with st.chat_message("human"):
+#                     st.write(message["content"])
+
+#     with st.container():
+#         with st.form(key="chat_form", clear_on_submit=True):
+#             user_query = st.text_input("Type your message here...", key="user_input")
+#             submit_button = st.form_submit_button("Send")
+
+#         if submit_button and user_query:
+#             # Get response
+#             response = get_response(user_query)
+#             st.session_state.chat_history.append({"author": "user", "content": user_query})
+#             st.session_state.chat_history.append({"author": "assistant", "content": response})
+
+#             # Dummy relevant docs for metrics demonstration
+#             retrieved_docs = ["doc1", "doc2", "doc3"]  # Replace with actual IDs from retriever
+#             relevant_docs = ["doc1", "doc4"]  # Replace with ground truth IDs
+#             recall_at_5 = calculate_recall_at_k(retrieved_docs, relevant_docs)
+#             st.session_state.metrics["recall_at_5"].append(recall_at_5)
+
+#             # Dummy reference and relevant text
+#             reference_response = "Gold-standard answer here."
+#             bert_scores = calculate_bertscore([response], [reference_response])
+#             st.session_state.metrics["bert_scores"].append(bert_scores["f1"])
+
+#             f1_score_value = calculate_f1_score(response, "Relevant text here")
+#             st.session_state.metrics["f1_scores"].append(f1_score_value)
+
+#             # Display evaluation metrics
+#             st.write("Evaluation Metrics:")
+#             st.write(f"Recall@5: {recall_at_5:.2f}")
+#             st.write(f"BERTScore F1: {bert_scores['f1']:.2f}")
+#             st.write(f"Faithfulness F1: {f1_score_value:.2f}")
+
+#             st.rerun()
+
+
+
+import os  
+import streamlit as st  
+import google.generativeai as genai  
+from langchain_google_genai import ChatGoogleGenerativeAI  
+from langchain_community.document_loaders import Docx2txtLoader   
+from langchain.text_splitter import RecursiveCharacterTextSplitter  
+from langchain_community.vectorstores import Chroma  
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder  
+from langchain_core.messages import HumanMessage, SystemMessage  
+from langchain.chains import create_history_aware_retriever, create_retrieval_chain 
+from langchain.chains.combine_documents import create_stuff_documents_chain 
+from langchain.embeddings import HuggingFaceEmbeddings
+import pysqlite3
+import sys
+sys.modules['sqlite3'] = pysqlite3
+
+# Set the Google API key
+GOOGLE_API_KEY = "AIzaSyCvkV4v4NPnPE2TcDGpIaJx56OIf_vUCnU"
+if not GOOGLE_API_KEY:
+    raise ValueError("Gemini API key not found. Please set it in the .env file.")
+
+os.environ["GOOGLE_API_KEY"] = GOOGLE_API_KEY
+
+# Streamlit app configuration
+st.set_page_config(page_title="College Data Chatbot", layout="centered")
+st.title("PreCollege Chatbot GEMINI+ HuggingFace Embeddings")
+
+# Initialize the Google Gemini LLM
+llm = ChatGoogleGenerativeAI(
+    model="gemini-1.5-pro-latest",
+    temperature=0.2, 
+    max_tokens=None,
+    timeout=None,
+    max_retries=2,
+)
+
+# Initialize embeddings using HuggingFace
+embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')
+
+def load_preprocessed_vectorstore():
+    """Loads documents, splits them, and creates a Chroma vector store."""
+    try:
+        loader = Docx2txtLoader("./Updated_structred_aman.docx")
+        documents = loader.load()
+
+        text_splitter = RecursiveCharacterTextSplitter(
+            separators=["\n\n", "\n", ". ", " ", ""],
+            chunk_size=3000, 
+            chunk_overlap=1000
+        )
+        
+        document_chunks = text_splitter.split_documents(documents)
+
+        vector_store = Chroma.from_documents(
+            embedding=embeddings,
+            documents=document_chunks,
+            persist_directory="./data32"
+        )
+        return vector_store
+    except Exception as e:
+        st.error(f"Error creating vector store: {e}")
+        return None
+
+def get_context_retriever_chain(vector_store):
+    """Creates a history-aware retriever chain."""
+    retriever = vector_store.as_retriever()
+
+    # Define the prompt for the retriever chain
+    prompt = ChatPromptTemplate.from_messages([
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("human", "{input}"),
+        ("system", """Given the chat history, context, and the latest user question, formulate a standalone question 
+that can be understood without the chat history. Use the context to provide a relevant answer if possible.
+If the question is beyond the scope of the context, return:
+'This question is beyond the scope of the available information. Please contact your mentor for further assistance.'
+""")
+    ])
+
+    retriever_chain = create_history_aware_retriever(llm, retriever, prompt)
+    return retriever_chain
+
+def get_conversational_chain(retriever_chain):
+    """Creates a conversational chain using the retriever chain."""
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", """Hello! I'm your PreCollege AI assistant, here to help you with your JEE Mains journey. 
+Please provide your JEE Mains rank and preferred engineering branches or colleges, 
+and I'll give you tailored advice based on our verified database. 
+Note: I will only provide information that is available within our database to ensure accuracy. Let's get started!
+\n\n{context}"""),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("human", "{input}")
+    ])
+
+    stuff_documents_chain = create_stuff_documents_chain(llm, prompt)
+    return create_retrieval_chain(retriever_chain, stuff_documents_chain)
+
+def get_response(user_query):
+    """Gets a response from the conversational RAG chain."""
+    retriever_chain = get_context_retriever_chain(st.session_state.vector_store)
+    conversation_rag_chain = get_conversational_chain(retriever_chain)
+    
+    formatted_chat_history = []
+    for message in st.session_state.chat_history:
+        if isinstance(message, HumanMessage):
+            formatted_chat_history.append({"author": "user", "content": message.content})
+        elif isinstance(message, SystemMessage):
+            formatted_chat_history.append({"author": "assistant", "content": message.content})
+    
+    response = conversation_rag_chain.invoke({
+        "chat_history": formatted_chat_history,
+        "input": user_query
+    })
+    
+    return response['answer']
+
+# Load the preprocessed vector store from the local directory
+if "vector_store" not in st.session_state:
+    st.session_state.vector_store = load_preprocessed_vectorstore()
+
+# Initialize chat history if not present
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = []
+
+# Main app logic
+if st.session_state.vector_store is None:
+    st.error("Failed to load preprocessed data. Please ensure the data exists in './data32' directory.")
+else:
+    # Display chat history
+    with st.container():
+        for message in st.session_state.chat_history:
+            if message.get("author") == "assistant":
+                with st.chat_message("assistant"):
+                    st.write(message.get("content"))
+            elif message.get("author") == "user":
+                with st.chat_message("user"):
+                    st.write(message.get("content"))
+
+    # Add user input box below the chat
+    if user_query := st.chat_input("Type your message here..."):
+        # Append user query to chat history
+        st.session_state.chat_history.append({"author": "user", "content": user_query})
+
+        # Get bot response
+        response = get_response(user_query)
+
+        # Append response to chat history
+        st.session_state.chat_history.append({"author": "assistant", "content": response})
+
+        # Display response
+        with st.chat_message("assistant"):
+            st.write(response)

chatbot_gemini.py

@@ -0,0 +1,168 @@
+import os  
+import streamlit as st  
+import google.generativeai as genai  
+# from langchain_openai import OpenAI /
+from langchain_openai import OpenAIEmbeddings
+from langchain_google_genai import GoogleGenerativeAIEmbeddings  
+from langchain_google_genai import ChatGoogleGenerativeAI  
+# from langchain_openai import OpenAIEmbeddings  
+from langchain_community.document_loaders import Docx2txtLoader   
+from langchain.text_splitter import RecursiveCharacterTextSplitter  
+from langchain_community.vectorstores import Chroma  
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder  
+from langchain_core.messages import HumanMessage, SystemMessage  
+from langchain.chains import create_history_aware_retriever, create_retrieval_chain 
+from langchain.chains.combine_documents import create_stuff_documents_chain 
+from dotenv import load_dotenv  
+from langchain.embeddings import HuggingFaceEmbeddings
+import pysqlite3
+import sys
+sys.modules['sqlite3'] = pysqlite3
+
+import os
+os.environ["TRANSFORMERS_OFFLINE"] = "1"
+
+# Retrieve OpenAI API key from the .env file
+GOOGLE_API_KEY = "AIzaSyC1-QUzA45IlCosX__sKlzNAgVZGEaHc0c"
+# GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+# OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+if not GOOGLE_API_KEY:
+    raise ValueError("Gemini API key not found. Please set it in the .env file.")
+
+# Set OpenAI API key
+os.environ["GOOGLE_API_KEY"] = GOOGLE_API_KEY
+# os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+# Streamlit app configuration
+st.set_page_config(page_title="College Data Chatbot", layout="centered")
+st.title("PreCollege Chatbot GEMINI+ HuggingFace Embeddings")
+
+# Initialize OpenAI LLM 
+llm = ChatGoogleGenerativeAI(
+    model="gemini-1.5-pro-latest",
+    temperature=0.2,  # Slightly higher for varied responses
+    max_tokens=None,
+    timeout=None,
+    max_retries=2,
+)
+
+# Initialize embeddings using OpenAI
+embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')
+
+def load_preprocessed_vectorstore():
+    try:
+        loader = Docx2txtLoader("./Updated_structred_aman.docx")
+        documents = loader.load()
+
+        text_splitter = RecursiveCharacterTextSplitter(
+            separators=["\n\n", "\n", ". ", " ", ""],
+            chunk_size=3000, 
+            chunk_overlap=1000)
+        
+        document_chunks = text_splitter.split_documents(documents)
+
+        vector_store = Chroma.from_documents(
+            
+            embedding=embeddings,
+            documents=document_chunks,
+            persist_directory="./data32"
+        )
+        return vector_store
+    except Exception as e:
+        st.error(f"Error creating vector store: {e}")
+        return None
+
+def get_context_retriever_chain(vector_store):
+    """Creates a history-aware retriever chain."""
+    retriever = vector_store.as_retriever()
+
+    # Define the prompt for the retriever chain
+    prompt = ChatPromptTemplate.from_messages([
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("human", "{input}"),
+        ("system", """Given the chat history and the latest user question, which might reference context in the chat history, 
+formulate a standalone question that can be understood without the chat history.
+If the question is directly addressed within the provided document, provide a relevant answer. 
+If the question is not explicitly addressed in the document, return the following message: 
+'This question is beyond the scope of the available information. Please contact your mentor for further assistance.'
+Do NOT answer the question directly, just reformulate it if needed and otherwise return it as is.""")
+    ])
+
+    retriever_chain = create_history_aware_retriever(llm, retriever, prompt)
+    return retriever_chain
+
+def get_conversational_chain(retriever_chain):
+    """Creates a conversational chain using the retriever chain."""
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", """Hello! I'm your PreCollege AI assistant, here to help you with your JEE Mains journey. 
+Please provide your JEE Mains rank and preferred engineering branches or colleges, 
+and I'll give you tailored advice based on our verified database. 
+Note: I will only provide information that is available within our database to ensure accuracy. Let's get started!
+"""
+         "\n\n"
+         "{context}"),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("human", "{input}")
+    ])
+
+    stuff_documents_chain = create_stuff_documents_chain(llm, prompt)
+    return create_retrieval_chain(retriever_chain, stuff_documents_chain)
+
+def get_response(user_query):
+    retriever_chain = get_context_retriever_chain(st.session_state.vector_store)
+    conversation_rag_chain = get_conversational_chain(retriever_chain)
+    
+    formatted_chat_history = []
+    for message in st.session_state.chat_history:
+        if isinstance(message, HumanMessage):
+            formatted_chat_history.append({"author": "user", "content": message.content})
+        elif isinstance(message, SystemMessage):
+            formatted_chat_history.append({"author": "assistant", "content": message.content})
+    
+    response = conversation_rag_chain.invoke({
+        "chat_history": formatted_chat_history,
+        "input": user_query
+    })
+    
+    return response['answer']
+
+# Load the preprocessed vector store from the local directory
+st.session_state.vector_store = load_preprocessed_vectorstore()
+
+# Initialize chat history if not present
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = [
+        {"author": "assistant", "content": "Hello, I am Precollege. How can I help you?"}
+    ]
+
+# Main app logic
+if st.session_state.get("vector_store") is None:
+    st.error("Failed to load preprocessed data. Please ensure the data exists in './data' directory.")
+else:
+    # Display chat history
+    with st.container():
+        for message in st.session_state.chat_history:
+            if message["author"] == "assistant":
+                with st.chat_message("system"):
+                    st.write(message["content"])
+            elif message["author"] == "user":
+                with st.chat_message("human"):
+                    st.write(message["content"])
+
+    # Add user input box below the chat
+    with st.container():
+        with st.form(key="chat_form", clear_on_submit=True):
+            user_query = st.text_input("Type your message here...", key="user_input")
+            submit_button = st.form_submit_button("Send")
+
+        if submit_button and user_query:
+            # Get bot response
+            response = get_response(user_query)
+            st.session_state.chat_history.append({"author": "user", "content": user_query})
+            st.session_state.chat_history.append({"author": "assistant", "content": response})
+
+            # Rerun the app to refresh the chat display
+            st.rerun()
+
+
+""""""

chatbot_openai.py

@@ -0,0 +1,502 @@
+import os
+import streamlit as st
+from langchain_openai import OpenAI
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.document_loaders import Docx2txtLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Chroma
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder, PromptTemplate
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain.chains import create_history_aware_retriever, create_retrieval_chain
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain_core.output_parsers import StrOutputParser
+from dotenv import load_dotenv
+
+# Retrieve OpenAI API key from the .env file
+load_dotenv()
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+if not OPENAI_API_KEY:
+    raise ValueError("OpenAI API key not found. Please set it in the .env file.")
+
+# Set OpenAI API key
+os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+
+# Streamlit app configuration
+st.set_page_config(page_title="College Data Chatbot", layout="centered")
+st.title("PreCollege Chatbot")
+
+# Initialize OpenAI LLM
+llm = OpenAI(
+    model="gpt-3.5-turbo-instruct",
+    temperature=0,
+)
+
+# Initialize embeddings using OpenAI
+embeddings = OpenAIEmbeddings(model="text-embedding-ada-002")
+
+def load_preprocessed_vectorstore():
+    try:
+        loader = Docx2txtLoader("./Updated_structred_aman.docx")
+        documents = loader.load()
+
+        text_splitter = RecursiveCharacterTextSplitter(
+            separators=["\n\n", "\n", ". ", " ", ""],
+            chunk_size=3000, 
+            chunk_overlap=200)
+        
+        document_chunks = text_splitter.split_documents(documents)
+
+        vector_store = Chroma.from_documents(
+            embedding=embeddings,
+            documents=document_chunks,
+            persist_directory="./data11"
+        )
+        return vector_store
+    except Exception as e:
+        st.error(f"Error creating vector store: {e}")
+        return None
+
+import logging
+
+# Function to create the retriever and prompt chain
+def get_context_retriever_chain(vector_store):
+    """Creates a context-aware retriever and prompt chain."""
+    retriever = vector_store.as_retriever(k=3)  # Hybrid retrieval for better results
+
+    rag_prompt = PromptTemplate(
+        template="""
+        Act as a PreCollege AI assistant dedicated to guiding students through their JEE Mains journey. Your goal is to provide personalized, accurate, and interactive advice for students seeking college admissions guidance. Tailor your responses to address students' individual needs, including:
+
+        1. College Selection and Counseling: Help students identify colleges they qualify for based on their JEE Mains rank and preferences, including IIITs institutions. Consider factors like location, course offerings, placement records, and fees.
+
+        2. Admission Process Guidance: Clarify the college admission procedures, including JoSAA counseling, spot rounds, document verification, and category-specific quotas (if applicable).
+
+        3. Career and Branch Selection Advice: Assist students in making informed decisions about their preferred engineering branches based on interest, market trends, and scope of opportunities.
+
+        Interactive Sessions: Engage students in Q&A sessions to answer their doubts related to preparation, counseling, and career choices.
+
+        Maintain a professional and friendly tone. Use your expertise to ensure students receive relevant and clear information. Provide examples, stats, and other insights to support your advice wherever needed.
+
+        QUESTION: {question} 
+        CONTEXT: {context} 
+        Answer in a detailed yet concise manner, also highlight relevant information and do not give unnecessary information or negative responses:
+        """,
+        input_variables=["question", "context"],
+    )
+
+    rag_prompt_chain = rag_prompt | llm | StrOutputParser()
+
+    return retriever, rag_prompt_chain
+
+
+def get_response(user_query):
+    """Processes the user query and generates a response."""
+    # Define a set of common greetings
+    greetings = ["hi", "hello", "hey", "greetings", "hi there"]
+
+    # Check if the user query is a greeting
+    if user_query.lower().strip() in greetings:
+        return "Hello! How can I assist you with your college search today?"
+
+    # Ensure the vector store is initialized
+    if "vector_store" not in st.session_state:
+        logging.error("Vector store is not initialized in session state.")
+        return "Vector store is not initialized. Please preprocess the document first."
+
+    retriever, rag_prompt_chain = get_context_retriever_chain(st.session_state.vector_store)
+
+    # Format chat history from session state
+    formatted_chat_history = []
+    for message in st.session_state.chat_history:
+        if message["author"] == "user":
+            formatted_chat_history.append({"author": "user", "content": message["content"]})
+        elif message["author"] == "assistant":
+            formatted_chat_history.append({"author": "assistant", "content": message["content"]})
+
+    try:
+        # Retrieve context
+        context = retriever.invoke(user_query)
+        logging.info(f"Retrieved context: {context}")
+
+        if not context:
+            logging.error("No relevant context retrieved.")
+            return "I couldn't retrieve relevant information. Please try a different query."
+
+        # Generate response
+        response = rag_prompt_chain.invoke({
+            "chat_history": formatted_chat_history,
+            "question": user_query,
+            "context": context
+        })
+        logging.info(f"Generated response: {response}")
+
+        # Check the response format
+        if isinstance(response, dict) and "answer" in response:
+            return response["answer"]
+        elif isinstance(response, str):  # Handle raw string outputs
+            return response
+        else:
+            logging.error(f"Unexpected response format: {response}")
+            return "Unexpected error occurred. Please try again later."
+    except Exception as e:
+        logging.error(f"Error generating response: {e}")
+        return "Sorry, I encountered an issue while processing your request. Please try again later."
+
+
+
+# Load the preprocessed vector store from the local directory
+st.session_state.vector_store = load_preprocessed_vectorstore()
+
+# Initialize chat history if not present
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = [
+        {"author": "assistant", "content": "Hello, I am Precollege. How can I help you?"}
+    ]
+
+# Main app logic
+if st.session_state.get("vector_store") is None:
+    st.error("Failed to load preprocessed data. Please ensure the data exists in './data' directory.")
+else:
+    # Display chat history
+    with st.container():
+        for message in st.session_state.chat_history:
+            if message["author"] == "assistant":
+                with st.chat_message("system"):
+                    st.write(message["content"])
+            elif message["author"] == "user":
+                with st.chat_message("human"):
+                    st.write(message["content"])
+
+    # Add user input box below the chat
+    with st.container():
+        with st.form(key="chat_form", clear_on_submit=True):
+            user_query = st.text_input("Type your message here...", key="user_input")
+            submit_button = st.form_submit_button("Send")
+
+        if submit_button and user_query:
+            # Get bot response
+            response = get_response(user_query)
+            st.session_state.chat_history.append({"author": "user", "content": user_query})
+            st.session_state.chat_history.append({"author": "assistant", "content": response})
+
+            # Rerun the app to refresh the chat display
+            st.rerun()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# import os
+# import streamlit as st
+# from langchain_openai import OpenAI
+# from langchain_openai import OpenAIEmbeddings
+# from langchain_community.document_loaders import Docx2txtLoader 
+# from langchain.text_splitter import RecursiveCharacterTextSplitter
+# from langchain_community.vectorstores import Chroma
+# from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+# from langchain_core.messages import HumanMessage, SystemMessage
+# from langchain.chains import create_history_aware_retriever, create_retrieval_chain
+# from langchain.chains.combine_documents import create_stuff_documents_chain
+# from dotenv import load_dotenv
+
+
+# # Retrieve OpenAI API key from the .env file
+# OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+# if not OPENAI_API_KEY:
+#     raise ValueError("OpenAI API key not found. Please set it in the .env file.")
+
+# # Set OpenAI API key
+# os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+# # Streamlit app configuration
+# st.set_page_config(page_title="College Data Chatbot", layout="centered")
+# st.title("PreCollege Chatbot")
+
+# # Initialize OpenAI LLM
+# llm = OpenAI(
+#     model="gpt-3.5-turbo-instruct",
+#     temperature=0,
+# )
+
+# # Initialize embeddings using OpenAI
+# embeddings = OpenAIEmbeddings(model="text-embedding-ada-002")
+
+# def load_preprocessed_vectorstore():
+#     try:
+#         loader = Docx2txtLoader("./Updated_structred_aman.docx")
+#         documents = loader.load()
+
+#         text_splitter = RecursiveCharacterTextSplitter(
+#             separators=["\n\n", "\n", ". ", " ", ""],
+#             chunk_size=3000, 
+#             chunk_overlap=200)
+        
+#         document_chunks = text_splitter.split_documents(documents)
+
+#         vector_store = Chroma.from_documents(
+            
+#             embedding=embeddings,
+#             documents=document_chunks,
+#             persist_directory="./data11"
+#         )
+#         return vector_store
+#     except Exception as e:
+#         st.error(f"Error creating vector store: {e}")
+#         return None
+
+# def get_context_retriever_chain(vector_store):
+#     """Creates a history-aware retriever chain."""
+#     retriever = vector_store.as_retriever()
+
+#     # Define the prompt for the retriever chain
+#     prompt = ChatPromptTemplate.from_messages([
+#         MessagesPlaceholder(variable_name="chat_history"),
+#         ("user", "{input}"),
+#         ("system", "You are a PreCollege AI assistant helping students with JEE Mains college guidance. Answer interactively and provide relevant, accurate information.")
+#     ])
+
+#     retriever_chain = create_history_aware_retriever(llm, retriever, prompt)
+#     return retriever_chain
+
+# def get_conversational_chain(retriever_chain):
+#     """Creates a conversational chain using the retriever chain."""
+#     prompt = ChatPromptTemplate.from_messages([
+#         ("system", "Answer the user's questions based on the context below:\n\n{context}"),
+#         MessagesPlaceholder(variable_name="chat_history"),
+#         ("user", "{input}")
+#     ])
+
+#     stuff_documents_chain = create_stuff_documents_chain(llm, prompt)
+#     return create_retrieval_chain(retriever_chain, stuff_documents_chain)
+
+# def get_response(user_query):
+#     retriever_chain = get_context_retriever_chain(st.session_state.vector_store)
+#     conversation_rag_chain = get_conversational_chain(retriever_chain)
+    
+#     formatted_chat_history = []
+#     for message in st.session_state.chat_history:
+#         if isinstance(message, HumanMessage):
+#             formatted_chat_history.append({"author": "user", "content": message.content})
+#         elif isinstance(message, SystemMessage):
+#             formatted_chat_history.append({"author": "assistant", "content": message.content})
+    
+#     response = conversation_rag_chain.invoke({
+#         "chat_history": formatted_chat_history,
+#         "input": user_query
+#     })
+    
+#     return response['answer']
+
+# # Load the preprocessed vector store from the local directory
+# st.session_state.vector_store = load_preprocessed_vectorstore()
+
+# # Initialize chat history if not present
+# if "chat_history" not in st.session_state:
+#     st.session_state.chat_history = [
+#         {"author": "assistant", "content": "Hello, I am Precollege. How can I help you?"}
+#     ]
+
+# # Main app logic
+# if st.session_state.get("vector_store") is None:
+#     st.error("Failed to load preprocessed data. Please ensure the data exists in './data' directory.")
+# else:
+#     # Display chat history
+#     with st.container():
+#         for message in st.session_state.chat_history:
+#             if message["author"] == "assistant":
+#                 with st.chat_message("system"):
+#                     st.write(message["content"])
+#             elif message["author"] == "user":
+#                 with st.chat_message("human"):
+#                     st.write(message["content"])
+
+#     # Add user input box below the chat
+#     with st.container():
+#         with st.form(key="chat_form", clear_on_submit=True):
+#             user_query = st.text_input("Type your message here...", key="user_input")
+#             submit_button = st.form_submit_button("Send")
+
+#         if submit_button and user_query:
+#             # Get bot response
+#             response = get_response(user_query)
+#             st.session_state.chat_history.append({"author": "user", "content": user_query})
+#             st.session_state.chat_history.append({"author": "assistant", "content": response})
+
+#             # Rerun the app to refresh the chat display
+#             st.rerun()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# import os 
+# import tempfile 
+# import streamlit as st 
+# from langchain_openai import OpenAI 
+# from langchain_openai import OpenAIEmbeddings 
+# from langchain_community.vectorstores import Chroma 
+# from langchain_community.document_loaders import Docx2txtLoader 
+# from langchain.text_splitter import RecursiveCharacterTextSplitter 
+# from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder 
+# from langchain_core.messages import HumanMessage, SystemMessage 
+# from langchain.chains import create_history_aware_retriever, create_retrieval_chain  
+# from langchain.chains.combine_documents import create_stuff_documents_chain 
+
+# # Load environment variables for API keys 
+# # load_dotenv() 
+# import os 
+# os.environ["OPENAI_API_KEY"]="sk-HQoHO1UganCjwF-tK2Hs-0wmwUHmVdiZIVwa_2SYBuT3BlbkFJSiebrtoqIo83LPDi-LaPHeLqndbP3I9tguwSnw3AMA" 
+
+# # Initialize OpenAI LLM 
+# llm = OpenAI( 
+#     model="gpt-3.5-turbo-instruct",
+#     temperature=0, 
+# )
+
+# # Initialize embeddings using OpenAI 
+# embeddings = OpenAIEmbeddings(model="text-embedding-ada-002")
+
+# def get_vectorstore_from_docx(docx_file):
+#     """Processes a .docx file to create a vector store.""" 
+#     try:
+#         with tempfile.NamedTemporaryFile(delete=False, suffix='.docx') as temp_file: 
+#             temp_file.write(docx_file.read())
+#             temp_file_path = temp_file.name
+
+#         loader = Docx2txtLoader(temp_file_path)
+#         documents = loader.load()
+
+#         text_splitter = RecursiveCharacterTextSplitter(chunk_size=3000, chunk_overlap=200)
+#         document_chunks = text_splitter.split_documents(documents)
+
+#         vector_store = Chroma.from_documents(
+#             embedding=embeddings,
+#             documents=document_chunks,
+#             persist_directory="./data1"
+#         )
+#         os.remove(temp_file_path)
+#         return vector_store
+#     except Exception as e:
+#         st.error(f"Error creating vector store: {e}")
+#         return None
+
+# def get_context_retriever_chain(vector_store):
+#     """Creates a history-aware retriever chain."""
+#     retriever = vector_store.as_retriever()
+    
+#     prompt = ChatPromptTemplate.from_messages([
+#         MessagesPlaceholder(variable_name="chat_history"),
+#         ("user", "{input}"),
+#         ("system", "You are a PreCollege AI assistant helping students with JEE Mains college guidance. Answer interactively and provide relevant, accurate information.")
+#     ])
+    
+#     retriever_chain = create_history_aware_retriever(llm, retriever, prompt)
+#     return retriever_chain
+
+# def get_conversational_chain(retriever_chain):
+#     """Creates a conversational chain using the retriever chain."""
+#     prompt = ChatPromptTemplate.from_messages([
+#         ("system", "Answer the user's questions based on the context below:\n\n{context}"),
+#         MessagesPlaceholder(variable_name="chat_history"),
+#         ("user", "{input}")
+#     ])
+    
+#     stuff_documents_chain = create_stuff_documents_chain(llm, prompt)
+#     return create_retrieval_chain(retriever_chain, stuff_documents_chain)
+
+# def get_response(user_query):
+#     retriever_chain = get_context_retriever_chain(st.session_state.vector_store)
+#     conversation_rag_chain = get_conversational_chain(retriever_chain)
+    
+#     formatted_chat_history = []
+#     for message in st.session_state.chat_history:
+#         if isinstance(message, HumanMessage):
+#             formatted_chat_history.append({"author": "user", "content": message.content})
+#         elif isinstance(message, SystemMessage):
+#             formatted_chat_history.append({"author": "assistant", "content": message.content})
+    
+#     response = conversation_rag_chain.invoke({
+#         "chat_history": formatted_chat_history,
+#         "input": user_query
+#     })
+    
+#     return response['answer']
+
+# # Streamlit app configuration
+# st.set_page_config(page_title="College Data Chatbot")
+# st.title("College Data Chatbot")
+
+# # Sidebar for document upload and automatic processing
+# with st.sidebar:
+#     st.header("Upload College Data Document")
+#     docx_file = st.file_uploader("Upload a .docx file")
+
+#     if docx_file:
+#         # Automatically process the uploaded file
+#         st.session_state.vector_store = get_vectorstore_from_docx(docx_file)
+#         if st.session_state.vector_store:
+#             st.session_state.docx_name = docx_file.name
+#             st.success("Document processed successfully!")
+
+# # Initialize chat history if not present
+# if "chat_history" not in st.session_state:
+#     st.session_state.chat_history = [
+#         {"author": "assistant", "content": "Hello, I am precollege. How can I help you?"}
+#     ]
+
+# # Main chat section
+# if st.session_state.get("vector_store") is None:
+#     st.info("Please upload and process a .docx file to get started.")
+# else:
+#     # Display the chat history first
+#     with st.container():
+#         for message in st.session_state.chat_history:
+#             if message["author"] == "assistant":
+#                 with st.chat_message("system"):
+#                     st.write(message["content"])
+#             elif message["author"] == "user":
+#                 with st.chat_message("human"):
+#                     st.write(message["content"])
+
+#     # User input at the bottom of the chat
+#     with st.container():
+#         with st.form(key="chat_form", clear_on_submit=True):
+#             user_query = st.text_input("Type your message here...", key="user_input")
+#             submit_button = st.form_submit_button("Send")
+
+#         if submit_button and user_query:
+#             # Process the user query and get the bot's response
+#             response = get_response(user_query)
+#             st.session_state.chat_history.append({"author": "user", "content": user_query})
+#             st.session_state.chat_history.append({"author": "assistant", "content": response})
+
+#             # Scroll to the bottom of the chat
+#             # st.experimental_rerun()

requirements.txt

@@ -0,0 +1,91 @@
+altair==5.4.1
+langchain-community==0.3.7
+annotated-types==0.7.0
+anyio==4.6.2.post1
+attrs==24.2.0
+blinker==1.9.0
+cachetools==5.5.0
+bert-score
+certifi==2024.8.30
+charset-normalizer==3.4.0
+db-sqlite3
+pysqlite3-binary
+pydantic
+langchain-core
+click==8.1.7
+colorama==0.4.6
+distro==1.9.0
+gitdb==4.0.11
+GitPython==3.1.43
+google-ai-generativelanguage==0.6.10
+google-api-core==2.23.0
+google-api-python-client==2.154.0
+google-auth==2.36.0
+google-auth-httplib2==0.2.0
+google-generativeai==0.8.3
+googleapis-common-protos==1.66.0
+grpcio==1.68.0
+grpcio-status==1.68.0
+h11==0.14.0
+huggingface-hub
+httpcore==1.0.7
+httplib2==0.22.0
+httpx==0.27.2
+idna==3.10
+Jinja2==3.1.4
+jiter==0.7.1
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+langchain-core==0.3.19
+langchain-google-genai==2.0.5
+langchain-openai==0.2.9
+langsmith==0.1.144
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+narwhals==1.14.1
+numpy==1.26.4
+openai==1.55.0
+orjson==3.10.11
+packaging==24.2
+pandas==2.2.3
+pillow==11.0.0
+proto-plus==1.25.0
+protobuf==5.28.3
+pyarrow==18.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pydantic==2.10.1
+pydantic_core==2.27.1
+pydeck==0.9.1
+Pygments==2.18.0
+pyparsing==3.2.0
+python-dateutil==2.9.0.post0
+pytz==2024.2
+PyYAML==6.0.2
+referencing==0.35.1
+regex==2024.11.6
+requests==2.32.3
+requests-toolbelt==1.0.0
+rich==13.9.4
+rpds-py==0.21.0
+rsa==4.9
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+streamlit==1.40.1
+tenacity==9.0.0
+tiktoken==0.8.0
+toml==0.10.2
+tornado==6.4.2
+tqdm==4.67.0
+typing_extensions==4.12.2
+tzdata==2024.2
+uritemplate==4.1.1
+urllib3==2.2.3
+watchdog==6.0.0
+docx2txt
+sentence-transformers==3.2.1
+chromadb