Update app.py

app.py

@@ -108,7 +108,6 @@ class MistralRAGChatbot:
     def __init__(self, vector_db_path: str, annoy_index_path: str):
         self.embeddings, self.texts = self.load_vector_db(vector_db_path)
         self.annoy_index = self.load_annoy_index(annoy_index_path, self.embeddings.shape[1])
-        # self.tfidf_matrix, self.tfidf_vectorizer = self.calculate_tfidf(self.texts)
         self.bm25 = BM25Okapi([text.split() for text in self.texts])
         self.word2vec_model = self.train_word2vec(self.texts)
         self.reranking_methods = {
@@ -131,11 +130,6 @@ class MistralRAGChatbot:
         logging.info(f"Loaded Annoy index from {annoy_index_path}.")
         return annoy_index
 
-    # def calculate_tfidf(self, texts: List[str]) -> Tuple[np.ndarray, TfidfVectorizer]:
-    #     vectorizer = TfidfVectorizer(stop_words='english')
-    #     tfidf_matrix = vectorizer.fit_transform(texts)
-    #     logging.info("TF-IDF matrix calculated.")
-    #     return tfidf_matrix, vectorizer
 
     def train_word2vec(self, texts: List[str]) -> Word2Vec:
         tokenized_texts = [text.split() for text in texts]
@@ -247,12 +241,6 @@ class MistralRAGChatbot:
         logging.debug(f"Annoy retrieval returned {len(indices)} documents.")
         return indices, scores
 
-    # def retrieve_with_tfidf(self, user_query: str, query_embedding: np.ndarray, top_k: int) -> Tuple[List[int], List[float]]:
-    #     query_vec = self.tfidf_vectorizer.transform([user_query])
-    #     similarities = cosine_similarity(query_vec, self.tfidf_matrix).flatten()
-    #     indices = np.argsort(-similarities)[:top_k]
-    #     logging.debug(f"TF-IDF retrieval returned {len(indices)} documents.")
-    #     return indices, similarities[indices].tolist()
 
     def retrieve_with_bm25(self, user_query: str, query_embedding: np.ndarray, top_k: int) -> Tuple[List[int], List[float]]:
         tokenized_query = user_query.split()
@@ -275,24 +263,6 @@ class MistralRAGChatbot:
         indices = np.argsort(-similarities)[:top_k]
         return indices, similarities[indices].tolist()
 
-    # def retrieve_with_euclidean(self, user_query: str, query_embedding: np.ndarray, top_k: int) -> Tuple[List[int], List[float]]:
-    #     distances = euclidean_distances([query_embedding], self.embeddings).flatten()
-    #     indices = np.argsort(distances)[:top_k]
-    #     logging.debug(f"Euclidean retrieval returned {len(indices)} documents.")
-    #     return indices, distances[indices].tolist()
-
-    # def retrieve_with_jaccard(self, user_query: str, query_embedding: np.ndarray, top_k: int) -> Tuple[List[int], List[float]]:
-    #     query_set = set(user_query.lower().split())
-    #     scores = []
-    #     for doc in self.texts:
-    #         doc_set = set(doc.lower().split())
-    #         intersection = query_set.intersection(doc_set)
-    #         union = query_set.union(doc_set)
-    #         score = float(len(intersection)) / len(union) if union else 0
-    #         scores.append(score)
-    #     indices = np.argsort(-np.array(scores))[:top_k]
-    #     logging.debug(f"Jaccard retrieval returned {len(indices)} documents.")
-    #     return indices.tolist(), [scores[i] for i in indices]
 
     def rerank_documents(
         self,