UPDATE: trainChatbot

app.py

@@ -271,7 +271,7 @@ async def loadPDF(vectorstore: str, pdf: UploadFile = File(...)):
  .insert({"username": username, 
  "chatbotName": chatbotName, 
  "dataSourceName": fileName, 
- "sourceEndpoint": "\loadPDF", 
+ "sourceEndpoint": "/loadPDF", 
  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
  .execute()
  )
@@ -299,7 +299,7 @@ async def loadImagePDF(vectorstore: str, pdf: UploadFile = File(...)):
  .insert({"username": username, 
  "chatbotName": chatbotName, 
  "dataSourceName": fileName, 
- "sourceEndpoint": "\loadImagePDF", 
+ "sourceEndpoint": "/loadImagePDF", 
  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
  .execute()
  )
@@ -330,7 +330,7 @@ async def loadText(addTextConfig: AddText):
  .insert({"username": username, 
  "chatbotName": chatbotName, 
  "dataSourceName": fileName, 
- "sourceEndpoint": "\loadText", 
+ "sourceEndpoint": "/loadText", 
  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
  .execute()
  )
@@ -339,28 +339,6 @@ async def loadText(addTextConfig: AddText):
  }
 
 
-
-@app.post("/addText")
-async def addText(addTextConfig: AddText):
- vectorstore, text, source = addTextConfig.vectorstore, addTextConfig.text, addTextConfig.source
- text = base64.b64decode(text.encode("utf-8")).decode("utf-8")
- username, chatbotname = vectorstore.split("$")[1], vectorstore.split("$")[2]
- df = pd.DataFrame(supabase.table("ConversAI_ChatbotInfo").select("*").execute().data)
- currentCount = df[(df["user_id"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
- newCount = currentCount + len(text)
- limit = supabase.table("ConversAI_UserConfig").select("tokenLimit").eq("user_id", username).execute().data[0][
- "tokenLimit"]
- if newCount < int(limit):
- supabase.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("user_id", username).eq(
- "chatbotname", chatbotname).execute()
- output = addDocuments(text=text, source=source, vectorstore=vectorstore)
- return output
- else:
- return {
- "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
- }
-
-
 class AddQAPair(BaseModel):
  vectorstore: str
  question: str
@@ -410,7 +388,7 @@ async def loadWebURLs(loadWebsite: LoadWebsite):
  .insert({"username": username, 
  "chatbotName": chatbotName, 
  "dataSourceName": fileName, 
- "sourceEndpoint": "\loadWebURLs", 
+ "sourceEndpoint": "/loadWebURLs", 
  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
  .execute()
  )
@@ -467,8 +445,8 @@ class YtTranscript(BaseModel):
  urls: list[str]
 
 
-@app.post("/getYoutubeTranscript")
-async def getYoutubeTranscript(ytTranscript: YtTranscript):
+@app.post("/loadYoutubeTranscript")
+async def loadYoutubeTranscript(ytTranscript: YtTranscript):
  vectorstore, urls = ytTranscript.vectorstore, ytTranscript.urls
  username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
  text = getTranscript(urls=urls)
@@ -484,7 +462,7 @@ async def getYoutubeTranscript(ytTranscript: YtTranscript):
  .insert({"username": username, 
  "chatbotName": chatbotName, 
  "dataSourceName": fileName, 
- "sourceEndpoint": "\getYoutubeTranscript", 
+ "sourceEndpoint": "/getYoutubeTranscript", 
  "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
  .execute()
  )
@@ -523,7 +501,42 @@ async def chatHistory(vectorstore: str):
  return response
 
 
-# @app.post("/trainChatbot")
-# async def chatHistory(vectorstore: str):
-# username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
-# return response
+@app.post("/listChatbotSources")
+async def listChatbotSources(vectorstore: str):
+ username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
+ result = supabase.table("ConversAI_ChatbotDataSources").select("*").eq("username", username).eq("chatbotName", chatbotName).execute().data
+ return result
+
+
+
+@app.post("/trainChatbot")
+async def trainChatbot(vectorstore: str):
+ texts = []
+ sources = []
+ fileTypes = [supabase.table("ConversAI_ChatbotDataSources").select("sourceEndpoint").eq("sourceContentURL", x).execute().data[0]["sourceEndpoint"] for x in sources]
+ for source, fileType in zip(sources, fileTypes):
+ if ((fileType == "/loadPDF") | (fileType == "/loadImagePDF")):
+ r = requests.get(source)
+ file = eval(r.content.decode("utf-8"))
+ content = file["output"]
+ fileSource = file["source"]
+ texts.append(".".join([base64.b64decode(content[key].encode("utf-8")).decode("utf-8") for key in content.keys()]).replace("\n", " "))
+ sources.append(fileSource)
+ elif fileType == "/loadText":
+ r = requests.get(source)
+ file = eval(r.content.decode("utf-8"))
+ content = file["output"]
+ fileSource = file["source"]
+ texts.append(content.replace("\n", " "))
+ sources.append(fileSource)
+ elif ((fileType == "/loadWebURLs") | (fileType == "/loadYoutubeTranscript")):
+ r = requests.get(source)
+ file = eval(r.content.decode("utf-8"))
+ content = file["output"]
+ fileSource = file["source"]
+ texts.append(".".join([base64.b64decode(content[key].encode("utf-8")).decode("utf-8") for key in content.keys()]).replace("\n", " "))
+ sources.append(fileSource)
+ else:
+ pass
+ texts = [(text, source) for text, source in zip(texts, sources)]
+ return addDocuments(texts = texts, vectorstore = vectorstore) 

functions.py

@@ -113,6 +113,7 @@ def createTable(tablename: str):
  prefer_grpc=True,
  api_key=os.environ["QDRANT_API_KEY"],
  collection_name=tablename,
+ force_recreate=True,
  retrieval_mode=RetrievalMode.HYBRID
  )
  return {
@@ -120,7 +121,7 @@ def createTable(tablename: str):
  }
 
 
-def addDocuments(text: str, source: str, vectorstore: str):
+def addDocuments(texts: list[tuple[str]], vectorstore: str):
  global vectorEmbeddings
  global sparseEmbeddings
  splitter = RecursiveCharacterTextSplitter(
@@ -128,20 +129,22 @@ def addDocuments(text: str, source: str, vectorstore: str):
  chunk_overlap=250,
  add_start_index=True
  )
- text = text.replace("\n", " ")
- text = text.translate(str.maketrans('', '', string.punctuation.replace(".", "")))
- texts = [Document(page_content=text, metadata={"source": source})]
- texts = splitter.split_documents(texts)
- ids = [str(uuid4()) for _ in range(len(texts))]
- vectorstore = QdrantVectorStore.from_existing_collection(
+ sources = [textTuple[1] for textTuple in texts]
+ texts = [textTuple[0].replace("\n", " ") for textTuple in texts]
+ texts = [text.translate(str.maketrans('', '', string.punctuation.replace(".", ""))) for text in texts]
+ texts = [Document(page_content=text, metadata={"source": source}) for text, source in zip(texts, sources)]
+ documents = splitter.split_documents(texts)
+ vectorstore = QdrantVectorStore.from_documents(
+ documents=documents,
  embedding=vectorEmbeddings,
  sparse_embedding=sparseEmbeddings,
- collection_name=vectorstore,
  url=os.environ["QDRANT_URL"],
+ prefer_grpc=True,
  api_key=os.environ["QDRANT_API_KEY"],
+ collection_name=vectorstore,
+ force_recreate=True,
  retrieval_mode=RetrievalMode.HYBRID
  )
- vectorstore.add_documents(documents=texts, ids=ids)
  return {
  "output": "SUCCESS"
  }