DEBUG: WebURLs

app.py

@@ -294,7 +294,7 @@ async def loadImagePDF(vectorstore: str, pdf: UploadFile = File(...)):
  "output": text,
  "source": source
  }
- dct = json.dumps(dct, indent=1).encode("utf-8")
+ dct = json.dumps(dct, indent=1).encode("utf-8", errors = "replace")
  fileName = createDataSourceName(sourceName=source)
  numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
  response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
@@ -329,7 +329,7 @@ async def loadText(addTextConfig: AddText):
  "source": "Text"
  }
  numTokens = len(text.translate(str.maketrans('', '', string.punctuation)).split(" "))
- dct = json.dumps(dct, indent=1).encode("utf-8")
+ dct = json.dumps(dct, indent=1).encode("utf-8", errors = "replace")
  fileName = createDataSourceName(sourceName="Text")
  response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
  response = (
@@ -389,7 +389,7 @@ async def loadWebURLs(loadWebsite: LoadWebsite):
  "source": source
  }
  numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
- dct = json.dumps(dct, indent=1).encode("utf-8")
+ dct = json.dumps(dct, indent=1).encode("utf-8", errors = "replace")
  fileName = createDataSourceName(sourceName=source)
  response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
  response = (
@@ -472,7 +472,7 @@ async def loadYoutubeTranscript(ytTranscript: YtTranscript):
  "source": "www.youtube.com"
  }
  numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
- dct = json.dumps(dct, indent=1).encode("utf-8")
+ dct = json.dumps(dct, indent=1).encode("utf-8", errors = "replace")
  fileName = createDataSourceName(sourceName="youtube")
  response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
  response = (
@@ -534,7 +534,7 @@ async def listChatbotSources(vectorstore: str):
 async def getDataSource(vectorstore: str, sourceUrl: str):
  trackUsage(vectorstore=vectorstore, endpoint="/getDataSource")
  r = requests.get(sourceUrl)
- return encodeToBase64(eval(r.content.decode("utf-8")))
+ return encodeToBase64(eval(r.content.decode("utf-8", errors = "replace")))
 
 
 @app.post("/deleteChatbotSource")
@@ -624,7 +624,7 @@ async def trainChatbot(trainChatbotConfig: TrainChatbot):
  for source, fileType in zip(UrlSources, fileTypes):
  if ((fileType == "/loadPDF") | (fileType == "/loadImagePDF")):
  r = requests.get(source)
- file = eval(r.content.decode("utf-8"))
+ file = eval(r.content.decode("utf-8", errors = "replace"))
  content = file["output"]
  fileSource = file["source"]
  texts.append(".".join(
@@ -633,14 +633,14 @@ async def trainChatbot(trainChatbotConfig: TrainChatbot):
  sources.append(fileSource)
  elif fileType == "/loadText":
  r = requests.get(source)
- file = eval(r.content.decode("utf-8"))
+ file = eval(r.content.decode("utf-8", errors = "replace"))
  content = file["output"]
  fileSource = file["source"]
  texts.append(content.replace("\n", " "))
  sources.append(fileSource)
  elif ((fileType == "/loadWebURLs") | (fileType == "/loadYoutubeTranscript")):
  r = requests.get(source)
- file = eval(r.content.decode("utf-8"))
+ file = eval(r.content.decode("utf-8", errors = "replace"))
  content = file["output"]
  fileSource = file["source"]
  texts.append(".".join(

functions.py

@@ -348,7 +348,7 @@ def analyzeData(query, dataframe):
  response = df.chat(query)
  if os.path.isfile(response):
  with open(response, "rb") as file:
- b64string = base64.b64encode(file.read()).decode("utf-8")
+ b64string = base64.b64encode(file.read()).decode("utf-8", errors = "replace")
  return f"data:image/png;base64,{b64string}"
  else:
  return response
@@ -384,7 +384,7 @@ def extractTextFromUrlList(urls):
 def encodeToBase64(dct: dict):
  for key in dct:
  if type(dct[key]) == str:
- dct[key] = base64.b64encode(dct[key].encode("utf-8")).decode("utf-8", errors = "replace")
+ dct[key] = base64.b64encode(dct[key].encode("utf-8", errors = "replace")).decode("utf-8", errors = "replace")
  elif type(dct[key]) == dict:
  dct[key] = encodeToBase64(dct[key])
  return dct
@@ -392,10 +392,10 @@ def encodeToBase64(dct: dict):
 
 def decodeBase64(dct: dict):
  if type(dct["output"]) == str:
- dct["output"] = base64.b64decode(dct["output"].encode("utf-8")).decode("utf-8", errors = "replace")
+ dct["output"] = base64.b64decode(dct["output"].encode("utf-8", errors = "replace")).decode("utf-8", errors = "replace")
  else:
  for key in dct["output"]:
- dct["output"][key] = base64.b64decode(dct["output"][key].encode("utf-8")).decode("utf-8", errors = "replace")
+ dct["output"][key] = base64.b64decode(dct["output"][key].encode("utf-8", errors = "replace")).decode("utf-8", errors = "replace")
  return dct