DEBUG: base64 -> plain text

app.py

@@ -329,7 +329,7 @@ async def loadText(addTextConfig: AddText):
  "output": text,
  "source": "Text"
  }
- numTokens = len(" ".join([base64.b64decode(text[x].encode("utf-8")).decode("utf-8") for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
+ numTokens = len(" ".join([text[x] for x in text]).translate(str.maketrans('', '', string.punctuation)).split(" "))
  dct = json.dumps(dct, indent=1).encode("utf-8")
  fileName = createDataSourceName(sourceName="Text")
  response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")

functions.py

@@ -297,7 +297,7 @@ def getTextFromImagePDF(pdfBytes):
  return cleanText(text = text)
 
  allImages = convert_from_bytes(pdfBytes)
- texts = [base64.b64encode(getText(image).encode("utf-8")).decode("utf-8") for image in allImages]
+ texts = [getText(image) for image in allImages]
  return {x + 1: y for x, y in enumerate(texts)}
 
 
@@ -313,7 +313,6 @@ def getTranscript(urls: str):
  except:
  doc = ""
  texts.append(doc)
- texts = [base64.b64encode(text.encode("utf-8")).decode("utf-8") for text in texts]
  return {x: y for x, y in zip(urls, texts)}
 
 
@@ -331,8 +330,7 @@ def analyzeData(query, dataframe):
 
 
 def extractTextFromPage(page):
- text = cleanText(text = page.get_text())
- return base64.b64encode(text.encode("utf-8")).decode("utf-8")
+ return cleanText(text = page.get_text())
 
 
 def extractTextFromPdf(pdf_path):
@@ -349,8 +347,7 @@ def extractTextFromUrl(url):
  response.raise_for_status()
  html = response.text
  soup = BeautifulSoup(html, 'lxml')
- text = cleanText(text = soup.get_text(separator=' ', strip=True))
- return base64.b64encode(text.encode("utf-8")).decode("utf-8")
+ return cleanText(text = soup.get_text(separator=' ', strip=True))
 
 
 def extractTextFromUrlList(urls):