Update functions.py

functions.py

@@ -106,10 +106,16 @@ def preprocess_plain_text(text,window_size=3):
  return passages
  
 @st.experimental_memo(suppress_st_warning=True) 
-def chunk_clean_text(text):
+def chunk_and_process_text(text):
  
  """Chunk text longer than 500 tokens"""
  
+ text = text.encode("ascii", "ignore").decode() # unicode
+ text = re.sub(r"https*\S+", " ", text) # url
+ text = re.sub(r"@\S+", " ", text) # mentions
+ text = re.sub(r"#\S+", " ", text) # hastags
+ text = re.sub(r"\s{2,}", " ", text) # over spaces
+
  article = nlp(text)
  sentences = [i.text for i in list(article.sents)]