Update preprocessing.py

preprocessing.py

@@ -0,0 +1,73 @@
+import hazm
+from cleantext import clean
+import regex as re
+
+def cleanhtml(raw_html):
+ cleanr = re.compile('<.*?>')
+ cleantext = re.sub(cleanr, '', raw_html)
+ return cleantext
+
+def cleaning(text):
+ text = text.strip()
+ 
+ # regular cleaning
+ # https://pypi.org/project/clean-text/ >> works well for eng and de languages
+ text = clean(text,
+ fix_unicode=True,
+ to_ascii=False,
+ lower=True,
+ no_line_breaks=True,
+ no_urls=True,
+ no_emails=True,
+ no_phone_numbers=True,
+ no_numbers=False,
+ no_digits=False,
+ no_currency_symbols=True,
+ no_punct=False, #Keep the punc
+ replace_with_url="",
+ replace_with_email="",
+ replace_with_phone_number="",
+ replace_with_number="",
+ replace_with_digit="0",
+ replace_with_currency_symbol="",
+ )
+
+ # cleaning htmls
+ text = cleanhtml(text)
+ 
+ # normalizing > https://github.com/sobhe/hazm
+ normalizer = hazm.Normalizer()
+ text = normalizer.normalize(text)
+ 
+ # removing wierd patterns
+ wierd_pattern = re.compile("["
+ u"\U0001F600-\U0001F64F" # emoticons
+ u"\U0001F300-\U0001F5FF" # symbols & pictographs
+ u"\U0001F680-\U0001F6FF" # transport & map symbols
+ u"\U0001F1E0-\U0001F1FF" # flags (iOS)
+ u"\U00002702-\U000027B0"
+ u"\U000024C2-\U0001F251"
+ u"\U0001f926-\U0001f937"
+ u'\U00010000-\U0010ffff'
+ u"\u200d"
+ u"\u2640-\u2642"
+ u"\u2600-\u2B55"
+ u"\u23cf"
+ u"\u23e9"
+ u"\u231a"
+ u"\u3030"
+ u"\ufe0f"
+ u"\u2069"
+ u"\u2066"
+ # u"\u200c"
+ u"\u2068"
+ u"\u2067"
+ "]+", flags=re.UNICODE)
+ 
+ text = wierd_pattern.sub(r'', text)
+ 
+ # removing extra spaces, hashtags
+ text = re.sub("#", "", text)
+ text = re.sub("\s+", " ", text)
+ 
+ return text