gpt2-medium-persian / src /data_utils.py
m3hrdadfi's picture
Merge remote-tracking branch 'origin/saied' into develop
8918872
raw
history blame
1.12 kB
from hazm import word_tokenize
from hazm import sent_tokenize
import re
import six
from normalizer import normalize
persian_regex = "0-9۰۱۲۳۴۵۶۷۸۹ءآئابتثجحخدذرزسشصضطظعغفقلمنهوپچژکگیە\u200c"
def filter_by_lang_regex(text, ratio=0.7, regex="0-9۰۱۲۳۴۵۶۷۸۹ءآئابتثجحخدذرزسشصضطظعغفقلمنهوپچژکگیە\u200c"):
candidate_text = re.sub(r"[^" + regex + "]+", " ", six.ensure_str(text)).replace(" ", "")
text = text.replace(" ", "")
return (len(candidate_text) / len(text)) > ratio
def filter_by_num_tokens(text, gt=64):
return len(word_tokenize(text)) > gt
def filter_by_num_sents(text, gt=2):
return len(sent_tokenize(text)) > gt
def remove_adds(text,ratio=50):
comma = text.split(",")
colon = re.findall(r'(?:([^\W]+):([^\W]+))',text)
virgool = text.split("،")
length_add = len(comma)+len(colon)+len(virgool)
return True if length_add < ratio else False
def normalizer(text, do_lowercase=False):
text = normalize(text)
if do_lowercase:
text = text.lower()
return text