mlengineer-ai
/

kenlm-sp-jomleh

Model card Files Files and versions Community

mehran commited on May 21, 2023

Commit

576f1fa

•

1 Parent(s): be62ffe

Update model.py

Files changed (1) hide show

model.py +1 -1

model.py CHANGED Viewed

@@ -451,7 +451,7 @@ class KenlmModel:
         if normalize_numbers:
             norm_list += [normalizers.Replace(Regex("[۱۲۳۴۵۶۷۸۹]"), "۰")]
         if remove_puctuation:
-            norm_list += [normalizers.Replace(Regex("[.!؛،؟]"), "")]
         if remove_non_farsi:
             norm_list += [normalizers.Replace(Regex("[^\u060c\u061b\u061f\u0622\u0623\u0624\u0626\u0627"
                                                     "\u0628\u062a\u062b\u062c\u062d\u062e\u062f\u0630\u0631"

         if normalize_numbers:
             norm_list += [normalizers.Replace(Regex("[۱۲۳۴۵۶۷۸۹]"), "۰")]
         if remove_puctuation:
+            norm_list += [normalizers.Replace(Regex("[\\.!؛،؟]"), "")]
         if remove_non_farsi:
             norm_list += [normalizers.Replace(Regex("[^\u060c\u061b\u061f\u0622\u0623\u0624\u0626\u0627"
                                                     "\u0628\u062a\u062b\u062c\u062d\u062e\u062f\u0630\u0631"