add unicodedata

app.py

@@ -10,6 +10,7 @@ from pathlib import Path
 import pandas as pd
 import plotly.express as px
 import keras
+import unicodedata as ud
 
 from underthesea import word_tokenize
 
@@ -81,6 +82,8 @@ def judge(x):
   result = []
   judge_result = []
 
+  x = ud.normalize('NFKC', x)
+
   lstm_pred = LSTM_predict(x)
   gru_pred = GRU_predict(x)
   # bert_pred = BERT_predict(x)

requirements.txt

@@ -9,4 +9,5 @@ pandas
 keras==2.15.0
 underthesea
 torch
-transformers
+transformers
+unicodedata