Update app.py
Browse files
app.py
CHANGED
|
@@ -56,7 +56,7 @@ def preprocess(text):
|
|
| 56 |
# tokens = T.tokenize(text)
|
| 57 |
# tokens = text_processor.pre_process_docs(text)
|
| 58 |
|
| 59 |
-
tokens = social_tokenizer(
|
| 60 |
print(tokens, file=sys.stderr)
|
| 61 |
ptokens = []
|
| 62 |
for index, token in enumerate(tokens):
|
|
|
|
| 56 |
# tokens = T.tokenize(text)
|
| 57 |
# tokens = text_processor.pre_process_docs(text)
|
| 58 |
|
| 59 |
+
tokens = social_tokenizer(text)
|
| 60 |
print(tokens, file=sys.stderr)
|
| 61 |
ptokens = []
|
| 62 |
for index, token in enumerate(tokens):
|