updated spacy
Browse files- categories/fluency.py +2 -3
categories/fluency.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
import language_tool_python
|
2 |
import numpy as np
|
3 |
-
import
|
4 |
import torch
|
5 |
import wordfreq
|
6 |
from transformers import AutoModelForMaskedLM, AutoTokenizer
|
@@ -18,8 +18,7 @@ model.eval()
|
|
18 |
tokenizer = AutoTokenizer.from_pretrained(model_name) # tokenizer
|
19 |
|
20 |
# spacy model for parsing
|
21 |
-
nlp =
|
22 |
-
|
23 |
|
24 |
def __get_rarity(word: str, lang: str = "en") -> float:
|
25 |
"""
|
|
|
1 |
import language_tool_python
|
2 |
import numpy as np
|
3 |
+
import en_core_web_sm
|
4 |
import torch
|
5 |
import wordfreq
|
6 |
from transformers import AutoModelForMaskedLM, AutoTokenizer
|
|
|
18 |
tokenizer = AutoTokenizer.from_pretrained(model_name) # tokenizer
|
19 |
|
20 |
# spacy model for parsing
|
21 |
+
nlp = en_core_web_sm.load()
|
|
|
22 |
|
23 |
def __get_rarity(word: str, lang: str = "en") -> float:
|
24 |
"""
|