Update README.md
Browse files
README.md
CHANGED
@@ -21,6 +21,7 @@ from tokenizers import Tokenizer, models, pre_tokenizers, trainers, Regex
|
|
21 |
import tokenizers
|
22 |
from tokenizers import Tokenizer, models, decoders, processors
|
23 |
from tokenizers import pre_tokenizers, trainers, Regex
|
|
|
24 |
|
25 |
class Attention(tf.keras.layers.Layer):
|
26 |
def __init__(self,
|
@@ -51,7 +52,9 @@ class Attention(tf.keras.layers.Layer):
|
|
51 |
return config
|
52 |
|
53 |
#download model
|
54 |
-
model = tf.keras.models.load_model(
|
|
|
|
|
55 |
compile=False,
|
56 |
custom_objects={'Attention':Attention})
|
57 |
|
@@ -101,9 +104,14 @@ class BPETokenizer:
|
|
101 |
else:
|
102 |
return decoded
|
103 |
#download tokenizer
|
104 |
-
tokenizer = BPETokenizer(vocab
|
105 |
-
|
|
|
|
|
|
|
|
|
106 |
)
|
|
|
107 |
```
|
108 |
|
109 |
|
|
|
21 |
import tokenizers
|
22 |
from tokenizers import Tokenizer, models, decoders, processors
|
23 |
from tokenizers import pre_tokenizers, trainers, Regex
|
24 |
+
import huggingface_hub
|
25 |
|
26 |
class Attention(tf.keras.layers.Layer):
|
27 |
def __init__(self,
|
|
|
52 |
return config
|
53 |
|
54 |
#download model
|
55 |
+
model = tf.keras.models.load_model(huggingface_hub.hf_hub_download('vkovenko/deep_lstm_attention_ukr_reviews_rating_estimation',
|
56 |
+
'deep_lstm_attention_w2v_huber.h5',
|
57 |
+
local_dir='model'),
|
58 |
compile=False,
|
59 |
custom_objects={'Attention':Attention})
|
60 |
|
|
|
104 |
else:
|
105 |
return decoded
|
106 |
#download tokenizer
|
107 |
+
tokenizer = BPETokenizer(vocab=huggingface_hub.hf_hub_download('vkovenko/deep_lstm_attention_ukr_reviews_rating_estimation',
|
108 |
+
'tokenizer_30k.json',
|
109 |
+
local_dir='model'),
|
110 |
+
merges=huggingface_hub.hf_hub_download('vkovenko/deep_lstm_attention_ukr_reviews_rating_estimation',
|
111 |
+
'merges_tokenizer.txt',
|
112 |
+
local_dir='model')
|
113 |
)
|
114 |
+
|
115 |
```
|
116 |
|
117 |
|