hieunguyen1053
commited on
Upload tokenizer
Browse files- added_tokens.json +1 -0
- tokenizer.json +9 -0
- tokenizer_config.json +8 -0
added_tokens.json
CHANGED
@@ -1,4 +1,5 @@
|
|
1 |
{
|
|
|
2 |
"[NEG]": 36096,
|
3 |
"[NEU]": 36097,
|
4 |
"[POS]": 36098
|
|
|
1 |
{
|
2 |
+
"#": 36099,
|
3 |
"[NEG]": 36096,
|
4 |
"[NEU]": 36097,
|
5 |
"[POS]": 36098
|
tokenizer.json
CHANGED
@@ -920,6 +920,15 @@
|
|
920 |
"rstrip": false,
|
921 |
"normalized": false,
|
922 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
923 |
}
|
924 |
],
|
925 |
"normalizer": {
|
|
|
920 |
"rstrip": false,
|
921 |
"normalized": false,
|
922 |
"special": true
|
923 |
+
},
|
924 |
+
{
|
925 |
+
"id": 36099,
|
926 |
+
"content": "#",
|
927 |
+
"single_word": false,
|
928 |
+
"lstrip": false,
|
929 |
+
"rstrip": false,
|
930 |
+
"normalized": true,
|
931 |
+
"special": false
|
932 |
}
|
933 |
],
|
934 |
"normalizer": {
|
tokenizer_config.json
CHANGED
@@ -815,6 +815,14 @@
|
|
815 |
"rstrip": false,
|
816 |
"single_word": false,
|
817 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
818 |
}
|
819 |
},
|
820 |
"additional_special_tokens": [
|
|
|
815 |
"rstrip": false,
|
816 |
"single_word": false,
|
817 |
"special": true
|
818 |
+
},
|
819 |
+
"36099": {
|
820 |
+
"content": "#",
|
821 |
+
"lstrip": false,
|
822 |
+
"normalized": true,
|
823 |
+
"rstrip": false,
|
824 |
+
"single_word": false,
|
825 |
+
"special": false
|
826 |
}
|
827 |
},
|
828 |
"additional_special_tokens": [
|