Upload 7 files
Browse files- added_tokens.json +1 -0
- config.json +1 -1
- model.safetensors +2 -2
- special_tokens_map.json +7 -0
- tokenizer_config.json +10 -1
added_tokens.json
CHANGED
@@ -148,6 +148,7 @@
|
|
148 |
"[PLURIFORM_PREFIX:R]": 32141,
|
149 |
"[PLURIFORM_PREFIX:S]": 32154,
|
150 |
"[ROOT]": 32163,
|
|
|
151 |
"[SUBJECT:1ppe]": 32158,
|
152 |
"[SUBJECT:1ppi]": 32120,
|
153 |
"[SUBJECT:1ps]": 32112,
|
|
|
148 |
"[PLURIFORM_PREFIX:R]": 32141,
|
149 |
"[PLURIFORM_PREFIX:S]": 32154,
|
150 |
"[ROOT]": 32163,
|
151 |
+
"[SPACE]": 32195,
|
152 |
"[SUBJECT:1ppe]": 32158,
|
153 |
"[SUBJECT:1ppi]": 32120,
|
154 |
"[SUBJECT:1ps]": 32112,
|
config.json
CHANGED
@@ -57,5 +57,5 @@
|
|
57 |
"torch_dtype": "float32",
|
58 |
"transformers_version": "4.38.1",
|
59 |
"use_cache": true,
|
60 |
-
"vocab_size":
|
61 |
}
|
|
|
57 |
"torch_dtype": "float32",
|
58 |
"transformers_version": "4.38.1",
|
59 |
"use_cache": true,
|
60 |
+
"vocab_size": 32196
|
61 |
}
|
model.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7407150199f25a66d3268c2b4d94f2f965616a84fc9451a83bd1b8faba868982
|
3 |
+
size 242181160
|
special_tokens_map.json
CHANGED
@@ -734,6 +734,13 @@
|
|
734 |
"normalized": false,
|
735 |
"rstrip": false,
|
736 |
"single_word": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
737 |
}
|
738 |
],
|
739 |
"eos_token": {
|
|
|
734 |
"normalized": false,
|
735 |
"rstrip": false,
|
736 |
"single_word": false
|
737 |
+
},
|
738 |
+
{
|
739 |
+
"content": "[SPACE]",
|
740 |
+
"lstrip": false,
|
741 |
+
"normalized": false,
|
742 |
+
"rstrip": false,
|
743 |
+
"single_word": false
|
744 |
}
|
745 |
],
|
746 |
"eos_token": {
|
tokenizer_config.json
CHANGED
@@ -1672,6 +1672,14 @@
|
|
1672 |
"rstrip": false,
|
1673 |
"single_word": false,
|
1674 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1675 |
}
|
1676 |
},
|
1677 |
"additional_special_tokens": [
|
@@ -1779,7 +1787,8 @@
|
|
1779 |
"[IMPERATIVE_PREFIX:2ps]",
|
1780 |
"[OBJECT_MARKER:3p:MONOSYLLABIC]",
|
1781 |
"opo",
|
1782 |
-
"îo"
|
|
|
1783 |
],
|
1784 |
"clean_up_tokenization_spaces": true,
|
1785 |
"eos_token": "</s>",
|
|
|
1672 |
"rstrip": false,
|
1673 |
"single_word": false,
|
1674 |
"special": true
|
1675 |
+
},
|
1676 |
+
"32195": {
|
1677 |
+
"content": "[SPACE]",
|
1678 |
+
"lstrip": false,
|
1679 |
+
"normalized": false,
|
1680 |
+
"rstrip": false,
|
1681 |
+
"single_word": false,
|
1682 |
+
"special": true
|
1683 |
}
|
1684 |
},
|
1685 |
"additional_special_tokens": [
|
|
|
1787 |
"[IMPERATIVE_PREFIX:2ps]",
|
1788 |
"[OBJECT_MARKER:3p:MONOSYLLABIC]",
|
1789 |
"opo",
|
1790 |
+
"îo",
|
1791 |
+
"[SPACE]"
|
1792 |
],
|
1793 |
"clean_up_tokenization_spaces": true,
|
1794 |
"eos_token": "</s>",
|