Upload tokenizer
Browse files- tokenizer_config.json +2 -1
tokenizer_config.json
CHANGED
|
@@ -126,7 +126,6 @@
|
|
| 126 |
"<extra_id_123>",
|
| 127 |
"<extra_id_124>"
|
| 128 |
],
|
| 129 |
-
"clean_up_tokenization_spaces": true,
|
| 130 |
"eos_token": {
|
| 131 |
"__type": "AddedToken",
|
| 132 |
"content": "</s>",
|
|
@@ -137,6 +136,7 @@
|
|
| 137 |
},
|
| 138 |
"extra_ids": 125,
|
| 139 |
"model_max_length": 1000000000000000019884624838656,
|
|
|
|
| 140 |
"pad_token": {
|
| 141 |
"__type": "AddedToken",
|
| 142 |
"content": "<pad>",
|
|
@@ -145,6 +145,7 @@
|
|
| 145 |
"rstrip": false,
|
| 146 |
"single_word": false
|
| 147 |
},
|
|
|
|
| 148 |
"tokenizer_class": "ByT5Tokenizer",
|
| 149 |
"unk_token": {
|
| 150 |
"__type": "AddedToken",
|
|
|
|
| 126 |
"<extra_id_123>",
|
| 127 |
"<extra_id_124>"
|
| 128 |
],
|
|
|
|
| 129 |
"eos_token": {
|
| 130 |
"__type": "AddedToken",
|
| 131 |
"content": "</s>",
|
|
|
|
| 136 |
},
|
| 137 |
"extra_ids": 125,
|
| 138 |
"model_max_length": 1000000000000000019884624838656,
|
| 139 |
+
"name_or_path": "google/byt5-small",
|
| 140 |
"pad_token": {
|
| 141 |
"__type": "AddedToken",
|
| 142 |
"content": "<pad>",
|
|
|
|
| 145 |
"rstrip": false,
|
| 146 |
"single_word": false
|
| 147 |
},
|
| 148 |
+
"special_tokens_map_file": "/scratch/p284491/.cache/huggingface/transformers/models--google--byt5-small/snapshots/68377bdc18a2ffec8a0533fef03b1c513a4dd49d/special_tokens_map.json",
|
| 149 |
"tokenizer_class": "ByT5Tokenizer",
|
| 150 |
"unk_token": {
|
| 151 |
"__type": "AddedToken",
|