Updated tokenizer
Browse files- tokenizer.json +0 -9
tokenizer.json
CHANGED
|
@@ -200,15 +200,6 @@
|
|
| 200 |
"rstrip": false,
|
| 201 |
"normalized": false,
|
| 202 |
"special": true
|
| 203 |
-
},
|
| 204 |
-
{
|
| 205 |
-
"id": 49155,
|
| 206 |
-
"content": "<sep>",
|
| 207 |
-
"single_word": false,
|
| 208 |
-
"lstrip": false,
|
| 209 |
-
"rstrip": false,
|
| 210 |
-
"normalized": false,
|
| 211 |
-
"special": true
|
| 212 |
}
|
| 213 |
],
|
| 214 |
"normalizer": null,
|
|
|
|
| 200 |
"rstrip": false,
|
| 201 |
"normalized": false,
|
| 202 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
}
|
| 204 |
],
|
| 205 |
"normalizer": null,
|