Upload tokenizer.json
Browse files- tokenizer.json +2 -2
tokenizer.json
CHANGED
@@ -140,7 +140,7 @@
|
|
140 |
},
|
141 |
{
|
142 |
"id": 128015,
|
143 |
-
"content": "
|
144 |
"single_word": false,
|
145 |
"lstrip": false,
|
146 |
"rstrip": false,
|
@@ -149,7 +149,7 @@
|
|
149 |
},
|
150 |
{
|
151 |
"id": 128016,
|
152 |
-
"content": "
|
153 |
"single_word": false,
|
154 |
"lstrip": false,
|
155 |
"rstrip": false,
|
|
|
140 |
},
|
141 |
{
|
142 |
"id": 128015,
|
143 |
+
"content": "<scratch_pad>",
|
144 |
"single_word": false,
|
145 |
"lstrip": false,
|
146 |
"rstrip": false,
|
|
|
149 |
},
|
150 |
{
|
151 |
"id": 128016,
|
152 |
+
"content": "</scratch_pad>",
|
153 |
"single_word": false,
|
154 |
"lstrip": false,
|
155 |
"rstrip": false,
|