teknium commited on
Commit
ef35118
·
verified ·
1 Parent(s): 0fe3160

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -2
tokenizer.json CHANGED
@@ -140,7 +140,7 @@
140
  },
141
  {
142
  "id": 128015,
143
- "content": "<|reserved_special_token_10|>",
144
  "single_word": false,
145
  "lstrip": false,
146
  "rstrip": false,
@@ -149,7 +149,7 @@
149
  },
150
  {
151
  "id": 128016,
152
- "content": "<|reserved_special_token_11|>",
153
  "single_word": false,
154
  "lstrip": false,
155
  "rstrip": false,
 
140
  },
141
  {
142
  "id": 128015,
143
+ "content": "<scratch_pad>",
144
  "single_word": false,
145
  "lstrip": false,
146
  "rstrip": false,
 
149
  },
150
  {
151
  "id": 128016,
152
+ "content": "</scratch_pad>",
153
  "single_word": false,
154
  "lstrip": false,
155
  "rstrip": false,