Upload processor
Browse files- added_tokens.json +4 -3
- tokenizer.json +13 -4
added_tokens.json
CHANGED
@@ -10,13 +10,14 @@
|
|
10 |
"<invoice/>": 57530,
|
11 |
"<letter/>": 57531,
|
12 |
"<memo/>": 57532,
|
13 |
-
"<news/>":
|
14 |
"<news_article/>": 57533,
|
15 |
-
"<note/>":
|
16 |
"<passport/>": 57544,
|
17 |
"<presentation/>": 57534,
|
18 |
"<questionnaire/>": 57535,
|
19 |
-
"<
|
|
|
20 |
"<resume/>": 57536,
|
21 |
"<s_class>": 57537,
|
22 |
"<s_iitcdip>": 57538,
|
|
|
10 |
"<invoice/>": 57530,
|
11 |
"<letter/>": 57531,
|
12 |
"<memo/>": 57532,
|
13 |
+
"<news/>": 57547,
|
14 |
"<news_article/>": 57533,
|
15 |
+
"<note/>": 57548,
|
16 |
"<passport/>": 57544,
|
17 |
"<presentation/>": 57534,
|
18 |
"<questionnaire/>": 57535,
|
19 |
+
"<receipt/>": 57546,
|
20 |
+
"<report/>": 57549,
|
21 |
"<resume/>": 57536,
|
22 |
"<s_class>": 57537,
|
23 |
"<s_iitcdip>": 57538,
|
tokenizer.json
CHANGED
@@ -149,8 +149,8 @@
|
|
149 |
"single_word": false,
|
150 |
"lstrip": false,
|
151 |
"rstrip": false,
|
152 |
-
"normalized":
|
153 |
-
"special":
|
154 |
},
|
155 |
{
|
156 |
"id": 57532,
|
@@ -280,7 +280,7 @@
|
|
280 |
},
|
281 |
{
|
282 |
"id": 57546,
|
283 |
-
"content": "<
|
284 |
"single_word": false,
|
285 |
"lstrip": false,
|
286 |
"rstrip": false,
|
@@ -289,7 +289,7 @@
|
|
289 |
},
|
290 |
{
|
291 |
"id": 57547,
|
292 |
-
"content": "<
|
293 |
"single_word": false,
|
294 |
"lstrip": false,
|
295 |
"rstrip": false,
|
@@ -298,6 +298,15 @@
|
|
298 |
},
|
299 |
{
|
300 |
"id": 57548,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
301 |
"content": "<report/>",
|
302 |
"single_word": false,
|
303 |
"lstrip": false,
|
|
|
149 |
"single_word": false,
|
150 |
"lstrip": false,
|
151 |
"rstrip": false,
|
152 |
+
"normalized": false,
|
153 |
+
"special": true
|
154 |
},
|
155 |
{
|
156 |
"id": 57532,
|
|
|
280 |
},
|
281 |
{
|
282 |
"id": 57546,
|
283 |
+
"content": "<receipt/>",
|
284 |
"single_word": false,
|
285 |
"lstrip": false,
|
286 |
"rstrip": false,
|
|
|
289 |
},
|
290 |
{
|
291 |
"id": 57547,
|
292 |
+
"content": "<news/>",
|
293 |
"single_word": false,
|
294 |
"lstrip": false,
|
295 |
"rstrip": false,
|
|
|
298 |
},
|
299 |
{
|
300 |
"id": 57548,
|
301 |
+
"content": "<note/>",
|
302 |
+
"single_word": false,
|
303 |
+
"lstrip": false,
|
304 |
+
"rstrip": false,
|
305 |
+
"normalized": true,
|
306 |
+
"special": false
|
307 |
+
},
|
308 |
+
{
|
309 |
+
"id": 57549,
|
310 |
"content": "<report/>",
|
311 |
"single_word": false,
|
312 |
"lstrip": false,
|