Mijavier commited on
Commit
441232c
1 Parent(s): 3eddd9a

Upload processor

Browse files
Files changed (2) hide show
  1. added_tokens.json +4 -3
  2. tokenizer.json +13 -4
added_tokens.json CHANGED
@@ -10,13 +10,14 @@
10
  "<invoice/>": 57530,
11
  "<letter/>": 57531,
12
  "<memo/>": 57532,
13
- "<news/>": 57546,
14
  "<news_article/>": 57533,
15
- "<note/>": 57547,
16
  "<passport/>": 57544,
17
  "<presentation/>": 57534,
18
  "<questionnaire/>": 57535,
19
- "<report/>": 57548,
 
20
  "<resume/>": 57536,
21
  "<s_class>": 57537,
22
  "<s_iitcdip>": 57538,
 
10
  "<invoice/>": 57530,
11
  "<letter/>": 57531,
12
  "<memo/>": 57532,
13
+ "<news/>": 57547,
14
  "<news_article/>": 57533,
15
+ "<note/>": 57548,
16
  "<passport/>": 57544,
17
  "<presentation/>": 57534,
18
  "<questionnaire/>": 57535,
19
+ "<receipt/>": 57546,
20
+ "<report/>": 57549,
21
  "<resume/>": 57536,
22
  "<s_class>": 57537,
23
  "<s_iitcdip>": 57538,
tokenizer.json CHANGED
@@ -149,8 +149,8 @@
149
  "single_word": false,
150
  "lstrip": false,
151
  "rstrip": false,
152
- "normalized": true,
153
- "special": false
154
  },
155
  {
156
  "id": 57532,
@@ -280,7 +280,7 @@
280
  },
281
  {
282
  "id": 57546,
283
- "content": "<news/>",
284
  "single_word": false,
285
  "lstrip": false,
286
  "rstrip": false,
@@ -289,7 +289,7 @@
289
  },
290
  {
291
  "id": 57547,
292
- "content": "<note/>",
293
  "single_word": false,
294
  "lstrip": false,
295
  "rstrip": false,
@@ -298,6 +298,15 @@
298
  },
299
  {
300
  "id": 57548,
 
 
 
 
 
 
 
 
 
301
  "content": "<report/>",
302
  "single_word": false,
303
  "lstrip": false,
 
149
  "single_word": false,
150
  "lstrip": false,
151
  "rstrip": false,
152
+ "normalized": false,
153
+ "special": true
154
  },
155
  {
156
  "id": 57532,
 
280
  },
281
  {
282
  "id": 57546,
283
+ "content": "<receipt/>",
284
  "single_word": false,
285
  "lstrip": false,
286
  "rstrip": false,
 
289
  },
290
  {
291
  "id": 57547,
292
+ "content": "<news/>",
293
  "single_word": false,
294
  "lstrip": false,
295
  "rstrip": false,
 
298
  },
299
  {
300
  "id": 57548,
301
+ "content": "<note/>",
302
+ "single_word": false,
303
+ "lstrip": false,
304
+ "rstrip": false,
305
+ "normalized": true,
306
+ "special": false
307
+ },
308
+ {
309
+ "id": 57549,
310
  "content": "<report/>",
311
  "single_word": false,
312
  "lstrip": false,