hieunguyen1053 commited on
Commit
8f06854
·
verified ·
1 Parent(s): aacac6f

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +1 -0
  2. tokenizer.json +9 -0
  3. tokenizer_config.json +8 -0
added_tokens.json CHANGED
@@ -1,4 +1,5 @@
1
  {
 
2
  "[NEG]": 36096,
3
  "[NEU]": 36097,
4
  "[POS]": 36098
 
1
  {
2
+ "#": 36099,
3
  "[NEG]": 36096,
4
  "[NEU]": 36097,
5
  "[POS]": 36098
tokenizer.json CHANGED
@@ -920,6 +920,15 @@
920
  "rstrip": false,
921
  "normalized": false,
922
  "special": true
 
 
 
 
 
 
 
 
 
923
  }
924
  ],
925
  "normalizer": {
 
920
  "rstrip": false,
921
  "normalized": false,
922
  "special": true
923
+ },
924
+ {
925
+ "id": 36099,
926
+ "content": "#",
927
+ "single_word": false,
928
+ "lstrip": false,
929
+ "rstrip": false,
930
+ "normalized": true,
931
+ "special": false
932
  }
933
  ],
934
  "normalizer": {
tokenizer_config.json CHANGED
@@ -815,6 +815,14 @@
815
  "rstrip": false,
816
  "single_word": false,
817
  "special": true
 
 
 
 
 
 
 
 
818
  }
819
  },
820
  "additional_special_tokens": [
 
815
  "rstrip": false,
816
  "single_word": false,
817
  "special": true
818
+ },
819
+ "36099": {
820
+ "content": "#",
821
+ "lstrip": false,
822
+ "normalized": true,
823
+ "rstrip": false,
824
+ "single_word": false,
825
+ "special": false
826
  }
827
  },
828
  "additional_special_tokens": [