Corianas commited on
Commit
d37067b
·
verified ·
1 Parent(s): e8b403e

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +8 -0
tokenizer_config.json CHANGED
@@ -25,6 +25,14 @@
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
 
 
 
 
 
 
 
 
28
  }
29
  },
30
  "bos_token": "<s>",
 
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
+ },
29
+ "3": {
30
+ "content": "<END>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
  }
37
  },
38
  "bos_token": "<s>",