Update tokenizer_config.json

#12
Files changed (1) hide show
  1. tokenizer_config.json +8 -0
tokenizer_config.json CHANGED
@@ -27,6 +27,14 @@
27
  "single_word": false,
28
  "special": true
29
  },
 
 
 
 
 
 
 
 
30
  "7": {
31
  "content": "<|im_end|>",
32
  "lstrip": false,
 
27
  "single_word": false,
28
  "special": true
29
  },
30
+ "6": {
31
+ "content": "<|im_start|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
  "7": {
39
  "content": "<|im_end|>",
40
  "lstrip": false,