DrNicefellow commited on
Commit
d5a450a
1 Parent(s): f9951a8

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -12
tokenizer_config.json CHANGED
@@ -2051,18 +2051,6 @@
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
2054
- "clean_up_tokenization_spaces": false,
2055
- "eos_token": "<|endoftext|>",
2056
- "errors": "replace",
2057
- "model_max_length": 131072,
2058
- "pad_token": "<|PAD_TOKEN|>",
2059
- "padding_side": "left",
2060
- "split_special_tokens": false,
2061
- "tokenizer_class": "Qwen2Tokenizer",
2062
- "unk_token": null
2063
- }
2064
-
2065
-
2066
  "clean_up_tokenization_spaces": true,
2067
  "eos_token": "<|end_of_text|>",
2068
  "model_input_names": [
 
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
 
 
 
 
 
 
 
 
 
 
 
 
2054
  "clean_up_tokenization_spaces": true,
2055
  "eos_token": "<|end_of_text|>",
2056
  "model_input_names": [