tokenizer_config.json

{
 "add_prefix_space": false,
 "added_tokens_decoder": {
 "151643": {
 "content": "<|endoftext|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151644": {
 "content": "<|im_start|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151645": {
 "content": "<|im_end|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151647": {
 "content": "<|userprompt|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151648": {
 "content": "<|endofuserprompt|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151649": {
 "content": "<|response|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151650": {
 "content": "<|endofresponse|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151651": {
 "content": "<|system|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151652": {
 "content": "<|endofsystem|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151653": {
 "content": "<|observation|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151654": {
 "content": "<|endofobservation|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151655": {
 "content": "<|execution|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151656": {
 "content": "<|endofexecution|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151657": {
 "content": "<|reject-unknown|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151658": {
 "content": "<|sec-cot|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "151659": {
 "content": "<|sec-end-cot|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 }
 },
 "additional_special_tokens": ["<|im_start|>", "<|im_end|>", "<|userprompt|>", "<|endofuserprompt|>", "<|response|>", "<|endofresponse|>", "<|system|>", "<|endofsystem|>", "<|observation|>", "<|endofobservation|>", "<|execution|>", "<|endofexecution|>", "<|reject-unknown|>", "<|sec-cot|>", "<|sec-end-cot|>"],
 "bos_token": null,
 "chat_template": "{% if messages[0]['role'] == 'system' %}<|system|>{{ messages[0]['content'] }}<|endofsystem|>{% set start_idx = 1 %}{% else %}<|system|>You are a helpful assistant<|endofsystem|>{% set start_idx = 0 %}{% endif %}{% for idx in range(start_idx, messages|length) %}{% if messages[idx]['role'] == 'user' %}<|userprompt|>{{ messages[idx]['content'] }}<|endofuserprompt|>{% elif messages[idx]['role'] == 'assistant' %}<|response|>{{ messages[idx]['content'] }}<|endofresponse|>{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] == 'user' %}<|response|>{% endif %}",
 "clean_up_tokenization_spaces": false,
 "eos_token": "<|endofresponse|>",
 "errors": "replace",
 "model_max_length": 131072,
 "pad_token": "<|endofresponse|>",
 "split_special_tokens": false,
 "tokenizer_class": "Qwen2Tokenizer",
 "unk_token": null
}