{
"add_prefix_space": false,
"added_tokens_decoder": {
"100256": {
"content": "<|_unuse_missing_100256|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100257": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100258": {
"content": "<|fim_prefix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100259": {
"content": "<|fim_middle|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100260": {
"content": "<|fim_suffix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100261": {
"content": "<|_unuse_missing_100261|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100262": {
"content": "<|_unuse_missing_100262|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100263": {
"content": "<|_unuse_missing_100263|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100264": {
"content": "<|_unuse_missing_100264|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100265": {
"content": "<|_unuse_missing_100265|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100266": {
"content": "<|_unuse_missing_100266|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100267": {
"content": "<|_unuse_missing_100267|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100268": {
"content": "<|_unuse_missing_100268|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100269": {
"content": "<|_unuse_missing_100269|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100270": {
"content": "<|_unuse_missing_100270|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100271": {
"content": "<|_unuse_missing_100271|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100272": {
"content": "<|_unuse_missing_100272|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100273": {
"content": "<|_unuse_missing_100273|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100274": {
"content": "<|_unuse_missing_100274|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100275": {
"content": "<|_unuse_missing_100275|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100276": {
"content": "<|endofprompt|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<|endoftext|>",
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>",
"<|endofprompt|>",
"<|_unuse_missing_100256|>",
"<|_unuse_missing_100261|>",
"<|_unuse_missing_100262|>",
"<|_unuse_missing_100263|>",
"<|_unuse_missing_100264|>",
"<|_unuse_missing_100265|>",
"<|_unuse_missing_100266|>",
"<|_unuse_missing_100267|>",
"<|_unuse_missing_100268|>",
"<|_unuse_missing_100269|>",
"<|_unuse_missing_100270|>",
"<|_unuse_missing_100271|>",
"<|_unuse_missing_100272|>",
"<|_unuse_missing_100273|>",
"<|_unuse_missing_100274|>",
"<|_unuse_missing_100275|>"
],
"bos_token": "<|endoftext|>",
"chat_template": "{% if messages[0]['role'] == 'system' %}{% set merged_content = messages[0]['content'] + ' ' + messages[1]['content'] %}{% set merged_messages = [{'role': messages[1]['role'], 'content': merged_content}] + messages[2:] %}{% else %}{% set merged_messages = messages %}{% endif %}{% for message in merged_messages %}{{('human' if message['role'] == 'user' else message['role']) + ': ' + (message['content'].split('')|first + message['content'].split('')|last if message['role'] == 'assistant' and '' in message['content'] else message['content'])}}{% if (loop.last and add_generation_prompt and merged_messages[-1]['role'] != 'assistant') or not loop.last %}{{ ' ' }}{% endif %}{% endfor %}{% if add_generation_prompt and merged_messages[-1]['role'] != 'assistant' %}{{ 'assistant:' }}{% endif %}",
"clean_up_tokenization_spaces": true,
"eos_token": "<|endoftext|>",
"model_max_length": 32768,
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "<|endoftext|>"
}