Commit
•
458bfe9
1
Parent(s):
1ea70b6
Add chat_template to tokenizer_config.json (#1)
Browse files- Add chat_template to tokenizer_config.json (6f648292e82677c6914b6fc3f833037e4ee8093a)
Co-authored-by: Neural Magic Research <[email protected]>
- tokenizer_config.json +1 -0
tokenizer_config.json
CHANGED
@@ -45,6 +45,7 @@
|
|
45 |
}
|
46 |
},
|
47 |
"bos_token": "<s>",
|
|
|
48 |
"clean_up_tokenization_spaces": false,
|
49 |
"eos_token": "</s>",
|
50 |
"legacy": false,
|
|
|
45 |
}
|
46 |
},
|
47 |
"bos_token": "<s>",
|
48 |
+
"chat_template": "{% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>\n' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}",
|
49 |
"clean_up_tokenization_spaces": false,
|
50 |
"eos_token": "</s>",
|
51 |
"legacy": false,
|