Upload tokenizer

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+ "bos_token": "<|endoftext|>",
+ "eos_token": "<|endoftext|>",
+ "pad_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+ "_commit_hash": "fc8e001871f4a6be8e6079093b33de334a2316c9",
+ "_from_auto": true,
+ "added_tokens_decoder": {},
+ "auto_map": {
+ "AutoTokenizer": [
+ "tokenization_phi3_small.Phi3SmallTokenizer",
+ null
+ ]
+ },
+ "bos_token": "<|endoftext|>",
+ "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+ "clean_up_tokenization_spaces": true,
+ "eos_token": "<|endoftext|>",
+ "model_max_length": 8192,
+ "pad_token": "<|endoftext|>",
+ "tokenizer_class": "Phi3SmallTokenizer",
+ "trust_remote_code": true
+}