Upload tokenizer

README.md

@@ -1,13 +1,13 @@
 ---
 license: cc-by-sa-3.0
-datasets:
-- euclaise/TinyCoT
-- euclaise/reddit-instruct
-- sablo/oasst2_curated
 library_name: transformers
 tags:
 - supertrainer2000
 - human-data
+datasets:
+- euclaise/TinyCoT
+- euclaise/reddit-instruct
+- sablo/oasst2_curated
 metrics:
 - accuracy
 ---

tokenizer_config.json

@@ -203,6 +203,7 @@
     }
   },
   "bos_token": "<|endoftext|>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### User:\n' + message['content'].strip() + '\n' }}{% elif message['role'] == 'assistant' %}{{ '### Assistant:\n' + message['content'].strip() + '\n' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
   "model_max_length": 1000000000000000019884624838656,