Add <step> token, chat template (#5)

- Add <step> token, chat template (0a378c1610f3711b26a521ecf070c7acb2caeb08)


Co-authored-by: Pedro Cuenca <[email protected]>

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<step>": 32015
+}

tokenizer.json

@@ -29,6 +29,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 32015,
+      "content": "<step>",
+      "single_word": true,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": false
     }
   ],
   "normalizer": {
@@ -32149,8 +32158,7 @@
       "▁<EOT><EOT>": 32011,
       "▁<EOT><EOT><EOT>": 32012,
       "▁<EOT><EOT><EOT><EOT>": 32013,
-      "▁<EOT><EOT><EOT><EOT><EOT>": 32014,
-      "<step>": 32015
+      "▁<EOT><EOT><EOT><EOT><EOT>": 32014
     },
     "merges": [
       "▁ t",
@@ -93415,4 +93423,4 @@
       "▁<EOT >"
     ]
   }
-}
+}

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set user_index = 1 %}{% else %}{% set user_index = 0 %}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != ((loop.index0 + user_index) % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 %}{{ '<s>' }}{% endif %}{% set content = 'Source: ' + message['role'] + '\n\n ' + message['content'].strip() %}{{ content + ' <step> ' }}{% endfor %}{{'Source: assistant\nDestination: user\n\n '}}",
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -25,6 +26,14 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "32015": {
+      "content": "<step>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": true,
+      "special": false
     }
   },
   "bos_token": "<s>",