Upload tokenizer

added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "<image>": 151646,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

special_tokens_map.json

@@ -3,13 +3,6 @@
     "<|im_start|>",
     "<|im_end|>"
   ],
-  "bos_token": {
-    "content": "<|im_start|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "eos_token": {
     "content": "<|im_end|>",
     "lstrip": false,

tokenizer.json

@@ -63,58 +63,10 @@
     ]
   },
   "post_processor": {
-    "type": "TemplateProcessing",
-    "single": [
-      {
-        "SpecialToken": {
-          "id": "<|im_start|>",
-          "type_id": 0
-        }
-      },
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      }
-    ],
-    "pair": [
-      {
-        "SpecialToken": {
-          "id": "<|im_start|>",
-          "type_id": 0
-        }
-      },
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      },
-      {
-        "SpecialToken": {
-          "id": "<|im_start|>",
-          "type_id": 1
-        }
-      },
-      {
-        "Sequence": {
-          "id": "B",
-          "type_id": 1
-        }
-      }
-    ],
-    "special_tokens": {
-      "<|im_start|>": {
-        "id": "<|im_start|>",
-        "ids": [
-          151644
-        ],
-        "tokens": [
-          "<|im_start|>"
-        ]
-      }
-    }
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
   },
   "decoder": {
     "type": "ByteLevel",
@@ -130,6 +82,7 @@
     "end_of_word_suffix": "",
     "fuse_unk": false,
     "byte_fallback": false,
+    "ignore_merges": false,
     "vocab": {
       "!": 0,
       "\"": 1,

tokenizer_config.json

@@ -1,6 +1,5 @@
 {
-  "add_bos_token": true,
-  "add_eos_token": false,
+  "add_prefix_space": false,
   "added_tokens_decoder": {
     "151643": {
       "content": "<|endoftext|>",
@@ -39,7 +38,7 @@
     "<|im_start|>",
     "<|im_end|>"
   ],
-  "bos_token": "<|im_start|>",
+  "bos_token": null,
   "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
@@ -47,9 +46,7 @@
   "model_max_length": 32768,
   "pad_token": "<|endoftext|>",
   "padding_side": "right",
-  "processor_class": "LlavaProcessor",
   "split_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": null,
-  "use_default_system_prompt": false
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
 }