Upload tokenizer.json

tokenizer.json

@@ -50,7 +50,7 @@
     },
     {
       "id": 5,
-      "content": "<CLS|LLM-jp>",
+      "content": "<|im_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -59,7 +59,7 @@
     },
     {
       "id": 6,
-      "content": "<SEP|LLM-jp>",
+      "content": "<|im_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,12 +104,6 @@
           "id": "A",
           "type_id": 0
         }
-      },
-      {
-        "SpecialToken": {
-          "id": "<EOD|LLM-jp>",
-          "type_id": 0
-        }
       }
     ],
     "pair": [
@@ -119,23 +113,11 @@
           "type_id": 0
         }
       },
-      {
-        "SpecialToken": {
-          "id": "<EOD|LLM-jp>",
-          "type_id": 0
-        }
-      },
       {
         "Sequence": {
           "id": "B",
           "type_id": 1
         }
-      },
-      {
-        "SpecialToken": {
-          "id": "<EOD|LLM-jp>",
-          "type_id": 1
-        }
       }
     ],
     "special_tokens": {
@@ -200,11 +182,11 @@
         0.0
       ],
       [
-        "<CLS|LLM-jp>",
+        "<|im_start|>",
         0.0
       ],
       [
-        "<SEP|LLM-jp>",
+        "<|im_end|>",
         0.0
       ],
       [