Upload 6 files

added_tokens.json

@@ -1,40 +1,13 @@
 {
-  "<|/code|>": 32014,
-  "<|/data|>": 32033,
-  "<|/inst|>": 32037,
-  "<|/query|>": 32031,
-  "<|/sys|>": 32035,
-  "<|assistant_mask|>": 32017,
-  "<|assistant|>": 32001,
-  "<|calc|>": 32012,
-  "<|code|>": 32013,
-  "<|continue|>": 32009,
-  "<|data|>": 32032,
-  "<|diff_marker|>": 32025,
-  "<|disc_sep|>": 32029,
-  "<|disc_start|>": 32028,
-  "<|disc_thread|><|query|>": 32030,
   "<|endoftext|>": 32000,
-  "<|end|>": 32007,
-  "<|fim_middle|>": 32021,
-  "<|fim_prefix|>": 32020,
-  "<|fim_suffix|>": 32022,
-  "<|function_call|>": 32005,
-  "<|function_list|>": 32011,
-  "<|function_output|>": 32003,
-  "<|ghissue|>": 32026,
-  "<|ghreview|>": 32027,
-  "<|inst|>": 32036,
-  "<|ipynb_marker|>": 32024,
-  "<|message|>": 32019,
-  "<|meta_start|>": 32023,
-  "<|raw|>": 32008,
-  "<|resource|>": 32016,
-  "<|start|>": 32018,
-  "<|step|>": 32002,
-  "<|summary|>": 32015,
+  "<|assistant|>": 32001,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
   "<|system|>": 32006,
-  "<|sys|>": 32034,
-  "<|tag|>": 32004,
+  "<|end|>": 32007,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
   "<|user|>": 32010
 }

generation_config.json

@@ -8,4 +8,4 @@
   ],
   "pad_token_id": 32000,
   "transformers_version": "4.39.3"
-}
+}

special_tokens_map.json

@@ -1,7 +1,4 @@
 {
-  "additional_special_tokens": [
-    "<|/inst|>"
-  ],
   "bos_token": {
     "content": "<s>",
     "lstrip": false,
@@ -30,4 +27,4 @@
     "rstrip": false,
     "single_word": false
   }
-}
+}

tokenizer.json

@@ -50,7 +50,7 @@
     },
     {
       "id": 32002,
-      "content": "<|step|>",
+      "content": "<|placeholder1|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
@@ -59,7 +59,7 @@
     },
     {
       "id": 32003,
-      "content": "<|function_output|>",
+      "content": "<|placeholder2|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
@@ -68,7 +68,7 @@
     },
     {
       "id": 32004,
-      "content": "<|tag|>",
+      "content": "<|placeholder3|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
@@ -77,7 +77,7 @@
     },
     {
       "id": 32005,
-      "content": "<|function_call|>",
+      "content": "<|placeholder4|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
@@ -104,7 +104,7 @@
     },
     {
       "id": 32008,
-      "content": "<|raw|>",
+      "content": "<|placeholder5|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
@@ -113,7 +113,7 @@
     },
     {
       "id": 32009,
-      "content": "<|continue|>",
+      "content": "<|placeholder6|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": true,
@@ -128,250 +128,7 @@
       "rstrip": true,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 32011,
-      "content": "<|function_list|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32012,
-      "content": "<|calc|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32013,
-      "content": "<|code|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32014,
-      "content": "<|/code|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32015,
-      "content": "<|summary|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32016,
-      "content": "<|resource|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32017,
-      "content": "<|assistant_mask|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32018,
-      "content": "<|start|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32019,
-      "content": "<|message|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32020,
-      "content": "<|fim_prefix|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32021,
-      "content": "<|fim_middle|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32022,
-      "content": "<|fim_suffix|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32023,
-      "content": "<|meta_start|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32024,
-      "content": "<|ipynb_marker|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32025,
-      "content": "<|diff_marker|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32026,
-      "content": "<|ghissue|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32027,
-      "content": "<|ghreview|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32028,
-      "content": "<|disc_start|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32029,
-      "content": "<|disc_sep|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32030,
-      "content": "<|disc_thread|><|query|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32031,
-      "content": "<|/query|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32032,
-      "content": "<|data|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32033,
-      "content": "<|/data|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32034,
-      "content": "<|sys|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32035,
-      "content": "<|/sys|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32036,
-      "content": "<|inst|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32037,
-      "content": "<|/inst|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    }
+    } 
   ],
   "normalizer": {
     "type": "Sequence",
@@ -393,12 +150,6 @@
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
-      {
-        "SpecialToken": {
-          "id": "<s>",
-          "type_id": 0
-        }
-      },
       {
         "Sequence": {
           "id": "A",
@@ -407,24 +158,12 @@
       }
     ],
     "pair": [
-      {
-        "SpecialToken": {
-          "id": "<s>",
-          "type_id": 0
-        }
-      },
       {
         "Sequence": {
           "id": "A",
           "type_id": 0
         }
       },
-      {
-        "SpecialToken": {
-          "id": "<s>",
-          "type_id": 1
-        }
-      },
       {
         "Sequence": {
           "id": "B",
@@ -432,17 +171,7 @@
         }
       }
     ],
-    "special_tokens": {
-      "<s>": {
-        "id": "<s>",
-        "ids": [
-          1
-        ],
-        "tokens": [
-          "<s>"
-        ]
-      }
-    }
+    "special_tokens": {}
   },
   "decoder": {
     "type": "Sequence",

tokenizer_config.json

@@ -127,4 +127,4 @@
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
-}
+}