Hugging Face's logo

NousResearch
/
Meta-Llama-3-8B-Alternate-Tokenizer

Text Generation
Transformers
Safetensors
PyTorch
English
llama
facebook
meta
llama-3
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
2
teknium commited on
Commit
9fe5e73
·
verified ·
1 Parent(s): 576f17d

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +13 -13
tokenizer.json CHANGED
@@ -23,7 +23,7 @@
23
  },
24
  {
25
  "id": 128002,
26
- "content": "<|reserved_special_token_0|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
@@ -32,7 +32,7 @@
32
  },
33
  {
34
  "id": 128003,
35
- "content": "<|reserved_special_token_1|>",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
@@ -41,12 +41,12 @@
41
  },
42
  {
43
  "id": 128004,
44
- "content": "<|reserved_special_token_2|>",
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
- "special": true
50
  },
51
  {
52
  "id": 128005,
@@ -55,7 +55,7 @@
55
  "lstrip": false,
56
  "rstrip": false,
57
  "normalized": false,
58
- "special": true
59
  },
60
  {
61
  "id": 128006,
@@ -77,12 +77,12 @@
77
  },
78
  {
79
  "id": 128008,
80
- "content": "<|reserved_special_token_4|>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
84
  "normalized": false,
85
- "special": true
86
  },
87
  {
88
  "id": 128009,
@@ -95,30 +95,30 @@
95
  },
96
  {
97
  "id": 128010,
98
- "content": "<|reserved_special_token_5|>",
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
102
  "normalized": false,
103
- "special": true
104
  },
105
  {
106
  "id": 128011,
107
- "content": "<|reserved_special_token_6|>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
- "special": true
113
  },
114
  {
115
  "id": 128012,
116
- "content": "<|reserved_special_token_7|>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
- "special": true
122
  },
123
  {
124
  "id": 128013,
 
23
  },
24
  {
25
  "id": 128002,
26
+ "content": "<|im_start|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
 
32
  },
33
  {
34
  "id": 128003,
35
+ "content": "<|im_end|>",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
 
41
  },
42
  {
43
  "id": 128004,
44
+ "content": "<tool_call>",
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
+ "special": false
50
  },
51
  {
52
  "id": 128005,
 
55
  "lstrip": false,
56
  "rstrip": false,
57
  "normalized": false,
58
+ "special": false
59
  },
60
  {
61
  "id": 128006,
 
77
  },
78
  {
79
  "id": 128008,
80
+ "content": "<tools>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
84
  "normalized": false,
85
+ "special": false
86
  },
87
  {
88
  "id": 128009,
 
95
  },
96
  {
97
  "id": 128010,
98
+ "content": "</tools>",
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
102
  "normalized": false,
103
+ "special": false
104
  },
105
  {
106
  "id": 128011,
107
+ "content": "</tool_call>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
+ "special": false
113
  },
114
  {
115
  "id": 128012,
116
+ "content": "</tool_response>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
+ "special": false
122
  },
123
  {
124
  "id": 128013,