Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +3 -3
tokenizer.json
CHANGED
@@ -59,7 +59,7 @@
|
|
59 |
},
|
60 |
{
|
61 |
"id": 128006,
|
62 |
-
"content": "<|
|
63 |
"single_word": false,
|
64 |
"lstrip": false,
|
65 |
"rstrip": false,
|
@@ -86,7 +86,7 @@
|
|
86 |
},
|
87 |
{
|
88 |
"id": 128009,
|
89 |
-
"content": "<|
|
90 |
"single_word": false,
|
91 |
"lstrip": false,
|
92 |
"rstrip": false,
|
@@ -410560,4 +410560,4 @@
|
|
410560 |
"éĶ ¦"
|
410561 |
]
|
410562 |
}
|
410563 |
-
}
|
|
|
59 |
},
|
60 |
{
|
61 |
"id": 128006,
|
62 |
+
"content": "<|im_start|>",
|
63 |
"single_word": false,
|
64 |
"lstrip": false,
|
65 |
"rstrip": false,
|
|
|
86 |
},
|
87 |
{
|
88 |
"id": 128009,
|
89 |
+
"content": "<|im_end|>",
|
90 |
"single_word": false,
|
91 |
"lstrip": false,
|
92 |
"rstrip": false,
|
|
|
410560 |
"éĶ ¦"
|
410561 |
]
|
410562 |
}
|
410563 |
+
}
|