Hugging Face's logo

SmallDoge
/
Doge-20M

Text Generation
Transformers
Safetensors
English
doge
conversational
custom_code
Model card Files Community
1
JingzeShi commited on
Commit
405f5ad
·
verified ·
1 Parent(s): 8c9adc3

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +8 -8
  2. tokenizer_config.json +4 -4
tokenizer.json CHANGED
@@ -77,7 +77,7 @@
77
  },
78
  {
79
  "id": 8,
80
- "content": "<think>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
@@ -86,7 +86,7 @@
86
  },
87
  {
88
  "id": 9,
89
- "content": "</think>",
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
@@ -95,7 +95,7 @@
95
  },
96
  {
97
  "id": 10,
98
- "content": "<answer>",
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
@@ -104,7 +104,7 @@
104
  },
105
  {
106
  "id": 11,
107
- "content": "</answer>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
@@ -2417,10 +2417,10 @@
2417
  "<|eom_id|>": 5,
2418
  "<|eot_id|>": 6,
2419
  "<|python_tag|>": 7,
2420
- "<think>": 8,
2421
- "</think>": 9,
2422
- "<answer>": 10,
2423
- "</answer>": 11,
2424
  "<|reserved_special_token_0|>": 12,
2425
  "<|reserved_special_token_1|>": 13,
2426
  "<|reserved_special_token_2|>": 14,
 
77
  },
78
  {
79
  "id": 8,
80
+ "content": "<|begin_of_thought|>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
 
86
  },
87
  {
88
  "id": 9,
89
+ "content": "<|end_of_thought|>",
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
 
95
  },
96
  {
97
  "id": 10,
98
+ "content": "<|begin_of_solution|>",
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
 
104
  },
105
  {
106
  "id": 11,
107
+ "content": "<|end_of_solution|>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
 
2417
  "<|eom_id|>": 5,
2418
  "<|eot_id|>": 6,
2419
  "<|python_tag|>": 7,
2420
+ "<|begin_of_thought|>": 8,
2421
+ "<|end_of_thought|>": 9,
2422
+ "<|begin_of_solution|>": 10,
2423
+ "<|end_of_solution|>": 11,
2424
  "<|reserved_special_token_0|>": 12,
2425
  "<|reserved_special_token_1|>": 13,
2426
  "<|reserved_special_token_2|>": 14,
tokenizer_config.json CHANGED
@@ -65,7 +65,7 @@
65
  "special": true
66
  },
67
  "8": {
68
- "content": "<think>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
@@ -73,7 +73,7 @@
73
  "special": true
74
  },
75
  "9": {
76
- "content": "</think>",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
@@ -81,7 +81,7 @@
81
  "special": true
82
  },
83
  "10": {
84
- "content": "<answer>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
@@ -89,7 +89,7 @@
89
  "special": true
90
  },
91
  "11": {
92
- "content": "</answer>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
 
65
  "special": true
66
  },
67
  "8": {
68
+ "content": "<|begin_of_thought|>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
 
73
  "special": true
74
  },
75
  "9": {
76
+ "content": "<|end_of_thought|>",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
 
81
  "special": true
82
  },
83
  "10": {
84
+ "content": "<|begin_of_solution|>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
 
89
  "special": true
90
  },
91
  "11": {
92
+ "content": "<|end_of_solution|>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,