danielhanchen
commited on
Commit
•
a834080
1
Parent(s):
802b5cc
Fix pad
Browse files- tokenizer_config.json +44 -3
tokenizer_config.json
CHANGED
@@ -1,3 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
{
|
2 |
"add_bos_token": true,
|
3 |
"add_eos_token": false,
|
@@ -39,12 +80,12 @@
|
|
39 |
"clean_up_tokenization_spaces": false,
|
40 |
"eos_token": "<eos>",
|
41 |
"legacy": null,
|
42 |
-
"model_max_length":
|
43 |
-
"pad_token": "<
|
44 |
-
"padding_side": "right",
|
45 |
"sp_model_kwargs": {},
|
46 |
"spaces_between_special_tokens": false,
|
47 |
"tokenizer_class": "GemmaTokenizer",
|
48 |
"unk_token": "<unk>",
|
49 |
"use_default_system_prompt": false
|
50 |
}
|
|
|
|
1 |
+
|
2 |
+
Hugging Face's logo Hugging Face
|
3 |
+
|
4 |
+
Models
|
5 |
+
Datasets
|
6 |
+
Spaces
|
7 |
+
Posts
|
8 |
+
Docs
|
9 |
+
Pricing
|
10 |
+
|
11 |
+
google
|
12 |
+
/
|
13 |
+
gemma-2b
|
14 |
+
Text Generation
|
15 |
+
Transformers
|
16 |
+
Safetensors
|
17 |
+
GGUF
|
18 |
+
gemma
|
19 |
+
Inference Endpoints
|
20 |
+
text-generation-inference
|
21 |
+
23 papers
|
22 |
+
Model card
|
23 |
+
Files and versions
|
24 |
+
Community
|
25 |
+
13
|
26 |
+
gemma-2b
|
27 |
+
/ tokenizer_config.json
|
28 |
+
osanseviero's picture
|
29 |
+
osanseviero
|
30 |
+
HF staff
|
31 |
+
Squashing commit
|
32 |
+
5e8e4ea
|
33 |
+
verified
|
34 |
+
about 20 hours ago
|
35 |
+
raw
|
36 |
+
history
|
37 |
+
blame
|
38 |
+
contribute
|
39 |
+
delete
|
40 |
+
No virus
|
41 |
+
1.11 kB
|
42 |
{
|
43 |
"add_bos_token": true,
|
44 |
"add_eos_token": false,
|
|
|
80 |
"clean_up_tokenization_spaces": false,
|
81 |
"eos_token": "<eos>",
|
82 |
"legacy": null,
|
83 |
+
"model_max_length": 1000000000000000019884624838656,
|
84 |
+
"pad_token": "<pad>",
|
|
|
85 |
"sp_model_kwargs": {},
|
86 |
"spaces_between_special_tokens": false,
|
87 |
"tokenizer_class": "GemmaTokenizer",
|
88 |
"unk_token": "<unk>",
|
89 |
"use_default_system_prompt": false
|
90 |
}
|
91 |
+
|