Hugging Face's logo

pszemraj
/
perSLIMmon-8b-base

Text Generation
Transformers
Safetensors
English
persimmon
Inference Endpoints
Model card Files Community
pszemraj commited on
Commit
d763102
•
1 Parent(s): 16e8fde

update eos

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +14 -3
special_tokens_map.json CHANGED
@@ -1,16 +1,27 @@
1
  {
 
 
 
 
2
  "bos_token": {
3
  "content": "|ENDOFTEXT|",
4
  "lstrip": false,
5
- "normalized": true,
 
 
 
 
 
 
 
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "unk_token": {
10
  "content": "<unk>",
11
  "lstrip": false,
12
- "normalized": true,
13
  "rstrip": false,
14
  "single_word": false
15
  }
16
- }
 
1
  {
2
+ "additional_special_tokens": [
3
+ "<unk>",
4
+ "|ENDOFTEXT|"
5
+ ],
6
  "bos_token": {
7
  "content": "|ENDOFTEXT|",
8
  "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false
12
+ },
13
+ "eos_token": {
14
+ "content": "|ENDOFTEXT|",
15
+ "lstrip": false,
16
+ "normalized": false,
17
  "rstrip": false,
18
  "single_word": false
19
  },
20
  "unk_token": {
21
  "content": "<unk>",
22
  "lstrip": false,
23
+ "normalized": false,
24
  "rstrip": false,
25
  "single_word": false
26
  }
27
+ }