CreitinGameplays commited on
Commit
c3abf6b
1 Parent(s): bf5e89a
Files changed (2) hide show
  1. special_tokens_map.json +10 -22
  2. tokenizer_config.json +1 -1
special_tokens_map.json CHANGED
@@ -1,26 +1,8 @@
1
  {
2
  "additional_special_tokens": [
3
- {
4
- "content": "<|system|>",
5
- "lstrip": false,
6
- "normalized": false,
7
- "rstrip": false,
8
- "single_word": false
9
- },
10
- {
11
- "content": "<|prompter|>",
12
- "lstrip": false,
13
- "normalized": false,
14
- "rstrip": false,
15
- "single_word": false
16
- },
17
- {
18
- "content": "<|assistant|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- }
24
  ],
25
  "bos_token": {
26
  "content": "<s>",
@@ -36,7 +18,13 @@
36
  "rstrip": false,
37
  "single_word": false
38
  },
39
- "pad_token": "<pad>",
 
 
 
 
 
 
40
  "unk_token": {
41
  "content": "<unk>",
42
  "lstrip": false,
 
1
  {
2
  "additional_special_tokens": [
3
+ "<|system|>",
4
+ "<|prompter|>",
5
+ "<|assistant|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  ],
7
  "bos_token": {
8
  "content": "<s>",
 
18
  "rstrip": false,
19
  "single_word": false
20
  },
21
+ "pad_token": {
22
+ "content": "<pad>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
  "unk_token": {
29
  "content": "<unk>",
30
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -65,7 +65,7 @@
65
  "<|assistant|>"
66
  ],
67
  "bos_token": "<s>",
68
- "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|prompter|>' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
69
  "clean_up_tokenization_spaces": false,
70
  "eos_token": "</s>",
71
  "model_max_length": 1000000000000000019884624838656,
 
65
  "<|assistant|>"
66
  ],
67
  "bos_token": "<s>",
68
+ "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '<|prompter|>' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ '<|system|>' + message['content'] + eos_token }}{% elif message['role'] == 'assistant' %}{{ '<|assistant|>' + message['content'] + eos_token }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>' }}{% endif %}{% endfor %}",
69
  "clean_up_tokenization_spaces": false,
70
  "eos_token": "</s>",
71
  "model_max_length": 1000000000000000019884624838656,