Hugging Face's logo

momo
/
7B-safety-lora

Safetensors
Model card Files Community
momo commited on
Commit
9898012
·
1 Parent(s): 38cd8a0

remove_tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +0 -24
  2. tokenizer.json +0 -0
  3. tokenizer_config.json +0 -98
special_tokens_map.json DELETED
@@ -1,24 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<s>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "</s>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": {
17
- "content": "</s>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- },
23
- "unk_token": "<unk>"
24
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tokenizer.json DELETED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json DELETED
@@ -1,98 +0,0 @@
1
- {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "add_prefix_space": null,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<unk>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
- },
14
- "1": {
15
- "content": "<s>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "2": {
23
- "content": "</s>",
24
- "lstrip": false,
25
- "normalized": false,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- },
30
- "32000": {
31
- "content": "<|tel|>",
32
- "lstrip": false,
33
- "normalized": false,
34
- "rstrip": false,
35
- "single_word": false,
36
- "special": false
37
- },
38
- "32001": {
39
- "content": "<|rnn|>",
40
- "lstrip": false,
41
- "normalized": false,
42
- "rstrip": false,
43
- "single_word": false,
44
- "special": false
45
- },
46
- "32002": {
47
- "content": "<|email|>",
48
- "lstrip": false,
49
- "normalized": false,
50
- "rstrip": false,
51
- "single_word": false,
52
- "special": false
53
- },
54
- "32003": {
55
- "content": "<|crd|>",
56
- "lstrip": false,
57
- "normalized": false,
58
- "rstrip": false,
59
- "single_word": false,
60
- "special": false
61
- },
62
- "32004": {
63
- "content": "<|acc|>",
64
- "lstrip": false,
65
- "normalized": false,
66
- "rstrip": false,
67
- "single_word": false,
68
- "special": false
69
- },
70
- "51510": {
71
- "content": "<|im_start|>",
72
- "lstrip": false,
73
- "normalized": false,
74
- "rstrip": false,
75
- "single_word": false,
76
- "special": false
77
- },
78
- "51511": {
79
- "content": "<|im_end|>",
80
- "lstrip": false,
81
- "normalized": false,
82
- "rstrip": false,
83
- "single_word": false,
84
- "special": false
85
- }
86
- },
87
- "bos_token": "<s>",
88
- "chat_template": "{{('<|im_start|>assistant\n' + messages[0]['content'].strip() + '<|im_end|>' + '\n' if messages[0]['role'] == 'system' else '')}}{% for message in (messages[1:] if messages[0]['role'] == 'system' else messages) %}{% if message['role'] == 'user' %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>' + '\n' + '<|im_start|>assistant\n'}}{% elif message['role'] == 'assistant' %}{{message['content']}}{% if loop.last %}{% if add_generation_prompt %}{{'<|im_end|>'}}{% endif %}{% else %}{{'<|im_end|>'}}{% endif %}{% endif %}{% endfor %}",
89
- "clean_up_tokenization_spaces": true,
90
- "eos_token": "</s>",
91
- "legacy": true,
92
- "model_max_length": 1000000000000000019884624838656,
93
- "model_name": "kylin-7b",
94
- "pad_token": "</s>",
95
- "tokenizer_class": "LlamaTokenizer",
96
- "unk_token": "<unk>",
97
- "use_default_system_prompt": false
98
- }