Hugging Face's logo

M-A-E
/
TinyLlama-1.1B-Chat-v0.1-hotel_data

Text Generation
Transformers
PyTorch
English
llama
text-generation-inference
unsloth
trl
sft
Inference Endpoints
Model card Files Community
M-A-E commited on
Commit
529458c
·
verified ·
1 Parent(s): a0962c0

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +28 -4
  2. tokenizer.json +8 -3
  3. tokenizer_config.json +7 -6
special_tokens_map.json CHANGED
@@ -1,6 +1,30 @@
1
  {
2
- "bos_token": "<s>",
3
- "eos_token": "<|im_end|>",
4
- "pad_token": "[PAD]",
5
- "unk_token": "<unk>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  }
 
1
  {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "[PAD]",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
  }
tokenizer.json CHANGED
@@ -1,6 +1,11 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
@@ -23,7 +28,7 @@
23
  },
24
  {
25
  "id": 2,
26
- "content": "<|im_end|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
@@ -147,7 +152,7 @@
147
  "vocab": {
148
  "<unk>": 0,
149
  "<s>": 1,
150
- "<|im_end|>": 2,
151
  "<0x00>": 3,
152
  "<0x01>": 4,
153
  "<0x02>": 5,
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 2048,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
  "padding": null,
10
  "added_tokens": [
11
  {
 
28
  },
29
  {
30
  "id": 2,
31
+ "content": "</s>",
32
  "single_word": false,
33
  "lstrip": false,
34
  "rstrip": false,
 
152
  "vocab": {
153
  "<unk>": 0,
154
  "<s>": 1,
155
+ "</s>": 2,
156
  "<0x00>": 3,
157
  "<0x01>": 4,
158
  "<0x02>": 5,
tokenizer_config.json CHANGED
@@ -19,7 +19,7 @@
19
  "special": true
20
  },
21
  "2": {
22
- "content": "<|im_end|>",
23
  "lstrip": false,
24
  "normalized": false,
25
  "rstrip": false,
@@ -36,13 +36,14 @@
36
  }
37
  },
38
  "bos_token": "<s>",
39
- "chat_template": "{% for message in messages %}{% if message['from'] == 'human' %}{{'<|im_start|>user\n' + message['value'] + '<|im_end|>\n'}}{% elif message['from'] == 'gpt' %}{{'<|im_start|>assistant\n' + message['value'] + '<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['value'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
40
  "clean_up_tokenization_spaces": false,
41
- "eos_token": "<|im_end|>",
42
- "legacy": true,
43
- "model_max_length": 1000000000000000019884624838656,
44
  "pad_token": "[PAD]",
 
 
45
  "tokenizer_class": "LlamaTokenizer",
46
  "unk_token": "<unk>",
47
- "use_default_system_prompt": false
48
  }
 
19
  "special": true
20
  },
21
  "2": {
22
+ "content": "</s>",
23
  "lstrip": false,
24
  "normalized": false,
25
  "rstrip": false,
 
36
  }
37
  },
38
  "bos_token": "<s>",
 
39
  "clean_up_tokenization_spaces": false,
40
+ "eos_token": "</s>",
41
+ "legacy": false,
42
+ "model_max_length": 2048,
43
  "pad_token": "[PAD]",
44
+ "padding_side": "left",
45
+ "sp_model_kwargs": {},
46
  "tokenizer_class": "LlamaTokenizer",
47
  "unk_token": "<unk>",
48
+ "use_default_system_prompt": true
49
  }