teknium TheBloke commited on
Commit
7df4ea5
1 Parent(s): 8b0bf33

Mistral tokenizer_config.json fix (#1)

Browse files

- Mistral tokenizer_config.json fix (e0316eb32781334f9543d0b15aab28261967dd49)


Co-authored-by: Tom Jobbins <[email protected]>

Files changed (1) hide show
  1. tokenizer_config.json +6 -8
tokenizer_config.json CHANGED
@@ -4,25 +4,25 @@
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
7
- "lstrip": true,
8
  "normalized": false,
9
- "rstrip": true,
10
  "single_word": false,
11
  "special": true
12
  },
13
  "1": {
14
  "content": "<s>",
15
- "lstrip": true,
16
  "normalized": false,
17
- "rstrip": true,
18
  "single_word": false,
19
  "special": true
20
  },
21
  "2": {
22
  "content": "</s>",
23
- "lstrip": true,
24
  "normalized": false,
25
- "rstrip": true,
26
  "single_word": false,
27
  "special": true
28
  }
@@ -37,9 +37,7 @@
37
  "sp_model_kwargs": {},
38
  "spaces_between_special_tokens": false,
39
  "tokenizer_class": "LlamaTokenizer",
40
- "tokenizer_file": "/home/teknium/.cache/huggingface/hub/models--mistralai--Mistral-7B-v0.1/snapshots/ae9d75c6b4eb39515def78c685fb4d71d49fc2cf/tokenizer.json",
41
  "trust_remote_code": false,
42
  "unk_token": "<unk>",
43
  "use_default_system_prompt": true,
44
- "use_fast": true
45
  }
 
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
7
+ "lstrip": false,
8
  "normalized": false,
9
+ "rstrip": false,
10
  "single_word": false,
11
  "special": true
12
  },
13
  "1": {
14
  "content": "<s>",
15
+ "lstrip": false,
16
  "normalized": false,
17
+ "rstrip": false,
18
  "single_word": false,
19
  "special": true
20
  },
21
  "2": {
22
  "content": "</s>",
23
+ "lstrip": false,
24
  "normalized": false,
25
+ "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
  }
 
37
  "sp_model_kwargs": {},
38
  "spaces_between_special_tokens": false,
39
  "tokenizer_class": "LlamaTokenizer",
 
40
  "trust_remote_code": false,
41
  "unk_token": "<unk>",
42
  "use_default_system_prompt": true,
 
43
  }