pszemraj commited on
Commit
dfe1d58
·
1 Parent(s): 0cdda3c

add fast tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -41,5 +41,5 @@
41
  "unk_token": "<unk>",
42
  "use_default_system_prompt": true,
43
  "use_fast": true,
44
- "use_fast_tokenizer": false
45
  }
 
41
  "unk_token": "<unk>",
42
  "use_default_system_prompt": true,
43
  "use_fast": true,
44
+ "use_fast_tokenizer": true
45
  }