floriangardin commited on
Commit
51b4b9b
1 Parent(s): 0b985ed

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -9,6 +9,7 @@
9
  "rstrip": false,
10
  "single_word": false
11
  },
 
12
  "eos_token": {
13
  "__type": "AddedToken",
14
  "content": "<|endoftext|>",
@@ -20,7 +21,6 @@
20
  "errors": "replace",
21
  "model_max_length": 1000000000000000019884624838656,
22
  "pad_token": null,
23
- "special_tokens_map_file": null,
24
  "tokenizer_class": "GPT2Tokenizer",
25
  "unk_token": {
26
  "__type": "AddedToken",
 
9
  "rstrip": false,
10
  "single_word": false
11
  },
12
+ "clean_up_tokenization_spaces": true,
13
  "eos_token": {
14
  "__type": "AddedToken",
15
  "content": "<|endoftext|>",
 
21
  "errors": "replace",
22
  "model_max_length": 1000000000000000019884624838656,
23
  "pad_token": null,
 
24
  "tokenizer_class": "GPT2Tokenizer",
25
  "unk_token": {
26
  "__type": "AddedToken",