Safetensors
llama
pchizhov commited on
Commit
1a44a71
·
verified ·
1 Parent(s): 90a928c

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +5 -1
tokenizer_config.json CHANGED
@@ -271,5 +271,9 @@
271
  ],
272
  "clean_up_tokenization_spaces": true,
273
  "model_max_length": 1000000000000000019884624838656,
274
- "tokenizer_class": "PreTrainedTokenizerFast"
 
 
 
 
275
  }
 
271
  ],
272
  "clean_up_tokenization_spaces": true,
273
  "model_max_length": 1000000000000000019884624838656,
274
+ "tokenizer_class": "PreTrainedTokenizerFast",
275
+ "bos_token": "<|end_of_text|>",
276
+ "eos_token": "<|end_of_text|>",
277
+ "unk_token": "[UNK]",
278
+ "vocab_size": 65536
279
  }