tokenizer_config.json

{
 "add_prefix_space": false,
 "additional_special_tokens": [
 "<present>",
 "<absent>",
 "<category>",
 "<infill>",
 "<seealso>",
 "<header>",
 "<|endoftext|>",
 "<sep>",
 "<mask>",
 "<mixed>",
 "<number>",
 "<phrase>"
 ],
 "bos_token": {
 "__type": "AddedToken",
 "content": "<s>",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false
 },
 "cls_token": {
 "__type": "AddedToken",
 "content": "<s>",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false
 },
 "eos_token": {
 "__type": "AddedToken",
 "content": "</s>",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false
 },
 "errors": "replace",
 "mask_token": {
 "__type": "AddedToken",
 "content": "<mask>",
 "lstrip": true,
 "normalized": true,
 "rstrip": false,
 "single_word": false
 },
 "model_max_length": 512,
 "name_or_path": "memray/bart_wikikp",
 "pad_token": {
 "__type": "AddedToken",
 "content": "<pad>",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false
 },
 "sep": "<sep>",
 "sep_token": "<sep>",
 "special_tokens_map_file": "/home/ashraf.haddad/.cache/huggingface/transformers/4b7a3619321a39f6780e0c775802e3523e52c1efd3a46e6d1baac9e1e8e234e6.898eb95aac9bb57440c2f57caa963ae18b9b10ba4731cc81020283286b0391fc",
 "tokenizer_class": "BartTokenizer",
 "trim_offsets": true,
 "unk_token": {
 "__type": "AddedToken",
 "content": "<unk>",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false
 }
}