tokenizer_config.json

{"errors": "replace", "unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "max_len": 1024, "special_tokens_map_file": "/home/nikon/.cache/torch/transformers/2ed126dd981c54d35b6a4739020bca74e2187e0f035bcb84f05de5841e8ffe72.3ae9ae72462581d20e36bc528e9c47bb30cd671bb21add40ca0b24a0be9fac22", "tokenizer_file": null, "name_or_path": "nikokons/gpt2-greek"}