tokenizer.json

{
    "model_type": "bert",
    "vocab_size": 32000,
    "type_vocab_size": 2,
    "padding_id": 0,
    "unk_id": 1,
    "cls_id": 2,
    "sep_id": 3,
    "mask_id": 4,
    "normalizer": null,
    "pretokenizer": {
        "type": "ByteLevel",
        "add_prefix_space": true
    },
    "postprocessor": {
        "type": "TemplateProcessing",
        "single": {
            "template": "{input} </s>",
            "special_tokens": [
                {
                    "id": 2,
                    "content": "</s>"
                }
            ]
        },
        "pair": {
            "template": "{pair} </s>",
            "special_tokens": [
                {
                    "id": 2,
                    "content": "</s>"
                }
            ]
        }
    }
}