tokenizer.json

{
 "version": "1.0",
 "truncation": null,
 "padding": null,
 "added_tokens": [
 {
 "id": 0,
 "content": "<|endoftext|>",
 "single_word": false,
 "lstrip": false,
 "rstrip": false,
 "normalized": false,
 "special": true
 }
 ],
 "normalizer": null,
 "pre_tokenizer": {
 "type": "ByteLevel",
 "add_prefix_space": false,
 "trim_offsets": true,
 "use_regex": true
 },
 "post_processor": {
 "type": "ByteLevel",
 "add_prefix_space": true,
 "trim_offsets": true,
 "use_regex": true
 },
 "decoder": null,
 "model": {
 "type": "BPE",
 "dropout": null,
 "unk_token": "<|endoftext|>",
 "continuing_subword_prefix": null,
 "end_of_word_suffix": null,
 "fuse_unk": false,
 "byte_fallback": false,
 "ignore_merges": false,
 "vocab": {
 "<|endoftext|>": 0,
 "\u0000": 1,
 "\u0001": 2,
 "\u0002": 3,
 "\u0003": 4,
 "\u0004": 5,
 "\u0005": 6,
 "\u0006": 7,
 "\u0007": 8,
 "\b": 9,
 "\t": 10,
 "\n": 11,
 "\u000b": 12,
 "\f": 13,
 "\r": 14,
 "\u000e": 15,
 "\u000f": 16,
 "\u0010": 17,
 "\u0011": 18,
 "\u0012": 19,
 "\u0013": 20,
 "\u0014": 21,
 "\u0015": 22,
 "\u0016": 23,
 "\u0017": 24,
 "\u0018": 25,
 "\u0019": 26,
 "\u001a": 27,
 "\u001b": 28,
 "\u001c": 29,
 "\u001d": 30,
 "\u001e": 31,
 "\u001f": 32,
 " ": 33,
 "!": 34,
 "\"": 35,
 "#": 36,
 "$": 37,
 "%": 38,
 "&": 39,
 "'": 40,
 "(": 41,
 ")": 42,
 "*": 43,
 "+": 44,
 ",": 45,
 "-": 46,
 ".": 47,
 "/": 48,
 "0": 49,
 "1": 50,
 "2": 51,
 "3": 52,
 "4": 53,
 "5": 54,
 "6": 55,
 "7": 56,
 "8": 57,
 "9": 58,
 ":": 59,
 ";": 60,
 "<": 61,
 "=": 62,
 ">": 63,
 "?": 64,
 "@": 65,
 "A": 66,
 "B": 67,
 "C": 68,
 "D": 69,
 "E": 70,
 "F": 71,
 "G": 72,
 "H": 73,
 "I": 74,
 "J": 75,
 "K": 76,
 "L": 77,
 "M": 78,
 "N": 79,
 "O": 80,
 "P": 81,
 "Q": 82,
 "R": 83,
 "S": 84,
 "T": 85,
 "U": 86,
 "V": 87,
 "W": 88,
 "X": 89,
 "Y": 90,
 "Z": 91,
 "[": 92,
 "\\": 93,
 "]": 94,
 "^": 95,
 "_": 96,
 "`": 97,
 "a": 98,
 "b": 99,
 "c": 100,
 "d": 101,
 "e": 102,
 "f": 103,
 "g": 104,
 "h": 105,
 "i": 106,
 "j": 107,
 "k": 108,
 "l": 109,
 "m": 110,
 "n": 111,
 "o": 112,
 "p": 113,
 "q": 114,
 "r": 115,
 "s": 116,
 "t": 117,
 "u": 118,
 "v": 119,
 "w": 120,
 "x": 121,
 "y": 122,
 "z": 123,
 "{": 124,
 "|": 125,
 "}": 126,
 "~": 127,
 "": 128,
 "€": 129,
 "": 130,
 "‚": 131,
 "ƒ": 132,
 "„": 133,
 "…": 134,
 "†": 135,
 "‡": 136,
 "ˆ": 137,
 "‰": 138,
 "Š": 139,
 "‹": 140,
 "Œ": 141,
 "": 142,
 "Ž": 143,
 "": 144,
 "": 145,
 "‘": 146,
 "’": 147,
 "“": 148,
 "”": 149,
 "•": 150,
 "–": 151,
 "—": 152,
 "˜": 153,
 "™": 154,
 "š": 155,
 "›": 156,
 "œ": 157,
 "": 158,
 "ž": 159,
 "Ÿ": 160,
 " ": 161,
 "¡": 162,
 "¢": 163,
 "£": 164,
 "¤": 165,
 "¥": 166,
 "¦": 167,
 "§": 168,
 "¨": 169,
 "©": 170,
 "ª": 171,
 "«": 172,
 "¬": 173,
 "­": 174,
 "®": 175,
 "¯": 176,
 "°": 177,
 "±": 178,
 "²": 179,
 "³": 180,
 "´": 181,
 "µ": 182,
 "¶": 183,
 "·": 184,
 "¸": 185,
 "¹": 186,
 "º": 187,
 "»": 188,
 "¼": 189,
 "½": 190,
 "¾": 191,
 "¿": 192,
 "À": 193,
 "Á": 194,
 "Â": 195,
 "Ã": 196,
 "Ä": 197,
 "Å": 198,
 "Æ": 199,
 "Ç": 200,
 "È": 201,
 "É": 202,
 "Ê": 203,
 "Ë": 204,
 "Ì": 205,
 "Í": 206,
 "Î": 207,
 "Ï": 208,
 "Ð": 209,
 "Ñ": 210,
 "Ò": 211,
 "Ó": 212,
 "Ô": 213,
 "Õ": 214,
 "Ö": 215,
 "×": 216,
 "Ø": 217,
 "Ù": 218,
 "Ú": 219,
 "Û": 220,
 "Ü": 221,
 "Ý": 222,
 "Þ": 223,
 "ß": 224,
 "à": 225,
 "á": 226,
 "â": 227,
 "ã": 228,
 "ä": 229,
 "å": 230,
 "æ": 231,
 "ç": 232,
 "è": 233,
 "é": 234,
 "ê": 235,
 "ë": 236,
 "ì": 237,
 "í": 238,
 "î": 239,
 "ï": 240,
 "ð": 241,
 "ñ": 242,
 "ò": 243,
 "ó": 244,
 "ô": 245,
 "õ": 246,
 "ö": 247,
 "÷": 248,
 "ø": 249,
 "ù": 250,
 "ú": 251,
 "û": 252,
 "ü": 253,
 "ý": 254,
 "þ": 255,
 "ÿ": 256,
 "Ā": 257,
 "ā": 258,
 "Ă": 259,
 "ă": 260,
 "Ą": 261,
 "ą": 262,
 "Ć": 263
 },
 "merges": []
 }
}