vantubk1386's picture
Upload tokenizer
004ff58 verified
raw
history blame
2.32 kB
{
" ": 16,
"!": 5,
"\"": 11,
"'": 176,
",": 3,
".": 4,
":": 2,
";": 1,
"?": 6,
"A": 17,
"B": 18,
"C": 19,
"D": 20,
"E": 21,
"F": 22,
"G": 23,
"H": 24,
"I": 25,
"J": 26,
"K": 27,
"L": 28,
"M": 29,
"N": 30,
"O": 31,
"P": 32,
"Q": 33,
"R": 34,
"S": 35,
"T": 36,
"U": 37,
"V": 38,
"W": 39,
"X": 40,
"Y": 41,
"Z": 42,
"_": 0,
"a": 43,
"b": 44,
"c": 45,
"d": 46,
"e": 47,
"f": 48,
"g": 49,
"h": 50,
"i": 51,
"j": 52,
"k": 53,
"l": 54,
"m": 55,
"n": 56,
"o": 57,
"p": 58,
"q": 59,
"r": 60,
"s": 61,
"t": 62,
"u": 63,
"v": 64,
"w": 65,
"x": 66,
"y": 67,
"z": 68,
"¡": 7,
"«": 12,
"»": 13,
"¿": 8,
"æ": 72,
"ç": 78,
"ð": 81,
"ø": 116,
"ħ": 98,
"ŋ": 112,
"œ": 120,
"ǀ": 152,
"ǁ": 153,
"ǂ": 154,
"ǃ": 155,
"ɐ": 70,
"ɑ": 69,
"ɒ": 71,
"ɓ": 73,
"ɔ": 76,
"ɕ": 77,
"ɖ": 80,
"ɗ": 79,
"ɘ": 84,
"ə": 83,
"ɚ": 85,
"ɛ": 86,
"ɜ": 87,
"ɝ": 88,
"ɞ": 89,
"ɟ": 90,
"ɠ": 93,
"ɡ": 92,
"ɢ": 94,
"ɣ": 139,
"ɤ": 140,
"ɥ": 99,
"ɦ": 96,
"ɧ": 97,
"ɨ": 101,
"ɪ": 102,
"ɫ": 106,
"ɬ": 105,
"ɭ": 104,
"ɮ": 107,
"ɯ": 110,
"ɰ": 111,
"ɱ": 109,
"ɲ": 114,
"ɳ": 113,
"ɴ": 115,
"ɵ": 117,
"ɶ": 121,
"ɸ": 118,
"ɹ": 123,
"ɺ": 124,
"ɻ": 126,
"ɽ": 129,
"ɾ": 125,
"ʀ": 127,
"ʁ": 128,
"ʂ": 130,
"ʃ": 131,
"ʄ": 91,
"ʈ": 132,
"ʉ": 134,
"ʊ": 135,
"ʋ": 136,
"ʌ": 138,
"ʍ": 141,
"ʎ": 143,
"ʏ": 144,
"ʐ": 146,
"ʑ": 145,
"ʒ": 147,
"ʔ": 148,
"ʕ": 150,
"ʘ": 122,
"ʙ": 74,
"ʛ": 95,
"ʜ": 100,
"ʝ": 103,
"ʟ": 108,
"ʡ": 149,
"ʢ": 151,
"ʤ": 82,
"ʧ": 133,
"ʰ": 162,
"ʱ": 163,
"ʲ": 164,
"ʴ": 161,
"ʷ": 165,
"ʼ": 160,
"ˈ": 156,
"ˌ": 157,
"ː": 158,
"ˑ": 159,
"˞": 168,
"ˠ": 166,
"ˤ": 167,
"̩": 175,
"β": 75,
"θ": 119,
"χ": 142,
"ᵻ": 177,
"—": 9,
"“": 14,
"”": 15,
"…": 10,
"↑": 170,
"→": 171,
"↓": 169,
"↗": 172,
"↘": 173,
"ⱱ": 137
}