adkhamboy's picture
Upload tokenizer
b672dca
raw
history blame
1.22 kB
{
"&": 1,
"[PAD]": 102,
"[UNK]": 101,
"\\": 2,
"_": 3,
"`": 4,
"a": 5,
"b": 6,
"c": 7,
"d": 8,
"e": 9,
"f": 10,
"g": 11,
"h": 12,
"i": 13,
"j": 14,
"k": 15,
"l": 16,
"m": 17,
"n": 18,
"o": 19,
"p": 20,
"q": 21,
"r": 22,
"s": 23,
"t": 24,
"u": 25,
"v": 26,
"w": 27,
"x": 28,
"y": 29,
"z": 30,
"|": 0,
"©": 32,
"«": 33,
"¬": 34,
"´": 35,
"µ": 36,
"»": 37,
"ë": 38,
"ò": 39,
"ó": 40,
"õ": 41,
"ü": 42,
"ģ": 43,
"ı": 44,
"ş": 45,
"ƣ": 46,
"ƶ": 47,
"ʙ": 48,
"ʻ": 49,
"ʼ": 50,
"а": 51,
"б": 52,
"в": 53,
"г": 54,
"д": 55,
"е": 56,
"ж": 57,
"з": 58,
"и": 59,
"й": 60,
"к": 61,
"л": 62,
"м": 63,
"н": 64,
"о": 65,
"п": 66,
"р": 67,
"с": 68,
"т": 69,
"у": 70,
"ф": 71,
"х": 72,
"ч": 73,
"ш": 74,
"ъ": 75,
"ы": 76,
"ь": 77,
"э": 78,
"ю": 79,
"ё": 80,
"ј": 81,
"ў": 82,
"ғ": 83,
"қ": 84,
"ҳ": 85,
"ӯ": 86,
"ֹ": 87,
"‒": 88,
"–": 89,
"—": 90,
"―": 91,
"’": 92,
"„": 93,
"•": 94,
"…": 95,
"−": 96,
"☺": 97,
"♂": 98,
"✅": 99,
"✔": 100,
"️": 101
}