Sri3010's picture
Upload tokenizer
356578f verified
raw
history blame
1.16 kB
{
"&": 1,
"(": 2,
")": 3,
"[PAD]": 94,
"[UNK]": 93,
"\\": 4,
"_": 5,
"`": 6,
"a": 7,
"b": 8,
"c": 9,
"d": 10,
"e": 11,
"f": 12,
"g": 13,
"h": 14,
"i": 15,
"j": 16,
"k": 17,
"l": 18,
"m": 19,
"n": 20,
"o": 21,
"p": 22,
"q": 23,
"r": 24,
"s": 25,
"t": 26,
"u": 27,
"v": 28,
"w": 29,
"x": 30,
"y": 31,
"z": 32,
"|": 0,
"·": 33,
"½": 34,
"¾": 35,
"á": 36,
"é": 37,
"ô": 38,
"š": 39,
"அ": 40,
"ஆ": 41,
"இ": 42,
"ஈ": 43,
"உ": 44,
"ஊ": 45,
"எ": 46,
"ஏ": 47,
"ஐ": 48,
"ஒ": 49,
"ஓ": 50,
"ஔ": 51,
"க": 52,
"ங": 53,
"ச": 54,
"ஞ": 55,
"ட": 56,
"ண": 57,
"த": 58,
"ந": 59,
"ன": 60,
"ப": 61,
"ம": 62,
"ய": 63,
"ர": 64,
"ற": 65,
"ல": 66,
"ள": 67,
"ழ": 68,
"வ": 69,
"ஷ": 70,
"ா": 71,
"ி": 72,
"ீ": 73,
"ு": 74,
"ூ": 75,
"ெ": 76,
"ே": 77,
"ை": 78,
"ொ": 79,
"ோ": 80,
"்": 81,
"ௗ": 82,
"ഥ": 83,
"–": 84,
"—": 85,
"’": 86,
"‚": 87,
"•": 88,
"…": 89,
"″": 90,
"●": 91,
"◯": 92
}