Upload tokenizer
ec946f1
verified
|
{
|
|
"(": 1,
|
|
")": 2,
|
|
"[PAD]": 96,
|
|
"[UNK]": 95,
|
|
"_": 3,
|
|
"a": 4,
|
|
"b": 5,
|
|
"c": 6,
|
|
"d": 7,
|
|
"e": 8,
|
|
"f": 9,
|
|
"g": 10,
|
|
"h": 11,
|
|
"i": 12,
|
|
"j": 13,
|
|
"k": 14,
|
|
"l": 15,
|
|
"m": 16,
|
|
"n": 17,
|
|
"o": 18,
|
|
"p": 19,
|
|
"q": 20,
|
|
"r": 21,
|
|
"s": 22,
|
|
"t": 23,
|
|
"u": 24,
|
|
"v": 25,
|
|
"w": 26,
|
|
"x": 27,
|
|
"y": 28,
|
|
"|": 0,
|
|
"«": 29,
|
|
"»": 30,
|
|
"،": 31,
|
|
"؛": 32,
|
|
"؟": 33,
|
|
"ء": 34,
|
|
"آ": 35,
|
|
"أ": 36,
|
|
"ؤ": 37,
|
|
"إ": 38,
|
|
"ئ": 39,
|
|
"ا": 40,
|
|
"ب": 41,
|
|
"ة": 42,
|
|
"ت": 43,
|
|
"ث": 44,
|
|
"ج": 45,
|
|
"ح": 46,
|
|
"خ": 47,
|
|
"د": 48,
|
|
"ذ": 49,
|
|
"ر": 50,
|
|
"ز": 51,
|
|
"س": 52,
|
|
"ش": 53,
|
|
"ص": 54,
|
|
"ض": 55,
|
|
"ط": 56,
|
|
"ظ": 57,
|
|
"ع": 58,
|
|
"غ": 59,
|
|
"ـ": 60,
|
|
"ف": 61,
|
|
"ق": 62,
|
|
"ك": 63,
|
|
"ل": 64,
|
|
"م": 65,
|
|
"ن": 66,
|
|
"ه": 67,
|
|
"و": 68,
|
|
"ى": 69,
|
|
"ي": 70,
|
|
"ً": 71,
|
|
"ٌ": 72,
|
|
"ٍ": 73,
|
|
"َ": 74,
|
|
"ُ": 75,
|
|
"ِ": 76,
|
|
"ّ": 77,
|
|
"ْ": 78,
|
|
"ٰ": 79,
|
|
"چ": 80,
|
|
"ڨ": 81,
|
|
"ک": 82,
|
|
"ھ": 83,
|
|
"ی": 84,
|
|
"ۖ": 85,
|
|
"ۗ": 86,
|
|
"ۘ": 87,
|
|
"ۚ": 88,
|
|
"ۛ": 89,
|
|
"—": 90,
|
|
"…": 91,
|
|
"☭": 92,
|
|
"ﺃ": 93,
|
|
"ﻻ": 94
|
|
}
|
|
|
