Upload tokenizer
4ef6074
verified
|
{ |
|
"__af__": 128004, |
|
"__am__": 128005, |
|
"__ar__": 128006, |
|
"__ast__": 128007, |
|
"__az__": 128008, |
|
"__ba__": 128009, |
|
"__be__": 128010, |
|
"__bg__": 128011, |
|
"__bn__": 128012, |
|
"__br__": 128013, |
|
"__bs__": 128014, |
|
"__ca__": 128015, |
|
"__ceb__": 128016, |
|
"__cs__": 128017, |
|
"__cy__": 128018, |
|
"__da__": 128019, |
|
"__de__": 128020, |
|
"__el__": 128021, |
|
"__en__": 128022, |
|
"__es__": 128023, |
|
"__et__": 128024, |
|
"__fa__": 128025, |
|
"__ff__": 128026, |
|
"__fi__": 128027, |
|
"__fr__": 128028, |
|
"__fy__": 128029, |
|
"__ga__": 128030, |
|
"__gd__": 128031, |
|
"__gl__": 128032, |
|
"__gu__": 128033, |
|
"__ha__": 128034, |
|
"__he__": 128035, |
|
"__hi__": 128036, |
|
"__hr__": 128037, |
|
"__ht__": 128038, |
|
"__hu__": 128039, |
|
"__hy__": 128040, |
|
"__id__": 128041, |
|
"__ig__": 128042, |
|
"__ilo__": 128043, |
|
"__is__": 128044, |
|
"__it__": 128045, |
|
"__ja__": 128046, |
|
"__jv__": 128047, |
|
"__ka__": 128048, |
|
"__kk__": 128049, |
|
"__km__": 128050, |
|
"__kn__": 128051, |
|
"__ko__": 128052, |
|
"__lb__": 128053, |
|
"__lg__": 128054, |
|
"__ln__": 128055, |
|
"__lo__": 128056, |
|
"__lt__": 128057, |
|
"__lv__": 128058, |
|
"__mg__": 128059, |
|
"__mk__": 128060, |
|
"__ml__": 128061, |
|
"__mn__": 128062, |
|
"__mr__": 128063, |
|
"__ms__": 128064, |
|
"__my__": 128065, |
|
"__ne__": 128066, |
|
"__nl__": 128067, |
|
"__no__": 128068, |
|
"__ns__": 128069, |
|
"__oc__": 128070, |
|
"__or__": 128071, |
|
"__pa__": 128072, |
|
"__pl__": 128073, |
|
"__ps__": 128074, |
|
"__pt__": 128075, |
|
"__ro__": 128076, |
|
"__ru__": 128077, |
|
"__sd__": 128078, |
|
"__si__": 128079, |
|
"__sk__": 128080, |
|
"__sl__": 128081, |
|
"__so__": 128082, |
|
"__sq__": 128083, |
|
"__sr__": 128084, |
|
"__ss__": 128085, |
|
"__su__": 128086, |
|
"__sv__": 128087, |
|
"__sw__": 128088, |
|
"__ta__": 128089, |
|
"__th__": 128090, |
|
"__tl__": 128091, |
|
"__tn__": 128092, |
|
"__tr__": 128093, |
|
"__uk__": 128094, |
|
"__ur__": 128095, |
|
"__uz__": 128096, |
|
"__vi__": 128097, |
|
"__wo__": 128098, |
|
"__xh__": 128099, |
|
"__yi__": 128100, |
|
"__yo__": 128101, |
|
"__zh__": 128102, |
|
"__zu__": 128103 |
|
} |
|
|