Upload tiv_latn_5mb tokenizer.

added_tokens.json

@@ -0,0 +1 @@
+{"[XXXXX98]": 15634, "[XXXXX606]": 16142, "[XXXXX601]": 16137, "[XXXXX744]": 16280, "[XXXXX564]": 16100, "[XXXXX743]": 16279, "[XXXXX159]": 15695, "[XXXXX675]": 16211, "[XXXXX691]": 16227, "[XXXXX668]": 16204, "[XXXXX0]": 15536, "[XXXXX539]": 16075, "[XXXXX823]": 16359, "[XXXXX838]": 16374, "[XXXXX92]": 15628, "[XXXXX298]": 15834, "[XXXXX271]": 15807, "[XXXXX222]": 15758, "[XXXXX349]": 15885, "[XXXXX761]": 16297, "[XXXXX431]": 15967, "[XXXXX610]": 16146, "[XXXXX197]": 15733, "[XXXXX79]": 15615, "[XXXXX442]": 15978, "[XXXXX708]": 16244, "[XXXXX18]": 15554, "[XXXXX745]": 16281, "[XXXXX478]": 16014, "[XXXXX417]": 15953, "[XXXXX609]": 16145, "[XXXXX517]": 16053, "[XXXXX33]": 15569, "[XXXXX153]": 15689, "[XXXXX471]": 16007, "[XXXXX163]": 15699, "[XXXXX390]": 15926, "[XXXXX582]": 16118, "[XXXXX410]": 15946, "[XXXXX523]": 16059, "[XXXXX682]": 16218, "[XXXXX815]": 16351, "[XXXXX585]": 16121, "[XXXXX60]": 15596, "[XXXXX625]": 16161, "[XXXXX522]": 16058, "[XXXXX191]": 15727, "[XXXXX560]": 16096, "[XXXXX614]": 16150, "[XXXXX71]": 15607, "[XXXXX781]": 16317, "[XXXXX573]": 16109, "[XXXXX777]": 16313, "[XXXXX356]": 15892, "[XXXXX620]": 16156, "[XXXXX109]": 15645, "[XXXXX674]": 16210, "[XXXXX534]": 16070, "[XXXXX318]": 15854, "[XXXXX666]": 16202, "[XXXXX154]": 15690, "[XXXXX634]": 16170, "[XXXXX340]": 15876, "[XXXXX5]": 15541, "[XXXXX842]": 16378, "[XXXXX795]": 16331, "[XXXXX406]": 15942, "[XXXXX521]": 16057, "[XXXXX671]": 16207, "[XXXXX557]": 16093, "[XXXXX790]": 16326, "[XXXXX556]": 16092, "[XXXXX352]": 15888, "[XXXXX139]": 15675, "[XXXXX526]": 16062, "[XXXXX819]": 16355, "[XXXXX499]": 16035, "[XXXXX599]": 16135, "[XXXXX142]": 15678, "[XXXXX121]": 15657, "[XXXXX552]": 16088, "[XXXXX454]": 15990, "[XXXXX489]": 16025, "[XXXXX418]": 15954, "[XXXXX296]": 15832, "[XXXXX623]": 16159, "[XXXXX457]": 15993, "[XXXXX337]": 15873, "[XXXXX290]": 15826, "[XXXXX128]": 15664, "[XXXXX624]": 16160, "[XXXXX387]": 15923, "[XXXXX683]": 16219, "[XXXXX181]": 15717, "[XXXXX302]": 15838, "[XXXXX192]": 15728, "[XXXXX166]": 15702, "[XXXXX280]": 15816, "[XXXXX788]": 16324, "[XXXXX347]": 15883, "[XXXXX359]": 15895, "[XXXXX11]": 15547, "[XXXXX785]": 16321, "[XXXXX542]": 16078, "[XXXXX214]": 15750, "[XXXXX1]": 15537, "[XXXXX797]": 16333, "[XXXXX572]": 16108, "[XXXXX772]": 16308, "[XXXXX334]": 15870, "[XXXXX667]": 16203, "[XXXXX160]": 15696, "[XXXXX812]": 16348, "[XXXXX770]": 16306, "[XXXXX562]": 16098, "[XXXXX143]": 15679, "[XXXXX85]": 15621, "[XXXXX450]": 15986, "[XXXXX53]": 15589, "[XXXXX221]": 15757, "[XXXXX423]": 15959, "[XXXXX10]": 15546, "[XXXXX527]": 16063, "[XXXXX847]": 16383, "[XXXXX336]": 15872, "[XXXXX279]": 15815, "[XXXXX446]": 15982, "[XXXXX54]": 15590, "[XXXXX783]": 16319, "[XXXXX162]": 15698, "[XXXXX498]": 16034, "[XXXXX100]": 15636, "[XXXXX64]": 15600, "[XXXXX392]": 15928, "[XXXXX179]": 15715, "[XXXXX816]": 16352, "[XXXXX90]": 15626, "[XXXXX792]": 16328, "[XXXXX346]": 15882, "[XXXXX48]": 15584, "[XXXXX500]": 16036, "[XXXXX479]": 16015, "[XXXXX591]": 16127, "[XXXXX712]": 16248, "[XXXXX203]": 15739, "[XXXXX381]": 15917, "[XXXXX554]": 16090, "[XXXXX621]": 16157, "[XXXXX422]": 15958, "[XXXXX266]": 15802, "[SEP]": 15533, "[XXXXX409]": 15945, "[XXXXX223]": 15759, "[XXXXX19]": 15555, "[XXXXX309]": 15845, "[XXXXX201]": 15737, "[XXXXX277]": 15813, "[XXXXX775]": 16311, "[XXXXX424]": 15960, "[XXXXX230]": 15766, "[XXXXX45]": 15581, "[XXXXX669]": 16205, "[XXXXX687]": 16223, "[XXXXX182]": 15718, "[XXXXX170]": 15706, "[XXXXX401]": 15937, "[XXXXX600]": 16136, "[XXXXX461]": 15997, "[XXXXX193]": 15729, "[XXXXX43]": 15579, "[XXXXX320]": 15856, "[XXXXX167]": 15703, "[XXXXX200]": 15736, "[XXXXX383]": 15919, "[XXXXX656]": 16192, "[XXXXX72]": 15608, "[XXXXX536]": 16072, "[XXXXX604]": 16140, "[XXXXX15]": 15551, "[XXXXX415]": 15951, "[XXXXX736]": 16272, "[XXXXX196]": 15732, "[XXXXX462]": 15998, "[XXXXX453]": 15989, "[XXXXX580]": 16116, "[XXXXX7]": 15543, "[XXXXX275]": 15811, "[XXXXX742]": 16278, "[XXXXX69]": 15605, "[XXXXX210]": 15746, "[XXXXX132]": 15668, "[XXXXX158]": 15694, "[XXXXX574]": 16110, "[XXXXX135]": 15671, "[XXXXX63]": 15599, "[XXXXX116]": 15652, "[XXXXX779]": 16315, "[XXXXX276]": 15812, "[XXXXX684]": 16220, "[XXXXX844]": 16380, "[XXXXX235]": 15771, "[XXXXX810]": 16346, "[XXXXX131]": 15667, "[XXXXX493]": 16029, "[XXXXX782]": 16318, "[XXXXX341]": 15877, "[XXXXX396]": 15932, "[XXXXX328]": 15864, "[XXXXX617]": 16153, "[XXXXX538]": 16074, "[XXXXX123]": 15659, "[XXXXX194]": 15730, "[XXXXX316]": 15852, "[XXXXX332]": 15868, "[XXXXX713]": 16249, "[XXXXX358]": 15894, "[XXXXX618]": 16154, "[XXXXX330]": 15866, "[XXXXX515]": 16051, "[XXXXX747]": 16283, "[XXXXX717]": 16253, "[XXXXX460]": 15996, "[XXXXX4]": 15540, "[XXXXX73]": 15609, "[XXXXX391]": 15927, "[XXXXX773]": 16309, "[XXXXX696]": 16232, "[XXXXX578]": 16114, "[XXXXX608]": 16144, "[XXXXX165]": 15701, "[XXXXX430]": 15966, "[XXXXX567]": 16103, "[XXXXX20]": 15556, "[XXXXX124]": 15660, "[XXXXX821]": 16357, "[XXXXX278]": 15814, "[XXXXX394]": 15930, "[XXXXX445]": 15981, "[XXXXX631]": 16167, "[XXXXX207]": 15743, "[XXXXX144]": 15680, "[XXXXX249]": 15785, "[XXXXX174]": 15710, "[XXXXX270]": 15806, "[XXXXX411]": 15947, "[XXXXX259]": 15795, "[XXXXX245]": 15781, "[XXXXX384]": 15920, "[XXXXX404]": 15940, "[XXXXX830]": 16366, "[XXXXX533]": 16069, "[XXXXX155]": 15691, "[XXXXX507]": 16043, "[XXXXX292]": 15828, "[XXXXX117]": 15653, "[XXXXX312]": 15848, "[XXXXX569]": 16105, "[XXXXX354]": 15890, "[XXXXX751]": 16287, "[XXXXX106]": 15642, "[XXXXX130]": 15666, "[XXXXX306]": 15842, "[XXXXX149]": 15685, "[XXXXX111]": 15647, "[XXXXX385]": 15921, "[XXXXX561]": 16097, "[XXXXX168]": 15704, "[XXXXX112]": 15648, "[XXXXX451]": 15987, "[XXXXX333]": 15869, "[XXXXX699]": 16235, "[XXXXX726]": 16262, "[XXXXX432]": 15968, "[XXXXX151]": 15687, "[XXXXX707]": 16243, "[XXXXX710]": 16246, "[XXXXX388]": 15924, "[XXXXX426]": 15962, "[XXXXX438]": 15974, "[XXXXX584]": 16120, "[XXXXX30]": 15566, "[XXXXX148]": 15684, "[XXXXX377]": 15913, "[XXXXX814]": 16350, "[XXXXX771]": 16307, "[XXXXX293]": 15829, "[XXXXX137]": 15673, "[XXXXX104]": 15640, "[XXXXX55]": 15591, "[XXXXX344]": 15880, "[XXXXX735]": 16271, "[XXXXX362]": 15898, "[XXXXX635]": 16171, "[XXXXX421]": 15957, "[XXXXX141]": 15677, "[XXXXX593]": 16129, "[XXXXX503]": 16039, "[XXXXX32]": 15568, "[XXXXX681]": 16217, "[XXXXX429]": 15965, "[XXXXX565]": 16101, "[XXXXX727]": 16263, "[XXXXX35]": 15571, "[XXXXX700]": 16236, "[XXXXX251]": 15787, "[XXXXX319]": 15855, "[XXXXX482]": 16018, "[XXXXX437]": 15973, "[XXXXX134]": 15670, "[XXXXX304]": 15840, "[XXXXX122]": 15658, "[XXXXX110]": 15646, "[XXXXX188]": 15724, "[XXXXX250]": 15786, "[XXXXX26]": 15562, "[XXXXX837]": 16373, "[XXXXX80]": 15616, "[XXXXX86]": 15622, "[XXXXX364]": 15900, "[XXXXX803]": 16339, "[XXXXX807]": 16343, "[XXXXX403]": 15939, "[XXXXX511]": 16047, "[XXXXX794]": 16330, "[XXXXX719]": 16255, "[XXXXX571]": 16107, "[XXXXX763]": 16299, "[XXXXX632]": 16168, "[XXXXX173]": 15709, "[XXXXX786]": 16322, "[XXXXX299]": 15835, "[XXXXX229]": 15765, "[XXXXX93]": 15629, "[XXXXX497]": 16033, "[XXXXX317]": 15853, "[XXXXX57]": 15593, "[XXXXX2]": 15538, "[XXXXX649]": 16185, "[XXXXX399]": 15935, "[XXXXX804]": 16340, "[XXXXX81]": 15617, "[XXXXX76]": 15612, "[XXXXX96]": 15632, "[XXXXX272]": 15808, "[XXXXX405]": 15941, "[XXXXX568]": 16104, "[XXXXX61]": 15597, "[XXXXX9]": 15545, "[XXXXX107]": 15643, "[XXXXX83]": 15619, "[XXXXX397]": 15933, "[XXXXX444]": 15980, "[XXXXX281]": 15817, "[XXXXX436]": 15972, "[XXXXX367]": 15903, "[XXXXX267]": 15803, "[XXXXX282]": 15818, "[XXXXX648]": 16184, "[XXXXX89]": 15625, "[XXXXX339]": 15875, "[XXXXX99]": 15635, "[XXXXX835]": 16371, "[XXXXX321]": 15857, "[XXXXX759]": 16295, "[XXXXX514]": 16050, "[XXXXX288]": 15824, "[XXXXX765]": 16301, "[XXXXX793]": 16329, "[XXXXX258]": 15794, "[XXXXX509]": 16045, "[XXXXX828]": 16364, "[XXXXX590]": 16126, "[XXXXX841]": 16377, "[XXXXX722]": 16258, "[XXXXX360]": 15896, "[XXXXX663]": 16199, "[XXXXX178]": 15714, "[XXXXX612]": 16148, "[XXXXX711]": 16247, "[XXXXX697]": 16233, "[XXXXX475]": 16011, "[XXXXX66]": 15602, "[XXXXX233]": 15769, "[XXXXX575]": 16111, "[XXXXX202]": 15738, "[XXXXX760]": 16296, "[CLS]": 15532, "[XXXXX506]": 16042, "[XXXXX448]": 15984, "[XXXXX563]": 16099, "[XXXXX532]": 16068, "[XXXXX199]": 15735, "[XXXXX274]": 15810, "[XXXXX826]": 16362, "[XXXXX840]": 16376, "[XXXXX537]": 16073, "[XXXXX190]": 15726, "[XXXXX373]": 15909, "[XXXXX297]": 15833, "[XXXXX435]": 15971, "[XXXXX753]": 16289, "[XXXXX630]": 16166, "[XXXXX216]": 15752, "[XXXXX467]": 16003, "[XXXXX659]": 16195, "[XXXXX126]": 15662, "[XXXXX14]": 15550, "[XXXXX324]": 15860, "[XXXXX480]": 16016, "[XXXXX813]": 16349, "[XXXXX56]": 15592, "[XXXXX543]": 16079, "[XXXXX832]": 16368, "[XXXXX310]": 15846, "[XXXXX184]": 15720, "[XXXXX138]": 15674, "[XXXXX709]": 16245, "[XXXXX23]": 15559, "[XXXXX25]": 15561, "[XXXXX702]": 16238, "[XXXXX84]": 15620, "[XXXXX466]": 16002, "[XXXXX136]": 15672, "[XXXXX36]": 15572, "[XXXXX44]": 15580, "[XXXXX24]": 15560, "[XXXXX428]": 15964, "[XXXXX653]": 16189, "[XXXXX157]": 15693, "[XXXXX211]": 15747, "[XXXXX650]": 16186, "[XXXXX473]": 16009, "[XXXXX596]": 16132, "<pad>": 15534, "[XXXXX156]": 15692, "[XXXXX546]": 16082, "[XXXXX642]": 16178, "[XXXXX218]": 15754, "[XXXXX836]": 16372, "[XXXXX811]": 16347, "[XXXXX47]": 15583, "[XXXXX541]": 16077, "[XXXXX516]": 16052, "[XXXXX22]": 15558, "[XXXXX342]": 15878, "[XXXXX768]": 16304, "[XXXXX228]": 15764, "[XXXXX70]": 15606, "[XXXXX244]": 15780, "[XXXXX325]": 15861, "[XXXXX491]": 16027, "[XXXXX105]": 15641, "[XXXXX357]": 15893, "[XXXXX470]": 16006, "[XXXXX215]": 15751, "[XXXXX459]": 15995, "[XXXXX247]": 15783, "[XXXXX486]": 16022, "[XXXXX180]": 15716, "[XXXXX91]": 15627, "[XXXXX688]": 16224, "[XXXXX50]": 15586, "[XXXXX545]": 16081, "[XXXXX551]": 16087, "[MASK]": 15535, "[XXXXX257]": 15793, "[XXXXX227]": 15763, "[XXXXX685]": 16221, "[XXXXX706]": 16242, "[XXXXX754]": 16290, "[XXXXX825]": 16361, "[XXXXX594]": 16130, "[XXXXX689]": 16225, "[XXXXX331]": 15867, "[XXXXX353]": 15889, "[XXXXX729]": 16265, "[XXXXX791]": 16327, "[XXXXX672]": 16208, "[XXXXX484]": 16020, "[XXXXX187]": 15723, "[XXXXX226]": 15762, "[XXXXX531]": 16067, "[XXXXX739]": 16275, "[XXXXX420]": 15956, "[XXXXX774]": 16310, "[XXXXX508]": 16044, "[XXXXX468]": 16004, "[XXXXX413]": 15949, "[XXXXX525]": 16061, "[XXXXX758]": 16294, "[XXXXX655]": 16191, "[XXXXX616]": 16152, "[XXXXX291]": 15827, "[XXXXX382]": 15918, "[XXXXX355]": 15891, "[XXXXX637]": 16173, "[XXXXX483]": 16019, "[XXXXX414]": 15950, "[XXXXX256]": 15792, "[XXXXX217]": 15753, "[XXXXX495]": 16031, "[XXXXX452]": 15988, "[XXXXX553]": 16089, "[XXXXX41]": 15577, "[XXXXX46]": 15582, "[XXXXX314]": 15850, "[XXXXX239]": 15775, "[XXXXX647]": 16183, "[XXXXX169]": 15705, "[XXXXX645]": 16181, "[XXXXX714]": 16250, "[XXXXX776]": 16312, "[XXXXX6]": 15542, "[XXXXX829]": 16365, "[XXXXX439]": 15975, "[XXXXX731]": 16267, "[XXXXX433]": 15969, "[XXXXX587]": 16123, "[XXXXX327]": 15863, "[XXXXX27]": 15563, "[XXXXX295]": 15831, "[XXXXX577]": 16113, "[XXXXX633]": 16169, "[XXXXX161]": 15697, "[XXXXX605]": 16141, "[XXXXX802]": 16338, "[XXXXX286]": 15822, "[XXXXX615]": 16151, "[XXXXX704]": 16240, "[XXXXX676]": 16212, "[XXXXX805]": 16341, "[XXXXX598]": 16134, "[XXXXX38]": 15574, "[XXXXX627]": 16163, "[XXXXX389]": 15925, "[XXXXX393]": 15929, "[XXXXX315]": 15851, "[XXXXX125]": 15661, "[XXXXX447]": 15983, "[XXXXX58]": 15594, "[XXXXX626]": 16162, "[XXXXX629]": 16165, "[XXXXX720]": 16256, "[XXXXX595]": 16131, "[XXXXX285]": 15821, "[XXXXX17]": 15553, "[XXXXX827]": 16363, "[XXXXX371]": 15907, "[XXXXX548]": 16084, "[XXXXX176]": 15712, "[XXXXX97]": 15633, "[XXXXX283]": 15819, "[XXXXX255]": 15791, "[XXXXX652]": 16188, "[XXXXX171]": 15707, "[XXXXX338]": 15874, "[XXXXX789]": 16325, "[XXXXX305]": 15841, "[XXXXX692]": 16228, "[XXXXX639]": 16175, "[XXXXX570]": 16106, "[XXXXX660]": 16196, "[XXXXX350]": 15886, "[XXXXX120]": 15656, "[XXXXX586]": 16122, "[XXXXX127]": 15663, "[XXXXX273]": 15809, "[XXXXX234]": 15770, "[XXXXX750]": 16286, "[XXXXX496]": 16032, "[XXXXX308]": 15844, "[XXXXX52]": 15588, "[XXXXX118]": 15654, "[XXXXX664]": 16200, "[XXXXX183]": 15719, "[XXXXX502]": 16038, "[XXXXX602]": 16138, "[XXXXX65]": 15601, "[XXXXX204]": 15740, "[XXXXX657]": 16193, "[XXXXX21]": 15557, "[XXXXX703]": 16239, "[XXXXX762]": 16298, "[XXXXX519]": 16055, "[XXXXX581]": 16117, "[XXXXX698]": 16234, "[XXXXX198]": 15734, "[XXXXX345]": 15881, "[XXXXX520]": 16056, "[XXXXX307]": 15843, "[XXXXX455]": 15991, "[XXXXX737]": 16273, "[XXXXX102]": 15638, "[XXXXX31]": 15567, "[XXXXX651]": 16187, "[XXXXX209]": 15745, "[XXXXX818]": 16354, "[XXXXX680]": 16216, "[XXXXX240]": 15776, "[XXXXX769]": 16305, "[XXXXX654]": 16190, "[XXXXX755]": 16291, "[XXXXX510]": 16046, "[XXXXX469]": 16005, "[XXXXX303]": 15839, "[XXXXX846]": 16382, "[XXXXX441]": 15977, "[XXXXX740]": 16276, "[XXXXX376]": 15912, "[XXXXX343]": 15879, "[XXXXX284]": 15820, "[XXXXX263]": 15799, "[XXXXX108]": 15644, "[XXXXX801]": 16337, "[XXXXX395]": 15931, "[XXXXX820]": 16356, "[XXXXX348]": 15884, "[XXXXX379]": 15915, "[XXXXX75]": 15611, "[XXXXX544]": 16080, "[XXXXX380]": 15916, "[XXXXX185]": 15721, "[XXXXX566]": 16102, "[XXXXX241]": 15777, "[XXXXX254]": 15790, "[XXXXX501]": 16037, "[XXXXX402]": 15938, "[XXXXX764]": 16300, "[XXXXX177]": 15713, "[XXXXX253]": 15789, "[XXXXX287]": 15823, "[XXXXX363]": 15899, "[XXXXX504]": 16040, "[XXXXX88]": 15624, "[XXXXX311]": 15847, "[XXXXX778]": 16314, "[XXXXX686]": 16222, "[XXXXX175]": 15711, "[XXXXX264]": 15800, "[XXXXX724]": 16260, "[XXXXX419]": 15955, "[XXXXX603]": 16139, "[XXXXX119]": 15655, "[XXXXX248]": 15784, "[XXXXX172]": 15708, "[XXXXX487]": 16023, "[XXXXX427]": 15963, "[XXXXX661]": 16197, "[XXXXX476]": 16012, "[XXXXX94]": 15630, "[XXXXX13]": 15549, "[XXXXX505]": 16041, "[XXXXX678]": 16214, "[XXXXX206]": 15742, "[XXXXX366]": 15902, "[XXXXX638]": 16174, "[XXXXX749]": 16285, "[XXXXX370]": 15906, "[XXXXX547]": 16083, "[XXXXX449]": 15985, "[XXXXX592]": 16128, "[XXXXX40]": 15576, "[XXXXX690]": 16226, "[XXXXX695]": 16231, "[XXXXX767]": 16303, "[XXXXX718]": 16254, "[XXXXX784]": 16320, "[XXXXX799]": 16335, "[XXXXX540]": 16076, "[XXXXX434]": 15970, "[XXXXX114]": 15650, "[XXXXX559]": 16095, "[XXXXX262]": 15798, "[XXXXX732]": 16268, "[XXXXX246]": 15782, "[XXXXX386]": 15922, "[XXXXX252]": 15788, "[XXXXX369]": 15905, "[XXXXX351]": 15887, "[XXXXX665]": 16201, "[XXXXX822]": 16358, "[XXXXX766]": 16302, "[XXXXX619]": 16155, "[XXXXX796]": 16332, "[XXXXX555]": 16091, "[XXXXX658]": 16194, "[XXXXX725]": 16261, "[XXXXX611]": 16147, "[XXXXX673]": 16209, "[XXXXX734]": 16270, "[XXXXX622]": 16158, "[XXXXX294]": 15830, "[XXXXX576]": 16112, "[XXXXX300]": 15836, "[XXXXX232]": 15768, "[XXXXX464]": 16000, "[XXXXX806]": 16342, "[XXXXX748]": 16284, "[XXXXX265]": 15801, "[XXXXX49]": 15585, "[XXXXX646]": 16182, "[XXXXX78]": 15614, "[XXXXX715]": 16251, "[XXXXX164]": 15700, "[XXXXX558]": 16094, "[XXXXX374]": 15910, "[XXXXX798]": 16334, "[XXXXX3]": 15539, "[XXXXX400]": 15936, "[XXXXX456]": 15992, "[XXXXX730]": 16266, "[XXXXX757]": 16293, "[XXXXX644]": 16180, "[XXXXX62]": 15598, "[XXXXX800]": 16336, "[XXXXX329]": 15865, "[XXXXX463]": 15999, "[XXXXX51]": 15587, "[XXXXX323]": 15859, "[XXXXX8]": 15544, "[XXXXX220]": 15756, "[XXXXX716]": 16252, "[XXXXX242]": 15778, "[XXXXX636]": 16172, "[XXXXX231]": 15767, "[XXXXX787]": 16323, "[XXXXX834]": 16370, "[XXXXX39]": 15575, "[XXXXX481]": 16017, "[XXXXX728]": 16264, "[XXXXX607]": 16143, "[XXXXX103]": 15639, "[XXXXX513]": 16049, "[XXXXX741]": 16277, "[XXXXX29]": 15565, "[XXXXX95]": 15631, "[XXXXX693]": 16229, "[XXXXX236]": 15772, "[XXXXX752]": 16288, "[XXXXX721]": 16257, "[XXXXX443]": 15979, "[XXXXX129]": 15665, "[XXXXX589]": 16125, "[XXXXX269]": 15805, "[XXXXX268]": 15804, "[XXXXX224]": 15760, "[XXXXX831]": 16367, "[XXXXX145]": 15681, "[XXXXX640]": 16176, "[XXXXX301]": 15837, "[XXXXX458]": 15994, "[XXXXX67]": 15603, "[XXXXX261]": 15797, "[XXXXX472]": 16008, "[XXXXX694]": 16230, "[XXXXX361]": 15897, "[XXXXX212]": 15748, "[XXXXX518]": 16054, "[XXXXX205]": 15741, "[XXXXX628]": 16164, "[XXXXX34]": 15570, "[XXXXX260]": 15796, "[XXXXX115]": 15651, "[XXXXX372]": 15908, "[XXXXX597]": 16133, "[XXXXX477]": 16013, "[XXXXX133]": 15669, "[XXXXX529]": 16065, "[XXXXX398]": 15934, "[XXXXX588]": 16124, "[XXXXX101]": 15637, "[XXXXX412]": 15948, "[XXXXX512]": 16048, "[XXXXX425]": 15961, "[XXXXX219]": 15755, "[XXXXX368]": 15904, "[XXXXX12]": 15548, "[XXXXX152]": 15688, "[XXXXX613]": 16149, "[XXXXX322]": 15858, "[XXXXX677]": 16213, "[XXXXX465]": 16001, "[XXXXX365]": 15901, "[XXXXX494]": 16030, "[XXXXX326]": 15862, "[XXXXX780]": 16316, "[XXXXX59]": 15595, "[XXXXX824]": 16360, "[XXXXX738]": 16274, "[XXXXX225]": 15761, "[XXXXX213]": 15749, "[XXXXX746]": 16282, "[XXXXX238]": 15774, "[XXXXX87]": 15623, "[XXXXX723]": 16259, "[XXXXX733]": 16269, "[XXXXX833]": 16369, "[XXXXX490]": 16026, "[XXXXX208]": 15744, "[XXXXX113]": 15649, "[XXXXX530]": 16066, "[XXXXX407]": 15943, "[XXXXX16]": 15552, "[XXXXX550]": 16086, "[XXXXX186]": 15722, "[XXXXX408]": 15944, "[XXXXX440]": 15976, "[XXXXX416]": 15952, "[XXXXX147]": 15683, "[XXXXX378]": 15914, "[XXXXX839]": 16375, "[XXXXX28]": 15564, "[XXXXX37]": 15573, "[XXXXX662]": 16198, "[XXXXX42]": 15578, "[XXXXX809]": 16345, "[XXXXX77]": 15613, "[XXXXX643]": 16179, "[XXXXX705]": 16241, "[XXXXX843]": 16379, "[XXXXX535]": 16071, "[XXXXX243]": 15779, "[XXXXX817]": 16353, "[XXXXX237]": 15773, "[XXXXX488]": 16024, "[XXXXX189]": 15725, "[XXXXX641]": 16177, "[XXXXX579]": 16115, "[XXXXX756]": 16292, "[XXXXX335]": 15871, "[XXXXX82]": 15618, "[XXXXX845]": 16381, "[XXXXX150]": 15686, "[XXXXX670]": 16206, "[XXXXX474]": 16010, "[XXXXX289]": 15825, "[XXXXX492]": 16028, "[XXXXX485]": 16021, "[XXXXX679]": 16215, "[XXXXX146]": 15682, "[XXXXX195]": 15731, "[XXXXX701]": 16237, "[XXXXX68]": 15604, "[XXXXX375]": 15911, "[XXXXX74]": 15610, "[XXXXX583]": 16119, "[XXXXX549]": 16085, "[XXXXX808]": 16344, "[XXXXX524]": 16060, "[XXXXX528]": 16064, "[XXXXX313]": 15849, "[XXXXX140]": 15676}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}, "additional_special_tokens": ["[XXXXX0]", "[XXXXX1]", "[XXXXX2]", "[XXXXX3]", "[XXXXX4]", "[XXXXX5]", "[XXXXX6]", "[XXXXX7]", "[XXXXX8]", "[XXXXX9]", "[XXXXX10]", "[XXXXX11]", "[XXXXX12]", "[XXXXX13]", "[XXXXX14]", "[XXXXX15]", "[XXXXX16]", "[XXXXX17]", "[XXXXX18]", "[XXXXX19]", "[XXXXX20]", "[XXXXX21]", "[XXXXX22]", "[XXXXX23]", "[XXXXX24]", "[XXXXX25]", "[XXXXX26]", "[XXXXX27]", "[XXXXX28]", "[XXXXX29]", "[XXXXX30]", "[XXXXX31]", "[XXXXX32]", "[XXXXX33]", "[XXXXX34]", "[XXXXX35]", "[XXXXX36]", "[XXXXX37]", "[XXXXX38]", "[XXXXX39]", "[XXXXX40]", "[XXXXX41]", "[XXXXX42]", "[XXXXX43]", "[XXXXX44]", "[XXXXX45]", "[XXXXX46]", "[XXXXX47]", "[XXXXX48]", "[XXXXX49]", "[XXXXX50]", "[XXXXX51]", "[XXXXX52]", "[XXXXX53]", "[XXXXX54]", "[XXXXX55]", "[XXXXX56]", "[XXXXX57]", "[XXXXX58]", "[XXXXX59]", "[XXXXX60]", "[XXXXX61]", "[XXXXX62]", "[XXXXX63]", "[XXXXX64]", "[XXXXX65]", "[XXXXX66]", "[XXXXX67]", "[XXXXX68]", "[XXXXX69]", "[XXXXX70]", "[XXXXX71]", "[XXXXX72]", "[XXXXX73]", "[XXXXX74]", "[XXXXX75]", "[XXXXX76]", "[XXXXX77]", "[XXXXX78]", "[XXXXX79]", "[XXXXX80]", "[XXXXX81]", "[XXXXX82]", "[XXXXX83]", "[XXXXX84]", "[XXXXX85]", "[XXXXX86]", "[XXXXX87]", "[XXXXX88]", "[XXXXX89]", "[XXXXX90]", "[XXXXX91]", "[XXXXX92]", "[XXXXX93]", "[XXXXX94]", "[XXXXX95]", "[XXXXX96]", "[XXXXX97]", "[XXXXX98]", "[XXXXX99]", "[XXXXX100]", "[XXXXX101]", "[XXXXX102]", "[XXXXX103]", "[XXXXX104]", "[XXXXX105]", "[XXXXX106]", "[XXXXX107]", "[XXXXX108]", "[XXXXX109]", "[XXXXX110]", "[XXXXX111]", "[XXXXX112]", "[XXXXX113]", "[XXXXX114]", "[XXXXX115]", "[XXXXX116]", "[XXXXX117]", "[XXXXX118]", "[XXXXX119]", "[XXXXX120]", "[XXXXX121]", "[XXXXX122]", "[XXXXX123]", "[XXXXX124]", "[XXXXX125]", "[XXXXX126]", "[XXXXX127]", "[XXXXX128]", "[XXXXX129]", "[XXXXX130]", "[XXXXX131]", "[XXXXX132]", "[XXXXX133]", "[XXXXX134]", "[XXXXX135]", "[XXXXX136]", "[XXXXX137]", "[XXXXX138]", "[XXXXX139]", "[XXXXX140]", "[XXXXX141]", "[XXXXX142]", "[XXXXX143]", "[XXXXX144]", "[XXXXX145]", "[XXXXX146]", "[XXXXX147]", "[XXXXX148]", "[XXXXX149]", "[XXXXX150]", "[XXXXX151]", "[XXXXX152]", "[XXXXX153]", "[XXXXX154]", "[XXXXX155]", "[XXXXX156]", "[XXXXX157]", "[XXXXX158]", "[XXXXX159]", "[XXXXX160]", "[XXXXX161]", "[XXXXX162]", "[XXXXX163]", "[XXXXX164]", "[XXXXX165]", "[XXXXX166]", "[XXXXX167]", "[XXXXX168]", "[XXXXX169]", "[XXXXX170]", "[XXXXX171]", "[XXXXX172]", "[XXXXX173]", "[XXXXX174]", "[XXXXX175]", "[XXXXX176]", "[XXXXX177]", "[XXXXX178]", "[XXXXX179]", "[XXXXX180]", "[XXXXX181]", "[XXXXX182]", "[XXXXX183]", "[XXXXX184]", "[XXXXX185]", "[XXXXX186]", "[XXXXX187]", "[XXXXX188]", "[XXXXX189]", "[XXXXX190]", "[XXXXX191]", "[XXXXX192]", "[XXXXX193]", "[XXXXX194]", "[XXXXX195]", "[XXXXX196]", "[XXXXX197]", "[XXXXX198]", "[XXXXX199]", "[XXXXX200]", "[XXXXX201]", "[XXXXX202]", "[XXXXX203]", "[XXXXX204]", "[XXXXX205]", "[XXXXX206]", "[XXXXX207]", "[XXXXX208]", "[XXXXX209]", "[XXXXX210]", "[XXXXX211]", "[XXXXX212]", "[XXXXX213]", "[XXXXX214]", "[XXXXX215]", "[XXXXX216]", "[XXXXX217]", "[XXXXX218]", "[XXXXX219]", "[XXXXX220]", "[XXXXX221]", "[XXXXX222]", "[XXXXX223]", "[XXXXX224]", "[XXXXX225]", "[XXXXX226]", "[XXXXX227]", "[XXXXX228]", "[XXXXX229]", "[XXXXX230]", "[XXXXX231]", "[XXXXX232]", "[XXXXX233]", "[XXXXX234]", "[XXXXX235]", "[XXXXX236]", "[XXXXX237]", "[XXXXX238]", "[XXXXX239]", "[XXXXX240]", "[XXXXX241]", "[XXXXX242]", "[XXXXX243]", "[XXXXX244]", "[XXXXX245]", "[XXXXX246]", "[XXXXX247]", "[XXXXX248]", "[XXXXX249]", "[XXXXX250]", "[XXXXX251]", "[XXXXX252]", "[XXXXX253]", "[XXXXX254]", "[XXXXX255]", "[XXXXX256]", "[XXXXX257]", "[XXXXX258]", "[XXXXX259]", "[XXXXX260]", "[XXXXX261]", "[XXXXX262]", "[XXXXX263]", "[XXXXX264]", "[XXXXX265]", "[XXXXX266]", "[XXXXX267]", "[XXXXX268]", "[XXXXX269]", "[XXXXX270]", "[XXXXX271]", "[XXXXX272]", "[XXXXX273]", "[XXXXX274]", "[XXXXX275]", "[XXXXX276]", "[XXXXX277]", "[XXXXX278]", "[XXXXX279]", "[XXXXX280]", "[XXXXX281]", "[XXXXX282]", "[XXXXX283]", "[XXXXX284]", "[XXXXX285]", "[XXXXX286]", "[XXXXX287]", "[XXXXX288]", "[XXXXX289]", "[XXXXX290]", "[XXXXX291]", "[XXXXX292]", "[XXXXX293]", "[XXXXX294]", "[XXXXX295]", "[XXXXX296]", "[XXXXX297]", "[XXXXX298]", "[XXXXX299]", "[XXXXX300]", "[XXXXX301]", "[XXXXX302]", "[XXXXX303]", "[XXXXX304]", "[XXXXX305]", "[XXXXX306]", "[XXXXX307]", "[XXXXX308]", "[XXXXX309]", "[XXXXX310]", "[XXXXX311]", "[XXXXX312]", "[XXXXX313]", "[XXXXX314]", "[XXXXX315]", "[XXXXX316]", "[XXXXX317]", "[XXXXX318]", "[XXXXX319]", "[XXXXX320]", "[XXXXX321]", "[XXXXX322]", "[XXXXX323]", "[XXXXX324]", "[XXXXX325]", "[XXXXX326]", "[XXXXX327]", "[XXXXX328]", "[XXXXX329]", "[XXXXX330]", "[XXXXX331]", "[XXXXX332]", "[XXXXX333]", "[XXXXX334]", "[XXXXX335]", "[XXXXX336]", "[XXXXX337]", "[XXXXX338]", "[XXXXX339]", "[XXXXX340]", "[XXXXX341]", "[XXXXX342]", "[XXXXX343]", "[XXXXX344]", "[XXXXX345]", "[XXXXX346]", "[XXXXX347]", "[XXXXX348]", "[XXXXX349]", "[XXXXX350]", "[XXXXX351]", "[XXXXX352]", "[XXXXX353]", "[XXXXX354]", "[XXXXX355]", "[XXXXX356]", "[XXXXX357]", "[XXXXX358]", "[XXXXX359]", "[XXXXX360]", "[XXXXX361]", "[XXXXX362]", "[XXXXX363]", "[XXXXX364]", "[XXXXX365]", "[XXXXX366]", "[XXXXX367]", "[XXXXX368]", "[XXXXX369]", "[XXXXX370]", "[XXXXX371]", "[XXXXX372]", "[XXXXX373]", "[XXXXX374]", "[XXXXX375]", "[XXXXX376]", "[XXXXX377]", "[XXXXX378]", "[XXXXX379]", "[XXXXX380]", "[XXXXX381]", "[XXXXX382]", "[XXXXX383]", "[XXXXX384]", "[XXXXX385]", "[XXXXX386]", "[XXXXX387]", "[XXXXX388]", "[XXXXX389]", "[XXXXX390]", "[XXXXX391]", "[XXXXX392]", "[XXXXX393]", "[XXXXX394]", "[XXXXX395]", "[XXXXX396]", "[XXXXX397]", "[XXXXX398]", "[XXXXX399]", "[XXXXX400]", "[XXXXX401]", "[XXXXX402]", "[XXXXX403]", "[XXXXX404]", "[XXXXX405]", "[XXXXX406]", "[XXXXX407]", "[XXXXX408]", "[XXXXX409]", "[XXXXX410]", "[XXXXX411]", "[XXXXX412]", "[XXXXX413]", "[XXXXX414]", "[XXXXX415]", "[XXXXX416]", "[XXXXX417]", "[XXXXX418]", "[XXXXX419]", "[XXXXX420]", "[XXXXX421]", "[XXXXX422]", "[XXXXX423]", "[XXXXX424]", "[XXXXX425]", "[XXXXX426]", "[XXXXX427]", "[XXXXX428]", "[XXXXX429]", "[XXXXX430]", "[XXXXX431]", "[XXXXX432]", "[XXXXX433]", "[XXXXX434]", "[XXXXX435]", "[XXXXX436]", "[XXXXX437]", "[XXXXX438]", "[XXXXX439]", "[XXXXX440]", "[XXXXX441]", "[XXXXX442]", "[XXXXX443]", "[XXXXX444]", "[XXXXX445]", "[XXXXX446]", "[XXXXX447]", "[XXXXX448]", "[XXXXX449]", "[XXXXX450]", "[XXXXX451]", "[XXXXX452]", "[XXXXX453]", "[XXXXX454]", "[XXXXX455]", "[XXXXX456]", "[XXXXX457]", "[XXXXX458]", "[XXXXX459]", "[XXXXX460]", "[XXXXX461]", "[XXXXX462]", "[XXXXX463]", "[XXXXX464]", "[XXXXX465]", "[XXXXX466]", "[XXXXX467]", "[XXXXX468]", "[XXXXX469]", "[XXXXX470]", "[XXXXX471]", "[XXXXX472]", "[XXXXX473]", "[XXXXX474]", "[XXXXX475]", "[XXXXX476]", "[XXXXX477]", "[XXXXX478]", "[XXXXX479]", "[XXXXX480]", "[XXXXX481]", "[XXXXX482]", "[XXXXX483]", "[XXXXX484]", "[XXXXX485]", "[XXXXX486]", "[XXXXX487]", "[XXXXX488]", "[XXXXX489]", "[XXXXX490]", "[XXXXX491]", "[XXXXX492]", "[XXXXX493]", "[XXXXX494]", "[XXXXX495]", "[XXXXX496]", "[XXXXX497]", "[XXXXX498]", "[XXXXX499]", "[XXXXX500]", "[XXXXX501]", "[XXXXX502]", "[XXXXX503]", "[XXXXX504]", "[XXXXX505]", "[XXXXX506]", "[XXXXX507]", "[XXXXX508]", "[XXXXX509]", "[XXXXX510]", "[XXXXX511]", "[XXXXX512]", "[XXXXX513]", "[XXXXX514]", "[XXXXX515]", "[XXXXX516]", "[XXXXX517]", "[XXXXX518]", "[XXXXX519]", "[XXXXX520]", "[XXXXX521]", "[XXXXX522]", "[XXXXX523]", "[XXXXX524]", "[XXXXX525]", "[XXXXX526]", "[XXXXX527]", "[XXXXX528]", "[XXXXX529]", "[XXXXX530]", "[XXXXX531]", "[XXXXX532]", "[XXXXX533]", "[XXXXX534]", "[XXXXX535]", "[XXXXX536]", "[XXXXX537]", "[XXXXX538]", "[XXXXX539]", "[XXXXX540]", "[XXXXX541]", "[XXXXX542]", "[XXXXX543]", "[XXXXX544]", "[XXXXX545]", "[XXXXX546]", "[XXXXX547]", "[XXXXX548]", "[XXXXX549]", "[XXXXX550]", "[XXXXX551]", "[XXXXX552]", "[XXXXX553]", "[XXXXX554]", "[XXXXX555]", "[XXXXX556]", "[XXXXX557]", "[XXXXX558]", "[XXXXX559]", "[XXXXX560]", "[XXXXX561]", "[XXXXX562]", "[XXXXX563]", "[XXXXX564]", "[XXXXX565]", "[XXXXX566]", "[XXXXX567]", "[XXXXX568]", "[XXXXX569]", "[XXXXX570]", "[XXXXX571]", "[XXXXX572]", "[XXXXX573]", "[XXXXX574]", "[XXXXX575]", "[XXXXX576]", "[XXXXX577]", "[XXXXX578]", "[XXXXX579]", "[XXXXX580]", "[XXXXX581]", "[XXXXX582]", "[XXXXX583]", "[XXXXX584]", "[XXXXX585]", "[XXXXX586]", "[XXXXX587]", "[XXXXX588]", "[XXXXX589]", "[XXXXX590]", "[XXXXX591]", "[XXXXX592]", "[XXXXX593]", "[XXXXX594]", "[XXXXX595]", "[XXXXX596]", "[XXXXX597]", "[XXXXX598]", "[XXXXX599]", "[XXXXX600]", "[XXXXX601]", "[XXXXX602]", "[XXXXX603]", "[XXXXX604]", "[XXXXX605]", "[XXXXX606]", "[XXXXX607]", "[XXXXX608]", "[XXXXX609]", "[XXXXX610]", "[XXXXX611]", "[XXXXX612]", "[XXXXX613]", "[XXXXX614]", "[XXXXX615]", "[XXXXX616]", "[XXXXX617]", "[XXXXX618]", "[XXXXX619]", "[XXXXX620]", "[XXXXX621]", "[XXXXX622]", "[XXXXX623]", "[XXXXX624]", "[XXXXX625]", "[XXXXX626]", "[XXXXX627]", "[XXXXX628]", "[XXXXX629]", "[XXXXX630]", "[XXXXX631]", "[XXXXX632]", "[XXXXX633]", "[XXXXX634]", "[XXXXX635]", "[XXXXX636]", "[XXXXX637]", "[XXXXX638]", "[XXXXX639]", "[XXXXX640]", "[XXXXX641]", "[XXXXX642]", "[XXXXX643]", "[XXXXX644]", "[XXXXX645]", "[XXXXX646]", "[XXXXX647]", "[XXXXX648]", "[XXXXX649]", "[XXXXX650]", "[XXXXX651]", "[XXXXX652]", "[XXXXX653]", "[XXXXX654]", "[XXXXX655]", "[XXXXX656]", "[XXXXX657]", "[XXXXX658]", "[XXXXX659]", "[XXXXX660]", "[XXXXX661]", "[XXXXX662]", "[XXXXX663]", "[XXXXX664]", "[XXXXX665]", "[XXXXX666]", "[XXXXX667]", "[XXXXX668]", "[XXXXX669]", "[XXXXX670]", "[XXXXX671]", "[XXXXX672]", "[XXXXX673]", "[XXXXX674]", "[XXXXX675]", "[XXXXX676]", "[XXXXX677]", "[XXXXX678]", "[XXXXX679]", "[XXXXX680]", "[XXXXX681]", "[XXXXX682]", "[XXXXX683]", "[XXXXX684]", "[XXXXX685]", "[XXXXX686]", "[XXXXX687]", "[XXXXX688]", "[XXXXX689]", "[XXXXX690]", "[XXXXX691]", "[XXXXX692]", "[XXXXX693]", "[XXXXX694]", "[XXXXX695]", "[XXXXX696]", "[XXXXX697]", "[XXXXX698]", "[XXXXX699]", "[XXXXX700]", "[XXXXX701]", "[XXXXX702]", "[XXXXX703]", "[XXXXX704]", "[XXXXX705]", "[XXXXX706]", "[XXXXX707]", "[XXXXX708]", "[XXXXX709]", "[XXXXX710]", "[XXXXX711]", "[XXXXX712]", "[XXXXX713]", "[XXXXX714]", "[XXXXX715]", "[XXXXX716]", "[XXXXX717]", "[XXXXX718]", "[XXXXX719]", "[XXXXX720]", "[XXXXX721]", "[XXXXX722]", "[XXXXX723]", "[XXXXX724]", "[XXXXX725]", "[XXXXX726]", "[XXXXX727]", "[XXXXX728]", "[XXXXX729]", "[XXXXX730]", "[XXXXX731]", "[XXXXX732]", "[XXXXX733]", "[XXXXX734]", "[XXXXX735]", "[XXXXX736]", "[XXXXX737]", "[XXXXX738]", "[XXXXX739]", "[XXXXX740]", "[XXXXX741]", "[XXXXX742]", "[XXXXX743]", "[XXXXX744]", "[XXXXX745]", "[XXXXX746]", "[XXXXX747]", "[XXXXX748]", "[XXXXX749]", "[XXXXX750]", "[XXXXX751]", "[XXXXX752]", "[XXXXX753]", "[XXXXX754]", "[XXXXX755]", "[XXXXX756]", "[XXXXX757]", "[XXXXX758]", "[XXXXX759]", "[XXXXX760]", "[XXXXX761]", "[XXXXX762]", "[XXXXX763]", "[XXXXX764]", "[XXXXX765]", "[XXXXX766]", "[XXXXX767]", "[XXXXX768]", "[XXXXX769]", "[XXXXX770]", "[XXXXX771]", "[XXXXX772]", "[XXXXX773]", "[XXXXX774]", "[XXXXX775]", "[XXXXX776]", "[XXXXX777]", "[XXXXX778]", "[XXXXX779]", "[XXXXX780]", "[XXXXX781]", "[XXXXX782]", "[XXXXX783]", "[XXXXX784]", "[XXXXX785]", "[XXXXX786]", "[XXXXX787]", "[XXXXX788]", "[XXXXX789]", "[XXXXX790]", "[XXXXX791]", "[XXXXX792]", "[XXXXX793]", "[XXXXX794]", "[XXXXX795]", "[XXXXX796]", "[XXXXX797]", "[XXXXX798]", "[XXXXX799]", "[XXXXX800]", "[XXXXX801]", "[XXXXX802]", "[XXXXX803]", "[XXXXX804]", "[XXXXX805]", "[XXXXX806]", "[XXXXX807]", "[XXXXX808]", "[XXXXX809]", "[XXXXX810]", "[XXXXX811]", "[XXXXX812]", "[XXXXX813]", "[XXXXX814]", "[XXXXX815]", "[XXXXX816]", "[XXXXX817]", "[XXXXX818]", "[XXXXX819]", "[XXXXX820]", "[XXXXX821]", "[XXXXX822]", "[XXXXX823]", "[XXXXX824]", "[XXXXX825]", "[XXXXX826]", "[XXXXX827]", "[XXXXX828]", "[XXXXX829]", "[XXXXX830]", "[XXXXX831]", "[XXXXX832]", "[XXXXX833]", "[XXXXX834]", "[XXXXX835]", "[XXXXX836]", "[XXXXX837]", "[XXXXX838]", "[XXXXX839]", "[XXXXX840]", "[XXXXX841]", "[XXXXX842]", "[XXXXX843]", "[XXXXX844]", "[XXXXX845]", "[XXXXX846]", "[XXXXX847]"]}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da72b7dfa1cda898c7ff360880b58c6963a08218b42fbce4baf61028d33639ed
+size 481344

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "remove_space": true, "keep_accents": true, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false, "__type": "AddedToken"}, "sp_model_kwargs": {}, "name_or_path": "models/5mb/tiv_latn_5mb", "model_input_names": ["input_ids", "attention_mask"], "special_tokens_map_file": "models/5mb/tiv_latn_5mb/special_tokens_map.json", "tokenizer_class": "AlbertTokenizer"}