BioTokenizer-BFD-UNI-400 / tokenizer.json
dotan1111's picture
Upload 2 files
b516756
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<UNK>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Lowercase"
},
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
"<UNK>",
0.0
],
[
"a",
-3.632944090450515
],
[
"l",
-3.7068570202096325
],
[
"g",
-3.9042950954442333
],
[
"r",
-3.9545991750240184
],
[
"v",
-3.998234837926354
],
[
"s",
-4.14952275224373
],
[
"t",
-4.287456837510817
],
[
"e",
-4.289392028132756
],
[
"p",
-4.290234200036018
],
[
"d",
-4.3477089813135485
],
[
"i",
-4.476037236923364
],
[
"m",
-4.510006645253128
],
[
"k",
-4.708311207049434
],
[
"q",
-4.735006170352175
],
[
"f",
-4.789893467104555
],
[
"h",
-5.0101739158699665
],
[
"n",
-5.01778111762699
],
[
"aa",
-5.022472558408127
],
[
"rr",
-5.128207662147673
],
[
"c",
-5.145198760398738
],
[
"al",
-5.180538576122947
],
[
"ll",
-5.201140477172862
],
[
"y",
-5.2091061310437485
],
[
"av",
-5.23780321892715
],
[
"ag",
-5.2607155787369955
],
[
"rl",
-5.269063013646287
],
[
"lv",
-5.280324374261429
],
[
"vl",
-5.2821850373789125
],
[
"la",
-5.287911107396141
],
[
"gg",
-5.2907629902747395
],
[
"lr",
-5.296613327734477
],
[
"ar",
-5.317839521134866
],
[
"w",
-5.3283983119940075
],
[
"ga",
-5.342320152037463
],
[
"lg",
-5.354316588369922
],
[
"gr",
-5.36421754910026
],
[
"ls",
-5.365998834295857
],
[
"ra",
-5.368576042445021
],
[
"el",
-5.3742851295036225
],
[
"sa",
-5.3767074957757295
],
[
"pa",
-5.38691039440727
],
[
"va",
-5.391132319944489
],
[
"sl",
-5.391970768048152
],
[
"ta",
-5.395723638820606
],
[
"gl",
-5.397241434678124
],
[
"vv",
-5.402501062230561
],
[
"as",
-5.4054133935553565
],
[
"dl",
-5.408363687465943
],
[
"da",
-5.419504788703245
],
[
"lp",
-5.421171023058353
],
[
"sg",
-5.425624703056531
],
[
"vg",
-5.43958578125789
],
[
"pg",
-5.443116085521833
],
[
"ld",
-5.460648843958097
],
[
"gv",
-5.461726128843196
],
[
"at",
-5.4623194350596
],
[
"tl",
-5.476791998017722
],
[
"ae",
-5.485646676180689
],
[
"ss",
-5.494587867484871
],
[
"ad",
-5.503243044744844
],
[
"le",
-5.513539597107796
],
[
"er",
-5.525114213305434
],
[
"dg",
-5.52631343875059
],
[
"lt",
-5.535693817027219
],
[
"ea",
-5.535755605860464
],
[
"ia",
-5.536650327927395
],
[
"tg",
-5.545634533570794
],
[
"rg",
-5.547461100096692
],
[
"vr",
-5.5482882187858085
],
[
"rv",
-5.5960473758838525
],
[
"x",
-5.608946077998157
],
[
"rs",
-5.611870379615395
],
[
"gs",
-5.626777482170128
],
[
"ap",
-5.631965142077629
],
[
"ai",
-5.636142688236619
],
[
"ve",
-5.639894023082112
],
[
"ge",
-5.655452846064064
],
[
"pl",
-5.660462720181497
],
[
"gd",
-5.664400349464522
],
[
"ee",
-5.671898580128399
],
[
"vd",
-5.672600958807292
],
[
"dv",
-5.674250319761869
],
[
"tv",
-5.675826469215492
],
[
"gt",
-5.682517311383533
],
[
"rp",
-5.687456812768204
],
[
"vt",
-5.69613081375628
],
[
"pv",
-5.701851465097279
],
[
"vs",
-5.7061688041113765
],
[
"dr",
-5.729755803330297
],
[
"ig",
-5.740135021060222
],
[
"sr",
-5.740787547090417
],
[
"re",
-5.753341772270069
],
[
"sv",
-5.773162831092586
],
[
"ev",
-5.773357017945507
],
[
"gi",
-5.779110480802128
],
[
"sp",
-5.78792820379083
],
[
"tt",
-5.788142314600666
],
[
"li",
-5.7894561148560495
],
[
"tp",
-5.811521012006784
],
[
"de",
-5.81161181019956
],
[
"pp",
-5.817117213994678
],
[
"aq",
-5.8208411958823
],
[
"vp",
-5.82086135309334
],
[
"pr",
-5.8300324993501444
],
[
"rd",
-5.832959938409839
],
[
"ps",
-5.833765158121672
],
[
"iv",
-5.834924940666063
],
[
"st",
-5.836422116950422
],
[
"ts",
-5.845334860786274
],
[
"qa",
-5.8602742392000575
],
[
"dp",
-5.889141466606141
],
[
"lf",
-5.891988029154746
],
[
"ei",
-5.8942462729591
],
[
"ql",
-5.896280144036849
],
[
"pe",
-5.8989822786508554
],
[
"dd",
-5.899947270574684
],
[
"fa",
-5.908091119842952
],
[
"il",
-5.909313751415439
],
[
"pd",
-5.916882430976356
],
[
"lk",
-5.926334580525451
],
[
"kk",
-5.929554253156709
],
[
"af",
-5.9304097490202015
],
[
"fg",
-5.934233305800573
],
[
"eg",
-5.935506392701038
],
[
"gk",
-5.943123839548651
],
[
"ak",
-5.947812328653333
],
[
"fl",
-5.948919578195277
],
[
"id",
-5.95000787138518
],
[
"ri",
-5.953187755007269
],
[
"kl",
-5.954798705995039
],
[
"vi",
-5.9632196203527315
],
[
"lq",
-5.981225013933381
],
[
"ie",
-5.987842234328983
],
[
"gp",
-5.996294580750687
],
[
"ek",
-6.006345600102266
],
[
"rt",
-6.0121209086764775
],
[
"ka",
-6.01491898744084
],
[
"gf",
-6.015842456861838
],
[
"qr",
-6.016464987489098
],
[
"is",
-6.045176758128992
],
[
"nl",
-6.085959225557872
],
[
"pt",
-6.0880399997621435
],
[
"si",
-6.0889675409573325
],
[
"ti",
-6.089641985742588
],
[
"rq",
-6.093718164152271
],
[
"tr",
-6.102011832883294
],
[
"sd",
-6.106249171332308
],
[
"gq",
-6.1155110312256795
],
[
"eq",
-6.147800209212976
],
[
"ln",
-6.173015268781688
],
[
"ng",
-6.181732269229789
],
[
"se",
-6.198317089876394
],
[
"sf",
-6.200053977367045
],
[
"na",
-6.2011622959247745
],
[
"fv",
-6.21270323523432
],
[
"et",
-6.213211850957885
],
[
"ed",
-6.220732944550861
],
[
"vf",
-6.235612361132397
],
[
"it",
-6.236237188173279
],
[
"hl",
-6.242607870761024
],
[
"rf",
-6.242719164870595
],
[
"ke",
-6.243551778618629
],
[
"fs",
-6.247993226234106
],
[
"an",
-6.26119987064193
],
[
"ma",
-6.264278246418684
],
[
"ep",
-6.265933814263626
],
[
"gn",
-6.2690329158590625
],
[
"yl",
-6.273341632364284
],
[
"fd",
-6.276005658681607
],
[
"td",
-6.280432226339416
],
[
"qv",
-6.2888497616309
],
[
"ha",
-6.296053201668862
],
[
"lh",
-6.30246972284626
],
[
"ki",
-6.3175004329885045
],
[
"ml",
-6.328114226448536
],
[
"hg",
-6.33088519345314
],
[
"gy",
-6.333659448005497
],
[
"es",
-6.334612606722317
],
[
"rk",
-6.341736778775461
],
[
"hr",
-6.354408701195906
],
[
"kv",
-6.3558736445353805
],
[
"di",
-6.358575255803077
],
[
"kt",
-6.367157900996915
],
[
"ah",
-6.369617063929681
],
[
"ks",
-6.376100446248731
],
[
"qq",
-6.37798503837473
],
[
"ir",
-6.3857457573601675
],
[
"rh",
-6.387183443208819
],
[
"kr",
-6.389145725185193
],
[
"ff",
-6.394310366090611
],
[
"np",
-6.394909773835787
],
[
"qp",
-6.400762509720662
],
[
"vk",
-6.408006270334974
],
[
"ip",
-6.426055131001776
],
[
"vq",
-6.436536338470235
],
[
"ya",
-6.436780445351445
],
[
"yg",
-6.437618271718964
],
[
"vn",
-6.439633057236559
],
[
"te",
-6.4462825523394685
],
[
"nv",
-6.447181836880107
],
[
"qg",
-6.450639119164498
],
[
"ay",
-6.459484467855098
],
[
"df",
-6.464086065081403
],
[
"gh",
-6.466932879944524
],
[
"tf",
-6.469809893138768
],
[
"sk",
-6.479002868749177
],
[
"ds",
-6.485484141843836
],
[
"sn",
-6.4878031190687935
],
[
"ii",
-6.504258965341522
],
[
"kg",
-6.507398478812792
],
[
"kp",
-6.518085192976523
],
[
"ly",
-6.522152923737886
],
[
"aaa",
-6.526671396118413
],
[
"ft",
-6.5295453018170875
],
[
"hp",
-6.53078009472407
],
[
"kd",
-6.547001972884521
],
[
"qi",
-6.556091048164445
],
[
"pi",
-6.557328188755392
],
[
"qs",
-6.55968638147019
],
[
"dt",
-6.579466923820265
],
[
"ns",
-6.581085923267036
],
[
"sq",
-6.583571636718558
],
[
"kn",
-6.601121373063897
],
[
"en",
-6.606175960425073
],
[
"fe",
-6.60685998733843
],
[
"tn",
-6.615442326280007
],
[
"wl",
-6.618302734526351
],
[
"pq",
-6.621155961329553
],
[
"ni",
-6.624879415351742
],
[
"yr",
-6.625910436920403
],
[
"qt",
-6.6307505077769555
],
[
"pf",
-6.6308412369595455
],
[
"rn",
-6.631740741295436
],
[
"in",
-6.639584636482613
],
[
"hv",
-6.640247883065955
],
[
"ik",
-6.650348904814281
],
[
"ry",
-6.655536274374892
],
[
"dq",
-6.662650543901028
],
[
"fr",
-6.677637695424519
],
[
"if",
-6.681523029800585
],
[
"mr",
-6.701260323121653
],
[
"ef",
-6.721187968715531
],
[
"am",
-6.722392548689049
],
[
"ms",
-6.722957219864604
],
[
"qe",
-6.723551837092796
],
[
"sy",
-6.728733788049803
],
[
"mt",
-6.739373333977653
],
[
"yv",
-6.742995012080062
],
[
"eh",
-6.74876854304447
],
[
"vh",
-6.750406955107264
],
[
"nn",
-6.7566005866277745
],
[
"yd",
-6.760908295045155
],
[
"tk",
-6.762557739296788
],
[
"nr",
-6.7630435456712235
],
[
"nt",
-6.767721256103645
],
[
"mv",
-6.769331033226527
],
[
"dk",
-6.781027506796054
],
[
"nd",
-6.7895149322464246
],
[
"mg",
-6.7954010994947005
],
[
"kq",
-6.796318304309551
],
[
"qk",
-6.80278943044512
],
[
"gw",
-6.810481762215055
],
[
"dy",
-6.813654384938911
],
[
"ys",
-6.822655724067237
],
[
"vy",
-6.82632505735617
],
[
"wr",
-6.829081855085716
],
[
"mp",
-6.839761205069106
],
[
"fi",
-6.85048550044306
],
[
"aw",
-6.863259418010875
],
[
"pk",
-6.865582196294758
],
[
"hd",
-6.884016503575918
],
[
"ala",
-6.885081628055053
],
[
"pn",
-6.886470616505994
],
[
"fp",
-6.888007121641019
],
[
"ty",
-6.89272350275464
],
[
"cg",
-6.893033028167876
],
[
"tq",
-6.894648525208451
],
[
"ac",
-6.922264253254053
],
[
"dh",
-6.940052819724661
],
[
"th",
-6.946611361274389
],
[
"nf",
-6.94773061409305
],
[
"yt",
-6.947895582263566
],
[
"ne",
-6.950186435523683
],
[
"laa",
-6.9618966428821984
],
[
"mk",
-6.963876569469246
],
[
"fn",
-6.9651216440287556
],
[
"mi",
-6.973364756865921
],
[
"rw",
-6.976192205590854
],
[
"sh",
-6.976252466655415
],
[
"yf",
-6.98466102259021
],
[
"ye",
-6.985376860426236
],
[
"iy",
-6.997339619581055
],
[
"ph",
-7.002791342761229
],
[
"ca",
-7.00714306958699
],
[
"nk",
-7.01679175195992
],
[
"vc",
-7.026717686171381
],
[
"dn",
-7.0329619448909995
],
[
"lc",
-7.04162123422458
],
[
"iq",
-7.052050568293268
],
[
"qd",
-7.056415484136933
],
[
"ht",
-7.057894344010668
],
[
"cr",
-7.062175308655888
],
[
"ey",
-7.067369296531744
],
[
"py",
-7.06762170686601
],
[
"gc",
-7.068569810281758
],
[
"he",
-7.068724837052839
],
[
"ws",
-7.083834967007803
],
[
"qn",
-7.094736754675786
],
[
"hs",
-7.097107793695214
],
[
"gm",
-7.102912164950451
],
[
"cl",
-7.117371301671321
],
[
"hh",
-7.119373071454689
],
[
"cs",
-7.123202520839005
],
[
"nq",
-7.123573885267396
],
[
"me",
-7.140295705363691
],
[
"fy",
-7.145687063516517
],
[
"ggg",
-7.148129052637575
],
[
"aal",
-7.149286650265301
],
[
"yp",
-7.156139099185067
],
[
"qf",
-7.157060249855512
],
[
"wa",
-7.171499064266852
],
[
"ky",
-7.1761559132310495
],
[
"ny",
-7.181344251441002
],
[
"lw",
-7.18428827548775
],
[
"sc",
-7.188554190683416
],
[
"md",
-7.189176149884691
],
[
"rc",
-7.2023406563778956
],
[
"aag",
-7.215624965192319
],
[
"lm",
-7.21766907832032
],
[
"kf",
-7.220208888998105
],
[
"sw",
-7.235424094169154
],
[
"yy",
-7.248760750503438
],
[
"rrr",
-7.256636321517087
],
[
"fk",
-7.26446436072386
],
[
"qh",
-7.272573702000296
],
[
"fq",
-7.277283326780614
],
[
"yn",
-7.279879376992396
],
[
"yq",
-7.282357112300778
],
[
"hf",
-7.294922693842759
],
[
"aga",
-7.3089624623105145
],
[
"mn",
-7.309472360282593
],
[
"ih",
-7.33053451667163
],
[
"ava",
-7.335882965161225
],
[
"cv",
-7.339016321207151
],
[
"hq",
-7.343025081428985
],
[
"wt",
-7.3434986168208845
],
[
"mq",
-7.354400689372477
],
[
"wg",
-7.36382580580741
],
[
"yi",
-7.3644973873263915
],
[
"dw",
-7.366350376841645
],
[
"em",
-7.374811392160398
],
[
"lla",
-7.392768462701696
],
[
"vaa",
-7.41721359052082
],
[
"lll",
-7.425657863607588
],
[
"wv",
-7.428509738612526
],
[
"yk",
-7.431757248014991
],
[
"vm",
-7.433809580239302
],
[
"vw",
-7.445818765018977
],
[
"aar",
-7.4717401761921565
],
[
"hi",
-7.486342911357944
],
[
"tw",
-7.488981567298911
],
[
"cp",
-7.492932113247878
],
[
"fh",
-7.504062101936077
],
[
"wi",
-7.50616719068525
],
[
"qy",
-7.513321084614569
],
[
"wp",
-7.518799984375065
],
[
"all",
-7.53302752114244
],
[
"gag",
-7.533965743298678
],
[
"ara",
-7.537367311945358
],
[
"pw",
-7.5446201708160405
],
[
"raa",
-7.5669264012957616
],
[
"kh",
-7.577026433974584
],
[
"wq",
-7.581697650093153
],
[
"lar",
-7.584606758968199
],
[
"lag",
-7.590419078832234
],
[
"tc",
-7.59811062974657
],
[
"ppp",
-7.611586621905673
],
[
"mf",
-7.616872823329912
],
[
"cd",
-7.624203031434293
],
[
"lgl",
-7.635044047159843
],
[
"rar",
-7.648183656824516
],
[
"lal",
-7.656940108094723
],
[
"arr",
-7.661151098652345
],
[
"ic",
-7.683928182319702
],
[
"wd",
-7.688213319192755
],
[
"fc",
-7.688873646632947
],
[
"hy",
-7.701008250661593
],
[
"wn",
-7.707791318520682
],
[
"ew",
-7.708192951886026
],
[
"wk",
-7.727612230894543
],
[
"vla",
-7.729698852587845
],
[
"agg",
-7.737282862442257
],
[
"rlr",
-7.744727930333475
],
[
"lae",
-7.766894216302047
],
[
"pap",
-7.775081139986716
],
[
"u",
-17.47096861423773
],
[
"b",
-17.671612117272954
],
[
"z",
-18.91242536750401
],
[
"o",
-20.362425367503242
]
]
}
}