nllb-200-1.3b-kbd-v0.2 / added_tokens.json
panagoa's picture
Upload tokenizer
e44846b verified
{
"<mask>": 271020,
"ace_Arab": 271021,
"ace_Latn": 271022,
"acm_Arab": 271023,
"acq_Arab": 271024,
"aeb_Arab": 271025,
"afr_Latn": 271026,
"ajp_Arab": 271027,
"aka_Latn": 271028,
"als_Latn": 271029,
"amh_Ethi": 271030,
"apc_Arab": 271031,
"arb_Arab": 271032,
"ars_Arab": 271033,
"ary_Arab": 271034,
"arz_Arab": 271035,
"asm_Beng": 271036,
"ast_Latn": 271037,
"awa_Deva": 271038,
"ayr_Latn": 271039,
"azb_Arab": 271040,
"azj_Latn": 271041,
"bak_Cyrl": 271042,
"bam_Latn": 271043,
"ban_Latn": 271044,
"bel_Cyrl": 271045,
"bem_Latn": 271046,
"ben_Beng": 271047,
"bho_Deva": 271048,
"bjn_Arab": 271049,
"bjn_Latn": 271050,
"bod_Tibt": 271051,
"bos_Latn": 271052,
"bug_Latn": 271053,
"bul_Cyrl": 271054,
"cat_Latn": 271055,
"ceb_Latn": 271056,
"ces_Latn": 271057,
"cjk_Latn": 271058,
"ckb_Arab": 271059,
"crh_Latn": 271060,
"cym_Latn": 271061,
"dan_Latn": 271062,
"deu_Latn": 271063,
"dik_Latn": 271064,
"dyu_Latn": 271065,
"dzo_Tibt": 271066,
"ell_Grek": 271067,
"eng_Latn": 271068,
"epo_Latn": 271069,
"est_Latn": 271070,
"eus_Latn": 271071,
"ewe_Latn": 271072,
"fao_Latn": 271073,
"fij_Latn": 271074,
"fin_Latn": 271075,
"fon_Latn": 271076,
"fra_Latn": 271077,
"fur_Latn": 271078,
"fuv_Latn": 271079,
"gaz_Latn": 271080,
"gla_Latn": 271081,
"gle_Latn": 271082,
"glg_Latn": 271083,
"grn_Latn": 271084,
"guj_Gujr": 271085,
"hat_Latn": 271086,
"hau_Latn": 271087,
"heb_Hebr": 271088,
"hin_Deva": 271089,
"hne_Deva": 271090,
"hrv_Latn": 271091,
"hun_Latn": 271092,
"hye_Armn": 271093,
"ibo_Latn": 271094,
"ilo_Latn": 271095,
"ind_Latn": 271096,
"isl_Latn": 271097,
"ita_Latn": 271098,
"jav_Latn": 271099,
"jpn_Jpan": 271100,
"kab_Latn": 271101,
"kac_Latn": 271102,
"kam_Latn": 271103,
"kan_Knda": 271104,
"kas_Arab": 271105,
"kas_Deva": 271106,
"kat_Geor": 271107,
"kaz_Cyrl": 271108,
"kbd_Cyrl": 271109,
"kbp_Latn": 271110,
"kea_Latn": 271111,
"khk_Cyrl": 271112,
"khm_Khmr": 271113,
"kik_Latn": 271114,
"kin_Latn": 271115,
"kir_Cyrl": 271116,
"kmb_Latn": 271117,
"kmr_Latn": 271118,
"knc_Arab": 271119,
"knc_Latn": 271120,
"kon_Latn": 271121,
"kor_Hang": 271122,
"lao_Laoo": 271123,
"lij_Latn": 271124,
"lim_Latn": 271125,
"lin_Latn": 271126,
"lit_Latn": 271127,
"lmo_Latn": 271128,
"ltg_Latn": 271129,
"ltz_Latn": 271130,
"lua_Latn": 271131,
"lug_Latn": 271132,
"luo_Latn": 271133,
"lus_Latn": 271134,
"lvs_Latn": 271135,
"mag_Deva": 271136,
"mai_Deva": 271137,
"mal_Mlym": 271138,
"mar_Deva": 271139,
"min_Latn": 271140,
"mkd_Cyrl": 271141,
"mlt_Latn": 271142,
"mni_Beng": 271143,
"mos_Latn": 271144,
"mri_Latn": 271145,
"mya_Mymr": 271146,
"nld_Latn": 271147,
"nno_Latn": 271148,
"nob_Latn": 271149,
"npi_Deva": 271150,
"nso_Latn": 271151,
"nus_Latn": 271152,
"nya_Latn": 271153,
"oci_Latn": 271154,
"ory_Orya": 271155,
"pag_Latn": 271156,
"pan_Guru": 271157,
"pap_Latn": 271158,
"pbt_Arab": 271159,
"pes_Arab": 271160,
"plt_Latn": 271161,
"pol_Latn": 271162,
"por_Latn": 271163,
"prs_Arab": 271164,
"quy_Latn": 271165,
"ron_Latn": 271166,
"run_Latn": 271167,
"rus_Cyrl": 271168,
"sag_Latn": 271169,
"san_Deva": 271170,
"sat_Beng": 271171,
"scn_Latn": 271172,
"shn_Mymr": 271173,
"sin_Sinh": 271174,
"slk_Latn": 271175,
"slv_Latn": 271176,
"smo_Latn": 271177,
"sna_Latn": 271178,
"snd_Arab": 271179,
"som_Latn": 271180,
"sot_Latn": 271181,
"spa_Latn": 271182,
"srd_Latn": 271183,
"srp_Cyrl": 271184,
"ssw_Latn": 271185,
"sun_Latn": 271186,
"swe_Latn": 271187,
"swh_Latn": 271188,
"szl_Latn": 271189,
"tam_Taml": 271190,
"taq_Latn": 271191,
"taq_Tfng": 271192,
"tat_Cyrl": 271193,
"tel_Telu": 271194,
"tgk_Cyrl": 271195,
"tgl_Latn": 271196,
"tha_Thai": 271197,
"tir_Ethi": 271198,
"tpi_Latn": 271199,
"tsn_Latn": 271200,
"tso_Latn": 271201,
"tuk_Latn": 271202,
"tum_Latn": 271203,
"tur_Latn": 271204,
"twi_Latn": 271205,
"tzm_Tfng": 271206,
"uig_Arab": 271207,
"ukr_Cyrl": 271208,
"umb_Latn": 271209,
"urd_Arab": 271210,
"uzn_Latn": 271211,
"vec_Latn": 271212,
"vie_Latn": 271213,
"war_Latn": 271214,
"wol_Latn": 271215,
"xho_Latn": 271216,
"ydd_Hebr": 271217,
"yor_Latn": 271218,
"yue_Hant": 271219,
"zho_Hans": 271220,
"zho_Hant": 271221,
"zsm_Latn": 271222,
"zul_Latn": 271223
}