g2pt-guacamol-base-deg / tokenizer.json
xchen16's picture
Upload tokenizer
3302334 verified
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 112,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 113,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 114,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 115,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 116,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Sequence",
"normalizers": []
},
"pre_tokenizer": {
"type": "WhitespaceSplit"
},
"post_processor": null,
"decoder": {
"type": "WordPiece",
"prefix": "##",
"cleanup": true
},
"model": {
"type": "WordLevel",
"vocab": {
"<boc>": 0,
"<eoc>": 1,
"<sepc>": 2,
"<bog>": 3,
"<eog>": 4,
"<sepg>": 5,
"IDX_0": 6,
"IDX_1": 7,
"IDX_2": 8,
"IDX_3": 9,
"IDX_4": 10,
"IDX_5": 11,
"IDX_6": 12,
"IDX_7": 13,
"IDX_8": 14,
"IDX_9": 15,
"IDX_10": 16,
"IDX_11": 17,
"IDX_12": 18,
"IDX_13": 19,
"IDX_14": 20,
"IDX_15": 21,
"IDX_16": 22,
"IDX_17": 23,
"IDX_18": 24,
"IDX_19": 25,
"IDX_20": 26,
"IDX_21": 27,
"IDX_22": 28,
"IDX_23": 29,
"IDX_24": 30,
"IDX_25": 31,
"IDX_26": 32,
"IDX_27": 33,
"IDX_28": 34,
"IDX_29": 35,
"IDX_30": 36,
"IDX_31": 37,
"IDX_32": 38,
"IDX_33": 39,
"IDX_34": 40,
"IDX_35": 41,
"IDX_36": 42,
"IDX_37": 43,
"IDX_38": 44,
"IDX_39": 45,
"IDX_40": 46,
"IDX_41": 47,
"IDX_42": 48,
"IDX_43": 49,
"IDX_44": 50,
"IDX_45": 51,
"IDX_46": 52,
"IDX_47": 53,
"IDX_48": 54,
"IDX_49": 55,
"IDX_50": 56,
"IDX_51": 57,
"IDX_52": 58,
"IDX_53": 59,
"IDX_54": 60,
"IDX_55": 61,
"IDX_56": 62,
"IDX_57": 63,
"IDX_58": 64,
"IDX_59": 65,
"IDX_60": 66,
"IDX_61": 67,
"IDX_62": 68,
"IDX_63": 69,
"IDX_64": 70,
"IDX_65": 71,
"IDX_66": 72,
"IDX_67": 73,
"IDX_68": 74,
"IDX_69": 75,
"IDX_70": 76,
"IDX_71": 77,
"IDX_72": 78,
"IDX_73": 79,
"IDX_74": 80,
"IDX_75": 81,
"IDX_76": 82,
"IDX_77": 83,
"IDX_78": 84,
"IDX_79": 85,
"IDX_80": 86,
"IDX_81": 87,
"IDX_82": 88,
"IDX_83": 89,
"IDX_84": 90,
"IDX_85": 91,
"IDX_86": 92,
"IDX_87": 93,
"IDX_88": 94,
"IDX_89": 95,
"ATOM_C": 96,
"ATOM_N": 97,
"ATOM_O": 98,
"ATOM_F": 99,
"ATOM_B": 100,
"ATOM_Br": 101,
"ATOM_Cl": 102,
"ATOM_I": 103,
"ATOM_P": 104,
"ATOM_S": 105,
"ATOM_Se": 106,
"ATOM_Si": 107,
"BOND_SINGLE": 108,
"BOND_DOUBLE": 109,
"BOND_TRIPLE": 110,
"BOND_AROMATIC": 111
},
"unk_token": "[UNK]"
}
}