scenario-NON-KD-PR-COPY-CDF-ALL-D2_data-cardiffnlp_tweet_sentiment_multilingual_
/
tokenizer_config.json
{ | |
"added_tokens_decoder": { | |
"0": { | |
"content": "[PAD]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"1": { | |
"content": "[CLS]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"2": { | |
"content": "[SEP]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"3": { | |
"content": "[UNK]", | |
"lstrip": false, | |
"normalized": true, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"250001": { | |
"content": "β<extra_id_99>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250002": { | |
"content": "β<extra_id_98>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250003": { | |
"content": "β<extra_id_97>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250004": { | |
"content": "β<extra_id_96>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250005": { | |
"content": "β<extra_id_95>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250006": { | |
"content": "β<extra_id_94>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250007": { | |
"content": "β<extra_id_93>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250008": { | |
"content": "β<extra_id_92>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250009": { | |
"content": "β<extra_id_91>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250010": { | |
"content": "β<extra_id_90>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250011": { | |
"content": "β<extra_id_89>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250012": { | |
"content": "β<extra_id_88>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250013": { | |
"content": "β<extra_id_87>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250014": { | |
"content": "β<extra_id_86>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250015": { | |
"content": "β<extra_id_85>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250016": { | |
"content": "β<extra_id_84>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250017": { | |
"content": "β<extra_id_83>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250018": { | |
"content": "β<extra_id_82>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250019": { | |
"content": "β<extra_id_81>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250020": { | |
"content": "β<extra_id_80>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250021": { | |
"content": "β<extra_id_79>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250022": { | |
"content": "β<extra_id_78>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250023": { | |
"content": "β<extra_id_77>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250024": { | |
"content": "β<extra_id_76>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250025": { | |
"content": "β<extra_id_75>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250026": { | |
"content": "β<extra_id_74>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250027": { | |
"content": "β<extra_id_73>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250028": { | |
"content": "β<extra_id_72>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250029": { | |
"content": "β<extra_id_71>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250030": { | |
"content": "β<extra_id_70>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250031": { | |
"content": "β<extra_id_69>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250032": { | |
"content": "β<extra_id_68>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250033": { | |
"content": "β<extra_id_67>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250034": { | |
"content": "β<extra_id_66>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250035": { | |
"content": "β<extra_id_65>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250036": { | |
"content": "β<extra_id_64>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250037": { | |
"content": "β<extra_id_63>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250038": { | |
"content": "β<extra_id_62>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250039": { | |
"content": "β<extra_id_61>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250040": { | |
"content": "β<extra_id_60>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250041": { | |
"content": "β<extra_id_59>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250042": { | |
"content": "β<extra_id_58>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250043": { | |
"content": "β<extra_id_57>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250044": { | |
"content": "β<extra_id_56>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250045": { | |
"content": "β<extra_id_55>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250046": { | |
"content": "β<extra_id_54>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250047": { | |
"content": "β<extra_id_53>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250048": { | |
"content": "β<extra_id_52>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250049": { | |
"content": "β<extra_id_51>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250050": { | |
"content": "β<extra_id_50>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250051": { | |
"content": "β<extra_id_49>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250052": { | |
"content": "β<extra_id_48>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250053": { | |
"content": "β<extra_id_47>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250054": { | |
"content": "β<extra_id_46>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250055": { | |
"content": "β<extra_id_45>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250056": { | |
"content": "β<extra_id_44>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250057": { | |
"content": "β<extra_id_43>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250058": { | |
"content": "β<extra_id_42>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250059": { | |
"content": "β<extra_id_41>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250060": { | |
"content": "β<extra_id_40>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250061": { | |
"content": "β<extra_id_39>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250062": { | |
"content": "β<extra_id_38>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250063": { | |
"content": "β<extra_id_37>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250064": { | |
"content": "β<extra_id_36>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250065": { | |
"content": "β<extra_id_35>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250066": { | |
"content": "β<extra_id_34>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250067": { | |
"content": "β<extra_id_33>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250068": { | |
"content": "β<extra_id_32>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250069": { | |
"content": "β<extra_id_31>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250070": { | |
"content": "β<extra_id_30>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250071": { | |
"content": "β<extra_id_29>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250072": { | |
"content": "β<extra_id_28>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250073": { | |
"content": "β<extra_id_27>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250074": { | |
"content": "β<extra_id_26>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250075": { | |
"content": "β<extra_id_25>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250076": { | |
"content": "β<extra_id_24>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250077": { | |
"content": "β<extra_id_23>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250078": { | |
"content": "β<extra_id_22>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250079": { | |
"content": "β<extra_id_21>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250080": { | |
"content": "β<extra_id_20>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250081": { | |
"content": "β<extra_id_19>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250082": { | |
"content": "β<extra_id_18>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250083": { | |
"content": "β<extra_id_17>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250084": { | |
"content": "β<extra_id_16>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250085": { | |
"content": "β<extra_id_15>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250086": { | |
"content": "β<extra_id_14>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250087": { | |
"content": "β<extra_id_13>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250088": { | |
"content": "β<extra_id_12>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250089": { | |
"content": "β<extra_id_11>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250090": { | |
"content": "β<extra_id_10>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250091": { | |
"content": "β<extra_id_9>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250092": { | |
"content": "β<extra_id_8>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250093": { | |
"content": "β<extra_id_7>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250094": { | |
"content": "β<extra_id_6>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250095": { | |
"content": "β<extra_id_5>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250096": { | |
"content": "β<extra_id_4>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250097": { | |
"content": "β<extra_id_3>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250098": { | |
"content": "β<extra_id_2>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250099": { | |
"content": "β<extra_id_1>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250100": { | |
"content": "β<extra_id_0>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"250101": { | |
"content": "[MASK]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
} | |
}, | |
"bos_token": "[CLS]", | |
"clean_up_tokenization_spaces": true, | |
"cls_token": "[CLS]", | |
"do_lower_case": false, | |
"eos_token": "[SEP]", | |
"mask_token": "[MASK]", | |
"model_max_length": 1000000000000000019884624838656, | |
"pad_token": "[PAD]", | |
"sep_token": "[SEP]", | |
"sp_model_kwargs": {}, | |
"split_by_punct": false, | |
"tokenizer_class": "DebertaV2Tokenizer", | |
"unk_token": "[UNK]", | |
"vocab_type": "spm" | |
} | |