File size: 2,016 Bytes
dd9a529 9d3af0b dd9a529 5488b1b 9d3af0b dd9a529 5488b1b 9d3af0b dd9a529 5488b1b 9d3af0b dd9a529 5488b1b 9d3af0b dd9a529 5488b1b dd9a529 5488b1b 9d3af0b dd9a529 9d3af0b 5488b1b 9d3af0b 5488b1b 9d3af0b 5488b1b dd9a529 5488b1b dd9a529 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 |
{
"added_tokens_decoder": {
"0": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "[رابط]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"6": {
"content": "[بريد]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"7": {
"content": "[مستخدم]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
}
},
"clean_up_tokenization_spaces": true,
"cls_token": "[CLS]",
"do_basic_tokenize": true,
"do_lower_case": false,
"mask_token": "[MASK]",
"max_len": 512,
"max_length": 512,
"model_max_length": 512,
"never_split": [
"[بريد]",
"[مستخدم]",
"[رابط]"
],
"pad_to_multiple_of": null,
"pad_token": "[PAD]",
"pad_token_type_id": 0,
"padding_side": "right",
"sep_token": "[SEP]",
"stride": 0,
"strip_accents": null,
"tokenize_chinese_chars": true,
"tokenizer_class": "BertTokenizer",
"truncation_side": "right",
"truncation_strategy": "longest_first",
"unk_token": "[UNK]"
}
|