Upload tokenizer
280b43c
verified
|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 24, |
|
"content": "[UNK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 25, |
|
"content": "[PAD]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 26, |
|
"content": "[CLS]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 27, |
|
"content": "[SEP]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 28, |
|
"content": "[MASK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": null, |
|
"pre_tokenizer": { |
|
"type": "Split", |
|
"pattern": { |
|
"String": "" |
|
}, |
|
"behavior": "Isolated", |
|
"invert": false |
|
}, |
|
"post_processor": null, |
|
"decoder": null, |
|
"model": { |
|
"type": "WordLevel", |
|
"vocab": { |
|
"A": 0, |
|
"R": 1, |
|
"N": 2, |
|
"D": 3, |
|
"C": 4, |
|
"Q": 5, |
|
"E": 6, |
|
"G": 7, |
|
"H": 8, |
|
"I": 9, |
|
"L": 10, |
|
"K": 11, |
|
"M": 12, |
|
"F": 13, |
|
"P": 14, |
|
"S": 15, |
|
"T": 16, |
|
"W": 17, |
|
"Y": 18, |
|
"V": 19, |
|
"B": 20, |
|
"Z": 21, |
|
"X": 22, |
|
"-": 23, |
|
"[UNK]": 24, |
|
"[PAD]": 25, |
|
"[CLS]": 26, |
|
"[SEP]": 27, |
|
"[MASK]": 28 |
|
}, |
|
"unk_token": "[UNK]" |
|
} |
|
} |
