adibm's picture
Upload tokenizer
1d4e99c
{
"additional_special_tokens": [
"<|begintarget|>",
"<|endtarget|>",
"<|begincontext|>",
"<|endcontext|>",
"<SYSTEM>",
"<USER>",
"<|beginlastuserutterance|>",
"<|endlastuserutterance|>",
"<|begindsts|>",
"<|enddsts|>",
"<|begindst|>",
"<|enddst|>",
"<|beginbelief|>",
"<|endbelief|>",
"<|beginresponse|>",
"<|endresponse|>",
"<|beginaction|>",
"<|endaction|>",
"<|beginuseraction|>",
"<|enduseraction|>",
"<|sysactions|>",
"<|beginintent|>",
"<|endintent|>",
"<|beginrequestedslots|>",
"<|endrequestedslots|>",
"<|promptdst|>",
"<|promptaction|>",
"<|promptresponse|>",
"<|pad|>",
"<|endoftext|>",
"<|startoftext|>",
"<|beginschema|>",
"<|endschema|>",
"<|schemaname|>",
"<|schemadescription|>",
"<|beginschemaintent|>",
"<|endschemaintent|>",
"<|intentrequiredslots|>",
"<|intentresultslots|>",
"<|intentoptionalslots|>",
"<|possiblevalues|>",
"<|endpossiblevalues|>",
"<|beginschemaslot|>",
"<|endschemaslot|>",
"<|schemaslotvalues|>",
"<|beginserviceresults|>",
"<|endserviceresults|>",
"<|value|>",
"<|domain|>",
"<|item|>",
"<|actionvalue|>"
],
"bos_token": "<|startoftext|>",
"eos_token": "<|endtarget|>",
"pad_token": "<|pad|>",
"unk_token": "<unk>"
}