{ "additional_special_tokens": [ { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|startoftranscript|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|en|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|zh|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|de|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|es|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ru|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ko|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|fr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ja|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|pt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|tr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|pl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ca|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|nl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ar|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|it|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|id|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|hi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|fi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|vi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|he|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|uk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|el|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ms|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|cs|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ro|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|da|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|hu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ta|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|no|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|th|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ur|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|hr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|bg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|lt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|la|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|mi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ml|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|cy|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|te|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|fa|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|lv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|bn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|az|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|kn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|et|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|mk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|br|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|eu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|is|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|hy|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ne|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|mn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|bs|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|kk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sq|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sw|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|gl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|mr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|pa|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|si|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|km|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|yo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|so|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|af|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|oc|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ka|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|be|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|tg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sd|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|gu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|am|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|yi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|lo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|uz|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|fo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ht|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ps|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|tk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|nn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|mt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sa|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|lb|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|my|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|bo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|tl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|mg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|as|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|tt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|haw|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ln|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ha|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ba|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|jw|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|su|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|cv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ky|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|sh|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|ug|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|translate|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|transcribe|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|startoflm|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|startofprev|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|nocaptions|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, { "content": "<|notimestamps|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false } ], "bos_token": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, "eos_token": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, "pad_token": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false }, "unk_token": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false } }