ocisd4/llama_tokenizer_ext_zhtw

from transformers import LlamaTokenizer

tokenizer = LlamaTokenizer.from_pretrained(  
        'ocisd4/llama_tokenizer_ext_zhtw',  
        pad_token='<unk>',  
        add_bos_token=True,  
        add_eos_token=False  
)  

#vocab size: 36128     

print(tokenizer.tokenize('今天天氣真好！'))   
#['▁', '今', '天', '天', '氣', '真', '好', '！']

print(tokenizer.encode('今天天氣真好！'))  
#[1, 29871, 31482, 30408, 30408, 32045, 30848, 31076, 30584]

print(tokenizer.decode(tokenizer.encode('今天天氣真好！')))  
# <s>今天天氣真好！