xu-song's picture
update
0ce6477
raw
history blame
234 Bytes
from transformers import LlamaTokenizer
tokenizer = LlamaTokenizer.from_pretrained("ziqingyang/chinese-llama-2-7b")
tokenizer.comments = "重新设计了新词表(大小:55296),进一步提升了中文字词的覆盖程度"