xu-song's picture
add more tokenizers
f4973d4
raw
history blame
192 Bytes
"""
https://huggingface.co/internlm/internlm-chat-7b
"""
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("internlm/internlm-chat-7b", trust_remote_code=True)