xu-song's picture
add more tokenizers
a1b0cd0
raw
history blame
226 Bytes
"""
https://huggingface.co/spaces/deepseek-ai/deepseek-coder-7b-instruct
"""
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-33b-instruct", trust_remote_code=True)