from transformers import AutoTokenizer | |
tokenizer = AutoTokenizer.from_pretrained("yhavinga/dutch-llama-tokenizer", trust_remote_code=True) | |
from transformers import AutoTokenizer | |
tokenizer = AutoTokenizer.from_pretrained("yhavinga/dutch-llama-tokenizer", trust_remote_code=True) | |