|
import os |
|
import config |
|
from transformers import AutoTokenizer |
|
from vocab import TokenizerType |
|
|
|
|
|
|
|
|
|
|
|
if config.USE_REMOTE: |
|
tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan2-7B-Chat", trust_remote_code=True) |
|
else: |
|
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) |
|
TOKENIZER_DIR = os.path.join(CURRENT_DIR, "Baichuan2-7B-Chat") |
|
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True) |
|
|
|
|
|
|
|
tokenizer.type = TokenizerType.ByteBPE |
|
|
|
tokenizer.comments = "expand the vocabulary size from 64000 in Baichuan1 to 125696" |
|
|
