vocab/baichuan2/special_token.py

import os
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model
import sentencepiece as spm

baichuan_spm = sp_pb2_model.ModelProto()
baichuan_spm.ParseFromString(open("Baichuan2-7B-Chat/tokenizer.model", "rb").read())


vocab_size = len(baichuan_spm.pieces)
for i in range(vocab_size):
    piece = baichuan_spm.pieces[i]
    if "reser" in piece.piece:
        print(i, str(piece).strip().replace("\n", ", "))