vocab/qwen/__init__.py

"""

依赖 torch tiktoken
依赖 transformer 4.31.0 及以上，
"""

from transformers import AutoTokenizer

# 请注意：分词器默认行为已更改为默认关闭特殊token攻击防护。
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-VL-Chat", trust_remote_code=True)

def test():
    encoding = tokenizer.encode("测试华为手机10086        8个空格")
    for token_id in encoding:
        token = tokenizer.convert_ids_to_tokens([token_id])[0].decode("utf-8")
        print(token_id, ":", token)

if __name__ == "__main__":
    test()