utils/oov_util.py

import os
import json
from vocab import all_tokenizer_config, load_tokenizer, TokenizerImpl


text = "hello; Замглавы управления развития; 특히 주소 15~17번 홀에선 3연속;" \
       " 確実に春が近づいてること;  a közoktatással? _ Belföld;" \
       " pumë, i vjetër, vjeç; ئەردوغان ۋە قىرغىزىستان ;" \
       " निम्न में से कौन सा हारडवेयर; ተለዋዋጭ የግድግዳ ; Дзейныя асобы:;" \
       " « અમરેલીનાં મહિલા વિકાસ; 🦙❤❥웃유♋☮✊;" \
       "װיקיװערטערבוך "
whitespace = "\t   \n\n\r  "
bytes = b"\x00\x01\x02\x03\x04".decode('utf-8')

text += whitespace


def get_unk(tokenizer_config):
    tokenizer = load_tokenizer(tokenizer_config)
    if hasattr(tokenizer, "unk_token"):
        return f"{tokenizer.unk_token}, {tokenizer.unk_token_id}"
    else:
        return "unk_token not found"


# def infer_tokenizer_impl(tokenizer_config):
def infer_tokenizer_type(tokenizer_config):
    tokenizer = load_tokenizer(tokenizer_config)
    if tokenizer_config.impl == TokenizerImpl.TikToken:
        return "tiktoken"
    if hasattr(tokenizer, "backend_tokenizer"):
        return str(type(tokenizer.backend_tokenizer.model))  # type(tokenizer._tokenizer.model))
    # orion: sp_model.Load(vocab_file)，继承 PreTrainedTokenizer
    elif hasattr(tokenizer, "sp_model"):  # 基于 sentencepiece 包
        # for i in range(tokenizer.sp_model.piece_size()):
        #     if tokenizer.sp_model.is_byte(i):
        #         print("")
        return f"sp_model, byte_num: {sum([tokenizer.sp_model.is_byte(i) for i in range(tokenizer.sp_model.piece_size())])}"

    # sp.Load(model_path)  ，并且包括image_tokenizer
    elif "glm-" in tokenizer_config.name_or_path:
        return f"byte_num: {sum([tokenizer.sp_tokenizer.text_tokenizer.sp.is_byte(i) for i in range(tokenizer.sp_tokenizer.text_tokenizer.sp.piece_size())])}"
    # sp.Load(model_path)  ，没有image_tokenizer
    elif "glm2-" in tokenizer_config.name_or_path \
            or "glm3-" in tokenizer_config.name_or_path \
            or "CharacterGLM-6B" in tokenizer_config.name_or_path:
        return f"byte_num: {sum([tokenizer.tokenizer.sp_model.is_byte(i) for i in range(tokenizer.tokenizer.sp_model.piece_size())])}"
    elif "abeja/gpt-neox-japanese-2.7b" == tokenizer_config.name_or_path:  # 支持 byte-level，解决oov问题
        return f"japanese-bpe: https://github.com/tanreinama/Japanese-BPEEncoder_V2"
    # bert-base-japanese： 特殊的地方在于 "word_tokenizer_type": "mecab"，见 https://huggingface.co/tohoku-nlp/bert-base-japanese/blob/main/tokenizer_config.json
    elif "bert-base-japanese" in tokenizer_config.name_or_path:
        return "wordpiece.MecabTokenizer, 支持byte-level https://taku910.github.io/mecab/"
    elif "moss" in tokenizer_config.name_or_path:
        return "应该是 sentencepiece.byte_bpe,待确认"
    elif "byt5" in tokenizer_config.name_or_path:
        return "未知，待定"
    else:
        print("catch", tokenizer_config.name_or_path)
        raise "error"





def test_lossless(tokenizer_config):
    """
    xlm-roberta-base 为什么oov这么少？是因为有 byte吗？
    :param tokenizer_config:
    :return:
    """
    tokenizer = load_tokenizer(tokenizer_config)
    encoding = tokenizer.encode(text, add_special_tokens=False)
    decoding = tokenizer.decode(encoding)

    if text in decoding:
        # print(tokenizer_config.name, tokenizer_config.impl, "lossless: true")
        pass
    else:
        unk_count = sum([1 for token_id in encoding if token_id == tokenizer.unk_token_id])
        oov_tokens = []
        # if tokenizer_config.impl == TokenizerImpl.SentencePiece:
        #     print(sum([tokenizer.is_byte(i) for i in range(tokenizer.piece_size())]))

        print("#######"*5)
        print(f"{tokenizer_config.name_or_path}, {infer_tokenizer_type(tokenizer_config)}\n"
              f"lossless: false; unk_token: {get_unk(tokenizer_config)},"
              f" unk_ratio: {unk_count/len(encoding):.4f}; oov: []")
        for i in range(len(text)):
            if text[i] != decoding[i]:
                # print(f"text[{i}]     = {str(bytes(text[i:], 'utf-8'))}\n"
                #       f"decoding[{i}] = {str(bytes(decoding[i:], 'utf-8'))}")
                print(f"text[{i}]     = {json.dumps(text[i:], ensure_ascii=False)}, \n"
                      f"decoding[{i}] = {json.dumps(decoding[i:], ensure_ascii=False)}")

                break



for config in all_tokenizer_config:
    # if "xlm-roberta-base" in config.name:
    # if "xlm-roberta-base" in config.name:
    # if "chatglm3-6b" in config.name:
    # if "bert-base-japanese" in config.name:
    # if "moss" in config.name:
    # if "byt5" in config.name:
    if "baichuan" in config.name_or_path:
    # if "CharacterGLM-6B" in config.name:
    # if "fastchat-t5" in config.name:  # 报错 pyo3_runtime.PanicException: AddedVocabulary bad split
    # if True:
        # test_unk(config)
         test_lossless(config)