vocab/bloom/test_oov_bloom.py

import difflib
import webbrowser
from transformers import AutoTokenizer
from data_sample.oov_base import space_tokens, jd_vocab_tokens, docs


tokenizer = AutoTokenizer.from_pretrained("tokenizer")

def test_oov():
    d = difflib.HtmlDiff(wrapcolumn=50)

    raw_lines = []
    decode_lines = []
    for line in space_tokens + jd_vocab_tokens + docs:
        tokens = tokenizer.encode(line)
        decode_line = tokenizer.decode(tokens)
        if line != decode_line:
            raw_lines.append(line)
            decode_lines.append(decode_line)

    q = d.make_file(raw_lines, decode_lines)
    with open('diff.html', 'w', encoding="utf-8") as f_new:
        f_new.write(q)
    webbrowser.open('diff.html')
if __name__ == "__main__":
    test_oov()