Hugging Face's logo

Spaces:
xu-song
/
tokenizer-arena
Running

App Files Community
1
tokenizer-arena
File size: 2,964 Bytes 
751936e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
"""

tokenizer类型：HFTokenizer


## Run



## 来源

- https://github.com/EleutherAI/gpt-neox/blob/main/tools/preprocess_data.py
- https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/tokenizer

"""

import json
import ftfy
from gpt_nexo_20b.tokenizer import build_tokenizer


class Encoder(object):
    def __init__(self, args):
        self.args = args

    def initializer(self):
        # Use Encoder class as a container for global data
        Encoder.tokenizer = build_tokenizer(self.args)

    def encode(self, text):
        if self.args.ftfy:
            text = ftfy.fix_text(text)
        ids = {}
        text_ids = Encoder.tokenizer.tokenize(text)
        return text_ids


class HFConfig:
    """
    jsonl_keys 是干嘛的？

    对应的配置文件：https://github.com/EleutherAI/gpt-neox/blob/main/configs/20B.yml
    "vocab-file": "./20B_checkpoints/20B_tokenizer.json",
    "tokenizer_type": "HFTokenizer",
    """
    def __init__(self):
        self.append_eod = True
        self.ftfy = False
        self.keep_empty = False
        self.log_interval = 100
        self.make_vocab_size_divisible_by = 128
        self.model_parallel_size = 1
        self.padded_vocab_size = 50304
        self.rank = 0
        self.tokenizer_type = 'HFTokenizer'
        self.vocab_file = '20B_tokenizer.json'


class GPTConfig:
    """
    对应的配置文件：https://github.com/EleutherAI/gpt-neox/blob/main/configs/local_setup.yml
    "vocab-file": "data/gpt2-vocab.json",
    "merge-file": "data/gpt2-merges.txt",

    "tokenizer_type": Default = GPT2BPETokenizer  # 默认值
    """
    def __init__(self):
        self.input = './data/enwik8/enwik8.zip'
        self.merge_file = './data/gpt2-merges.txt'
        self.workers = 1

class BERTConfig:
    """ 好像不支持
    "vocab-file": "./20B_checkpoints/20B_tokenizer.json",
    "tokenizer_type": "HFTokenizer",
    """
    pass


def test():
    args = HFConfig()
    encoder = Encoder(args)
    tokenizer = build_tokenizer(args)
    print(f"Vocab size: {tokenizer.vocab_size}")
    encoder.initializer()

    tokens = encoder.encode("中国\ngood job一个人去哪里")
    # 13609 中
    # 23197 国
    print(tokens)
    for token in tokens:
        print(token, Encoder.tokenizer.detokenize([token]))



def convert_vocab():
    vocab = json.load(open("20B_tokenizer.json", "r", encoding="utf-8"))
    json.dump(vocab, open("20B_tokenizer.zh.json", "w", encoding="utf-8"), ensure_ascii=False, indent=2)


def dump_vocab():
    args = HFConfig()
    tokenizer = build_tokenizer(args)
    print(f"Vocab size: {tokenizer.vocab_size}")
    with open("20B.vocab.txt", "w", encoding="utf-8") as f_out:
        for token in tokenizer.vocab:
            f_out.write(token + "\n")

"""
13609 中
23197 国
187 

12311 good
2628  job
27896 一个
13484 人
44781 去
20833 �
105 �
42013 里
"""



if __name__ == "__main__":
    test()
    # convert_vocab()
    # dump_vocab()