vocab/gpt_neox_chinese_v1/mock.py

import copy
import json

input_path = "20B_tokenizer_chinese.json"

tokenizer = json.load(open(input_path, "r", encoding="utf-8"))

vocab = tokenizer["model"]["vocab"]


for k, v in copy.deepcopy(vocab).items():
    vocab[str(v)] = v
    vocab.pop(k)

out_path = input_path.replace(".json", ".mock.json")
with open(out_path, "w", encoding="utf-8") as f_out:
    f_out.write(json.dumps(tokenizer, ensure_ascii=False, indent=2))