Spaces:

xu-song
/

tokenizer-arena

Running

App Files Files Community

xu-song commited on Sep 6, 2023

Commit

0ce6477

•

1 Parent(s): 6551d2c

update

Browse files

Files changed (21) hide show

app.py +1 -1
vocab/__init__.py +8 -4
vocab/{chinese_llama_lora_7b → chinese_llama}/README.md +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/__init__.py +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/README.md +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/chinese_sp.model +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merge_tokenizers.py +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merged_tokenizer_hf/special_tokens_map.json +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merged_tokenizer_hf/tokenizer.model +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merged_tokenizer_hf/tokenizer_config.json +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merged_tokenizer_sp/chinese_llama.model +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/tokenizer/config.json +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/tokenizer/special_tokens_map.json +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/tokenizer/tokenizer.model +0 -0
vocab/{chinese_llama_lora_7b → chinese_llama}/tokenizer/tokenizer_config.json +0 -0
vocab/chinese_llama2/__init__.py +5 -0
vocab/gpt_nexo_20b/20B_tokenizer.txt +0 -0
vocab/llama/README.md +4 -1
vocab/llama/gpt_neox/append_zh.txt +3884 -0
vocab/llama/gpt_neox/get_oov_zh_tokens.py +20 -0
vocab/llama/gpt_neox/load_by_gpt_neox.py +18 -0

app.py CHANGED Viewed

@@ -19,7 +19,7 @@
 - OOV
 - feedback位置
 - gpt4, gpt3.5 的overlap tokens 有问题。
 plots

 - OOV
 - feedback位置
 - gpt4, gpt3.5 的overlap tokens 有问题。
+- qwen:  ValueError: Unclosed image token
 plots

vocab/__init__.py CHANGED Viewed

@@ -11,6 +11,8 @@ tokenizer.vocab_size
 tokenizer.get_vocab()   # gpt-neox-20b, llama
 tokenizer.type = TokenizerType.ByteBPE.name
 tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.com/facebookresearch/llama/blob/main/llama/tokenizer.py
   - bert
     - 特征
     - 示例：
@@ -22,10 +24,11 @@ tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.c
   - tiktoken
   - icetk
   - hf_tokenizer
-    - 特征：.model 是 tokenizer.models.BPE 类型，词典有 Ġ  "\u0120" 开头，有 merge.txt
     - 示例：gpt_neox_20b, moss
-  - gpt3.5 gpt4
-    - 特征：tiktoken
 tokenizer.comments = "split all numbers into individual digits, " \
                      "and fallback to bytes to decompose unknown UTF-8 characters"
@@ -72,7 +75,8 @@ all_tokenizers = [
     #
     # #### llama alpaca系列
     "llama",  # '中文单字': 700, '中文多字': 0
-    "chinese_llama_lora_7b",  #
     # "chinese_alpaca_lora_7b",  # 中文Alpaca模型在上述中文LLaMA模型的基础上进一步使用了指令数据进行精调。
     # "belle_llama_ext_7b",
     # "alpaca_7b",

 tokenizer.get_vocab()   # gpt-neox-20b, llama
 tokenizer.type = TokenizerType.ByteBPE.name
 tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.com/facebookresearch/llama/blob/main/llama/tokenizer.py
+  "HFGPT2Tokenizer", "HFTokenizer", "GPT2BPETokenizer", "CharLevelTokenizer", "TiktokenTokenizer", "SPMTokenizer", https://github.com/EleutherAI/gpt-neox/blob/main/tools/preprocess_data.py
   - bert
     - 特征
     - 示例：
   - tiktoken
   - icetk
   - hf_tokenizer
+    - 特征：.model 是 tokenizer.models.BPE 类型，词典有 Ġ  "\u0120" 开头，有1个tokenizer.json(包括 merge vocab)，或者分开独立文件
     - 示例：gpt_neox_20b, moss
+  - tiktoken
+    - 特征：空格就是空格，
+    - 示例：gpt3.5 gpt4
 tokenizer.comments = "split all numbers into individual digits, " \
                      "and fallback to bytes to decompose unknown UTF-8 characters"
     #
     # #### llama alpaca系列
     "llama",  # '中文单字': 700, '中文多字': 0
+    "chinese_llama",  #
+    "chinese_llama2",  #
     # "chinese_alpaca_lora_7b",  # 中文Alpaca模型在上述中文LLaMA模型的基础上进一步使用了指令数据进行精调。
     # "belle_llama_ext_7b",
     # "alpaca_7b",

vocab/{chinese_llama_lora_7b → chinese_llama}/README.md RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/__init__.py RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/README.md RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/chinese_sp.model RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merge_tokenizers.py RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merged_tokenizer_hf/special_tokens_map.json RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merged_tokenizer_hf/tokenizer.model RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merged_tokenizer_hf/tokenizer_config.json RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/merge_tokenizer/merged_tokenizer_sp/chinese_llama.model RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/tokenizer/config.json RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/tokenizer/special_tokens_map.json RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/tokenizer/tokenizer.model RENAMED Viewed

File without changes

vocab/{chinese_llama_lora_7b → chinese_llama}/tokenizer/tokenizer_config.json RENAMED Viewed

File without changes

vocab/chinese_llama2/__init__.py ADDED Viewed

	@@ -0,0 +1,5 @@

+from transformers import LlamaTokenizer
+tokenizer = LlamaTokenizer.from_pretrained("ziqingyang/chinese-llama-2-7b")
+tokenizer.comments = "重新设计了新词表（大小：55296），进一步提升了中文字词的覆盖程度"

vocab/gpt_nexo_20b/20B_tokenizer.txt DELETED Viewed

The diff for this file is too large to render. See raw diff

vocab/llama/README.md CHANGED Viewed

@@ -1,6 +1,9 @@
-## ss
 ## 完整性

+##
+tokenizer.json 和 tokenizer.model 是 都需要吗？
 ## 完整性

vocab/llama/gpt_neox/append_zh.txt ADDED Viewed

	@@ -0,0 +1,3884 @@

+倏
+衣
+徙
+晾
+逃
+匿
+帐
+氧
+呢
+氦
+馀
+乜
+夺
+镗
+珙
+凶
+奥
+撵
+僵
+农
+锺
+橘
+奋
+朦
+淫
+罄
+桌
+演
+読
+巍
+咕
+班
+垩
+袄
+珂
+酪
+锐
+议
+遏
+黜
+觅
+胧
+戒
+楣
+买
+咬
+沁
+惊
+桃
+羨
+哑
+撂
+把
+阆
+雷
+谤
+虽
+枭
+孔
+佯
+袭
+悍
+皖
+冤
+笏
+笠
+租
+仲
+冇
+尝
+费
+癀
+匾
+亩
+浜
+席
+欺
+般
+润
+匪
+勾
+畹
+遥
+掐
+择
+衬
+板
+痈
+柢
+礴
+姆
+耆
+缤
+祜
+揶
+偻
+傻
+绰
+昙
+谈
+徽
+盼
+圩
+冼
+尹
+寨
+侗
+剧
+缅
+谭
+糟
+袒
+挥
+槛
+谕
+宙
+艮
+惶
+贡
+琵
+泫
+屈
+丙
+怠
+挣
+夸
+浑
+纷
+冷
+辆
+觞
+脆
+惧
+犷
+斎
+跩
+寒
+苇
+筋
+癣
+凿
+惮
+綦
+舅
+贩
+啼
+缰
+颦
+善
+祗
+柑
+绅
+扪
+奄
+稠
+槓
+彬
+胫
+啷
+琶
+渋
+惦
+囝
+叟
+仆
+抛
+锟
+挈
+诘
+扩
+巿
+徊
+份
+愁
+泵
+扣
+派
+牠
+团
+埋
+恍
+阚
+郦
+傣
+巳
+悯
+碁
+可以
+沏
+柞
+豊
+牧
+绶
+桩
+锯
+栄
+裙
+夔
+谍
+匍
+佬
+抿
+脩
+辽
+闪
+绚
+髻
+巡
+曦
+祈
+镭
+降
+咤
+庵
+沦
+堿
+廷
+睇
+逸
+芮
+挚
+胯
+颂
+抓
+丽
+妣
+琪
+杞
+另
+穂
+垫
+楔
+妤
+师
+淤
+筐
+笈
+莓
+忏
+兀
+庐
+阎
+邋
+披
+覃
+囧
+旦
+叨
+肠
+侑
+啫
+恵
+桡
+酱
+肩
+倒
+幅
+干
+歴
+墉
+频
+脐
+圳
+卒
+绒
+罕
+差
+厝
+苓
+忌
+炕
+粽
+芎
+偾
+党
+惩
+哨
+臀
+诏
+撩
+甕
+绮
+氾
+伟
+楦
+杠
+氐
+跑
+栀
+匀
+验
+棕
+凄
+痺
+苫
+赧
+沥
+厕
+剪
+欢
+杳
+衾
+悽
+罗
+担
+谱
+顶
+掉
+厢
+炊
+冯
+苗
+彿
+讶
+冏
+疝
+単
+盯
+汕
+纶
+讪
+逢
+镛
+急
+框
+枫
+阜
+晗
+勐
+莘
+弧
+承
+霉
+备
+臣
+缢
+殓
+擞
+购
+髋
+甩
+叶
+咲
+杷
+眸
+秣
+骄
+扭
+谲
+寿
+剤
+匹
+柄
+债
+晰
+释
+哧
+床
+攻
+乖
+躯
+辫
+幕
+笹
+喉
+楝
+琅
+臾
+呃
+液
+焚
+账
+总
+恪
+沧
+闇
+埕
+裟
+曝
+积
+漱
+啦
+桨
+泾
+巾
+孛
+応
+泡
+税
+透
+妞
+推
+甥
+隋
+亡
+灿
+亮
+紫
+郷
+催
+楞
+叵
+惚
+绊
+凯
+迩
+骆
+筑
+纂
+凋
+宠
+署
+惜
+炭
+纣
+肫
+捋
+楂
+逆
+霜
+缉
+离
+汴
+揍
+肢
+惇
+糜
+殡
+怆
+碇
+仿
+玫
+刃
+啧
+徴
+栈
+辉
+励
+逵
+遶
+廃
+秭
+哀
+沌
+赁
+寥
+扈
+貔
+傲
+着
+盥
+叛
+舍
+売
+兹
+呤
+护
+沈
+砾
+阡
+抹
+卍
+挹
+偕
+寸
+姐
+违
+职
+碴
+抒
+僖
+耍
+皑
+尼
+较
+颛
+跨
+浓
+咪
+剃
+浴
+匈
+予
+睨
+霓
+囤
+毂
+刨
+捩
+锥
+衔
+杂
+见
+让
+苞
+轼
+逐
+殇
+袅
+觎
+仍
+焘
+芈
+迤
+邹
+衢
+怕
+擂
+喵
+妒
+黯
+悸
+屉
+舀
+匐
+锻
+帛
+勋
+伞
+漪
+疡
+渣
+舌
+畏
+卿
+庙
+济
+忿
+跳
+貅
+韩
+芊
+固
+胤
+伐
+企
+呷
+玷
+堡
+眠
+敷
+啤
+祕
+辅
+玻
+禧
+谅
+慨
+浆
+筛
+喺
+挪
+谒
+霭
+覆
+骷
+骨
+栽
+缄
+谢
+车
+谴
+吞
+夹
+凡
+舒
+弱
+拙
+震
+芃
+橡
+浃
+雎
+细
+貘
+躺
+汨
+禁
+骼
+叽
+氲
+虞
+沭
+羸
+宽
+珲
+卞
+气
+漓
+揩
+霁
+背
+皮
+乙
+盲
+穴
+咫
+噤
+忤
+怖
+枝
+怦
+攘
+苟
+兿
+宓
+禺
+叡
+畲
+盐
+锤
+衮
+坤
+枱
+缆
+渚
+溥
+笋
+阴
+黎
+泌
+吧
+远
+极
+溼
+灰
+凛
+谓
+掰
+恣
+袜
+轴
+豹
+儋
+步
+髪
+涧
+责
+递
+竣
+噩
+奏
+恸
+浏
+舎
+怯
+浙
+亲
+堺
+斐
+莅
+叫
+谁
+噎
+忻
+仏
+坝
+墓
+夭
+桑
+宁
+灼
+庸
+屎
+拿
+莜
+愉
+奖
+妪
+惘
+皋
+轻
+粝
+耷
+酯
+霄
+捕
+筱
+庹
+硕
+枇
+衄
+晟
+感
+棋
+肋
+染
+箩
+挖
+囔
+桀
+觉
+吹
+擢
+倜
+半
+徼
+终
+趟
+戕
+昱
+胖
+妻
+斟
+肺
+哂
+损
+恺
+秘
+癞
+馗
+侄
+碾
+捂
+柬
+桢
+淑
+余
+呱
+甭
+栢
+晖
+霸
+锁
+汎
+莲
+炫
+绫
+揸
+莎
+绔
+弼
+听
+诧
+陟
+苒
+禀
+耐
+障
+氓
+聂
+徐
+慧
+乎
+镍
+樱
+镯
+芫
+矾
+痰
+患
+己
+镰
+虢
+択
+忍
+溃
+噔
+勤
+箭
+亨
+値
+棂
+隣
+芪
+顼
+颜
+慷
+橄
+价
+绍
+喧
+泗
+课
+绾
+坳
+扇
+拒
+硅
+傍
+骜
+莱
+帯
+习
+砗
+喫
+倭
+呻
+珉
+斫
+桧
+汤
+礼
+臂
+泥
+战
+勺
+劫
+侏
+吁
+挂
+淙
+笔
+苑
+咚
+姻
+谗
+稣
+酢
+壹
+棣
+衅
+罚
+棱
+赴
+尧
+痊
+痤
+脏
+阈
+镜
+畴
+収
+绪
+慰
+浩
+乍
+敛
+甬
+佛
+糊
+啸
+甜
+肏
+屿
+禹
+胁
+跋
+橹
+轨
+泊
+甚
+赞
+纨
+韦
+囟
+吕
+刹
+秸
+玑
+丰
+樑
+佰
+泠
+弹
+扑
+珈
+聋
+袋
+血
+哔
+汛
+圭
+堃
+飨
+额
+弛
+渍
+讲
+绻
+忘
+囫
+陇
+侃
+匣
+碰
+洹
+迄
+洩
+疗
+似
+呛
+吊
+栾
+洽
+睡
+顿
+槻
+焜
+纾
+骡
+沿
+枋
+骇
+徒
+廿
+噗
+旭
+抜
+揉
+衷
+锰
+剁
+乂
+坎
+杜
+稳
+堕
+寡
+衿
+曙
+晞
+吖
+架
+肇
+卧
+橼
+凳
+胶
+奠
+灬
+抽
+溉
+黛
+它
+払
+奔
+傥
+拟
+泼
+锚
+举
+拎
+歉
+艋
+肥
+渠
+跎
+飒
+恚
+髌
+芦
+筷
+竜
+棚
+頼
+戡
+仑
+畸
+臓
+碎
+喱
+樫
+践
+悩
+柯
+畔
+伤
+満
+咿
+寞
+庠
+痢
+喇
+裾
+侨
+亵
+伫
+夯
+附
+涛
+害
+釉
+韭
+咧
+仄
+掘
+姒
+碱
+琇
+弓
+炙
+诫
+馊
+奚
+忖
+毓
+挽
+艘
+诿
+混
+漉
+数据
+迹
+洌
+俏
+厚
+又
+妖
+屑
+眨
+芾
+叉
+曹
+贬
+粱
+虫
+诣
+厮
+擘
+逛
+韶
+削
+脯
+证
+掲
+肆
+宥
+晶
+毎
+贻
+咀
+徘
+昔
+谋
+逝
+础
+羚
+珥
+缀
+剌
+匕
+顺
+乗
+奉
+掩
+履
+肴
+诛
+俑
+袢
+芒
+啪
+巩
+灸
+颉
+冶
+继
+阶
+诲
+一个
+犊
+互
+撤
+咏
+糖
+聚
+偶
+楫
+倚
+罹
+镣
+温
+执
+昊
+短
+赎
+汝
+冨
+闻
+锂
+韧
+畑
+稷
+诀
+仅
+総
+谎
+氮
+労
+挠
+珐
+忽
+斡
+泻
+涞
+裁
+祚
+劣
+阁
+袍
+堰
+跪
+捐
+肪
+昴
+虑
+雇
+呼
+沱
+诡
+睑
+氡
+赫
+冲
+俯
+兑
+忙
+缈
+咐
+帽
+挤
+艳
+倡
+谆
+阀
+飞
+虻
+胀
+准
+玄
+兖
+免
+径
+衰
+锌
+键
+浸
+筹
+竖
+芽
+痧
+曼
+侠
+胞
+锵
+晚
+捏
+纠
+糯
+掂
+减
+刘
+噼
+従
+哽
+栉
+糍
+阐
+馨
+硬
+掷
+困
+囲
+慎
+済
+牲
+谟
+児
+弗
+炝
+刀
+谣
+枳
+棉
+漩
+卖
+屡
+痣
+聆
+羞
+趵
+留
+卢
+胭
+味
+営
+挺
+琐
+迁
+酷
+冉
+句
+悱
+闯
+碉
+馐
+酿
+际
+镀
+颐
+囊
+皲
+沅
+坡
+徇
+柚
+私
+壮
+弈
+邀
+桎
+奸
+珑
+绛
+票
+巨
+囡
+叠
+声
+衫
+坛
+冒
+识
+芙
+墘
+锒
+巢
+拓
+凰
+惭
+什
+歓
+隼
+惕
+涣
+耀
+汞
+爆
+皿
+僧
+坨
+旖
+轮
+逅
+臆
+签
+哐
+霆
+樟
+噙
+晨
+様
+舂
+桐
+详
+誊
+朴
+勳
+棵
+霊
+挨
+耳
+爹
+辺
+遛
+如果
+祺
+精
+枕
+溶
+涿
+尖
+俳
+惠
+讳
+冕
+袖
+舐
+污
+整
+吒
+方法
+宋
+插
+攀
+逍
+冗
+蜓
+许
+划
+袈
+恩
+掬
+盆
+苜
+骋
+眯
+辗
+続
+酌
+咽
+仰
+映
+桦
+耨
+淅
+伸
+邵
+闹
+靛
+裆
+闸
+堆
+虹
+辱
+讷
+裳
+兜
+哗
+臃
+弭
+豺
+姬
+俘
+卸
+淖
+隘
+镲
+吼
+佚
+汩
+痿
+闽
+券
+抵
+楚
+眺
+宦
+吏
+悉
+吋
+噱
+氹
+抠
+爽
+袁
+祎
+彗
+旱
+储
+臭
+脘
+朗
+尕
+珏
+歙
+毫
+琴
+纺
+谏
+丘
+趾
+杖
+姊
+郭
+锋
+圏
+朵
+缜
+曜
+撰
+毛
+仪
+哉
+别
+刎
+乌
+强
+币
+炳
+颊
+揣
+粿
+糕
+丝
+坞
+炜
+卜
+挑
+谥
+乏
+拮
+衲
+忧
+胎
+歆
+矶
+幄
+审
+恒
+掀
+甙
+捧
+揄
+巻
+靖
+粪
+陵
+飚
+凤
+痕
+办
+裔
+曾
+戏
+觥
+脳
+庞
+肱
+権
+棒
+陷
+殿
+撅
+诃
+遮
+疫
+徕
+诉
+游
+旅
+谪
+霑
+谛
+睹
+抉
+胃
+虚
+刁
+挙
+卤
+委
+癸
+沣
+卟
+缺
+孬
+锣
+劳
+奂
+痉
+寛
+偃
+沃
+彭
+耘
+裥
+仟
+扔
+趁
+悴
+郑
+氷
+沄
+爷
+汆
+貌
+囗
+恢
+愫
+絵
+缴
+绯
+啶
+禅
+秒
+肉
+涮
+拥
+撞
+円
+耿
+耒
+托
+油
+朕
+渤
+砝
+咛
+诵
+閒
+坊
+彼
+刈
+呈
+旌
+盈
+肓
+孑
+坪
+尸
+迢
+撷
+泸
+佼
+阔
+颚
+觑
+啻
+泪
+亦
+酉
+沸
+籺
+拄
+依
+攫
+拾
+邓
+剽
+圃
+幢
+琛
+汽
+闺
+欧
+罡
+隧
+泳
+描
+质
+桠
+船
+祟
+偈
+価
+皴
+缸
+趣
+馍
+稗
+裘
+骏
+佑
+轩
+啬
+缘
+逊
+纤
+顽
+喀
+愈
+亘
+噶
+夕
+规
+妩
+玮
+玲
+肝
+裸
+割
+睬
+酊
+叱
+溜
+衍
+遵
+您
+吗
+季
+吱
+尉
+絮
+纭
+惫
+亜
+苹
+姗
+玠
+睢
+貂
+扦
+泯
+孩
+祁
+赂
+舱
+帼
+权
+孙
+毗
+哲
+咘
+绩
+聒
+筒
+酋
+苷
+溅
+哆
+柴
+玟
+颌
+珅
+殴
+郁
+雒
+至
+跌
+転
+胗
+颠
+頫
+桶
+艰
+册
+寄
+扼
+剉
+悖
+撃
+礁
+羯
+锃
+槭
+舷
+犹
+庾
+稔
+甑
+旧
+宴
+揆
+涪
+晴
+渊
+溢
+竿
+儒
+凪
+眶
+渲
+呆
+洇
+养
+牵
+途
+凉
+叔
+淌
+邯
+距
+孳
+肤
+贱
+怏
+豔
+资
+聘
+扉
+殃
+兔
+纬
+豁
+靥
+读
+悚
+锆
+辛
+倪
+斤
+采
+孜
+拱
+骞
+竞
+停
+溧
+盱
+盾
+穿
+赵
+厥
+漠
+敍
+咣
+埔
+奘
+呡
+怒
+矫
+恻
+杉
+谙
+文件
+沾
+哼
+堇
+惯
+庚
+淬
+昆
+楼
+观
+芍
+問
+碓
+兽
+邕
+涨
+扳
+柔
+赊
+掴
+疼
+酮
+珠
+随
+亏
+呓
+玛
+训
+芩
+弦
+捻
+揖
+裕
+轭
+倔
+俾
+帰
+杯
+疖
+晷
+枯
+绸
+逶
+溺
+坍
+壅
+栎
+赳
+但
+躬
+悄
+乔
+洗
+涓
+寻
+髅
+讴
+雀
+雹
+戴
+筝
+屹
+逻
+寇
+危
+撼
+覇
+顷
+贫
+扬
+缨
+邢
+掳
+惹
+析
+雯
+哏
+甫
+哦
+洪
+溏
+硒
+侬
+孟
+痍
+贴
+弾
+胛
+挟
+苻
+浅
+仔
+宾
+乘
+艹
+坏
+恬
+耕
+徜
+砍
+胝
+绘
+遨
+镖
+罪
+俞
+吃
+埃
+疟
+绞
+坟
+待
+斛
+烽
+俗
+陛
+冢
+偿
+丫
+拈
+盂
+简
+丑
+亟
+辨
+艾
+痴
+啃
+伉
+勇
+叻
+袱
+响
+捶
+夥
+祭
+箇
+绝
+抖
+殖
+籁
+僱
+圈
+揪
+伏
+胚
+疤
+楸
+肾
+厂
+授
+亀
+倘
+淼
+隶
+革
+楮
+汰
+赠
+愍
+粑
+纲
+豚
+牯
+槌
+桜
+呋
+产
+捅
+歧
+渭
+垭
+琬
+颁
+犒
+樽
+伍
+综
+乾
+枸
+莫
+墒
+愣
+咆
+医
+肯
+険
+咁
+冬
+施
+靴
+柿
+樊
+端
+霾
+况
+倌
+伪
+韵
+呀
+僻
+胡
+抻
+域
+墟
+怙
+査
+琲
+佟
+泽
+借
+隠
+洁
+紧
+逼
+皙
+氚
+贰
+皎
+宫
+镂
+孵
+骁
+既
+晕
+渺
+胍
+哒
+恕
+洸
+眙
+栖
+炎
+箫
+贝
+贸
+阵
+陕
+戮
+彝
+避
+佣
+括
+涝
+彫
+颧
+绦
+傩
+冥
+疸
+敢
+喟
+裱
+诚
+琨
+舸
+漕
+敕
+辐
+恼
+镶
+啊
+珀
+迷
+侥
+甄
+挫
+砌
+爪
+纵
+啲
+妮
+睐
+敦
+俐
+戗
+尴
+缙
+辇
+粒
+咻
+弋
+惟
+毙
+眞
+睪
+呸
+洙
+伶
+衞
+豉
+赅
+笑
+抨
+适
+疙
+靠
+弥
+旁
+虔
+哈
+咗
+戈
+幔
+拜
+吐
+胄
+贮
+佳
+劝
+孽
+戬
+废
+挛
+泞
+炖
+遍
+玩
+髯
+羁
+隙
+恤
+增
+耶
+渴
+碌
+莹
+沂
+室
+砼
+疥
+浪
+莞
+趋
+毒
+眉
+姥
+讧
+噜
+迅
+柜
+堑
+脂
+零
+埵
+抟
+哮
+隅
+圄
+舾
+厅
+牢
+浔
+症
+��
+毁
+衩
+慕
+漳
+缝
+跟
+申
+砷
+灏
+祖
+诈
+亥
+眩
+樯
+樾
+雌
+秃
+便
+剜
+砧
+脲
+疏
+绵
+吾
+悔
+喔
+兰
+氤
+旯
+撑
+促
+慑
+砂
+销
+芥
+醒
+吓
+撕
+淋
+谬
+虐
+愿
+牺
+苏
+遐
+兮
+惨
+捷
+臻
+轧
+釜
+埗
+槎
+稿
+個
+竽
+尬
+郫
+辘
+笛
+妙
+戻
+毡
+盏
+箍
+赢
+骐
+逗
+折
+吠
+挲
+盟
+羹
+做
+叩
+惴
+殁
+氪
+灌
+俊
+约
+邈
+谨
+痞
+悌
+树
+巽
+涂
+睫
+帮
+泱
+拖
+胰
+浦
+趺
+袂
+骶
+俪
+却
+妨
+栗
+邺
+浄
+父
+炯
+廊
+沽
+环
+维
+浯
+焼
+佘
+揹
+祐
+粼
+笙
+髓
+悬
+戎
+撮
+疹
+漏
+惺
+譲
+炽
+琮
+续
+笃
+籤
+诊
+庁
+擀
+霏
+杈
+砥
+吵
+妊
+喂
+玥
+辞
+譬
+锏
+酵
+扶
+捌
+団
+幻
+旷
+陪
+哥
+亚
+祀
+贼
+疑
+堤
+拯
+継
+贯
+睾
+亢
+羣
+众
+喃
+坠
+涡
+占
+卷
+怎
+苔
+氟
+胜
+胥
+童
+硌
+惑
+替
+畀
+啵
+豫
+姓
+谄
+乃
+旮
+盹
+俺
+棘
+剔
+歪
+咂
+垒
+哭
+掖
+汶
+匆
+彰
+晏
+豆
+骅
+乩
+劭
+釦
+辍
+堀
+琢
+泷
+霖
+癫
+邨
+傀
+侯
+霰
+埤
+壊
+氙
+黏
+噪
+桂
+囹
+闫
+柏
+粳
+斜
+浒
+络
+戚
+仨
+吨
+宵
+牛
+肚
+昂
+挝
+汲
+诋
+肃
+儆
+往
+且
+轿
+肌
+陶
+粥
+牦
+奢
+卵
+孺
+啰
+杨
+攸
+郤
+绥
+咯
+聊
+抬
+悼
+遑
+遁
+恳
+壳
+昀
+耙
+咱
+焕
+喰
+酰
+辄
+休
+喷
+祛
+恙
+锉
+祉
+冽
+曳
+脾
+苋
+胱
+绕
+槃
+预
+诞
+焉
+欠
+升
+叹
+琉
+羔
+阇
+兲
+爬
+忑
+孪
+询
+邸
+谶
+丞
+偷
+穷
+羿
+郧
+栅
+愚
+氨
+盛
+壆
+爻
+聴
+笞
+増
+苡
+脱
+邛
+痫
+闵
+偎
+埠
+砲
+赌
+牴
+赐
+圜
+疣
+凑
+涠
+莺
+堵
+祸
+壑
+垮
+夙
+套
+末
+漫
+莆
+档
+沛
+盔
+牡
+绎
+喽
+姣
+喝
+橙
+酣
+颅
+橐
+稍
+缮
+丢
+抄
+稽
+跆
+擎
+祯
+恫
+沓
+迦
+赭
+馄
+缎
+僭
+耄
+雑
+擦
+焊
+厌
+溯
+揭
+矗
+袤
+胴
+壶
+攥
+镐
+旋
+黠
+尿
+帑
+疾
+痨
+怩
+砸
+笆
+谘
+聿
+埼
+慌
+杓
+跖
+呦
+凇
+淞
+剩
+握
+易
+芷
+氰
+丧
+赝
+骂
+貉
+枪
+朔
+趸
+観
+辕
+啾
+泓
+粮
+裂
+箕
+匠
+垂
+涩
+芯
+誉
+晌
+棍
+纱
+咋
+寮
+遅
+剂
+碘
+苁
+捞
+舵
+壬
+渗
+骤
+捆
+哌
+夷
+矍
+侧
+噢
+俩
+勒
+领
+够
+炸
+洄
+惬
+俟
+斗
+呎
+妹
+努
+毋
+碚
+綑
+缔
+忆
+吝
+泮
+帚
+碧
+邂
+痪
+几
+颡
+呗
+伴
+髦
+睿
+哇
+纳
+芜
+讚
+柘
+很
+抑
+剑
+泛
+飙
+祇
+策
+毘
+糸
+巖
+射
+酶
+跄
+幂
+旺
+拢
+妆
+洛
+彷
+辈
+啱
+竺
+呜
+隐
+午
+惰
+栃
+怵
+袪
+怔
+乓
+柳
+痠
+舫
+黢
+遣
+苕
+谩
+邻
+坷
+姹
+妾
+姿
+尻
+劾
+肛
+覚
+诶
+喏
+込
+拣
+倖
+厦
+淆
+累
+讹
+刷
+氯
+坯
+垄
+睽
+軽
+酬
+愧
+庶
+佶
+杰
+孤
+睥
+癡
+汾
+哟
+估
+眷
+豪
+锭
+劵
+廖
+刊
+矿
+财
+拽
+压
+埸
+欲
+牟
+哄
+乳
+孚
+硫
+播
+炉
+靶
+赛
+俱
+妓
+彧
+矜
+诰
+柒
+豌
+走
+壕
+矢
+仝
+歩
+仓
+栓
+粗
+琥
+纪
+临
+诅
+抢
+汁
+枷
+灯
+邦
+臬
+圆
+弑
+舺
+诽
+肘
+丐
+脍
+咩
+姨
+掏
+卅
+低
+刽
+缠
+阅
+屄
+需要
+酸
+供
+防
+姚
+灵
+箸
+棺
+拂
+殒
+碟
+宜
+粹
+秽
+日本
+辊
+淀
+假
+埂
+斩
+涟
+恨
+呕
+畿
+帘
+痘
+诬
+傚
+绡
+补
+馏
+尘
+筠
+巷
+苎
+垛
+箔
+莿
+啜
+墅
+风
+叁
+辖
+尊
+杆
+扎
+呲
+封
+囉
+叮
+妈
+胳
+奕
+痔
+洼
+苛
+稲
+勉
+遇
+募
+孰
+戳
+绑
+郢
+眈
+胆
+哎
+旻
+筊
+怜
+臼
+疮
+扱
+馔
+弩
+喻
+昵
+羌
+锈
+疆
+奴
+辙
+脊
+闲
+誓
+均
+吽
+阖
+穆
+帜
+彤
+阉
+赏
+弎
+找
+皂
+束
+屌
+淦
+碳
+睛
+稹
+禛
+绷
+孕
+吴
+寅
+沫
+胺
+啕
+洒
+笼
+丶
+织
+贺
+靓
+衡
+乱
+帅
+伎
+诟
+徨
+批
+纽
+雉
+絃
+队
+隔
+痱
+枞
+裤
+玳
+裨
+墀
+幡
+怀
+箐
+昨
+觊
+斌
+谀
+酝
+孱
+绣
+闰
+严
+槟
+咭
+扯
+捺
+楹
+杭
+珣
+悟
+艇
+庆
+洱
+紊
+漂
+舆
+欸
+舛
+髫
+氢
+奎
+姫
+敖
+炮
+糙
+印
+报
+培
+软
+罂
+锲
+圧
+爸
+险
+徉
+勘
+粧
+彩
+羟
+丨
+挎
+漯
+舔
+旎
+弁
+毕
+衙
+讫
+飓
+逮
+陨
+镊
+纸
+汀
+镌
+洵
+啉
+忪
+颔
+译
+捯
+屐
+坦
+仕
+桁
+酚
+拌
+早
+雏
+辣
+吟
+焙
+琰
+雾
+骛
+拚
+迸
+脖
+拆
+役
+帷
+笕
+央
+馒
+乒
+病
+吩
+侮
+牒
+陞
+闢
+拷
+涌
+躇
+喳
+囚
+骗
+咸
+擅
+徳
+捣
+犍
+纹
+俄
+渥
+箴
+含
+琳
+苍
+殭
+戛
+诺
+静
+甦
+尤
+捲
+妲
+馋
+卯
+拴
+弯
+喆
+跤
+陀
+园
+胸
+谔
+倶
+靼
+垃
+洮
+溴
+粉
+肿
+栩
+耋
+証
+诗
+肖
+恃
+级
+案
+觐
+図
+锅
+莉
+圾
+闘
+晔
+僮
+刺
+闾
+拐
+答
+牍
+垣
+芳
+撬
+率
+争
+核
+汇
+缚
+凭
+卓
+虱
+副
+炬
+啮
+刑
+耦
+垢
+呐
+悦
+噬
+协
+哋
+畦
+秧
+矛
+赦
+押
+术
+敝
+虾
+吮
+锹
+焖
+谯
+吡
+卑
+郸
+豢
+雨
+慵
+涔
+倾
+悻
+拭
+犀
+抱
+缥
+玖
+享
+侈
+叙
+措
+隍
+股
+诩
+拧
+秦
+怂
+喊
+浮
+厘
+扮
+枉
+槽
+颗
+难
+耸
+啥
+祢
+匮
+寂
+绉
+拇
+遽
+骊
+延
+掸
+贤
+跛
+决
+柠
+盘
+脣
+阑
+徬
+负
+侍
+宅
+碛
+甘
+芹
+刍
+勖
+宕
+辟
+寐
+诠
+艺
+优
+她
+琊
+讨
+巫
+炼
+晦
+漾
+轲
+糅
+斥
+撇
+邃
+斧
+朋
+舖
+豇
+锡
+旬
+秆
+���
+壁
+穹
+讼
+锦
+阱
+忐
+殊
+氛
+淄
+俸
+帖
+察
+彪
+玺
+灾
+纰
+赈
+迈
+届
+惋
+捍
+允
+俭
+横
+稞
+粤
+尽
+硝
+宸
+宰
+歎
+昧
+紮
+碑
+围
+绳
+丹
+泣
+糬
+陌
+迫
+阕
+乞
+屯
+贾
+辑
+雰
+働
+咒
+杀
+灞
+孖
+柩
+纫
+勧
+锄
+馥
+逾
+渝
+珞
+疽
+硼
+戊
+欣
+痹
+裹
+剐
+带
+攞
+订
+哺
+念
+锑
+筏
+怼
+凹
+痛
+硐
+忸
+隽
+凝
+亭
+皈
+黑
+冀
+挡
+谜
+镳
+秉
+槐
+癜
+啡
+撸
+糠
+绀
+淹
+姪
+戍
+墙
+忒
+斋
+皆
+秤
+肄
+讽
+凸
+盗
+羡
+惆
+涕
+虬
+遴
+镁
+昼
+苄
+堪
+倦
+恐
+慾
+闭
+涙
+遗
+卮
+汪
+敞
+警
+赓
+橱
+氖
+振
+颇
+遂
+皓
+沟
+筵
+邱
+讥
+飕
+泄
+缭
+佩
+佻
+晁
+疱
+脓
+杏
+宛
+枢
+某
+沉
+邑
+躏
+各
+颢
+抗
+诸
+辜
+佢
+层
+顾
+圻
+矩
+噘
+拘
+残
+丛
+伕
+刻
+厳
+啄
+粟
+破
+掇
+酥
+偏
+忱
+沪
+楷
+骸
+朐
+爱
+诙
+贞
+禾
+効
+悲
+桔
+庇
+骈
+稼
+捡
+槿
+误
+颍
+馅
+骑
+底
+愆
+尺
+吸
+旃
+禽
+虏
+排
+碣
+籼
+苯
+浚
+涸
+莠
+臊
+焰
+巧
+怡
+弃
+芭
+锢
+歳
+丈
+亳
+佔
+览
+祂
+邪
+郝
+闷
+黍
+琦
+宪
+逞
+坑
+须
+住
+俨
+召
+灶
+妥
+敌
+粄
+谑
+赃
+酒
+匡
+迺
+邬
+囿
+锗
+趴
+杵
+剖
+歹
+喋
+镉
+邳
+庖
+畅
+骧
+赚
+敏
+晋
+芡
+朽
+敎
+枚
+睦
+芝
+舶
+桅
+渎
+陆
+咙
+汗
+吻
+捉
+渐
+探
+髂
+倍
+晩
+耗
+陲
+昇
+堙
+贲
+啖
+揃
+矮
+擡
+赘
+可能
+毽
+砺
+盒
+糗
+虎
+郜
+甯
+疚
+妃
+轶
+垦
+契
+炅
+缇
+缕
+晃
+処
+哪
+渉
+礡
+伙
+骠
+弟
+桓
+箱
+経
+贿
+掺
+伽
+坐
+喙
+涉
+霈
+拡
+眦
+坵
+痒
+雳
+歇
+疔
+肮
+郊
+厄
+帕
+邝
+漆
+両
+広
+掣
+詹
+犁
+拗
+靡
+伺
+籍
+跚
+阂
+赣
+札
+俚
+淘
+略
+芬
+跺
+骥
+颤
+检
+我们
+绗
+涅
+监
+劲
+触
+宣
+咳
+追
+快
+棹
+竭
+焦
+盎
+兼
+讯
+癒
+砭
+酗
+仇
+黝
+偌
+稜
+馆
+练
+圣
+迟
+吭
+词
+轱
+浣
+枣
+绽
+冈
+咎
+拨
+赔
+涤
+霂
+喹
+剿
+慢
+犬
+罔
+戌
+甾
+佝
+儿
+犄
+倬
+儡
+孢
+甸
+愤
+撚
+淡
+仗
+爰
+屠
+疯
+督
+斓
+杼
+遒
+陡
+罩
+忡
+跃
+匝
+攒
+砖
+帏
+弔
+妇
+戟
+楠
+霍
+牙
+冰
+吆
+馈
+昉
+遭
+捎
+琍
+莽
+戾
+犽
+秩
+谦
+迭
+怅
+招
+穗
+沮
+厩
+涎
+籐
+缱
+扰
+轰
+慈
+勃
+睁
+疲
+沐
+咄
+诱
+剣
+飘
+启
+酐
+掌
+揽
+谚
+専
+倩
+拼
+珺
+恁
+脉
+毅
+奶
+拔
+癌
+棠
+槁
+焯
+援
+諡
+咨
+雍
+炒
+伢
+爵
+珊
+脚
+枰
+彅
+畜
+淮
+矣
+霞
+弄
+珮
+莴
+材
+辻
+阻
+姉
+廓
+迥
+韬
+垠
+粲
+谐
+佗
+癖
+砒
+丕
+笺
+具
+兢
+樵
+兆
+谧
+阙
+恿
+悪
+羧
+壤
+考
+境
+昏
+恶
+舜
+晤
+馁
+佤
+涵
+毯
+咖
+寝
+郴
+砰
+旗
+颖
+喘
+谌
+卫
+邮
+芸
+敲
+截
+退
+阮
+哩
+贪
+祠
+囱
+祥
+稻
+焱
+耻
+脑
+肽
+赋
+俎
+脇
+砣
+劈
+扛
+苦
+露
+箝
+侷
+俬
+怨
+扁
+货
+巅
+畳
+凌
+碗
+廉
+仃
+卦
+辩
+幼
+酩
+叭
+罐
+访
+珩
+侣
+撒
+妄
+赖
+充
+投
+舟
+汹
+骚
+即
+缪
+伯
+尚
+浊
+拍
+挞
+稚
+牌
+呵
+隈
+渔
+霎
+昶
+橇
+訳
+镕
+碍
+致
+刚
+殉
+溟
+遢
+纯
+墩
+颓
+块
+坚
+躲
+异
+邰
+剥
+祷
+散
+囍
+恭
+粕
+歯
+匙
+姝
+桉
+捱
+卡
+芋
+炀
+羲
+皱
+噻
+聪
+覧
+靳
+愕
+侦
+宏
+拦
+冻
+斑
+焗
+笨
+歼
+净
+竟
+昕
+郅
+抡
+赶
+颞
+矽
+跷
+镑
+籽
+赡
+迪
+晓
+脸
+罢
+亿
+佥
+咔
+缓
+羊
+冾
+卉
+刮
+琼
+対
+归
+跻
+颈
+兄
+厉
+稀
+祝
+曰
+擒
+征
+恰
+眼
+侵
+寰
+羽
+仙
+噌
+寓
+殷
+势
+浬
+款
+莪
+屏
+骰
+丼
+扫
+旨
+佃
+受
+歛
+珪
+沼
+犸
+贵
+妳
+循
+悠
+墨
+幽
+孀
+槲
+栋
+迎
+论
+耽
+幌
+抚
+痼
+禄
+臧
+贷
+佞
+掠
+柱
+咦
+偲
+雕
+涯
+敬
+灭
+陂
+舰
+苣
+犯
+炷
+炔
+雁
+帆
+莒
+伦
+谊
+疳
+剷
+淇
+述
+丿
+盅
+历
+盖
+汐
+疵
+傅
+普
+辰
+妘
+冠
+殆
+评
+孃
+舗
+垚
+缩
+陋
+栏
+迂
+姜
+绿
+弊
+救
+彻
+裴
+牝
+溱
+黔
+犇
+庭
+痂
+叼
+炁
+躁
+屁
+晒
+浇
+妍
+姑
+粘
+霹
+珍
+跹
+譁
+律
+淳
+琏
+使用
+益
+砚
+帧
+诤
+仞
+厨
+扒
+勿
+绢

vocab/llama/gpt_neox/get_oov_zh_tokens.py ADDED Viewed

	@@ -0,0 +1,20 @@

+from utils.zh_util import is_chinese
+from transformers import LlamaTokenizer
+llama_vocab = LlamaTokenizer.from_pretrained("../tokenizer").get_vocab()
+from vocab.gpt_neox_chinese_v1 import tokenizer
+vocab = tokenizer.get_vocab()
+f_out = open("append_zh.txt", "w", encoding="utf-8")
+for token, token_id in vocab.items():
+    token = tokenizer.decode([token_id])
+    # token = token.strip("Ġ")
+    if len(token) < 1:
+        continue
+    if is_chinese(token[0]):
+        if token not in llama_vocab:
+            f_out.write(token + "\n")

vocab/llama/gpt_neox/load_by_gpt_neox.py ADDED Viewed

	@@ -0,0 +1,18 @@

+# HF_tokenizer
+# from tokenizers import Tokenizer
+# tokenizer = Tokenizer.from_file("tokenizer/tokenizer.json")
+import sentencepiece as spm
+text = "nice job 华为手机"
+tokenizer = spm.SentencePieceProcessor(model_file="tokenizer/tokenizer.model")
+tokens = tokenizer.encode(text)  # [7575, 4982, 29871, 31266, 30573, 30880, 31429]
+print(tokens)
+from transformers import LlamaTokenizer
+tokenizer = LlamaTokenizer.from_pretrained("tokenizer")
+tokens = tokenizer.encode(text)  # [1, 7575, 4982, 29871, 31266, 30573, 30880, 31429]
+print(tokens)