vocab/icetk/README.md

简介

num_image_tokens = 20000   image_tokenizer
num_text_tokens = 130000   text_tokenizer

一共 150000

text_tokenizer

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[0]   对应 20000+0
piece: "<unk>"
score: 0.0
type: UNKNOWN

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[1]   对应 20000+1
piece: "<s>"
score: 0.0
type: CONTROL

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[2]   对应 20000+2
piece: "</s>"
score: 0.0
type: CONTROL

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[3]   对应 20000+3
piece: "<pad>"
score: 0.0
type: CONTROL

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[4]   对应 20000+4
piece: "<n>"
score: 0.0
type: USER_DEFINED

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[5]   对应 20000+5
piece: "\342\226\201"
score: -2.6171817779541016
type: NORMAL

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[6]
piece: ","
score: -3.151700019836426
type: NORMAL


tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[50]
piece: "{"
score: -7.532660961151123
type: NORMAL

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[100]
piece: "\342\226\201the"           # "\342\226\201"  这是啥？？ 
score: -3.922896385192871
type: NORMAL

tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[200]
piece: "\342\226\201This"
score: -7.821105480194092
type: NORMAL


tokenizer.sp_tokenizer.text_tokenizer.proto.pieces[128293]
piece: "\342\226\201pa\303\255ses"
score: -14.182646751403809
type: NORMAL