update

app.py

@@ -5,13 +5,16 @@
 """
 ## TODO:
 - http get方式获取参数，
-- 自启动
 - iter_vocab 的 warmup
 - add_special_token 开关
 - theme 开关 light/dark
 - token_id/tokens/bytes 开关
 - 通过 javascript 添加 hover_text
 - i18
+- 给方法 + 缓存，避免重复调用
+- 英文 utf-8编码
+- 词典支持下载
+- 中文字词统计，是否要包括 _ G 等字符
 
 
 
@@ -30,7 +33,6 @@ table
 [ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ]
 """
 
-
 import gradio as gr
 
 from vocab import all_tokenizers
@@ -63,8 +65,6 @@ default_stats_overlap_token_size = get_overlap_token_size(default_tokenizer_type
 default_output_text_1, default_output_table_1 = tokenize(default_user_input, default_tokenizer_type_1, update=False)
 default_output_text_2, default_output_table_2 = tokenize(default_user_input, default_tokenizer_type_2, update=False)
 
-
-
 with gr.Blocks(css="style.css") as demo:
     gr.HTML("""<h1 align="center">Tokenizer Arena ⚔️</h1>""")
     # links: https://www.coderstool.com/utf8-encoding-decoding
@@ -97,7 +97,6 @@ with gr.Blocks(css="style.css") as demo:
     #     None,
     # )
 
-
     gr.Markdown("## Tokenization")
 
     with gr.Row():
@@ -139,12 +138,12 @@ with gr.Blocks(css="style.css") as demo:
         # https://www.onlinewebfonts.com/icon/418591
         gr.Image("images/VS.svg", scale=1, show_label=False,
                  show_download_button=False, container=False,
-                 show_share_button=False)  # height=10,
+                 show_share_button=False)
         with gr.Column(scale=6):
             with gr.Group():
                 tokenizer_type_2 = gr.Dropdown(
                     all_tokenizers,
-                    value="baichuan_7b",
+                    value=default_tokenizer_type_2,
                     label="Tokenizer 2",
                 )
                 with gr.Group():
@@ -229,7 +228,6 @@ with gr.Blocks(css="style.css") as demo:
     # start up 初始化
     # user_input.update(user_input.value + "___")
 
-
 if __name__ == "__main__":
     demo.queue(max_size=20).launch()
     # demo.launch()