update

app.py

@@ -17,6 +17,7 @@
 - 中文字词统计，是否要包括 _ G 等字符
 - baichuan的单字数量怎么两万多个？
 - OOV
+- feedback位置
 
 
 plots
@@ -40,11 +41,12 @@ from util import *
 
 # llama chatglm_6b gpt_nexo_20b baichuan  baichuan_7b
 examples = [
-    # ["空格测试：  2个空格        8个空格", "llama", "chatglm_6b"],  # chatglm 有blank_n,
+    ["空格测试：  2个空格        8个空格", "llama", "chatglm_6b"],  # chatglm 有blank_n,
     ["标点测试：，。！？；", "baichuan_7b", "llama"],
     ["符号测试：🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan_7b", "llama"],
-    ["中文简体：宽带，繁体：樂來", "baichuan_7b", "llama"],
     ["数字测试：(10086 + 98) = 100184", "baichuan_7b", "llama"],
+    ["中文简体：宽带，繁体：樂來", "baichuan_7b", "llama"],
+
 ]
 
 # jieba.enable_parallel()  # flask中没办法parallel
@@ -66,8 +68,8 @@ default_tokenizer_type_2 = "internlm_chat_7b"
 default_stats_vocab_size_1, default_stats_zh_token_size_1 = basic_count(default_tokenizer_type_1)
 default_stats_vocab_size_2, default_stats_zh_token_size_2 = basic_count(default_tokenizer_type_2)
 default_stats_overlap_token_size = get_overlap_token_size(default_tokenizer_type_1, default_tokenizer_type_2)[0]
-default_output_text_1, default_output_table_1 = tokenize(default_user_input, default_tokenizer_type_1, update=False)
-default_output_text_2, default_output_table_2 = tokenize(default_user_input, default_tokenizer_type_2, update=False)
+default_output_text_1, default_output_table_1, default_output_len_1 = tokenize(default_user_input, default_tokenizer_type_1, update=False)
+default_output_text_2, default_output_table_2, default_output_len_2 = tokenize(default_user_input, default_tokenizer_type_2, update=False)
 
 with gr.Blocks(css="style.css") as demo:
     gr.HTML("""<h1 align="center">Tokenizer Arena ⚔️</h1>""")
@@ -81,7 +83,7 @@ with gr.Blocks(css="style.css") as demo:
     with gr.Row():
         gr.Markdown("## Input Text")
         dropdown_examples = gr.Dropdown(
-            ["Example1", "Example2", "Example3"],
+            ["空格测试", "标点测试", "符号测试", "数字测试"],
             value="Examples",
             type="index",
             show_label=False,
@@ -181,14 +183,14 @@ with gr.Blocks(css="style.css") as demo:
         with gr.Column():
             output_text_1 = gr.Highlightedtext(
                 value=default_output_text_1,
-                label="Tokens 1",
+                label=f"Tokens: {default_output_len_1}",
                 show_legend=True,
                 elem_classes="space-show"
             )
         with gr.Column():
             output_text_2 = gr.Highlightedtext(
                 value=default_output_text_2,
-                label="Tokens 2",
+                label=f"Tokens: {default_output_len_2}",
                 show_legend=True,
                 elem_classes="space-show"
             )

util.py

@@ -59,7 +59,7 @@ def tokenize(text, tokenizer_type, color_num=5, update=True):
     if update:
         return gr.update(value=pos_tokens, label=f"Tokens: {len(encoding)}"), table_df
     else:
-        return pos_tokens, table_df
+        return pos_tokens, table_df, len(encoding)
 
 
 def tokenize_pair(text, tokenizer_type_1, tokenizer_type_2):