Update tokenization and visualization logic

app.py

@@ -6,6 +6,10 @@ from datasets import load_dataset
 import random
 from pathlib import Path
 
+# tokenizer = AutoTokenizer.from_pretrained("Xenova/gpt-4o")
+# token_ids = tokenizer.encode("السلام عليكم ورحمة الله")
+# exit()
+
 initial_list_of_models = [
     "Xenova/gpt-4o",
     "NousResearch/Meta-Llama-3-8B",
@@ -18,12 +22,12 @@ dataset = load_dataset("MohamedRashad/rasaif-translations", split="train")["arab
 
 dataframe_path = Path(__file__).parent / "arabic_tokenizers_leaderboard.jsonl"
 if dataframe_path.exists():
-    df = pd.read_jsonl(dataframe_path, lines=True)
+    df = pd.read_json(dataframe_path, lines=True)
 else:
-    df = pd.DataFrame(columns=["Models", "Total Number of Tokens", "Vocab Size", "Tokenizer Class"])
+    df = pd.DataFrame(columns=["📛 Models", "➕ Total Number of Tokens", "📘 Vocab Size", "Tokenizer Class"])
 
 for model_name in tqdm(initial_list_of_models):
-    if model_name in df["Models"].values:
+    if model_name in df["📛 Models"].values:
         continue
     tokenizer = AutoTokenizer.from_pretrained(
         model_name, use_fast=True, trust_remote_code=True
@@ -32,16 +36,16 @@ for model_name in tqdm(initial_list_of_models):
     number_of_tokens = sum([len(x) for x in tokenizer(dataset).input_ids])
     df = df._append(
         {
-            "Models": model_name,
-            "Vocab Size": vocab_size,
-            "Total Number of Tokens": number_of_tokens,
+            "📛 Models": model_name,
+            "📘 Vocab Size": vocab_size,
+            "➕ Total Number of Tokens": number_of_tokens,
             "Tokenizer Class": tokenizer.__class__.__name__,
         },
         ignore_index=True,
     )
 
 # Sort the dataframe by the number of tokens
-df = df.sort_values(by="Total Number of Tokens", ascending=True)
+df = df.sort_values(by="➕ Total Number of Tokens", ascending=True)
 
 # Save the dataframe to a csv file
 df.to_json(dataframe_path, lines=True, orient="records")
@@ -49,8 +53,8 @@ df.to_json(dataframe_path, lines=True, orient="records")
 # Gradio Functions
 def refresh():
     global df
-    df = df.sort_values(by="Total Number of Tokens", ascending=True)
-    return gr.Dataframe(df), gr.BarPlot(df)
+    df = df.sort_values(by="➕ Total Number of Tokens", ascending=True)
+    return gr.Dataframe(df), gr.BarPlot(df), gr.Dropdown(choices=df["📛 Models"].tolist())
 
 def submit(model_name):
     global df
@@ -61,9 +65,9 @@ def submit(model_name):
     number_of_tokens = sum([len(x) for x in tokenizer(dataset).input_ids])
     df = df._append(
         {
-            "Models": model_name,
-            "Vocab Size": vocab_size,
-            "Total Number of Tokens": number_of_tokens,
+            "📛 Models": model_name,
+            "➕ Total Number of Tokens": number_of_tokens,
+            "📘 Vocab Size": vocab_size,
             "Tokenizer Class": tokenizer.__class__.__name__,
         },
         ignore_index=True,
@@ -144,13 +148,23 @@ def decode_arabic_tokens(tokens):
 def tokenize_text(text, chosen_model):
     tokenizer = AutoTokenizer.from_pretrained(chosen_model)
     tokenized_text = decode_bpe_tokens(tokenizer.tokenize(text))
-    # tokenized_text = decode_arabic_tokens(tokenizer.tokenize(text))
     random_colors = generate_distinct_colors(len(tokenized_text))
-    print(tokenized_text)
+
+    final_tokenized_text = []
+    for token in tokenized_text:
+        correct_tokenized_text = ""
+        for char in text:
+            correct_tokenized_text += char 
+            current_token = decode_bpe_tokens(tokenizer.tokenize(correct_tokenized_text))
+            if current_token[0] == token:
+                final_tokenized_text.append(correct_tokenized_text)
+                text = text[len(correct_tokenized_text):]
+                break
+    print(final_tokenized_text)
 
     output = []
     color_map = {}
-    for idx, token in enumerate(tokenized_text):
+    for idx, token in enumerate(final_tokenized_text):
         output.append((token, str(idx)))
         color_map[str(idx+1)] = random_colors[idx % len(random_colors)]
 
@@ -169,16 +183,15 @@ with gr.Blocks() as demo:
         with gr.Accordion("Barplot", open=False):
             barplot = gr.BarPlot(
                 df,
-                x="Models",
-                y="Total Number of Tokens",
+                x="📛 Models",
+                y="➕ Total Number of Tokens",
                 x_title=" ",
                 y_title=" ",
                 width=1000,
                 height=400,
-                tooltip=["Vocab Size", "Total Number of Tokens"],
+                tooltip=["📘 Vocab Size", "➕ Total Number of Tokens"],
                 vertical=False,
                 x_label_angle=30,
-                caption="Total Number of Tokens",
             )
         model_name = gr.Textbox(
             label="Model Name from Hugging Face (e.g. Xenova/gpt-4o)"
@@ -190,14 +203,14 @@ with gr.Blocks() as demo:
         text = gr.Textbox(label="Enter a text", lines=5, value="السلام عليكم ورحمة الله", rtl=True, text_align="right")
         dropdown = gr.Dropdown(
             label="Select a model",
-            choices=df["Models"].tolist(),
-            value=df["Models"].tolist()[0],
+            choices=df["📛 Models"].tolist(),
+            value=df["📛 Models"].tolist()[0],
         )
         submit_text_btn = gr.Button(value="Submit", variant="primary")
         tokenized_textbox = gr.HighlightedText(label="Tokenized text")
 
     submit_new_model_btn.click(submit, model_name)
-    refresh_btn.click(refresh, outputs=[dataframe, barplot])
+    refresh_btn.click(refresh, outputs=[dataframe, barplot, dropdown])
     submit_text_btn.click(tokenize_text, inputs=[text, dropdown], outputs=[tokenized_textbox])