app.py

import gradio as gr
from anthropic import Anthropic
import tiktoken


def tokenize(text : str) -> int:
    # About Anthropic tokenizer
    # https://github.com/anthropics/anthropic-sdk-python
    anthropic = Anthropic()
    anthropic_count = anthropic.count_tokens(text)

    # About Open AI tokenizer
    # https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
    gpt_35_encoder = tiktoken.get_encoding("cl100k_base")
    tokens = gpt_35_encoder.encode(text)

    return anthropic_count, len(tokens)


counter = gr.Interface(
    title="LLM tokenizer",
    description="左側のボックスにテキストを入れると、 Anthropic / OpenAI それぞれで分割したときのトークン数を計算します",
    fn=tokenize,
    inputs=gr.Textbox(lines=10, placeholder="Text Here..."),
    outputs=[
        gr.Number(value=0, label="Anthropic's token", interactive=False),
        gr.Number(value=0, label="OpenAI GPT 3.5 's token", interactive=False)
    ],
    live=True,
    allow_flagging="never"
)

counter.launch()