Spaces:
Running
Running
File size: 1,460 Bytes
7c73423 a4208a2 7c73423 a4208a2 7c73423 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
"""Gradio app to showcase the LLM tokenization."""
import os
import gradio as gr
from huggingface_hub import login
from playground_app import demo as playground_tab
from compression_app import demo as compression_tab
from character_app import demo as character_tab
auth_token = os.environ.get('HF_TOKEN', None)
if auth_token:
login(token=auth_token)
title = '<div align="center">Tokenizer Arena ⚔️</div>'
interface_list = [playground_tab, compression_tab, character_tab]
tab_names = [" ⚔️ Playground", "🏆 Compression Leaderboard", "📊 Character Statistics"]
# interface_list = [compression_tab, character_tab]
# tab_names = ["🏆 Compression Leaderboard", "📊 Character Statistics"]
with gr.Blocks(css="css/style.css", js="js/onload.js") as demo:
gr.HTML(
f"<h1 style='text-align: center; margin-bottom: 1rem'>{title}</h1>"
)
with gr.Tabs():
for interface, tab_name in zip(interface_list, tab_names):
with gr.Tab(label=tab_name):
interface.render()
# model_name = gr.Textbox(
# placeholder="🔍 Add tokenizer from Hugging Face (e.g. Xenova/gpt-4o) and press ENTER...",
# show_label=False,
# )
#
# model_name.submit()
# demo.load(js=open("js/onload.js", "r", encoding="utf-8").read())
if __name__ == "__main__":
demo.launch()
# demo.queue(max_size=1024, default_concurrency_limit=80).launch()
|