yhavinga's picture
Add Llama tokenizer creation for Dutch, English, Code, Markdown and TeX.
c78da21
raw
history blame contribute delete
565 Bytes
import gradio as gr
from app_playground import demo as tab_playground
from app_compression import demo as tab_compression
from patcher.gr_interface import TabbedInterface
demo = TabbedInterface(
[tab_playground, tab_compression],
[" ⚔️ Playground", "🏆 Compression Leaderboard",], # 编码速度,解码速度,字符分类(zh、num等,支持正则),支持的语言,机构,。
title='Tokenizer Arena ⚔️ (with some Dutch 🇳🇱🇧🇪🇸🇷 hacked in)',
css="css/style.css"
)
if __name__ == "__main__":
demo.launch()