Hugging Face's logo

Spaces:
xu-song
/
tokenizer-arena
Running

App Files Community
1
xu-song commited on
Commit
44c3329
•
1 Parent(s): 2442c83

add aya

Browse files
Files changed (2) hide show
  1. vocab/__init__.py +1 -1
  2. vocab/aya_101/__init__.py +5 -0
vocab/__init__.py CHANGED
@@ -158,7 +158,7 @@ all_tokenizers = [
158
  "deepseek_llm_7b_base",
159
  "gemma_7b",
160
  "olmo_7b",
161
-
162
  ]
163
 
164
  all_tokenizers = sorted(all_tokenizers)
 
158
  "deepseek_llm_7b_base",
159
  "gemma_7b",
160
  "olmo_7b",
161
+ "aya_101",
162
  ]
163
 
164
  all_tokenizers = sorted(all_tokenizers)
vocab/aya_101/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+
2
+
3
+ from transformers import AutoTokenizer
4
+
5
+ tokenizer = AutoTokenizer.from_pretrained("CohereForAI/aya-101")