add more tokenizer

vocab/__init__.py

@@ -96,8 +96,8 @@ all_tokenizers = [
     # "alpaca_7b",
     "baichuan",
     "baichuan2",
-    "qwen",
     "internlm_chat_7b",
+    "falcon_7b",
     "falcon_180b",
     # "goat",
 
@@ -109,9 +109,18 @@ all_tokenizers = [
     "skywork_13b_base",
     "skywork_13b_math",
     "mistral",
+    "t5_small",
+    "t5_base",
+    "t5_large",
+    "flan_t5_base",
+    "fastchat_t5_3b",
+    "pko_t5_large",
+
 
 ]
 
+all_tokenizers = sorted(all_tokenizers)
+
 class TokenizerType(Enum):
     """
     - https://huggingface.co/docs/transformers/tokenizer_summary

vocab/falcon_7b/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b", trust_remote_code=True)

vocab/fastchat_t5_3b/__init__.py

@@ -0,0 +1,3 @@
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("lmsys/fastchat-t5-3b-v1.0", trust_remote_code=True)

vocab/flan_t5_base/__init__.py

@@ -0,0 +1,3 @@
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-base", trust_remote_code=True)

vocab/pko_t5_large/__init__.py

@@ -0,0 +1,3 @@
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("paust/pko-t5-large", trust_remote_code=True)

vocab/t5/__init__.py

@@ -1,7 +0,0 @@
-"""
-
-
-SentencePiece
-"""
-
-

vocab/t5_base/__init__.py

@@ -0,0 +1,8 @@
+"""
+https://huggingface.co/t5-base
+"""
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("t5-base", trust_remote_code=True)

vocab/t5_large/__init__.py

@@ -0,0 +1,8 @@
+"""
+https://huggingface.co/t5-large
+"""
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("t5-large", trust_remote_code=True)

vocab/t5_small/__init__.py

@@ -0,0 +1,8 @@
+"""
+https://huggingface.co/t5-large
+"""
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("t5-small", trust_remote_code=True)