Spaces:
Running
Running
remove lru_cache: has some bugs
Browse files- character_app.py +0 -1
- playground_examples.py +1 -1
- playground_util.py +1 -1
character_app.py
CHANGED
@@ -18,7 +18,6 @@ abbr2name = {column[1]: column[0].split('-')[-1] for column in all_columns}
|
|
18 |
|
19 |
|
20 |
def get_column_info(columns):
|
21 |
-
print(columns)
|
22 |
markdown = ""
|
23 |
for column in columns:
|
24 |
markdown += f"- `num({column})`: num of tokens containing {abbr2name[column]} characters\n" \
|
|
|
18 |
|
19 |
|
20 |
def get_column_info(columns):
|
|
|
21 |
markdown = ""
|
22 |
for column in columns:
|
23 |
markdown += f"- `num({column})`: num of tokens containing {abbr2name[column]} characters\n" \
|
playground_examples.py
CHANGED
@@ -37,7 +37,7 @@ examples = {
|
|
37 |
|
38 |
more_examples = [
|
39 |
# bert系列
|
40 |
-
("bert-base-cased", "bert-base-uncased", "", ""), # # clue VS kplug, bert VS clue
|
41 |
("bert-base-cased", "clue", "", "增加了[]()"),
|
42 |
("roberta-chinese-clue", "kplug", "", ""),
|
43 |
|
|
|
37 |
|
38 |
more_examples = [
|
39 |
# bert系列
|
40 |
+
("google-bert/bert-base-cased", "google-bert/bert-base-uncased", "", ""), # # clue VS kplug, bert VS clue
|
41 |
("bert-base-cased", "clue", "", "增加了[]()"),
|
42 |
("roberta-chinese-clue", "kplug", "", ""),
|
43 |
|
playground_util.py
CHANGED
@@ -15,7 +15,7 @@ Buenos días!
|
|
15 |
default_tokenizer_name_1 = "gradientai/Llama-3-8B-Instruct-Gradient-1048k"
|
16 |
default_tokenizer_name_2 = "openai/gpt-4"
|
17 |
|
18 |
-
@lru_cache
|
19 |
def tokenize(
|
20 |
text: str,
|
21 |
tokenizer_name: str,
|
|
|
15 |
default_tokenizer_name_1 = "gradientai/Llama-3-8B-Instruct-Gradient-1048k"
|
16 |
default_tokenizer_name_2 = "openai/gpt-4"
|
17 |
|
18 |
+
# @lru_cache
|
19 |
def tokenize(
|
20 |
text: str,
|
21 |
tokenizer_name: str,
|