Spaces:
Running
Running
File size: 291 Bytes
f4973d4 d27a756 f4973d4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 |
from vocab.gpt_35_turbo import tokenizer
text = "你好，请告诉我聚乙烯是什么"
encoding = tokenizer.encode(text)
print(tokenizer.decode([6744]))
print(tokenizer.convert_ids_to_tokens([6744]))
print(tokenizer.decode([100256]))
print(tokenizer.convert_ids_to_tokens([100256])) |