baoule_tokenizer / exemple.py
Adjoumani's picture
Upload exemple.py with huggingface_hub
817ba05 verified
raw
history blame
256 Bytes
from baoule_tokenizer import BaouleTokenizer
tokenizer = BaouleTokenizer()
tokenizer.load("baoule_tokenizer.model")
text = "Votre texte en baoulé ici"
encoded = tokenizer.encode(text)
decoded = tokenizer.decode(encoded)
print(encoded)
print(decoded)