krishnamishra8848/Nepali_Tokenizer

# Example Code: State of Art Nepali Tokenizer

from transformers import AutoTokenizer

# Load the tokenizer directly from Hugging Face
tokenizer = AutoTokenizer.from_pretrained("krishnamishra8848/Nepali_Tokenizer")

# Test tokenization with some text
text = "तपाईंले आफ्ना लक्ष्यलाई प्राप्त गर्नका लागि मिहिनेत र धैर्यको महत्त्व बुझ्नुपर्छ।"
tokens = tokenizer(text)

print("Tokenized Output:", tokens)