Update README.md
Browse files
README.md
CHANGED
@@ -33,4 +33,8 @@ tokenizer = AutoTokenizer.from_pretrained("hugohow/creole_reunion_tokenizer")
|
|
33 |
# Example of tokenization
|
34 |
text = "Comment i lé zot tout ?"
|
35 |
tokens = tokenizer.encode(text)
|
36 |
-
print(tokens)
|
|
|
|
|
|
|
|
|
|
33 |
# Example of tokenization
|
34 |
text = "Comment i lé zot tout ?"
|
35 |
tokens = tokenizer.encode(text)
|
36 |
+
print(tokens)
|
37 |
+
```
|
38 |
+
|
39 |
+
|
40 |
+
Hugo How-Choong
|