Commit
·
553792f
1
Parent(s):
14fc818
Upload tokenizer
Browse files- added_tokens.json +2 -2
- vocab.json +3 -4
added_tokens.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"</s>":
|
| 3 |
-
"<s>":
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"</s>": 63,
|
| 3 |
+
"<s>": 62
|
| 4 |
}
|
vocab.json
CHANGED
|
@@ -1,8 +1,7 @@
|
|
| 1 |
{
|
| 2 |
-
"[PAD]":
|
| 3 |
-
"[UNK]":
|
| 4 |
-
"
|
| 5 |
-
"।": 2,
|
| 6 |
"ଁ": 3,
|
| 7 |
"ଂ": 4,
|
| 8 |
"ଃ": 5,
|
|
|
|
| 1 |
{
|
| 2 |
+
"[PAD]": 63,
|
| 3 |
+
"[UNK]": 62,
|
| 4 |
+
"।": 1,
|
|
|
|
| 5 |
"ଁ": 3,
|
| 6 |
"ଂ": 4,
|
| 7 |
"ଃ": 5,
|