mallikrao2 commited on
Commit
8e057c4
·
1 Parent(s): af978af

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +1 -1
vocab.json CHANGED
@@ -1 +1 @@
1
- {"x": 0, "j": 1, "r": 2, "b": 3, "d": 5, "o": 6, "s": 7, "l": 8, "'": 9, "c": 10, "v": 11, "e": 12, "y": 13, "u": 14, "i": 15, "k": 16, "p": 17, "w": 18, "z": 19, "h": 20, "q": 21, "n": 22, "f": 23, "a": 24, "m": 25, "t": 26, "g": 27, "|": 4, "[unq]": 28, "[padded]": 29}
 
1
+ {"g": 0, "l": 1, "t": 2, "m": 3, "c": 4, "a": 5, "z": 6, "r": 7, "y": 8, "e": 9, "q": 10, "v": 11, "k": 12, "u": 13, "i": 14, "'": 15, "b": 16, "x": 17, "d": 19, "w": 20, "s": 21, "f": 22, "h": 23, "j": 24, "p": 25, "o": 26, "n": 27, "|": 18, "[unq]": 28, "[padded]": 29}