add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/May06_13-42-02_hydcorp-esxi09/1651824734.3689704/events.out.tfevents.1651824734.hydcorp-esxi09.1531983.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70eb5b901a62c030aa502f6555ed357a2170768227114abf1ef6c7d32f2a3011
+size 4792

runs/May06_13-42-02_hydcorp-esxi09/events.out.tfevents.1651824734.hydcorp-esxi09.1531983.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37ba34ffe16e1f98eb01bc9384761fdd52b4ed2c264547d0261b13f661f4cc2f
+size 4757

vocab.json

@@ -1 +1 @@
-{"d": 0, "h": 1, "t": 2, "n": 3, "s": 4, "b": 5, "0": 6, "q": 7, "j": 8, "9": 9, "2": 10, "z": 11, "5": 12, "g": 13, "'": 14, "3": 15, "a": 16, "w": 17, "v": 19, "4": 20, "p": 21, "i": 22, "6": 23, "8": 24, "u": 25, "r": 26, "m": 27, "e": 28, "y": 29, "x": 30, "o": 31, "k": 32, "1": 33, "c": 34, "l": 35, "f": 36, "|": 18, "[UNK]": 37, "[PAD]": 38}
+{"j": 0, "9": 1, "v": 2, "3": 3, "8": 4, "4": 5, "h": 7, "n": 8, "e": 9, "1": 10, "x": 11, "f": 12, "o": 13, "y": 14, "p": 15, "5": 16, "0": 17, "b": 18, "l": 19, "d": 20, "6": 21, "c": 22, "2": 23, "z": 24, "q": 25, "a": 26, "m": 27, "u": 28, "'": 29, "i": 30, "g": 31, "w": 32, "t": 33, "k": 34, "r": 35, "s": 36, "|": 6, "[UNK]": 37, "[PAD]": 38}