add tokenizer

runs/May08_22-17-15_hydcorp-esxi09/1652028450.5587265/events.out.tfevents.1652028450.hydcorp-esxi09.1602021.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b8d2b3d343094595b8f0ffb081165e84ae958beaab9f4e8fb9d9ca73d277fe0
+size 4482

runs/May08_22-17-15_hydcorp-esxi09/events.out.tfevents.1652028450.hydcorp-esxi09.1602021.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:785b975fa7f6f025872e513b1576008e912451b52cb32f9d991ef56c1da2dacc
+size 4144

vocab.json

@@ -1 +1 @@
-{"p": 0, "d": 2, "h": 3, "1": 4, "g": 5, "v": 6, "i": 7, "8": 8, "s": 9, "a": 10, "e": 11, "x": 12, "t": 13, "2": 14, "0": 15, "9": 16, "m": 17, "6": 18, "k": 19, "z": 20, "3": 21, "u": 22, "4": 23, "l": 24, "n": 25, "q": 26, "j": 27, "y": 28, "r": 29, "f": 30, "b": 31, "w": 32, "'": 33, "o": 34, "c": 35, "5": 36, "|": 1, "[UNK]": 37, "[PAD]": 38}
+{"i": 0, "k": 1, "l": 2, "y": 3, "x": 4, "w": 5, "4": 6, "1": 7, "n": 8, "3": 9, "j": 10, "z": 11, "o": 12, "8": 13, "g": 14, "p": 15, "6": 16, "b": 17, "m": 18, "u": 19, "r": 21, "e": 22, "5": 23, "q": 24, "'": 25, "s": 26, "h": 27, "0": 28, "a": 29, "v": 30, "2": 31, "9": 32, "d": 33, "t": 34, "f": 35, "c": 36, "|": 20, "[UNK]": 37, "[PAD]": 38}