add tokenizer

runs/May06_13-54-21_hydcorp-esxi09/1651825472.9060152/events.out.tfevents.1651825472.hydcorp-esxi09.1534756.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85d259d708d5a7dc010fa900a0acaa220cdb57af327587a76adb490de3a897c3
+size 4792

runs/May06_13-54-21_hydcorp-esxi09/events.out.tfevents.1651825472.hydcorp-esxi09.1534756.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1fe1b481334f9f22d661967ab1599a1ed0bb7440e7c32b3f041b3f3641ca72d
+size 4757

vocab.json

@@ -1 +1 @@
-{"k": 0, "c": 1, "e": 2, "s": 3, "v": 4, "m": 5, "0": 6, "y": 7, "2": 9, "1": 10, "j": 11, "9": 12, "r": 13, "6": 14, "f": 15, "5": 16, "l": 17, "g": 18, "a": 19, "8": 20, "'": 21, "i": 22, "w": 23, "b": 24, "p": 25, "x": 26, "d": 27, "t": 28, "n": 29, "u": 30, "z": 31, "3": 32, "o": 33, "h": 34, "4": 35, "q": 36, "|": 8, "[UNK]": 37, "[PAD]": 38}
+{"2": 0, "p": 1, "q": 2, "u": 3, "v": 4, "l": 5, "3": 6, "5": 7, "j": 8, "o": 9, "r": 10, "f": 11, "g": 12, "6": 13, "m": 14, "d": 15, "8": 16, "y": 17, "e": 18, "h": 19, "t": 20, "k": 21, "i": 22, "4": 23, "0": 24, "x": 25, "'": 26, "1": 27, "n": 28, "w": 29, "s": 30, "c": 32, "a": 33, "b": 34, "9": 35, "z": 36, "|": 31, "[UNK]": 37, "[PAD]": 38}