add tokenizer

runs/May06_13-46-10_hydcorp-esxi09/1651824982.611327/events.out.tfevents.1651824982.hydcorp-esxi09.1532990.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cbfb6a01bbe552af2c053df485255ea99f1f45cbf16c4a52b6f0d7b0927008f
+size 4792

runs/May06_13-46-10_hydcorp-esxi09/events.out.tfevents.1651824982.hydcorp-esxi09.1532990.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e330e740237cd821989e9f9f9ad06595a7122926bc71668452bb2d7e362fb83f
+size 4757

vocab.json

@@ -1 +1 @@
-{"'": 0, "q": 1, "c": 2, "t": 3, "r": 4, "d": 5, "b": 6, "p": 7, "8": 8, "n": 10, "2": 11, "u": 12, "o": 13, "x": 14, "a": 15, "0": 16, "6": 17, "5": 18, "m": 19, "h": 20, "i": 21, "f": 22, "z": 23, "9": 24, "4": 25, "y": 26, "g": 27, "3": 28, "s": 29, "l": 30, "k": 31, "e": 32, "w": 33, "1": 34, "j": 35, "v": 36, "|": 9, "[UNK]": 37, "[PAD]": 38}
+{"f": 0, "h": 1, "a": 2, "y": 3, "5": 4, "l": 6, "6": 7, "9": 8, "o": 9, "1": 10, "'": 11, "e": 12, "i": 13, "2": 14, "b": 15, "3": 16, "v": 17, "p": 18, "w": 19, "z": 20, "u": 21, "d": 22, "m": 23, "x": 24, "8": 25, "k": 26, "0": 27, "j": 28, "4": 29, "g": 30, "c": 31, "q": 32, "s": 33, "t": 34, "n": 35, "r": 36, "|": 5, "[UNK]": 37, "[PAD]": 38}