add tokenizer

runs/May06_14-00-35_hydcorp-esxi09/1651825846.8922675/events.out.tfevents.1651825846.hydcorp-esxi09.1535644.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e48d3d4bfe5394aab174d1dc6bd272792e71c8b7791e15308d577837e98a092b
+size 4792

runs/May06_14-00-35_hydcorp-esxi09/events.out.tfevents.1651825846.hydcorp-esxi09.1535644.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a94d6b7bfa0fd6b7389d53ab8eb63d04e48ce01acb23156cd516df051a4eaf75
+size 4757

vocab.json

@@ -1 +1 @@
-{"2": 0, "p": 1, "q": 2, "u": 3, "v": 4, "l": 5, "3": 6, "5": 7, "j": 8, "o": 9, "r": 10, "f": 11, "g": 12, "6": 13, "m": 14, "d": 15, "8": 16, "y": 17, "e": 18, "h": 19, "t": 20, "k": 21, "i": 22, "4": 23, "0": 24, "x": 25, "'": 26, "1": 27, "n": 28, "w": 29, "s": 30, "c": 32, "a": 33, "b": 34, "9": 35, "z": 36, "|": 31, "[UNK]": 37, "[PAD]": 38}
+{"[PAD]": 0, "p": 1, "q": 2, "u": 3, "v": 4, "l": 5, "3": 6, "5": 7, "j": 8, "o": 9, "r": 10, "f": 11, "g": 12, "6": 13, "m": 14, "d": 15, "8": 16, "y": 17, "e": 18, "h": 19, "t": 20, "k": 21, "i": 22, "4": 23, "0": 24, "x": 25, "'": 26, "1": 27, "n": 28, "w": 29, "s": 30, "c": 32, "a": 33, "b": 34, "9": 35, "z": 36, "|": 31, "[UNK]": 37, "2": 38}