add tokenizer

runs/May08_22-19-52_hydcorp-esxi09/1652028606.7162893/events.out.tfevents.1652028606.hydcorp-esxi09.1602656.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39dd68547f3c02f4bc10b3d05d47d6d1cd06add150e0122340740978380872f2
+size 4482

runs/May08_22-19-52_hydcorp-esxi09/events.out.tfevents.1652028606.hydcorp-esxi09.1602656.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be670498a415814d6b8380da3e6dc0937157bb3cd8f6c0a0e8f3de9da842f623
+size 4145

vocab.json

@@ -1 +1 @@
-{"i": 0, "k": 1, "l": 2, "y": 3, "x": 4, "w": 5, "4": 6, "1": 7, "n": 8, "3": 9, "j": 10, "z": 11, "o": 12, "8": 13, "g": 14, "p": 15, "6": 16, "b": 17, "m": 18, "u": 19, "r": 21, "e": 22, "5": 23, "q": 24, "'": 25, "s": 26, "h": 27, "0": 28, "a": 29, "v": 30, "2": 31, "9": 32, "d": 33, "t": 34, "f": 35, "c": 36, "|": 20, "[UNK]": 37, "[PAD]": 38}
+{"k": 0, "c": 1, "'": 2, "&": 3, "a": 4, "y": 5, "i": 6, "_": 7, "n": 8, "x": 9, "q": 10, "*": 11, "™": 12, "s": 13, "b": 14, "w": 15, "€": 16, "®": 17, "d": 18, "p": 19, "v": 20, "â": 21, "u": 22, "+": 23, "m": 24, "e": 25, "r": 26, "j": 27, "o": 29, ")": 30, "/": 31, "g": 32, "#": 33, "t": 34, "h": 35, "z": 36, "@": 37, "f": 38, "(": 39, "l": 40, "%": 41, "|": 28, "[UNK]": 42, "[PAD]": 43}