add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/May08_22-06-36_hydcorp-esxi09/1652027810.6546953/events.out.tfevents.1652027810.hydcorp-esxi09.1601020.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94c32d5acdac41a083fcb99c5836d4007687a83c0999a705bfceea6e7a31a2da
+size 4482

runs/May08_22-06-36_hydcorp-esxi09/events.out.tfevents.1652027810.hydcorp-esxi09.1601020.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9e4d1e6f1a221cd8c4969831c09375c9a84095b42385778745800424f54f67b
+size 4145

vocab.json

@@ -1 +1 @@
-{"r": 0, "x": 1, "w": 2, "f": 3, "3": 4, "9": 5, "h": 6, "2": 7, "y": 8, "'": 9, "a": 10, "1": 11, "b": 12, "z": 13, "4": 15, "5": 16, "n": 17, "0": 18, "k": 19, "8": 20, "c": 21, "6": 22, "u": 23, "d": 24, "i": 25, "p": 26, "q": 27, "v": 28, "e": 29, "g": 30, "t": 31, "l": 32, "j": 33, "s": 34, "m": 35, "o": 36, "|": 14, "[UNK]": 37, "[PAD]": 38}
+{"r": 0, "5": 1, "3": 2, "p": 3, "g": 4, "e": 5, "b": 6, "0": 8, "j": 9, "w": 10, "d": 11, "q": 12, "c": 13, "x": 14, "6": 15, "s": 16, "1": 17, "a": 18, "m": 19, "4": 20, "n": 21, "i": 22, "9": 23, "f": 24, "2": 25, "k": 26, "l": 27, "8": 28, "'": 29, "y": 30, "z": 31, "h": 32, "u": 33, "o": 34, "v": 35, "t": 36, "|": 7, "[UNK]": 37, "[PAD]": 38}