add tokenizer

runs/May30_11-00-21_5fadd493bdeb/1653908607.0582464/events.out.tfevents.1653908607.5fadd493bdeb.86.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86bffb8889241a9ab96a0937f87180da23dcfdff7642a71cf3e792e039cf108b
+size 4528

runs/May30_11-00-21_5fadd493bdeb/events.out.tfevents.1653908607.5fadd493bdeb.86.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c90ff4d8a2aba8d85df6d7ccfff785872e1560322caf3df122e3a13747f33ca
+size 4403

runs/May30_11-05-36_5fadd493bdeb/1653908922.4678671/events.out.tfevents.1653908922.5fadd493bdeb.86.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75b5250b6bb230c0c79d210b6860328add29b3e4c4fbe27a1e878064a447bd67
+size 4528

runs/May30_11-05-36_5fadd493bdeb/events.out.tfevents.1653908922.5fadd493bdeb.86.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcabe975e41c31f4582e635f7c81d82ecd800e5f0d44460a779367092e512f31
+size 4403

vocab.json

@@ -1 +1 @@
-{"<pad>": 0, "<s>": 1, "</s>": 2, "<unk>": 3, "|": 4, "ء": 5, "آ": 6, "ئ": 7, "ا": 8, "ب": 9, "ت": 10, "ث": 11, "ج": 12, "ح": 13, "خ": 14, "د": 15, "ذ": 16, "ر": 17, "ز": 18, "س": 19, "ش": 20, "ص": 21, "ض": 22, "ط": 23, "ظ": 24, "ع": 25, "غ": 26, "ف": 27, "ق": 28, "ل": 29, "م": 30, "ن": 31, "ه": 32, "و": 33, "پ": 34, "چ": 35, "ژ": 36, "ک": 37, "گ": 38, "ی": 39, "[UNK]": 40, "[PAD]": 41}
+{"<pad>": 0, "<s>": 1, "</s>": 2, "<unk>": 3, "|": 4, "ء": 5, "آ": 6, "ئ": 7, "ا": 8, "ب": 9, "ت": 10, "ث": 11, "ج": 12, "ح": 13, "خ": 14, "د": 15, "ذ": 16, "ر": 17, "ز": 18, "س": 19, "ش": 20, "ص": 21, "ض": 22, "ط": 23, "ظ": 24, "ع": 25, "غ": 26, "ف": 27, "ق": 28, "ل": 29, "م": 30, "ن": 31, "ه": 32, "و": 33, "پ": 34, "چ": 35, "ژ": 36, "ک": 37, "گ": 38, "ی": 39}