add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/May06_15-10-21_hydcorp-esxi09/1651830045.849557/events.out.tfevents.1651830045.hydcorp-esxi09.1557537.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab87540348bf6b48e73766e14b1a1fc9982adc4be78a2f8099115a97a1bbeb31
+size 4539

runs/May06_15-10-21_hydcorp-esxi09/events.out.tfevents.1651830045.hydcorp-esxi09.1557537.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2790c90ee1d954973a264e3764e3d199f156722b681b31861f2b4028933df43c
+size 4201

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer"}