Training in progress, step 500

.ipynb_checkpoints/eval-checkpoint.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 import argparse
+import functools
 import re
 from typing import Dict
 
-from datasets import Audio, Dataset, load_dataset, load_metric
+from datasets import Audio, Dataset, DatasetDict, load_dataset, load_metric
 
-from transformers import AutoFeatureExtractor, pipeline
+from transformers import AutoFeatureExtractor, AutoTokenizer, pipeline
 
 
 def log_results(result: Dataset, args: Dict[str, str]):
@@ -63,12 +64,50 @@ def normalize_text(text: str) -> str:
     return text
 
 
+def create_vocabulary_from_data(
+    datasets: DatasetDict,
+    word_delimiter_token = None,
+    unk_token = None,
+    pad_token = None,
+):
+    # Given training and test labels create vocabulary
+    def extract_all_chars(batch):
+        all_text = " ".join(batch["target"])
+        vocab = list(set(all_text))
+        return {"vocab": [vocab], "all_text": [all_text]}
+
+    vocabs = datasets.map(
+        extract_all_chars,
+        batched=True,
+        batch_size=-1,
+        keep_in_memory=True,
+        remove_columns=datasets["test"].column_names,
+    )
+
+
+    vocab_dict = {v: k for k, v in enumerate(sorted(list(vocabs["test"]["vocab"])))}
+
+    # replace white space with delimiter token
+    if word_delimiter_token is not None:
+        vocab_dict[word_delimiter_token] = vocab_dict[" "]
+        del vocab_dict[" "]
+
+    # add unk and pad token
+    if unk_token is not None:
+        vocab_dict[unk_token] = len(vocab_dict)
+
+    if pad_token is not None:
+        vocab_dict[pad_token] = len(vocab_dict)
+
+    return vocab_dict
+
+
 def main(args):
     # load dataset
     dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
 
     # for testing: only process the first two examples as a test
-    # dataset = dataset.select(range(10))
+    dataset = dataset.select(range(10))
 
     # load processor
     feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)
@@ -90,6 +129,7 @@ def main(args):
         batch["target"] = normalize_text(batch["sentence"])
         return batch
 
+
     # run inference on all examples
     result = dataset.map(map_to_pred, remove_columns=dataset.column_names)
 
@@ -97,6 +137,21 @@ def main(args):
     # do not change function below
     log_results(result, args)
 
+    if args.check_vocab:
+        tokenizer = AutoTokenizer.from_pretrained(args.model_id)
+        unk_token = "[UNK]"
+        pad_token = "[PAD]"
+        word_delimiter_token = "|"
+        raw_datasets = DatasetDict({"test": result})
+        vocab_dict = create_vocabulary_from_data(
+                    raw_datasets,
+                    word_delimiter_token=word_delimiter_token,
+                    unk_token=unk_token,
+                    pad_token=pad_token,
+                )
+        print(vocab_dict)
+        print(set(vocab_dict) - set(tokenizer.get_vocab()))
+    
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -123,6 +178,9 @@ if __name__ == "__main__":
     parser.add_argument(
         "--log_outputs", action="store_true", help="If defined, write outputs to log file for analysis."
     )
+    parser.add_argument(
+        "--check_vocab", action="store_true", help="Verify that normalized target text is within character set"
+    )
     args = parser.parse_args()
 
     main(args)

.ipynb_checkpoints/run-checkpoint.sh

@@ -16,6 +16,7 @@ python xls-r-uzbek-cv8/run_speech_recognition_ctc.py \
 	--save_steps="500" \
 	--eval_steps="500" \
 	--logging_steps="100" \
+	--min_duration_in_seconds="0.5" \
 	--layerdrop="0.0" \
 	--activation_dropout="0.1" \
 	--save_total_limit="3" \

.ipynb_checkpoints/run_speech_recognition_ctc-checkpoint.py

@@ -553,6 +553,7 @@ def main():
             "gradient_checkpointing": training_args.gradient_checkpointing,
             "layerdrop": model_args.layerdrop,
             "ctc_loss_reduction": model_args.ctc_loss_reduction,
+            "zero_infinity": True,
             "pad_token_id": tokenizer.pad_token_id,
             "vocab_size": len(tokenizer),
             "activation_dropout": model_args.activation_dropout,

config.json

@@ -100,8 +100,9 @@
     1
   ],
   "torch_dtype": "float32",
-  "transformers_version": "4.16.0.dev0",
+  "transformers_version": "4.17.0.dev0",
   "use_weighted_layer_sum": false,
   "vocab_size": 32,
-  "xvector_output_dim": 512
+  "xvector_output_dim": 512,
+  "zero_infinity": true
 }

eval.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 import argparse
+import functools
 import re
 from typing import Dict
 
-from datasets import Audio, Dataset, load_dataset, load_metric
+from datasets import Audio, Dataset, DatasetDict, load_dataset, load_metric
 
-from transformers import AutoFeatureExtractor, pipeline
+from transformers import AutoFeatureExtractor, AutoTokenizer, pipeline
 
 
 def log_results(result: Dataset, args: Dict[str, str]):
@@ -63,12 +64,50 @@ def normalize_text(text: str) -> str:
     return text
 
 
+def create_vocabulary_from_data(
+    datasets: DatasetDict,
+    word_delimiter_token = None,
+    unk_token = None,
+    pad_token = None,
+):
+    # Given training and test labels create vocabulary
+    def extract_all_chars(batch):
+        all_text = " ".join(batch["target"])
+        vocab = list(set(all_text))
+        return {"vocab": [vocab], "all_text": [all_text]}
+
+    vocabs = datasets.map(
+        extract_all_chars,
+        batched=True,
+        batch_size=-1,
+        keep_in_memory=True,
+        remove_columns=datasets["test"].column_names,
+    )
+
+
+    vocab_dict = {v: k for k, v in enumerate(sorted(list(vocabs["test"]["vocab"])))}
+
+    # replace white space with delimiter token
+    if word_delimiter_token is not None:
+        vocab_dict[word_delimiter_token] = vocab_dict[" "]
+        del vocab_dict[" "]
+
+    # add unk and pad token
+    if unk_token is not None:
+        vocab_dict[unk_token] = len(vocab_dict)
+
+    if pad_token is not None:
+        vocab_dict[pad_token] = len(vocab_dict)
+
+    return vocab_dict
+
+
 def main(args):
     # load dataset
     dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
 
     # for testing: only process the first two examples as a test
-    # dataset = dataset.select(range(10))
+    dataset = dataset.select(range(10))
 
     # load processor
     feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)
@@ -90,6 +129,7 @@ def main(args):
         batch["target"] = normalize_text(batch["sentence"])
         return batch
 
+
     # run inference on all examples
     result = dataset.map(map_to_pred, remove_columns=dataset.column_names)
 
@@ -97,6 +137,21 @@ def main(args):
     # do not change function below
     log_results(result, args)
 
+    if args.check_vocab:
+        tokenizer = AutoTokenizer.from_pretrained(args.model_id)
+        unk_token = "[UNK]"
+        pad_token = "[PAD]"
+        word_delimiter_token = "|"
+        raw_datasets = DatasetDict({"test": result})
+        vocab_dict = create_vocabulary_from_data(
+                    raw_datasets,
+                    word_delimiter_token=word_delimiter_token,
+                    unk_token=unk_token,
+                    pad_token=pad_token,
+                )
+        print(vocab_dict)
+        print(set(vocab_dict) - set(tokenizer.get_vocab()))
+    
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -123,6 +178,9 @@ if __name__ == "__main__":
     parser.add_argument(
         "--log_outputs", action="store_true", help="If defined, write outputs to log file for analysis."
     )
+    parser.add_argument(
+        "--check_vocab", action="store_true", help="Verify that normalized target text is within character set"
+    )
     args = parser.parse_args()
 
     main(args)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f992aa6854d16e2db041cc2d1e625cd6f276fca51497aacd3b9d9563932eab38
+oid sha256:8b71f72aac1d492daee452de8218f59a78d50f29656419218b12834c50edc15f
 size 1262054897

requirements.txt

@@ -0,0 +1,3 @@
+unidecode
+tensorboard
+

run.sh

@@ -16,6 +16,7 @@ python xls-r-uzbek-cv8/run_speech_recognition_ctc.py \
 	--save_steps="500" \
 	--eval_steps="500" \
 	--logging_steps="100" \
+	--min_duration_in_seconds="0.5" \
 	--layerdrop="0.0" \
 	--activation_dropout="0.1" \
 	--save_total_limit="3" \

run_speech_recognition_ctc.py

@@ -553,6 +553,7 @@ def main():
             "gradient_checkpointing": training_args.gradient_checkpointing,
             "layerdrop": model_args.layerdrop,
             "ctc_loss_reduction": model_args.ctc_loss_reduction,
+            "zero_infinity": True,
             "pad_token_id": tokenizer.pad_token_id,
             "vocab_size": len(tokenizer),
             "activation_dropout": model_args.activation_dropout,

runs/Feb02_06-54-25_job-699ba53c-fea9-4eb2-81af-a97f440eaa45/1643785646.6555233/events.out.tfevents.1643785646.job-699ba53c-fea9-4eb2-81af-a97f440eaa45.33872.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9500cee755dde8faf816fe7422af4bc642d5207e5a7715fc6d36a685700c4666
+size 4799

runs/Feb02_06-54-25_job-699ba53c-fea9-4eb2-81af-a97f440eaa45/events.out.tfevents.1643785646.job-699ba53c-fea9-4eb2-81af-a97f440eaa45.33872.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:412cb17226cf304713e4986e1eab9dffa422ea44ea63de0bdd747546e911336e
+size 5852

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ef714526a959c262fd3874035daab89db41dfb4f72a127557e4538085974074
+oid sha256:39c0625450d0afa8d2e897190721a9173256a42e1f889cdecc94feee325632c3
 size 3055