Add Wav2Vec ASR Model Files

wav2vecasr/MispronounciationDetector.py

@@ -0,0 +1,129 @@
+from pandas.core.construction import T
+import torch
+import jiwer
+
+class MispronounciationDetector:
+  def __init__(self, l2_phoneme_recogniser, l2_phoneme_recogniser_processor, g2p, device):
+    self.l2_phoneme_recogniser = l2_phoneme_recogniser
+    self.l2_phoneme_recogniser_processor = l2_phoneme_recogniser_processor
+    self.g2p = g2p
+    self.device = device
+
+  def detect(self, audio, text):
+    l2_phones = self.get_l2_phoneme_sequence(audio)
+    native_speaker_phones = self.get_native_speaker_phoneme_sequence(text)
+    raw_info = self.get_mispronounciation_output(text, l2_phones, native_speaker_phones)
+    return raw_info
+
+  def get_l2_phoneme_sequence(self, audio):
+    input_dict = self.l2_phoneme_recogniser_processor(audio, sampling_rate=16000, return_tensors="pt", padding=True)
+    logits = self.l2_phoneme_recogniser(input_dict.input_values.to(self.device)).logits
+    pred_ids = torch.argmax(logits, dim=-1)[0]
+    pred_phones = [phoneme for phoneme in self.l2_phoneme_recogniser_processor.batch_decode(pred_ids) if phoneme != ""]
+    return pred_phones
+
+  def get_native_speaker_phoneme_sequence(self, text):
+    phonemes = self.g2p(text)
+    return phonemes
+
+  def get_mispronounciation_output(self, text, pred_phones, org_label_phones):
+    # get per
+    label_phones = [phone for phone in org_label_phones if phone != " "]
+    reference = " ".join(label_phones) # dummy phones
+    hypothesis = " ".join(pred_phones) # dummy l2 speaker phones
+    res = jiwer.process_words(reference, hypothesis)
+    per = res.wer
+    # print(jiwer.visualize_alignment(res))
+
+    # get phoneme alignments
+    alignments = res.alignments
+    error_bool = []
+    ref, hyp = [],[]
+    for alignment_chunk in alignments[0]:
+      alignment_type = alignment_chunk.type
+      ref_start_idx = alignment_chunk.ref_start_idx
+      ref_end_idx = alignment_chunk.ref_end_idx
+      hyp_start_idx = alignment_chunk.hyp_start_idx
+      hyp_end_idx = alignment_chunk.hyp_end_idx
+      if alignment_type != "equal":
+        if alignment_type == "insert":
+          for i in range(hyp_start_idx, hyp_end_idx):
+            ref.append("*" * len(pred_phones[i]))
+            space_padding = " " * (len(pred_phones[i])-1)
+            error_bool.append(space_padding + "a")
+          hyp.extend(pred_phones[hyp_start_idx:hyp_end_idx])
+        elif alignment_type == "delete":
+          ref.extend(label_phones[ref_start_idx:ref_end_idx])
+          for i in range(ref_start_idx, ref_end_idx):
+            hyp.append("*" * len(label_phones[i]))
+            space_padding = " " * (len(label_phones[i])-1)
+            error_bool.append(space_padding + alignment_type[0])
+        else:
+          for i in range(ref_end_idx - ref_start_idx):
+            correct_phone = label_phones[ref_start_idx+i]
+            pred_phone = pred_phones[hyp_start_idx+i]
+            if len(correct_phone) > len(pred_phone):
+              space_padding = " " * (len(correct_phone) - len(pred_phone))
+              ref.append(correct_phone)
+              hyp.append(space_padding + pred_phone)
+              error_bool.append(" " * (len(correct_phone)-1) + alignment_type[0])
+            else:
+              space_padding = " " * (len(pred_phone) - len(correct_phone))
+              ref.append(space_padding + correct_phone)
+              hyp.append(pred_phone)
+              error_bool.append(" " * (len(pred_phone)-1) + alignment_type[0])
+      else:
+        ref.extend(label_phones[ref_start_idx:ref_end_idx])
+        hyp.extend(pred_phones[hyp_start_idx:hyp_end_idx])
+        # ref or hyp does not matter
+        for i in range(ref_start_idx, ref_end_idx):
+          space_padding = "-" * (len(label_phones[i]))
+          error_bool.append(space_padding)
+
+    delimiter_idx = 0
+    for phone in org_label_phones:
+      if phone == " ":
+        hyp.insert(delimiter_idx+1, "|")
+        ref.insert(delimiter_idx+1, "|")
+        error_bool.insert(delimiter_idx+1, "|")
+        continue
+      while delimiter_idx < len(ref) and ref[delimiter_idx].strip() != phone:
+        delimiter_idx += 1
+    # word ends
+    ref.append("|")
+    hyp.append("|")
+
+    # get mispronounced words
+    aligned_word_error_output = ""
+    words = text.split(" ")
+    word_error_bool = self.get_mispronounced_words(error_bool)
+    wer = sum(word_error_bool) / len(words)
+
+    raw_info = {"ref":ref, "hyp": hyp, "per":per, "phoneme_errors": error_bool, "wer": wer, "words": words, "word_errors":word_error_bool}
+
+    return raw_info
+
+
+  def get_mispronounced_words(self, phoneme_error_bool):
+    # map mispronounced phones back to words that were mispronounce to get WER
+    word_error_bool = []
+    phoneme_error_bool.append("|")
+    word_phones = self.split_lst_by_delim(phoneme_error_bool, "|")
+    for phones in word_phones:
+      if "s" in phones or "d" in phones or "a" in phones:
+        word_error_bool.append(True)
+      else:
+        word_error_bool.append(False)
+    return word_error_bool
+
+
+  def split_lst_by_delim(self, lst, delimiter):
+    temp = []
+    res = []
+    for item in lst:
+      if item != delimiter:
+        temp.append(item.strip())
+      else:
+        res.append(temp);
+        temp = []
+    return res

wav2vecasr/data/arctic_a0003.txt

@@ -0,0 +1 @@
+For the twentieth time that evening the two men shook hands

wav2vecasr/demo.py

@@ -0,0 +1,32 @@
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+from speechbrain.pretrained import GraphemeToPhoneme
+import datasets
+import os
+import torchaudio
+from MispronounciationDetector import MispronounciationDetector
+
+# Load sample data
+audio_path, transcript_path = os.path.join(os.getcwd(), "data", "arctic_a0003.wav"), os.path.join(os.getcwd(), "data", "arctic_a0003.txt")
+audio, org_sr = torchaudio.load(audio_path)
+audio = torchaudio.functional.resample(audio, orig_freq=org_sr, new_freq=16000)
+audio = audio.view(audio.shape[1])
+with open(transcript_path) as f:
+  text = f.read()
+f.close()
+print("Done loading sample data")
+
+# Load processors and models
+device = "cpu"
+path = os.path.join(os.getcwd(), "model", "checkpoint-1200")
+model = Wav2Vec2ForCTC.from_pretrained(path).to(device)
+processor = Wav2Vec2Processor.from_pretrained(path)
+g2p = GraphemeToPhoneme.from_hparams("speechbrain/soundchoice-g2p")
+mispronounciation_detector = MispronounciationDetector(model, processor, g2p, "cpu")
+print("Done loading models and processors")
+
+# Predict
+raw_info = mispronounciation_detector.detect(audio, text)
+aligned_phoneme_output_delimited_by_words = " ".join(raw_info['ref']) + "\n" + " ".join(raw_info['hyp']) + "\n" +\
+                                            " ".join(raw_info['phoneme_errors'])
+print(f"PER: {raw_info['per']}\n")
+print(f"Phoneme level errors:\n{raw_info['phoneme_output']}\n")

wav2vecasr/model/checkpoint-1200/config.json

@@ -0,0 +1,108 @@
+{
+  "_name_or_path": "/content/drive/MyDrive/NUS/Y4S1/Sound and Music Computing/CS4347 Project/Experiments/Wav2Vec Baselines/L2 Artic 3 Speakers: Baseline 2/wav2vec-baseline2-model-checkpoints/checkpoint-600",
+  "activation_dropout": 0.0,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForCTC"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 768,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": true,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "mean",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.0,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.0,
+  "mask_feature_length": 64,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.25,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.75,
+  "model_type": "wav2vec2",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "output_hidden_size": 1024,
+  "pad_token_id": 82,
+  "proj_codevector_dim": 768,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.17.0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 83,
+  "xvector_output_dim": 512
+}

wav2vecasr/model/checkpoint-1200/preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

wav2vecasr/model/checkpoint-1200/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44bd3813b64d85faa8f88091f160cc107e340ee71372b470dd6c4b09cb00906d
+size 1262269741

wav2vecasr/model/checkpoint-1200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6686b1782523e688cd46835b2db33ae51a6ffd852401967b311db1a20efad2ee
+size 14639

wav2vecasr/model/checkpoint-1200/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:662b2a6102fe369b78bf169eb2bcea08b4dc636d31dfd2652b32a63eda7e03e8
+size 557

wav2vecasr/model/checkpoint-1200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcbd8882ac5e67f1b9d59f4eaa2583483d429dfecec5ce45fd99da4d06e47847
+size 627

wav2vecasr/model/checkpoint-1200/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}

wav2vecasr/model/checkpoint-1200/tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

wav2vecasr/model/checkpoint-1200/trainer_state.json

@@ -0,0 +1,106 @@
+{
+  "best_metric": 17.541647491946616,
+  "best_model_checkpoint": "/content/drive/MyDrive/NUS/Y4S1/Sound and Music Computing/CS4347 Project/Experiments/Wav2Vec Baselines/L2 Artic 3 Speakers: Baseline 2/wav2vec-baseline2-model-checkpoints/checkpoint-200",
+  "epoch": 4.411764705882353,
+  "global_step": 1200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001194,
+      "loss": 4.1351,
+      "step": 200
+    },
+    {
+      "epoch": 0.74,
+      "eval_loss": 1.9543204307556152,
+      "eval_per": 17.541647491946616,
+      "eval_runtime": 143.4439,
+      "eval_samples_per_second": 15.149,
+      "eval_steps_per_second": 1.896,
+      "step": 200
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 0.0002394,
+      "loss": 1.7915,
+      "step": 400
+    },
+    {
+      "epoch": 1.47,
+      "eval_loss": 1.692239761352539,
+      "eval_per": 7.960756135308424,
+      "eval_runtime": 137.113,
+      "eval_samples_per_second": 15.848,
+      "eval_steps_per_second": 1.984,
+      "step": 400
+    },
+    {
+      "epoch": 2.21,
+      "learning_rate": 0.0002971387283236994,
+      "loss": 1.2246,
+      "step": 600
+    },
+    {
+      "epoch": 2.21,
+      "eval_loss": 0.5273078083992004,
+      "eval_per": 0.31805393535991217,
+      "eval_runtime": 136.1021,
+      "eval_samples_per_second": 15.966,
+      "eval_steps_per_second": 1.999,
+      "step": 600
+    },
+    {
+      "epoch": 2.94,
+      "learning_rate": 0.0002913872832369942,
+      "loss": 0.9433,
+      "step": 800
+    },
+    {
+      "epoch": 2.94,
+      "eval_loss": 0.41386935114860535,
+      "eval_per": 0.2565853269749339,
+      "eval_runtime": 136.2091,
+      "eval_samples_per_second": 15.953,
+      "eval_steps_per_second": 1.997,
+      "step": 800
+    },
+    {
+      "epoch": 3.68,
+      "learning_rate": 0.000285606936416185,
+      "loss": 0.8842,
+      "step": 1000
+    },
+    {
+      "epoch": 3.68,
+      "eval_loss": 0.3962230980396271,
+      "eval_per": 0.24980343554684897,
+      "eval_runtime": 139.9847,
+      "eval_samples_per_second": 15.523,
+      "eval_steps_per_second": 1.943,
+      "step": 1000
+    },
+    {
+      "epoch": 4.41,
+      "learning_rate": 0.00027982658959537567,
+      "loss": 0.8542,
+      "step": 1200
+    },
+    {
+      "epoch": 4.41,
+      "eval_loss": 0.3784765601158142,
+      "eval_per": 0.24003603985584057,
+      "eval_runtime": 136.6045,
+      "eval_samples_per_second": 15.907,
+      "eval_steps_per_second": 1.991,
+      "step": 1200
+    }
+  ],
+  "max_steps": 10880,
+  "num_train_epochs": 40,
+  "total_flos": 4.4309288969819863e+18,
+  "trial_name": null,
+  "trial_params": null
+}

wav2vecasr/model/checkpoint-1200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d30a9efd4c6d95a24828b0e52d71ed7ad4f3c83075c158d1488ebe5f50b6719
+size 3323

wav2vecasr/model/checkpoint-1200/vocab.json

@@ -0,0 +1 @@
+{"K*": 0, "Z*": 1, "AE*": 2, "B": 3, "UH*": 4, "W": 5, "SIL": 6, "CH": 7, "EH*": 8, "T": 9, "D_": 10, "W*": 11, "K": 12, "spn": 13, "AH": 14, "AH*": 15, "NG": 16, "P*": 17, "B*": 18, "G": 19, "OY": 20, "D": 21, "ZH": 22, "sp": 23, "V": 24, "EY": 25, "V``": 26, "UW": 27, "s": 28, "P": 29, "UW*": 30, "ER*": 31, "sil": 32, "R*": 33, "IH": 34, "OW": 35, "HH*": 36, "Y": 37, "AO": 38, "AW*": 39, "ER": 40, "OW*": 41, "AY": 42, "M": 43, "T*": 44, "DH": 45, "AA*": 46, "L": 47, "AX": 48, "N*": 49, "EH": 50, "DH*": 51, "t": 52, "ERR": 53, "AO*": 54, "Z": 55, "S": 56, "ZH*": 57, "EY*": 58, "JH*": 59, "F": 60, "L*": 61, "Y*": 62, "R": 63, "G*": 64, "JH": 65, "W`": 66, "D*": 67, "AA": 68, "IY": 69, "AE": 70, "Ah": 71, "AW": 72, "SH": 73, "TH": 74, "N": 75, "V*": 76, "HH": 77, "UH": 78, "err": 79, "|": 80, "[UNK]": 81, "[PAD]": 82}