initial commit

README.md

@@ -1,3 +1,351 @@
 ---
+tags:
+- espnet
+- audio
+- audio-to-audio
+language: en
+datasets:
+- vctk_noisy
+- dns_ins20
+- chime4
+- reverb
+- whamr
 license: cc-by-4.0
 ---
+
+## ESPnet2 ENH model
+
+### `espnet/Wangyou_Zhang_universal_train_enh_uses_refch0_2mem_raw`
+
+This model was trained by Wangyou Zhang using the wsj0_2mix recipe in [espnet](https://github.com/espnet/espnet/).
+
+### Demo: How to use in ESPnet2
+
+Follow the [ESPnet installation instructions](https://espnet.github.io/espnet/installation.html)
+if you haven't done that already.
+
+```bash
+cd espnet
+
+pip install -e .
+cd egs2/universal_se/enh1
+./run.sh --skip_data_prep false --skip_train true --is_tse_task false --download_model espnet/Wangyou_Zhang_universal_train_enh_uses_refch0_2mem_raw
+```
+
+<!-- Generated by scripts/utils/show_enh_score.sh -->
+# RESULTS
+## Environments
+- date: `Sat Jul 15 12:50:47 CST 2023`
+- python version: `3.8.16 (default, Mar  2 2023, 03:21:46)  [GCC 11.2.0]`
+- espnet version: `espnet 202301`
+- pytorch version: `pytorch 2.0.1`
+- Git hash: ``
+  - Commit date: ``
+
+
+## USES (ref_channel=0, 2 groups of memory tokens)
+
+|dataset|condition|PESQ_WB|STOI|SAR|SDR|SIR|SI_SNR|OVRL|SIG|BAK|P808_MOS|
+|---|---|---|---|---|---|---|---|---|---|---|---|
+|vctk_noisy_tt_2spk|1ch, 48kHz||93.05|10.97|10.97|0.00|8.36|3.14|3.39|4.05|3.57|
+|vctk_noisy_tt_2spk_16k|1ch, 16kHz|3.11|95.03|21.51|21.51|0.00|19.45|3.19|3.46|4.06|3.57|
+|dns20_tt_synthetic_no_reverb|1ch, 16kHz|3.23|97.77|19.63|19.63|0.00|19.72|3.32|3.56|4.10|4.04|
+|dns20_tt_synthetic_with_reverb|1ch, 16kHz|2.75|89.87|13.40|13.40|0.00|12.90|2.36|2.85|3.21|3.37|
+|chime4_et05_simu_isolated_6ch_track|5ch, 16kHz|2.95|97.82|18.30|18.30|0.00|17.24|3.22|3.47|4.07|3.75|
+|reverb_et_simu_8ch_multich|8ch, 16kHz|2.09|89.83|11.94|11.94|0.00|-10.12|2.98|3.35|3.79|3.90|
+|whamr_tt_mix_single_anechoic_max_16k|2ch, 16kHz|2.55|96.36|15.78|15.78|0.00|15.46|3.33|3.55|4.16|3.86|
+|whamr_tt_mix_single_reverb_max_16k|2ch, 16kHz|2.51|95.98|13.75|13.75|0.00|12.51|3.32|3.54|4.15|3.86|
+|chime4_et05_real_isolated_6ch_track_1ch|5ch, 16kHz|1.23|55.11|-2.34|-2.34|0.00|-30.45|3.07|3.36|3.98|3.75|
+|reverb_et_real_8ch_multich|8ch, 16kHz|1.17|75.30|4.39|4.39|0.00|1.62|3.11|3.42|3.97|3.99|
+
+## ENH config
+
+<details><summary>expand</summary>
+
+```
+config: conf/tuning/train_enh_uses_refch0_2mem.yaml
+print_config: false
+log_level: INFO
+dry_run: false
+iterator_type: chunk
+output_dir: exp/enh_train_enh_uses_refch0_2mem_raw
+ngpu: 1
+seed: 0
+num_workers: 4
+num_att_plot: 3
+dist_backend: nccl
+dist_init_method: env://
+dist_world_size: 4
+dist_rank: 0
+local_rank: 0
+dist_master_addr: localhost
+dist_master_port: 33702
+dist_launcher: null
+multiprocessing_distributed: true
+unused_parameters: true
+sharded_ddp: false
+cudnn_enabled: true
+cudnn_benchmark: false
+cudnn_deterministic: true
+collect_stats: false
+write_collected_feats: false
+skip_stats_npz: false
+max_epoch: 150
+patience: 20
+val_scheduler_criterion:
+- valid
+- loss
+early_stopping_criterion:
+- valid
+- loss
+- min
+best_model_criterion:
+-   - valid
+    - loss
+    - min
+keep_nbest_models: 1
+nbest_averaging_interval: 0
+grad_clip: 5.0
+grad_clip_type: 2.0
+grad_noise: false
+accum_grad: 1
+no_forward_run: false
+resume: true
+train_dtype: float32
+use_amp: false
+log_interval: null
+use_matplotlib: true
+use_tensorboard: true
+create_graph_in_tensorboard: false
+use_wandb: false
+wandb_project: null
+wandb_id: null
+wandb_entity: null
+wandb_name: null
+wandb_model_log_interval: -1
+detect_anomaly: false
+pretrain_path: null
+init_param: []
+ignore_init_mismatch: false
+freeze_param: []
+num_iters_per_epoch: 8000
+batch_size: 4
+valid_batch_size: null
+batch_bins: 1000000
+valid_batch_bins: null
+train_shape_file:
+- exp/enh_stats_16k/train/speech_mix_shape
+- exp/enh_stats_16k/train/speech_ref1_shape
+- exp/enh_stats_16k/train/dereverb_ref1_shape
+valid_shape_file:
+- exp/enh_stats_16k/valid/speech_mix_shape
+- exp/enh_stats_16k/valid/speech_ref1_shape
+- exp/enh_stats_16k/valid/dereverb_ref1_shape
+batch_type: folded
+valid_batch_type: null
+fold_length:
+- 80000
+- 80000
+- 80000
+sort_in_batch: descending
+sort_batch: descending
+multiple_iterator: false
+chunk_length: 32000
+chunk_shift_ratio: 0.5
+num_cache_chunks: 1024
+chunk_excluded_key_prefixes: []
+chunk_discard_short_samples: false
+train_data_path_and_name_and_type:
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/wav.scp
+    - speech_mix
+    - sound
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/spk1.scp
+    - speech_ref1
+    - sound
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/dereverb1.scp
+    - dereverb_ref1
+    - sound
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/utt2category
+    - category
+    - text
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/utt2fs
+    - fs
+    - text_int
+valid_data_path_and_name_and_type:
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/wav.scp
+    - speech_mix
+    - sound
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/spk1.scp
+    - speech_ref1
+    - sound
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/dereverb1.scp
+    - dereverb_ref1
+    - sound
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/utt2category
+    - category
+    - text
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/utt2fs
+    - fs
+    - text_int
+allow_variable_data_keys: false
+max_cache_size: 0.0
+max_cache_fd: 32
+allow_multi_rates: true
+valid_max_cache_size: null
+exclude_weight_decay: false
+exclude_weight_decay_conf: {}
+optim: adam
+optim_conf:
+    lr: 0.0004
+    eps: 1.0e-08
+    weight_decay: 1.0e-05
+scheduler: warmupreducelronplateau
+scheduler_conf:
+    warmup_steps: 25000
+    mode: min
+    factor: 0.5
+    patience: 2
+init: null
+model_conf:
+    normalize_variance: true
+    categories:
+    - 1ch_48k
+    - 1ch_16k
+    - 1ch_16k_r
+    - 2ch_16k
+    - 2ch_16k_r
+    - 5ch_16k
+    - 8ch_16k_r
+criterions:
+-   name: mr_l1_tfd
+    conf:
+        window_sz:
+        - 256
+        - 512
+        - 768
+        - 1024
+        hop_sz: null
+        eps: 1.0e-08
+        time_domain_weight: 0.5
+        normalize_variance: true
+    wrapper: fixed_order
+    wrapper_conf:
+        weight: 1.0
+-   name: si_snr
+    conf:
+        eps: 1.0e-07
+    wrapper: fixed_order
+    wrapper_conf:
+        weight: 0.0
+speech_volume_normalize: null
+rir_scp: null
+rir_apply_prob: 1.0
+noise_scp: null
+noise_apply_prob: 1.0
+noise_db_range: '13_15'
+short_noise_thres: 0.5
+use_reverberant_ref: false
+num_spk: 1
+num_noise_type: 1
+sample_rate: 8000
+force_single_channel: false
+channel_reordering: true
+categories:
+- 1ch_48k
+- 1ch_16k
+- 1ch_16k_r
+- 2ch_16k
+- 2ch_16k_r
+- 5ch_16k
+- 8ch_16k_r
+dynamic_mixing: false
+utt2spk: null
+dynamic_mixing_gain_db: 0.0
+encoder: stft
+encoder_conf:
+    n_fft: 256
+    hop_length: 128
+    use_builtin_complex: false
+separator: uses
+separator_conf:
+    num_spk: 1
+    enc_channels: 256
+    bottleneck_size: 64
+    num_blocks: 6
+    num_spatial_blocks: 3
+    segment_size: 64
+    memory_size: 20
+    memory_types: 2
+    rnn_type: lstm
+    bidirectional: true
+    hidden_size: 128
+    att_heads: 4
+    dropout: 0.0
+    norm_type: cLN
+    activation: relu
+    ch_mode: tac
+    ch_att_dim: 256
+    eps: 1.0e-05
+    ref_channel: 0
+decoder: stft
+decoder_conf:
+    n_fft: 256
+    hop_length: 128
+mask_module: multi_mask
+mask_module_conf: {}
+preprocessor: enh
+preprocessor_conf: {}
+required:
+- output_dir
+version: '202301'
+distributed: true
+```
+
+</details>
+
+
+
+### Citing ESPnet
+
+```BibTex
+@inproceedings{watanabe2018espnet,
+  author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai},
+  title={{ESPnet}: End-to-End Speech Processing Toolkit},
+  year={2018},
+  booktitle={Proceedings of Interspeech},
+  pages={2207--2211},
+  doi={10.21437/Interspeech.2018-1456},
+  url={http://dx.doi.org/10.21437/Interspeech.2018-1456}
+}
+
+
+@inproceedings{ESPnet-SE,
+  author = {Chenda Li and Jing Shi and Wangyou Zhang and Aswin Shanmugam Subramanian and Xuankai Chang and
+  Naoyuki Kamo and Moto Hira and Tomoki Hayashi and Christoph B{"{o}}ddeker and Zhuo Chen and Shinji Watanabe},
+  title = {ESPnet-SE: End-To-End Speech Enhancement and Separation Toolkit Designed for {ASR} Integration},
+  booktitle = {{IEEE} Spoken Language Technology Workshop, {SLT} 2021, Shenzhen, China, January 19-22, 2021},
+  pages = {785--792},
+  publisher = {{IEEE}},
+  year = {2021},
+  url = {https://doi.org/10.1109/SLT48900.2021.9383615},
+  doi = {10.1109/SLT48900.2021.9383615},
+  timestamp = {Mon, 12 Apr 2021 17:08:59 +0200},
+  biburl = {https://dblp.org/rec/conf/slt/Li0ZSCKHHBC021.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+
+
+```
+
+or arXiv:
+
+```bibtex
+@misc{watanabe2018espnet,
+  title={ESPnet: End-to-End Speech Processing Toolkit},
+  author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai},
+  year={2018},
+  eprint={1804.00015},
+  archivePrefix={arXiv},
+  primaryClass={cs.CL}
+}
+```

exp/enh_train_enh_uses_refch0_2mem_raw/20epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be693ffb237fe0e44dc647d4df4f0ca8795833cdba63b914235918854402e03c
+size 12296793

exp/enh_train_enh_uses_refch0_2mem_raw/RESULTS.md

@@ -0,0 +1,25 @@
+<!-- Generated by scripts/utils/show_enh_score.sh -->
+# RESULTS
+## Environments
+- date: `Sat Jul 15 12:50:47 CST 2023`
+- python version: `3.8.16 (default, Mar  2 2023, 03:21:46)  [GCC 11.2.0]`
+- espnet version: `espnet 202301`
+- pytorch version: `pytorch 2.0.1`
+- Git hash: ``
+  - Commit date: ``
+
+
+## USES (ref_channel=0, 2 groups of memory tokens)
+
+|dataset|condition|PESQ_WB|STOI|SAR|SDR|SIR|SI_SNR|OVRL|SIG|BAK|P808_MOS|
+|---|---|---|---|---|---|---|---|---|---|---|---|
+|vctk_noisy_tt_2spk|1ch, 48kHz||93.05|10.97|10.97|0.00|8.36|3.14|3.39|4.05|3.57|
+|vctk_noisy_tt_2spk_16k|1ch, 16kHz|3.11|95.03|21.51|21.51|0.00|19.45|3.19|3.46|4.06|3.57|
+|dns20_tt_synthetic_no_reverb|1ch, 16kHz|3.23|97.77|19.63|19.63|0.00|19.72|3.32|3.56|4.10|4.04|
+|dns20_tt_synthetic_with_reverb|1ch, 16kHz|2.75|89.87|13.40|13.40|0.00|12.90|2.36|2.85|3.21|3.37|
+|chime4_et05_simu_isolated_6ch_track|5ch, 16kHz|2.95|97.82|18.30|18.30|0.00|17.24|3.22|3.47|4.07|3.75|
+|reverb_et_simu_8ch_multich|8ch, 16kHz|2.09|89.83|11.94|11.94|0.00|-10.12|2.98|3.35|3.79|3.90|
+|whamr_tt_mix_single_anechoic_max_16k|2ch, 16kHz|2.55|96.36|15.78|15.78|0.00|15.46|3.33|3.55|4.16|3.86|
+|whamr_tt_mix_single_reverb_max_16k|2ch, 16kHz|2.51|95.98|13.75|13.75|0.00|12.51|3.32|3.54|4.15|3.86|
+|chime4_et05_real_isolated_6ch_track_1ch|5ch, 16kHz|1.23|55.11|-2.34|-2.34|0.00|-30.45|3.07|3.36|3.98|3.75|
+|reverb_et_real_8ch_multich|8ch, 16kHz|1.17|75.30|4.39|4.39|0.00|1.62|3.11|3.42|3.97|3.99|

exp/enh_train_enh_uses_refch0_2mem_raw/config.yaml

@@ -0,0 +1,236 @@
+config: conf/tuning/train_enh_uses_refch0_2mem.yaml
+print_config: false
+log_level: INFO
+dry_run: false
+iterator_type: chunk
+output_dir: exp/enh_train_enh_uses_refch0_2mem_raw
+ngpu: 1
+seed: 0
+num_workers: 4
+num_att_plot: 3
+dist_backend: nccl
+dist_init_method: env://
+dist_world_size: 4
+dist_rank: 0
+local_rank: 0
+dist_master_addr: localhost
+dist_master_port: 33702
+dist_launcher: null
+multiprocessing_distributed: true
+unused_parameters: true
+sharded_ddp: false
+cudnn_enabled: true
+cudnn_benchmark: false
+cudnn_deterministic: true
+collect_stats: false
+write_collected_feats: false
+skip_stats_npz: false
+max_epoch: 150
+patience: 20
+val_scheduler_criterion:
+- valid
+- loss
+early_stopping_criterion:
+- valid
+- loss
+- min
+best_model_criterion:
+-   - valid
+    - loss
+    - min
+keep_nbest_models: 1
+nbest_averaging_interval: 0
+grad_clip: 5.0
+grad_clip_type: 2.0
+grad_noise: false
+accum_grad: 1
+no_forward_run: false
+resume: true
+train_dtype: float32
+use_amp: false
+log_interval: null
+use_matplotlib: true
+use_tensorboard: true
+create_graph_in_tensorboard: false
+use_wandb: false
+wandb_project: null
+wandb_id: null
+wandb_entity: null
+wandb_name: null
+wandb_model_log_interval: -1
+detect_anomaly: false
+pretrain_path: null
+init_param: []
+ignore_init_mismatch: false
+freeze_param: []
+num_iters_per_epoch: 8000
+batch_size: 4
+valid_batch_size: null
+batch_bins: 1000000
+valid_batch_bins: null
+train_shape_file:
+- exp/enh_stats_16k/train/speech_mix_shape
+- exp/enh_stats_16k/train/speech_ref1_shape
+- exp/enh_stats_16k/train/dereverb_ref1_shape
+valid_shape_file:
+- exp/enh_stats_16k/valid/speech_mix_shape
+- exp/enh_stats_16k/valid/speech_ref1_shape
+- exp/enh_stats_16k/valid/dereverb_ref1_shape
+batch_type: folded
+valid_batch_type: null
+fold_length:
+- 80000
+- 80000
+- 80000
+sort_in_batch: descending
+sort_batch: descending
+multiple_iterator: false
+chunk_length: 32000
+chunk_shift_ratio: 0.5
+num_cache_chunks: 1024
+chunk_excluded_key_prefixes: []
+chunk_discard_short_samples: false
+train_data_path_and_name_and_type:
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/wav.scp
+    - speech_mix
+    - sound
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/spk1.scp
+    - speech_ref1
+    - sound
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/dereverb1.scp
+    - dereverb_ref1
+    - sound
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/utt2category
+    - category
+    - text
+-   - dump/raw/train_dns20_vctk_whamr_chime4_reverb/utt2fs
+    - fs
+    - text_int
+valid_data_path_and_name_and_type:
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/wav.scp
+    - speech_mix
+    - sound
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/spk1.scp
+    - speech_ref1
+    - sound
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/dereverb1.scp
+    - dereverb_ref1
+    - sound
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/utt2category
+    - category
+    - text
+-   - dump/raw/valid_dns20_vctk_whamr_chime4/utt2fs
+    - fs
+    - text_int
+allow_variable_data_keys: false
+max_cache_size: 0.0
+max_cache_fd: 32
+allow_multi_rates: true
+valid_max_cache_size: null
+exclude_weight_decay: false
+exclude_weight_decay_conf: {}
+optim: adam
+optim_conf:
+    lr: 0.0004
+    eps: 1.0e-08
+    weight_decay: 1.0e-05
+scheduler: warmupreducelronplateau
+scheduler_conf:
+    warmup_steps: 25000
+    mode: min
+    factor: 0.5
+    patience: 2
+init: null
+model_conf:
+    normalize_variance: true
+    categories:
+    - 1ch_48k
+    - 1ch_16k
+    - 1ch_16k_r
+    - 2ch_16k
+    - 2ch_16k_r
+    - 5ch_16k
+    - 8ch_16k_r
+criterions:
+-   name: mr_l1_tfd
+    conf:
+        window_sz:
+        - 256
+        - 512
+        - 768
+        - 1024
+        hop_sz: null
+        eps: 1.0e-08
+        time_domain_weight: 0.5
+        normalize_variance: true
+    wrapper: fixed_order
+    wrapper_conf:
+        weight: 1.0
+-   name: si_snr
+    conf:
+        eps: 1.0e-07
+    wrapper: fixed_order
+    wrapper_conf:
+        weight: 0.0
+speech_volume_normalize: null
+rir_scp: null
+rir_apply_prob: 1.0
+noise_scp: null
+noise_apply_prob: 1.0
+noise_db_range: '13_15'
+short_noise_thres: 0.5
+use_reverberant_ref: false
+num_spk: 1
+num_noise_type: 1
+sample_rate: 8000
+force_single_channel: false
+channel_reordering: true
+categories:
+- 1ch_48k
+- 1ch_16k
+- 1ch_16k_r
+- 2ch_16k
+- 2ch_16k_r
+- 5ch_16k
+- 8ch_16k_r
+dynamic_mixing: false
+utt2spk: null
+dynamic_mixing_gain_db: 0.0
+encoder: stft
+encoder_conf:
+    n_fft: 256
+    hop_length: 128
+    use_builtin_complex: false
+separator: uses
+separator_conf:
+    num_spk: 1
+    enc_channels: 256
+    bottleneck_size: 64
+    num_blocks: 6
+    num_spatial_blocks: 3
+    segment_size: 64
+    memory_size: 20
+    memory_types: 2
+    rnn_type: lstm
+    bidirectional: true
+    hidden_size: 128
+    att_heads: 4
+    dropout: 0.0
+    norm_type: cLN
+    activation: relu
+    ch_mode: tac
+    ch_att_dim: 256
+    eps: 1.0e-05
+    ref_channel: 0
+decoder: stft
+decoder_conf:
+    n_fft: 256
+    hop_length: 128
+mask_module: multi_mask
+mask_module_conf: {}
+preprocessor: enh
+preprocessor_conf: {}
+required:
+- output_dir
+version: '202301'
+distributed: true

exp/enh_train_enh_uses_refch0_2mem_raw/latest.pth

@@ -0,0 +1 @@
+20epoch.pth

exp/enh_train_enh_uses_refch0_2mem_raw/valid.loss.ave.pth

@@ -0,0 +1 @@
+20epoch.pth

exp/enh_train_enh_uses_refch0_2mem_raw/valid.loss.best.pth

@@ -0,0 +1 @@
+20epoch.pth

meta.yaml

@@ -0,0 +1,8 @@
+espnet: '202301'
+files:
+  model_file: exp/enh_train_enh_uses_refch0_2mem_raw/20epoch.pth
+python: "3.8.16 (default, Mar  2 2023, 03:21:46) \n[GCC 11.2.0]"
+timestamp: 1696137339.402181
+torch: 2.0.1
+yaml_files:
+  train_config: exp/enh_train_enh_uses_refch0_2mem_raw/config.yaml