feat(demo-v1): support Chinese song with pretrained VISinger2

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.gradio

app.py

@@ -0,0 +1,185 @@
+import numpy as np
+import gradio as gr
+from pypinyin import lazy_pinyin
+
+from pinyin_dict import PINYIN_DICT
+
+from espnet_model_zoo.downloader import ModelDownloader
+from espnet2.fileio.read_text import read_label
+from espnet2.bin.svs_inference import SingingGenerate
+
+
+spks = {
+    "singer1 (man)": 1,
+    "singer2 (man)": 2,
+    "singer3 (female)": 5,
+    "singer4 (female)": 9,
+    "singer5 (man)": 18,
+    "singer6 (female)": 15,
+    "singer7 (man)": 23,
+    "singer8 (man)": 25,
+    "singer9 (female)": 29,
+    "singer10 (man)": 27,
+}
+
+def gen_song(lang, tempo, texts, durs, pitchs, spk):
+    if lang == "zh":
+        PRETRAIN_MODEL = "espnet/aceopencpop_svs_visinger2_40singer_pretrain"
+        fs = 44100
+        text_list = lazy_pinyin(texts)
+
+    # preprocess
+    if texts is None:
+        return (fs, np.array([0.0])), "Error: No Text provided!"
+    if durs is None:
+        return (fs, np.array([0.0])), "Error: No Dur provided!"
+    if pitchs is None:
+        return (fs, np.array([0.0])), "Error: No Pitch provided!"
+
+    dur_list = durs.strip().split()
+    pitch_list = pitchs.strip().split()
+
+    if len(text_list) != len(dur_list):
+        return (fs, np.array([0.0])), f"Error: len in text({len(text_list)}) mismatch with duration({len(dur_list)})!"
+    if len(text_list) != len(pitch_list):
+        return (fs, np.array([0.0])), f"Error: len in text({len(text_list)}) mismatch with pitch({len(pitch_list)})!"
+
+    ## text to phoneme
+    sybs = []
+    if lang == "zh":
+        pinyin_dict = PINYIN_DICT
+        for text in text_list:
+            text = text.lower()
+            if text not in pinyin_dict:
+                return (fs, np.array([0.0])), f"Error: pinyin `{text}` is invalid!"
+            phns = "_".join(pinyin_dict[text])
+            sybs.append(phns)
+            
+    ## pitch
+    pitch_dict = {}
+    with open("./midi-note.scp", "r", encoding="utf-8") as f:
+        for line in f:
+            items = line.strip().split()
+            pitch_dict[items[0]] = int(items[1])
+            pitch_dict[items[1]] = int(items[1])
+
+    labels = []
+    notes = []
+    st = 0
+    for phns, dur, pitch in zip(sybs, dur_list, pitch_list):
+        if pitch not in pitch_dict:
+            return (fs, np.array([0.0])), f"Error: pitch `{pitch}` is invalid!"
+        pitch = pitch_dict[pitch]
+        dur = float(dur)
+        phn_list = phns.split("_")
+        lyric = "".join(phn_list)
+        note = [st, st + dur, lyric, pitch, phns]
+        st += dur
+        notes.append(note)
+        for phn in phn_list:
+            labels.append(phn)
+
+    phns_str = " ".join(labels)
+    batch = {
+        "score": (
+            int(tempo),
+            notes,
+        ),
+        "text": phns_str,
+    }
+    
+    # Infer
+    device = "cpu"
+    # device = "cuda" if torch.cuda.is_available() else "cpu"
+    d = ModelDownloader()
+    pretrain_downloaded = d.download_and_unpack(PRETRAIN_MODEL)
+    svs = SingingGenerate(
+        train_config = pretrain_downloaded["train_config"],
+        model_file = pretrain_downloaded["model_file"],
+        device = device
+    )
+    sid = spks[spk]
+    output_dict = svs(batch, sids=np.array([sid]))
+    wav_info = output_dict["wav"].cpu().numpy()
+    return (fs, wav_info), "success!"
+
+
+title = "Demo of Singing Voice Synthesis in Muskits-ESPnet"
+
+description = """
+This is the demo page of our toolkit <b>Muskits-ESPnet: A Comprehensive Toolkit for Singing Voice Synthesis in New Paradigm<b>.
+
+
+<p>How to use:</p>
+<ol>
+  <li> Choose language ID. Language id </li>
+  <li> Input tempo in integer </li>
+  <li> Input text, duration, pitch of equal length </li>
+  <li> Choose ons singer </li>
+  <li> Click submit button </li>
+</ol>
+
+
+"""
+
+article = """
+<div style='margin:20px auto;'>
+
+<p>References: <a href="https://arxiv.org/abs/2409.07226">Muskits-ESPnet paper</a> |
+<a href="https://github.com/espnet/espnet">espnet GitHub</a> |
+<a href="https://huggingface.co/espnet/aceopencpop_svs_visinger2_40singer_pretrain">pretrained model</a></p>
+
+<pre>
+@inproceedings{wu2024muskits,
+  title = {{Muskits-ESPnet}: A Comprehensive Toolkit for Singing Voice Synthesis in New Paradigm},
+  author = {Yuning Wu and Jiatong Shi and Yifeng Yu and Yuxun Tang and Tao Qian and Yueqian Lin and Jionghao Han and Xinyi Bai and Shinji Watanabe and Qin Jin},
+  booktitle={Proc. ACM Multimedia},
+  year={2024},
+}
+</pre>
+
+</div>
+"""
+
+
+# SP: silence, AP: aspirate.
+examples = [
+    ["zh", 89, "雨淋湿了SP天空AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "60 62 62 62 0 62 58 0", "singer1 (man)"],
+    ["zh", 89, "雨淋湿了SP天空AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "C4 D4 D4 D4 rest D4 A#3 rest", "singer2 (man)"],
+    ["zh", 89, "雨淋湿了SP天空AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "C4 D4 D4 D4 rest D4 Bb3 rest", "singer3 (female)"],
+]
+
+gr.Interface(
+    fn=gen_song,
+    inputs=[
+        gr.Radio(label="language", choices=["zh"], value="zh"),
+        gr.Textbox(label="Tempo"),
+        gr.Textbox(label="Text"),
+        gr.Textbox(label="Duration"),
+        gr.Textbox(label="Pitch"),
+        gr.Radio(
+            label="Singer",
+            choices=[
+                "singer1 (man)",
+                "singer2 (man)",
+                "singer3 (female)",
+                "singer4 (female)",
+                "singer5 (man)",
+                "singer6 (female)",
+                "singer7 (man)",
+                "singer8 (man)",
+                "singer9 (female)",
+                "singer10 (man)",
+            ],
+            value="singer1 (man)"
+        ),
+    ],
+    outputs=[
+        gr.Audio(label="Generated Song", type="numpy"),
+        gr.Textbox(label="Running Status"),
+    ],
+    title=title,
+    description=description,
+    article=article,
+    examples=examples,
+).launch()

midi-note.scp

@@ -0,0 +1,152 @@
+G9 127
+F#9 126
+Gb9 126
+F9 125
+E9 124
+D#9 123
+Eb9 123
+D9 122
+C#9 121
+Db9 121
+C9 120
+B8 119
+A#8 118
+Bb8 118
+A8 117
+G#8 116
+Ab8 116
+G8 115
+F#8 114
+Gb8 114
+F8 113
+E8 112
+D#8 111
+Eb8 111
+D8 110
+C#8 109
+Db8 109
+C8 108
+B7 107
+A#7 106
+Bb7 106
+A7 105
+G#7 104
+Ab7 104
+G7 103
+F#7 102
+Gb7 102
+F7 101
+E7 100
+D#7 99
+Eb7 99
+D7 98
+C#7 97
+Db7 97
+C7 96
+B6 95
+A#6 94
+Bb6 94
+A6 93
+G#6 92
+Ab6 92
+G6 91
+F#6 90
+Gb6 90
+F6 89
+E6 88
+D#6 87
+Eb6 87
+D6 86
+C#6 85
+Db6 85
+C6 84
+B5 83
+A#5 82
+Bb5 82
+A5 81
+G#5 80
+Ab5 80
+G5 79
+F#5 78
+Gb5 78
+F5 77
+E5 76
+D#5 75
+Eb5 75
+D5 74
+C#5 73
+Db5 73
+C5 72
+B4 71
+A#4 70
+Bb4 70
+A4 69
+G#4 68
+Ab4 68
+G4 67
+F#4 66
+Gb4 66
+F4 65
+E4 64
+D#4 63
+Eb4 63
+D4 62
+C#4 61
+Db4 61
+C4 60
+B3 59
+A#3 58
+Bb3 58
+A3 57
+G#3 56
+Ab3 56
+G3 55
+F#3 54
+Gb3 54
+F3 53
+E3 52
+D#3 51
+Eb3 51
+D3 50
+C#3 49
+Db3 49
+C3 48
+B2 47
+A#2 46
+Bb2 46
+A2 45
+G#2 44
+Ab2 44
+G2 43
+F#2 42
+Gb2 42
+F2 41
+E2 40
+D#2 39
+Eb2 39
+D2 38
+C#2 37
+Db2 37
+C2 36
+B1 35
+A#1 34
+Bb1 34
+A1 33
+G#1 32
+Ab1 32
+G1 31
+F#1 30
+Gb1 30
+F1 29
+E1 28
+D#1 27
+Eb1 27
+D1 26
+C#1 25
+Db1 25
+C1 24
+B0 23
+A#0 22
+Bb0 22
+A0 21
+rest 0

pinyin_dict.py

@@ -0,0 +1,425 @@
+# Adapted from Opencpop's pinyin to phoneme mapping table:
+# https://wenet.org.cn/opencpop/resources/annotationformat/
+PINYIN_DICT = {
+    "a": ("a",),
+    "ai": ("ai",),
+    "an": ("an",),
+    "ang": ("ang",),
+    "ao": ("ao",),
+    "ba": ("b", "a"),
+    "bai": ("b", "ai"),
+    "ban": ("b", "an"),
+    "bang": ("b", "ang"),
+    "bao": ("b", "ao"),
+    "bei": ("b", "ei"),
+    "ben": ("b", "en"),
+    "beng": ("b", "eng"),
+    "bi": ("b", "i"),
+    "bian": ("b", "ian"),
+    "biao": ("b", "iao"),
+    "bie": ("b", "ie"),
+    "bin": ("b", "in"),
+    "bing": ("b", "ing"),
+    "bo": ("b", "o"),
+    "bu": ("b", "u"),
+    "ca": ("c", "a"),
+    "cai": ("c", "ai"),
+    "can": ("c", "an"),
+    "cang": ("c", "ang"),
+    "cao": ("c", "ao"),
+    "ce": ("c", "e"),
+    "cei": ("c", "ei"),
+    "cen": ("c", "en"),
+    "ceng": ("c", "eng"),
+    "cha": ("ch", "a"),
+    "chai": ("ch", "ai"),
+    "chan": ("ch", "an"),
+    "chang": ("ch", "ang"),
+    "chao": ("ch", "ao"),
+    "che": ("ch", "e"),
+    "chen": ("ch", "en"),
+    "cheng": ("ch", "eng"),
+    "chi": ("ch", "i"),
+    "chong": ("ch", "ong"),
+    "chou": ("ch", "ou"),
+    "chu": ("ch", "u"),
+    "chua": ("ch", "ua"),
+    "chuai": ("ch", "uai"),
+    "chuan": ("ch", "uan"),
+    "chuang": ("ch", "uang"),
+    "chui": ("ch", "ui"),
+    "chun": ("ch", "un"),
+    "chuo": ("ch", "uo"),
+    "ci": ("c", "i"),
+    "cong": ("c", "ong"),
+    "cou": ("c", "ou"),
+    "cu": ("c", "u"),
+    "cuan": ("c", "uan"),
+    "cui": ("c", "ui"),
+    "cun": ("c", "un"),
+    "cuo": ("c", "uo"),
+    "da": ("d", "a"),
+    "dai": ("d", "ai"),
+    "dan": ("d", "an"),
+    "dang": ("d", "ang"),
+    "dao": ("d", "ao"),
+    "de": ("d", "e"),
+    "dei": ("d", "ei"),
+    "den": ("d", "en"),
+    "deng": ("d", "eng"),
+    "di": ("d", "i"),
+    "dia": ("d", "ia"),
+    "dian": ("d", "ian"),
+    "diao": ("d", "iao"),
+    "die": ("d", "ie"),
+    "ding": ("d", "ing"),
+    "diu": ("d", "iu"),
+    "dong": ("d", "ong"),
+    "dou": ("d", "ou"),
+    "du": ("d", "u"),
+    "duan": ("d", "uan"),
+    "dui": ("d", "ui"),
+    "dun": ("d", "un"),
+    "duo": ("d", "uo"),
+    "e": ("e",),
+    "ei": ("ei",),
+    "en": ("en",),
+    "eng": ("eng",),
+    "er": ("er",),
+    "fa": ("f", "a"),
+    "fan": ("f", "an"),
+    "fang": ("f", "ang"),
+    "fei": ("f", "ei"),
+    "fen": ("f", "en"),
+    "feng": ("f", "eng"),
+    "fo": ("f", "o"),
+    "fou": ("f", "ou"),
+    "fu": ("f", "u"),
+    "ga": ("g", "a"),
+    "gai": ("g", "ai"),
+    "gan": ("g", "an"),
+    "gang": ("g", "ang"),
+    "gao": ("g", "ao"),
+    "ge": ("g", "e"),
+    "gei": ("g", "ei"),
+    "gen": ("g", "en"),
+    "geng": ("g", "eng"),
+    "gong": ("g", "ong"),
+    "gou": ("g", "ou"),
+    "gu": ("g", "u"),
+    "gua": ("g", "ua"),
+    "guai": ("g", "uai"),
+    "guan": ("g", "uan"),
+    "guang": ("g", "uang"),
+    "gui": ("g", "ui"),
+    "gun": ("g", "un"),
+    "guo": ("g", "uo"),
+    "ha": ("h", "a"),
+    "hai": ("h", "ai"),
+    "han": ("h", "an"),
+    "hang": ("h", "ang"),
+    "hao": ("h", "ao"),
+    "he": ("h", "e"),
+    "hei": ("h", "ei"),
+    "hen": ("h", "en"),
+    "heng": ("h", "eng"),
+    "hm": ("h", "m"),
+    "hng": ("h", "ng"),
+    "hong": ("h", "ong"),
+    "hou": ("h", "ou"),
+    "hu": ("h", "u"),
+    "hua": ("h", "ua"),
+    "huai": ("h", "uai"),
+    "huan": ("h", "uan"),
+    "huang": ("h", "uang"),
+    "hui": ("h", "ui"),
+    "hun": ("h", "un"),
+    "huo": ("h", "uo"),
+    "ji": ("j", "i"),
+    "jia": ("j", "ia"),
+    "jian": ("j", "ian"),
+    "jiang": ("j", "iang"),
+    "jiao": ("j", "iao"),
+    "jie": ("j", "ie"),
+    "jin": ("j", "in"),
+    "jing": ("j", "ing"),
+    "jiong": ("j", "iong"),
+    "jiu": ("j", "iu"),
+    "ju": ("j", "v"),
+    "juan": ("j", "van"),
+    "jue": ("j", "ve"),
+    "jun": ("j", "vn"),
+    "ka": ("k", "a"),
+    "kai": ("k", "ai"),
+    "kan": ("k", "an"),
+    "kang": ("k", "ang"),
+    "kao": ("k", "ao"),
+    "ke": ("k", "e"),
+    "kei": ("k", "ei"),
+    "ken": ("k", "en"),
+    "keng": ("k", "eng"),
+    "kong": ("k", "ong"),
+    "kou": ("k", "ou"),
+    "ku": ("k", "u"),
+    "kua": ("k", "ua"),
+    "kuai": ("k", "uai"),
+    "kuan": ("k", "uan"),
+    "kuang": ("k", "uang"),
+    "kui": ("k", "ui"),
+    "kun": ("k", "un"),
+    "kuo": ("k", "uo"),
+    "la": ("l", "a"),
+    "lai": ("l", "ai"),
+    "lan": ("l", "an"),
+    "lang": ("l", "ang"),
+    "lao": ("l", "ao"),
+    "le": ("l", "e"),
+    "lei": ("l", "ei"),
+    "leng": ("l", "eng"),
+    "li": ("l", "i"),
+    "lia": ("l", "ia"),
+    "lian": ("l", "ian"),
+    "liang": ("l", "iang"),
+    "liao": ("l", "iao"),
+    "lie": ("l", "ie"),
+    "lin": ("l", "in"),
+    "ling": ("l", "ing"),
+    "liu": ("l", "iu"),
+    "lo": ("l", "o"),
+    "long": ("l", "ong"),
+    "lou": ("l", "ou"),
+    "lu": ("l", "u"),
+    "luan": ("l", "uan"),
+    "lun": ("l", "un"),
+    "luo": ("l", "uo"),
+    "lv": ("l", "v"),
+    "lve": ("l", "ve"),
+    "m": ("m",),
+    "ma": ("m", "a"),
+    "mai": ("m", "ai"),
+    "man": ("m", "an"),
+    "mang": ("m", "ang"),
+    "mao": ("m", "ao"),
+    "me": ("m", "e"),
+    "mei": ("m", "ei"),
+    "men": ("m", "en"),
+    "meng": ("m", "eng"),
+    "mi": ("m", "i"),
+    "mian": ("m", "ian"),
+    "miao": ("m", "iao"),
+    "mie": ("m", "ie"),
+    "min": ("m", "in"),
+    "ming": ("m", "ing"),
+    "miu": ("m", "iu"),
+    "mo": ("m", "o"),
+    "mou": ("m", "ou"),
+    "mu": ("m", "u"),
+    "n": ("n",),
+    "na": ("n", "a"),
+    "nai": ("n", "ai"),
+    "nan": ("n", "an"),
+    "nang": ("n", "ang"),
+    "nao": ("n", "ao"),
+    "ne": ("n", "e"),
+    "nei": ("n", "ei"),
+    "nen": ("n", "en"),
+    "neng": ("n", "eng"),
+    "ng": ("n", "g"),
+    "ni": ("n", "i"),
+    "nian": ("n", "ian"),
+    "niang": ("n", "iang"),
+    "niao": ("n", "iao"),
+    "nie": ("n", "ie"),
+    "nin": ("n", "in"),
+    "ning": ("n", "ing"),
+    "niu": ("n", "iu"),
+    "nong": ("n", "ong"),
+    "nou": ("n", "ou"),
+    "nu": ("n", "u"),
+    "nuan": ("n", "uan"),
+    "nun": ("n", "un"),
+    "nuo": ("n", "uo"),
+    "nv": ("n", "v"),
+    "nve": ("n", "ve"),
+    "o": ("o",),
+    "ou": ("ou",),
+    "pa": ("p", "a"),
+    "pai": ("p", "ai"),
+    "pan": ("p", "an"),
+    "pang": ("p", "ang"),
+    "pao": ("p", "ao"),
+    "pei": ("p", "ei"),
+    "pen": ("p", "en"),
+    "peng": ("p", "eng"),
+    "pi": ("p", "i"),
+    "pian": ("p", "ian"),
+    "piao": ("p", "iao"),
+    "pie": ("p", "ie"),
+    "pin": ("p", "in"),
+    "ping": ("p", "ing"),
+    "po": ("p", "o"),
+    "pou": ("p", "ou"),
+    "pu": ("p", "u"),
+    "qi": ("q", "i"),
+    "qia": ("q", "ia"),
+    "qian": ("q", "ian"),
+    "qiang": ("q", "iang"),
+    "qiao": ("q", "iao"),
+    "qie": ("q", "ie"),
+    "qin": ("q", "in"),
+    "qing": ("q", "ing"),
+    "qiong": ("q", "iong"),
+    "qiu": ("q", "iu"),
+    "qu": ("q", "v"),
+    "quan": ("q", "van"),
+    "que": ("q", "ve"),
+    "qun": ("q", "vn"),
+    "ran": ("r", "an"),
+    "rang": ("r", "ang"),
+    "rao": ("r", "ao"),
+    "re": ("r", "e"),
+    "ren": ("r", "en"),
+    "reng": ("r", "eng"),
+    "ri": ("r", "i"),
+    "rong": ("r", "ong"),
+    "rou": ("r", "ou"),
+    "ru": ("r", "u"),
+    "rua": ("r", "ua"),
+    "ruan": ("r", "uan"),
+    "rui": ("r", "ui"),
+    "run": ("r", "un"),
+    "ruo": ("r", "uo"),
+    "sa": ("s", "a"),
+    "sai": ("s", "ai"),
+    "san": ("s", "an"),
+    "sang": ("s", "ang"),
+    "sao": ("s", "ao"),
+    "se": ("s", "e"),
+    "sen": ("s", "en"),
+    "seng": ("s", "eng"),
+    "sha": ("sh", "a"),
+    "shai": ("sh", "ai"),
+    "shan": ("sh", "an"),
+    "shang": ("sh", "ang"),
+    "shao": ("sh", "ao"),
+    "she": ("sh", "e"),
+    "shei": ("sh", "ei"),
+    "shen": ("sh", "en"),
+    "sheng": ("sh", "eng"),
+    "shi": ("sh", "i"),
+    "shou": ("sh", "ou"),
+    "shu": ("sh", "u"),
+    "shua": ("sh", "ua"),
+    "shuai": ("sh", "uai"),
+    "shuan": ("sh", "uan"),
+    "shuang": ("sh", "uang"),
+    "shui": ("sh", "ui"),
+    "shun": ("sh", "un"),
+    "shuo": ("sh", "uo"),
+    "si": ("s", "i"),
+    "song": ("s", "ong"),
+    "sou": ("s", "ou"),
+    "su": ("s", "u"),
+    "suan": ("s", "uan"),
+    "sui": ("s", "ui"),
+    "sun": ("s", "un"),
+    "suo": ("s", "uo"),
+    "ta": ("t", "a"),
+    "tai": ("t", "ai"),
+    "tan": ("t", "an"),
+    "tang": ("t", "ang"),
+    "tao": ("t", "ao"),
+    "te": ("t", "e"),
+    "tei": ("t", "ei"),
+    "teng": ("t", "eng"),
+    "ti": ("t", "i"),
+    "tian": ("t", "ian"),
+    "tiao": ("t", "iao"),
+    "tie": ("t", "ie"),
+    "ting": ("t", "ing"),
+    "tong": ("t", "ong"),
+    "tou": ("t", "ou"),
+    "tu": ("t", "u"),
+    "tuan": ("t", "uan"),
+    "tui": ("t", "ui"),
+    "tun": ("t", "un"),
+    "tuo": ("t", "uo"),
+    "wa": ("w", "a"),
+    "wai": ("w", "ai"),
+    "wan": ("w", "an"),
+    "wang": ("w", "ang"),
+    "wei": ("w", "ei"),
+    "wen": ("w", "en"),
+    "weng": ("w", "eng"),
+    "wo": ("w", "o"),
+    "wu": ("w", "u"),
+    "xi": ("x", "i"),
+    "xia": ("x", "ia"),
+    "xian": ("x", "ian"),
+    "xiang": ("x", "iang"),
+    "xiao": ("x", "iao"),
+    "xie": ("x", "ie"),
+    "xin": ("x", "in"),
+    "xing": ("x", "ing"),
+    "xiong": ("x", "iong"),
+    "xiu": ("x", "iu"),
+    "xu": ("x", "v"),
+    "xuan": ("x", "van"),
+    "xue": ("x", "ve"),
+    "xun": ("x", "vn"),
+    "ya": ("y", "a"),
+    "yan": ("y", "an"),
+    "yang": ("y", "ang"),
+    "yao": ("y", "ao"),
+    "ye": ("y", "e"),
+    "yi": ("y", "i"),
+    "yin": ("y", "in"),
+    "ying": ("y", "ing"),
+    "yo": ("y", "o"),
+    "yong": ("y", "ong"),
+    "you": ("y", "ou"),
+    "yu": ("y", "v"),
+    "yuan": ("y", "van"),
+    "yue": ("y", "ve"),
+    "yun": ("y", "vn"),
+    "za": ("z", "a"),
+    "zai": ("z", "ai"),
+    "zan": ("z", "an"),
+    "zang": ("z", "ang"),
+    "zao": ("z", "ao"),
+    "ze": ("z", "e"),
+    "zei": ("z", "ei"),
+    "zen": ("z", "en"),
+    "zeng": ("z", "eng"),
+    "zha": ("zh", "a"),
+    "zhai": ("zh", "ai"),
+    "zhan": ("zh", "an"),
+    "zhang": ("zh", "ang"),
+    "zhao": ("zh", "ao"),
+    "zhe": ("zh", "e"),
+    "zhei": ("zh", "ei"),
+    "zhen": ("zh", "en"),
+    "zheng": ("zh", "eng"),
+    "zhi": ("zh", "i"),
+    "zhong": ("zh", "ong"),
+    "zhou": ("zh", "ou"),
+    "zhu": ("zh", "u"),
+    "zhua": ("zh", "ua"),
+    "zhuai": ("zh", "uai"),
+    "zhuan": ("zh", "uan"),
+    "zhuang": ("zh", "uang"),
+    "zhui": ("zh", "ui"),
+    "zhun": ("zh", "un"),
+    "zhuo": ("zh", "uo"),
+    "zi": ("z", "i"),
+    "zong": ("z", "ong"),
+    "zou": ("z", "ou"),
+    "zu": ("z", "u"),
+    "zuan": ("z", "uan"),
+    "zui": ("z", "ui"),
+    "zun": ("z", "un"),
+    "zuo": ("z", "uo"),
+    "sp": ("SP",),
+    "ap": ("AP",),
+}
+

requirements.txt

@@ -0,0 +1,8 @@
+git+https://github.com/South-Twilight/espnet
+torch
+numpy
+librosa
+espnet_model_zoo
+importlib
+pathlib
+pypinyin

util.py

@@ -0,0 +1,5 @@
+
+def split_pinyin(pinyin: str, pinyin_dict: dict) -> tuple[str]:
+    # load pinyin dict from local/pinyin.dict
+    pinyin = pinyin.lower()
+    return pinyin_dict[pinyin]