test adding photo and add speaker embedding:

1) test the function to add new logo
2) add pretrained speaker embedding file

app.py

@@ -1,34 +1,31 @@
+import os
 import numpy as np
 import gradio as gr
+import pyopenjtalk
 from pypinyin import lazy_pinyin
-
-from pinyin_dict import PINYIN_DICT
+from util import preprocess_input, get_tokenizer, load_pitch_dict
 
 from espnet_model_zoo.downloader import ModelDownloader
 from espnet2.fileio.read_text import read_label
 from espnet2.bin.svs_inference import SingingGenerate
 
 
-spks = {
-    "singer1 (man)": 1,
-    "singer2 (man)": 2,
-    "singer3 (female)": 5,
-    "singer4 (female)": 9,
-    "singer5 (man)": 18,
-    "singer6 (female)": 15,
-    "singer7 (man)": 23,
-    "singer8 (man)": 25,
-    "singer9 (female)": 29,
-    "singer10 (man)": 27,
+singer_embeddings = {
+    "singer1 (female)": "resource/singer/singer_embedding_opencpop.npy",
 }
 
-def gen_song(lang, tempo, texts, durs, pitchs, spk):
-    if lang == "zh":
-        PRETRAIN_MODEL = "espnet/aceopencpop_svs_visinger2_40singer_pretrain"
-        fs = 44100
-        text_list = lazy_pinyin(texts)
+langs = {
+    "zh": 2,
+    "jp": 1,
+}
 
-    # preprocess
+def gen_song(lang, tempo, texts, durs, pitchs, spk):
+    fs = 44100
+    # PRETRAIN_MODEL = "espnet/aceopencpop_svs_visinger2_40singer_pretrain"
+    # pretrain_downloaded = {
+    #     "train_config": "/data7/tyx/espnet/egs2/mixed/svs1/exp/svs_train_visinger2_spk_embed_lang_raw_phn_None_mix_all/config.yaml",
+    #     "model_file": "/data7/tyx/espnet/egs2/mixed/svs1/exp/svs_train_visinger2_spk_embed_lang_raw_phn_None_mix_all/500epoch.pth",
+    # }
     if texts is None:
         return (fs, np.array([0.0])), "Error: No Text provided!"
     if durs is None:
@@ -36,7 +33,16 @@ def gen_song(lang, tempo, texts, durs, pitchs, spk):
     if pitchs is None:
         return (fs, np.array([0.0])), "Error: No Pitch provided!"
 
+    # preprocess
+    if lang == "zh":
+        texts = preprocess_input(texts, "")
+        text_list = lazy_pinyin(texts)
+    elif lang == "jp":
+        texts = preprocess_input(texts, " ")
+        text_list = texts.strip().split()
+    durs = preprocess_input(durs, " ")
     dur_list = durs.strip().split()
+    pitchs = preprocess_input(pitchs, " ")
     pitch_list = pitchs.strip().split()
 
     if len(text_list) != len(dur_list):
@@ -45,23 +51,20 @@ def gen_song(lang, tempo, texts, durs, pitchs, spk):
         return (fs, np.array([0.0])), f"Error: len in text({len(text_list)}) mismatch with pitch({len(pitch_list)})!"
 
     ## text to phoneme
+    tokenizer = get_tokenizer(lang)
     sybs = []
-    if lang == "zh":
-        pinyin_dict = PINYIN_DICT
-        for text in text_list:
-            text = text.lower()
-            if text not in pinyin_dict:
-                return (fs, np.array([0.0])), f"Error: pinyin `{text}` is invalid!"
-            phns = "_".join(pinyin_dict[text])
-            sybs.append(phns)
-            
-    ## pitch
-    pitch_dict = {}
-    with open("./midi-note.scp", "r", encoding="utf-8") as f:
-        for line in f:
-            items = line.strip().split()
-            pitch_dict[items[0]] = int(items[1])
-            pitch_dict[items[1]] = int(items[1])
+    for text in text_list:
+        if text == "AP" or text == "SP":
+            rev = [text]
+        else:
+            rev = tokenizer(text)
+            rev = [phn + f"@{lang}" for phn in rev]
+        if rev == False:
+            return (fs, np.array([0.0])), f"Error: text `{text}` is invalid!"
+        phns = "_".join(rev)
+        sybs.append(phns)
+
+    pitch_dict = load_pitch_dict()
 
     labels = []
     notes = []
@@ -87,19 +90,24 @@ def gen_song(lang, tempo, texts, durs, pitchs, spk):
         ),
         "text": phns_str,
     }
-    
+    print(batch)
+    return (fs, np.array([0.0])), "success!"
+
     # Infer
     device = "cpu"
     # device = "cuda" if torch.cuda.is_available() else "cpu"
-    d = ModelDownloader()
-    pretrain_downloaded = d.download_and_unpack(PRETRAIN_MODEL)
+    # d = ModelDownloader()
+    # pretrain_downloaded = d.download_and_unpack(PRETRAIN_MODEL)
     svs = SingingGenerate(
         train_config = pretrain_downloaded["train_config"],
         model_file = pretrain_downloaded["model_file"],
         device = device
     )
-    sid = spks[spk]
-    output_dict = svs(batch, sids=np.array([sid]))
+    # sid = spks[spk]
+    lid = langs[lang]
+    spk_embed = np.load(singer_embeddings[spk])
+    # output_dict = svs(batch, sids=np.array([sid]))
+    output_dict = svs(batch, lids=np.array([lid]), spembs=spk_embed)
     wav_info = output_dict["wav"].cpu().numpy()
     return (fs, wav_info), "success!"
 
@@ -107,19 +115,24 @@ def gen_song(lang, tempo, texts, durs, pitchs, spk):
 title = "Demo of Singing Voice Synthesis in Muskits-ESPnet"
 
 description = """
-This is the demo page of our toolkit <b>Muskits-ESPnet: A Comprehensive Toolkit for Singing Voice Synthesis in New Paradigm<b>.
-
-
-<p>How to use:</p>
-<ol>
-  <li> Choose language ID. Language id </li>
-  <li> Input tempo in integer </li>
-  <li> Input text, duration, pitch of equal length </li>
-  <li> Choose ons singer </li>
-  <li> Click submit button </li>
-</ol>
-
-
+<div style="font-size: 20px;">
+  <p>This is the demo page of our toolkit <b>Muskits-ESPnet: A Comprehensive Toolkit for Singing Voice Synthesis in New Paradigm</b>.</p>
+  <p>Singing Voice Synthesis (SVS) takes a music score as input and generates singing vocal with the voice of a specific singer.
+  Music score contains information about tempo (singing speed), lyrics, as well as duration and pitch of each word in lyrics.</p>
+
+  <img src="resource/Muskits_ESPnet_logo.png" alt="Muskits-ESPnet logo" style="width: 300px; height: auto;" />
+
+  <p>How to use:</p>
+  <ol>
+    <li> <b>Choose language ID</b>: "zh" indicates lyrics input in Chinese, and "jp" indicates lyrics input in Japanese. </li>
+    <li> <b>Input tempo</b>: tempo is a positive integer, typically ranging between 60 and 180, representing the number of beats per minute. </li>
+    <li> <b>Input lyrics, duration, pitch in equal length</b>: lyrics should match the language setting. Each word of input should be split with " " (blankspace) or "\\n" (newline) without quotes. Examples are provided.</li>
+    <li> <b>Choose one singer</b> </li>
+    <li> <b>Click submit button</b> </li>
+  </ol>
+
+  <b>Notice</b>: Values outside this range may result in suboptimal generation quality!
+</div>
 """
 
 article = """
@@ -133,7 +146,7 @@ article = """
 @inproceedings{wu2024muskits,
   title = {{Muskits-ESPnet}: A Comprehensive Toolkit for Singing Voice Synthesis in New Paradigm},
   author = {Yuning Wu and Jiatong Shi and Yifeng Yu and Yuxun Tang and Tao Qian and Yueqian Lin and Jionghao Han and Xinyi Bai and Shinji Watanabe and Qin Jin},
-  booktitle={Proc. ACM Multimedia},
+  booktitle={Proceedings of the 32st ACM International Conference on Multimedia},
   year={2024},
 }
 </pre>
@@ -144,15 +157,16 @@ article = """
 
 # SP: silence, AP: aspirate.
 examples = [
-    ["zh", 89, "雨淋湿了SP天空AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "60 62 62 62 0 62 58 0", "singer1 (man)"],
-    ["zh", 89, "雨淋湿了SP天空AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "C4 D4 D4 D4 rest D4 A#3 rest", "singer2 (man)"],
-    ["zh", 89, "雨淋湿了SP天空AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "C4 D4 D4 D4 rest D4 Bb3 rest", "singer3 (female)"],
+    ["zh", 89, "雨 淋 湿 了 SP 天 空 AP\n毁 的 SP 很 讲 究 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.5 0.21 0.3 0.12 0.12 0.25 0.5 0.48 0.34", "60 62 62 62 0 62 58 0 58 58 0 58 58 63 0", "singer1 (female)"],
+    # ["zh", 89, "雨 淋 湿 了 SP 天 空 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "C4 D4 D4 D4 rest D4 A#3 rest", "singer2 (male)"],
+    # ["zh", 89, "雨 淋 湿 了 SP 天 空 AP", "0.23 0.16 0.36 0.16 0.07 0.28 0.50 0.21", "C4 D4 D4 D4 rest D4 Bb3 rest", "singer3 (female)"],
+    ["jp", 152, "きっ と と べ ば そ ら ま で と ど く AP", "0.39 2.76 0.2 0.2 0.39 0.39 0.2 0.2 0.39 0.2 0.2 0.59 1.08", "64 71 68 69 71 71 69 68 66 68 69 68 0", "singer1 (female)"],
 ]
 
-gr.Interface(
+app = gr.Interface(
     fn=gen_song,
     inputs=[
-        gr.Radio(label="language", choices=["zh"], value="zh"),
+        gr.Radio(label="language", choices=["zh", "jp"], value="zh"),
         gr.Textbox(label="Tempo"),
         gr.Textbox(label="Text"),
         gr.Textbox(label="Duration"),
@@ -160,18 +174,9 @@ gr.Interface(
         gr.Radio(
             label="Singer",
             choices=[
-                "singer1 (man)",
-                "singer2 (man)",
-                "singer3 (female)",
-                "singer4 (female)",
-                "singer5 (man)",
-                "singer6 (female)",
-                "singer7 (man)",
-                "singer8 (man)",
-                "singer9 (female)",
-                "singer10 (man)",
+                "singer1 (female)",
             ],
-            value="singer1 (man)"
+            value="singer1 (female)"
         ),
     ],
     outputs=[
@@ -182,4 +187,6 @@ gr.Interface(
     description=description,
     article=article,
     examples=examples,
-).launch()
+)
+
+app.launch()

pinyin_dict.py → resource/pinyin_dict.py

@@ -419,7 +419,5 @@ PINYIN_DICT = {
     "zui": ("z", "ui"),
     "zun": ("z", "un"),
     "zuo": ("z", "uo"),
-    "sp": ("SP",),
-    "ap": ("AP",),
 }
 

resource/singer/singer_embedding_ace-1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac07d73eb172609c87329b35af91f0d27c1c20c30dcd90f23f1d4eb8c6450ca6
+size 896

resource/singer/singer_embedding_ace-10.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6b4da3cb369b0d4f2bcdb76986e6fbfb3ebba727a8e5a899227d6b70f73769d
+size 896

resource/singer/singer_embedding_ace-11.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf65222a416d9786feea3ac91672683727973aa7f7a89b533117a2bdf5dd706b
+size 896

resource/singer/singer_embedding_ace-12.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f07e7da7629c222318beaafee2b43f913be6f5137842f7d771225bb76268c7f
+size 896

resource/singer/singer_embedding_ace-13.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edfc4aba22a1adb4680a70ab872d2522a12587180d91176d64b03bdb4f181dfd
+size 896

resource/singer/singer_embedding_ace-14.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f43b58502afc550af2eff7c358e370b72433163b319b97b4c3b3d3eff9557ce
+size 896

resource/singer/singer_embedding_ace-15.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee8771b428b1f2bd5f680d54287eff436b3a9b84c6dd15797502942fe22ad18b
+size 896

resource/singer/singer_embedding_ace-16.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edea4a167679d619d66fc972a273b310290f9e5eac75a7857e81a7f25286b67b
+size 896

resource/singer/singer_embedding_ace-17.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bffeb7f3364168310939320f885d3ccf1502915365873050602eac7b557b12f
+size 896

resource/singer/singer_embedding_ace-18.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e3ffe2c7f0e583f97d58c13bc7318e64f1a33c0549e171f01d90c1fc0659a10
+size 896

resource/singer/singer_embedding_ace-19.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32d8abcf1ce64bdbe38c959a43103da2f08b7e0399de1de44ab0ba64caf63deb
+size 896

resource/singer/singer_embedding_ace-2.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11410fa00026e13862d6636e77e3f8932b6f1986ceec3039dc5ebfcfb865cd8e
+size 896

resource/singer/singer_embedding_ace-20.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:911d7f112b52b8644742d18c97346a5d10731dc2f001c3523297e01641e22b21
+size 896

resource/singer/singer_embedding_ace-21.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c62c1b06d0448bdc3709d994bb54593a41d27356f1cabd6ec5818cf523ee66f
+size 896

resource/singer/singer_embedding_ace-22.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db4b2821e3a955da34a3369f36f7eac366fd7c7e742f43676d753ca5f60ba7fa
+size 896

resource/singer/singer_embedding_ace-23.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9e98c6885c860bd9393c6829a430c9956dd6d0a7e286dc0e19e58a8cdb26ddc
+size 896

resource/singer/singer_embedding_ace-24.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5753ebb8bed09c7d03565166282c38def512c9a686f8cc6ce672b0ad71d7fd7c
+size 896

resource/singer/singer_embedding_ace-25.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfdeb67333e1c316bc50ef9ce05cd4e894f9ad3af307a5c41e5cf2314a021fc3
+size 896

resource/singer/singer_embedding_ace-26.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea4a02e11fa0287f247f057396e70bde203883c00f0fbb266bc28ecff44a1533
+size 896

resource/singer/singer_embedding_ace-27.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eecbf3c017c1b6cffbc5ae393a6f78544154ab1e88f714904f56889a1065a74f
+size 896

resource/singer/singer_embedding_ace-28.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f23bf949e26a7486aa0d8905b79bd80af6ab77afa1f7f5e6c8d6ac074558ff
+size 896

resource/singer/singer_embedding_ace-29.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b6807858aff6652a354e2945ca36de2cce5f766d09e0a14efcca6cc5576a2bd
+size 896

resource/singer/singer_embedding_ace-3.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab0dd74df2d47ee5de50eab7adc0a771b52e1cb10ebabdf47620001027a6f113
+size 896

resource/singer/singer_embedding_ace-30.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d0f99a41d7b459e5e96bea65ac793d68d62e92c75c42e0bfb0e04d8c0313ad3
+size 896

resource/singer/singer_embedding_ace-4.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c0361d54e0df795e1d60b8d10d32e459aadcd0a867ba1888c36a9c6f09b2eb2
+size 896

resource/singer/singer_embedding_ace-5.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50a9b21470c213422954ce1ce3f1cc8d5c295862896acc9fb772fa5c54f8bb57
+size 896

resource/singer/singer_embedding_ace-6.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3a7de10b1f824cc6ed4f4c203a434fa0990bc6cf58e9e1c73c3103b92041a0c
+size 896

resource/singer/singer_embedding_ace-7.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce7059ef8ae835777f53e3c89af722773f0e5792eeced0a4c131c1aeeeb342f1
+size 896

resource/singer/singer_embedding_ace-8.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b27ddeda52d4e2298d95c77ae85d22d35ac83393f3b00417d21041be38ebacf3
+size 896

resource/singer/singer_embedding_ace-9.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17547452ee7b536dbf3ccda567cd0230bf53f103c27d66687a1ca8959782b45c
+size 896

resource/singer/singer_embedding_ameboshi.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58f65d89d76060bb5406cdde4001bde87c29b0ef3fefd3a2072ab937a379cc9c
+size 896

resource/singer/singer_embedding_itako.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c94e137f090bb8376114a3aadf42423db4e59d0236a46049f8cee9c387fd5fd6
+size 896

resource/singer/singer_embedding_kiritan.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68583154587392d29729c270bc82778c203bbfe2f2101f5ec5bc68c8aa40bd22
+size 896

resource/singer/singer_embedding_kising_barber.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5643c312089790dc95d97e4a8b24f152d7e4caa703170143f1c1f9ecb0179564
+size 896

resource/singer/singer_embedding_kising_blanca.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fec31eacdb733f13185daf155f75d73f0fdcf1eda1cce92d02b93d02878da3e0
+size 896

resource/singer/singer_embedding_kising_changge.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0ad51d26b771f2460b146a8984bab73f929e943ce15932c287f6252a241026a
+size 896

resource/singer/singer_embedding_kising_chuci.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:764e6424b167dd4ff798f7c608985de31e6dadea68ee578b5dd66eba938cc835
+size 896

resource/singer/singer_embedding_kising_chuming.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4d1847080412e765208bdc09db64cf1e117e998e410e3afe6c061171afcab92
+size 896

resource/singer/singer_embedding_kising_crimson.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c6a67b36899aa0f4c62144fa63fc719e2448618fc730a0073ad66b16ee59593
+size 896

resource/singer/singer_embedding_kising_david.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eef8cd6416850059b5e8fc7c11a8fc2fb2165ea59177623133e33a30497a41b0
+size 896

resource/singer/singer_embedding_kising_dvaid.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ae99f524321f6836c71e83115308ba1117f7235b352f8c2135489bcd54dde5f
+size 896

resource/singer/singer_embedding_kising_ghost.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed8b0f26a4bf60b5dea57c2bad28c1757052d04fcb131b64515da95cd2addcfa
+size 896

resource/singer/singer_embedding_kising_growl.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca44c134bf0fcbc808743e6da1e441782ffd4043e5aeea29e3dd8fc74cfe37f1
+size 896

resource/singer/singer_embedding_kising_hiragi-yuki.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fbb4a7243b36a2c5ebdcd2d6b42b1280b552a159303c976f4c9d28fe3c0a431
+size 896

resource/singer/singer_embedding_kising_huolian.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db470188fcc158e1a410fd7854884b51e142070adbce3f8be09a3055bf575886
+size 896