import os import gradio as gr import numpy as np import spaces import torch import TTS from huggingface_hub import snapshot_download from omegaconf import OmegaConf from TTS.utils.synthesizer import Synthesizer from ipa.ipa import get_ipa, parse_ipa from replace.tts import ChangedVitsConfig TTS.tts.configs.vits_config.VitsConfig = ChangedVitsConfig def load_model(model_id): model_dir = snapshot_download(model_id) config_file_path = os.path.join(model_dir, "config.json") model_ckpt_path = os.path.join(model_dir, "model.pth") speaker_file_path = os.path.join(model_dir, "speakers.pth") language_file_path = os.path.join(model_dir, "language_ids.json") speaker_embedding_file_path = os.path.join(model_dir, "speaker_embs.pth") temp_config_path = "temp_config.json" with open(config_file_path, "r") as f: content = f.read() content = content.replace("speakers.pth", speaker_file_path) content = content.replace("language_ids.json", language_file_path) content = content.replace("speaker_embs.pth", speaker_embedding_file_path) f.close() with open(temp_config_path, "w") as f: f.write(content) f.close() return Synthesizer( tts_checkpoint=model_ckpt_path, tts_config_path=temp_config_path, use_cuda=torch.cuda.is_available(), ) OmegaConf.register_new_resolver("load_model", load_model) models_config = OmegaConf.to_object(OmegaConf.load("configs/models.yaml")) @spaces.GPU def _do_tts(model_id, ipa, language_name, speaker_name=None, speaker_wav=None): model = models_config[model_id]["model"] if speaker_wav is not None: return model.tts( ipa, speaker_wav=speaker_wav, language_name=language_name, split_sentences=False, ) return model.tts( ipa, speaker_name=speaker_name, language_name=language_name, split_sentences=False, ) def text_to_speech( model_id: str, use_default_emb_or_custom: str, speaker_wav, speaker: str, dialect, text: str, ): if len(text) == 0: raise gr.Error("請勿輸入空字串。") words, ipa, pinyin, missing_words = get_ipa(text, dialect=dialect) if len(missing_words) > 0: raise gr.Error( f"句子中的[{','.join(missing_words)}]目前無法轉成 ipa。請嘗試其他句子。" ) parsed_ipa = parse_ipa(ipa) if dialect == "nansixian": dialect = "sixian" if use_default_emb_or_custom == "預設語者": wav = _do_tts( model_id, parsed_ipa, speaker_name=speaker if len(models_config[model_id]["speaker_mapping"]) > 1 else None, language_name=dialect, ) else: wav = _do_tts( model_id, parsed_ipa, speaker_wav=speaker_wav, language_name=dialect, ) return ( words, pinyin, ( models_config[model_id]["model"].tts_model.config.audio.sample_rate, np.array(wav), ), ) def when_model_selected(model_id): model_config = models_config[model_id] speaker_drop_down_choices = [ (k, v) for k, v in model_config["speaker_mapping"].items() ] dialect_drop_down_choices = [ (k, v) for k, v in model_config["dialect_mapping"].items() ] use_default_emb_or_ref_radio_visible = False if model_config["model"].tts_model.config.model_args.speaker_encoder_model_path: use_default_emb_or_ref_radio_visible = True return ( gr.update( choices=speaker_drop_down_choices, value=speaker_drop_down_choices[0][1] if len(speaker_drop_down_choices) > 0 else None, interactive=len(speaker_drop_down_choices) > 1, ), gr.update( choices=dialect_drop_down_choices, value=dialect_drop_down_choices[0][1], interactive=len(dialect_drop_down_choices) > 1, ), gr.update(visible=use_default_emb_or_ref_radio_visible, value="預設語者"), ) def use_default_emb_or_custom_radio_input(use_default_emb_or_custom): if use_default_emb_or_custom == "客製化語者": return gr.update(visible=True), gr.update(visible=False) return gr.update(visible=False), gr.update(visible=True) demo = gr.Blocks( title="臺灣客語語音生成系統", css="@import url(https://tauhu.tw/tauhu-oo.css);", theme=gr.themes.Default( font=( "tauhu-oo", gr.themes.GoogleFont("Source Sans Pro"), "ui-sans-serif", "system-ui", "sans-serif", ) ), ) with demo: default_model_id = list(models_config.keys())[0] model_drop_down = gr.Dropdown( models_config.keys(), value=default_model_id, label="模型", ) use_default_emb_or_custom_radio = gr.Radio( label="語者類型", choices=["客製化語者"], value="客製化語者", visible=True, show_label=False, interactive=False, ) speaker_wav = gr.Audio( label="客製化語音", visible=True, editable=False, type="filepath", waveform_options=gr.WaveformOptions( show_controls=False, sample_rate=16000, ), ) speaker_drop_down = gr.Dropdown( choices=[ (k, v) for k, v in models_config[default_model_id]["speaker_mapping"].items() ], value=list(models_config[default_model_id]["speaker_mapping"].values())[0], label="語者", interactive=len(models_config[default_model_id]["speaker_mapping"]) > 1, visible=False, ) use_default_emb_or_custom_radio.input( use_default_emb_or_custom_radio_input, inputs=[use_default_emb_or_custom_radio], outputs=[speaker_wav, speaker_drop_down], ) dialect_radio = gr.Radio( choices=[ (k, v) for k, v in models_config[default_model_id]["dialect_mapping"].items() ], value=list(models_config[default_model_id]["dialect_mapping"].values())[0], label="腔調", interactive=len(models_config[default_model_id]["dialect_mapping"]) > 1, ) model_drop_down.input( when_model_selected, inputs=[model_drop_down], outputs=[speaker_drop_down, dialect_radio, use_default_emb_or_custom_radio], ) input_text = gr.Textbox( label="輸入文字", value="", ) gr.Markdown( """ # 臺灣客語語音合成系統 ### Taiwanese Hakka Text-to-Speech System ### 研發團隊 - **[李鴻欣 Hung-Shin Lee](mailto:hungshinlee@gmail.com)（[聯和科創](https://www.104.com.tw/company/1a2x6bmu75)）** - **[陳力瑋 Li-Wei Chen](mailto:wayne900619@gmail.com)（[聯和科創](https://www.104.com.tw/company/1a2x6bmu75)）** ### 合作單位 - **[國立聯合大學智慧客家實驗室](https://www.gohakka.org)** """ ) gr.Interface( text_to_speech, inputs=[ model_drop_down, use_default_emb_or_custom_radio, speaker_wav, speaker_drop_down, dialect_radio, input_text, ], outputs=[ gr.Textbox(interactive=False, label="斷詞"), gr.Textbox(interactive=False, label="客語拼音"), gr.Audio(interactive=False, label="合成語音", show_download_button=True), ], allow_flagging="auto", ) gr.Examples( [ [ "預設語者", "江芮敏", "sixian", "歸條路吊等長長个花燈，祈求風調雨順，歸屋下人个心願，親像花燈下燒暖个光華", ], [ "預設語者", "戴君儒", "hailu", "男女平等个時代，平平做得受教育", ], [ "預設語者", "宋涵葳", "dapu", "客家山城乜跈緊鬧熱䟘來咧", ], [ "預設語者", "江芮敏", "raoping", "頭擺匱人，戴个毋係菅草屋，个創商品哦", ], [ "預設語者", "洪藝晅", "zhaoan", "歇熱个時務，阿松歸屋下轉去在客莊个老屋", ], [ "預設語者", "江芮敏", "nansixian", "在𠊎讀小學一年生个時節，阿爸輒常用自轉車載𠊎去學校讀書", ], ], label="範例", inputs=[ use_default_emb_or_custom_radio, speaker_drop_down, dialect_radio, input_text, ], visible=False, ) demo.launch()