VITS/XTTS.py

import os
import subprocess
import uuid
import time
import torch
import torchaudio

# langid is used to detect language for longer text
# Most users expect text to be their own language, there is checkbox to disable it
import langid
import re
import gradio as gr
from TTS.tts.configs.xtts_config import XttsConfig
from TTS.tts.models.xtts import Xtts
from TTS.utils.generic_utils import get_user_data_dir

class XTTS():
 def __init__(self):
 model_name = "tts_models/multilingual/multi-dataset/xtts_v2"
 # ModelManager().download_model(model_name)
 model_path = os.path.join(get_user_data_dir("tts"), model_name.replace("/", "--"))
 # print("XTTS downloaded")

 config = XttsConfig()
 config.load_json(os.path.join(model_path, "config.json"))

 self.model = Xtts.init_from_config(config)
 self.model.load_checkpoint(
 config,
 checkpoint_path=os.path.join(model_path, "model.pth"),
 vocab_path=os.path.join(model_path, "vocab.json"),
 eval=True,
 use_deepspeed=True,
 )
 self.model.cuda()

 self.supported_languages = config.languages

 def predict(self, 
 prompt,
 language,
 audio_file_pth,
 voice_cleanup,
 ):
 # 模型不支持语言
 if language not in self.supported_languages:
 gr.Warning(
 f"Language you put {language} in is not in is not in our Supported Languages, please choose from dropdown"
 )

 return (
 None,
 None,
 None,
 None,
 )

 language_predicted = langid.classify(prompt)[
 0
 ].strip() # strip need as there is space at end!

 # tts expects chinese as zh-cn
 if language_predicted == "zh":
 # we use zh-cn
 language_predicted = "zh-cn"

 print(f"Detected language:{language_predicted}, Chosen language:{language}")

 speaker_wav = audio_file_pth

 # Filtering for microphone input, as it has BG noise, maybe silence in beginning and end
 # This is fast filtering not perfect

 # Apply all on demand
 lowpassfilter = denoise = trim = loudness = True

 if lowpassfilter:
 lowpass_highpass = "lowpass=8000,highpass=75,"
 else:
 lowpass_highpass = ""

 if trim:
 # better to remove silence in beginning and end for microphone
 trim_silence = "areverse,silenceremove=start_periods=1:start_silence=0:start_threshold=0.02,areverse,silenceremove=start_periods=1:start_silence=0:start_threshold=0.02,"
 else:
 trim_silence = ""

 if voice_cleanup:
 try:
 out_filename = (
 speaker_wav + str(uuid.uuid4()) + ".wav"
 ) # ffmpeg to know output format

 # we will use newer ffmpeg as that has afftn denoise filter
 shell_command = f"ffmpeg -y -i {speaker_wav} -af {lowpass_highpass}{trim_silence} {out_filename}".split(
 " "
 )

 command_result = subprocess.run(
 [item for item in shell_command],
 capture_output=False,
 text=True,
 check=True,
 )
 speaker_wav = out_filename
 print("Filtered microphone input")
 except subprocess.CalledProcessError:
 # There was an error - command exited with non-zero code
 print("Error: failed filtering, use original microphone input")
 else:
 speaker_wav = speaker_wav

 if len(prompt) < 2:
 gr.Warning("Please give a longer prompt text")
 return (
 None,
 None,
 None,
 None,
 )
 
 metrics_text = ""
 t_latent = time.time()

 # note diffusion_conditioning not used on hifigan (default mode), it will be empty but need to pass it to model.inference
 try:
 (
 gpt_cond_latent,
 speaker_embedding,
 ) = self.model.get_conditioning_latents(audio_path=speaker_wav, gpt_cond_len=30, gpt_cond_chunk_len=4, max_ref_length=60)
 except Exception as e:
 print("Speaker encoding error", str(e))
 gr.Warning(
 "It appears something wrong with reference, did you unmute your microphone?"
 )
 return (
 None,
 None,
 None,
 )

 latent_calculation_time = time.time() - t_latent
 # metrics_text=f"Embedding calculation time: {latent_calculation_time:.2f} seconds\n"

 # temporary comma fix
 prompt= re.sub("([^\x00-\x7F]|\w)(\.|\。|\?)",r"\1 \2\2",prompt)

 wav_chunks = []
 ## Direct mode
 
 print("I: Generating new audio...")
 t0 = time.time()
 out = self.model.inference(
 prompt,
 language,
 gpt_cond_latent,
 speaker_embedding,
 repetition_penalty=5.0,
 temperature=0.75,
 )
 inference_time = time.time() - t0
 print(f"I: Time to generate audio: {round(inference_time*1000)} milliseconds")
 metrics_text+=f"Time to generate audio: {round(inference_time*1000)} milliseconds\n"
 real_time_factor= (time.time() - t0) / out['wav'].shape[-1] * 24000
 print(f"Real-time factor (RTF): {real_time_factor}")
 metrics_text+=f"Real-time factor (RTF): {real_time_factor:.2f}\n"
 torchaudio.save("output.wav", torch.tensor(out["wav"]).unsqueeze(0), 24000)
 return (
 "output.wav",
 metrics_text,
 speaker_wav,
 )

title = "语音克隆 Coqui🐸 XTTS"

examples = [
 [
 "Once when I was six years old I saw a magnificent picture",
 "en",
 "examples/female.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Lorsque j'avais six ans j'ai vu, une fois, une magnifique image",
 "fr",
 "examples/male.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Als ich sechs war, sah ich einmal ein wunderbares Bild",
 "de",
 "examples/female.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Cuando tenía seis años, vi una vez una imagen magnífica",
 "es",
 "examples/male.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Quando eu tinha seis anos eu vi, uma vez, uma imagem magnífica",
 "pt",
 "examples/female.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Kiedy miałem sześć lat, zobaczyłem pewnego razu wspaniały obrazek",
 "pl",
 "examples/male.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Un tempo lontano, quando avevo sei anni, vidi un magnifico disegno",
 "it",
 "examples/female.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Bir zamanlar, altı yaşındayken, muhteşem bir resim gördüm",
 "tr",
 "examples/female.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Когда мне было шесть лет, я увидел однажды удивительную картинку",
 "ru",
 "examples/female.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Toen ik een jaar of zes was, zag ik op een keer een prachtige plaat",
 "nl",
 "examples/male.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "Když mi bylo šest let, viděl jsem jednou nádherný obrázek",
 "cs",
 "examples/female.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "当我还只有六岁的时候， 看到了一副精彩的插画",
 "zh-cn",
 "examples/female.wav",
 None,
 False,
 False,
 False,
 True,
 ],
 [
 "かつて 六歳のとき、素晴らしい絵を見ました",
 "ja",
 "examples/female.wav",
 None,
 False,
 True,
 False,
 True,
 ],
 [
 "한번은 내가 여섯 살이었을 때 멋진 그림을 보았습니다.",
 "ko",
 "examples/female.wav",
 None,
 False,
 True,
 False,
 True,
 ],
 [
 "Egyszer hat éves koromban láttam egy csodálatos képet",
 "hu",
 "examples/male.wav",
 None,
 False,
 True,
 False,
 True,
 ],
]



def main():
 with gr.Blocks(analytics_enabled=False) as demo:
 with gr.Row():
 with gr.Column():
 gr.Markdown(
 """
 ## 语音克隆 (Coqui TTS)
 """
 )
 with gr.Column():
 # 用于对齐图像的占位符
 pass

 with gr.Row():
 with gr.Column():
 gr.Markdown("文本提示: 一次一两个句子最好。最多200个文本字符。")
 with gr.Column():
 gr.Markdown("语言: 选择用于合成语音的输出语言。")

 with gr.Row():
 with gr.Column():
 input_text_gr = gr.Textbox(
 label="文本提示",
 info="一次一两个句子最好。最多200个文本字符。",
 value="嗨，我是你的新语音克隆。请尽量上传高质量的音频。",
 )
 language_gr = gr.Dropdown(
 label="语言",
 info="选择用于合成语音的输出语言",
 choices=[
 "en",
 "es",
 "fr",
 "de",
 "it",
 "pt",
 "pl",
 "tr",
 "ru",
 "nl",
 "cs",
 "ar",
 "zh-cn",
 "ja",
 "ko",
 "hu",
 "hi"
 ],
 max_choices=1,
 value="en",
 )
 ref_gr = gr.Audio(
 label="参考音频",
 type="filepath",
 value="examples/female.wav",
 sources=["microphone", "upload"],
 )
 clean_ref_gr = gr.Checkbox(
 label="清理参考语音",
 value=False,
 info="如果您的麦克风或参考语音有噪音，此选项可以改善输出。",
 )
 tts_button = gr.Button("发送", elem_id="send-btn", visible=True)


 with gr.Column():
 audio_gr = gr.Audio(label="合成音频", autoplay=True)
 out_text_gr = gr.Text(label="指标")
 ref_audio_gr = gr.Audio(label="使用的参考音频")
 with gr.Row():
 gr.Examples(examples,
 label="示例",
 inputs=[input_text_gr, language_gr, ref_gr, clean_ref_gr],
 outputs=[audio_gr, out_text_gr, ref_audio_gr],
 fn=XTTS().predict,
 cache_examples=False,)
 tts_button.click(XTTS().predict, [input_text_gr, language_gr, ref_gr, clean_ref_gr], outputs=[audio_gr, out_text_gr, ref_audio_gr])
 return demo

if __name__ == "__main__": 
 demo = main() 
 demo.launch()