app.py

# flake8: noqa: E402
import logging
logging.getLogger("numba").setLevel(logging.WARNING)
logging.getLogger("markdown_it").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("matplotlib").setLevel(logging.WARNING)

logging.basicConfig(
    level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
)

logger = logging.getLogger(__name__)
import datetime
import numpy as np
import torch
from ebooklib import epub
import PyPDF2
from PyPDF2 import PdfReader
import zipfile
import shutil
import sys, os
import json
from bs4 import BeautifulSoup
import argparse
import commons
import utils
from models import SynthesizerTrn
from text.symbols import symbols
from text import cleaned_text_to_sequence, get_bert
from text.cleaner import clean_text
import gradio as gr
import webbrowser
import re
from scipy.io.wavfile import write
from datetime import datetime
net_g = None
BandList = {
    
        "PoppinParty":["香澄","有咲","たえ","りみ","沙綾"],
        "Afterglow":["蘭","モカ","ひまり","巴","つぐみ"],
        "HelloHappyWorld":["こころ","美咲","薫","花音","はぐみ"],
        "PastelPalettes":["彩","日菜","千聖","イヴ","麻弥"],
        "Roselia":["友希那","紗夜","リサ","燐子","あこ"],
        "RaiseASuilen":["レイヤ","ロック","ますき","チュチュ","パレオ"],
        "Morfonica":["ましろ","瑠唯","つくし","七深","透子"],
        "MyGo&AveMujica(Part)":["燈","愛音","そよ","立希","楽奈","祥子","睦","海鈴"],
        "圣翔音乐学园":["華戀","光","香子","雙葉","真晝","純那","克洛迪娜","真矢","奈奈"],
        "凛明馆女子学校":["珠緒","壘","文","悠悠子","一愛"],
        "弗隆提亚艺术学校":["艾露","艾露露","菈樂菲","司","靜羽"],
        "西克菲尔特音乐学院":["晶","未知留","八千代","栞","美帆"]
}

if sys.platform == "darwin" and torch.backends.mps.is_available():
    device = "mps"
    os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
else:
    device = "cuda"

def is_japanese(string):
        for ch in string:
            if ord(ch) > 0x3040 and ord(ch) < 0x30FF:
                return True
        return False

def extrac(text):
    text = re.sub("<[^>]*>","",text)
    result_list = re.split(r'\n', text)
    final_list = []
    for i in result_list:
        i = i.replace('\n','').replace(' ','')
        #Current length of single sentence: 20 
        if len(i)>1:
            if len(i) > 20:
                try:
                    cur_list = re.split(r'。|！', i)
                    for i in cur_list:
                        if len(i)>1:
                            final_list.append(i+'。')
                except:
                    pass
            else:
                final_list.append(i)
            '''
        final_list.append(i)
        '''
    final_list = [x for x in final_list if x != '']
    return final_list

def get_text(text, language_str, hps):
    norm_text, phone, tone, word2ph = clean_text(text, language_str)
    phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)

    if hps.data.add_blank:
        phone = commons.intersperse(phone, 0)
        tone = commons.intersperse(tone, 0)
        language = commons.intersperse(language, 0)
        for i in range(len(word2ph)):
            word2ph[i] = word2ph[i] * 2
        word2ph[0] += 1
    bert = get_bert(norm_text, word2ph, language_str, device)
    del word2ph
    assert bert.shape[-1] == len(phone), phone

    if language_str == "ZH":
        bert = bert
        ja_bert = torch.zeros(768, len(phone))
    elif language_str == "JA":
        ja_bert = bert
        bert = torch.zeros(1024, len(phone))
    else:
        bert = torch.zeros(1024, len(phone))
        ja_bert = torch.zeros(768, len(phone))

    assert bert.shape[-1] == len(
        phone
    ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"

    phone = torch.LongTensor(phone)
    tone = torch.LongTensor(tone)
    language = torch.LongTensor(language)
    return bert, ja_bert, phone, tone, language


def infer(text, sdp_ratio, noise_scale, noise_scale_w, length_scale, sid, language):
    global net_g
    bert, ja_bert, phones, tones, lang_ids = get_text(text, language, hps)
    with torch.no_grad():
        x_tst = phones.to(device).unsqueeze(0)
        tones = tones.to(device).unsqueeze(0)
        lang_ids = lang_ids.to(device).unsqueeze(0)
        bert = bert.to(device).unsqueeze(0)
        ja_bert = ja_bert.to(device).unsqueeze(0)
        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
        del phones
        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
        audio = (
            net_g.infer(
                x_tst,
                x_tst_lengths,
                speakers,
                tones,
                lang_ids,
                bert,
                ja_bert,
                sdp_ratio=sdp_ratio,
                noise_scale=noise_scale,
                noise_scale_w=noise_scale_w,
                length_scale=length_scale,
            )[0][0, 0]
            .data.cpu()
            .float()
            .numpy()
        )
        current_time = datetime.now()
        print(str(current_time)+':'+str(sid))
        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers
        return audio


def tts_fn(
    text, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,LongSentence
):
    if not LongSentence:
        with torch.no_grad():
            audio = infer(
                text,
                sdp_ratio=sdp_ratio,
                noise_scale=noise_scale,
                noise_scale_w=noise_scale_w,
                length_scale=length_scale,
                sid=speaker,
                language= "JP" if is_japanese(text) else "ZH",
            )
            torch.cuda.empty_cache()
        return (hps.data.sampling_rate, audio)
    else:
        audiopath = 'voice.wav'
        a = ['【','[','(','（']
        b = ['】',']',')','）']
        for i in a:
            text = text.replace(i,'<')
        for i in b:
            text = text.replace(i,'>')
        final_list = extrac(text.replace('“','').replace('”',''))
        audio_fin = []
        for sentence in final_list:
            with torch.no_grad():
                audio = infer(
                    sentence,
                    sdp_ratio=sdp_ratio,
                    noise_scale=noise_scale,
                    noise_scale_w=noise_scale_w,
                    length_scale=length_scale,
                    sid=speaker,
                    language= "JP" if is_japanese(text) else "ZH",
                )
            audio_fin.append(audio)
        return (hps.data.sampling_rate, np.concatenate(audio_fin))

def split_into_sentences(text):
    """将文本分割为句子，基于中文的标点符号"""
    sentences = re.split(r'(?<=[。！？…\n])', text)
    return [sentence.strip() for sentence in sentences if sentence]


def seconds_to_ass_time(seconds):
    """将秒数转换为ASS时间格式"""
    hours = int(seconds / 3600)
    minutes = int((seconds % 3600) / 60)
    seconds = int(seconds) % 60
    milliseconds = int((seconds - int(seconds)) * 1000)
    return "{:01d}:{:02d}:{:02d}.{:02d}".format(hours, minutes, seconds, int(milliseconds / 10))

def generate_audio_and_srt_for_group(group, outputPath, group_index, sampling_rate, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime):
    audio_fin = []
    ass_entries = []
    start_time = 0

    ass_header = """[Script Info]
; Script generated by OpenAI Assistant
Title: Audiobook
ScriptType: v4.00+
WrapStyle: 0
PlayResX: 640
PlayResY: 360
ScaledBorderAndShadow: yes
[V4+ Styles]
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
Style: Default,Arial,20,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,1,1,2,10,10,10,1
[Events]
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
"""

    for sentence in group:
        try:
            print(sentence)
            FakeSpeaker = sentence.split("|")[0]
            print(FakeSpeaker)
            SpeakersList = re.split('\n', spealerList)
            if FakeSpeaker in list(hps.data.spk2id.keys()):
                speaker = FakeSpeaker
            for i in SpeakersList:
                if FakeSpeaker == i.split("|")[1]:
                    speaker = i.split("|")[0]
            speaker_ids = hps.data.spk2id
            
            _, audio = tts_fn(sentence.split("|")[-1], speaker=speaker, sdp_ratio=sdp_ratio, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale, LongSentence=True)
            silence_frames = int(silenceTime * 44010)
            silence_data = np.zeros((silence_frames,), dtype=audio.dtype)
            audio_fin.append(audio)
            audio_fin.append(silence_data)

            duration = len(audio) / sampling_rate
            end_time = start_time + duration + silenceTime
            ass_entries.append("Dialogue: 0,{},{},".format(seconds_to_ass_time(start_time), seconds_to_ass_time(end_time)) + "Default,,0,0,0,,{}".format(sentence.replace("|","：")))
            start_time = end_time
        except:
            pass
    wav_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.wav')
    ass_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.ass')

    write(wav_filename, sampling_rate, np.concatenate(audio_fin))

    with open(ass_filename, 'w', encoding='utf-8') as f:
        f.write(ass_header + '\n'.join(ass_entries))
    return (hps.data.sampling_rate, np.concatenate(audio_fin))
def extract_text_from_epub(file_path):
    book = epub.read_epub(file_path)
    content = []
    for item in book.items:
        if isinstance(item, epub.EpubHtml):
            soup = BeautifulSoup(item.content, 'html.parser')
            content.append(soup.get_text())
    return '\n'.join(content)

def extract_text_from_pdf(file_path):
    with open(file_path, 'rb') as file:
        reader = PdfReader(file)
        content = [page.extract_text() for page in reader.pages]
    return '\n'.join(content)

def extract_text_from_game2(data):
    current_content = []

    def _extract(data, current_data=None):
        nonlocal current_content
        
        if current_data is None:
            current_data = {}

        if isinstance(data, dict):
            if 'name' in data and 'body' in data:
                current_name = data['name']
                current_body = data['body'].replace('\n', '')
                current_content.append(f"{current_name}|{current_body}")

            for key, value in data.items():
                _extract(value, dict(current_data))

        elif isinstance(data, list):
            for item in data:
                _extract(item, dict(current_data))

    _extract(data)
    return '\n'.join(current_content)

def extract_text_from_file(inputFile):
    file_extension = os.path.splitext(inputFile)[1].lower()
    
    if file_extension == ".epub":
        return extract_text_from_epub(inputFile)
    elif file_extension == ".pdf":
        return extract_text_from_pdf(inputFile)
    elif file_extension == ".txt":
        with open(inputFile, 'r', encoding='utf-8') as f:
            return f.read()
    elif file_extension == ".asset":
        with open(inputFile, 'r', encoding='utf-8') as f:
            content =  json.load(f)
        return extract_text_from_game2(content) if extract_text_from_game2(content) != '' else extract_text_from_game2(content)
    else:
        raise ValueError(f"Unsupported file format: {file_extension}")

def audiobook(inputFile, groupsize, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime):
    directory_path = "books"
    output_path = "books/audiobook_part_1.wav"
 
    if os.path.exists(directory_path):
        shutil.rmtree(directory_path)

    os.makedirs(directory_path)
    text = extract_text_from_file(inputFile.name)
    sentences = split_into_sentences(text)
    GROUP_SIZE = groupsize
    for i in range(0, len(sentences), GROUP_SIZE):
        group = sentences[i:i+GROUP_SIZE]
        if spealerList == "":
            spealerList = "无"
        result = generate_audio_and_srt_for_group(group,directory_path, i//GROUP_SIZE + 1, 44100, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime)
        if not torch.cuda.is_available():
            return result
    return result

def loadmodel(model):
    _ = net_g.eval()
    _ = utils.load_checkpoint(model, net_g, None, skip_optimizer=True)
    return "success"


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-m", "--model", default="./logs/Bangdream/G_10000.pth", help="path of your model"
    )
    parser.add_argument(
        "-c",
        "--config",
        default="configs/config.json",
        help="path of your config file",
    )
    parser.add_argument(
        "--share", default=True, help="make link public", action="store_true"
    )
    parser.add_argument(
        "-d", "--debug", action="store_true", help="enable DEBUG-LEVEL log"
    )

    args = parser.parse_args()
    if args.debug:
        logger.info("Enable DEBUG-LEVEL log")
        logging.basicConfig(level=logging.DEBUG)
    device = (
        "cuda:0"
        if torch.cuda.is_available()
        else (
            "mps"
            if sys.platform == "darwin" and torch.backends.mps.is_available()
            else "cpu"
        )
    )
    hps = utils.get_hparams_from_file(args.config)
    net_g = SynthesizerTrn(
        len(symbols),
        hps.data.filter_length // 2 + 1,
        hps.train.segment_size // hps.data.hop_length,
        n_speakers=hps.data.n_speakers,
        **hps.model,
    ).to(device)
    loadmodel(args.model)
    speaker_ids = hps.data.spk2id
    speakers = list(speaker_ids.keys())
    languages = ["ZH", "JP"]
    examples = [
        ["filelist/Scenarioband6-018.asset", 500, "つくし", "ましろ|真白\n七深|七深\n透子|透子\nつくし|筑紫\n瑠唯|瑠唯\nそよ|素世\n祥子|祥子", "扩展功能"],
    ]
    modelPaths = []
    for dirpath, dirnames, filenames in os.walk("./logs/Bangdream/"):
        for filename in filenames:
            modelPaths.append(os.path.join(dirpath, filename))
    with gr.Blocks() as app:
        gr.Markdown(
             f"少歌邦邦全员TTS,使用本模型请严格遵守法律法规!\n 发布二创作品请注明项目和本模型作者<a href='https://space.bilibili.com/19874615/'>B站@Mahiroshi</a>及项目链接\n从 <a href='https://nijigaku.top/2023/10/03/BangDreamTTS/'>我的博客站点</a> 查看使用说明</a>"
        )
        for band in BandList:
            with gr.TabItem(band):
                for name in BandList[band]:
                    with gr.TabItem(name):
                        with gr.Row():
                            with gr.Column():
                                with gr.Row():
                                    gr.Markdown(
                                        '<div align="center">'
                                        f'<img style="width:auto;height:400px;" src="file/image/{name}.png">' 
                                        '</div>'
                                    )
                                length_scale = gr.Slider(
                                        minimum=0.1, maximum=2, value=1, step=0.01, label="语速调节"
                                    )
                                with gr.Accordion(label="切换模型(合成中文建议切换为早期模型)", open=False):
                                    modelstrs = gr.Dropdown(label = "模型", choices = modelPaths, value = modelPaths[0], type = "value")
                                    btnMod = gr.Button("载入模型")
                                    statusa = gr.TextArea()
                                    btnMod.click(loadmodel, inputs=[modelstrs], outputs = [statusa])
                            with gr.Column():
                                text = gr.TextArea(
                                    label="输入纯日语或者中文",
                                    placeholder="输入纯日语或者中文",
                                    value="有个人躺在地上，哀嚎......\n有个人睡着了，睡在盒子里。\n我要把它打开，看看他的梦是什么。",
                                )                                
                                btn = gr.Button("点击生成", variant="primary")
                                audio_output = gr.Audio(label="Output Audio")
                                with gr.Accordion(label="其它参数设定", open=False):
                                    sdp_ratio = gr.Slider(
                                    minimum=0, maximum=1, value=0.2, step=0.01, label="SDP/DP混合比"
                                    )
                                    noise_scale = gr.Slider(
                                        minimum=0.1, maximum=2, value=0.6, step=0.01, label="感情调节"
                                    )
                                    noise_scale_w = gr.Slider(
                                        minimum=0.1, maximum=2, value=0.8, step=0.01, label="音素长度"
                                    )
                                    LongSentence = gr.Checkbox(value=True, label="Generate LongSentence")
                                    speaker = gr.Dropdown(
                                        choices=speakers, value=name, label="说话人"
                                    )
                    btn.click(
                        tts_fn,
                        inputs=[
                            text,
                            speaker,
                            sdp_ratio,
                            noise_scale,
                            noise_scale_w,
                            length_scale,
                            LongSentence,
                        ],
                        outputs=[audio_output],
                    )
        for i in examples:
            with gr.Tab(i[-1]):
                with gr.Row():
                    with gr.Column():
                        gr.Markdown(
                                        f"从 <a href='https://nijigaku.top/2023/10/03/BangDreamTTS/'>我的博客站点</a> 查看自制galgame使用说明\n</a>"
                                    )
                        inputFile = gr.inputs.File(label="上传txt(可设置角色对应表)、epub或mobi文件")
                        groupSize = gr.Slider(
                        minimum=10, maximum=1000,value = i[1], step=1, label="当个音频文件包含的最大字数"
                        )
                        silenceTime = gr.Slider(
                        minimum=0, maximum=1, value=0.5, step=0.1, label="句子的间隔"
                        )
                        spealerList = gr.TextArea(
                                            label="角色对应表",
                                            placeholder="左边是你想要在每一句话合成中用到的speaker(见角色清单)右边是你上传文本时分隔符左边设置的说话人:{ChoseSpeakerFromConfigList1}|{SeakerInUploadText1}\n{ChoseSpeakerFromConfigList2}|{SeakerInUploadText2}\n{ChoseSpeakerFromConfigList3}|{SeakerInUploadText3}\n",
                                            value = i[3],
                        )                  
                        speaker = gr.Dropdown(
                            choices=speakers, value = i[2], label="选择默认说话人"
                        )
                    with gr.Column():
                        sdp_ratio = gr.Slider(
                        minimum=0, maximum=1, value=0.2, step=0.01, label="SDP/DP混合比"
                        )
                        noise_scale = gr.Slider(
                            minimum=0.1, maximum=2, value=0.6, step=0.01, label="感情调节"
                        )
                        noise_scale_w = gr.Slider(
                            minimum=0.1, maximum=2, value=0.8, step=0.01, label="音素长度"
                        )
                        length_scale = gr.Slider(
                            minimum=0.1, maximum=2, value=1, step=0.01, label="生成长度"
                        )
                        LastAudioOutput = gr.Audio(label="当用cuda在本地运行时才能在book文件夹下浏览全部合成内容")
                        btn2 = gr.Button("点击生成", variant="primary")
                    btn2.click(
                        audiobook,
                        inputs=[
                            inputFile,
                            groupSize,
                            speaker,
                            sdp_ratio,
                            noise_scale,
                            noise_scale_w,
                            length_scale,
                            spealerList,
                            silenceTime
                        ],
                        outputs=[LastAudioOutput],
                    )
app.launch()