import spaces
import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.models.speecht5.number_normalizer import EnglishNumberNormalizer
from string import punctuation
import re
from parler_tts import ParlerTTSForConditionalGeneration
from transformers import AutoTokenizer, AutoFeatureExtractor, set_seed
# Device setup
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# SmolLM Instruct setup
checkpoint = "HuggingFaceTB/SmolLM-360M-Instruct"
smol_tokenizer = AutoTokenizer.from_pretrained(checkpoint)
smol_model = AutoModelForCausalLM.from_pretrained(checkpoint, device_map="auto", torch_dtype=torch.bfloat16)
# Original model setup
repo_id = "ylacombe/p-m-e"
model = ParlerTTSForConditionalGeneration.from_pretrained(repo_id).to(device)
text_tokenizer = AutoTokenizer.from_pretrained(repo_id)
description_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large")
feature_extractor = AutoFeatureExtractor.from_pretrained(repo_id)
SAMPLE_RATE = feature_extractor.sampling_rate
SEED = 42
default_text = "La voix humaine est un instrument de musique au-dessus de tous les autres."
default_description = "a woman with a slightly low-pitched voice speaks slowly in a clear and close-sounding environment, but her delivery is quite monotone."
examples = [
[
"La voix humaine est un instrument de musique au-dessus de tous les autres.",
"a woman with a slightly low-pitched voice speaks slowly in a clear and close-sounding environment, but her delivery is quite monotone.",
True,
None,
],
[
"The human voice is nature's most perfect instrument.",
"A woman with a slightly low-pitched voice speaks slowly in a very distant-sounding environment with a clean audio quality, delivering her message in a very monotone manner.",
True,
None,
],
]
number_normalizer = EnglishNumberNormalizer()
def format_description(raw_description, do_format=True):
if not do_format:
return raw_description
messages = [{
"role": "system",
"content": "You are a helpful assistant that formats voice descriptions precisely according to the template provided."
}, {
"role": "user",
"content": f"""Format this voice description exactly as:
"a [gender] with a [pitch] voice speaks [speed] in a [environment], [delivery style]"
Required format:
- gender: man/woman
- pitch: slightly low-pitched/moderate pitch/high-pitched
- speed: slowly/moderately/quickly
- environment: close-sounding and clear/distant-sounding and noisy
- delivery style: with monotone delivery/with animated delivery
Input description: {raw_description}
Return only the formatted description, nothing else."""
}]
input_text = smol_tokenizer.apply_chat_template(messages, tokenize=False)
inputs = smol_tokenizer.encode(input_text, return_tensors="pt").to(device)
outputs = smol_model.generate(
inputs,
max_new_tokens=100,
temperature=0.2,
top_p=0.9,
do_sample=True
)
formatted = smol_tokenizer.decode(outputs[0], skip_special_tokens=True)
# Extract just the formatted description
if "a woman" in formatted.lower() or "a man" in formatted.lower():
return formatted.strip()
return raw_description
def preprocess(text):
text = number_normalizer(text).strip()
text = text.replace("-", " ")
if text[-1] not in punctuation:
text = f"{text}."
abbreviations_pattern = r'\b[A-Z][A-Z\.]+\b'
def separate_abb(chunk):
chunk = chunk.replace(".","")
return " ".join(chunk)
abbreviations = re.findall(abbreviations_pattern, text)
for abv in abbreviations:
if abv in text:
text = text.replace(abv, separate_abb(abv))
return text
@spaces.GPU
def gen_tts(text, description, do_format=True):
formatted_desc = format_description(description, do_format)
inputs = description_tokenizer(formatted_desc.strip(), return_tensors="pt").to(device)
prompt = text_tokenizer(preprocess(text), return_tensors="pt").to(device)
set_seed(SEED)
generation = model.generate(
input_ids=inputs.input_ids,
prompt_input_ids=prompt.input_ids,
attention_mask=inputs.attention_mask,
prompt_attention_mask=prompt.attention_mask,
do_sample=True,
temperature=1.0
)
audio_arr = generation.cpu().numpy().squeeze()
return formatted_desc, (SAMPLE_RATE, audio_arr)
# Rest of the code remains unchanged
css = """
#share-btn-container {
display: flex;
padding-left: 0.5rem !important;
padding-right: 0.5rem !important;
background-color: #000000;
justify-content: center;
align-items: center;
border-radius: 9999px !important;
width: 13rem;
margin-top: 10px;
margin-left: auto;
flex: unset !important;
}
#share-btn {
all: initial;
color: #ffffff;
font-weight: 600;
cursor: pointer;
font-family: 'IBM Plex Sans', sans-serif;
margin-left: 0.5rem !important;
padding-top: 0.25rem !important;
padding-bottom: 0.25rem !important;
right:0;
}
#share-btn * {
all: unset !important;
}
#share-btn-container div:nth-child(-n+2){
width: auto !important;
min-height: 0px !important;
}
#share-btn-container .wrap {
display: none !important;
}
"""
with gr.Blocks(css=css) as block:
gr.HTML(
"""
"""
)
gr.HTML(
"""Parler-TTS is a training and inference library for
high-fidelity text-to-speech (TTS) models.
This multilingual model supports French, Spanish, Italian, Portuguese, Polish, German, Dutch, and English. It generates high-quality speech with features that can be controlled using a simple text prompt.
By default, Parler-TTS generates 🎲 random voice characteristics. To ensure 🎯 speaker consistency across generations, try to use consistent descriptions in your prompts.
"""
)
with gr.Row():
with gr.Column():
input_text = gr.Textbox(
label="Input Text",
lines=2,
value=default_text
)
raw_description = gr.Textbox(
label="Voice Description",
lines=2,
value=default_description
)
do_format = gr.Checkbox(
label="Reformat description using SmolLM",
value=True
)
formatted_description = gr.Textbox(
label="Used Description",
lines=2
)
generate_button = gr.Button("Generate Audio", variant="primary")
with gr.Column():
audio_out = gr.Audio(label="Parler-TTS generation", type="numpy")
generate_button.click(
fn=gen_tts,
inputs=[input_text, raw_description, do_format],
outputs=[formatted_description, audio_out]
)
gr.Examples(
examples=examples,
fn=gen_tts,
inputs=[input_text, raw_description, do_format],
outputs=[formatted_description, audio_out],
cache_examples=True
)
gr.HTML(
"""Tips for ensuring good generation:
- Include the term "very clear audio" to generate the highest quality audio, and "very noisy audio" for high levels of background noise
- Punctuation can be used to control the prosody of the generations
- The remaining speech features (gender, speaking rate, pitch and reverberation) can be controlled directly through the prompt
"""
)
block.queue()
block.launch(share=True)