v02 (#10)

- v02 (d95adfe36996e391bf9178e2e3189b24588ca230)

app.py

@@ -12,9 +12,11 @@ from transformers import AutoTokenizer, AutoFeatureExtractor, set_seed
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
 
-repo_id = "parler-tts/parler_tts_mini_v0.1"
+repo_id =  "parler-tts/parler-tts-mini-v1"
+repo_id_large = "ylacombe/parler-large-v1-og"
 
 model = ParlerTTSForConditionalGeneration.from_pretrained(repo_id).to(device)
+model_large = ParlerTTSForConditionalGeneration.from_pretrained(repo_id_large).to(device)
 tokenizer = AutoTokenizer.from_pretrained(repo_id)
 feature_extractor = AutoFeatureExtractor.from_pretrained(repo_id)
 
@@ -22,23 +24,33 @@ feature_extractor = AutoFeatureExtractor.from_pretrained(repo_id)
 SAMPLE_RATE = feature_extractor.sampling_rate
 SEED = 42
 
-default_text = "Please surprise me and speak in whatever voice you enjoy."
+default_text = "All of the data, pre-processing, training code, and weights are released publicly under a permissive license, enabling the community to build on our work and develop their own powerful models."
+default_description = "Laura's voice is monotone yet slightly fast in delivery, with a very close recording that almost has no background noise."
 examples = [
     [
-        "Remember - this is only the first iteration of the model! To improve the prosody and naturalness of the speech further, we're scaling up the amount of training data by a factor of five times.",
-        "A male speaker with a low-pitched voice delivering his words at a fast pace in a small, confined space with a very clear audio and an animated tone."
+        "This version introduces speaker consistency across generations, characterized by their name. For example, Jon, Lea, Gary, Jenna, Mike and Laura.",
+        "Gary's voice is monotone yet slightly fast in delivery, with a very close recording that has no background noise.",
+        None,
+    ],
+    [
+        '''There's 34 speakers. To take advantage of this, simply adapt your text description to specify which speaker to use: "Mike speaks animatedly...".''',
+        "Gary speaks slightly animatedly and slightly slowly in delivery, with a very close recording that has no background noise.",
+        None
     ],
     [
         "'This is the best time of my life, Bartley,' she said happily.",
-        "A female speaker with a slightly low-pitched, quite monotone voice delivers her words at a slightly faster-than-average pace in a confined space with very clear audio.",
+        "A female speaker delivers a slightly expressive and animated speech with a moderate speed. The recording features a low-pitch voice and slight background noise, creating a close-sounding audio experience.",
+        None,
     ],
     [
         "Montrose also, after having experienced still more variety of good and bad fortune, threw down his arms, and retired out of the kingdom.",
-        "A male speaker with a slightly high-pitched voice delivering his words at a slightly slow pace in a small, confined space with a touch of background noise and a quite monotone tone.",
+        "A man voice speaks slightly slowly with very noisy background, carrying a low-pitch tone and displaying a touch of expressiveness and animation. The sound is very distant, adding an air of intrigue.",
+        None
     ],
     [
-        "Montrose also, after having experienced still more variety of good and bad fortune, threw down his arms, and retired out of the kingdom.",
-        "A male speaker with a low-pitched voice delivers his words at a fast pace and an animated tone, in a very spacious environment, accompanied by noticeable background noise.",
+        "Once upon a time, in the depth of winter, when the flakes of snow fell like feathers from the clouds, a queen sat sewing at her pal-ace window, which had a carved frame of black wood.",
+        "In a very poor recording quality, a female speaker delivers her slightly expressive and animated words with a fast pace. There's high level of background noise and a very distant-sounding reverberation. Her voice is slightly higher pitched than average.",
+        None,
     ],
 ]
 
@@ -64,14 +76,19 @@ def preprocess(text):
     return text
 
 @spaces.GPU
-def gen_tts(text, description):
-    inputs = tokenizer(description, return_tensors="pt").to(device)
+def gen_tts(text, description, use_large=False):
+    inputs = tokenizer(description.strip(), return_tensors="pt").to(device)
     prompt = tokenizer(preprocess(text), return_tensors="pt").to(device)
 
     set_seed(SEED)
-    generation = model.generate(
-        input_ids=inputs.input_ids, prompt_input_ids=prompt.input_ids, do_sample=True, temperature=1.0
-    )
+    if use_large:
+        generation = model_large.generate(
+            input_ids=inputs.input_ids, prompt_input_ids=prompt.input_ids, attention_mask=inputs.attention_mask, prompt_attention_mask=prompt.attention_mask, do_sample=True, temperature=1.0
+        )      
+    else:
+        generation = model.generate(
+            input_ids=inputs.input_ids, prompt_input_ids=prompt.input_ids, attention_mask=inputs.attention_mask, prompt_attention_mask=prompt.attention_mask, do_sample=True, temperature=1.0
+        )
     audio_arr = generation.cpu().numpy().squeeze()
 
     return SAMPLE_RATE, audio_arr
@@ -132,41 +149,48 @@ with gr.Blocks(css=css) as block:
     gr.HTML(
         f"""
         <p><a href="https://github.com/huggingface/parler-tts"> Parler-TTS</a> is a training and inference library for
-        high-fidelity text-to-speech (TTS) models. The model demonstrated here, <a href="https://huggingface.co/parler-tts/parler_tts_mini_v0.1"> Parler-TTS Mini v0.1</a>, 
-        is the first iteration model trained using 10k hours of narrated audiobooks. It generates high-quality speech 
+        high-fidelity text-to-speech (TTS) models.</p> 
+        <p>The models demonstrated here, Parler-TTS <a href="https://huggingface.co/parler-tts/parler-tts-mini-v1">Mini v1</a> and <a href="https://huggingface.co/parler-tts/parler-tts-large-v1">Large v1</a>, 
+        are trained using 45k hours of narrated English audiobooks. It generates high-quality speech 
         with features that can be controlled using a simple text prompt (e.g. gender, background noise, speaking rate, pitch and reverberation).</p>
 
-        <p>Tips for ensuring good generation:
-        <ul>
-            <li>Include the term "very clear audio" to generate the highest quality audio, and "very noisy audio" for high levels of background noise</li>
-            <li>Punctuation can be used to control the prosody of the generations, e.g. use commas to add small breaks in speech</li>
-            <li>The remaining speech features (gender, speaking rate, pitch and reverberation) can be controlled directly through the prompt</li>
-        </ul>
-        </p>
+        <p>By default, Parler-TTS generates 🎲 random voice. To ensure 🎯 <b> speaker consistency </b> across generations, these checkpoints were also trained on 34 speakers, characterized by name (e.g. Jon, Lea, Gary, Jenna, Mike, Laura).</p>
+        
+        <p>To take advantage of this, simply adapt your text description to specify which speaker to use: `Jon's voice is monotone...`</p>
         """
     )
     with gr.Row():
         with gr.Column():
             input_text = gr.Textbox(label="Input Text", lines=2, value=default_text, elem_id="input_text")
-            description = gr.Textbox(label="Description", lines=2, value="", elem_id="input_description")
+            description = gr.Textbox(label="Description", lines=2, value=default_description, elem_id="input_description")
+            use_large = gr.Checkbox(value=False, label="Use Large checkpoint", info="Generate with Parler-TTS Large v1 instead of Mini v1 - Better but way slower.")
             run_button = gr.Button("Generate Audio", variant="primary")
         with gr.Column():
             audio_out = gr.Audio(label="Parler-TTS generation", type="numpy", elem_id="audio_out")
 
-    inputs = [input_text, description]
+    inputs = [input_text, description, use_large]
     outputs = [audio_out]
-    gr.Examples(examples=examples, fn=gen_tts, inputs=inputs, outputs=outputs, cache_examples=True)
     run_button.click(fn=gen_tts, inputs=inputs, outputs=outputs, queue=True)
+    gr.Examples(examples=examples, fn=gen_tts, inputs=inputs, outputs=outputs, cache_examples=True)
     gr.HTML(
         """
-        <p>To improve the prosody and naturalness of the speech further, we're scaling up the amount of training data to 50k hours of speech.
-        The v1 release of the model will be trained on this data, as well as inference optimisations, such as flash attention
-        and torch compile, that will improve the latency by 2-4x. If you want to find out more about how this model was trained and even fine-tune it yourself, check-out the 
+        <p>Tips for ensuring good generation:
+        <ul>
+            <li>Include the term "very clear audio" to generate the highest quality audio, and "very noisy audio" for high levels of background noise</li>
+            <li>Punctuation can be used to control the prosody of the generations, e.g. use commas to add small breaks in speech</li>
+            <li>The remaining speech features (gender, speaking rate, pitch and reverberation) can be controlled directly through the prompt</li>
+        </ul>
+        </p>
+
+        <p>Parler-TTS can be much faster. We give some tips on how to generate much more quickly in this <a href="https://github.com/huggingface/parler-tts/blob/main/INFERENCE.md"> inference guide</a>. Think SDPA, torch.compile, batching and streaming!</p>
+        
+        <p>If you want to find out more about how this model was trained and even fine-tune it yourself, check-out the 
         <a href="https://github.com/huggingface/parler-tts"> Parler-TTS</a> repository on GitHub.</p>
         
         <p>The Parler-TTS codebase and its associated checkpoints are licensed under <a href='https://github.com/huggingface/parler-tts?tab=Apache-2.0-1-ov-file#readme'> Apache 2.0</a>.</p>
         """
     )
 
+
 block.queue()
 block.launch(share=True)