Spaces:
Runtime error
Runtime error
File size: 8,461 Bytes
884c49d 19d9763 70814d8 0c8b1e1 22dcbc5 e4b4acb 0c8b1e1 19d9763 0c8b1e1 19d9763 0c8b1e1 19d9763 70814d8 19d9763 70814d8 19d9763 70814d8 19d9763 70814d8 19d9763 0c8b1e1 19d9763 0c8b1e1 19d9763 0c8b1e1 19d9763 70814d8 19d9763 70814d8 19d9763 70814d8 19d9763 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 |
import gradio as gr
import torch
from transformers import BitsAndBytesConfig, HqqConfig
from whisperplus import (
SpeechToTextPipeline,
download_youtube_to_mp3,
download_youtube_to_mp4,
format_speech_to_dialogue,
)
from whisperplus.pipelines.long_text_summarization import LongTextSummarizationPipeline
from whisperplus.pipelines.summarization import TextSummarizationPipeline
from whisperplus.pipelines.text2speech import TextToSpeechPipeline
from whisperplus.pipelines.whisper_autocaption import WhisperAutoCaptionPipeline
from whisperplus.pipelines.whisper_diarize import ASRDiarizationPipeline
import subprocess
subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
def youtube_url_to_text(url, model_id, language_choice):
"""
Main function that downloads and converts a video to MP3 format, performs speech-to-text conversion using
a specified model, and returns the transcript along with the video path.
Args:
url (str): The URL of the video to download and convert.
model_id (str): The ID of the speech-to-text model to use.
language_choice (str): The language choice for the speech-to-text conversion.
Returns:
transcript (str): The transcript of the speech-to-text conversion.
"""
audio_path = download_youtube_to_mp3(url, output_dir="downloads", filename="test")
hqq_config = HqqConfig(
nbits=4,
group_size=64,
quant_zero=False,
quant_scale=False,
axis=0,
offload_meta=False,
) # axis=0 is used by default
pipeline = SpeechToTextPipeline(
model_id=model_id,
quant_config=hqq_config,
flash_attention_2=True,
)
transcript = pipeline(
audio_path=audio_path,
chunk_length_s=30,
stride_length_s=5,
max_new_tokens=128,
batch_size=100,
language=language_choice,
return_timestamps=False,
)
return transcript
def summarization(text, model_id="facebook/bart-large-cnn"):
"""
Main function that performs summarization using a specified model and returns the summary.
Args:
text (str): The text to summarize.
model_id (str): The ID of the summarization model to use.
Returns:
summary (str): The summary of the text.
"""
summarizer = TextSummarizationPipeline(model_id=model_id)
summary = summarizer.summarize(text)
return summary[0]["summary_text"]
def long_text_summarization(text, model_id="facebook/bart-large-cnn"):
"""
Main function that performs summarization using a specified model and returns the summary.
Args:
text (str): The text to summarize.
model_id (str): The ID of the summarization model to use.
Returns:
summary (str): The summary of the text.
"""
summarizer = LongTextSummarizationPipeline(model_id=model_id)
summary_text = summarizer.summarize(text)
return summary_text
def speaker_diarization(url, model_id, device, num_speakers, min_speaker, max_speaker):
"""
Main function that downloads and converts a video to MP3 format, performs speech-to-text conversion using
a specified model, and returns the transcript along with the video path.
Args:
url (str): The URL of the video to download and convert.
model_id (str): The ID of the speech-to-text model to use.
language_choice (str): The language choice for the speech-to-text conversion.
Returns:
transcript (str): The transcript of the speech-to-text conversion.
video_path (str): The path of the downloaded video.
"""
pipeline = ASRDiarizationPipeline.from_pretrained(
asr_model=model_id,
diarizer_model="pyannote/speaker-diarization",
use_auth_token=False,
chunk_length_s=30,
device=device,
)
audio_path = download_youtube_to_mp3(url)
output_text = pipeline(
audio_path, num_speakers=num_speakers, min_speaker=min_speaker, max_speaker=max_speaker)
dialogue = format_speech_to_dialogue(output_text)
return dialogue, audio_path
def text2spech_bark(text, model_id="suno/bark", voice_preset="v2/en_speaker_6"):
tts = TextToSpeechPipeline(model_id=model_id)
audio = tts(text=text, voice_preset=voice_preset)
return audio
def whisper_autocaption(url, language, model_id="openai/whisper-large-v3"):
video_path = download_youtube_to_mp4(url)
caption = WhisperAutoCaptionPipeline(model_id=model_id)
output = caption(video_path=video_path, output_path="output.mp4", language=language)
return output
with gr.Blocks() as demo:
with gr.Tab("YouTube URL to Text"):
with gr.Row():
with gr.Column():
url_input = gr.Textbox(label="Enter YouTube URL")
model_id_input = gr.Textbox(label="Enter Model ID", value="openai/whisper-medium")
language_input = gr.Textbox(label="Enter Language", value="en")
submit_btn1 = gr.Button("Submit")
with gr.Column():
output1 = gr.Textbox(label="Transcript")
submit_btn1.click(
youtube_url_to_text, inputs=[url_input, model_id_input, language_input], outputs=output1)
with gr.Tab("Text Summarization"):
with gr.Row():
with gr.Column():
text_input = gr.Textbox(label="Enter Text", lines=5)
model_id_input2 = gr.Textbox(label="Enter Model ID", value="facebook/bart-large-cnn")
submit_btn2 = gr.Button("Summarize")
with gr.Column():
output2 = gr.Textbox(label="Summary")
submit_btn2.click(summarization, inputs=[text_input, model_id_input2], outputs=output2)
with gr.Tab("Long Text Summarization"):
with gr.Row():
with gr.Column():
long_text_input = gr.Textbox(label="Enter Long Text", lines=10)
model_id_input3 = gr.Textbox(label="Enter Model ID", value="facebook/bart-large-cnn")
submit_btn3 = gr.Button("Summarize Long Text")
with gr.Column():
output3 = gr.Textbox(label="Long Text Summary")
submit_btn3.click(long_text_summarization, inputs=[long_text_input, model_id_input3], outputs=output3)
with gr.Tab("Speaker Diarization"):
with gr.Row():
with gr.Column():
url_input2 = gr.Textbox(label="Enter YouTube URL")
model_id_input4 = gr.Textbox(label="Enter Model ID")
num_speakers = gr.Number(label="Number of Speakers", value=2)
min_speakers = gr.Number(label="Min Speakers", value=1)
max_speakers = gr.Number(label="Max Speakers", value=4)
device = gr.Textbox(label="Device", value="cpu")
submit_btn4 = gr.Button("Diarize")
with gr.Column():
output4 = gr.DataFrame(headers=["Speaker", "Text"], datatype=["str", "str"])
submit_btn4.click(
speaker_diarization,
inputs=[url_input2, model_id_input4, device, num_speakers, min_speakers, max_speakers],
outputs=output4)
with gr.Tab("Text to Speech"):
with gr.Row():
with gr.Column():
text_input2 = gr.Textbox(label="Enter Text", lines=3)
model_id_input5 = gr.Textbox(label="Enter Model ID", value="suno/bark")
voice_preset = gr.Textbox(label="Voice Preset", value="v2/en_speaker_6")
submit_btn5 = gr.Button("Generate Audio")
with gr.Column():
output5 = gr.Audio(label="Generated Audio")
submit_btn5.click(
text2spech_bark, inputs=[text_input2, model_id_input5, voice_preset], outputs=output5)
with gr.Tab("Whisper Autocaption"):
with gr.Row():
with gr.Column():
url_input3 = gr.Textbox(label="Enter YouTube URL")
language = gr.Textbox(label="Language", value="en")
model_id_input6 = gr.Textbox(label="Enter Model ID", value="openai/whisper-large-v2")
submit_btn6 = gr.Button("Generate Captions")
with gr.Column():
output6 = gr.Video(label="Captioned Video")
submit_btn6.click(
whisper_autocaption, inputs=[url_input3, language, model_id_input6], outputs=output6)
demo.launch() |