Spaces:
Runtime error
Runtime error
# Libraries | |
import gradio as gr | |
import whisper | |
from pytube import YouTube | |
from transformers import pipeline, T5Tokenizer, T5ForConditionalGeneration, AutoTokenizer, AutoModelForSeq2SeqLM | |
import torch | |
from wordcloud import WordCloud | |
import re | |
import os | |
class GradioInference: | |
def __init__(self): | |
# OpenAI's Whisper model sizes | |
self.sizes = list(whisper._MODELS.keys()) | |
# Whisper's available languages for ASR | |
self.langs = ["none"] + sorted(list(whisper.tokenizer.LANGUAGES.values())) | |
# Default size | |
self.current_size = "base" | |
# Default model size | |
self.loaded_model = whisper.load_model(self.current_size) | |
# Initialize Pytube Object | |
self.yt = None | |
# Initialize summary model for English | |
self.bart_summarizer = pipeline("summarization", model="facebook/bart-large-cnn", truncation=True) | |
# Initialize Multilingual summary model | |
self.mt5_tokenizer = AutoTokenizer.from_pretrained("csebuetnlp/mT5_multilingual_XLSum", truncation=True) | |
self.mt5_model = AutoModelForSeq2SeqLM.from_pretrained("csebuetnlp/mT5_multilingual_XLSum") | |
# Initialize VoiceLabT5 model and tokenizer | |
self.keyword_model = T5ForConditionalGeneration.from_pretrained( | |
"Voicelab/vlt5-base-keywords" | |
) | |
self.keyword_tokenizer = T5Tokenizer.from_pretrained( | |
"Voicelab/vlt5-base-keywords" | |
) | |
# Sentiment Classifier | |
self.classifier = pipeline("text-classification", model="lxyuan/distilbert-base-multilingual-cased-sentiments-student", return_all_scores=False) | |
def __call__(self, link, lang, size, progress=gr.Progress()): | |
""" | |
Call the Gradio Inference python class. | |
This class gets access to a YouTube video using python's library Pytube and downloads its audio. | |
Then it uses the Whisper model to perform Automatic Speech Recognition (i.e Speech-to-Text). | |
Once the function has the transcription of the video it proccess it to obtain: | |
- Summary: using Facebook's BART transformer. | |
- KeyWords: using VoiceLabT5 keyword extractor. | |
- Sentiment Analysis: using Hugging Face's default sentiment classifier | |
- WordCloud: using the wordcloud python library. | |
""" | |
gr.Info("Starting process") | |
progress(0, desc="Starting analysis") | |
if self.yt is None: | |
self.yt = YouTube(link) | |
# Pytube library to access to YouTube audio stream | |
path = self.yt.streams.filter(only_audio=True)[0].download(filename="tmp.mp4") | |
if lang == "none": | |
lang = None | |
if size != self.current_size: | |
self.loaded_model = whisper.load_model(size) | |
self.current_size = size | |
progress(0.20, desc="Transcribing") | |
# Transcribe the audio extracted from pytube | |
results = self.loaded_model.transcribe(path, language=lang) | |
progress(0.40, desc="Summarizing") | |
# Perform summarization on the transcription | |
transcription_summary = self.bart_summarizer( | |
results["text"], | |
max_length=256, | |
min_length=30, | |
do_sample=False, | |
truncation=True | |
) | |
# Multilingual summary with mt5 | |
WHITESPACE_HANDLER = lambda k: re.sub('\s+', ' ', re.sub('\n+', ' ', k.strip())) | |
input_ids_sum = self.mt5_tokenizer( | |
[WHITESPACE_HANDLER(results["text"])], | |
return_tensors="pt", | |
padding="max_length", | |
truncation=True, | |
max_length=512 | |
)["input_ids"] | |
output_ids_sum = self.mt5_model.generate( | |
input_ids=input_ids_sum, | |
max_length=256, | |
no_repeat_ngram_size=2, | |
num_beams=4 | |
)[0] | |
summary = self.mt5_tokenizer.decode( | |
output_ids_sum, | |
skip_special_tokens=True, | |
clean_up_tokenization_spaces=False | |
) | |
# End multilingual summary | |
progress(0.60, desc="Extracting Keywords") | |
# Extract keywords using VoiceLabT5 | |
task_prefix = "Keywords: " | |
input_sequence = task_prefix + results["text"] | |
input_ids = self.keyword_tokenizer( | |
input_sequence, | |
return_tensors="pt", | |
truncation=False | |
).input_ids | |
output = self.keyword_model.generate( | |
input_ids, | |
no_repeat_ngram_size=3, | |
num_beams=4 | |
) | |
predicted = self.keyword_tokenizer.decode(output[0], skip_special_tokens=True) | |
keywords = [x.strip() for x in predicted.split(",") if x.strip()] | |
formatted_keywords = "\n".join([f"โข {keyword}" for keyword in keywords]) | |
progress(0.80, desc="Extracting Sentiment") | |
# Define a dictionary to map labels to emojis | |
sentiment_emojis = { | |
"positive": "Positive ๐๐ผ", | |
"negative": "Negative ๐๐ผ", | |
"neutral": "Neutral ๐ถ", | |
} | |
# Sentiment label | |
label = self.classifier(summary)[0]["label"] | |
# Format the label with emojis | |
formatted_sentiment = sentiment_emojis.get(label, label) | |
progress(0.90, desc="Generating Wordcloud") | |
# Generate WordCloud object | |
wordcloud = WordCloud(colormap = "Oranges").generate(results["text"]) | |
# WordCloud image to display | |
wordcloud_image = wordcloud.to_image() | |
if lang == "english" or lang == "none": | |
return ( | |
results["text"], | |
transcription_summary[0]["summary_text"], | |
formatted_keywords, | |
formatted_sentiment, | |
wordcloud_image, | |
) | |
else: | |
return ( | |
results["text"], | |
summary, | |
formatted_keywords, | |
formatted_sentiment, | |
wordcloud_image, | |
) | |
def populate_metadata(self, link): | |
""" | |
Access to the YouTube video title and thumbnail image to further display it | |
params: | |
- link: a YouTube URL. | |
""" | |
if not link: | |
return None, None | |
self.yt = YouTube(link) | |
return self.yt.thumbnail_url, self.yt.title | |
def from_audio_input(self, lang, size, audio_file, progress=gr.Progress()): | |
""" | |
Call the Gradio Inference python class. | |
Uses it directly the Whisper model to perform Automatic Speech Recognition (i.e Speech-to-Text). | |
Once the function has the transcription of the video it proccess it to obtain: | |
- Summary: using Facebook's BART transformer. | |
- KeyWords: using VoiceLabT5 keyword extractor. | |
- Sentiment Analysis: using Hugging Face's default sentiment classifier | |
- WordCloud: using the wordcloud python library. | |
""" | |
gr.Info("Starting process") | |
progress(0, desc="Starting analysis") | |
if lang == "none": | |
lang = None | |
if size != self.current_size: | |
self.loaded_model = whisper.load_model(size) | |
self.current_size = size | |
progress(0.20, desc="Transcribing") | |
results = self.loaded_model.transcribe(audio_file, language=lang) | |
progress(0.40, desc="Summarizing") | |
# Perform summarization on the transcription | |
transcription_summary = self.bart_summarizer( | |
results["text"], max_length=150, min_length=30, do_sample=False, truncation=True | |
) | |
# Multilingual summary with mt5 | |
WHITESPACE_HANDLER = lambda k: re.sub('\s+', ' ', re.sub('\n+', ' ', k.strip())) | |
input_ids_sum = self.mt5_tokenizer( | |
[WHITESPACE_HANDLER(results["text"])], | |
return_tensors="pt", | |
padding="max_length", | |
truncation=True, | |
max_length=512 | |
)["input_ids"] | |
output_ids_sum = self.mt5_model.generate( | |
input_ids=input_ids_sum, | |
max_length=130, | |
no_repeat_ngram_size=2, | |
num_beams=4 | |
)[0] | |
summary = self.mt5_tokenizer.decode( | |
output_ids_sum, | |
skip_special_tokens=True, | |
clean_up_tokenization_spaces=False | |
) | |
# End multilingual summary | |
progress(0.60, desc="Extracting Keywords") | |
# Extract keywords using VoiceLabT5 | |
task_prefix = "Keywords: " | |
input_sequence = task_prefix + results["text"] | |
input_ids = self.keyword_tokenizer( | |
input_sequence, | |
return_tensors="pt", | |
truncation=False | |
).input_ids | |
output = self.keyword_model.generate( | |
input_ids, | |
no_repeat_ngram_size=3, | |
num_beams=4 | |
) | |
predicted = self.keyword_tokenizer.decode(output[0], skip_special_tokens=True) | |
keywords = [x.strip() for x in predicted.split(",") if x.strip()] | |
formatted_keywords = "\n".join([f"โข {keyword}" for keyword in keywords]) | |
progress(0.80, desc="Extracting Sentiment") | |
# Define a dictionary to map labels to emojis | |
sentiment_emojis = { | |
"positive": "Positive ๐๐ผ", | |
"negative": "Negative ๐๐ผ", | |
"neutral": "Neutral ๐ถ", | |
} | |
# Sentiment label | |
label = self.classifier(summary)[0]["label"] | |
# Format the label with emojis | |
formatted_sentiment = sentiment_emojis.get(label, label) | |
progress(0.90, desc="Generating Wordcloud") | |
# WordCloud object | |
wordcloud = WordCloud(colormap = "Oranges").generate( | |
results["text"] | |
) | |
wordcloud_image = wordcloud.to_image() | |
if lang == "english" or lang == "none": | |
return ( | |
results["text"], | |
transcription_summary[0]["summary_text"], | |
formatted_keywords, | |
formatted_sentiment, | |
wordcloud_image, | |
) | |
else: | |
return ( | |
results["text"], | |
summary, | |
formatted_keywords, | |
formatted_sentiment, | |
wordcloud_image, | |
) | |
def from_article(self, article, progress=gr.Progress()): | |
""" | |
Call the Gradio Inference python class. | |
Acepts the user's text imput, then it performs: | |
- Summary: using Facebook's BART transformer. | |
- KeyWords: using VoiceLabT5 keyword extractor. | |
- Sentiment Analysis: using Hugging Face's default sentiment classifier | |
- WordCloud: using the wordcloud python library. | |
""" | |
gr.Info("Starting process") | |
progress(0, desc="Starting analysis") | |
progress(0.30, desc="Summarizing") | |
# Perform summarization on the transcription | |
transcription_summary = self.bart_summarizer( | |
article, max_length=150, min_length=30, do_sample=False, truncation=True | |
) | |
# Multilingual summary with mt5 | |
WHITESPACE_HANDLER = lambda k: re.sub('\s+', ' ', re.sub('\n+', ' ', k.strip())) | |
input_ids_sum = self.mt5_tokenizer( | |
[WHITESPACE_HANDLER(article)], | |
return_tensors="pt", | |
padding="max_length", | |
truncation=True, | |
max_length=512 | |
)["input_ids"] | |
output_ids_sum = self.mt5_model.generate( | |
input_ids=input_ids_sum, | |
max_length=130, | |
no_repeat_ngram_size=2, | |
num_beams=4 | |
)[0] | |
summary = self.mt5_tokenizer.decode( | |
output_ids_sum, | |
skip_special_tokens=True, | |
clean_up_tokenization_spaces=False | |
) | |
# End multilingual summary | |
progress(0.60, desc="Extracting Keywords") | |
# Extract keywords using VoiceLabT5 | |
task_prefix = "Keywords: " | |
input_sequence = task_prefix + article | |
input_ids = self.keyword_tokenizer( | |
input_sequence, | |
return_tensors="pt", | |
truncation=False | |
).input_ids | |
output = self.keyword_model.generate( | |
input_ids, | |
no_repeat_ngram_size=3, | |
num_beams=4 | |
) | |
predicted = self.keyword_tokenizer.decode(output[0], skip_special_tokens=True) | |
keywords = [x.strip() for x in predicted.split(",") if x.strip()] | |
formatted_keywords = "\n".join([f"โข {keyword}" for keyword in keywords]) | |
progress(0.80, desc="Extracting Sentiment") | |
# Define a dictionary to map labels to emojis | |
sentiment_emojis = { | |
"positive": "Positive ๐๐ผ", | |
"negative": "Negative ๐๐ผ", | |
"neutral": "Neutral ๐ถ", | |
} | |
# Sentiment label | |
label = self.classifier(summary)[0]["label"] | |
# Format the label with emojis | |
formatted_sentiment = sentiment_emojis.get(label, label) | |
progress(0.90, desc="Generating Wordcloud") | |
# WordCloud object | |
wordcloud = WordCloud(colormap = "Oranges").generate( | |
article | |
) | |
wordcloud_image = wordcloud.to_image() | |
return ( | |
transcription_summary[0]["summary_text"], | |
formatted_keywords, | |
formatted_sentiment, | |
wordcloud_image, | |
) | |
gio = GradioInference() | |
title = "Media Insights" | |
description = "Your AI-powered video analytics tool" | |
theme = gr.themes.Soft(spacing_size="md", radius_size="md") | |
block = gr.Blocks(theme=theme) | |
with block as demo: | |
gr.HTML( | |
""" | |
<div style="text-align: center; max-width: 500px; margin: 0 auto;"> | |
<div> | |
<h1 style="font-family: Montserrat, sans-serif;">MEDIA <span style="color: #433ccb;">INSIGHTS</span> ๐ก</h1> | |
</div> | |
<h4> | |
Your AI-powered media analytics tool โจ | |
</h4> | |
</div> | |
""" | |
) | |
with gr.Group(): | |
with gr.Tab("From YouTube ๐น"): | |
with gr.Box(): | |
with gr.Row().style(equal_height=True): | |
size = gr.Dropdown( | |
label="Speech-to-text Model Size", choices=gio.sizes, value="base" | |
) | |
lang = gr.Dropdown( | |
label="Language (Optional)", choices=gio.langs, value="none" | |
) | |
link = gr.Textbox( | |
label="YouTube Link", placeholder="Enter YouTube link..." | |
) | |
with gr.Row().style(equal_height=True): | |
with gr.Column(variant="panel", scale=1): | |
title = gr.Label(label="Video Title") | |
img = gr.Image(label="Thumbnail").style(height=350) | |
with gr.Column(variant="panel", scale=1): | |
text = gr.Textbox( | |
label="Transcription", | |
placeholder="Transcription Output...", | |
lines=18, | |
).style(show_copy_button=True) | |
with gr.Row().style(equal_height=True): | |
summary = gr.Textbox( | |
label="Summary", placeholder="Summary Output...", lines=5 | |
).style(show_copy_button=True) | |
keywords = gr.Textbox( | |
label="Keywords", placeholder="Keywords Output...", lines=5 | |
).style(show_copy_button=True) | |
label = gr.Label(label="Sentiment Analysis") | |
wordcloud_image = gr.Image(label="WordCloud") | |
with gr.Row(): | |
btn = gr.Button("Get Video Insights ๐", variant="primary", scale=1) | |
clear = gr.ClearButton( | |
[link, title, img, text, summary, keywords, label, wordcloud_image], | |
value="Clear ๐๏ธ", scale=1 | |
) | |
btn.click( | |
gio, | |
inputs=[link, lang, size], | |
outputs=[text, summary, keywords, label, wordcloud_image], | |
) | |
link.change(gio.populate_metadata, inputs=[link], outputs=[img, title]) | |
with gr.Tab("From Audio file ๐๏ธ"): | |
with gr.Box(): | |
with gr.Row().style(equal_height=True): | |
size = gr.Dropdown( | |
label="Model Size", choices=gio.sizes, value="base" | |
) | |
lang = gr.Dropdown( | |
label="Language (Optional)", choices=gio.langs, value="none" | |
) | |
audio_file = gr.Audio(type="filepath") | |
with gr.Row().style(equal_height=True): | |
text = gr.Textbox( | |
label="Transcription", | |
placeholder="Transcription Output...", | |
lines=10, | |
).style(show_copy_button=True) | |
with gr.Row().style(equal_height=True): | |
summary = gr.Textbox( | |
label="Summary", placeholder="Summary Output...", lines=5 | |
) | |
keywords = gr.Textbox( | |
label="Keywords", placeholder="Keywords Output...", lines=5 | |
) | |
label = gr.Label(label="Sentiment Analysis") | |
wordcloud_image = gr.Image(label="WordCloud") | |
with gr.Row(): | |
btn = gr.Button( | |
"Get Audio Insights ๐", variant="primary" | |
) | |
clear = gr.ClearButton([audio_file,text, summary, keywords, label, wordcloud_image], value="Clear ๐๏ธ") | |
btn.click( | |
gio.from_audio_input, | |
inputs=[lang, size, audio_file], | |
outputs=[text, summary, keywords, label, wordcloud_image], | |
) | |
with gr.Tab("From Article ๐"): | |
with gr.Box(): | |
with gr.Row().style(equal_height=True): | |
article = gr.Textbox( | |
label="Text", | |
placeholder="Paste your text...", | |
lines=10, | |
).style(show_copy_button=True) | |
with gr.Row().style(equal_height=True): | |
summary = gr.Textbox( | |
label="Summary", placeholder="Summary Output...", lines=5 | |
) | |
keywords = gr.Textbox( | |
label="Keywords", placeholder="Keywords Output...", lines=5 | |
) | |
label = gr.Label(label="Sentiment Analysis") | |
wordcloud_image = gr.Image(label="WordCloud") | |
with gr.Row(): | |
btn = gr.Button( | |
"Get Text insights ๐", variant="primary") | |
clear = gr.ClearButton([article, summary, keywords, label, wordcloud_image], value="Clear") | |
btn.click( | |
gio.from_article, | |
inputs=[article], | |
outputs=[summary, keywords, label, wordcloud_image], | |
) | |
with block: | |
gr.Markdown("### Video Examples") | |
gr.Examples(["https://www.youtube.com/shorts/xDNzz8yAH7I", | |
"https://youtu.be/MnrJzXM7a6o", | |
"https://youtu.be/FKjj1tNcbtM"], | |
inputs=link) | |
gr.Markdown("### Audio Examples") | |
gr.Examples([[os.path.join(os.path.dirname(__file__),"audios/EnglishLecture.mp4")]], inputs=audio_file) | |
gr.Markdown("###Text Examples") | |
gr.Examples(["""Relations between India and Canada have been rocky for some time. But this week, they hit new lows. | |
After Prime Minister Justin Trudeau on Monday accused India of assassinating a Canadian Sikh in British Columbia, India angrily denied the charges. Both countries are up in arms, and the diplomatic row shows no sign of easing anytime soon. | |
We asked a pair of commentators, Barkha Dutt from India and David Moscrop from Canada, to give us a sense of the mood in their respective countries. โ Damir Marusic | |
Barkha Dutt: India and Canada stand at the precipice of a breakdown in relations. Trudeau has only himself to blame. | |
The Canadian prime ministerโs habit of turning a blind eye toward extremism in his country โ secessionists who violently advocate for Khalistan, a separate homeland for Sikhs โ has made him the most disliked world leader in India. | |
His claim that Prime Minister Narendra Modiโs Indian government is behind the murder of one such person โ Hardeep Singh Nijjar, the leader of the self-proclaimed Khalistan Tiger Force, and a designated terrorist in India โ has cemented his poor reputation. | |
Nijjar was wanted for multiple crimes in India. In 2016, Indian authorities filed a red notice with Interpol for his arrest. On June 18, he was fatally shot in the parking lot of a Sikh temple by unidentified assailants. At the time, some Canadian journalists pointed to internecine battles within Khalistani groups. โIt has all the makings of a gangland hit,โ Terry Milewski, an expert on the global Khalistan network, told me. | |
Now, almost four months later, Trudeau is declaring Nijjarโs death an assassination. To Indians, the timing is as suspect as the claim."""],inputs=article, label="News_1") | |
gr.Markdown("### About the app:") | |
with gr.Accordion("What is Media Insights?", open=False): | |
gr.Markdown( | |
"Media Insights is a tool developed for academic purposes that allows you to analyze YouTube videos, audio files or some text. It provides features like transcription, summarization, keyword extraction, sentiment analysis, and word cloud generation for multimedia content." | |
) | |
with gr.Accordion("How does Media Insights work?", open=False): | |
gr.Markdown( | |
"Media Insights leverages several powerful AI models and libraries. It uses OpenAI's Whisper for Automatic Speech Recognition (ASR) to transcribe audio content. It summarizes the transcribed text using Facebook's BART model, extracts keywords with VoiceLabT5, performs sentiment analysis with DistilBERT, and generates word clouds." | |
) | |
with gr.Accordion("What languages are supported for the analysis?", open=False): | |
gr.Markdown( | |
"Media Insights supports multiple languages for transcription and analysis. You can select your preferred language from the available options when using the app." | |
) | |
with gr.Accordion("Can I analyze audio files instead of YouTube videos?", open=False): | |
gr.Markdown( | |
"Yes, you can analyze audio files directly. Simply upload your audio file to the app, and it will provide the same transcription, summarization, keyword extraction, sentiment analysis, and word cloud generation features. In addition, you can also paste your article or text of your preference, to get all the insights directly from it." | |
) | |
with gr.Accordion("What are the different model sizes available for transcription?", open=False): | |
gr.Markdown( | |
"The app uses a Speech-to-text model that has different training sizes, from tiny to large. Hence, the bigger the model the accurate the transcription." | |
) | |
with gr.Accordion("How long does it take to analyze a video or audio file?", open=False): | |
gr.Markdown( | |
"The time taken for analysis may vary based on the duration of the video or audio file and the selected model size. Shorter content will be processed more quickly." | |
) | |
with gr.Accordion("Who developed Media Insights?" ,open=False): | |
gr.Markdown( | |
"Media Insights was developed by students as part of the 2022/23 Master's in Big Data & Data Science program at Universidad Complutense de Madrid for academic purposes (Trabajo de Fin de Master)." | |
) | |
gr.HTML( | |
""" | |
<div style="text-align: center; margin: 0 auto;"> | |
<p style="margin-bottom: 10px; font-size: 96%"> | |
Trabajo de Fin de Mรกster - Grupo 3 | |
</p> | |
<p> | |
2022/23 Master in Big Data & Data Science - Universidad Complutense de Madrid | |
</p> | |
</div> | |
""" | |
) | |
demo.launch() | |