Upload 5 files

README.txt

@@ -0,0 +1,13 @@
+---
+title: 📹NLP Video Transcript SL📝
+emoji: 📹📝
+colorFrom: red
+colorTo: white
+sdk: streamlit
+sdk_version: 1.2.0
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,119 @@
+from collections import deque
+import streamlit as st
+import torch
+from streamlit_player import st_player
+from transformers import AutoModelForCTC, Wav2Vec2Processor
+from streaming import ffmpeg_stream
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+player_options = {
+    "events": ["onProgress"],
+    "progress_interval": 200,
+    "volume": 1.0,
+    "playing": True,
+    "loop": False,
+    "controls": False,
+    "muted": False,
+    "config": {"youtube": {"playerVars": {"start": 1}}},
+}
+
+# disable rapid fading in and out on `st.code` updates
+st.markdown("<style>.element-container{opacity:1 !important}</style>", unsafe_allow_html=True)
+
+@st.cache(hash_funcs={torch.nn.parameter.Parameter: lambda _: None})
+def load_model(model_path="facebook/wav2vec2-large-robust-ft-swbd-300h"):
+    processor = Wav2Vec2Processor.from_pretrained(model_path)
+    model = AutoModelForCTC.from_pretrained(model_path).to(device)
+    return processor, model
+    
+processor, model = load_model()
+
+def stream_text(url, chunk_duration_ms, pad_duration_ms):
+    sampling_rate = processor.feature_extractor.sampling_rate
+
+    # calculate the length of logits to cut from the sides of the output to account for input padding
+    output_pad_len = model._get_feat_extract_output_lengths(int(sampling_rate * pad_duration_ms / 1000))
+
+    # define the audio chunk generator
+    stream = ffmpeg_stream(url, sampling_rate, chunk_duration_ms=chunk_duration_ms, pad_duration_ms=pad_duration_ms)
+
+    leftover_text = ""
+    for i, chunk in enumerate(stream):
+        input_values = processor(chunk, sampling_rate=sampling_rate, return_tensors="pt").input_values
+
+        with torch.no_grad():
+            logits = model(input_values.to(device)).logits[0]
+            if i > 0:
+                logits = logits[output_pad_len : len(logits) - output_pad_len]
+            else:  # don't count padding at the start of the clip
+                logits = logits[: len(logits) - output_pad_len]
+
+            predicted_ids = torch.argmax(logits, dim=-1).cpu().tolist()
+            if processor.decode(predicted_ids).strip():
+                leftover_ids = processor.tokenizer.encode(leftover_text)
+                # concat the last word (or its part) from the last frame with the current text
+                text = processor.decode(leftover_ids + predicted_ids)
+                # don't return the last word in case it's just partially recognized
+                text, leftover_text = text.rsplit(" ", 1)
+                yield text
+            else:
+                yield leftover_text
+                leftover_text = ""
+    yield leftover_text
+
+def main():
+    state = st.session_state
+    st.header("Video ASR Streamlit from Youtube Link")
+
+    with st.form(key="inputs_form"):
+    
+        # Our worlds best teachers on subjects of AI, Cognitive, Neuroscience for our Behavioral and Medical Health
+        ytJoschaBach="https://youtu.be/cC1HszE5Hcw?list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&t=8984"
+        ytSamHarris="https://www.youtube.com/watch?v=4dC_nRYIDZU&list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&index=2"
+        ytJohnAbramson="https://www.youtube.com/watch?v=arrokG3wCdE&list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&index=3"
+        ytElonMusk="https://www.youtube.com/watch?v=DxREm3s1scA&list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&index=4"
+        ytJeffreyShainline="https://www.youtube.com/watch?v=EwueqdgIvq4&list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&index=5"
+        ytJeffHawkins="https://www.youtube.com/watch?v=Z1KwkpTUbkg&list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&index=6"
+        ytSamHarris="https://youtu.be/Ui38ZzTymDY?list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L"
+        ytSamHarris="https://youtu.be/4dC_nRYIDZU?list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&t=7809"
+        ytSamHarris="https://youtu.be/4dC_nRYIDZU?list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&t=7809"
+        ytSamHarris="https://youtu.be/4dC_nRYIDZU?list=PLHgX2IExbFouJoqEr8JMF5MbZSbyC91-L&t=7809"
+        ytTimelapseAI="https://www.youtube.com/watch?v=63yr9dlI0cU&list=PLHgX2IExbFovQybyfltywXnqZi5YvaSS-"
+        state.youtube_url = st.text_input("YouTube URL", ytTimelapseAI)
+        
+        
+        state.chunk_duration_ms = st.slider("Audio chunk duration (ms)", 2000, 10000, 3000, 100)
+        state.pad_duration_ms = st.slider("Padding duration (ms)", 100, 5000, 1000, 100)
+        submit_button = st.form_submit_button(label="Submit")
+
+    if submit_button or "asr_stream" not in state:
+        # a hack to update the video player on value changes
+        state.youtube_url = (
+            state.youtube_url.split("&hash=")[0]
+            + f"&hash={state.chunk_duration_ms}-{state.pad_duration_ms}"
+        )
+        state.asr_stream = stream_text(
+            state.youtube_url, state.chunk_duration_ms, state.pad_duration_ms
+        )
+        state.chunks_taken = 0
+        
+        
+        state.lines = deque([], maxlen=100)  # limit to the last n lines of subs
+        
+
+    player = st_player(state.youtube_url, **player_options, key="youtube_player")
+
+    if "asr_stream" in state and player.data and player.data["played"] < 1.0:
+        # check how many seconds were played, and if more than processed - write the next text chunk
+        processed_seconds = state.chunks_taken * (state.chunk_duration_ms / 1000)
+        if processed_seconds < player.data["playedSeconds"]:
+            text = next(state.asr_stream)
+            state.lines.append(text)
+            state.chunks_taken += 1
+    if "lines" in state:
+        # print the lines of subs
+        st.code("\n".join(state.lines))
+
+
+if __name__ == "__main__":
+    main()

packages.txt

@@ -0,0 +1 @@
+ffmpeg

requirements.txt

@@ -0,0 +1,5 @@
+--find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+torch==1.10.0+cpu
+transformers
+streamlit-player
+yt-dlp

streaming.py

@@ -0,0 +1,66 @@
+import subprocess
+
+import numpy as np
+
+
+def ffmpeg_stream(youtube_url, sampling_rate=16_000, chunk_duration_ms=5000, pad_duration_ms=200):
+    """
+    Helper function to read an audio file through ffmpeg.
+    """
+    chunk_len = int(sampling_rate * chunk_duration_ms / 1000)
+    pad_len = int(sampling_rate * pad_duration_ms / 1000)
+    read_chunk_len = chunk_len + pad_len * 2
+
+    ar = f"{sampling_rate}"
+    ac = "1"
+    format_for_conversion = "f32le"
+    dtype = np.float32
+    size_of_sample = 4
+
+    ffmpeg_command = [
+        "ffmpeg",
+        "-i",
+        "pipe:",
+        "-ac",
+        ac,
+        "-ar",
+        ar,
+        "-f",
+        format_for_conversion,
+        "-hide_banner",
+        "-loglevel",
+        "quiet",
+        "pipe:1",
+    ]
+
+    ytdl_command = ["yt-dlp", "-f", "bestaudio", youtube_url, "--quiet", "-o", "-"]
+
+    try:
+        ffmpeg_process = subprocess.Popen(ffmpeg_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=-1)
+        ytdl_process = subprocess.Popen(ytdl_command, stdout=ffmpeg_process.stdin)
+    except FileNotFoundError:
+        raise ValueError("ffmpeg was not found but is required to stream audio files from filename")
+
+    acc = b""
+    leftover = np.zeros((0,), dtype=np.float32)
+    while ytdl_process.poll() is None:
+        buflen = read_chunk_len * size_of_sample
+
+        raw = ffmpeg_process.stdout.read(buflen)
+        if raw == b"":
+            break
+
+        if len(acc) + len(raw) > buflen:
+            acc = raw
+        else:
+            acc += raw
+
+        audio = np.frombuffer(acc, dtype=dtype)
+        audio = np.concatenate([leftover, audio])
+        if len(audio) < pad_len * 2:
+            # TODO: handle end of stream better than this
+            break
+        yield audio
+
+        leftover = audio[-pad_len * 2 :]
+        read_chunk_len = chunk_len