initial commit

app.py

@@ -0,0 +1,82 @@
+import gradio as gr
+# from gradio import ChatMessage
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
+import numpy as np
+import librosa
+import json
+import os
+
+
+from huggingface_hub import InferenceClient
+
+hf_token = os.getenv("HF_Token")
+
+# def get_token():
+#     with open("credentials.json","r") as f:
+#         credentials = json.load(f)
+#     return credentials['token']
+
+# hf_token = get_token()
+
+client = InferenceClient(
+    "meta-llama/Meta-Llama-3-8B-Instruct",
+    token=hf_token)
+
+
+
+
+processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
+model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
+model.config.forced_decoder_ids = None
+
+def chat(audio, chat:list):
+    transcription = transcribe(audio)
+    chat.append({'role':'user','content':transcription})
+    response = client.chat_completion(
+        messages=chat,
+        max_tokens=500,
+        stream=False,
+    ).choices[0].message.content
+    chat.append({'role':'assistant','content':response})
+    return chat
+
+def transcribe(audio):
+    sr, audio = audio
+    audio = audio.astype(np.float32)
+    if len(audio.shape) > 2 and audio.shape[1] > 1:
+        audio = np.mean(audio, axis=1)
+    audio = librosa.resample(audio, orig_sr=sr, target_sr=16000)
+    input_features = processor(audio, sampling_rate=16000, return_tensors="pt").input_features 
+    predicted_ids = model.generate(input_features)
+    transcription = processor.batch_decode(predicted_ids, skip_special_tokens=False)
+    transcription = processor.tokenizer.normalize(transcription[0])
+    return transcription
+
+with gr.Blocks() as app:
+
+    chatbot = gr.Chatbot(
+        value=[{
+            'role':'System',
+            'content':'You are a helpfull assitant for an Audio based Chatbot. You are helping Users to order their notes and thoughts.'
+        }],
+        bubble_full_width=False,
+        type="messages"
+    )
+
+    with gr.Row():
+
+        audio_input = gr.Audio(
+            sources=['microphone'],
+            interactive=True,
+            scale=8
+        )
+
+        # mode_option = gr.Radio(
+        #     choices=["online", "local"],
+        #     scale=1
+        # )
+
+    # Event listener for when the audio recording stops
+    audio_input.stop_recording(fn=chat, inputs=[audio_input, chatbot], outputs=chatbot)
+
+app.launch()

requirements.txt

@@ -0,0 +1,79 @@
+aiofiles==23.2.1
+annotated-types==0.7.0
+anyio==4.4.0
+audioread==3.0.1
+certifi==2024.7.4
+cffi==1.17.0
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.2.1
+cycler==0.12.1
+decorator==5.1.1
+exceptiongroup==1.2.2
+fastapi==0.112.1
+ffmpy==0.4.0
+filelock==3.15.4
+fonttools==4.53.1
+fsspec==2024.6.1
+gradio==4.41.0
+gradio_client==1.3.0
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.24.5
+idna==3.7
+importlib_resources==6.4.2
+Jinja2==3.1.4
+joblib==1.4.2
+kiwisolver==1.4.5
+lazy_loader==0.4
+librosa==0.10.2.post1
+llvmlite==0.43.0
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.9.2
+mdurl==0.1.2
+msgpack==1.0.8
+numba==0.60.0
+numpy==2.0.1
+orjson==3.10.7
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+platformdirs==4.2.2
+pooch==1.8.2
+pycparser==2.22
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.2
+regex==2024.7.24
+requests==2.32.3
+rich==13.7.1
+ruff==0.6.0
+safetensors==0.4.4
+scikit-learn==1.5.1
+scipy==1.14.0
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+soundfile==0.12.1
+soxr==0.4.0
+starlette==0.38.2
+threadpoolctl==3.5.0
+tokenizers==0.19.1
+tomlkit==0.12.0
+tqdm==4.66.5
+transformers==4.44.0
+typer==0.12.3
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.2
+uvicorn==0.30.6
+websockets==12.0