Integrated speech transcription

app.py

@@ -6,10 +6,10 @@ from fastapi import FastAPI, File, UploadFile
 from pydantic import BaseModel
 from fastapi.middleware.cors import CORSMiddleware
 from langchain_community.document_loaders import UnstructuredURLLoader
+from src.api.speech_api import speech_translator_router
 
+app = FastAPI(title="ConversAI", root_path="/api/v1")
 
-
-app = FastAPI(title = "ConversAI", root_path = "/api/v1")
 app.add_middleware(
  CORSMiddleware,
  allow_origins=["*"],
@@ -18,29 +18,33 @@ app.add_middleware(
  allow_headers=["*"],
 )
 
+app.include_router(speech_translator_router, prefix="/speech")
+
+
 @app.post("/signup")
 async def signup(username: str, password: str):
- response = createUser(username = username, password = password)
+ response = createUser(username=username, password=password)
  return response
 
 
 @app.post("/login")
 async def login(username: str, password: str):
- response = matchPassword(username = username, password = password)
+ response = matchPassword(username=username, password=password)
  return response
 
 
 @app.post("/newChatbot")
 async def newChatbot(chatbotName: str, username: str):
- currentBotCount = len(listTables(username = username)["output"])
- limit = client.table("ConversAI_UserConfig").select("chatbotLimit").eq("username", username).execute().data[0]["chatbotLimit"]
+ currentBotCount = len(listTables(username=username)["output"])
+ limit = client.table("ConversAI_UserConfig").select("chatbotLimit").eq("username", username).execute().data[0][
+ "chatbotLimit"]
  if currentBotCount >= int(limit):
  return {
  "output": "CHATBOT LIMIT EXCEEDED"
  }
  client.table("ConversAI_ChatbotInfo").insert({"username": username, "chatbotname": chatbotName}).execute()
  chatbotName = f"convai-{username}-{chatbotName}"
- return createTable(tablename = chatbotName)
+ return createTable(tablename=chatbotName)
 
 
 @app.post("/addPDF")
@@ -53,11 +57,13 @@ async def addPDFData(vectorstore: str, pdf: UploadFile = File(...)):
  username, chatbotname = vectorstore.split("-")[1], vectorstore.split("-")[2]
  df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
  currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
- limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0]["tokenLimit"]
+ limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0][
+ "tokenLimit"]
  newCount = currentCount + len(text)
  if newCount < int(limit):
- client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq("chatbotname", chatbotname).execute()
- return addDocuments(text = text, vectorstore = vectorstore)
+ client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq(
+ "chatbotname", chatbotname).execute()
+ return addDocuments(text=text, vectorstore=vectorstore)
  else:
  return {
  "output": "DOCUMENT EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
@@ -67,7 +73,7 @@ async def addPDFData(vectorstore: str, pdf: UploadFile = File(...)):
 @app.post("/scanAndReturnText")
 async def returnText(pdf: UploadFile = File(...)):
  pdf = await pdf.read()
- text = getTextFromImagePDF(pdfBytes = pdf)
+ text = getTextFromImagePDF(pdfBytes=pdf)
  return text
 
 
@@ -77,10 +83,12 @@ async def addText(vectorstore: str, text: str):
  df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
  currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
  newCount = currentCount + len(text)
- limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0]["tokenLimit"]
+ limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0][
+ "tokenLimit"]
  if newCount < int(limit):
- client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq("chatbotname", chatbotname).execute()
- return addDocuments(text = text, vectorstore = vectorstore)
+ client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq(
+ "chatbotname", chatbotname).execute()
+ return addDocuments(text=text, vectorstore=vectorstore)
  else:
  return {
  "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
@@ -100,10 +108,12 @@ async def addText(addQaPair: AddQAPair):
  currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
  qa = f"QUESTION: {addQaPair.question}\tANSWER: {addQaPair.answer}"
  newCount = currentCount + len(qa)
- limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0]["tokenLimit"]
+ limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0][
+ "tokenLimit"]
  if newCount < int(limit):
- client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq("chatbotname", chatbotname).execute()
- return addDocuments(text = qa, vectorstore = addQaPair.vectorstore)
+ client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq(
+ "chatbotname", chatbotname).execute()
+ return addDocuments(text=qa, vectorstore=addQaPair.vectorstore)
  else:
  return {
  "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
@@ -115,20 +125,24 @@ async def addWebsite(vectorstore: str, websiteUrls: list[str]):
  urls = websiteUrls
  loader = UnstructuredURLLoader(urls=urls)
  docs = loader.load()
- text = "\n\n".join([f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])
+ text = "\n\n".join(
+ [f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])
  username, chatbotname = vectorstore.split("-")[1], vectorstore.split("-")[2]
  df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
  currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
  newCount = currentCount + len(text)
- limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0]["tokenLimit"]
+ limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0][
+ "tokenLimit"]
  if newCount < int(limit):
- client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq("chatbotname", chatbotname).execute()
- return addDocuments(text = text, vectorstore = vectorstore)
+ client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq(
+ "chatbotname", chatbotname).execute()
+ return addDocuments(text=text, vectorstore=vectorstore)
  else:
  return {
  "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
  }
 
+
 @app.post("/answerQuery")
 async def answerQuestion(query: str, vectorstore: str, llmModel: str = "llama3-70b-8192"):
  return answerQuery(query=query, vectorstore=vectorstore, llmModel=llmModel)
@@ -140,15 +154,18 @@ async def delete(chatbotName: str):
  client.table('ConversAI_ChatbotInfo').delete().eq('username', username).eq('chatbotname', chatbotName).execute()
  return deleteTable(tableName=chatbotName)
 
+
 @app.post("/listChatbots")
 async def delete(username: str):
  return listTables(username=username)
 
+
 @app.post("/getLinks")
 async def crawlUrl(baseUrl: str):
  return {
  "urls": getLinks(url=baseUrl, timeout=30)
- }
+ }
+
 
 @app.post("/getCurrentCount")
 async def getCount(vectorstore: str):
@@ -156,11 +173,12 @@ async def getCount(vectorstore: str):
  df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
  return {
  "currentCount": df[(df['username'] == username) & (df['chatbotname'] == chatbotName)]['charactercount'].iloc[0]
- }
+ }
+
 
 @app.post("/getYoutubeTranscript")
 async def getYTTranscript(urls: str):
- return getTranscript(urls = urls)
+ return getTranscript(urls=urls)
 
 
 @app.post("/analyzeData")
@@ -169,10 +187,10 @@ async def analyzeAndAnswer(query: str, file: UploadFile = File(...)):
  try:
  if extension in ["xls", "xlsx", "xlsm", "xlsb"]:
  df = pd.read_excel(io.BytesIO(await file.read()))
- response = analyzeData(query = query, dataframe = df)
+ response = analyzeData(query=query, dataframe=df)
  elif extension == "csv":
  df = pd.read_csv(io.BytesIO(await file.read()))
- response = analyzeData(query = query, dataframe = df)
+ response = analyzeData(query=query, dataframe=df)
  else:
  response = "INVALID FILE TYPE"
  return {
@@ -181,4 +199,4 @@ async def analyzeAndAnswer(query: str, file: UploadFile = File(...)):
  except:
  return {
  "output": "UNABLE TO ANSWER QUERY"
- }
+ }

requirements.txt

@@ -24,4 +24,5 @@ pdf2image
 sentence-transformers
 supabase
 unstructured
-urllib3
+urllib3
+gtts

src/__init__.py

@@ -0,0 +1,19 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+
+import logging.config
+import yaml
+import os
+
+if os.path.exists("logs"):
+ pass
+else:
+ os.makedirs("logs")
+
+log_config_path = os.path.join(os.getcwd(), "logging_config.yaml")
+with open(log_config_path, 'r') as file:
+ config = yaml.safe_load(file.read())
+
+logging.config.dictConfig(config)

src/api/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/api/speech_api.py

@@ -0,0 +1,195 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import os
+import tempfile
+from fastapi.responses import JSONResponse
+from fastapi import Form
+from fastapi import UploadFile, HTTPException, status
+from src.models.models import TextToSpeechRequest
+from fastapi.routing import APIRouter
+from src.pipeline.speech_transcription_pipeline import SpeechTranscriptionPipeline
+from src import logging
+
+speech_translator_router = APIRouter(tags=["SpeechTranscription"])
+pipeline = SpeechTranscriptionPipeline()
+
+
+@speech_translator_router.post(
+ "/text_to_speech",
+ description="""
+ ** For language refer below points**
+ **Supported Locales:**
+
+ - **English:**
+ - **Australia:** 
+ - **Language:** en
+ - **TLD:** com.au
+ - **United Kingdom:**
+ - **Language:** en
+ - **TLD:** co.uk
+ - **United States:**
+ - **Language:** en
+ - **TLD:** us
+ - **Canada:**
+ - **Language:** en
+ - **TLD:** ca
+ - **India:**
+ - **Language:** en
+ - **TLD:** co.in
+ - **Ireland:**
+ - **Language:** en
+ - **TLD:** ie
+ - **South Africa:**
+ - **Language:** en
+ - **TLD:** co.za
+ - **Nigeria:**
+ - **Language:** en
+ - **TLD:** com.ng
+
+ - **French:**
+ - **Canada:** 
+ - **Language:** fr
+ - **TLD:** ca
+ - **France:**
+ - **Language:** fr
+ - **TLD:** fr
+
+ - **Mandarin:**
+ - **China Mainland:** 
+ - **Language:** zh-CN
+ - **TLD:** any
+ - **Taiwan:**
+ - **Language:** zh-TW
+ - **TLD:** any
+
+ - **Portuguese:**
+ - **Brazil:** 
+ - **Language:** pt
+ - **TLD:** com.br
+ - **Portugal:**
+ - **Language:** pt
+ - **TLD:** pt
+
+ - **Spanish:**
+ - **Mexico:** 
+ - **Language:** es
+ - **TLD:** com.mx
+ - **Spain:**
+ - **Language:** es
+ - **TLD:** es
+ - **United States:**
+ - **Language:** es
+ - **TLD:** us
+ """
+)
+async def text_to_speech(request: TextToSpeechRequest):
+ logging.info(f"Text to speech request received")
+ try:
+ audio_bytes = pipeline.text_to_speech(request.text, request.lang, request.tld)
+ if not audio_bytes:
+ logging.error(f"Audio generation failed.")
+ raise ValueError("Audio generation failed.")
+ logging.info(f"Text to speech request processed successfully")
+ return JSONResponse(content={"audio": audio_bytes, "status_code": status.HTTP_200_OK}, status_code=200)
+ except ValueError as ve:
+ logging.error(f"Error processing text to speech request: {str(ve)}")
+ raise HTTPException(status_code=400, detail=str(ve))
+ except Exception as e:
+ logging.error(f"Internal Server Error: {str(e)}")
+ raise HTTPException(status_code=500, detail="Internal Server Error")
+
+
+@speech_translator_router.post(
+ "/speech_to_text",
+ description="""
+ ** Specify the language used in the audio **
+ **Supported Languages:**
+
+ **Major Languages:**
+ - **English:** en
+ - **Mandarin Chinese:** zh
+ - **Spanish:** es
+ - **French:** fr
+ - **German:** de
+ - **Italian:** it
+ - **Japanese:** ja
+ - **Korean:** ko
+ - **Russian:** ru
+ - **Portuguese:** pt
+ - **Arabic:** ar
+
+ **Additional Languages:**
+
+ - **Indic Languages:**
+ - **Hindi:** hi
+ - **Bengali:** bn
+ - **Tamil:** ta
+ - **Telugu:** te
+
+ - **Southeast Asian Languages:**
+ - **Vietnamese:** vi
+ - **Thai:** th
+ - **Indonesian:** id
+ - **Malay:** ms
+
+ - **African Languages:**
+ - **Swahili:** sw
+ - **Yoruba:** yo
+ - **Hausa:** ha
+
+ - **European Languages:**
+ - **Polish:** pl
+ - **Dutch:** nl
+ - **Swedish:** sv
+ - **Norwegian:** no
+ """
+)
+async def speech_to_text(audio: UploadFile, lang: str = Form(...)):
+ logging.info(f"Speech to text request received")
+ try:
+ audio_bytes = await audio.read()
+ if not audio_bytes:
+ logging.error(f"Empty audio file")
+ raise ValueError("Empty audio file")
+ except Exception as e:
+ logging.error(f"Invalid audio file {e}")
+ raise HTTPException(
+ status_code=status.HTTP_400_BAD_REQUEST,
+ detail="Invalid audio file"
+ )
+
+ try:
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
+ temp_audio_file.write(audio_bytes)
+ temp_audio_file_path = temp_audio_file.name
+ logging.info(f"Temporary audio file created at {temp_audio_file_path}")
+ except Exception as e:
+ logging.error(f"Could not process audio file{e}")
+ raise HTTPException(
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ detail="Could not process audio file"
+ )
+
+ try:
+ logging.info(f"Transcribing audio to text")
+ transcript = pipeline.speech_to_text(temp_audio_file_path, lang)
+ except FileNotFoundError as fnfe:
+ logging.error(f"Temporary file not found{fnfel}")
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail="Temporary file not found"
+ )
+ except Exception as e:
+ logging.error(f"Error processing speech-to-text: {str(e)}")
+ raise HTTPException(
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ detail="Error processing speech-to-text"
+ )
+ finally:
+ logging.info(f"Cleaning up temporary audio file")
+ if os.path.exists(temp_audio_file_path):
+ os.remove(temp_audio_file_path)
+
+ return JSONResponse(content={"transcript": transcript, "status_code": status.HTTP_200_OK}, status_code=200)

src/components/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/components/speech_to_text.py

@@ -0,0 +1,48 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import torch
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
+
+
+class SpeechToText:
+ def __init__(self):
+ self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
+ self.torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+
+ model_id = "openai/whisper-large-v3"
+
+ self.model = AutoModelForSpeechSeq2Seq.from_pretrained(
+ model_id, torch_dtype=self.torch_dtype, low_cpu_mem_usage=True, use_safetensors=True
+ ).to(self.device)
+ self.processor = AutoProcessor.from_pretrained(model_id)
+ self.speech_to_text_pipeline = self.pipeline()
+
+ def pipeline(self):
+ pipe = pipeline(
+ "automatic-speech-recognition",
+ model=self.model,
+ tokenizer=self.processor.tokenizer,
+ feature_extractor=self.processor.feature_extractor,
+ max_new_tokens=128, # max number of tokens to generate at a time
+ chunk_length_s=30, # length of audio chunks to process at a time
+ batch_size=16, # number of chunks to process at a time
+ return_timestamps=True,
+ torch_dtype=self.torch_dtype,
+ device=self.device,
+
+ )
+ return pipe
+
+ def transcribe_audio(self, audio, language: str = "en"):
+ """
+ This function is for transcribing audio to text.
+ :param audio: upload your audio file
+ :param language: choose the languaage of the audio file
+ :return:
+ """
+
+ result = self.speech_to_text_pipeline(audio, return_timestamps=True,
+ generate_kwargs={"language": language, "task": "translate"})
+ return result["chunks"], result["text"]

src/components/text_to_speech_gtts.py

@@ -0,0 +1,33 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import base64
+from io import BytesIO
+
+from gtts import gTTS
+from gtts.tokenizer import pre_processors
+
+
+class TextToSpeech:
+ def __init__(self):
+ self.preprocessing = [pre_processors.tone_marks, pre_processors.end_of_line, pre_processors.word_sub,
+ pre_processors.abbreviations]
+
+ def conversion(self, text: str, lang: str, tld: str) -> str:
+ """
+ Convert text to speech and return the Base64-encoded MP3 data.
+ :param text: The text to convert to speech.
+ :param lang: The language in which to convert the text.
+ :return: Base64-encoded MP3 data as a string.
+ """
+ tts = gTTS(text=text, lang=lang, slow=False, tld=tld, pre_processor_funcs=self.preprocessing)
+ mp3_fp = BytesIO()
+ tts.write_to_fp(mp3_fp)
+ mp3_fp.seek(0)
+
+ mp3_binary = mp3_fp.getvalue()
+
+ base64_mp3 = base64.b64encode(mp3_binary).decode("utf-8")
+
+ return base64_mp3

src/models/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/models/models.py

@@ -0,0 +1,16 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from fastapi import UploadFile
+from pydantic import BaseModel
+
+
+class TextToSpeechRequest(BaseModel):
+ text: str
+ lang: str
+ tld: str
+
+
+class SpeechToTextRequest(BaseModel):
+ lang: str

src/pipeline/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/pipeline/speech_transcription_pipeline.py

@@ -0,0 +1,32 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from src.components.speech_to_text import SpeechToText
+from src.components.text_to_speech_gtts import TextToSpeech
+
+
+class SpeechTranscriptionPipeline:
+ def __init__(self):
+ self.speech_to_text_ = SpeechToText()
+ self.text_to_speech_ = TextToSpeech()
+
+ def text_to_speech(self, text: str, lang: str, tld: str) -> str:
+ """
+ Convert text to speech.
+ :param text: The text to convert to speech.
+ :param lang: The language in which to convert the text.
+ :return: The speech representation of the text.
+ """
+ speech = self.text_to_speech_.conversion(text, lang, tld)
+ return speech
+
+ def speech_to_text(self, audio, lang: str) -> str:
+ """
+ Convert speech to text.
+ :param audio: The audio data to convert to text.
+ :param lang: The language in which the audio is spoken.
+ :return: The text representation of the audio.
+ """
+ transcript_with_timestamp, transcript = self.speech_to_text_.transcribe_audio(audio=audio, language=lang)
+ return transcript

src/utils/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/utils/utils.py

@@ -0,0 +1,11 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-02
+"""
+import yaml
+
+
+def load_config(file_path):
+ with open(file_path, 'r') as file:
+ config = yaml.safe_load(file)
+ return config