An ugly solution for the transcription delay problem

api/audio.py

@@ -92,27 +92,50 @@ class STTManager:
         return np.array([], dtype=np.int16), audio_buffer
 
     def transcribe_audio(self, audio: np.ndarray, text: str = "") -> str:
+        """
+        Convert speech to text from a full audio segment.
+
+        :param audio: Numpy array containing audio data.
+        :param text: Text message to add.
+        :return: Transcribed text.
+        """
+
         if len(audio) < 500:
             return text
         else:
             transcript = self.transcribe_numpy_array(audio, context=text)
             return text + " " + transcript
 
-    def transcribe_and_add_to_chat(self, audio: np.ndarray, chat: List[List[Optional[str]]]) -> List[List[Optional[str]]]:
+    def add_to_chat(self, text: str, chat: List[List[Optional[str]]], editable_chat: bool = True) -> List[List[Optional[str]]]:
         """
-        Transcribe audio and add the transcription to the chat history.
+        Add a text message to the chat history.
 
-        :param audio: Numpy array containing audio data.
+        :param text: Text message to add.
         :param chat: List of chat messages.
         :return: Updated chat history.
         """
+        if not editable_chat or len(text) == 0:
+            return chat
+
         if len(chat) == 0 or chat[-1][0] is None:
             chat.append(["", None])
 
-        chat[-1][0] = self.transcribe_audio(audio, chat[-1][0])
+        chat[-1][0] = text
 
         return chat
 
+    def transcribe_and_add_to_chat(self, audio: np.ndarray, chat: List[List[Optional[str]]]) -> List[List[Optional[str]]]:
+        """
+        Transcribe audio and add the transcription to the chat history.
+
+        :param audio: Numpy array containing audio data.
+        :param chat: List of chat messages.
+        :return: Updated chat history.
+        """
+        text = self.transcribe_audio(audio)
+        chat = self.add_to_chat(text, chat)
+        return chat
+
     def transcribe_numpy_array(self, audio: np.ndarray, context: Optional[str] = None) -> str:
         """
         Convert speech to text from a full audio segment.

ui/coding.py

@@ -251,21 +251,53 @@ def get_problem_solving_ui(llm: LLMManager, tts: TTSManager, stt: STTManager, de
             fn=llm.end_interview, inputs=[description, chat_history, interview_type_select], outputs=[feedback]
         )
 
-        # TODO: add a counter for audio chunks to use for better delay handling
-        audio_counter = 0
+        is_recording = gr.State(False)
+        audio_input.start_recording(fn=lambda: True, outputs=[is_recording])
+
+        hidden_text = gr.State("")
+        is_transcribing = gr.State(False)
         audio_input.stream(
             stt.process_audio_chunk,
             inputs=[audio_input, audio_buffer],
             outputs=[audio_buffer, audio_to_transcribe],
             show_progress="hidden",
-        ).success(fn=stt.transcribe_and_add_to_chat, inputs=[audio_to_transcribe, chat], outputs=[chat], show_progress="hidden")
+        ).success(fn=lambda: True, outputs=[is_transcribing]).success(
+            fn=stt.transcribe_audio, inputs=[audio_to_transcribe, hidden_text], outputs=[hidden_text], show_progress="full"
+        ).success(
+            fn=stt.add_to_chat, inputs=[hidden_text, chat, is_recording], outputs=[chat], show_progress="full"
+        ).success(
+            fn=lambda: False, outputs=[is_transcribing]
+        )
 
-        # TODO: find a way to remove a delay
-        audio_input.stop_recording(fn=lambda: time.sleep(2)).success(
+        # Ugly but works, need to clean up the code and find a better way to handle the logic
+        # Main problem - we need to wait until the last chunk of audio is transcribed before sending the request
+        # The same time I don't want to have a fixed delay by default
+        # I didn't find a native way of gradio to handle this, so I used a workaround
+        # There should be a better way to handle this, but I didn't find it yet
+        # The solution below keeps waiting 0.5 second up to 8 times until the audio is transcribed
+        audio_input.stop_recording(fn=lambda x: time.sleep(int(x) / 2), inputs=[is_transcribing]).success(
+            fn=lambda x: time.sleep(int(x) / 2), inputs=[is_transcribing]
+        ).success(fn=lambda x: time.sleep(int(x) / 2), inputs=[is_transcribing]).success(
+            fn=lambda x: time.sleep(int(x) / 2), inputs=[is_transcribing]
+        ).success(
+            fn=lambda x: time.sleep(int(x) / 2), inputs=[is_transcribing]
+        ).success(
+            fn=lambda x: time.sleep(int(x) / 2), inputs=[is_transcribing]
+        ).success(
+            fn=lambda x: time.sleep(int(x) / 2), inputs=[is_transcribing]
+        ).success(
+            fn=lambda x: time.sleep(int(x) / 2), inputs=[is_transcribing]
+        ).success(
+            fn=lambda: False, outputs=[is_recording]
+        ).success(
             fn=send_request_partial,
             inputs=[code, previous_code, chat_history, chat],
             outputs=[chat_history, chat, previous_code, audio_output],
-        ).success(fn=lambda: np.array([], dtype=np.int16), outputs=[audio_buffer])
+        ).success(
+            fn=lambda: np.array([], dtype=np.int16), outputs=[audio_buffer]
+        ).success(
+            fn=lambda: "", outputs=[hidden_text]
+        )
 
         interview_type_select.change(
             fn=lambda x: gr.update(choices=topic_lists[x], value=np.random.choice(topic_lists[x])),

ui/instructions.py

@@ -33,7 +33,7 @@ This section is where the interaction happens:
 - **Code/Solution Area**: On the left side, you will find a space to write your solution. For codding problem you can use any language, although syntax highlighting is only available for Python and SQL.
 - **Communication Area**: On the right, this area includes:
   - **Chat History**: Displays the entire dialogue history, showing messages from both you and the AI interviewer. Your recognized speech will be shown here before being sent to the AI.
-  - **Audio Record Button**: Use this button to record your responses. Press to start recording, speak your thoughts, and press stop to send your audio. Your message will be sent to the chat, along with a snapshot of your code or any notes from solution text area."
+  - **Audio Record Button**: Use this button to record your responses. Press to start recording, speak your thoughts, and press stop to send your audio. Wait until everything you said is transcribed and then click stop. Your message will be sent to the chat, along with a snapshot of your code or any notes from solution text area."
 
 Engage with the AI as you would with a real interviewer. Provide concise responses and frequent updates rather than long monologues. Your interactions, including any commentary on your code, will be recorded and the AI's responses will be read aloud and displayed in the chat. Follow the AI's instructions and respond to any follow-up questions as they arise.