change to vad audio translate

config.py

@@ -50,9 +50,9 @@ MAX_LENTH_ZH = 4
 WHISPER_PROMPT_EN = ""# "The following is an English sentence."
 MAX_LENGTH_EN= 8
 
-WHISPER_MODEL = 'medium-q5_0'
+WHISPER_MODEL_EN = 'medium-q5_0'
 # WHISPER_MODEL = 'large-v3-turbo-q5_0'
-# WHISPER_MODEL = 'small'
+WHISPER_MODEL_ZH = 'small'
 
 # LLM
 LLM_MODEL_PATH = (MODEL_DIR / "qwen2.5-1.5b-instruct-q5_0.gguf").as_posix()

main.py

@@ -58,7 +58,6 @@ async def translate(websocket: WebSocket):
     query_parameters_dict = websocket.query_params
     from_lang, to_lang = query_parameters_dict.get('from'), query_parameters_dict.get('to')
     
-    pipe.reset()
     client = WhisperTranscriptionService(
         websocket,
         pipe,

moyoyo_asr_models/silero_vad.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2623a2953f6ff3d2c1e61740c6cdb7168133479b267dfef114a4a3cc5bdd788f
+size 2327524

transcribe/helpers/vadprocessor.py

@@ -1,276 +1,296 @@
-import gc
-import time
+from copy import deepcopy
+from queue import Queue, Empty
+from time import time
+from config import MODEL_DIR
+# from silero_vad import load_silero_vad
 import numpy as np
 import onnxruntime
-from datetime import timedelta
-from pydub import AudioSegment
-from silero_vad import load_silero_vad, get_speech_timestamps, VADIterator
-import os
-import logging
-
-class FixedVADIterator(VADIterator):
-    '''It fixes VADIterator by allowing to process any audio length, not only exactly 512 frames at once.
-    If audio to be processed at once is long and multiple voiced segments detected, 
-    then __call__ returns the start of the first segment, and end (or middle, which means no end) of the last segment. 
-    '''
 
-    def reset_states(self):
-        super().reset_states()
-        self.buffer = np.array([],dtype=np.float32)
-
-    def __call__(self, x, return_seconds=False):
-        self.buffer = np.append(self.buffer, x) 
-        ret = None
-        while len(self.buffer) >= 512:
-            r = super().__call__(self.buffer[:512], return_seconds=return_seconds)
-            self.buffer = self.buffer[512:]
-            if ret is None:
-                ret = r
-            elif r is not None:
-                if 'end' in r:
-                    ret['end'] = r['end']  # the latter end
-                if 'start' in r and 'end' in ret:  # there is an earlier start.
-                    # Remove end, merging this segment with the previous one.
-                    del ret['end']
-        return ret if ret != {} else None
-
-class SileroVADProcessor:
-    """
-    A class for processing audio files using Silero VAD to detect voice activity
-    and extract voice segments from audio files.
-    """
+class OnnxWrapper():
 
-    def __init__(self,
-                 activate_threshold=0.5,
-                 fusion_threshold=0.3,
-                 min_speech_duration=0.25,
-                 max_speech_duration=20,
-                 min_silence_duration=250,
-                 sample_rate=16000,
-                 ort_providers=None):
-        """
-        Initialize the SileroVADProcessor.
-
-        Args:
-            activate_threshold (float): Threshold for voice activity detection
-            fusion_threshold (float): Threshold for merging close speech segments (seconds)
-            min_speech_duration (float): Minimum duration of speech to be considered valid (seconds)
-            max_speech_duration (float): Maximum duration of speech (seconds)
-            min_silence_duration (int): Minimum silence duration (ms)
-            sample_rate (int): Sample rate of the audio (8000 or 16000 Hz)
-            ort_providers (list): ONNX Runtime providers for acceleration
-        """
-        # VAD parameters
-        self.activate_threshold = activate_threshold
-        self.fusion_threshold = fusion_threshold
-        self.min_speech_duration = min_speech_duration
-        self.max_speech_duration = max_speech_duration
-        self.min_silence_duration = min_silence_duration
-        self.sample_rate = sample_rate
-        self.ort_providers = ort_providers if ort_providers else []
-
-        # Initialize logger
-        self.logger = logging.getLogger(__name__)
-
-        # Load Silero VAD model
-        self._init_onnx_session()
-        self.silero_vad = load_silero_vad(onnx=True)
-
-    def _init_onnx_session(self):
-        """Initialize ONNX Runtime session with appropriate settings."""
-        session_opts = onnxruntime.SessionOptions()
-        session_opts.log_severity_level = 3
-        session_opts.inter_op_num_threads = 0
-        session_opts.intra_op_num_threads = 0
-        session_opts.enable_cpu_mem_arena = True
-        session_opts.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
-        session_opts.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
-
-        session_opts.add_session_config_entry("session.intra_op.allow_spinning", "1")
-        session_opts.add_session_config_entry("session.inter_op.allow_spinning", "1")
-        session_opts.add_session_config_entry("session.set_denormal_as_zero", "1")
-
-        # Set the session_opts to be used by silero_vad
-        # onnxruntime.capi._pybind_state.get_default_session_options(session_opts)
-
-    def load_audio(self, audio_path):
-        """
-        Load audio file and prepare it for VAD processing.
+    def __init__(self, path, force_onnx_cpu=False):
+        opts = onnxruntime.SessionOptions()
+        opts.inter_op_num_threads = 1
+        opts.intra_op_num_threads = 1
 
-        Args:
-            audio_path (str): Path to the audio file
+        if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers():
+            self.session = onnxruntime.InferenceSession(path, providers=['CPUExecutionProvider'], sess_options=opts)
+        else:
+            self.session = onnxruntime.InferenceSession(path, sess_options=opts)
 
-        Returns:
-            numpy.ndarray: Audio data as numpy array
-        """
-        self.logger.info(f"Loading audio from {audio_path}")
-        audio_segment = AudioSegment.from_file(audio_path)
-        audio_segment = audio_segment.set_channels(1).set_frame_rate(self.sample_rate)
+        self.reset_states()
+        self.sample_rates = [16000]
 
-        # Convert to numpy array and normalize
-        dtype = np.float16 if self.use_gpu_fp16 else np.float32
-        audio_array = np.array(audio_segment.get_array_of_samples(), dtype=dtype) * 0.000030517578  # 1/32768
+    def _validate_input(self, x: np.ndarray, sr: int):
+        if x.ndim == 1:
+            x = x[None]
+        if x.ndim > 2:
+            raise ValueError(f"Too many dimensions for input audio chunk {x.ndim}")
 
-        self.audio_segment = audio_segment  # Store for later use
-        return audio_array
-    
-    @property
-    def model(self):
-        return self.silero_vad
+        if sr != 16000 and (sr % 16000 == 0):
+            step = sr // 16000
+            x = x[:, ::step]
+            sr = 16000
 
-    def process_timestamps(self, timestamps):
-        """
-        Process VAD timestamps: filter short segments and merge close segments.
+        if sr not in self.sample_rates:
+            raise ValueError(f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)")
+        if sr / x.shape[1] > 31.25:
+            raise ValueError("Input audio chunk is too short")
 
-        Args:
-            timestamps (list): List of (start, end) tuples
+        return x, sr
 
-        Returns:
-            list: Processed list of (start, end) tuples
-        """
-        # Filter out short durations
-        filtered_timestamps = [(start, end) for start, end in timestamps
-                               if (end - start) >= self.min_speech_duration]
-
-        # Fuse timestamps in two passes for better merging
-        fused_timestamps_1st = []
-        for start, end in filtered_timestamps:
-            if fused_timestamps_1st and (start - fused_timestamps_1st[-1][1] <= self.fusion_threshold):
-                fused_timestamps_1st[-1] = (fused_timestamps_1st[-1][0], end)
-            else:
-                fused_timestamps_1st.append((start, end))
+    def reset_states(self, batch_size=1):
+        self._state = np.zeros((2, batch_size, 128)).astype(np.float32)
+        self._context = np.zeros(0)
+        self._last_sr = 0
+        self._last_batch_size = 0
 
-        fused_timestamps_2nd = []
-        for start, end in fused_timestamps_1st:
-            if fused_timestamps_2nd and (start - fused_timestamps_2nd[-1][1] <= self.fusion_threshold):
-                fused_timestamps_2nd[-1] = (fused_timestamps_2nd[-1][0], end)
-            else:
-                fused_timestamps_2nd.append((start, end))
+    def __call__(self, x, sr: int):
 
-        return fused_timestamps_2nd
+        x, sr = self._validate_input(x, sr)
+        num_samples = 512 if sr == 16000 else 256
 
-    def format_time(self, seconds):
-        """
-        Convert seconds to VTT time format 'hh:mm:ss.mmm'.
+        if x.shape[-1] != num_samples:
+            raise ValueError(
+                f"Provided number of samples is {x.shape[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)")
 
-        Args:
-            seconds (float): Time in seconds
+        batch_size = x.shape[0]
+        context_size = 64 if sr == 16000 else 32
 
-        Returns:
-            str: Formatted time string
-        """
-        td = timedelta(seconds=seconds)
-        td_sec = td.total_seconds()
-        total_seconds = int(td_sec)
-        milliseconds = int((td_sec - total_seconds) * 1000)
-        hours = total_seconds // 3600
-        minutes = (total_seconds % 3600) // 60
-        seconds = total_seconds % 60
-        return f"{hours:02}:{minutes:02}:{seconds:02}.{milliseconds:03}"
-
-    def detect_speech(self, audio:np.array):
-        """
-        Run VAD on the audio file to detect speech segments.
+        if not self._last_batch_size:
+            self.reset_states(batch_size)
+        if (self._last_sr) and (self._last_sr != sr):
+            self.reset_states(batch_size)
+        if (self._last_batch_size) and (self._last_batch_size != batch_size):
+            self.reset_states(batch_size)
 
-        Args:
-            audio_path (str): Path to the audio file
+        if not len(self._context):
+            self._context = np.zeros((batch_size, context_size)).astype(np.float32)
 
-        Returns:
-            list: List of processed timestamps as (start, end) tuples
-        """
-        self.logger.info("Starting VAD process")
-        start_time = time.time()
-        # Get speech timestamps
-        raw_timestamps = get_speech_timestamps(
-            audio,
-            model=self.silero_vad,
-            threshold=self.activate_threshold,
-            max_speech_duration_s=self.max_speech_duration,
-            min_speech_duration_ms=int(self.min_speech_duration * 1000),
-            min_silence_duration_ms=self.min_silence_duration,
-            return_seconds=True
-        )
-
-        # Convert to simple format and process
-        timestamps = [(item['start'], item['end']) for item in raw_timestamps]
-        processed_timestamps = self.process_timestamps(timestamps)
-
-        # Clean up
-        del audio
-        gc.collect()
-
-        self.logger.info(f"VAD completed in {time.time() - start_time:.3f} seconds")
-        return processed_timestamps
+        x = np.concatenate([self._context, x], axis=1)
+        if sr in [8000, 16000]:
+            ort_inputs = {'input': x, 'state': self._state, 'sr': np.array(sr, dtype='int64')}
+            ort_outs = self.session.run(None, ort_inputs)
+            out, state = ort_outs
+            self._state = state
+        else:
+            raise ValueError()
 
-        """
-        Save timestamps in both second and sample indices formats.
+        self._context = x[..., -context_size:]
+        self._last_sr = sr
+        self._last_batch_size = batch_size
 
-        Args:
-            timestamps (list): List of (start, end) tuples
-            output_prefix (str): Prefix for output files
-        """
-        # Save timestamps in seconds (VTT format)
-        seconds_path = f"{output_prefix}_timestamps_second.txt"
-        with open(seconds_path, "w", encoding='UTF-8') as file:
-            self.logger.info("Saving timestamps in seconds format")
-            for start, end in timestamps:
-                s_time = self.format_time(start)
-                e_time = self.format_time(end)
-                line = f"{s_time} --> {e_time}\n"
-                file.write(line)
-
-        # Save timestamps in sample indices
-        indices_path = f"{output_prefix}_timestamps_indices.txt"
-        with open(indices_path, "w", encoding='UTF-8') as file:
-            self.logger.info("Saving timestamps in indices format")
-            for start, end in timestamps:
-                line = f"{int(start * self.sample_rate)} --> {int(end * self.sample_rate)}\n"
-                file.write(line)
-
-        self.logger.info(f"Timestamps saved to {seconds_path} and {indices_path}")
-
-    def extract_speech_segments(self, audio_segment, timestamps):
-        """
-        Extract speech segments from the audio and combine them into a single audio file.
+        # out = torch.from_numpy(out)
+        return out
 
-        Args:
-            timestamps (list): List of (start, end) tuples indicating speech segments
+    def audio_forward(self, audio: np.ndarray, sr: int):
+        outs = []
+        x, sr = self._validate_input(audio, sr)
+        self.reset_states()
+        num_samples = 512 if sr == 16000 else 256
 
-        Returns:
-            AudioSegment: The combined speech segments
-        """
-        audio_segment = audio_segment.numpy()
-        combined_speech = np.array([], dtype=np.float32)
+        if x.shape[1] % num_samples:
+            pad_num = num_samples - (x.shape[1] % num_samples)
+            x = np.pad(x, ((0, 0), (0, pad_num)), 'constant', constant_values=(0.0, 0.0))
+
+        for i in range(0, x.shape[1], num_samples):
+            wavs_batch = x[:, i:i + num_samples]
+            out_chunk = self.__call__(wavs_batch, sr)
+            outs.append(out_chunk)
 
-        # Extract and combine each speech segment
-        for i, (start, end) in enumerate(timestamps):
-            # Convert seconds to milliseconds for pydub
-            start_ms = int(start * 1000)
-            end_ms = int(end * 1000)
+        stacked = np.concatenate(outs, axis=1)
+        return stacked
 
-            # Ensure the end time does not exceed the length of the audio segment
-            if end_ms > len(audio_segment):
-                end_ms = len(audio_segment)
 
-            # Extract the segment
-            segment = audio_segment[start_ms:end_ms]
+class VADIteratorOnnx:
+    def __init__(self,
+                 threshold: float = 0.5,
+                 sampling_rate: int = 16000,
+                 min_silence_duration_ms: int = 100,
+                 max_speech_duration_s: float = float('inf'),
+                 ):
+        self.model = OnnxWrapper(MODEL_DIR/ "silero_vad.onnx", True)
+        self.threshold = threshold
+        self.sampling_rate = sampling_rate
+
+        if sampling_rate not in [8000, 16000]:
+            raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]')
+
+        self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
+        self.max_speech_samples = int(sampling_rate * max_speech_duration_s)
+        # self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000
+        self.reset_states()
 
-            # Add to combined audio
-            combined_speech = np.append(combined_speech, segment)
+    def reset_states(self):
 
-        return combined_speech
+        self.model.reset_states()
+        self.triggered = False
+        self.temp_end = 0
+        self.current_sample = 0
+        self.start = 0
 
-    def process_audio(self, audio_array:np.array):
+    def __call__(self, x: np.ndarray, return_seconds=False):
         """
-        Complete processing pipeline: detect speech, save timestamps, and optionally extract speech.
+        x: np.ndarray
+            audio chunk (see examples in repo)
 
-        Returns:
-            tuple: (timestamps, output_speech_path if extract_speech else None)
+        return_seconds: bool (default - False)
+            whether return timestamps in seconds (default - samples)
         """
+        
+        window_size_samples = 512 if self.sampling_rate == 16000 else 256
+        x = x[:window_size_samples]
+        if len(x) < window_size_samples:
+            x = np.pad(x, ((0, 0), (0, window_size_samples - len(x))), 'constant', constant_values=0.0)
+
+        self.current_sample += window_size_samples
+
+        speech_prob = self.model(x, self.sampling_rate)[0,0]
+        # print(f"{self.current_sample/self.sampling_rate:.2f}: {speech_prob}")
+
+        if (speech_prob >= self.threshold) and self.temp_end:
+            self.temp_end = 0
+
+        if (speech_prob >= self.threshold) and not self.triggered:
+            self.triggered = True
+            speech_start = max(0, self.current_sample - window_size_samples)
+            self.start = speech_start
+            return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)}
+        
+        if (speech_prob >= self.threshold) and self.current_sample - self.start >= self.max_speech_samples:
+            if self.temp_end:
+                self.temp_end = 0
+            self.start = self.current_sample
+            return {'end': int(self.current_sample) if not return_seconds else round(self.current_sample / self.sampling_rate, 1)}
+
+        if (speech_prob < self.threshold - 0.15) and self.triggered:
+            if not self.temp_end:
+                self.temp_end = self.current_sample
+            if self.current_sample - self.temp_end < self.min_silence_samples:
+                return None
+            else:
+                speech_end = self.temp_end - window_size_samples
+                self.temp_end = 0
+                self.triggered = False
+                return {'end': int(speech_end) if not return_seconds else round(speech_end / self.sampling_rate, 1)}
 
-        # Run VAD to detect speech
-        timestamps = self.detect_speech(audio_array)
+        return None
+    
 
-        combined_speech = self.extract_speech_segments(audio_array, timestamps)
 
-        return timestamps, combined_speech
+class VadV2:
+    def __init__(self,
+                 threshold: float = 0.5,
+                 sampling_rate: int = 16000,
+                 min_silence_duration_ms: int = 100,
+                 speech_pad_ms: int = 30,
+                 max_speech_duration_s: float = float('inf')):
+        # self.vad_iterator = VADIterator(threshold, sampling_rate, min_silence_duration_ms)
+        self.vad_iterator = VADIteratorOnnx(threshold, sampling_rate, min_silence_duration_ms, max_speech_duration_s)
+        self.speech_pad_samples = int(sampling_rate * speech_pad_ms / 1000)
+        self.sampling_rate = sampling_rate
+        self.audio_buffer = np.array([], dtype=np.float32)
+        self.start = 0
+        self.end = 0
+        self.offset = 0
+        assert speech_pad_ms <= min_silence_duration_ms, "speech_pad_ms should be less than min_silence_duration_ms"
+        self.max_speech_samples = int(sampling_rate * max_speech_duration_s)
+
+        self.silence_chunk_size = 0
+        self.silence_chunk_threshold = 60 / (512 / self.sampling_rate)
+
+    def reset(self):
+        self.audio_buffer = np.array([], dtype=np.float32)
+        self.start = 0
+        self.end = 0
+        self.offset = 0
+        self.vad_iterator.reset_states()
+
+    def __call__(self, x: np.ndarray = None):
+        if x is None:
+            if self.start:
+                start = max(self.offset, self.start - self.speech_pad_samples)
+                end = self.offset + len(self.audio_buffer)
+                start_ts = round(start / self.sampling_rate, 1)
+                end_ts = round(end / self.sampling_rate, 1)
+                audio_data = self.audio_buffer[start - self.offset: end - self.offset]
+                result = {
+                    "start": start_ts,
+                    "end": end_ts,
+                    "audio": audio_data,
+                }
+            else:
+                result = None
+            self.reset()
+            return result
+
+        self.audio_buffer = np.append(self.audio_buffer, deepcopy(x))
+
+        result = self.vad_iterator(x)
+        if result is not None:
+            # self.start = result.get('start', self.start)
+            # self.end = result.get('end', self.end)
+            self.silence_chunk_size = 0
+
+            if 'start' in result:
+                self.start = result['start']
+            if 'end' in result:
+                self.end = result['end']
+        else:
+            self.silence_chunk_size += 1
+
+        if self.start == 0 and len(self.audio_buffer) > self.speech_pad_samples:
+            self.offset += len(self.audio_buffer) - self.speech_pad_samples
+            self.audio_buffer = self.audio_buffer[-self.speech_pad_samples:]
+
+        if self.silence_chunk_size >= self.silence_chunk_threshold:
+            self.offset += len(self.audio_buffer) - self.speech_pad_samples
+            self.audio_buffer = self.audio_buffer[-self.speech_pad_samples:]
+            self.silence_chunk_size = 0
+
+        if self.end > self.start:
+            start = max(self.offset, self.start - self.speech_pad_samples)
+            end = self.end + self.speech_pad_samples
+            start_ts = round(start / self.sampling_rate, 1)
+            end_ts = round(end / self.sampling_rate, 1)
+            audio_data = self.audio_buffer[start - self.offset: end - self.offset]
+            self.audio_buffer = self.audio_buffer[self.end - self.offset:]
+            self.offset = self.end
+            self.start = self.end
+            # self.start = 0
+            self.end = 0
+            result = {
+                "start": start_ts,
+                "end": end_ts,
+                "audio": audio_data,
+            }
+
+            return result
+        return None
+    
+
+    
+class VadProcessor:
+    def __init__(
+            self,
+            prob_threshold=0.5,
+            silence_s=0.5,
+            cache_s=0.25,
+            sr=16000
+    ):
+        self.prob_thres = prob_threshold
+        self.cache_s = cache_s
+        self.sr = sr
+        self.silence_s = silence_s
+
+        self.vad = VadV2(self.prob_thres, self.sr, self.silence_s * 1000, self.cache_s * 1000, max_speech_duration_s=15)
+    
+
+    def process_audio(self, audio_buffer: np.ndarray):
+        audio = np.array([], np.float32)
+        for i in range(0, len(audio_buffer), 512):
+            chunk = audio_buffer[i:i+512]
+            ret = self.vad(chunk)
+            if ret:
+                audio = np.append(audio, ret['audio'])
+        return audio

transcribe/helpers/whisper.py

@@ -9,10 +9,14 @@ logger = getLogger(__name__)
 
 class WhisperCPP:
 
-    def __init__(self, warmup=True) -> None:
+    def __init__(self, source_lange: str='en', warmup=True) -> None:
         models_dir = config.MODEL_DIR.as_posix()
+        if source_lange == "zh":
+            whisper_model = config.WHISPER_MODEL_ZH
+        else:
+            whisper_model = config.WHISPER_MODEL_EN
         self.model = Model(
-            model=config.WHISPER_MODEL, 
+            model=whisper_model, 
             models_dir=models_dir,
             print_realtime=False,
             print_progress=False,
@@ -47,9 +51,9 @@ class WhisperCPP:
                 audio_buffer,
                 initial_prompt=prompt,
                 language=language,  
-                token_timestamps=True, 
+                # token_timestamps=True, 
                 # split_on_word=True,
-                max_len=max_len
+                # max_len=max_len
             )
             return output
         except Exception as e:

transcribe/pipelines/__init__.py

@@ -1,5 +1,5 @@
 
 from .pipe_translate import TranslatePipe, Translate7BPipe
-from .pipe_whisper import WhisperPipe
+from .pipe_whisper import WhisperPipe, WhisperChinese
 from .pipe_vad import VadPipe
 from .base import MetaItem

transcribe/pipelines/pipe_translate.py

@@ -35,3 +35,6 @@ class Translate7BPipe(TranslatePipe):
         if cls.translator is None:
             cls.translator = QwenTranslator(LLM_LARGE_MODEL_PATH, LLM_SYS_PROMPT_EN, LLM_SYS_PROMPT_ZH)
         
+
+
+   

transcribe/pipelines/pipe_vad.py

@@ -1,165 +1,41 @@
 
 from .base import MetaItem, BasePipe
-from ..helpers.vadprocessor import SileroVADProcessor, FixedVADIterator
+from ..helpers.vadprocessor import VadV2
 import numpy as np
 from silero_vad import get_speech_timestamps
-import torch
 from typing import List
 import logging
 
 # import noisereduce as nr
 
-def collect_chunks(tss: List[dict], wav: torch.Tensor, sample_rate: int = 16000):
-    chunks = []
-    silent_samples = int(0.3 * sample_rate)  # 300ms 的静音样本数
-    silence = torch.zeros(silent_samples)  # 创建300ms的静音
-    
-    for i in range(len(tss)):
-        # 先添加当前语音片段
-        chunks.append(wav[tss[i]['start']: tss[i]['end']])
-        
-        # 如果不是最后一个片段，且与下一个片段间隔大于100ms，则添加静音
-        if i < len(tss) - 1:
-            gap = tss[i+1]['start'] - tss[i]['end']
-            if gap > 0.1 * sample_rate:  # 判断间隔是否大于100ms
-                chunks.append(silence)  # 添加300ms静音
-    
-    return torch.cat(chunks)
-
-def collect_chunks_improved(tss: List[dict], wav: torch.Tensor, sample_rate: int = 16000):
-    chunks = []
-    silent_samples = int(0.3 * sample_rate)  # 300ms 的静音样本数
-    # silence = torch.zeros(silent_samples)  # 创建300ms的静音
-    min_gap_samples = int(0.1 * sample_rate)  # 最小间隔阈值 (100ms)
-    
-    # 对时间戳进行简单的平滑处理
-    smoothed_tss = []
-    for i, ts in enumerate(tss):
-        if i > 0 and ts['start'] - tss[i-1]['end'] < 0.02 * sample_rate:  # 如果间隔小于20ms，认为是连续的
-            smoothed_tss[-1]['end'] = ts['end']  # 合并到前一个片段
-        else:
-            smoothed_tss.append(ts)
-    
-    for i in range(len(smoothed_tss)):
-        # 添加当前语音片段
-        chunks.append(wav[smoothed_tss[i]['start']: smoothed_tss[i]['end']])
-        
-        # 如果不是最后一个片段，且与下一个片段间隔大于阈值，则添加静音
-        if i < len(smoothed_tss) - 1:
-            gap = smoothed_tss[i+1]['start'] - smoothed_tss[i]['end']
-            if gap > min_gap_samples:
-                # 根据间隔大小动态调整静音长度，但最大不超过300ms
-                silence_length = min(gap // 2, silent_samples)
-                chunks.append(torch.zeros(silence_length))
-    
-    return torch.cat(chunks)
 
 class VadPipe(BasePipe):
-    model = None
+    vac = None
     sample_rate = 16000
     window_size_samples = 512
     chunk_size = 512
+    prob_threshold=0.5,
+    silence_s=0.5,
+    cache_s=0.25,
+
 
-    def __init__(self, in_queue=None, out_queue=None) -> None:
-        super().__init__(in_queue, out_queue)
-        self._offset = 0 # 处理的frame size offset
-        self._status = 'END'
-    
 
-    def reset(self):
-        self._offset = 0 
-        self._status = 'END'
- 
     @classmethod
     def init(cls):
-        if cls.model is None:
-            cls.model = SileroVADProcessor(
-            activate_threshold=0.3,       # 降低以捕获更多音频
-            fusion_threshold=0.45,        # 提高以更好地融合语音片段
-            min_speech_duration=0.2,      # 略微降低以捕获短音节
-            max_speech_duration=20,       # 保持不变
-            min_silence_duration=500,     # 增加到300毫秒，允许说话间的自然停顿
-            sample_rate=cls.sample_rate # 采样率，音频信号的采样频率
-        )
-            cls.vac = FixedVADIterator(
-                cls.model.silero_vad, 
-                sampling_rate=cls.sample_rate, 
-                threshold=0.3,
-                speech_pad_ms=10
-                )
-            cls.vac.reset_states()
-   
-
-
-    # def reduce_noise(self, data):
-    #     return nr.reduce_noise(y=data, sr=self.sample_rate)
-
-    def _process_speech_chunk(self, source_audio:np.ndarray):
-        speech_dict = self.vac(source_audio, return_seconds=False)
-        if speech_dict:
-            logging.debug(f"🔊 {speech_dict} {self._offset}")
-            relative_start_frame = None
-            relative_end_frame = None
-            start_frame, end_frame = speech_dict.get("start"), speech_dict.get("end")
-            if start_frame:
-                relative_start_frame = max(0, (start_frame - self._offset))
-            if end_frame:
-                relative_end_frame = min((end_frame+1 - self._offset),len(source_audio))
-            return relative_start_frame, relative_end_frame
+        if cls.vac is None:
+            cls.vac = VadV2(cls.prob_threshold, cls.sample_rate, cls.silence_s * 1000, cls.cache_s * 1000, max_speech_duration_s=15)
     
     def process(self, in_data: MetaItem) -> MetaItem:
-        if self._offset == 0:
-            self.vac.reset_states()
-        # silence_audio_100ms = np.zeros(int(0.1*self.sample_rate))
-        source_audio = np.frombuffer(in_data.source_audio, dtype=np.float32)
-        speech_data  = self._process_speech_chunk(source_audio)
-       
-        if speech_data: # 表示有音频的变化点出��
-            rel_start_frame, rel_end_frame = speech_data 
-            if rel_start_frame and not rel_end_frame:
-                self._status = "START" # 语音开始
-                target_audio = source_audio[rel_start_frame:]
-                logging.debug("🔊 Speech started, capturing audio from frame: {}".format(rel_start_frame))
-            elif not rel_start_frame and rel_end_frame:
-                self._status = "END" # 音频结束
-                target_audio = source_audio[:rel_end_frame]
-                logging.debug("🔚 Speech ended, capturing audio up to frame: {}".format(rel_end_frame))
-            elif rel_start_frame and rel_end_frame:
-                self._status = 'END'
-                target_audio = source_audio[rel_start_frame:rel_end_frame]
-                logging.debug("🔄 Speech segment captured from frame {} to frame {}".format(rel_start_frame, rel_end_frame))
-            else:
-                self._status = 'END'
-                target_audio = np.array([],dtype=np.float32)
-                # logging.debug("❌ No valid speech segment detected, setting status to END")
+        audio_buffer = np.frombuffer(in_data.source_audio)
+        vad_audio = self.vac(audio_buffer)
+        if vad_audio:
+            in_data.audio = vad_audio['audio']
         else:
-            if self._status == 'START':
-                target_audio = source_audio
-                # logging.debug("🔊 Continuing to capture audio as speech is still ongoing")
-            else: # end
-                target_audio = np.array([],dtype=np.float32)
-                # self._status = 'END'
-                # logging.debug("❌ No speech detected, setting status to END")
-
-        self._offset += len(source_audio)
-
-        in_data.audio = target_audio.tobytes()
-        in_data.source_audio = b''
+            in_data.audio = b""
         return in_data
 
 
+    # def reduce_noise(self, data):
+    #     return nr.reduce_noise(y=data, sr=self.sample_rate)
 
-    def process_all(self, in_data: MetaItem) -> MetaItem:
-        source_audio = in_data.source_audio
-        source_audio = np.frombuffer(source_audio, dtype=np.float32)
-        # source_audio = self.reduce_noise(source_audio)
-        send_audio = b""
-        speech_timestamps = get_speech_timestamps(torch.Tensor(source_audio), self.model.silero_vad, sampling_rate=16000)
-        
-        if speech_timestamps:
-            send_audio = collect_chunks_improved(speech_timestamps, torch.Tensor(source_audio))
-            send_audio = send_audio.numpy()
-            in_data.audio = send_audio
-            # send_audio = self.reduce_noise(send_audio).tobytes()
-        in_data.source_audio = b""
-        return in_data
+    

transcribe/pipelines/pipe_whisper.py

@@ -7,16 +7,18 @@ class WhisperPipe(BasePipe):
     whisper = None
 
 
+
     @classmethod
     def init(cls):
         if cls.whisper is None:
+            # cls.zh_whisper = WhisperCPP(source_lange='zh')
             cls.whisper = WhisperCPP()
-
+    
 
     def process(self, in_data: MetaItem) -> MetaItem:
         audio_data = in_data.audio
         source_language = in_data.source_language
-        segments = self.whisper.transcribe(audio_data, source_language) or []
+        segments = self.whisper.transcribe(audio_data, source_language)
         texts = "".join([s.text for s in segments])
         in_data.segments = [Segment(t0=s.t0, t1=s.t1, text=self.filter_chinese_printable(s.text)) for s in segments]
         in_data.transcribe_content = texts
@@ -30,3 +32,11 @@ class WhisperPipe(BasePipe):
             if unicodedata.category(char) != 'Cc':  # 不可打印字符的分类为 'Cc'
                 printable.append(char)
         return ''.join(printable).strip()
+    
+
+
+class WhisperChinese(WhisperPipe):
+    @classmethod
+    def init(cls):
+        if cls.whisper is None:
+            cls.whisper = WhisperCPP(source_lange='zh')

transcribe/translatepipes.py

@@ -1,4 +1,4 @@
-from transcribe.pipelines import WhisperPipe, TranslatePipe, MetaItem, VadPipe, Translate7BPipe
+from transcribe.pipelines import WhisperPipe, TranslatePipe, MetaItem, WhisperChinese, Translate7BPipe
 import multiprocessing  as mp
 import config
 
@@ -11,17 +11,18 @@ class TranslatePipes:
         # self.result_queue = mp.Queue()
         
         # whisper 转录
-        self._whisper_pipe = self._launch_process(WhisperPipe())
+        self._whisper_pipe_en = self._launch_process(WhisperPipe())
+        self._whisper_pipe_zh = self._launch_process(WhisperChinese())
         
         # llm 翻译
-        self._translate_pipe = self._launch_process(TranslatePipe())
+        # self._translate_pipe = self._launch_process(TranslatePipe())
 
         self._translate_7b_pipe = self._launch_process(Translate7BPipe())
         # vad 
-        self._vad_pipe = self._launch_process(VadPipe())
+        # self._vad_pipe = self._launch_process(VadPipe())
 
-    def reset(self):
-        self._vad_pipe.reset()
+    # def reset(self):
+    #     self._vad_pipe.reset()
     
     def _launch_process(self, process_obj):
         process_obj.daemon = True
@@ -29,9 +30,10 @@ class TranslatePipes:
         return process_obj
 
     def wait_ready(self):
-        self._whisper_pipe.wait()
-        self._translate_pipe.wait()
-        self._vad_pipe.wait()
+        self._whisper_pipe_zh.wait()
+        self._whisper_pipe_en.wait()
+        # self._translate_pipe.wait()
+        # self._vad_pipe.wait()
         self._translate_7b_pipe.wait()
     
     def translate(self, text, src_lang, dst_lang) -> MetaItem:
@@ -48,14 +50,20 @@ class TranslatePipes:
             transcribe_content=text,
             source_language=src_lang, 
             destination_language=dst_lang)
-        self._translate_pipe.input_queue.put(item)
-        return self._translate_pipe.output_queue.get()
+        self._translate_7b_pipe.input_queue.put(item)
+        return self._translate_7b_pipe.output_queue.get()
+    
+    def get_whisper_model(self, lang:str='en'):
+        if lang == 'zh':
+            return self._whisper_pipe_zh
+        return self._whisper_pipe_en
     
 
     def transcrible(self, audio_buffer:bytes, src_lang: str) -> MetaItem:
+        whisper_model = self.get_whisper_model(src_lang)
         item = MetaItem(audio=audio_buffer, source_language=src_lang)
-        self._whisper_pipe.input_queue.put(item)
-        return self._whisper_pipe.output_queue.get()
+        whisper_model.input_queue.put(item)
+        return whisper_model.output_queue.get()
     
     def voice_detect(self, audio_buffer:bytes) -> MetaItem:
         item = MetaItem(source_audio=audio_buffer)

transcribe/whisper_llm_serve.py

@@ -8,14 +8,15 @@ from typing import List, Optional, Iterator, Tuple, Any
 import asyncio
 import numpy as np
 import config
-# import wordninja
+
 from api_model import TransResult, Message, DebugResult
-# from .server import ServeClientBase
+
 from .utils import log_block, save_to_wave, TestDataWriter
 from .translatepipes import TranslatePipes
 from .strategy import (
     TranscriptStabilityAnalyzer, TranscriptToken)
-import csv
+from transcribe.helpers.vadprocessor import VadProcessor
+from transcribe.pipelines import MetaItem
 
 logger = getLogger("TranscriptionService")
 
@@ -39,15 +40,16 @@ class WhisperTranscriptionService:
 
         # 音频处理相关
         self.sample_rate = 16000
-        self.frames_np = None
+ 
         self.lock = threading.Lock()
         self._frame_queue = queue.Queue()
-        
+        self._vad_frame_queue = queue.Queue()
+
         # 文本分隔符，根据语言设置
         self.text_separator = self._get_text_separator(language)
         self.loop = asyncio.get_event_loop()
         # 发送就绪状态
-        self.send_ready_state()
+        
         self._transcrible_analysis = None
         # 启动处理线程
         self._translate_thread_stop = threading.Event()
@@ -55,9 +57,8 @@ class WhisperTranscriptionService:
 
         self.translate_thread = self._start_thread(self._transcription_processing_loop)
         self.frame_processing_thread = self._start_thread(self._frame_processing_loop)
-
-        self._vad_processed_offset = 0
-
+        self._vad = VadProcessor()
+        self.row_number = 0
         # for test
         self._transcrible_time_cost = 0.
         self._translate_time_cost = 0.
@@ -86,9 +87,9 @@ class WhisperTranscriptionService:
         """根据语言返回适当的文本分隔符"""
         return "" if language == "zh" else " "
 
-    def send_ready_state(self) -> None:
+    async def send_ready_state(self) -> None:
         """发送服务就绪状态消息"""
-        self.websocket.send(json.dumps({
+        await self.websocket.send(json.dumps({
             "uid": self.client_uid,
             "message": self.SERVER_READY,
             "backend": "whisper_transcription"
@@ -98,8 +99,8 @@ class WhisperTranscriptionService:
         """设置源语言和目标语言"""
         self.source_language = source_lang
         self.target_language = target_lang
-        self.text_separator = self._get_text_separator(source_lang)
-        self._transcrible_analysis = TranscriptStabilityAnalyzer(self.source_language, self.text_separator)
+        # self.text_separator = self._get_text_separator(source_lang)
+        # self._transcrible_analysis = TranscriptStabilityAnalyzer(self.source_language, self.text_separator)
 
     def add_frames(self, frame_np: np.ndarray) -> None:
         """添加音频帧到处理队列"""
@@ -109,69 +110,21 @@ class WhisperTranscriptionService:
         """从队列获取音频帧并合并到缓冲区"""
         while not self._frame_processing_thread_stop.is_set():
             try:
-                frame_np = self._frame_queue.get(timeout=0.1)
-                frame_np = self._apply_voice_activity_detection(frame_np)
-                if frame_np is None:
-                    logger.error("Received None frame, stopping thread")
+                audio = self._frame_queue.get(timeout=0.1)
+                # save_to_wave(f"{self._c}_before_vad.wav", audio)
+                processed_audio = self._vad.process_audio(audio)
+                if processed_audio.shape[0] > 0:
+                    # vad_processed_audio = processed_audio
+                    # save_to_wave(f"{self._c}_after_vad.wav", processed_audio)
+                    # vad_frame_obj = np.frombuffer(processed_audio.audio, dtype=np.float32)
+                    logger.debug(f"Vad frame: {processed_audio.shape[0]/self.sample_rate:.2f}")
                 # apply vad speech check:
-                
-                with self.lock:
-                    if self.frames_np is None:
-                        self.frames_np = frame_np.copy()
-                    else:
-                        self.frames_np = np.append(self.frames_np, frame_np)
+                    self._vad_frame_queue.put(processed_audio)
             except queue.Empty:
                 pass
 
-    def _apply_voice_activity_detection(self, frame_np:np.array) -> None:
-        """应用语音活动检测来优化音频缓冲区"""
-        # self._c+= 1
-        processed_audio = self._translate_pipe.voice_detect(frame_np.tobytes())
-        speech_audio =  np.frombuffer(processed_audio.audio, dtype=np.float32)
-        # if speech_audio:
-        # if len(frame) > self.sample_rate:
-        #     save_to_wave(f"{self._c}-org.wav", frame)
-        #     save_to_wave(f"{self._c}-vad.wav", self.frames_np)
-        return speech_audio
-
-    def _update_audio_buffer(self, offset: int) -> None:
-        """从音频缓冲区中移除已处理的部分"""
-        with self.lock:
-            if self.frames_np is not None and offset > 0:
-                # self._c += 1
-                # before = self.frames_np.copy()
-                self.frames_np = self.frames_np[offset:]
-                # after = self.frames_np.copy()
-                # save_to_wave(f"./tests/{self._c}_before_cut_{offset}.wav", before)
-                # save_to_wave(f"./tests/{self._c}_cut.wav", before[:offset])
-                # save_to_wave(f"./tests/{self._c}_after_cut.wav", after)
-
-
-    def _get_audio_for_processing(self) -> Optional[np.ndarray]:
-        """准备用于处理的音频块"""
-        # 应用VAD处理
-        # frame_np = self._apply_voice_activity_detection()
-        frame_np = self.frames_np.copy()
-        # 没有音频帧
-        if frame_np is None:
-            return None
-
-        frames = frame_np.copy()
-
-        # 音频过短时的处理
-        if len(frames) < self.sample_rate:
-            # 极短音频段，清空并返回None
-            # self._update_audio_buffer(len(frames))
-            return None
-        # if len(frames) < self.sample_rate:
-        #     # 不足一秒的音频，补充静音
-        #     silence_audio = np.zeros((self.sample_rate + 1000,), dtype=np.float32)
-        #     silence_audio[-len(frames):] = frames
-        #     return silence_audio.copy()
-
-        return frames
-
-    def _transcribe_audio(self, audio_buffer: np.ndarray) -> List[TranscriptToken]:
+
+    def _transcribe_audio(self, audio_buffer: np.ndarray)->MetaItem:
         """转录音频并返回转录片段"""
         log_block("Audio buffer length", f"{audio_buffer.shape[0]/self.sample_rate:.2f}", "s")
         start_time = time.perf_counter()
@@ -180,14 +133,11 @@ class WhisperTranscriptionService:
         segments = result.segments
         time_diff = (time.perf_counter() - start_time)
         logger.debug(f"📝 Transcrible Segments: {segments} ")
-        logger.debug(f"📝 Transcrible: {self.text_separator.join(seg.text for seg in segments)} ")
+        # logger.debug(f"📝 Transcrible: {self.text_separator.join(seg.text for seg in segments)} ")
         log_block("📝 Transcrible output", f"{self.text_separator.join(seg.text for seg in segments)}", "")
         log_block("📝 Transcrible time", f"{time_diff:.3f}", "s")
         self._transcrible_time_cost = round(time_diff, 3)
-        return [
-            TranscriptToken(text=s.text, t0=s.t0, t1=s.t1)
-            for s in segments
-        ]
+        return result
 
     def _translate_text(self, text: str) -> str:
         """将文本翻译为目标语言"""
@@ -221,37 +171,44 @@ class WhisperTranscriptionService:
         self._translate_time_cost = round(time_diff, 3)
         return translated_text
 
-
-
     def _transcription_processing_loop(self) -> None:
         """主转录处理循环"""
-        c = 0
-        while not self._translate_thread_stop.is_set():
-
-            # 等待音频数据
-            if self.frames_np is None:
-                time.sleep(0.2)
-                logger.info("Waiting for audio data...")
-                continue
 
-            # 获取音频块进行处理
-            audio_buffer = self._get_audio_for_processing()
-            if audio_buffer is None:
+        while not self._translate_thread_stop.is_set():
+            audio_buffer = self._vad_frame_queue.get()
+            if audio_buffer is None or len(audio_buffer) < int(self.sample_rate):
                 time.sleep(0.2)
                 continue
-            logger.debug(f"🥤 Buffer Length: {len(audio_buffer)/self.sample_rate:.2f} ")
-          
-
+                
+            logger.debug(f"audio buffer size: {len(audio_buffer) / self.sample_rate:.2f}s")
             # try:
-            segments = self._transcribe_audio(audio_buffer)
-
-            # 处理转录结果并发送到客户端
-            for result in self._process_transcription_results(segments, audio_buffer):
+            meta_item = self._transcribe_audio(audio_buffer)
+            segments = meta_item.segments
+            logger.debug(f"Segments: {segments}")
+            if len(segments):
+                result = self._process_transcription_results_2(segments)
                 self._send_result_to_client(result)
+                time.sleep(0.1)
+            # 处理转录结果并发送到客户端
+            # for result in self._process_transcription_results(segments, audio_buffer):
+            #     self._send_result_to_client(result)
 
             # except Exception as e:
             #     logger.error(f"Error processing audio: {e}")
 
+    def _process_transcription_results_2(self, segments: List[TranscriptToken],):
+        seg = segments[0]
+        item =  TransResult(
+                seg_id=self.row_number,
+                context=seg.text,
+                from_=self.source_language,
+                to=self.target_language,
+                tran_content=self._translate_text_large(seg.text),
+                partial=False
+            )
+        self.row_number += 1
+        return item
+
     def _process_transcription_results(self, segments: List[TranscriptToken], audio_buffer: np.ndarray) -> Iterator[TransResult]:
         """
         处理转录结果，生成翻译结果