Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat: extract DynamicStreamingVAD as reusable component
xiaoyunchong.xycca352d992f9f
diff --git a/docs/vllm_guide.md b/docs/vllm_guide.mdindex 4967a4039..f50cc8908 100644--- a/docs/vllm_guide.md+++ b/docs/vllm_guide.md@@ -512,3 +512,91 @@ Chrome 要求 HTTPS 或 localhost。远程服务器用 SSH 端口转发:`ssh -### Q: 多个并发连接会互相影响吗?不会。每个 WebSocket 连接有独立的 session(VAD/ASR 状态隔离)。vLLM 内部会自动调度。++---++## 附录:DynamicStreamingVAD++`funasr.utils.dynamic_vad.DynamicStreamingVAD` 是通用的动态阈值流式 VAD 封装,+在 fsmn-vad 基础上根据当前语音段的累积时长动态调整静音切分阈值。++### 设计动机++fsmn-vad 默认使用固定静音阈值(800ms)。实际场景中:+- 短句(如"好的")需要等更长的静音才切,否则会把一句话切碎+- 长段(如会议发言 30s+)需要更快切分,否则 ASR 输入过长导致质量下降++### 用法++```python+from funasr import AutoModel+from funasr.utils.dynamic_vad import DynamicStreamingVAD++vad_model = AutoModel(model="fsmn-vad", device="cuda:0")++# 使用默认阈值配置+vad = DynamicStreamingVAD(vad_model)++# 或自定义阈值+vad = DynamicStreamingVAD(+ vad_model,+ silence_schedule=[+ (3000, 1500), # 累积 <=3s: 等 1.5s 静音+ (10000, 800), # 累积 3-10s: 等 0.8s+ (float('inf'), 300), # 累积 >10s: 等 0.3s+ ],+ speech_noise_thres=0.5,+)+```++#### 流式调用++```python+import torch++for audio_chunk in audio_stream: # 实时音频流+ segments = vad.feed(torch.from_numpy(audio_chunk).float())+ for seg in segments:+ print(f"Speech: {seg[0]}-{seg[1]}ms")++# 结束时+final_segments = vad.finalize()+```++#### 非流式调用++```python+segments = vad.process(full_audio_tensor)+for seg in segments:+ print(f"Speech: {seg[0]}-{seg[1]}ms")+```++### 默认阈值配置++| 累积时长 | 静音阈值 | 说明 |+|---------|---------|------|+| ≤ 5s | 2.0s | 短句不切碎 |+| 5-10s | 1.5s | 正常分句 |+| 10-15s | 1.0s | 开始收紧 |+| 15-30s | 0.8s | 较快切分 |+| 30-45s | 0.4s | 防止过长 |+| > 45s | 0.1s | 强制切分 |++### 参数++| 参数 | 默认值 | 说明 |+|------|--------|------|+| `vad_model` | - | FunASR AutoModel 加载的 fsmn-vad 实例 |+| `chunk_size_ms` | 60 | VAD 内部处理 chunk 大小 |+| `speech_noise_thres` | 0.5 | 语音/噪声判别阈值 |+| `speech_to_sil_thres_ms` | 150 | 语音转静音基础时间 |+| `silence_schedule` | 见上表 | 动态阈值配置 `[(上限ms, 静音ms), ...]` |+| `sample_rate` | 16000 | 采样率 |++### 属性++| 属性 | 说明 |+|------|------|+| `vad.is_speaking` | 当前是否在语音状态中 |+| `vad.current_duration_ms` | 当前段已累积时长 |+| `vad.current_threshold_ms` | 当前使用的静音阈值 |diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.pyindex 2b4a75a1d..a54e047bb 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py@@ -71,40 +71,7 @@ def _clean_asr_text(text):return text.strip()-class StreamingVAD:- """Streaming VAD with dynamic silence threshold."""-- def __init__(self, vad_model):- self.model = vad_model- self.cache = {}- self.confirmed_segments = []- self.current_speech_start = None-- def feed(self, audio_chunk_tensor, is_final=False):- res = self.model.generate(- input=[audio_chunk_tensor], cache=self.cache,- is_final=is_final, chunk_size=60,- )- signals = res[0].get("value", [])- new_confirmed = []- for sig in signals:- if sig[0] >= 0 and sig[1] == -1:- self.current_speech_start = sig[0]- elif sig[0] == -1 and sig[1] >= 0:- start = self.current_speech_start if self.current_speech_start is not None else 0- self.confirmed_segments.append([start, sig[1]])- new_confirmed.append([start, sig[1]])- self.current_speech_start = None- elif sig[0] >= 0 and sig[1] >= 0:- self.confirmed_segments.append(sig)- new_confirmed.append(sig)- self.current_speech_start = None- return new_confirmed-- def reset(self):- self.cache = {}- self.confirmed_segments = []- self.current_speech_start = None+from funasr.utils.dynamic_vad import DynamicStreamingVADclass HybridSpeakerTracker:@@ -275,7 +242,6 @@ class RealtimeASRSession:self.last_decode_samples = 0self.locked_sentences = []self.prev_seg_text = ""- self.accumulated_since_cut_ms = 0self.spk_tracker = spk_trackerself.use_context = Trueself.is_active = False@@ -287,31 +253,12 @@ class RealtimeASRSession:new_audio = self.audio_buffer[self.vad_fed_samples:]if len(new_audio) > 0:- self.accumulated_since_cut_ms += len(new_audio) * 1000 // self.sample_rate- if "stats" in self.vad.cache:- self.vad.cache["stats"].speech_noise_thres = 0.5- if self.accumulated_since_cut_ms <= 5000:- desired_silence_ms = 2000- elif self.accumulated_since_cut_ms <= 10000:- desired_silence_ms = 1500- elif self.accumulated_since_cut_ms <= 15000:- desired_silence_ms = 1000- elif self.accumulated_since_cut_ms <= 30000:- desired_silence_ms = 800- elif self.accumulated_since_cut_ms <= 45000:- desired_silence_ms = 400- else:- desired_silence_ms = 100- new_thresh = max(desired_silence_ms - 150, 0)- self.vad.cache["stats"].max_end_sil_frame_cnt_thresh = new_thresh-new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False)self.vad_fed_samples = len(self.audio_buffer)for seg in new_confirmed:seg_text = self._decode_segment(seg)self.prev_text = ""- self.accumulated_since_cut_ms = 0if not seg_text.strip():continueself.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})@@ -442,7 +389,6 @@ class RealtimeASRSession:self.last_partial_text = ""self.last_decode_samples = 0self.locked_sentences = []- self.accumulated_since_cut_ms = 0if self.spk_tracker:self.spk_tracker.reset()@@ -492,7 +438,7 @@ def load_models(args):async def handle_client(websocket, args):vllm_engine, asr_kwargs, vad_model, spk_model = load_models(args)- vad = StreamingVAD(vad_model)+ vad = DynamicStreamingVAD(vad_model)spk_tracker = HybridSpeakerTracker(spk_model, args.device)session = RealtimeASRSession(vllm_engine, asr_kwargs, vad, spk_tracker=spk_tracker)logger.info(f"Client connected: {websocket.remote_address}")diff --git a/funasr/utils/dynamic_vad.py b/funasr/utils/dynamic_vad.pynew file mode 100644index 000000000..35df65ac4--- /dev/null+++ b/funasr/utils/dynamic_vad.py@@ -0,0 +1,222 @@+#!/usr/bin/env python3+"""DynamicStreamingVAD — 动态阈值流式 VAD 封装。++在 fsmn-vad 基础上,根据当前语音段的累积时长动态调整静音切分阈值:+短句等待更长静音(避免切碎),长句快速切分(避免堆积)。++支持流式(逐帧喂入)和非流式(一次性处理完整音频)两种调用方式。++Usage (流式):+ from funasr import AutoModel+ from funasr.utils.dynamic_vad import DynamicStreamingVAD++ vad_model = AutoModel(model="fsmn-vad", device="cuda:0")+ vad = DynamicStreamingVAD(vad_model)++ for audio_chunk in audio_stream:+ segments = vad.feed(audio_chunk)+ for seg in segments:+ print(f"Speech: {seg[0]}-{seg[1]}ms")++ # 结束时+ final_segments = vad.finalize()++Usage (非流式):+ segments = vad.process(full_audio_tensor)+ for seg in segments:+ print(f"Speech: {seg[0]}-{seg[1]}ms")+"""++from typing import List, Optional, Tuple++import torch+import numpy as np+++# 默认动态阈值配置:(累积时长上限ms, 静音阈值ms)+DEFAULT_SILENCE_SCHEDULE = [+ (5000, 2000),+ (10000, 1500),+ (15000, 1000),+ (30000, 800),+ (45000, 400),+ (float('inf'), 100),+]+++class DynamicStreamingVAD:+ """动态阈值流式 VAD。++ 在 fsmn-vad 的流式推理基础上,根据当前语音段已累积的时长+ 动态调整静音切分阈值,实现「短句不切碎、长句快切分」。++ Args:+ vad_model: FunASR AutoModel 加载的 fsmn-vad 模型实例。+ chunk_size_ms: 每次喂入 VAD 的 chunk 大小(毫秒),默认 60。+ speech_noise_thres: 语音/噪声判别阈值,默认 0.5。+ speech_to_sil_thres_ms: 语音转静音的基础时间(毫秒),默认 150。+ silence_schedule: 动态阈值配置表,格式为+ [(累积时长上限ms, 对应的静音阈值ms), ...]。+ 当累积时长 <= 上限时,使用对应的静音阈值。+ 默认值适合实时对话场景。设为 None 禁用动态调整(使用固定阈值)。+ sample_rate: 采样率,默认 16000。++ Example:+ # 自定义阈值:更激进的切分+ vad = DynamicStreamingVAD(+ vad_model,+ silence_schedule=[+ (3000, 1500),+ (8000, 800),+ (15000, 400),+ (float('inf'), 200),+ ],+ )+ """++ def __init__(+ self,+ vad_model,+ chunk_size_ms: int = 60,+ speech_noise_thres: float = 0.5,+ speech_to_sil_thres_ms: int = 150,+ silence_schedule: Optional[List[Tuple[float, int]]] = None,+ sample_rate: int = 16000,+ ):+ self.model = vad_model+ self.chunk_size_ms = chunk_size_ms+ self.speech_noise_thres = speech_noise_thres+ self.speech_to_sil_thres_ms = speech_to_sil_thres_ms+ self.silence_schedule = silence_schedule if silence_schedule is not None else DEFAULT_SILENCE_SCHEDULE+ self.sample_rate = sample_rate++ self.cache = {}+ self.confirmed_segments: List[List[int]] = []+ self.current_speech_start: Optional[int] = None+ self.accumulated_since_cut_ms: int = 0++ def _get_silence_threshold(self) -> int:+ """根据当前累积时长,从 schedule 中查询静音阈值。"""+ for limit_ms, silence_ms in self.silence_schedule:+ if self.accumulated_since_cut_ms <= limit_ms:+ return silence_ms+ return self.silence_schedule[-1][1]++ def _apply_dynamic_threshold(self):+ """将动态阈值应用到 VAD 内部 cache。"""+ if "stats" not in self.cache:+ return+ stats = self.cache["stats"]+ stats.speech_noise_thres = self.speech_noise_thres+ desired_silence_ms = self._get_silence_threshold()+ stats.max_end_sil_frame_cnt_thresh = max(desired_silence_ms - self.speech_to_sil_thres_ms, 0)++ def feed(self, audio_chunk: torch.Tensor, is_final: bool = False) -> List[List[int]]:+ """喂入一段音频,返回新确认的语音段。++ Args:+ audio_chunk: 音频数据(float32 tensor,16kHz)。+ 可以是任意长度,内部按 chunk_size_ms 处理。+ is_final: 是否为最后一段音频。设为 True 时会强制结束当前语音段。++ Returns:+ 新确认的语音段列表,每段为 [start_ms, end_ms]。+ 仅在检测到语音结束时返回非空列表。+ """+ if audio_chunk.dim() > 1:+ audio_chunk = audio_chunk.squeeze()++ chunk_samples = len(audio_chunk)+ self.accumulated_since_cut_ms += int(chunk_samples * 1000 / self.sample_rate)++ self._apply_dynamic_threshold()++ res = self.model.generate(+ input=[audio_chunk], cache=self.cache,+ is_final=is_final, chunk_size=self.chunk_size_ms,+ )++ signals = res[0].get("value", [])+ new_confirmed = []++ for sig in signals:+ if sig[0] >= 0 and sig[1] == -1:+ self.current_speech_start = sig[0]+ elif sig[0] == -1 and sig[1] >= 0:+ start = self.current_speech_start if self.current_speech_start is not None else 0+ seg = [start, sig[1]]+ self.confirmed_segments.append(seg)+ new_confirmed.append(seg)+ self.current_speech_start = None+ self.accumulated_since_cut_ms = 0+ elif sig[0] >= 0 and sig[1] >= 0:+ self.confirmed_segments.append(sig)+ new_confirmed.append(sig)+ self.current_speech_start = None+ self.accumulated_since_cut_ms = 0++ return new_confirmed++ def finalize(self) -> List[List[int]]:+ """结束流式处理,返回最后可能未结束的语音段。++ 调用此方法后,VAD 状态会被重置。+ 如果当前有正在进行的语音段,会被强制结束。++ Returns:+ 最后确认的语音段列表。+ """+ # Feed empty with is_final=True to flush+ empty = torch.zeros(int(self.sample_rate * 0.01), dtype=torch.float32)+ return self.feed(empty, is_final=True)++ def process(self, audio: torch.Tensor) -> List[List[int]]:+ """非流式接口:一次性处理完整音频,返回所有语音段。++ Args:+ audio: 完整音频(float32 tensor,16kHz)。++ Returns:+ 所有检测到的语音段 [[start_ms, end_ms], ...]。+ """+ self.reset()++ if isinstance(audio, np.ndarray):+ audio = torch.from_numpy(audio).float()+ if audio.dim() > 1:+ audio = audio.squeeze()++ # 分 chunk 喂入+ chunk_samples = int(self.sample_rate * self.chunk_size_ms / 1000)+ total = len(audio)+ all_segments = []++ for i in range(0, total, chunk_samples):+ chunk = audio[i:i + chunk_samples]+ is_last = (i + chunk_samples >= total)+ segs = self.feed(chunk, is_final=is_last)+ all_segments.extend(segs)++ return all_segments++ @property+ def is_speaking(self) -> bool:+ """当前是否在语音状态中。"""+ return self.current_speech_start is not None++ @property+ def current_duration_ms(self) -> int:+ """当前段已累积的时长(毫秒)。"""+ return self.accumulated_since_cut_ms++ @property+ def current_threshold_ms(self) -> int:+ """当前使用的静音阈值(毫秒)。"""+ return self._get_silence_threshold()++ def reset(self):+ """重置所有状态,开始新一轮检测。"""+ self.cache = {}+ self.confirmed_segments = []+ self.current_speech_start = None+ self.accumulated_since_cut_ms = 0