Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat: extract DynamicStreamingVAD as reusable component

xiaoyunchong.xycca352d992f9f
diff --git a/docs/vllm_guide.md b/docs/vllm_guide.md
index 4967a4039..f50cc8908 100644
--- a/docs/vllm_guide.md
+++ b/docs/vllm_guide.md
@@ -512,3 +512,91 @@ Chrome 要求 HTTPS 或 localhost。远程服务器用 SSH 端口转发:`ssh -
### Q: 多个并发连接会互相影响吗?
不会。每个 WebSocket 连接有独立的 session(VAD/ASR 状态隔离)。vLLM 内部会自动调度。
+
+---
+
+## 附录:DynamicStreamingVAD
+
+`funasr.utils.dynamic_vad.DynamicStreamingVAD` 是通用的动态阈值流式 VAD 封装,
+在 fsmn-vad 基础上根据当前语音段的累积时长动态调整静音切分阈值。
+
+### 设计动机
+
+fsmn-vad 默认使用固定静音阈值(800ms)。实际场景中:
+- 短句(如"好的")需要等更长的静音才切,否则会把一句话切碎
+- 长段(如会议发言 30s+)需要更快切分,否则 ASR 输入过长导致质量下降
+
+### 用法
+
+```python
+from funasr import AutoModel
+from funasr.utils.dynamic_vad import DynamicStreamingVAD
+
+vad_model = AutoModel(model="fsmn-vad", device="cuda:0")
+
+# 使用默认阈值配置
+vad = DynamicStreamingVAD(vad_model)
+
+# 或自定义阈值
+vad = DynamicStreamingVAD(
+ vad_model,
+ silence_schedule=[
+ (3000, 1500), # 累积 <=3s: 等 1.5s 静音
+ (10000, 800), # 累积 3-10s: 等 0.8s
+ (float('inf'), 300), # 累积 >10s: 等 0.3s
+ ],
+ speech_noise_thres=0.5,
+)
+```
+
+#### 流式调用
+
+```python
+import torch
+
+for audio_chunk in audio_stream: # 实时音频流
+ segments = vad.feed(torch.from_numpy(audio_chunk).float())
+ for seg in segments:
+ print(f"Speech: {seg[0]}-{seg[1]}ms")
+
+# 结束时
+final_segments = vad.finalize()
+```
+
+#### 非流式调用
+
+```python
+segments = vad.process(full_audio_tensor)
+for seg in segments:
+ print(f"Speech: {seg[0]}-{seg[1]}ms")
+```
+
+### 默认阈值配置
+
+| 累积时长 | 静音阈值 | 说明 |
+|---------|---------|------|
+| ≤ 5s | 2.0s | 短句不切碎 |
+| 5-10s | 1.5s | 正常分句 |
+| 10-15s | 1.0s | 开始收紧 |
+| 15-30s | 0.8s | 较快切分 |
+| 30-45s | 0.4s | 防止过长 |
+| > 45s | 0.1s | 强制切分 |
+
+### 参数
+
+| 参数 | 默认值 | 说明 |
+|------|--------|------|
+| `vad_model` | - | FunASR AutoModel 加载的 fsmn-vad 实例 |
+| `chunk_size_ms` | 60 | VAD 内部处理 chunk 大小 |
+| `speech_noise_thres` | 0.5 | 语音/噪声判别阈值 |
+| `speech_to_sil_thres_ms` | 150 | 语音转静音基础时间 |
+| `silence_schedule` | 见上表 | 动态阈值配置 `[(上限ms, 静音ms), ...]` |
+| `sample_rate` | 16000 | 采样率 |
+
+### 属性
+
+| 属性 | 说明 |
+|------|------|
+| `vad.is_speaking` | 当前是否在语音状态中 |
+| `vad.current_duration_ms` | 当前段已累积时长 |
+| `vad.current_threshold_ms` | 当前使用的静音阈值 |
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py
index 2b4a75a1d..a54e047bb 100644
--- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py
+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py
@@ -71,40 +71,7 @@ def _clean_asr_text(text):
return text.strip()
-class StreamingVAD:
- """Streaming VAD with dynamic silence threshold."""
-
- def __init__(self, vad_model):
- self.model = vad_model
- self.cache = {}
- self.confirmed_segments = []
- self.current_speech_start = None
-
- def feed(self, audio_chunk_tensor, is_final=False):
- res = self.model.generate(
- input=[audio_chunk_tensor], cache=self.cache,
- is_final=is_final, chunk_size=60,
- )
- signals = res[0].get("value", [])
- new_confirmed = []
- for sig in signals:
- if sig[0] >= 0 and sig[1] == -1:
- self.current_speech_start = sig[0]
- elif sig[0] == -1 and sig[1] >= 0:
- start = self.current_speech_start if self.current_speech_start is not None else 0
- self.confirmed_segments.append([start, sig[1]])
- new_confirmed.append([start, sig[1]])
- self.current_speech_start = None
- elif sig[0] >= 0 and sig[1] >= 0:
- self.confirmed_segments.append(sig)
- new_confirmed.append(sig)
- self.current_speech_start = None
- return new_confirmed
-
- def reset(self):
- self.cache = {}
- self.confirmed_segments = []
- self.current_speech_start = None
+from funasr.utils.dynamic_vad import DynamicStreamingVAD
class HybridSpeakerTracker:
@@ -275,7 +242,6 @@ class RealtimeASRSession:
self.last_decode_samples = 0
self.locked_sentences = []
self.prev_seg_text = ""
- self.accumulated_since_cut_ms = 0
self.spk_tracker = spk_tracker
self.use_context = True
self.is_active = False
@@ -287,31 +253,12 @@ class RealtimeASRSession:
new_audio = self.audio_buffer[self.vad_fed_samples:]
if len(new_audio) > 0:
- self.accumulated_since_cut_ms += len(new_audio) * 1000 // self.sample_rate
- if "stats" in self.vad.cache:
- self.vad.cache["stats"].speech_noise_thres = 0.5
- if self.accumulated_since_cut_ms <= 5000:
- desired_silence_ms = 2000
- elif self.accumulated_since_cut_ms <= 10000:
- desired_silence_ms = 1500
- elif self.accumulated_since_cut_ms <= 15000:
- desired_silence_ms = 1000
- elif self.accumulated_since_cut_ms <= 30000:
- desired_silence_ms = 800
- elif self.accumulated_since_cut_ms <= 45000:
- desired_silence_ms = 400
- else:
- desired_silence_ms = 100
- new_thresh = max(desired_silence_ms - 150, 0)
- self.vad.cache["stats"].max_end_sil_frame_cnt_thresh = new_thresh
-
new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False)
self.vad_fed_samples = len(self.audio_buffer)
for seg in new_confirmed:
seg_text = self._decode_segment(seg)
self.prev_text = ""
- self.accumulated_since_cut_ms = 0
if not seg_text.strip():
continue
self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})
@@ -442,7 +389,6 @@ class RealtimeASRSession:
self.last_partial_text = ""
self.last_decode_samples = 0
self.locked_sentences = []
- self.accumulated_since_cut_ms = 0
if self.spk_tracker:
self.spk_tracker.reset()
@@ -492,7 +438,7 @@ def load_models(args):
async def handle_client(websocket, args):
vllm_engine, asr_kwargs, vad_model, spk_model = load_models(args)
- vad = StreamingVAD(vad_model)
+ vad = DynamicStreamingVAD(vad_model)
spk_tracker = HybridSpeakerTracker(spk_model, args.device)
session = RealtimeASRSession(vllm_engine, asr_kwargs, vad, spk_tracker=spk_tracker)
logger.info(f"Client connected: {websocket.remote_address}")
diff --git a/funasr/utils/dynamic_vad.py b/funasr/utils/dynamic_vad.py
new file mode 100644
index 000000000..35df65ac4
--- /dev/null
+++ b/funasr/utils/dynamic_vad.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""DynamicStreamingVAD — 动态阈值流式 VAD 封装。
+
+在 fsmn-vad 基础上,根据当前语音段的累积时长动态调整静音切分阈值:
+短句等待更长静音(避免切碎),长句快速切分(避免堆积)。
+
+支持流式(逐帧喂入)和非流式(一次性处理完整音频)两种调用方式。
+
+Usage (流式):
+ from funasr import AutoModel
+ from funasr.utils.dynamic_vad import DynamicStreamingVAD
+
+ vad_model = AutoModel(model="fsmn-vad", device="cuda:0")
+ vad = DynamicStreamingVAD(vad_model)
+
+ for audio_chunk in audio_stream:
+ segments = vad.feed(audio_chunk)
+ for seg in segments:
+ print(f"Speech: {seg[0]}-{seg[1]}ms")
+
+ # 结束时
+ final_segments = vad.finalize()
+
+Usage (非流式):
+ segments = vad.process(full_audio_tensor)
+ for seg in segments:
+ print(f"Speech: {seg[0]}-{seg[1]}ms")
+"""
+
+from typing import List, Optional, Tuple
+
+import torch
+import numpy as np
+
+
+# 默认动态阈值配置:(累积时长上限ms, 静音阈值ms)
+DEFAULT_SILENCE_SCHEDULE = [
+ (5000, 2000),
+ (10000, 1500),
+ (15000, 1000),
+ (30000, 800),
+ (45000, 400),
+ (float('inf'), 100),
+]
+
+
+class DynamicStreamingVAD:
+ """动态阈值流式 VAD。
+
+ 在 fsmn-vad 的流式推理基础上,根据当前语音段已累积的时长
+ 动态调整静音切分阈值,实现「短句不切碎、长句快切分」。
+
+ Args:
+ vad_model: FunASR AutoModel 加载的 fsmn-vad 模型实例。
+ chunk_size_ms: 每次喂入 VAD 的 chunk 大小(毫秒),默认 60。
+ speech_noise_thres: 语音/噪声判别阈值,默认 0.5。
+ speech_to_sil_thres_ms: 语音转静音的基础时间(毫秒),默认 150。
+ silence_schedule: 动态阈值配置表,格式为
+ [(累积时长上限ms, 对应的静音阈值ms), ...]。
+ 当累积时长 <= 上限时,使用对应的静音阈值。
+ 默认值适合实时对话场景。设为 None 禁用动态调整(使用固定阈值)。
+ sample_rate: 采样率,默认 16000。
+
+ Example:
+ # 自定义阈值:更激进的切分
+ vad = DynamicStreamingVAD(
+ vad_model,
+ silence_schedule=[
+ (3000, 1500),
+ (8000, 800),
+ (15000, 400),
+ (float('inf'), 200),
+ ],
+ )
+ """
+
+ def __init__(
+ self,
+ vad_model,
+ chunk_size_ms: int = 60,
+ speech_noise_thres: float = 0.5,
+ speech_to_sil_thres_ms: int = 150,
+ silence_schedule: Optional[List[Tuple[float, int]]] = None,
+ sample_rate: int = 16000,
+ ):
+ self.model = vad_model
+ self.chunk_size_ms = chunk_size_ms
+ self.speech_noise_thres = speech_noise_thres
+ self.speech_to_sil_thres_ms = speech_to_sil_thres_ms
+ self.silence_schedule = silence_schedule if silence_schedule is not None else DEFAULT_SILENCE_SCHEDULE
+ self.sample_rate = sample_rate
+
+ self.cache = {}
+ self.confirmed_segments: List[List[int]] = []
+ self.current_speech_start: Optional[int] = None
+ self.accumulated_since_cut_ms: int = 0
+
+ def _get_silence_threshold(self) -> int:
+ """根据当前累积时长,从 schedule 中查询静音阈值。"""
+ for limit_ms, silence_ms in self.silence_schedule:
+ if self.accumulated_since_cut_ms <= limit_ms:
+ return silence_ms
+ return self.silence_schedule[-1][1]
+
+ def _apply_dynamic_threshold(self):
+ """将动态阈值应用到 VAD 内部 cache。"""
+ if "stats" not in self.cache:
+ return
+ stats = self.cache["stats"]
+ stats.speech_noise_thres = self.speech_noise_thres
+ desired_silence_ms = self._get_silence_threshold()
+ stats.max_end_sil_frame_cnt_thresh = max(desired_silence_ms - self.speech_to_sil_thres_ms, 0)
+
+ def feed(self, audio_chunk: torch.Tensor, is_final: bool = False) -> List[List[int]]:
+ """喂入一段音频,返回新确认的语音段。
+
+ Args:
+ audio_chunk: 音频数据(float32 tensor,16kHz)。
+ 可以是任意长度,内部按 chunk_size_ms 处理。
+ is_final: 是否为最后一段音频。设为 True 时会强制结束当前语音段。
+
+ Returns:
+ 新确认的语音段列表,每段为 [start_ms, end_ms]。
+ 仅在检测到语音结束时返回非空列表。
+ """
+ if audio_chunk.dim() > 1:
+ audio_chunk = audio_chunk.squeeze()
+
+ chunk_samples = len(audio_chunk)
+ self.accumulated_since_cut_ms += int(chunk_samples * 1000 / self.sample_rate)
+
+ self._apply_dynamic_threshold()
+
+ res = self.model.generate(
+ input=[audio_chunk], cache=self.cache,
+ is_final=is_final, chunk_size=self.chunk_size_ms,
+ )
+
+ signals = res[0].get("value", [])
+ new_confirmed = []
+
+ for sig in signals:
+ if sig[0] >= 0 and sig[1] == -1:
+ self.current_speech_start = sig[0]
+ elif sig[0] == -1 and sig[1] >= 0:
+ start = self.current_speech_start if self.current_speech_start is not None else 0
+ seg = [start, sig[1]]
+ self.confirmed_segments.append(seg)
+ new_confirmed.append(seg)
+ self.current_speech_start = None
+ self.accumulated_since_cut_ms = 0
+ elif sig[0] >= 0 and sig[1] >= 0:
+ self.confirmed_segments.append(sig)
+ new_confirmed.append(sig)
+ self.current_speech_start = None
+ self.accumulated_since_cut_ms = 0
+
+ return new_confirmed
+
+ def finalize(self) -> List[List[int]]:
+ """结束流式处理,返回最后可能未结束的语音段。
+
+ 调用此方法后,VAD 状态会被重置。
+ 如果当前有正在进行的语音段,会被强制结束。
+
+ Returns:
+ 最后确认的语音段列表。
+ """
+ # Feed empty with is_final=True to flush
+ empty = torch.zeros(int(self.sample_rate * 0.01), dtype=torch.float32)
+ return self.feed(empty, is_final=True)
+
+ def process(self, audio: torch.Tensor) -> List[List[int]]:
+ """非流式接口:一次性处理完整音频,返回所有语音段。
+
+ Args:
+ audio: 完整音频(float32 tensor,16kHz)。
+
+ Returns:
+ 所有检测到的语音段 [[start_ms, end_ms], ...]。
+ """
+ self.reset()
+
+ if isinstance(audio, np.ndarray):
+ audio = torch.from_numpy(audio).float()
+ if audio.dim() > 1:
+ audio = audio.squeeze()
+
+ # 分 chunk 喂入
+ chunk_samples = int(self.sample_rate * self.chunk_size_ms / 1000)
+ total = len(audio)
+ all_segments = []
+
+ for i in range(0, total, chunk_samples):
+ chunk = audio[i:i + chunk_samples]
+ is_last = (i + chunk_samples >= total)
+ segs = self.feed(chunk, is_final=is_last)
+ all_segments.extend(segs)
+
+ return all_segments
+
+ @property
+ def is_speaking(self) -> bool:
+ """当前是否在语音状态中。"""
+ return self.current_speech_start is not None
+
+ @property
+ def current_duration_ms(self) -> int:
+ """当前段已累积的时长(毫秒)。"""
+ return self.accumulated_since_cut_ms
+
+ @property
+ def current_threshold_ms(self) -> int:
+ """当前使用的静音阈值(毫秒)。"""
+ return self._get_silence_threshold()
+
+ def reset(self):
+ """重置所有状态,开始新一轮检测。"""
+ self.cache = {}
+ self.confirmed_segments = []
+ self.current_speech_start = None
+ self.accumulated_since_cut_ms = 0