Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(vad): exclude idle time from dynamic silence schedule
zhifu gao2df180c636b1
diff --git a/funasr/models/fsmn_vad_streaming/dynamic_vad.py b/funasr/models/fsmn_vad_streaming/dynamic_vad.pyindex f880fd2f8..944b60b9f 100644--- a/funasr/models/fsmn_vad_streaming/dynamic_vad.py+++ b/funasr/models/fsmn_vad_streaming/dynamic_vad.py@@ -94,6 +94,18 @@ class DynamicStreamingVAD:self.confirmed_segments: List[List[int]] = []self.current_speech_start: Optional[int] = Noneself.accumulated_since_cut_ms: int = 0+ self._total_audio_samples: int = 0+ self._needs_stream_reset: bool = False++ def _update_speech_duration(self):+ if self.current_speech_start is None:+ self.accumulated_since_cut_ms = 0+ else:+ # VAD start signals use the session clock, including preceding silence.+ audio_end_ms = self._total_audio_samples * 1000 // self.sample_rate+ self.accumulated_since_cut_ms = max(+ 0, audio_end_ms - int(self.current_speech_start)+ )def _get_silence_threshold(self) -> int:"""根据当前累积时长,从 schedule 中查询静音阈值。"""@@ -123,11 +135,14 @@ class DynamicStreamingVAD:新确认的语音段列表,每段为 [start_ms, end_ms]。仅在检测到语音结束时返回非空列表。"""+ if self._needs_stream_reset:+ self.reset()if audio_chunk.dim() > 1:audio_chunk = audio_chunk.squeeze()chunk_samples = len(audio_chunk)- self.accumulated_since_cut_ms += int(chunk_samples * 1000 / self.sample_rate)+ self._total_audio_samples += chunk_samples+ self._update_speech_duration()self._apply_dynamic_threshold()initial_cache_kwargs = {}@@ -163,17 +178,22 @@ class DynamicStreamingVAD:self.current_speech_start = Noneself.accumulated_since_cut_ms = 0+ self._update_speech_duration()+ # Preserve final state for callers' tail fallback; reset before the next stream.+ self._needs_stream_reset = is_finalreturn new_confirmeddef finalize(self) -> List[List[int]]:"""结束流式处理,返回最后可能未结束的语音段。- 调用此方法后,VAD 状态会被重置。- 如果当前有正在进行的语音段,会被强制结束。+ 本次状态保留到下一次 feed,便于调用方补齐未返回结束信号的尾段。+ 下一次 feed 会重置状态,开始新的流。Returns:最后确认的语音段列表。"""+ if self._needs_stream_reset:+ return []# Feed empty with is_final=True to flushempty = torch.zeros(int(self.sample_rate * 0.01), dtype=torch.float32)return self.feed(empty, is_final=True)@@ -228,3 +248,5 @@ class DynamicStreamingVAD:self.confirmed_segments = []self.current_speech_start = Noneself.accumulated_since_cut_ms = 0+ self._total_audio_samples = 0+ self._needs_stream_reset = Falsediff --git a/tests/test_dynamic_streaming_vad.py b/tests/test_dynamic_streaming_vad.pyindex 00f256f27..c4ae39995 100644--- a/tests/test_dynamic_streaming_vad.py+++ b/tests/test_dynamic_streaming_vad.py@@ -82,5 +82,166 @@ class TestDynamicStreamingVadFirstCall(unittest.TestCase):self.assertEqual(one_chunk_segments, [])+class _SignalModel(_ThresholdAwareModel):+ def __init__(self, signals):+ super().__init__()+ self.signals = iter(signals)+ self.thresholds = []++ def generate(self, input, cache, **kwargs):+ if not cache:+ self.model.init_cache(cache, **kwargs)+ self.thresholds.append(+ cache["stats"].max_end_sil_frame_cnt_thresh+ + self.model.vad_opts.speech_to_sil_time_thres+ )+ return [{"value": next(self.signals, [])}]+++class TestDynamicStreamingVadSpeechDuration(unittest.TestCase):+ def test_initial_silence_does_not_advance_speech_schedule(self):+ vad = DynamicStreamingVAD(_SignalModel([[]]))++ vad.feed(torch.zeros(60 * 16000))++ self.assertEqual(vad.current_duration_ms, 0)+ self.assertEqual(vad.current_threshold_ms, 2000)++ def test_speech_after_initial_silence_uses_short_utterance_threshold(self):+ model = _SignalModel([[], [[60000, -1]]])+ vad = DynamicStreamingVAD(model)+ vad.feed(torch.zeros(60 * 16000))++ vad.feed(torch.ones(16000))++ self.assertEqual(model.thresholds, [2000, 2000])+ self.assertEqual(vad.current_duration_ms, 1000)+ self.assertEqual(vad.current_threshold_ms, 2000)++ def test_inter_utterance_silence_does_not_age_next_utterance(self):+ model = _SignalModel([[[0, -1]], [[-1, 1000]], [], [[22000, -1]]])+ vad = DynamicStreamingVAD(model)+ vad.feed(torch.ones(16000))+ vad.feed(torch.zeros(2 * 16000))+ vad.feed(torch.zeros(19 * 16000))++ vad.feed(torch.ones(16000))++ self.assertEqual(model.thresholds[-1], 2000)+ self.assertEqual(vad.current_duration_ms, 1000)++ def test_duration_uses_detected_start_and_keeps_long_speech_schedule(self):+ vad = DynamicStreamingVAD(_SignalModel([[], [[6000, -1]], []]))+ vad.feed(torch.zeros(5 * 16000))+ vad.feed(torch.ones(2 * 16000))+ self.assertEqual(vad.current_duration_ms, 1000)++ vad.feed(torch.ones(6 * 16000))++ self.assertEqual(vad.current_duration_ms, 7000)+ self.assertEqual(vad.current_threshold_ms, 1500)++ def test_submillisecond_packets_do_not_lose_elapsed_samples(self):+ vad = DynamicStreamingVAD(_SignalModel([[[0, -1]], [], []]))+ for count in (1, 15, 160):+ vad.feed(torch.ones(count))++ self.assertEqual(vad.current_duration_ms, 11)++ def test_last_open_signal_in_packet_defines_current_duration(self):+ vad = DynamicStreamingVAD(_SignalModel([[[0, 1000], [2000, -1]]]))++ self.assertEqual(vad.feed(torch.ones(10 * 16000)), [[0, 1000]])++ self.assertEqual(vad.current_duration_ms, 8000)+ self.assertEqual(vad.current_threshold_ms, 1500)++ def test_reset_starts_a_new_sample_clock(self):+ vad = DynamicStreamingVAD(_SignalModel([[], [[0, -1]]]))+ vad.feed(torch.zeros(60 * 16000))+ vad.reset()++ vad.feed(torch.ones(16000))++ self.assertEqual(vad.current_duration_ms, 1000)+ self.assertEqual(vad.current_threshold_ms, 2000)++ def test_backdated_start_in_previous_packet_counts_entire_segment(self):+ vad = DynamicStreamingVAD(_SignalModel([[], [[4800, -1]]]))+ vad.feed(torch.zeros(5 * 16000))++ vad.feed(torch.ones(16000))++ self.assertEqual(vad.current_duration_ms, 1200)++ def test_start_end_start_in_one_packet_uses_last_start(self):+ vad = DynamicStreamingVAD(+ _SignalModel([[[1000, -1], [-1, 2000], [3000, -1]]])+ )++ self.assertEqual(vad.feed(torch.ones(10 * 16000)), [[1000, 2000]])+ self.assertEqual(vad.current_speech_start, 3000)+ self.assertEqual(vad.current_duration_ms, 7000)++ def test_finalize_without_end_preserves_start_for_service_fallback(self):+ vad = DynamicStreamingVAD(_SignalModel([[[500, -1]], []]))+ vad.feed(torch.ones(16000))++ self.assertEqual(vad.finalize(), [])++ self.assertEqual(vad.current_speech_start, 500)+ self.assertTrue(vad.is_speaking)++ def test_feed_after_finalize_starts_a_new_stream(self):+ model = _SignalModel([[], [], [[0, -1]]])+ vad = DynamicStreamingVAD(model)+ vad.feed(torch.zeros(60 * 16000))+ vad.finalize()++ vad.feed(torch.ones(16000))++ self.assertEqual(vad.current_duration_ms, 1000)+ self.assertEqual(model.thresholds[-1], 2000)++ def test_feed_after_direct_final_clears_previous_stream(self):+ model = _SignalModel([[[0, 1000], [2000, -1]], [[0, -1]]])+ vad = DynamicStreamingVAD(model)+ self.assertEqual(vad.feed(torch.ones(60 * 16000), is_final=True), [[0, 1000]])+ self.assertEqual(vad.current_speech_start, 2000)+ old_cache = vad.cache++ vad.feed(torch.ones(16000))++ self.assertEqual(vad.current_duration_ms, 1000)+ self.assertEqual(vad.confirmed_segments, [])+ self.assertIsNot(vad.cache, old_cache)+ self.assertEqual(model.thresholds[-1], 2000)++ def test_repeated_finalize_preserves_final_state_without_feeding_again(self):+ model = _SignalModel([[[0, 100], [500, -1]], []])+ vad = DynamicStreamingVAD(model)+ vad.feed(torch.ones(16000))+ vad.finalize()+ final_cache = vad.cache++ self.assertEqual(vad.finalize(), [])++ self.assertEqual(vad.current_speech_start, 500)+ self.assertEqual(vad.confirmed_segments, [[0, 100]])+ self.assertIs(vad.cache, final_cache)+ self.assertEqual(len(model.thresholds), 2)++ def test_finalize_after_process_preserves_results_and_fallback_state(self):+ model = _SignalModel([[[0, 30]], [[80, -1]]])+ vad = DynamicStreamingVAD(model)+ self.assertEqual(vad.process(torch.ones(1920)), [[0, 30]])++ self.assertEqual(vad.finalize(), [])++ self.assertEqual(vad.confirmed_segments, [[0, 30]])+ self.assertEqual(vad.current_speech_start, 80)+ self.assertEqual(len(model.thresholds), 2)++if __name__ == "__main__":unittest.main()