Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(vad): apply the offline dynamic silence schedule per frame
lingfengyaoc9a4febbb906
diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.pyindex d8537ed20..0b01c8384 100644--- a/funasr/models/fsmn_vad_streaming/model.py+++ b/funasr/models/fsmn_vad_streaming/model.py@@ -353,6 +353,7 @@ class Stats(object):self.output_data_buf_offset = 0self.frame_probs = []self.max_end_sil_frame_cnt_thresh = max_end_sil_frame_cnt_thresh+ self.silence_schedule = Noneself.speech_noise_thres = speech_noise_thresself.scores = Noneself.max_time_out = False@@ -1012,9 +1013,11 @@ class FsmnVADStreaming(nn.Module):"dynamic_silence", kwargs.get("max_end_silence_time") is None)silence_schedule = kwargs.get("silence_schedule", DEFAULT_SILENCE_SCHEDULE)- speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thres- accumulated_ms = cache.get("_dynamic_accumulated_ms", 0)- in_speech = cache.get("_dynamic_in_speech", False)+ # The schedule is looked up per frame in DetectOneFrame from the current segment's duration.+ if "stats" in cache:+ cache["stats"].silence_schedule = silence_schedule if dynamic_silence else None+ if dynamic_silence:+ cache["stats"].speech_noise_thres = 0.5frontend_inference_kwargs = {}if getattr(frontend, "supports_aligned_waveforms", False):frontend_inference_kwargs["return_waveform"] = True@@ -1023,21 +1026,6 @@ class FsmnVADStreaming(nn.Module):kwargs["is_final"] = _is_final and i == n - 1audio_sample_i = audio_sample[i * chunk_stride_samples : (i + 1) * chunk_stride_samples]- # Apply dynamic silence threshold (only accumulate while in speech)- if dynamic_silence and "stats" in cache:- vad_state = cache["stats"].vad_state_machine- # VadStateMachine: 2=kVadInStateStartPointNotDetected, 3=kVadInStateInSpeechSegment- if vad_state.value == 2 or in_speech: # kVadInStateInSpeechSegment- accumulated_ms += chunk_size- in_speech = True- for limit_ms, silence_ms in silence_schedule:- if accumulated_ms <= limit_ms:- cache["stats"].max_end_sil_frame_cnt_thresh = max(silence_ms - speech_to_sil_ms, 0)- cache["stats"].speech_noise_thres = 0.5- break- cache["_dynamic_accumulated_ms"] = accumulated_ms- cache["_dynamic_in_speech"] = in_speech-# extract fbank featsspeech, speech_lengths = extract_fbank([audio_sample_i],@@ -1074,11 +1062,6 @@ class FsmnVADStreaming(nn.Module):segments_i = self.forward(**batch)if len(segments_i) > 0:segments.extend(*segments_i)- if dynamic_silence:- accumulated_ms = 0- in_speech = False- cache["_dynamic_accumulated_ms"] = 0- cache["_dynamic_in_speech"] = Falsecache["prev_samples"] = audio_sample[-m:] if m > 0 else torch.empty(0)if _is_final:@@ -1264,6 +1247,17 @@ class FsmnVADStreaming(nn.Module):cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache)elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:+ silence_schedule = getattr(cache["stats"], "silence_schedule", None)+ if silence_schedule is not None:+ segment_ms = (+ cur_frm_idx - cache["stats"].confirmed_start_frame + 1+ ) * frm_shift_in_ms+ for limit_ms, silence_ms in silence_schedule:+ if segment_ms <= limit_ms:+ cache["stats"].max_end_sil_frame_cnt_thresh = max(+ silence_ms - self.vad_opts.speech_to_sil_time_thres, 0+ )+ breakif (cache["stats"].continous_silence_frame_count * frm_shift_in_ms>= cache["stats"].max_end_sil_frame_cnt_threshdiff --git a/tests/test_fsmn_vad_dynamic_silence.py b/tests/test_fsmn_vad_dynamic_silence.pyindex f396614f9..d8f39c345 100644--- a/tests/test_fsmn_vad_dynamic_silence.py+++ b/tests/test_fsmn_vad_dynamic_silence.py@@ -55,13 +55,83 @@ class TestFsmnVadDynamicSilence(unittest.TestCase):cache = self._run_inference(max_end_silence_time=300)self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200)- self.assertNotIn("_dynamic_accumulated_ms", cache)+ self.assertIsNone(cache["stats"].silence_schedule)+ self.assertEqual(cache["stats"].speech_noise_thres, 0.6)def test_explicit_dynamic_silence_still_enables_schedule(self):cache = self._run_inference(max_end_silence_time=300, dynamic_silence=True)- self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 1900)- self.assertEqual(cache["_dynamic_accumulated_ms"], 1000)+ self.assertEqual(cache["stats"].silence_schedule, vad_model.DEFAULT_SILENCE_SCHEDULE)+ self.assertEqual(cache["stats"].speech_noise_thres, 0.5)+ # The threshold is no longer advanced per chunk; DetectOneFrame applies the schedule.+ self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200)+ self.assertNotIn("_dynamic_accumulated_ms", cache)+++class TestFsmnVadDynamicSilencePerFrame(unittest.TestCase):+ """Drive DetectOneFrame with scripted window-detector transitions."""++ def _end_frames(self, speech_ms, silence_ms, schedule, start_frame=0, fixed_thresh_ms=650):+ vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)+ torch.nn.Module.__init__(vad)+ vad.vad_opts = vad_model.VADXOptions()+ ends = []++ def on_voice_end(end_frame, fake_result, is_last_frame, cache=None):+ ends.append(end_frame)++ def reset_detection(cache=None):+ cache["stats"].confirmed_start_frame = -1+ cache["stats"].vad_state_machine = vad_model.VadStateMachine.kVadInStateStartPointNotDetected++ vad.OnVoiceEnd = on_voice_end+ vad.PopDataToOutputBuf = lambda *args, **kwargs: None+ vad.PopDataBufTillFrame = lambda *args, **kwargs: None+ vad.LatencyFrmNumAtStartPoint = lambda cache=None: 0+ vad.ResetDetection = reset_detection++ stats = vad_model.Stats(+ sil_pdf_ids=[0], max_end_sil_frame_cnt_thresh=fixed_thresh_ms, speech_noise_thres=0.5+ )+ stats.silence_schedule = schedule+ speech_frames, silence_frames = speech_ms // 10, silence_ms // 10+ changes = (+ [vad_model.AudioChangeState.kChangeStateSil2Speech]+ + [vad_model.AudioChangeState.kChangeStateSpeech2Speech] * (speech_frames - 1)+ + [vad_model.AudioChangeState.kChangeStateSpeech2Sil]+ + [vad_model.AudioChangeState.kChangeStateSil2Sil] * silence_frames+ )+ script = iter(changes)+ cache = {+ "stats": stats,+ "windows_detector": SimpleNamespace(DetectOneFrame=lambda *a, **k: next(script)),+ }+ for i in range(len(changes)):+ vad.DetectOneFrame(vad_model.FrameState.kFrameStateSpeech, start_frame + i, False, cache=cache)+ return [e - start_frame for e in ends]++ def test_schedule_follows_current_segment_duration(self):+ schedule = vad_model.DEFAULT_SILENCE_SCHEDULE+ # Thresholds below are the schedule values minus speech_to_sil_time_thres (150 ms).+ # 5 s into a segment: 2000 ms -> 1850 ms, so 900 ms of silence does not end it.+ self.assertEqual(self._end_frames(5000, 900, schedule), [])+ # 15 s into a segment: 1000 ms -> 850 ms, so 900 ms ends it and 700 ms does not.+ self.assertEqual(len(self._end_frames(15000, 900, schedule)), 1)+ self.assertEqual(self._end_frames(15000, 700, schedule), [])+ # 55 s into a segment: 200 ms -> 50 ms, so 300 ms ends it.+ self.assertEqual(len(self._end_frames(55000, 300, schedule)), 1)++ def test_schedule_does_not_depend_on_absolute_position(self):+ schedule = vad_model.DEFAULT_SILENCE_SCHEDULE+ for speech_ms, silence_ms in [(5000, 900), (15000, 900), (55000, 300)]:+ at_zero = self._end_frames(speech_ms, silence_ms, schedule)+ shifted = self._end_frames(speech_ms, silence_ms, schedule, start_frame=6000)+ self.assertEqual(at_zero, shifted)++ def test_without_schedule_threshold_stays_fixed(self):+ # 650 ms fixed threshold: 700 ms of silence ends a 5 s segment, 600 ms does not.+ self.assertEqual(len(self._end_frames(5000, 700, None)), 1)+ self.assertEqual(self._end_frames(5000, 600, None), [])if __name__ == "__main__":