Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(vad): apply the offline dynamic silence schedule per frame

lingfengyaoc9a4febbb906
diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py
index d8537ed20..0b01c8384 100644
--- a/funasr/models/fsmn_vad_streaming/model.py
+++ b/funasr/models/fsmn_vad_streaming/model.py
@@ -353,6 +353,7 @@ class Stats(object):
self.output_data_buf_offset = 0
self.frame_probs = []
self.max_end_sil_frame_cnt_thresh = max_end_sil_frame_cnt_thresh
+ self.silence_schedule = None
self.speech_noise_thres = speech_noise_thres
self.scores = None
self.max_time_out = False
@@ -1012,9 +1013,11 @@ class FsmnVADStreaming(nn.Module):
"dynamic_silence", kwargs.get("max_end_silence_time") is None
)
silence_schedule = kwargs.get("silence_schedule", DEFAULT_SILENCE_SCHEDULE)
- speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thres
- accumulated_ms = cache.get("_dynamic_accumulated_ms", 0)
- in_speech = cache.get("_dynamic_in_speech", False)
+ # The schedule is looked up per frame in DetectOneFrame from the current segment's duration.
+ if "stats" in cache:
+ cache["stats"].silence_schedule = silence_schedule if dynamic_silence else None
+ if dynamic_silence:
+ cache["stats"].speech_noise_thres = 0.5
frontend_inference_kwargs = {}
if getattr(frontend, "supports_aligned_waveforms", False):
frontend_inference_kwargs["return_waveform"] = True
@@ -1023,21 +1026,6 @@ class FsmnVADStreaming(nn.Module):
kwargs["is_final"] = _is_final and i == n - 1
audio_sample_i = audio_sample[i * chunk_stride_samples : (i + 1) * chunk_stride_samples]
- # Apply dynamic silence threshold (only accumulate while in speech)
- if dynamic_silence and "stats" in cache:
- vad_state = cache["stats"].vad_state_machine
- # VadStateMachine: 2=kVadInStateStartPointNotDetected, 3=kVadInStateInSpeechSegment
- if vad_state.value == 2 or in_speech: # kVadInStateInSpeechSegment
- accumulated_ms += chunk_size
- in_speech = True
- for limit_ms, silence_ms in silence_schedule:
- if accumulated_ms <= limit_ms:
- cache["stats"].max_end_sil_frame_cnt_thresh = max(silence_ms - speech_to_sil_ms, 0)
- cache["stats"].speech_noise_thres = 0.5
- break
- cache["_dynamic_accumulated_ms"] = accumulated_ms
- cache["_dynamic_in_speech"] = in_speech
-
# extract fbank feats
speech, speech_lengths = extract_fbank(
[audio_sample_i],
@@ -1074,11 +1062,6 @@ class FsmnVADStreaming(nn.Module):
segments_i = self.forward(**batch)
if len(segments_i) > 0:
segments.extend(*segments_i)
- if dynamic_silence:
- accumulated_ms = 0
- in_speech = False
- cache["_dynamic_accumulated_ms"] = 0
- cache["_dynamic_in_speech"] = False
cache["prev_samples"] = audio_sample[-m:] if m > 0 else torch.empty(0)
if _is_final:
@@ -1264,6 +1247,17 @@ class FsmnVADStreaming(nn.Module):
cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache
)
elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
+ silence_schedule = getattr(cache["stats"], "silence_schedule", None)
+ if silence_schedule is not None:
+ segment_ms = (
+ cur_frm_idx - cache["stats"].confirmed_start_frame + 1
+ ) * frm_shift_in_ms
+ for limit_ms, silence_ms in silence_schedule:
+ if segment_ms <= limit_ms:
+ cache["stats"].max_end_sil_frame_cnt_thresh = max(
+ silence_ms - self.vad_opts.speech_to_sil_time_thres, 0
+ )
+ break
if (
cache["stats"].continous_silence_frame_count * frm_shift_in_ms
>= cache["stats"].max_end_sil_frame_cnt_thresh
diff --git a/tests/test_fsmn_vad_dynamic_silence.py b/tests/test_fsmn_vad_dynamic_silence.py
index f396614f9..d8f39c345 100644
--- a/tests/test_fsmn_vad_dynamic_silence.py
+++ b/tests/test_fsmn_vad_dynamic_silence.py
@@ -55,13 +55,83 @@ class TestFsmnVadDynamicSilence(unittest.TestCase):
cache = self._run_inference(max_end_silence_time=300)
self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200)
- self.assertNotIn("_dynamic_accumulated_ms", cache)
+ self.assertIsNone(cache["stats"].silence_schedule)
+ self.assertEqual(cache["stats"].speech_noise_thres, 0.6)
def test_explicit_dynamic_silence_still_enables_schedule(self):
cache = self._run_inference(max_end_silence_time=300, dynamic_silence=True)
- self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 1900)
- self.assertEqual(cache["_dynamic_accumulated_ms"], 1000)
+ self.assertEqual(cache["stats"].silence_schedule, vad_model.DEFAULT_SILENCE_SCHEDULE)
+ self.assertEqual(cache["stats"].speech_noise_thres, 0.5)
+ # The threshold is no longer advanced per chunk; DetectOneFrame applies the schedule.
+ self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200)
+ self.assertNotIn("_dynamic_accumulated_ms", cache)
+
+
+class TestFsmnVadDynamicSilencePerFrame(unittest.TestCase):
+ """Drive DetectOneFrame with scripted window-detector transitions."""
+
+ def _end_frames(self, speech_ms, silence_ms, schedule, start_frame=0, fixed_thresh_ms=650):
+ vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)
+ torch.nn.Module.__init__(vad)
+ vad.vad_opts = vad_model.VADXOptions()
+ ends = []
+
+ def on_voice_end(end_frame, fake_result, is_last_frame, cache=None):
+ ends.append(end_frame)
+
+ def reset_detection(cache=None):
+ cache["stats"].confirmed_start_frame = -1
+ cache["stats"].vad_state_machine = vad_model.VadStateMachine.kVadInStateStartPointNotDetected
+
+ vad.OnVoiceEnd = on_voice_end
+ vad.PopDataToOutputBuf = lambda *args, **kwargs: None
+ vad.PopDataBufTillFrame = lambda *args, **kwargs: None
+ vad.LatencyFrmNumAtStartPoint = lambda cache=None: 0
+ vad.ResetDetection = reset_detection
+
+ stats = vad_model.Stats(
+ sil_pdf_ids=[0], max_end_sil_frame_cnt_thresh=fixed_thresh_ms, speech_noise_thres=0.5
+ )
+ stats.silence_schedule = schedule
+ speech_frames, silence_frames = speech_ms // 10, silence_ms // 10
+ changes = (
+ [vad_model.AudioChangeState.kChangeStateSil2Speech]
+ + [vad_model.AudioChangeState.kChangeStateSpeech2Speech] * (speech_frames - 1)
+ + [vad_model.AudioChangeState.kChangeStateSpeech2Sil]
+ + [vad_model.AudioChangeState.kChangeStateSil2Sil] * silence_frames
+ )
+ script = iter(changes)
+ cache = {
+ "stats": stats,
+ "windows_detector": SimpleNamespace(DetectOneFrame=lambda *a, **k: next(script)),
+ }
+ for i in range(len(changes)):
+ vad.DetectOneFrame(vad_model.FrameState.kFrameStateSpeech, start_frame + i, False, cache=cache)
+ return [e - start_frame for e in ends]
+
+ def test_schedule_follows_current_segment_duration(self):
+ schedule = vad_model.DEFAULT_SILENCE_SCHEDULE
+ # Thresholds below are the schedule values minus speech_to_sil_time_thres (150 ms).
+ # 5 s into a segment: 2000 ms -> 1850 ms, so 900 ms of silence does not end it.
+ self.assertEqual(self._end_frames(5000, 900, schedule), [])
+ # 15 s into a segment: 1000 ms -> 850 ms, so 900 ms ends it and 700 ms does not.
+ self.assertEqual(len(self._end_frames(15000, 900, schedule)), 1)
+ self.assertEqual(self._end_frames(15000, 700, schedule), [])
+ # 55 s into a segment: 200 ms -> 50 ms, so 300 ms ends it.
+ self.assertEqual(len(self._end_frames(55000, 300, schedule)), 1)
+
+ def test_schedule_does_not_depend_on_absolute_position(self):
+ schedule = vad_model.DEFAULT_SILENCE_SCHEDULE
+ for speech_ms, silence_ms in [(5000, 900), (15000, 900), (55000, 300)]:
+ at_zero = self._end_frames(speech_ms, silence_ms, schedule)
+ shifted = self._end_frames(speech_ms, silence_ms, schedule, start_frame=6000)
+ self.assertEqual(at_zero, shifted)
+
+ def test_without_schedule_threshold_stays_fixed(self):
+ # 650 ms fixed threshold: 700 ms of silence ends a 5 s segment, 600 ms does not.
+ self.assertEqual(len(self._end_frames(5000, 700, None)), 1)
+ self.assertEqual(self._end_frames(5000, 600, None), [])
if __name__ == "__main__":