Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix: bound streaming VAD frame buffers (#3236)
zhifu gao3a071af17d84
diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.pyindex 072bebf82..02cbf6c36 100644--- a/funasr/frontends/wav_frontend.py+++ b/funasr/frontends/wav_frontend.py@@ -262,6 +262,8 @@ class WavFrontend(nn.Module):class WavFrontendOnline(nn.Module):"""Conventional frontend structure for streaming ASR/VAD."""+ supports_aligned_waveforms = True+def __init__(self,cmvn_file: str = None,@@ -514,17 +516,30 @@ class WavFrontendOnline(nn.Module):cache = kwargs.get("cache", {})if len(cache) == 0:self.init_cache(cache)+ cache.setdefault("waveform_buffer", None)+ cache.setdefault("waveform_buffer_start_sample", 0)+ cache.setdefault("emitted_lfr_frames", 0)+ cache["aligned_waveforms"] = torch.empty(0)+ return_waveform = kwargs.get("return_waveform", False)batch_size = input.shape[0]assert (batch_size == 1), "we support to extract feature online only when the batch size is equal to 1 now"+ # VAD opts in to exact score-aligned waveform spans.+ if return_waveform and input.shape[1] > 0:+ cache["waveform_buffer"] = (+ input.clone()+ if cache["waveform_buffer"] is None+ else torch.cat((cache["waveform_buffer"], input), dim=1)+ )waveforms, feats, feats_lengths = self.forward_fbank(input, input_lengths, cache=cache) # input shape: B T D+ has_fbank_frames = bool(feats.shape[0])- if feats.shape[0]:+ if has_fbank_frames:cache["waveforms"] = torch.cat((cache["reserve_waveforms"], waveforms), dim=1)@@ -587,6 +602,41 @@ class WavFrontendOnline(nn.Module):feats, feats_lengths, _ = self.forward_lfr_cmvn(feats, feats_lengths, is_final, cache=cache)++ if return_waveform and feats.ndim == 3 and feats.shape[1] > 0:+ score_frame_shift = self.lfr_n * self.frame_shift_sample_length+ aligned_start = cache["emitted_lfr_frames"] * score_frame_shift+ aligned_sample_count = (+ (feats.shape[1] - 1) * score_frame_shift + self.frame_sample_length+ )+ aligned_end = aligned_start + aligned_sample_count+ waveform_buffer = cache["waveform_buffer"]+ buffer_start = cache["waveform_buffer_start_sample"]+ buffer_end = buffer_start + (+ 0 if waveform_buffer is None else waveform_buffer.shape[1]+ )+ if (+ waveform_buffer is None+ or aligned_start < buffer_start+ or aligned_end > buffer_end+ ):+ raise RuntimeError(+ "Frontend emitted LFR frames without enough waveform samples: "+ f"need [{aligned_start}, {aligned_end}), "+ f"have [{buffer_start}, {buffer_end})"+ )+ local_start = aligned_start - buffer_start+ local_end = aligned_end - buffer_start+ cache["aligned_waveforms"] = waveform_buffer[+ :, local_start:local_end+ ].clone()+ cache["emitted_lfr_frames"] += feats.shape[1]++ next_frame_start = cache["emitted_lfr_frames"] * score_frame_shift+ next_buffer_start = min(next_frame_start, buffer_end)+ drop_samples = next_buffer_start - buffer_start+ cache["waveform_buffer"] = waveform_buffer[:, drop_samples:].clone()+ cache["waveform_buffer_start_sample"] = next_buffer_start# if is_final:# self.init_cache(cache)return feats, feats_lengths@@ -603,6 +653,10 @@ class WavFrontendOnline(nn.Module):cache["input_cache"] = torch.empty(0)cache["lfr_splice_cache"] = []cache["waveforms"] = None+ cache["aligned_waveforms"] = torch.empty(0)+ cache["waveform_buffer"] = None+ cache["waveform_buffer_start_sample"] = 0+ cache["emitted_lfr_frames"] = 0cache["fbanks"] = Nonecache["fbanks_lens"] = Nonereturn cachediff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.pyindex 47bedfae5..d8537ed20 100644--- a/funasr/models/fsmn_vad_streaming/model.py+++ b/funasr/models/fsmn_vad_streaming/model.py@@ -403,6 +403,35 @@ class FsmnVADStreaming(nn.Module):self.encoder = encoderself.encoder_conf = encoder_conf+ def DropCachedFrames(self, drop_frames: int, cache: dict = None) -> None:+ """Release score, decibel, and waveform history before an absolute frame."""+ if cache is None:+ cache = {}+ stats = cache["stats"]+ if drop_frames > stats.frm_cnt:+ raise RuntimeError(+ f"Cannot drop through frame {drop_frames}; only {stats.frm_cnt} frames exist"+ )++ real_drop_frames = drop_frames - stats.last_drop_frames+ if real_drop_frames <= 0:+ return++ frame_shift_length = int(+ self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000+ )+ drop_samples = real_drop_frames * frame_shift_length+ stats.data_buf_all = stats.data_buf_all[drop_samples:].clone()+ stats.decibel = stats.decibel[real_drop_frames:]+ stats.scores = stats.scores[:, real_drop_frames:, :].clone()+ stats.last_drop_frames = drop_frames++ data_buf_offset = max(+ stats.data_buf_start_frame - stats.last_drop_frames,+ 0,+ ) * frame_shift_length+ stats.data_buf = stats.data_buf_all[data_buf_offset:]+def ResetDetection(self, cache: dict = None):"""Resetdetection.@@ -424,44 +453,80 @@ class FsmnVADStreaming(nn.Module):if cache["stats"].output_data_buf:assert cache["stats"].output_data_buf[-1].contain_seg_end_point == Truedrop_frames = int(cache["stats"].output_data_buf[-1].end_ms / self.vad_opts.frame_in_ms)- real_drop_frames = drop_frames - cache["stats"].last_drop_frames- cache["stats"].last_drop_frames = drop_frames- cache["stats"].data_buf_all = cache["stats"].data_buf_all[- real_drop_frames- * int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) :- ]- cache["stats"].decibel = cache["stats"].decibel[real_drop_frames:]- cache["stats"].scores = cache["stats"].scores[:, real_drop_frames:, :]-- def ComputeDecibel(self, cache: dict = None) -> None:+ self.DropCachedFrames(drop_frames, cache=cache)++ def ComputeDecibel(+ self, cache: dict = None, frame_count: Optional[int] = None+ ) -> None:"""Computedecibel.Args:+ frame_count: Number of VAD score frames emitted for this waveform.cache: State cache dict for streaming inference."""if cache is None:cache = {}frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000)frame_shift_length = int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000)- if cache["stats"].data_buf_all is None:- cache["stats"].data_buf_all = cache["stats"].waveform[- 0- ] # cache["stats"].data_buf is pointed to cache["stats"].waveform[0]- cache["stats"].data_buf = cache["stats"].data_buf_all+ if frame_count is not None and int(frame_count) <= 0:+ return+ stats = cache["stats"]+ waveform = stats.waveform[0]++ frame_count_was_provided = frame_count is not None+ available_frames = max(+ (waveform.numel() - frame_sample_length) // frame_shift_length + 1,+ 0,+ )+ frame_count = available_frames if frame_count is None else int(frame_count)+ if frame_count <= 0:+ return++ # The frontend provides the exact waveform span for these VAD score frames.+ aligned_sample_count = (+ (frame_count - 1) * frame_shift_length + frame_sample_length+ )+ if waveform.numel() != aligned_sample_count:+ if frame_count_was_provided or waveform.numel() < aligned_sample_count:+ raise RuntimeError(+ "VAD score frames and waveform samples are not aligned: "+ f"expected {aligned_sample_count}, got {waveform.numel()}"+ )+ waveform = waveform[:aligned_sample_count]++ aligned_waveform = waveform++ if stats.data_buf_all is None:+ stats.data_buf_all = aligned_waveform.clone()else:- cache["stats"].data_buf_all = torch.cat(- (cache["stats"].data_buf_all, cache["stats"].waveform[0])+ new_sample_count = frame_count * frame_shift_length+ stats.data_buf_all = torch.cat(+ (stats.data_buf_all, aligned_waveform[-new_sample_count:]))-- waveform_numpy = cache["stats"].waveform.numpy()- offsets = np.arange(0, waveform_numpy.shape[1] - frame_sample_length + 1, frame_shift_length)- frames = waveform_numpy[0, offsets[:, np.newaxis] + np.arange(frame_sample_length)]+ data_buf_offset = max(+ stats.data_buf_start_frame - stats.last_drop_frames,+ 0,+ ) * frame_shift_length+ stats.data_buf = stats.data_buf_all[data_buf_offset:]- decibel_numpy = 10 * np.log10(np.sum(np.square(frames), axis=1) + 0.000001)+ waveform_numpy = aligned_waveform.detach().cpu().numpy()++ offsets = np.arange(+ 0,+ waveform_numpy.shape[0] - frame_sample_length + 1,+ frame_shift_length,+ )+ frames = waveform_numpy[+ offsets[:, np.newaxis] + np.arange(frame_sample_length)+ ]++ decibel_numpy = 10 * np.log10(+ np.sum(np.square(frames), axis=1) + 0.000001+ )decibel_numpy = decibel_numpy.tolist()- cache["stats"].decibel.extend(decibel_numpy)+ stats.decibel.extend(decibel_numpy)def ComputeScores(self, feats: torch.Tensor, cache: dict = None) -> None:@@ -710,7 +775,6 @@ class FsmnVADStreaming(nn.Module):# for each frame, calc log posterior probability of each stateif cur_decibel < self.vad_opts.decibel_thres:frame_state = FrameState.kFrameStateSil- self.DetectOneFrame(frame_state, t, False, cache=cache)return frame_statesum_score = 0.0@@ -777,17 +841,20 @@ class FsmnVADStreaming(nn.Module):"""if cache is None:cache = {}+ if feats.numel() == 0:+ return []# if len(cache) == 0:# self.AllResetDetection()# self.waveform = waveform # compute decibel for each framecache["stats"].waveform = waveformis_streaming_input = kwargs.get("is_streaming_input", True)- self.ComputeDecibel(cache=cache)+ self.ComputeDecibel(frame_count=feats.shape[1], cache=cache)self.ComputeScores(feats, cache=cache)if not is_final:self.DetectCommonFrames(cache=cache)else:self.DetectLastFrames(cache=cache)+ self.DropCachedFrames(cache["stats"].data_buf_start_frame, cache=cache)segments = []for batch_num in range(0, feats.shape[0]): # only support batch_size = 1 nowsegment_batch = []@@ -948,6 +1015,9 @@ class FsmnVADStreaming(nn.Module):speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thresaccumulated_ms = cache.get("_dynamic_accumulated_ms", 0)in_speech = cache.get("_dynamic_in_speech", False)+ frontend_inference_kwargs = {}+ if getattr(frontend, "supports_aligned_waveforms", False):+ frontend_inference_kwargs["return_waveform"] = Truefor i in range(n):kwargs["is_final"] = _is_final and i == n - 1@@ -975,6 +1045,7 @@ class FsmnVADStreaming(nn.Module):frontend=frontend,cache=cache["frontend"],is_final=kwargs["is_final"],+ **frontend_inference_kwargs,)time3 = time.perf_counter()meta_data["extract_feat"] = f"{time3 - time2:0.3f}"@@ -984,9 +1055,18 @@ class FsmnVADStreaming(nn.Module):speech = speech.to(device=kwargs["device"])speech_lengths = speech_lengths.to(device=kwargs["device"])+ waveform = cache["frontend"].get("aligned_waveforms")+ if waveform is None or waveform.numel() == 0:+ # Custom frontends may expose an exact span under the legacy key.+ waveform = cache["frontend"].get("waveforms")+ if speech.numel() > 0 and (waveform is None or waveform.numel() == 0):+ raise RuntimeError(+ "Streaming VAD frontend must provide aligned_waveforms or waveforms "+ "when emitting feature frames"+ )batch = {"feats": speech,- "waveform": cache["frontend"]["waveforms"],+ "waveform": waveform,"is_final": kwargs["is_final"],"cache": cache,"is_streaming_input": is_streaming_input,diff --git a/tests/test_fsmn_vad_streaming_buffers.py b/tests/test_fsmn_vad_streaming_buffers.pynew file mode 100644index 000000000..1f97f01ef--- /dev/null+++ b/tests/test_fsmn_vad_streaming_buffers.py@@ -0,0 +1,476 @@+from types import SimpleNamespace+from unittest.mock import patch++import pytest+import torch++from funasr.frontends.wav_frontend import WavFrontendOnline+from funasr.models.fsmn_vad_streaming import model as vad_model+++class DummyEncoder(torch.nn.Module):+ def forward(self, feats, cache=None):+ return torch.zeros(feats.shape[0], feats.shape[1], 2)+++class DummyWindowDetector:+ def Reset(self):+ return None+++def build_vad_and_cache():+ vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)+ torch.nn.Module.__init__(vad)+ vad.encoder = DummyEncoder()+ vad.vad_opts = SimpleNamespace(+ frame_length_ms=25,+ frame_in_ms=10,+ sample_rate=16000,+ nn_eval_block_size=0,+ )+ vad.DetectCommonFrames = lambda cache=None: 0+ vad.DetectLastFrames = lambda cache=None: 0++ cache = {+ "encoder": {},+ "stats": vad_model.Stats(+ sil_pdf_ids=[],+ max_end_sil_frame_cnt_thresh=0,+ speech_noise_thres=0.5,+ ),+ "windows_detector": DummyWindowDetector(),+ }+ return vad, cache+++def run_aligned_frontend_windows(vad, cache):+ source = torch.arange(4000, dtype=torch.float32) / 4000.0+ calls = (+ (source[0:560], 2),+ (source[320:1520], 6),+ (source[1280:2480], 6),+ )+ for waveform, frame_count in calls:+ vad.forward(+ feats=torch.zeros(1, frame_count, 4),+ waveform=waveform.unsqueeze(0),+ cache=cache,+ is_final=False,+ )+ return source+++def test_streaming_buffers_only_append_audio_aligned_with_score_frames():+ vad, cache = build_vad_and_cache()+ source = run_aligned_frontend_windows(vad, cache)+ stats = cache["stats"]++ expected_waveform = source[:2480]+ expected_frames = expected_waveform.unfold(0, 400, 160)+ expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001)++ assert stats.scores.shape[1] == 14+ assert len(stats.decibel) == stats.scores.shape[1]+ assert torch.allclose(torch.tensor(stats.decibel), expected_decibel)+ assert torch.equal(stats.data_buf_all, expected_waveform)+ assert torch.equal(stats.data_buf, expected_waveform)+++def test_reset_detection_drops_completed_aligned_frames():+ vad, cache = build_vad_and_cache()+ source = run_aligned_frontend_windows(vad, cache)+ stats = cache["stats"]++ segment = vad_model.E2EVadSpeechBufWithDoa()+ segment.Reset()+ segment.end_ms = 80+ segment.contain_seg_end_point = True+ stats.output_data_buf.append(segment)+ stats.data_buf_start_frame = 8++ vad.ResetDetection(cache=cache)++ assert stats.last_drop_frames == 8+ assert stats.scores.shape[1] == 6+ assert len(stats.decibel) == 6+ assert torch.equal(stats.data_buf_all, source[1280:2480])+ assert torch.equal(stats.data_buf, stats.data_buf_all)++ vad.forward(+ feats=torch.zeros(1, 2, 4),+ waveform=source[2240:2800].unsqueeze(0),+ cache=cache,+ is_final=False,+ )++ expected_waveform = source[1280:2800]+ expected_frames = expected_waveform.unfold(0, 400, 160)+ expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001)+ assert stats.scores.shape[1] == 8+ assert len(stats.decibel) == 8+ assert torch.allclose(torch.tensor(stats.decibel), expected_decibel)+ assert torch.equal(stats.data_buf_all, expected_waveform)+ assert torch.equal(stats.data_buf, expected_waveform)+++ ("final_size", "expected_frame_counts", "expected_ranges", "expected_sample_count"),+ (+ (160, (2, 6, 6, 3), ((0, 560), (320, 1520), (1280, 2480), (2240, 2960)), 2960),+ (0, (2, 6, 6, 2), ((0, 560), (320, 1520), (1280, 2480), (2240, 2800)), 2800),+ ),+)+def test_real_frontend_exposes_exact_waveforms_for_final_chunks(+ final_size,+ expected_frame_counts,+ expected_ranges,+ expected_sample_count,+):+ frontend = WavFrontendOnline(+ fs=16000,+ n_mels=80,+ frame_length=25,+ frame_shift=10,+ lfr_m=5,+ lfr_n=1,+ dither=0.0,+ upsacle_samples=False,+ )+ frontend_cache = {}+ vad, vad_cache = build_vad_and_cache()+ source = torch.arange(4000, dtype=torch.float32) / 4000.0+ offset = 0++ for index, (chunk_size, frame_count, sample_range) in enumerate(+ zip((960, 960, 960, final_size), expected_frame_counts, expected_ranges)+ ):+ chunk = source[offset : offset + chunk_size]+ offset += chunk_size+ is_final = index == len(expected_frame_counts) - 1+ feats, _ = frontend(+ chunk.unsqueeze(0),+ torch.tensor([chunk.numel()]),+ cache=frontend_cache,+ is_final=is_final,+ return_waveform=True,+ )+ aligned_waveform = frontend_cache["aligned_waveforms"]++ assert feats.shape[1] == frame_count+ assert torch.equal(aligned_waveform[0], source[slice(*sample_range)])+ assert frontend_cache["waveform_buffer"].shape[1] <= 640++ vad.forward(+ feats=feats,+ waveform=aligned_waveform,+ cache=vad_cache,+ is_final=is_final,+ )++ expected_waveform = source[:expected_sample_count]+ expected_frames = expected_waveform.unfold(0, 400, 160)+ expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001)+ stats = vad_cache["stats"]++ assert stats.scores.shape[1] == sum(expected_frame_counts)+ assert len(stats.decibel) == stats.scores.shape[1]+ assert torch.allclose(torch.tensor(stats.decibel), expected_decibel)+ assert torch.equal(stats.data_buf_all, expected_waveform)+++def test_compute_decibel_keeps_positional_cache_call_compatible():+ vad, cache = build_vad_and_cache()+ waveform = torch.arange(560, dtype=torch.float32) / 560.0+ cache["stats"].waveform = waveform.unsqueeze(0)++ vad.ComputeDecibel(cache)++ assert len(cache["stats"].decibel) == 2+ assert torch.equal(cache["stats"].data_buf_all, waveform)+++def test_real_frontend_tracks_initial_padding_across_small_chunks():+ frontend = WavFrontendOnline(+ fs=16000,+ n_mels=80,+ frame_length=25,+ frame_shift=10,+ lfr_m=5,+ lfr_n=1,+ dither=0.0,+ upsacle_samples=False,+ )+ cache = {}+ source = torch.arange(2000, dtype=torch.float32) / 2000.0+ emitted_frames = 0+ input_offset = 0++ for index, (chunk_size, expected_frames) in enumerate(+ zip((480, 480, 480, 160), (0, 2, 3, 3))+ ):+ chunk = source[input_offset : input_offset + chunk_size]+ input_offset += chunk_size+ feats, _ = frontend(+ chunk.unsqueeze(0),+ torch.tensor([chunk.numel()]),+ cache=cache,+ is_final=index == 3,+ return_waveform=True,+ )++ assert (feats.shape[1] if feats.ndim == 3 else 0) == expected_frames+ assert cache["waveform_buffer"].shape[1] <= 640+ if expected_frames == 0:+ assert cache["aligned_waveforms"].numel() == 0+ continue++ aligned_sample_count = (expected_frames - 1) * 160 + 400+ expected_waveform = source[+ emitted_frames * 160 : emitted_frames * 160 + aligned_sample_count+ ]+ assert torch.equal(cache["aligned_waveforms"][0], expected_waveform)+ emitted_frames += expected_frames++ assert emitted_frames == 8+++def test_frontend_cursor_does_not_advance_past_received_audio():+ frontend = WavFrontendOnline(+ fs=16000,+ n_mels=80,+ frame_length=25,+ frame_shift=10,+ lfr_m=7,+ lfr_n=6,+ dither=0.0,+ upsacle_samples=False,+ )+ cache = {}+ source = torch.arange(2400, dtype=torch.float32) / 2400.0++ first_feats, _ = frontend(+ source[:880].unsqueeze(0),+ torch.tensor([880]),+ cache=cache,+ is_final=False,+ return_waveform=True,+ )++ assert first_feats.shape[1] == 1+ assert torch.equal(cache["aligned_waveforms"][0], source[:400])+ assert cache["waveform_buffer_start_sample"] == 880++ second_feats, _ = frontend(+ source[880:1840].unsqueeze(0),+ torch.tensor([960]),+ cache=cache,+ is_final=False,+ return_waveform=True,+ )++ assert second_feats.shape[1] == 1+ assert torch.equal(cache["aligned_waveforms"][0], source[960:1360])+++def test_zero_score_frontend_batch_is_a_model_noop():+ frontend = WavFrontendOnline(+ fs=16000,+ n_mels=80,+ frame_length=25,+ frame_shift=10,+ lfr_m=5,+ lfr_n=1,+ dither=0.0,+ upsacle_samples=False,+ )+ frontend_cache = {}+ source = torch.arange(480, dtype=torch.float32) / 480.0+ feats, _ = frontend(+ source.unsqueeze(0),+ torch.tensor([source.numel()]),+ cache=frontend_cache,+ is_final=False,+ return_waveform=True,+ )+ vad, cache = build_vad_and_cache()++ assert feats.numel() == 0+ assert (+ vad.forward(+ feats=feats,+ waveform=frontend_cache["aligned_waveforms"],+ cache=cache,+ is_final=False,+ )+ == []+ )+ assert cache["stats"].data_buf_all is None+++def test_consumed_silence_history_is_compacted_without_an_endpoint():+ vad, cache = build_vad_and_cache()++ def consume_all_frames(cache=None):+ cache["stats"].data_buf_start_frame = cache["stats"].frm_cnt++ vad.DetectCommonFrames = consume_all_frames+ source = torch.arange(100 * 960 + 400, dtype=torch.float32) / 100000.0++ for index in range(100):+ start = index * 960+ vad.forward(+ feats=torch.zeros(1, 6, 4),+ waveform=source[start : start + 1200].unsqueeze(0),+ cache=cache,+ is_final=False,+ )++ stats = cache["stats"]+ assert stats.last_drop_frames == stats.frm_cnt == 600+ assert stats.data_buf_all.numel() == 240+ assert stats.data_buf.numel() == 240+ assert stats.scores.shape[1] == 0+ assert stats.decibel == []+++def test_waveform_alignment_is_opt_in_for_other_frontend_consumers():+ frontend = WavFrontendOnline(+ fs=16000,+ n_mels=80,+ frame_length=25,+ frame_shift=10,+ lfr_m=5,+ lfr_n=1,+ dither=0.0,+ upsacle_samples=False,+ )+ cache = {}+ waveform = torch.zeros(1, 960)++ frontend(+ waveform,+ torch.tensor([waveform.shape[1]]),+ cache=cache,+ is_final=False,+ )++ assert cache["waveform_buffer"] is None+ assert cache["aligned_waveforms"].numel() == 0+++def test_misaligned_waveform_fails_before_mutating_model_buffers():+ vad, cache = build_vad_and_cache()+ cache["stats"].waveform = torch.zeros(1, 880)++ with pytest.raises(RuntimeError, match="score frames and waveform samples"):+ vad.ComputeDecibel(cache=cache, frame_count=2)++ assert cache["stats"].data_buf_all is None+ assert cache["stats"].decibel == []+++def test_low_decibel_frame_advances_detector_once_with_absolute_index():+ vad, cache = build_vad_and_cache()+ stats = cache["stats"]+ stats.decibel = [-100.0]+ stats.scores = torch.zeros(1, 1, 2)+ stats.frm_cnt = 101+ stats.last_drop_frames = 100+ vad.vad_opts.decibel_thres = 0.0+ vad.vad_opts.nn_eval_block_size = 1+ detected_frames = []+ vad.DetectOneFrame = lambda frame_state, frame_index, is_final, cache=None: (+ detected_frames.append(frame_index)+ )++ vad_model.FsmnVADStreaming.DetectCommonFrames(vad, cache=cache)++ assert detected_frames == [100]++[email protected]("supports_alignment", (False, True))+def test_inference_negotiates_aligned_waveform_support(supports_alignment):+ vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)+ vad.vad_opts = SimpleNamespace(speech_to_sil_time_thres=100)+ vad.forward = lambda **batch: []+ cache = {+ "frontend": {},+ "prev_samples": torch.empty(0),+ "encoder": {},+ "stats": SimpleNamespace(),+ }+ frontend = SimpleNamespace(fs=16000, frame_shift=10, lfr_n=1)+ if supports_alignment:+ frontend.supports_aligned_waveforms = True+ extract_kwargs = []++ def fake_extract_fbank(*args, **kwargs):+ extract_kwargs.append(kwargs)+ cache["frontend"]["waveforms"] = torch.zeros(1, 560)+ return torch.zeros(1, 2, 4), torch.tensor([2])++ with (+ patch.object(+ vad_model,+ "load_audio_text_image_video",+ return_value=[torch.zeros(960)],+ ),+ patch.object(vad_model, "extract_fbank", side_effect=fake_extract_fbank),+ ):+ vad_model.FsmnVADStreaming.inference(+ vad,+ torch.zeros(960),+ frontend=frontend,+ cache=cache,+ key=["utt"],+ chunk_size=60,+ is_final=False,+ device="cpu",+ dynamic_silence=False,+ )++ assert extract_kwargs[0].get("return_waveform", False) is supports_alignment+++def test_inference_rejects_missing_waveform_before_forward():+ vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)+ vad.vad_opts = SimpleNamespace(speech_to_sil_time_thres=100)+ forward_batches = []+ vad.forward = lambda **batch: forward_batches.append(batch) or []+ cache = {+ "frontend": {},+ "prev_samples": torch.empty(0),+ "encoder": {},+ "stats": SimpleNamespace(),+ }+ frontend = SimpleNamespace(fs=16000, frame_shift=10, lfr_n=1)++ with (+ patch.object(+ vad_model,+ "load_audio_text_image_video",+ return_value=[torch.zeros(960)],+ ),+ patch.object(+ vad_model,+ "extract_fbank",+ return_value=(torch.zeros(1, 2, 4), torch.tensor([2])),+ ),+ pytest.raises(+ RuntimeError,+ match="must provide aligned_waveforms or waveforms",+ ),+ ):+ vad_model.FsmnVADStreaming.inference(+ vad,+ torch.zeros(960),+ frontend=frontend,+ cache=cache,+ key=["utt"],+ chunk_size=60,+ is_final=False,+ device="cpu",+ dynamic_silence=False,+ )++ assert forward_batches == []