Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix: bound streaming VAD frame buffers (#3236)

zhifu gao3a071af17d84
diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.py
index 072bebf82..02cbf6c36 100644
--- a/funasr/frontends/wav_frontend.py
+++ b/funasr/frontends/wav_frontend.py
@@ -262,6 +262,8 @@ class WavFrontend(nn.Module):
class WavFrontendOnline(nn.Module):
"""Conventional frontend structure for streaming ASR/VAD."""
+ supports_aligned_waveforms = True
+
def __init__(
self,
cmvn_file: str = None,
@@ -514,17 +516,30 @@ class WavFrontendOnline(nn.Module):
cache = kwargs.get("cache", {})
if len(cache) == 0:
self.init_cache(cache)
+ cache.setdefault("waveform_buffer", None)
+ cache.setdefault("waveform_buffer_start_sample", 0)
+ cache.setdefault("emitted_lfr_frames", 0)
+ cache["aligned_waveforms"] = torch.empty(0)
+ return_waveform = kwargs.get("return_waveform", False)
batch_size = input.shape[0]
assert (
batch_size == 1
), "we support to extract feature online only when the batch size is equal to 1 now"
+ # VAD opts in to exact score-aligned waveform spans.
+ if return_waveform and input.shape[1] > 0:
+ cache["waveform_buffer"] = (
+ input.clone()
+ if cache["waveform_buffer"] is None
+ else torch.cat((cache["waveform_buffer"], input), dim=1)
+ )
waveforms, feats, feats_lengths = self.forward_fbank(
input, input_lengths, cache=cache
) # input shape: B T D
+ has_fbank_frames = bool(feats.shape[0])
- if feats.shape[0]:
+ if has_fbank_frames:
cache["waveforms"] = torch.cat((cache["reserve_waveforms"], waveforms), dim=1)
@@ -587,6 +602,41 @@ class WavFrontendOnline(nn.Module):
feats, feats_lengths, _ = self.forward_lfr_cmvn(
feats, feats_lengths, is_final, cache=cache
)
+
+ if return_waveform and feats.ndim == 3 and feats.shape[1] > 0:
+ score_frame_shift = self.lfr_n * self.frame_shift_sample_length
+ aligned_start = cache["emitted_lfr_frames"] * score_frame_shift
+ aligned_sample_count = (
+ (feats.shape[1] - 1) * score_frame_shift + self.frame_sample_length
+ )
+ aligned_end = aligned_start + aligned_sample_count
+ waveform_buffer = cache["waveform_buffer"]
+ buffer_start = cache["waveform_buffer_start_sample"]
+ buffer_end = buffer_start + (
+ 0 if waveform_buffer is None else waveform_buffer.shape[1]
+ )
+ if (
+ waveform_buffer is None
+ or aligned_start < buffer_start
+ or aligned_end > buffer_end
+ ):
+ raise RuntimeError(
+ "Frontend emitted LFR frames without enough waveform samples: "
+ f"need [{aligned_start}, {aligned_end}), "
+ f"have [{buffer_start}, {buffer_end})"
+ )
+ local_start = aligned_start - buffer_start
+ local_end = aligned_end - buffer_start
+ cache["aligned_waveforms"] = waveform_buffer[
+ :, local_start:local_end
+ ].clone()
+ cache["emitted_lfr_frames"] += feats.shape[1]
+
+ next_frame_start = cache["emitted_lfr_frames"] * score_frame_shift
+ next_buffer_start = min(next_frame_start, buffer_end)
+ drop_samples = next_buffer_start - buffer_start
+ cache["waveform_buffer"] = waveform_buffer[:, drop_samples:].clone()
+ cache["waveform_buffer_start_sample"] = next_buffer_start
# if is_final:
# self.init_cache(cache)
return feats, feats_lengths
@@ -603,6 +653,10 @@ class WavFrontendOnline(nn.Module):
cache["input_cache"] = torch.empty(0)
cache["lfr_splice_cache"] = []
cache["waveforms"] = None
+ cache["aligned_waveforms"] = torch.empty(0)
+ cache["waveform_buffer"] = None
+ cache["waveform_buffer_start_sample"] = 0
+ cache["emitted_lfr_frames"] = 0
cache["fbanks"] = None
cache["fbanks_lens"] = None
return cache
diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py
index 47bedfae5..d8537ed20 100644
--- a/funasr/models/fsmn_vad_streaming/model.py
+++ b/funasr/models/fsmn_vad_streaming/model.py
@@ -403,6 +403,35 @@ class FsmnVADStreaming(nn.Module):
self.encoder = encoder
self.encoder_conf = encoder_conf
+ def DropCachedFrames(self, drop_frames: int, cache: dict = None) -> None:
+ """Release score, decibel, and waveform history before an absolute frame."""
+ if cache is None:
+ cache = {}
+ stats = cache["stats"]
+ if drop_frames > stats.frm_cnt:
+ raise RuntimeError(
+ f"Cannot drop through frame {drop_frames}; only {stats.frm_cnt} frames exist"
+ )
+
+ real_drop_frames = drop_frames - stats.last_drop_frames
+ if real_drop_frames <= 0:
+ return
+
+ frame_shift_length = int(
+ self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000
+ )
+ drop_samples = real_drop_frames * frame_shift_length
+ stats.data_buf_all = stats.data_buf_all[drop_samples:].clone()
+ stats.decibel = stats.decibel[real_drop_frames:]
+ stats.scores = stats.scores[:, real_drop_frames:, :].clone()
+ stats.last_drop_frames = drop_frames
+
+ data_buf_offset = max(
+ stats.data_buf_start_frame - stats.last_drop_frames,
+ 0,
+ ) * frame_shift_length
+ stats.data_buf = stats.data_buf_all[data_buf_offset:]
+
def ResetDetection(self, cache: dict = None):
"""Resetdetection.
@@ -424,44 +453,80 @@ class FsmnVADStreaming(nn.Module):
if cache["stats"].output_data_buf:
assert cache["stats"].output_data_buf[-1].contain_seg_end_point == True
drop_frames = int(cache["stats"].output_data_buf[-1].end_ms / self.vad_opts.frame_in_ms)
- real_drop_frames = drop_frames - cache["stats"].last_drop_frames
- cache["stats"].last_drop_frames = drop_frames
- cache["stats"].data_buf_all = cache["stats"].data_buf_all[
- real_drop_frames
- * int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) :
- ]
- cache["stats"].decibel = cache["stats"].decibel[real_drop_frames:]
- cache["stats"].scores = cache["stats"].scores[:, real_drop_frames:, :]
-
- def ComputeDecibel(self, cache: dict = None) -> None:
+ self.DropCachedFrames(drop_frames, cache=cache)
+
+ def ComputeDecibel(
+ self, cache: dict = None, frame_count: Optional[int] = None
+ ) -> None:
"""Computedecibel.
Args:
+ frame_count: Number of VAD score frames emitted for this waveform.
cache: State cache dict for streaming inference.
"""
if cache is None:
cache = {}
frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000)
frame_shift_length = int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000)
- if cache["stats"].data_buf_all is None:
- cache["stats"].data_buf_all = cache["stats"].waveform[
- 0
- ] # cache["stats"].data_buf is pointed to cache["stats"].waveform[0]
- cache["stats"].data_buf = cache["stats"].data_buf_all
+ if frame_count is not None and int(frame_count) <= 0:
+ return
+ stats = cache["stats"]
+ waveform = stats.waveform[0]
+
+ frame_count_was_provided = frame_count is not None
+ available_frames = max(
+ (waveform.numel() - frame_sample_length) // frame_shift_length + 1,
+ 0,
+ )
+ frame_count = available_frames if frame_count is None else int(frame_count)
+ if frame_count <= 0:
+ return
+
+ # The frontend provides the exact waveform span for these VAD score frames.
+ aligned_sample_count = (
+ (frame_count - 1) * frame_shift_length + frame_sample_length
+ )
+ if waveform.numel() != aligned_sample_count:
+ if frame_count_was_provided or waveform.numel() < aligned_sample_count:
+ raise RuntimeError(
+ "VAD score frames and waveform samples are not aligned: "
+ f"expected {aligned_sample_count}, got {waveform.numel()}"
+ )
+ waveform = waveform[:aligned_sample_count]
+
+ aligned_waveform = waveform
+
+ if stats.data_buf_all is None:
+ stats.data_buf_all = aligned_waveform.clone()
else:
- cache["stats"].data_buf_all = torch.cat(
- (cache["stats"].data_buf_all, cache["stats"].waveform[0])
+ new_sample_count = frame_count * frame_shift_length
+ stats.data_buf_all = torch.cat(
+ (stats.data_buf_all, aligned_waveform[-new_sample_count:])
)
-
- waveform_numpy = cache["stats"].waveform.numpy()
- offsets = np.arange(0, waveform_numpy.shape[1] - frame_sample_length + 1, frame_shift_length)
- frames = waveform_numpy[0, offsets[:, np.newaxis] + np.arange(frame_sample_length)]
+ data_buf_offset = max(
+ stats.data_buf_start_frame - stats.last_drop_frames,
+ 0,
+ ) * frame_shift_length
+ stats.data_buf = stats.data_buf_all[data_buf_offset:]
- decibel_numpy = 10 * np.log10(np.sum(np.square(frames), axis=1) + 0.000001)
+ waveform_numpy = aligned_waveform.detach().cpu().numpy()
+
+ offsets = np.arange(
+ 0,
+ waveform_numpy.shape[0] - frame_sample_length + 1,
+ frame_shift_length,
+ )
+ frames = waveform_numpy[
+ offsets[:, np.newaxis] + np.arange(frame_sample_length)
+ ]
+
+ decibel_numpy = 10 * np.log10(
+ np.sum(np.square(frames), axis=1) + 0.000001
+ )
decibel_numpy = decibel_numpy.tolist()
- cache["stats"].decibel.extend(decibel_numpy)
+ stats.decibel.extend(decibel_numpy)
def ComputeScores(self, feats: torch.Tensor, cache: dict = None) -> None:
@@ -710,7 +775,6 @@ class FsmnVADStreaming(nn.Module):
# for each frame, calc log posterior probability of each state
if cur_decibel < self.vad_opts.decibel_thres:
frame_state = FrameState.kFrameStateSil
- self.DetectOneFrame(frame_state, t, False, cache=cache)
return frame_state
sum_score = 0.0
@@ -777,17 +841,20 @@ class FsmnVADStreaming(nn.Module):
"""
if cache is None:
cache = {}
+ if feats.numel() == 0:
+ return []
# if len(cache) == 0:
# self.AllResetDetection()
# self.waveform = waveform # compute decibel for each frame
cache["stats"].waveform = waveform
is_streaming_input = kwargs.get("is_streaming_input", True)
- self.ComputeDecibel(cache=cache)
+ self.ComputeDecibel(frame_count=feats.shape[1], cache=cache)
self.ComputeScores(feats, cache=cache)
if not is_final:
self.DetectCommonFrames(cache=cache)
else:
self.DetectLastFrames(cache=cache)
+ self.DropCachedFrames(cache["stats"].data_buf_start_frame, cache=cache)
segments = []
for batch_num in range(0, feats.shape[0]): # only support batch_size = 1 now
segment_batch = []
@@ -948,6 +1015,9 @@ class FsmnVADStreaming(nn.Module):
speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thres
accumulated_ms = cache.get("_dynamic_accumulated_ms", 0)
in_speech = cache.get("_dynamic_in_speech", False)
+ frontend_inference_kwargs = {}
+ if getattr(frontend, "supports_aligned_waveforms", False):
+ frontend_inference_kwargs["return_waveform"] = True
for i in range(n):
kwargs["is_final"] = _is_final and i == n - 1
@@ -975,6 +1045,7 @@ class FsmnVADStreaming(nn.Module):
frontend=frontend,
cache=cache["frontend"],
is_final=kwargs["is_final"],
+ **frontend_inference_kwargs,
)
time3 = time.perf_counter()
meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
@@ -984,9 +1055,18 @@ class FsmnVADStreaming(nn.Module):
speech = speech.to(device=kwargs["device"])
speech_lengths = speech_lengths.to(device=kwargs["device"])
+ waveform = cache["frontend"].get("aligned_waveforms")
+ if waveform is None or waveform.numel() == 0:
+ # Custom frontends may expose an exact span under the legacy key.
+ waveform = cache["frontend"].get("waveforms")
+ if speech.numel() > 0 and (waveform is None or waveform.numel() == 0):
+ raise RuntimeError(
+ "Streaming VAD frontend must provide aligned_waveforms or waveforms "
+ "when emitting feature frames"
+ )
batch = {
"feats": speech,
- "waveform": cache["frontend"]["waveforms"],
+ "waveform": waveform,
"is_final": kwargs["is_final"],
"cache": cache,
"is_streaming_input": is_streaming_input,
diff --git a/tests/test_fsmn_vad_streaming_buffers.py b/tests/test_fsmn_vad_streaming_buffers.py
new file mode 100644
index 000000000..1f97f01ef
--- /dev/null
+++ b/tests/test_fsmn_vad_streaming_buffers.py
@@ -0,0 +1,476 @@
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+import torch
+
+from funasr.frontends.wav_frontend import WavFrontendOnline
+from funasr.models.fsmn_vad_streaming import model as vad_model
+
+
+class DummyEncoder(torch.nn.Module):
+ def forward(self, feats, cache=None):
+ return torch.zeros(feats.shape[0], feats.shape[1], 2)
+
+
+class DummyWindowDetector:
+ def Reset(self):
+ return None
+
+
+def build_vad_and_cache():
+ vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)
+ torch.nn.Module.__init__(vad)
+ vad.encoder = DummyEncoder()
+ vad.vad_opts = SimpleNamespace(
+ frame_length_ms=25,
+ frame_in_ms=10,
+ sample_rate=16000,
+ nn_eval_block_size=0,
+ )
+ vad.DetectCommonFrames = lambda cache=None: 0
+ vad.DetectLastFrames = lambda cache=None: 0
+
+ cache = {
+ "encoder": {},
+ "stats": vad_model.Stats(
+ sil_pdf_ids=[],
+ max_end_sil_frame_cnt_thresh=0,
+ speech_noise_thres=0.5,
+ ),
+ "windows_detector": DummyWindowDetector(),
+ }
+ return vad, cache
+
+
+def run_aligned_frontend_windows(vad, cache):
+ source = torch.arange(4000, dtype=torch.float32) / 4000.0
+ calls = (
+ (source[0:560], 2),
+ (source[320:1520], 6),
+ (source[1280:2480], 6),
+ )
+ for waveform, frame_count in calls:
+ vad.forward(
+ feats=torch.zeros(1, frame_count, 4),
+ waveform=waveform.unsqueeze(0),
+ cache=cache,
+ is_final=False,
+ )
+ return source
+
+
+def test_streaming_buffers_only_append_audio_aligned_with_score_frames():
+ vad, cache = build_vad_and_cache()
+ source = run_aligned_frontend_windows(vad, cache)
+ stats = cache["stats"]
+
+ expected_waveform = source[:2480]
+ expected_frames = expected_waveform.unfold(0, 400, 160)
+ expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001)
+
+ assert stats.scores.shape[1] == 14
+ assert len(stats.decibel) == stats.scores.shape[1]
+ assert torch.allclose(torch.tensor(stats.decibel), expected_decibel)
+ assert torch.equal(stats.data_buf_all, expected_waveform)
+ assert torch.equal(stats.data_buf, expected_waveform)
+
+
+def test_reset_detection_drops_completed_aligned_frames():
+ vad, cache = build_vad_and_cache()
+ source = run_aligned_frontend_windows(vad, cache)
+ stats = cache["stats"]
+
+ segment = vad_model.E2EVadSpeechBufWithDoa()
+ segment.Reset()
+ segment.end_ms = 80
+ segment.contain_seg_end_point = True
+ stats.output_data_buf.append(segment)
+ stats.data_buf_start_frame = 8
+
+ vad.ResetDetection(cache=cache)
+
+ assert stats.last_drop_frames == 8
+ assert stats.scores.shape[1] == 6
+ assert len(stats.decibel) == 6
+ assert torch.equal(stats.data_buf_all, source[1280:2480])
+ assert torch.equal(stats.data_buf, stats.data_buf_all)
+
+ vad.forward(
+ feats=torch.zeros(1, 2, 4),
+ waveform=source[2240:2800].unsqueeze(0),
+ cache=cache,
+ is_final=False,
+ )
+
+ expected_waveform = source[1280:2800]
+ expected_frames = expected_waveform.unfold(0, 400, 160)
+ expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001)
+ assert stats.scores.shape[1] == 8
+ assert len(stats.decibel) == 8
+ assert torch.allclose(torch.tensor(stats.decibel), expected_decibel)
+ assert torch.equal(stats.data_buf_all, expected_waveform)
+ assert torch.equal(stats.data_buf, expected_waveform)
+
+
+ ("final_size", "expected_frame_counts", "expected_ranges", "expected_sample_count"),
+ (
+ (160, (2, 6, 6, 3), ((0, 560), (320, 1520), (1280, 2480), (2240, 2960)), 2960),
+ (0, (2, 6, 6, 2), ((0, 560), (320, 1520), (1280, 2480), (2240, 2800)), 2800),
+ ),
+)
+def test_real_frontend_exposes_exact_waveforms_for_final_chunks(
+ final_size,
+ expected_frame_counts,
+ expected_ranges,
+ expected_sample_count,
+):
+ frontend = WavFrontendOnline(
+ fs=16000,
+ n_mels=80,
+ frame_length=25,
+ frame_shift=10,
+ lfr_m=5,
+ lfr_n=1,
+ dither=0.0,
+ upsacle_samples=False,
+ )
+ frontend_cache = {}
+ vad, vad_cache = build_vad_and_cache()
+ source = torch.arange(4000, dtype=torch.float32) / 4000.0
+ offset = 0
+
+ for index, (chunk_size, frame_count, sample_range) in enumerate(
+ zip((960, 960, 960, final_size), expected_frame_counts, expected_ranges)
+ ):
+ chunk = source[offset : offset + chunk_size]
+ offset += chunk_size
+ is_final = index == len(expected_frame_counts) - 1
+ feats, _ = frontend(
+ chunk.unsqueeze(0),
+ torch.tensor([chunk.numel()]),
+ cache=frontend_cache,
+ is_final=is_final,
+ return_waveform=True,
+ )
+ aligned_waveform = frontend_cache["aligned_waveforms"]
+
+ assert feats.shape[1] == frame_count
+ assert torch.equal(aligned_waveform[0], source[slice(*sample_range)])
+ assert frontend_cache["waveform_buffer"].shape[1] <= 640
+
+ vad.forward(
+ feats=feats,
+ waveform=aligned_waveform,
+ cache=vad_cache,
+ is_final=is_final,
+ )
+
+ expected_waveform = source[:expected_sample_count]
+ expected_frames = expected_waveform.unfold(0, 400, 160)
+ expected_decibel = 10 * torch.log10(expected_frames.square().sum(dim=1) + 0.000001)
+ stats = vad_cache["stats"]
+
+ assert stats.scores.shape[1] == sum(expected_frame_counts)
+ assert len(stats.decibel) == stats.scores.shape[1]
+ assert torch.allclose(torch.tensor(stats.decibel), expected_decibel)
+ assert torch.equal(stats.data_buf_all, expected_waveform)
+
+
+def test_compute_decibel_keeps_positional_cache_call_compatible():
+ vad, cache = build_vad_and_cache()
+ waveform = torch.arange(560, dtype=torch.float32) / 560.0
+ cache["stats"].waveform = waveform.unsqueeze(0)
+
+ vad.ComputeDecibel(cache)
+
+ assert len(cache["stats"].decibel) == 2
+ assert torch.equal(cache["stats"].data_buf_all, waveform)
+
+
+def test_real_frontend_tracks_initial_padding_across_small_chunks():
+ frontend = WavFrontendOnline(
+ fs=16000,
+ n_mels=80,
+ frame_length=25,
+ frame_shift=10,
+ lfr_m=5,
+ lfr_n=1,
+ dither=0.0,
+ upsacle_samples=False,
+ )
+ cache = {}
+ source = torch.arange(2000, dtype=torch.float32) / 2000.0
+ emitted_frames = 0
+ input_offset = 0
+
+ for index, (chunk_size, expected_frames) in enumerate(
+ zip((480, 480, 480, 160), (0, 2, 3, 3))
+ ):
+ chunk = source[input_offset : input_offset + chunk_size]
+ input_offset += chunk_size
+ feats, _ = frontend(
+ chunk.unsqueeze(0),
+ torch.tensor([chunk.numel()]),
+ cache=cache,
+ is_final=index == 3,
+ return_waveform=True,
+ )
+
+ assert (feats.shape[1] if feats.ndim == 3 else 0) == expected_frames
+ assert cache["waveform_buffer"].shape[1] <= 640
+ if expected_frames == 0:
+ assert cache["aligned_waveforms"].numel() == 0
+ continue
+
+ aligned_sample_count = (expected_frames - 1) * 160 + 400
+ expected_waveform = source[
+ emitted_frames * 160 : emitted_frames * 160 + aligned_sample_count
+ ]
+ assert torch.equal(cache["aligned_waveforms"][0], expected_waveform)
+ emitted_frames += expected_frames
+
+ assert emitted_frames == 8
+
+
+def test_frontend_cursor_does_not_advance_past_received_audio():
+ frontend = WavFrontendOnline(
+ fs=16000,
+ n_mels=80,
+ frame_length=25,
+ frame_shift=10,
+ lfr_m=7,
+ lfr_n=6,
+ dither=0.0,
+ upsacle_samples=False,
+ )
+ cache = {}
+ source = torch.arange(2400, dtype=torch.float32) / 2400.0
+
+ first_feats, _ = frontend(
+ source[:880].unsqueeze(0),
+ torch.tensor([880]),
+ cache=cache,
+ is_final=False,
+ return_waveform=True,
+ )
+
+ assert first_feats.shape[1] == 1
+ assert torch.equal(cache["aligned_waveforms"][0], source[:400])
+ assert cache["waveform_buffer_start_sample"] == 880
+
+ second_feats, _ = frontend(
+ source[880:1840].unsqueeze(0),
+ torch.tensor([960]),
+ cache=cache,
+ is_final=False,
+ return_waveform=True,
+ )
+
+ assert second_feats.shape[1] == 1
+ assert torch.equal(cache["aligned_waveforms"][0], source[960:1360])
+
+
+def test_zero_score_frontend_batch_is_a_model_noop():
+ frontend = WavFrontendOnline(
+ fs=16000,
+ n_mels=80,
+ frame_length=25,
+ frame_shift=10,
+ lfr_m=5,
+ lfr_n=1,
+ dither=0.0,
+ upsacle_samples=False,
+ )
+ frontend_cache = {}
+ source = torch.arange(480, dtype=torch.float32) / 480.0
+ feats, _ = frontend(
+ source.unsqueeze(0),
+ torch.tensor([source.numel()]),
+ cache=frontend_cache,
+ is_final=False,
+ return_waveform=True,
+ )
+ vad, cache = build_vad_and_cache()
+
+ assert feats.numel() == 0
+ assert (
+ vad.forward(
+ feats=feats,
+ waveform=frontend_cache["aligned_waveforms"],
+ cache=cache,
+ is_final=False,
+ )
+ == []
+ )
+ assert cache["stats"].data_buf_all is None
+
+
+def test_consumed_silence_history_is_compacted_without_an_endpoint():
+ vad, cache = build_vad_and_cache()
+
+ def consume_all_frames(cache=None):
+ cache["stats"].data_buf_start_frame = cache["stats"].frm_cnt
+
+ vad.DetectCommonFrames = consume_all_frames
+ source = torch.arange(100 * 960 + 400, dtype=torch.float32) / 100000.0
+
+ for index in range(100):
+ start = index * 960
+ vad.forward(
+ feats=torch.zeros(1, 6, 4),
+ waveform=source[start : start + 1200].unsqueeze(0),
+ cache=cache,
+ is_final=False,
+ )
+
+ stats = cache["stats"]
+ assert stats.last_drop_frames == stats.frm_cnt == 600
+ assert stats.data_buf_all.numel() == 240
+ assert stats.data_buf.numel() == 240
+ assert stats.scores.shape[1] == 0
+ assert stats.decibel == []
+
+
+def test_waveform_alignment_is_opt_in_for_other_frontend_consumers():
+ frontend = WavFrontendOnline(
+ fs=16000,
+ n_mels=80,
+ frame_length=25,
+ frame_shift=10,
+ lfr_m=5,
+ lfr_n=1,
+ dither=0.0,
+ upsacle_samples=False,
+ )
+ cache = {}
+ waveform = torch.zeros(1, 960)
+
+ frontend(
+ waveform,
+ torch.tensor([waveform.shape[1]]),
+ cache=cache,
+ is_final=False,
+ )
+
+ assert cache["waveform_buffer"] is None
+ assert cache["aligned_waveforms"].numel() == 0
+
+
+def test_misaligned_waveform_fails_before_mutating_model_buffers():
+ vad, cache = build_vad_and_cache()
+ cache["stats"].waveform = torch.zeros(1, 880)
+
+ with pytest.raises(RuntimeError, match="score frames and waveform samples"):
+ vad.ComputeDecibel(cache=cache, frame_count=2)
+
+ assert cache["stats"].data_buf_all is None
+ assert cache["stats"].decibel == []
+
+
+def test_low_decibel_frame_advances_detector_once_with_absolute_index():
+ vad, cache = build_vad_and_cache()
+ stats = cache["stats"]
+ stats.decibel = [-100.0]
+ stats.scores = torch.zeros(1, 1, 2)
+ stats.frm_cnt = 101
+ stats.last_drop_frames = 100
+ vad.vad_opts.decibel_thres = 0.0
+ vad.vad_opts.nn_eval_block_size = 1
+ detected_frames = []
+ vad.DetectOneFrame = lambda frame_state, frame_index, is_final, cache=None: (
+ detected_frames.append(frame_index)
+ )
+
+ vad_model.FsmnVADStreaming.DetectCommonFrames(vad, cache=cache)
+
+ assert detected_frames == [100]
+
+
[email protected]("supports_alignment", (False, True))
+def test_inference_negotiates_aligned_waveform_support(supports_alignment):
+ vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)
+ vad.vad_opts = SimpleNamespace(speech_to_sil_time_thres=100)
+ vad.forward = lambda **batch: []
+ cache = {
+ "frontend": {},
+ "prev_samples": torch.empty(0),
+ "encoder": {},
+ "stats": SimpleNamespace(),
+ }
+ frontend = SimpleNamespace(fs=16000, frame_shift=10, lfr_n=1)
+ if supports_alignment:
+ frontend.supports_aligned_waveforms = True
+ extract_kwargs = []
+
+ def fake_extract_fbank(*args, **kwargs):
+ extract_kwargs.append(kwargs)
+ cache["frontend"]["waveforms"] = torch.zeros(1, 560)
+ return torch.zeros(1, 2, 4), torch.tensor([2])
+
+ with (
+ patch.object(
+ vad_model,
+ "load_audio_text_image_video",
+ return_value=[torch.zeros(960)],
+ ),
+ patch.object(vad_model, "extract_fbank", side_effect=fake_extract_fbank),
+ ):
+ vad_model.FsmnVADStreaming.inference(
+ vad,
+ torch.zeros(960),
+ frontend=frontend,
+ cache=cache,
+ key=["utt"],
+ chunk_size=60,
+ is_final=False,
+ device="cpu",
+ dynamic_silence=False,
+ )
+
+ assert extract_kwargs[0].get("return_waveform", False) is supports_alignment
+
+
+def test_inference_rejects_missing_waveform_before_forward():
+ vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)
+ vad.vad_opts = SimpleNamespace(speech_to_sil_time_thres=100)
+ forward_batches = []
+ vad.forward = lambda **batch: forward_batches.append(batch) or []
+ cache = {
+ "frontend": {},
+ "prev_samples": torch.empty(0),
+ "encoder": {},
+ "stats": SimpleNamespace(),
+ }
+ frontend = SimpleNamespace(fs=16000, frame_shift=10, lfr_n=1)
+
+ with (
+ patch.object(
+ vad_model,
+ "load_audio_text_image_video",
+ return_value=[torch.zeros(960)],
+ ),
+ patch.object(
+ vad_model,
+ "extract_fbank",
+ return_value=(torch.zeros(1, 2, 4), torch.tensor([2])),
+ ),
+ pytest.raises(
+ RuntimeError,
+ match="must provide aligned_waveforms or waveforms",
+ ),
+ ):
+ vad_model.FsmnVADStreaming.inference(
+ vad,
+ torch.zeros(960),
+ frontend=frontend,
+ cache=cache,
+ key=["utt"],
+ chunk_size=60,
+ is_final=False,
+ device="cpu",
+ dynamic_silence=False,
+ )
+
+ assert forward_batches == []