Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(frontend): initialize consumed frames without LFR

zhifu gao7d8855a931f5
diff --git a/.github/workflows/test-kws-output.yml b/.github/workflows/test-kws-output.yml
index 55b2f4cb0..754e333e0 100644
--- a/.github/workflows/test-kws-output.yml
+++ b/.github/workflows/test-kws-output.yml
@@ -8,12 +8,14 @@ on:
- "funasr/models/sanm_kws/**"
- "funasr/models/sanm_kws_streaming/**"
- "funasr/frontends/wav_frontend.py"
+ - "funasr/models/fsmn_vad_streaming/model.py"
- "funasr/auto/auto_model.py"
- "funasr/utils/datadir_writer.py"
- "funasr/utils/kws_utils.py"
- "tests/test_kws_optional_output.py"
- "tests/test_kws_streaming_continuity.py"
- "tests/test_kws_utils.py"
+ - "tests/test_wav_frontend_no_lfr.py"
- "tests/test_fsmn_vad_streaming_buffers.py"
- "tests/test_dynamic_streaming_vad.py"
- "tests/test_fsmn_vad_dynamic_silence.py"
@@ -26,12 +28,14 @@ on:
- "funasr/models/sanm_kws/**"
- "funasr/models/sanm_kws_streaming/**"
- "funasr/frontends/wav_frontend.py"
+ - "funasr/models/fsmn_vad_streaming/model.py"
- "funasr/auto/auto_model.py"
- "funasr/utils/datadir_writer.py"
- "funasr/utils/kws_utils.py"
- "tests/test_kws_optional_output.py"
- "tests/test_kws_streaming_continuity.py"
- "tests/test_kws_utils.py"
+ - "tests/test_wav_frontend_no_lfr.py"
- "tests/test_fsmn_vad_streaming_buffers.py"
- "tests/test_dynamic_streaming_vad.py"
- "tests/test_fsmn_vad_dynamic_silence.py"
@@ -63,6 +67,7 @@ jobs:
tests/test_kws_optional_output.py \
tests/test_kws_streaming_continuity.py \
tests/test_kws_utils.py \
+ tests/test_wav_frontend_no_lfr.py \
tests/test_fsmn_vad_streaming_buffers.py \
tests/test_dynamic_streaming_vad.py \
tests/test_fsmn_vad_dynamic_silence.py
diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.py
index 593cc1446..cb671e6df 100644
--- a/funasr/frontends/wav_frontend.py
+++ b/funasr/frontends/wav_frontend.py
@@ -488,6 +488,7 @@ class WavFrontendOnline(nn.Module):
lfr_splice_frame_idxs = []
for i in range(batch_size):
mat = input[i, : input_lengths[i], :]
+ lfr_splice_frame_idx = mat.size(0)
if self.lfr_m != 1 or self.lfr_n != 1:
# update self.lfr_splice_cache in self.apply_lfr
# mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(mat, self.lfr_m, self.lfr_n, self.lfr_splice_cache[i],
diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py
index 0b01c8384..b59862e6b 100644
--- a/funasr/models/fsmn_vad_streaming/model.py
+++ b/funasr/models/fsmn_vad_streaming/model.py
@@ -1005,7 +1005,9 @@ class FsmnVADStreaming(nn.Module):
audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0]))
- n = int(len(audio_sample) // chunk_stride_samples + int(_is_final))
+ n = len(audio_sample) // chunk_stride_samples
+ if _is_final and len(audio_sample) % chunk_stride_samples:
+ n += 1
m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final)))
segments = []
# Keep explicit fixed-threshold requests from being overwritten by the dynamic schedule.
diff --git a/tests/test_wav_frontend_no_lfr.py b/tests/test_wav_frontend_no_lfr.py
new file mode 100644
index 000000000..a9af79662
--- /dev/null
+++ b/tests/test_wav_frontend_no_lfr.py
@@ -0,0 +1,133 @@
+import pytest
+import torch
+
+from funasr.frontends.wav_frontend import WavFrontendOnline
+from funasr.models.fsmn_vad_streaming import model as vad_model
+from funasr.utils import fbank as kaldi
+
+
+def make_frontend(tmp_path, cmvn):
+ cmvn_file = None
+ if cmvn:
+ cmvn_file = tmp_path / "no_lfr.mvn"
+ cmvn_file.write_text(
+ "<AddShift> 80 80\n<LearnRateCoef> 1 [ "
+ + " ".join(["0.25"] * 80)
+ + " ]\n<Rescale> 80 80\n<LearnRateCoef> 1 [ "
+ + " ".join(["2"] * 80)
+ + " ]\n"
+ )
+ return WavFrontendOnline(cmvn_file=cmvn_file, dither=0.0)
+
+
[email protected]("cmvn", (False, True))
[email protected]("is_final", (False, True))
[email protected]("lengths", ((5, 3, 0), (0, 0)))
+def test_no_lfr_reports_valid_consumed_frames(tmp_path, cmvn, is_final, lengths):
+ frontend = make_frontend(tmp_path, cmvn)
+ inputs = torch.arange(len(lengths) * 7 * 80, dtype=torch.float32).reshape(
+ len(lengths), 7, 80
+ )
+ original = inputs.clone()
+ cache = {}
+
+ feats, actual_lengths, consumed = frontend.forward_lfr_cmvn(
+ inputs, torch.tensor(lengths), is_final=is_final, cache=cache
+ )
+
+ assert actual_lengths.tolist() == list(lengths)
+ assert consumed.tolist() == list(lengths)
+ assert feats.shape == (len(lengths), max(lengths), 80)
+ assert feats.dtype == torch.float32
+ assert cache == {}
+ for index, length in enumerate(lengths):
+ expected = original[index, :length]
+ if cmvn:
+ expected = (expected + 0.25) * 2
+ torch.testing.assert_close(feats[index, :length], expected, rtol=0, atol=0)
+ assert torch.count_nonzero(feats[index, length:]) == 0
+
+
[email protected]("cmvn", (False, True))
[email protected]("empty_final", (False, True))
[email protected]("chunks", ((1600,), (400,) * 4, (160,) * 10, (80, 320, 40, 1160)))
+def test_no_lfr_stream_matches_full_filterbank(tmp_path, cmvn, empty_final, chunks):
+ frontend = make_frontend(tmp_path, cmvn)
+ audio = 0.2 * torch.sin(torch.arange(1600, dtype=torch.float32) * (2 * torch.pi * 440 / 16000))
+ expected = kaldi.fbank(
+ audio.unsqueeze(0) * (1 << 15), num_mel_bins=80, frame_length=25, frame_shift=10,
+ sample_frequency=16000, window_type="hamming", dither=0.0, energy_floor=0.0,
+ )
+ if cmvn:
+ expected = (expected + 0.25) * 2
+ cache, outputs, offset, emitted = {}, [], 0, 0
+ calls = chunks + ((0,) if empty_final else ())
+ for index, size in enumerate(calls):
+ chunk = audio[offset : offset + size]
+ offset += size
+ feats, lengths = frontend(
+ chunk.unsqueeze(0), torch.tensor([size]), cache=cache,
+ is_final=index == len(calls) - 1, return_waveform=True,
+ )
+ count = int(lengths[0]) if lengths.numel() else 0
+ if count:
+ outputs.append(feats[0, :count])
+ start = emitted * 160
+ stop = start + (count - 1) * 160 + 400
+ torch.testing.assert_close(cache["aligned_waveforms"][0], audio[start:stop], rtol=0, atol=0)
+ emitted += count
+ else:
+ assert cache["aligned_waveforms"].numel() == 0
+ assert cache["reserve_waveforms"].numel() == 0
+ assert emitted == expected.shape[0] == 8
+ torch.testing.assert_close(torch.cat(outputs), expected, rtol=1e-5, atol=1e-5)
+
+
[email protected]("size", (0, 160, 399))
[email protected]("empty_final", (False, True))
+def test_no_lfr_short_stream_has_no_frames(tmp_path, size, empty_final):
+ frontend = make_frontend(tmp_path, False)
+ cache = {}
+ for index, length in enumerate((size, 0) if empty_final else (size,)):
+ feats, lengths = frontend(
+ torch.zeros(1, length), torch.tensor([length]), cache=cache,
+ is_final=not empty_final or index == 1, return_waveform=True,
+ )
+ assert feats.numel() == 0
+ assert not lengths.numel() or lengths.sum() == 0
+ assert cache["aligned_waveforms"].numel() == 0
+
+
[email protected]("lfr_m", (1, 5))
[email protected]("offset_ms", (0, 1000))
[email protected]("chunk_ms", (20, 100, 5000))
+def test_offline_vad_finalizes_last_nonempty_chunk(lfr_m, offset_ms, chunk_ms):
+ class ScriptedEncoder(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.frames = 0
+
+ def forward(self, feats, cache=None):
+ time_ms = (torch.arange(feats.shape[1]) + self.frames) * 10
+ speech = ((time_ms >= 500 + offset_ms) & (time_ms < 1500 + offset_ms)) | (
+ (time_ms >= 2500 + offset_ms) & (time_ms < 3200 + offset_ms)
+ )
+ self.frames += feats.shape[1]
+ probability = torch.where(speech, 0.999, 0.001)
+ return torch.stack((1 - probability, probability), dim=-1).unsqueeze(0)
+
+ model = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)
+ torch.nn.Module.__init__(model)
+ model.encoder = ScriptedEncoder()
+ model.vad_opts = vad_model.VADXOptions()
+ audio = 0.2 * torch.sin(
+ torch.arange((4000 + offset_ms) * 16, dtype=torch.float32)
+ * (2 * torch.pi * 440 / 16000)
+ )
+ frontend = WavFrontendOnline(lfr_m=lfr_m, lfr_n=1, dither=0.0)
+ result, _ = model.inference(
+ [audio], key=["fixture"], cache={}, frontend=frontend, device="cpu",
+ chunk_size=chunk_ms, is_streaming_input=False, is_final=True, dynamic_silence=True,
+ )
+ assert model.encoder.frames == (4000 + offset_ms) // 10 - 2
+ assert result == [{"key": "fixture", "value": [[240 + offset_ms, 3980 + offset_ms]]}]