Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(frontend): initialize consumed frames without LFR
zhifu gao7d8855a931f5
diff --git a/.github/workflows/test-kws-output.yml b/.github/workflows/test-kws-output.ymlindex 55b2f4cb0..754e333e0 100644--- a/.github/workflows/test-kws-output.yml+++ b/.github/workflows/test-kws-output.yml@@ -8,12 +8,14 @@ on:- "funasr/models/sanm_kws/**"- "funasr/models/sanm_kws_streaming/**"- "funasr/frontends/wav_frontend.py"+ - "funasr/models/fsmn_vad_streaming/model.py"- "funasr/auto/auto_model.py"- "funasr/utils/datadir_writer.py"- "funasr/utils/kws_utils.py"- "tests/test_kws_optional_output.py"- "tests/test_kws_streaming_continuity.py"- "tests/test_kws_utils.py"+ - "tests/test_wav_frontend_no_lfr.py"- "tests/test_fsmn_vad_streaming_buffers.py"- "tests/test_dynamic_streaming_vad.py"- "tests/test_fsmn_vad_dynamic_silence.py"@@ -26,12 +28,14 @@ on:- "funasr/models/sanm_kws/**"- "funasr/models/sanm_kws_streaming/**"- "funasr/frontends/wav_frontend.py"+ - "funasr/models/fsmn_vad_streaming/model.py"- "funasr/auto/auto_model.py"- "funasr/utils/datadir_writer.py"- "funasr/utils/kws_utils.py"- "tests/test_kws_optional_output.py"- "tests/test_kws_streaming_continuity.py"- "tests/test_kws_utils.py"+ - "tests/test_wav_frontend_no_lfr.py"- "tests/test_fsmn_vad_streaming_buffers.py"- "tests/test_dynamic_streaming_vad.py"- "tests/test_fsmn_vad_dynamic_silence.py"@@ -63,6 +67,7 @@ jobs:tests/test_kws_optional_output.py \tests/test_kws_streaming_continuity.py \tests/test_kws_utils.py \+ tests/test_wav_frontend_no_lfr.py \tests/test_fsmn_vad_streaming_buffers.py \tests/test_dynamic_streaming_vad.py \tests/test_fsmn_vad_dynamic_silence.pydiff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.pyindex 593cc1446..cb671e6df 100644--- a/funasr/frontends/wav_frontend.py+++ b/funasr/frontends/wav_frontend.py@@ -488,6 +488,7 @@ class WavFrontendOnline(nn.Module):lfr_splice_frame_idxs = []for i in range(batch_size):mat = input[i, : input_lengths[i], :]+ lfr_splice_frame_idx = mat.size(0)if self.lfr_m != 1 or self.lfr_n != 1:# update self.lfr_splice_cache in self.apply_lfr# mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(mat, self.lfr_m, self.lfr_n, self.lfr_splice_cache[i],diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.pyindex 0b01c8384..b59862e6b 100644--- a/funasr/models/fsmn_vad_streaming/model.py+++ b/funasr/models/fsmn_vad_streaming/model.py@@ -1005,7 +1005,9 @@ class FsmnVADStreaming(nn.Module):audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0]))- n = int(len(audio_sample) // chunk_stride_samples + int(_is_final))+ n = len(audio_sample) // chunk_stride_samples+ if _is_final and len(audio_sample) % chunk_stride_samples:+ n += 1m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final)))segments = []# Keep explicit fixed-threshold requests from being overwritten by the dynamic schedule.diff --git a/tests/test_wav_frontend_no_lfr.py b/tests/test_wav_frontend_no_lfr.pynew file mode 100644index 000000000..a9af79662--- /dev/null+++ b/tests/test_wav_frontend_no_lfr.py@@ -0,0 +1,133 @@+import pytest+import torch++from funasr.frontends.wav_frontend import WavFrontendOnline+from funasr.models.fsmn_vad_streaming import model as vad_model+from funasr.utils import fbank as kaldi+++def make_frontend(tmp_path, cmvn):+ cmvn_file = None+ if cmvn:+ cmvn_file = tmp_path / "no_lfr.mvn"+ cmvn_file.write_text(+ "<AddShift> 80 80\n<LearnRateCoef> 1 [ "+ + " ".join(["0.25"] * 80)+ + " ]\n<Rescale> 80 80\n<LearnRateCoef> 1 [ "+ + " ".join(["2"] * 80)+ + " ]\n"+ )+ return WavFrontendOnline(cmvn_file=cmvn_file, dither=0.0)++[email protected]("cmvn", (False, True))[email protected]("is_final", (False, True))[email protected]("lengths", ((5, 3, 0), (0, 0)))+def test_no_lfr_reports_valid_consumed_frames(tmp_path, cmvn, is_final, lengths):+ frontend = make_frontend(tmp_path, cmvn)+ inputs = torch.arange(len(lengths) * 7 * 80, dtype=torch.float32).reshape(+ len(lengths), 7, 80+ )+ original = inputs.clone()+ cache = {}++ feats, actual_lengths, consumed = frontend.forward_lfr_cmvn(+ inputs, torch.tensor(lengths), is_final=is_final, cache=cache+ )++ assert actual_lengths.tolist() == list(lengths)+ assert consumed.tolist() == list(lengths)+ assert feats.shape == (len(lengths), max(lengths), 80)+ assert feats.dtype == torch.float32+ assert cache == {}+ for index, length in enumerate(lengths):+ expected = original[index, :length]+ if cmvn:+ expected = (expected + 0.25) * 2+ torch.testing.assert_close(feats[index, :length], expected, rtol=0, atol=0)+ assert torch.count_nonzero(feats[index, length:]) == 0++[email protected]("cmvn", (False, True))[email protected]("empty_final", (False, True))[email protected]("chunks", ((1600,), (400,) * 4, (160,) * 10, (80, 320, 40, 1160)))+def test_no_lfr_stream_matches_full_filterbank(tmp_path, cmvn, empty_final, chunks):+ frontend = make_frontend(tmp_path, cmvn)+ audio = 0.2 * torch.sin(torch.arange(1600, dtype=torch.float32) * (2 * torch.pi * 440 / 16000))+ expected = kaldi.fbank(+ audio.unsqueeze(0) * (1 << 15), num_mel_bins=80, frame_length=25, frame_shift=10,+ sample_frequency=16000, window_type="hamming", dither=0.0, energy_floor=0.0,+ )+ if cmvn:+ expected = (expected + 0.25) * 2+ cache, outputs, offset, emitted = {}, [], 0, 0+ calls = chunks + ((0,) if empty_final else ())+ for index, size in enumerate(calls):+ chunk = audio[offset : offset + size]+ offset += size+ feats, lengths = frontend(+ chunk.unsqueeze(0), torch.tensor([size]), cache=cache,+ is_final=index == len(calls) - 1, return_waveform=True,+ )+ count = int(lengths[0]) if lengths.numel() else 0+ if count:+ outputs.append(feats[0, :count])+ start = emitted * 160+ stop = start + (count - 1) * 160 + 400+ torch.testing.assert_close(cache["aligned_waveforms"][0], audio[start:stop], rtol=0, atol=0)+ emitted += count+ else:+ assert cache["aligned_waveforms"].numel() == 0+ assert cache["reserve_waveforms"].numel() == 0+ assert emitted == expected.shape[0] == 8+ torch.testing.assert_close(torch.cat(outputs), expected, rtol=1e-5, atol=1e-5)++[email protected]("size", (0, 160, 399))[email protected]("empty_final", (False, True))+def test_no_lfr_short_stream_has_no_frames(tmp_path, size, empty_final):+ frontend = make_frontend(tmp_path, False)+ cache = {}+ for index, length in enumerate((size, 0) if empty_final else (size,)):+ feats, lengths = frontend(+ torch.zeros(1, length), torch.tensor([length]), cache=cache,+ is_final=not empty_final or index == 1, return_waveform=True,+ )+ assert feats.numel() == 0+ assert not lengths.numel() or lengths.sum() == 0+ assert cache["aligned_waveforms"].numel() == 0++[email protected]("lfr_m", (1, 5))[email protected]("offset_ms", (0, 1000))[email protected]("chunk_ms", (20, 100, 5000))+def test_offline_vad_finalizes_last_nonempty_chunk(lfr_m, offset_ms, chunk_ms):+ class ScriptedEncoder(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.frames = 0++ def forward(self, feats, cache=None):+ time_ms = (torch.arange(feats.shape[1]) + self.frames) * 10+ speech = ((time_ms >= 500 + offset_ms) & (time_ms < 1500 + offset_ms)) | (+ (time_ms >= 2500 + offset_ms) & (time_ms < 3200 + offset_ms)+ )+ self.frames += feats.shape[1]+ probability = torch.where(speech, 0.999, 0.001)+ return torch.stack((1 - probability, probability), dim=-1).unsqueeze(0)++ model = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)+ torch.nn.Module.__init__(model)+ model.encoder = ScriptedEncoder()+ model.vad_opts = vad_model.VADXOptions()+ audio = 0.2 * torch.sin(+ torch.arange((4000 + offset_ms) * 16, dtype=torch.float32)+ * (2 * torch.pi * 440 / 16000)+ )+ frontend = WavFrontendOnline(lfr_m=lfr_m, lfr_n=1, dither=0.0)+ result, _ = model.inference(+ [audio], key=["fixture"], cache={}, frontend=frontend, device="cpu",+ chunk_size=chunk_ms, is_streaming_input=False, is_final=True, dynamic_silence=True,+ )+ assert model.encoder.frames == (4000 + offset_ms) // 10 - 2+ assert result == [{"key": "fixture", "value": [[240 + offset_ms, 3980 + offset_ms]]}]