Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(onnx): handle offline VAD inputs without feature frames
BillVan1945f2dfafb3848b
diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.pyindex cb9ba3d0d..5d9d8fd4e 100644--- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py+++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py@@ -103,6 +103,8 @@ class Fsmn_vad:end_idx = beg_idx + 1waveform = waveform_list[beg_idx:end_idx]feats, feats_len = self.extract_feat(waveform)+ if feats_len[0] == 0:+ continuewaveform = np.array(waveform)param_dict = kwargs.get("param_dict", dict())in_cache = param_dict.get("in_cache", list())@@ -183,8 +185,13 @@ class Fsmn_vad:def extract_feat(self, waveform_list: List[np.ndarray]) -> Tuple[np.ndarray, np.ndarray]:feats, feats_len = [], []for waveform in waveform_list:- speech, _ = self.frontend.fbank(waveform)- feat, feat_len = self.frontend.lfr_cmvn(speech)+ speech, speech_len = self.frontend.fbank(waveform)+ if speech_len == 0:+ # LFR needs a first frame; keep the empty input at its original position.+ feat = np.empty((0, speech.shape[1] * self.frontend.lfr_m), dtype=speech.dtype)+ feat_len = 0+ else:+ feat, feat_len = self.frontend.lfr_cmvn(speech)feats.append(feat)feats_len.append(feat_len)diff --git a/tests/test_funasr_onnx_vad_multiple_inputs.py b/tests/test_funasr_onnx_vad_multiple_inputs.pyindex fd2478114..a5cbaff48 100644--- a/tests/test_funasr_onnx_vad_multiple_inputs.py+++ b/tests/test_funasr_onnx_vad_multiple_inputs.py@@ -4,6 +4,7 @@ import importlib.utilimport osimport tempfileimport unittest+from unittest.mock import Mockimport numpy as np@@ -44,10 +45,10 @@ def fake_infer(inputs):return scores, inputs[1:]-def make_model(batch_size=1):+def make_model(batch_size=1, **frontend_conf):# Keep __call__, WavFrontend and E2EVadModel real; no model files.model = Fsmn_vad.__new__(Fsmn_vad)- model.frontend = WavFrontend(cmvn_file=None, **FRONTEND_CONF)+ model.frontend = WavFrontend(cmvn_file=None, **{**FRONTEND_CONF, **frontend_conf})model.batch_size = batch_size# E2EVadModel's defaults equal the template's model_conf.model.vad_scorer_config = {}@@ -99,6 +100,80 @@ class FsmnVadMultipleInputsTest(unittest.TestCase):result = model([self.wav[name] for name in names])self.assertEqual(result, [SEGMENTS[name] for name in names])+ def test_short_inputs_and_frame_boundary(self):+ for fs, frame_length, lfr_m in (+ (16000, 25, 5),+ (8000, 25, 5),+ (16000, 20, 5),+ (16000, 25, 1),+ ):+ window = int(fs * frame_length / 1000)+ for batch_size in (1, 2, 3):+ model = make_model(batch_size, fs=fs, frame_length=frame_length, lfr_m=lfr_m)+ model.infer = Mock(side_effect=fake_infer)+ for samples in (0, 1, window // 2, window - 1, window, window + 1, 2 * window):+ audio = np.zeros(samples, dtype=np.float32)+ path = os.path.join(os.path.dirname(self.wav["a"]), "short.wav")+ soundfile.write(path, audio, fs)+ for audio_in in (audio, path):+ with self.subTest(+ fs=fs,+ frame_length=frame_length,+ lfr_m=lfr_m,+ batch_size=batch_size,+ samples=samples,+ input_type=type(audio_in).__name__,+ ):+ model.infer.reset_mock()+ self.assertEqual(model(audio_in), [[]])+ self.assertEqual(model.infer.call_count, int(samples >= window))++ def test_short_files_keep_their_input_positions(self):+ for samples in (0, 160, 320, 399):+ path = os.path.join(os.path.dirname(self.wav["a"]), "short.wav")+ soundfile.write(path, np.zeros(samples, dtype=np.float32), 16000)+ paths = {**self.wav, "short": path}+ expected = {**SEGMENTS, "short": []}+ for batch_size in (1, 2, 3):+ model = make_model(batch_size)+ model.infer = Mock(side_effect=fake_infer)+ for names in (+ ["short", "a"],+ ["a", "short"],+ ["a", "short", "b"],+ ["short", "short"],+ ["short", "b", "short"],+ ):+ with self.subTest(samples=samples, batch_size=batch_size, names=names):+ model.infer.reset_mock()+ result = model([paths[name] for name in names])+ self.assertEqual(result, [expected[name] for name in names])+ self.assertEqual(+ model.infer.call_count, sum(name != "short" for name in names)+ )+ self.assertEqual(len({id(segments) for segments in result}), len(result))++ def test_extract_feat_with_empty_frames(self):+ for lfr_m, lfr_n in ((1, 1), (5, 1), (3, 2)):+ model = make_model(lfr_m=lfr_m, lfr_n=lfr_n)+ empty = np.zeros(0, dtype=np.float32)+ short = np.zeros(399, dtype=np.float32)+ normal = np.zeros(1600, dtype=np.float32)+ expected, expected_len = model.extract_feat([normal])+ for waveforms, lengths in (+ ([empty], [0]),+ ([short], [0]),+ ([empty, short, normal], [0, 0, expected_len[0]]),+ ):+ with self.subTest(lfr_m=lfr_m, lfr_n=lfr_n, lengths=lengths):+ feats, feats_len = model.extract_feat(waveforms)+ np.testing.assert_array_equal(feats_len, lengths)+ self.assertEqual(feats.shape, (len(waveforms), max(lengths), 80 * lfr_m))+ self.assertEqual(feats.dtype, np.float32)+ if len(waveforms) > 1:+ np.testing.assert_array_equal(feats[-1], expected[0])+ self.assertFalse(feats[:-1].any())+if __name__ == "__main__":unittest.main()