Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(onnx): handle offline VAD inputs without feature frames

BillVan1945f2dfafb3848b
diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
index cb9ba3d0d..5d9d8fd4e 100644
--- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
+++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
@@ -103,6 +103,8 @@ class Fsmn_vad:
end_idx = beg_idx + 1
waveform = waveform_list[beg_idx:end_idx]
feats, feats_len = self.extract_feat(waveform)
+ if feats_len[0] == 0:
+ continue
waveform = np.array(waveform)
param_dict = kwargs.get("param_dict", dict())
in_cache = param_dict.get("in_cache", list())
@@ -183,8 +185,13 @@ class Fsmn_vad:
def extract_feat(self, waveform_list: List[np.ndarray]) -> Tuple[np.ndarray, np.ndarray]:
feats, feats_len = [], []
for waveform in waveform_list:
- speech, _ = self.frontend.fbank(waveform)
- feat, feat_len = self.frontend.lfr_cmvn(speech)
+ speech, speech_len = self.frontend.fbank(waveform)
+ if speech_len == 0:
+ # LFR needs a first frame; keep the empty input at its original position.
+ feat = np.empty((0, speech.shape[1] * self.frontend.lfr_m), dtype=speech.dtype)
+ feat_len = 0
+ else:
+ feat, feat_len = self.frontend.lfr_cmvn(speech)
feats.append(feat)
feats_len.append(feat_len)
diff --git a/tests/test_funasr_onnx_vad_multiple_inputs.py b/tests/test_funasr_onnx_vad_multiple_inputs.py
index fd2478114..a5cbaff48 100644
--- a/tests/test_funasr_onnx_vad_multiple_inputs.py
+++ b/tests/test_funasr_onnx_vad_multiple_inputs.py
@@ -4,6 +4,7 @@ import importlib.util
import os
import tempfile
import unittest
+from unittest.mock import Mock
import numpy as np
@@ -44,10 +45,10 @@ def fake_infer(inputs):
return scores, inputs[1:]
-def make_model(batch_size=1):
+def make_model(batch_size=1, **frontend_conf):
# Keep __call__, WavFrontend and E2EVadModel real; no model files.
model = Fsmn_vad.__new__(Fsmn_vad)
- model.frontend = WavFrontend(cmvn_file=None, **FRONTEND_CONF)
+ model.frontend = WavFrontend(cmvn_file=None, **{**FRONTEND_CONF, **frontend_conf})
model.batch_size = batch_size
# E2EVadModel's defaults equal the template's model_conf.
model.vad_scorer_config = {}
@@ -99,6 +100,80 @@ class FsmnVadMultipleInputsTest(unittest.TestCase):
result = model([self.wav[name] for name in names])
self.assertEqual(result, [SEGMENTS[name] for name in names])
+ def test_short_inputs_and_frame_boundary(self):
+ for fs, frame_length, lfr_m in (
+ (16000, 25, 5),
+ (8000, 25, 5),
+ (16000, 20, 5),
+ (16000, 25, 1),
+ ):
+ window = int(fs * frame_length / 1000)
+ for batch_size in (1, 2, 3):
+ model = make_model(batch_size, fs=fs, frame_length=frame_length, lfr_m=lfr_m)
+ model.infer = Mock(side_effect=fake_infer)
+ for samples in (0, 1, window // 2, window - 1, window, window + 1, 2 * window):
+ audio = np.zeros(samples, dtype=np.float32)
+ path = os.path.join(os.path.dirname(self.wav["a"]), "short.wav")
+ soundfile.write(path, audio, fs)
+ for audio_in in (audio, path):
+ with self.subTest(
+ fs=fs,
+ frame_length=frame_length,
+ lfr_m=lfr_m,
+ batch_size=batch_size,
+ samples=samples,
+ input_type=type(audio_in).__name__,
+ ):
+ model.infer.reset_mock()
+ self.assertEqual(model(audio_in), [[]])
+ self.assertEqual(model.infer.call_count, int(samples >= window))
+
+ def test_short_files_keep_their_input_positions(self):
+ for samples in (0, 160, 320, 399):
+ path = os.path.join(os.path.dirname(self.wav["a"]), "short.wav")
+ soundfile.write(path, np.zeros(samples, dtype=np.float32), 16000)
+ paths = {**self.wav, "short": path}
+ expected = {**SEGMENTS, "short": []}
+ for batch_size in (1, 2, 3):
+ model = make_model(batch_size)
+ model.infer = Mock(side_effect=fake_infer)
+ for names in (
+ ["short", "a"],
+ ["a", "short"],
+ ["a", "short", "b"],
+ ["short", "short"],
+ ["short", "b", "short"],
+ ):
+ with self.subTest(samples=samples, batch_size=batch_size, names=names):
+ model.infer.reset_mock()
+ result = model([paths[name] for name in names])
+ self.assertEqual(result, [expected[name] for name in names])
+ self.assertEqual(
+ model.infer.call_count, sum(name != "short" for name in names)
+ )
+ self.assertEqual(len({id(segments) for segments in result}), len(result))
+
+ def test_extract_feat_with_empty_frames(self):
+ for lfr_m, lfr_n in ((1, 1), (5, 1), (3, 2)):
+ model = make_model(lfr_m=lfr_m, lfr_n=lfr_n)
+ empty = np.zeros(0, dtype=np.float32)
+ short = np.zeros(399, dtype=np.float32)
+ normal = np.zeros(1600, dtype=np.float32)
+ expected, expected_len = model.extract_feat([normal])
+ for waveforms, lengths in (
+ ([empty], [0]),
+ ([short], [0]),
+ ([empty, short, normal], [0, 0, expected_len[0]]),
+ ):
+ with self.subTest(lfr_m=lfr_m, lfr_n=lfr_n, lengths=lengths):
+ feats, feats_len = model.extract_feat(waveforms)
+ np.testing.assert_array_equal(feats_len, lengths)
+ self.assertEqual(feats.shape, (len(waveforms), max(lengths), 80 * lfr_m))
+ self.assertEqual(feats.dtype, np.float32)
+ if len(waveforms) > 1:
+ np.testing.assert_array_equal(feats[-1], expected[0])
+ self.assertFalse(feats[:-1].any())
+
if __name__ == "__main__":
unittest.main()