Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(onnx): return one segment list per input from Fsmn_vad

Tanishq76b5189e0afb
diff --git a/.github/workflows/test-funasr-onnx-package.yml b/.github/workflows/test-funasr-onnx-package.yml
index dc0f886ff..8ef0e63f0 100644
--- a/.github/workflows/test-funasr-onnx-package.yml
+++ b/.github/workflows/test-funasr-onnx-package.yml
@@ -12,6 +12,7 @@ on:
- "tests/test_funasr_onnx_paraformer_online_overlap.py"
- "tests/test_funasr_onnx_paraformer_online_empty_final.py"
- "tests/test_funasr_onnx_vad_online_short_chunks.py"
+ - "tests/test_funasr_onnx_vad_multiple_inputs.py"
- "tests/test_rich_transcription_postprocess.py"
- "tests/test_rich_transcription_event_segments.py"
push:
@@ -27,6 +28,7 @@ on:
- "tests/test_funasr_onnx_paraformer_online_overlap.py"
- "tests/test_funasr_onnx_paraformer_online_empty_final.py"
- "tests/test_funasr_onnx_vad_online_short_chunks.py"
+ - "tests/test_funasr_onnx_vad_multiple_inputs.py"
- "tests/test_rich_transcription_postprocess.py"
- "tests/test_rich_transcription_event_segments.py"
workflow_dispatch:
@@ -110,6 +112,9 @@ jobs:
- name: Verify streaming VAD short chunks
run: python tests/test_funasr_onnx_vad_online_short_chunks.py
+ - name: Verify offline VAD results per input
+ run: python tests/test_funasr_onnx_vad_multiple_inputs.py
+
- name: Verify source distribution error paths
run: |
python -m pip install --force-reinstall --no-deps dist/funasr-onnx/*.tar.gz
diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
index fc132d17c..cb9ba3d0d 100644
--- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
+++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
@@ -96,10 +96,11 @@ class Fsmn_vad:
waveform_list = self.load_data(audio_in, self.frontend.opts.frame_opts.samp_freq)
waveform_nums = len(waveform_list)
is_final = kwargs.get("kwargs", False)
- segments = [[]] * self.batch_size
- for beg_idx in range(0, waveform_nums, self.batch_size):
+ segments = [[] for _ in range(waveform_nums)]
+ # E2EVadModel and prepare_cache handle one waveform at a time, so batch_size is not used.
+ for beg_idx in range(0, waveform_nums):
vad_scorer = E2EVadModel(self.vad_scorer_config)
- end_idx = min(waveform_nums, beg_idx + self.batch_size)
+ end_idx = beg_idx + 1
waveform = waveform_list[beg_idx:end_idx]
feats, feats_len = self.extract_feat(waveform)
waveform = np.array(waveform)
@@ -137,8 +138,7 @@ class Fsmn_vad:
# segments = self.vad_scorer(scores, waveform[0][None, :], is_final=is_final, max_end_sil=self.max_end_sil)
if segments_part:
- for batch_num in range(0, self.batch_size):
- segments[batch_num] += segments_part[batch_num]
+ segments[beg_idx] += segments_part[0]
except ONNXRuntimeError:
# logging.warning(traceback.format_exc())
diff --git a/tests/test_funasr_onnx_vad_multiple_inputs.py b/tests/test_funasr_onnx_vad_multiple_inputs.py
new file mode 100644
index 000000000..fd2478114
--- /dev/null
+++ b/tests/test_funasr_onnx_vad_multiple_inputs.py
@@ -0,0 +1,104 @@
+"""Run against an installed funasr-onnx package."""
+
+import importlib.util
+import os
+import tempfile
+import unittest
+
+import numpy as np
+
+if importlib.util.find_spec("funasr_onnx") is None:
+ raise unittest.SkipTest("Install the standalone funasr-onnx distribution first.")
+
+import soundfile
+from funasr_onnx.utils.frontend import WavFrontend
+from funasr_onnx.vad_bin import Fsmn_vad
+
+# frontend_conf of funasr/models/fsmn_vad_streaming/template.yaml
+FRONTEND_CONF = {
+ "fs": 16000,
+ "window": "hamming",
+ "n_mels": 80,
+ "frame_length": 25,
+ "frame_shift": 10,
+ "dither": 0.0,
+ "lfr_m": 5,
+ "lfr_n": 1,
+}
+
+# 16 kHz test files: length and noise ranges in samples, silence elsewhere.
+AUDIO = {
+ "a": (48000, [(8000, 24000)]), # 3 s, noise at 0.5-1.5 s
+ "b": (72000, [(16000, 32000), (48000, 56000)]), # 4.5 s, noise at 1-2 s and 3-3.5 s
+ "silence": (32000, []), # 2 s
+}
+
+# Segments (ms) of each file on its own.
+SEGMENTS = {"a": [[220, 1660]], "b": [[720, 2160], [2720, 3660]], "silence": []}
+
+
+def fake_infer(inputs):
+ # Score loud frames as speech instead of running the ONNX session.
+ sil = np.where(inputs[0][0].mean(-1) > 5.0, 0.05, 0.95)
+ scores = np.stack((sil, 1.0 - sil), axis=-1)[None].astype(np.float32)
+ return scores, inputs[1:]
+
+
+def make_model(batch_size=1):
+ # Keep __call__, WavFrontend and E2EVadModel real; no model files.
+ model = Fsmn_vad.__new__(Fsmn_vad)
+ model.frontend = WavFrontend(cmvn_file=None, **FRONTEND_CONF)
+ model.batch_size = batch_size
+ # E2EVadModel's defaults equal the template's model_conf.
+ model.vad_scorer_config = {}
+ model.max_end_sil = 800
+ model.encoder_conf = {"fsmn_layers": 1, "proj_dim": 2, "lorder": 2}
+ model.infer = fake_infer
+ return model
+
+
+class FsmnVadMultipleInputsTest(unittest.TestCase):
+ def setUp(self):
+ tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(tmp.cleanup)
+ self.wav = {}
+ for name, (length, noise) in AUDIO.items():
+ rng = np.random.default_rng(0)
+ audio = np.zeros(length, dtype=np.float32)
+ for start, end in noise:
+ audio[start:end] = rng.uniform(-0.3, 0.3, end - start)
+ self.wav[name] = os.path.join(tmp.name, name + ".wav")
+ soundfile.write(self.wav[name], audio, 16000)
+
+ def test_single_input(self):
+ # A path or an array gives a list with one segment list.
+ model = make_model()
+ for name in AUDIO:
+ with self.subTest(name=name):
+ self.assertEqual(model(self.wav[name]), [SEGMENTS[name]])
+ audio, _ = soundfile.read(self.wav[name], dtype="float32")
+ self.assertEqual(model(audio), [SEGMENTS[name]])
+
+ def test_list_input(self):
+ # A list of paths gives one segment list per file, in input order.
+ model = make_model()
+ for names in ([], ["a", "b"], ["b", "a"], ["silence", "b"], ["a", "silence", "b"]):
+ with self.subTest(names=names):
+ result = model([self.wav[name] for name in names])
+ self.assertEqual(result, [SEGMENTS[name] for name in names])
+
+ def test_batch_size_above_one(self):
+ # batch_size=2 or 3 gives the batch_size=1 results, for one file and for lists.
+ for batch_size in (2, 3):
+ model = make_model(batch_size=batch_size)
+ for name in AUDIO:
+ with self.subTest(batch_size=batch_size, name=name):
+ self.assertEqual(model(self.wav[name]), [SEGMENTS[name]])
+ for names in (["a", "b"], ["a", "a"], ["silence", "b"], ["a", "silence", "b"]):
+ with self.subTest(batch_size=batch_size, names=names):
+ result = model([self.wav[name] for name in names])
+ self.assertEqual(result, [SEGMENTS[name] for name in names])
+
+
+if __name__ == "__main__":
+ unittest.main()