Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(audio): average the channels of 2-D floating-point waveforms

sohailshk8060eeb840d6
diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py
index b689d65ec..5e0bd2019 100644
--- a/funasr/utils/load_utils.py
+++ b/funasr/utils/load_utils.py
@@ -176,6 +176,23 @@ def load_audio_text_image_video(
pass
# print(f"unsupport data type: {data_or_path_or_list}, return raw data")
+ if (
+ isinstance(data_or_path_or_list, torch.Tensor)
+ and data_or_path_or_list.ndim == 2
+ and data_or_path_or_list.is_floating_point()
+ and data_type in (None, "sound")
+ and kwargs.get("reduce_channels", True)
+ ):
+ # In-memory waveforms follow either torchaudio's (channels, samples) or
+ # soundfile's (samples, channels) layout; the channel axis is the shorter one.
+ # Files are already reduced to mono above.
+ if data_or_path_or_list.numel() == 0:
+ data_or_path_or_list = data_or_path_or_list.reshape(-1)
+ else:
+ data_or_path_or_list = data_or_path_or_list.mean(
+ 0 if data_or_path_or_list.shape[0] <= data_or_path_or_list.shape[1] else 1
+ )
+
if audio_fs != fs and data_type != "text":
if torchaudio is not None:
resampler = torchaudio.transforms.Resample(audio_fs, fs)
diff --git a/tests/test_load_audio_multichannel_arrays.py b/tests/test_load_audio_multichannel_arrays.py
new file mode 100644
index 000000000..16a9685c0
--- /dev/null
+++ b/tests/test_load_audio_multichannel_arrays.py
@@ -0,0 +1,60 @@
+import unittest
+
+import numpy as np
+import torch
+
+from funasr.utils.load_utils import load_audio_text_image_video
+
+
+class TestMultichannelWaveformInput(unittest.TestCase):
+ def setUp(self):
+ t = np.arange(1600, dtype=np.float32)
+ self.left = np.sin(t / 20.0).astype(np.float32)
+ self.right = 0.5 * np.sin(t / 7.0).astype(np.float32)
+ self.mono = (self.left + self.right) / 2
+
+ def _check(self, waveform, **kwargs):
+ out = load_audio_text_image_video(waveform, **kwargs)
+ self.assertEqual(tuple(out.shape), (1600,))
+ np.testing.assert_allclose(out.numpy(), self.mono, atol=1e-6)
+
+ def test_channels_first_ndarray_is_averaged(self):
+ self._check(np.stack([self.left, self.right]))
+
+ def test_channels_last_ndarray_is_averaged(self):
+ self._check(np.stack([self.left, self.right], axis=1))
+
+ def test_channels_first_tensor_is_averaged(self):
+ self._check(torch.from_numpy(np.stack([self.left, self.right])))
+
+ def test_single_channel_2d_input_becomes_1d(self):
+ for waveform in (self.left[None, :], self.left[:, None]):
+ out = load_audio_text_image_video(waveform)
+ np.testing.assert_allclose(out.numpy(), self.left, atol=1e-6)
+
+ def test_stereo_is_reduced_before_resampling(self):
+ out = load_audio_text_image_video(
+ np.stack([self.left, self.right]), fs=16000, audio_fs=8000
+ )
+ self.assertEqual(out.ndim, 1)
+ self.assertEqual(out.shape[0], 3200)
+
+ def test_reduce_channels_false_keeps_channels(self):
+ out = load_audio_text_image_video(
+ np.stack([self.left, self.right]), reduce_channels=False
+ )
+ self.assertEqual(tuple(out.shape), (2, 1600))
+
+ def test_fbank_features_are_not_averaged(self):
+ feats = np.random.default_rng(0).standard_normal((50, 80)).astype(np.float32)
+ out = load_audio_text_image_video(feats, data_type="fbank")
+ self.assertEqual(tuple(out.shape), (50, 80))
+
+ def test_empty_2d_input_becomes_empty_1d(self):
+ for shape in ((0, 2), (2, 0)):
+ out = load_audio_text_image_video(np.zeros(shape, dtype=np.float32))
+ self.assertEqual(tuple(out.shape), (0,))
+
+
+if __name__ == "__main__":
+ unittest.main()