Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

Decode container-formatted audio bytes before inference (#3435)

zhifu gao763ee06e3710
diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py
index e30559a2a..585187d4a 100644
--- a/funasr/utils/load_utils.py
+++ b/funasr/utils/load_utils.py
@@ -127,12 +127,16 @@ def load_audio_text_image_video(
data_or_path_or_list = data_or_path_or_list.mean(0)
except:
try:
+ if hasattr(data_or_path_or_list, "seek"):
+ data_or_path_or_list.seek(0)
import soundfile as sf
data_np, audio_fs = sf.read(data_or_path_or_list, dtype="float32")
data_or_path_or_list = torch.from_numpy(data_np).squeeze()
if data_or_path_or_list.ndim > 1 and kwargs.get("reduce_channels", True):
data_or_path_or_list = data_or_path_or_list.mean(-1)
except:
+ if hasattr(data_or_path_or_list, "seek"):
+ data_or_path_or_list.seek(0)
data_or_path_or_list = _load_audio_ffmpeg(data_or_path_or_list, sr=fs)
data_or_path_or_list = torch.from_numpy(
data_or_path_or_list
@@ -175,6 +179,96 @@ def load_audio_text_image_video(
return data_or_path_or_list
+def _mp3_header_fields(data: bytes, offset: int):
+ """Return MPEG audio header fields, including free-format bitrate index zero."""
+ if offset + 4 > len(data):
+ return None
+
+ header = int.from_bytes(data[offset : offset + 4], "big")
+ if header & 0xFFE00000 != 0xFFE00000:
+ return None
+
+ version_id = (header >> 19) & 0x3
+ layer_id = (header >> 17) & 0x3
+ bitrate_index = (header >> 12) & 0xF
+ sample_rate_index = (header >> 10) & 0x3
+ padding = (header >> 9) & 0x1
+ if (
+ version_id == 1
+ or layer_id == 0
+ or bitrate_index == 15
+ or sample_rate_index == 3
+ ):
+ return None
+ return version_id, layer_id, bitrate_index, sample_rate_index, padding
+
+
+def _mp3_frame_length(data: bytes, offset: int) -> int:
+ """Return a fixed-bitrate MPEG audio frame length, or zero if unavailable."""
+ fields = _mp3_header_fields(data, offset)
+ if fields is None:
+ return 0
+ version_id, layer_id, bitrate_index, sample_rate_index, padding = fields
+ if bitrate_index == 0:
+ return 0
+
+ mpeg1_bitrates = {
+ 3: (32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448),
+ 2: (32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384),
+ 1: (32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320),
+ }
+ mpeg2_bitrates = {
+ 3: (32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256),
+ 2: (8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160),
+ 1: (8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160),
+ }
+ sample_rates = {
+ 3: (44100, 48000, 32000),
+ 2: (22050, 24000, 16000),
+ 0: (11025, 12000, 8000),
+ }
+ bitrate_table = mpeg1_bitrates if version_id == 3 else mpeg2_bitrates
+ bitrate = bitrate_table[layer_id][bitrate_index - 1] * 1000
+ sample_rate = sample_rates[version_id][sample_rate_index]
+
+ if layer_id == 3:
+ return (12 * bitrate // sample_rate + padding) * 4
+ coefficient = 144 if version_id == 3 or layer_id == 2 else 72
+ return coefficient * bitrate // sample_rate + padding
+
+
+def _has_consecutive_mp3_frames(data: bytes) -> bool:
+ """Avoid mistaking raw PCM that starts with one sync-like sample for MP3."""
+ fields = _mp3_header_fields(data, 0)
+ if fields is None:
+ return False
+ version_id, layer_id, bitrate_index, sample_rate_index, _ = fields
+ if bitrate_index == 0:
+ signature = version_id, layer_id, sample_rate_index
+ padding_slot = 4 if layer_id == 3 else 1
+ first_padding = fields[4] * padding_slot
+ for second_offset in range(24, min(len(data) - 3, 8192)):
+ next_fields = _mp3_header_fields(data, second_offset)
+ if next_fields is not None and (
+ next_fields[0], next_fields[1], next_fields[3]
+ ) == signature and next_fields[2] == 0:
+ base_frame_length = second_offset - first_padding
+ third_offset = (
+ second_offset
+ + base_frame_length
+ + next_fields[4] * padding_slot
+ )
+ third_fields = _mp3_header_fields(data, third_offset)
+ if third_fields is not None and (
+ third_fields[0], third_fields[1], third_fields[3]
+ ) == signature and third_fields[2] == 0:
+ return True
+ return False
+
+ first_frame_length = _mp3_frame_length(data, 0)
+ return first_frame_length > 0 and _mp3_frame_length(data, first_frame_length) > 0
+
+
def _is_audio_container(data: bytes) -> bool:
"""Return True if *data* starts with a recognised container-format magic header.
@@ -184,10 +278,15 @@ def _is_audio_container(data: bytes) -> bool:
if len(data) < 4:
return False
# WAV – RIFF....WAVE
- if data[:4] == b"RIFF":
+ if (
+ len(data) >= 12
+ and data[:4] in (b"RIFF", b"RIFX", b"RF64", b"BW64")
+ and data[8:12] == b"WAVE"
+ ):
return True
- # MP3 – ID3 tag or sync word (0xFF 0xEx)
- if data[:3] == b"ID3" or (data[0] == 0xFF and (data[1] & 0xE0) == 0xE0):
+ # MP3 – ID3 tag or at least two structurally valid MPEG audio frames
+ has_mpeg_sync = data[0] == 0xFF and (data[1] & 0xE0) == 0xE0
+ if data[:3] == b"ID3" or (has_mpeg_sync and _has_consecutive_mp3_frames(data)):
return True
# OGG
if data[:4] == b"OggS":
@@ -205,23 +304,28 @@ def _is_audio_container(data: bytes) -> bool:
def load_bytes(input):
- """Convert audio bytes to numpy array.
+ """Convert raw PCM or container-formatted audio bytes to a waveform.
Args:
- input (bytes): Raw audio bytes.
+ input (bytes): Raw int16 PCM or encoded audio-file bytes.
Returns:
- numpy.ndarray: Decoded audio samples.
+ numpy.ndarray: Mono float32 samples at 16 kHz.
"""
- # Only run the (expensive) frame-rate validation when the payload is an
- # actual audio container (WAV, MP3, OGG, …). Raw PCM buffers have no
- # recognisable header and would cause pydub to spend ~200 ms before
- # raising an exception that is then silently swallowed anyway.
if _is_audio_container(input):
try:
- input = validate_frame_rate(input)
- except Exception:
- pass
+ waveform = load_audio_text_image_video(BytesIO(input), fs=16000)
+ except Exception as exc:
+ raise RuntimeError(
+ "Failed to decode container-formatted audio bytes. Verify that the input is "
+ "a complete supported audio file and that torchaudio, soundfile, or ffmpeg "
+ "is available."
+ ) from exc
+ else:
+ if isinstance(waveform, torch.Tensor):
+ waveform = waveform.detach().cpu().numpy()
+ return np.asarray(waveform, dtype=np.float32)
+
middle_data = np.frombuffer(input, dtype=np.int16)
middle_data = np.asarray(middle_data)
if middle_data.dtype.kind not in "iu":
@@ -315,14 +419,14 @@ def extract_fbank(data, data_len=None, data_type: str = "sound", frontend=None,
return data.to(torch.float32), data_len.to(torch.int32)
-def _load_audio_ffmpeg(file: str, sr: int = 16000):
+def _load_audio_ffmpeg(file, sr: int = 16000):
"""
Open an audio file and read as mono waveform, resampling as necessary
Parameters
----------
- file: str
- The audio file to open
+ file: str or file-like object
+ The audio file or byte stream to open
sr: int
The sample rate to resample the audio if necessary
@@ -336,7 +440,15 @@ def _load_audio_ffmpeg(file: str, sr: int = 16000):
# and resampling as necessary. Requires the ffmpeg CLI in PATH.
# fmt: off
pcm_params = []
- if file.lower().endswith('.pcm'):
+ stdin_data = None
+ if hasattr(file, "read"):
+ if hasattr(file, "seek"):
+ file.seek(0)
+ stdin_data = file.read()
+ input_source = "pipe:0"
+ else:
+ input_source = os.fspath(file)
+ if isinstance(input_source, str) and input_source.lower().endswith('.pcm'):
pcm_params = [
"-f", "s16le",
"-ar", str(sr),
@@ -348,7 +460,7 @@ def _load_audio_ffmpeg(file: str, sr: int = 16000):
"-nostdin",
"-threads", "0",
*pcm_params, # PCM files need input format specified before -i since PCM is raw data without headers
- "-i", file,
+ "-i", input_source,
"-f", "s16le",
"-ac", "1",
"-acodec", "pcm_s16le",
@@ -357,7 +469,7 @@ def _load_audio_ffmpeg(file: str, sr: int = 16000):
]
# fmt: on
try:
- out = run(cmd, capture_output=True, check=True).stdout
+ out = run(cmd, input=stdin_data, capture_output=True, check=True).stdout
except CalledProcessError as e:
raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e
diff --git a/tests/test_load_audio_bytes.py b/tests/test_load_audio_bytes.py
new file mode 100644
index 000000000..d23ae04a7
--- /dev/null
+++ b/tests/test_load_audio_bytes.py
@@ -0,0 +1,235 @@
+import base64
+import importlib.util
+import io
+from pathlib import Path
+import struct
+import sys
+import tempfile
+import types
+import unittest
+from unittest import mock
+import wave
+
+import numpy as np
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _load_utils_module():
+ module_names = ["funasr", "funasr.download", "funasr.download.file"]
+ previous = {name: sys.modules.get(name) for name in module_names}
+
+ funasr_package = types.ModuleType("funasr")
+ funasr_package.__path__ = []
+ download_package = types.ModuleType("funasr.download")
+ download_package.__path__ = []
+ download_file = types.ModuleType("funasr.download.file")
+ download_file.download_from_url = lambda url: url
+
+ sys.modules["funasr"] = funasr_package
+ sys.modules["funasr.download"] = download_package
+ sys.modules["funasr.download.file"] = download_file
+ try:
+ spec = importlib.util.spec_from_file_location(
+ "load_utils_audio_bytes_test", ROOT / "funasr/utils/load_utils.py"
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+ finally:
+ for name, original in previous.items():
+ if original is None:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = original
+
+
+LOAD_UTILS = _load_utils_module()
+
+# 80 ms, 16 kHz mono, generated without ID3 or Xing metadata by ffmpeg 6.1.1.
+NO_ID3_MP3 = base64.b64decode(
+ "//NIxAAa0LJcBVlIABsLTlpy05ZMsmWnLloB0Uy2BhEGUUDiEXzRhOvE88TrdOJY1lDSQMoYBCIAFBGIO4CYJgHAGBsEw2TtwUQIECBBBwnB8H8oGJTlA/wcOYgB/WDhzIA/wI7n+jhgHz+BDnfg+BAQBDB//B8H1QJghAAQSMFwa3j/mFYgGAQU1hUCfLMm//NIxBcg+gJkDZ2gADIa5vcxhhUcxwIo5huQh2CQ4GXugbI6BkioGWkgiPkQAyWIBoMXQsSBs0DYYRz/hlkMijHCghC3/kNFyi5SaHOHO//KJFSKmReJox//yKkVMi8XjEul1L//8mi8Yl0upF42EoS//Pf///XYuxb//41DPjII3hgHzBwACWYI5Npk2mcG//NIxBYiUm5MAZ6oABXAdm1MCaSE6GJWZaBm8zAWVAg8DDsXAAA46gGGABlEABoH+F1A6xxid//IwiAuAskT//HGWCcIIdJ///J84VCcOm5P///nDQuHTcvqNC5///+s3N1Ghos3N1Ghz//WD4gCIPiAIg////gwEQuQYFyiD4Eqijt1s/+gX9S2f8CFjqrn//NIxA8hCpaUAZqYAPlgf8AdQXUCtxZpIEW4sIuMgA7jJMuo+OAmyIkTLikklo/KxOF8qk+YIqSUtH8rE4ZlUvoGy6SqK/5cTPF9R8uLPUlUV0lf8vqPmizyaj6Cz1FdJVFdJX/6c+hPJz5wBpK2LsX/hgBmwwCYHDAJgcMKtVaq1VVMQU1FMy4xMDBVVVVV//NIxA0AAANIAcAAAFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV"
+)[:720]
+
+
+def _sine_pcm(sample_rate, duration=0.1):
+ sample_count = round(sample_rate * duration)
+ times = np.arange(sample_count, dtype=np.float64) / sample_rate
+ return np.round(np.sin(2 * np.pi * 440 * times) * 12000).astype(np.int16)
+
+
+def _wav_bytes(samples, sample_rate):
+ output = io.BytesIO()
+ with wave.open(output, "wb") as wav_file:
+ wav_file.setnchannels(1)
+ wav_file.setsampwidth(2)
+ wav_file.setframerate(sample_rate)
+ wav_file.writeframes(samples.tobytes())
+ return output.getvalue()
+
+
+def _rifx_bytes(samples, sample_rate):
+ big_endian_samples = samples.astype(">i2").tobytes()
+ return (
+ b"RIFX"
+ + struct.pack(">I", 36 + len(big_endian_samples))
+ + b"WAVEfmt "
+ + struct.pack(">IHHIIHH", 16, 1, 1, sample_rate, sample_rate * 2, 2, 16)
+ + b"data"
+ + struct.pack(">I", len(big_endian_samples))
+ + big_endian_samples
+ )
+
+
+def _free_format_mp3_bytes():
+ data = bytearray(NO_ID3_MP3)
+ for offset in (0, 144, 288, 432, 576):
+ data[offset + 2] &= 0x0F
+ return bytes(data)
+
+
+class TestLoadAudioBytes(unittest.TestCase):
+ def test_decodes_wav_container_without_treating_header_as_pcm(self):
+ samples = _sine_pcm(16000)
+
+ actual = LOAD_UTILS.load_bytes(_wav_bytes(samples, 16000))
+
+ expected = samples.astype(np.float32) / 32768.0
+ self.assertEqual(actual.dtype, np.float32)
+ np.testing.assert_allclose(actual, expected, atol=1e-6, rtol=0)
+
+ def test_resamples_wav_container_to_16khz(self):
+ samples = _sine_pcm(8000)
+
+ actual = LOAD_UTILS.load_bytes(_wav_bytes(samples, 8000))
+
+ self.assertEqual(actual.dtype, np.float32)
+ self.assertEqual(len(actual), 1600)
+ self.assertTrue(np.isfinite(actual).all())
+ self.assertGreater(float(np.max(np.abs(actual))), 0.1)
+
+ def test_decodes_big_endian_rifx_wav(self):
+ samples = np.array([-32768, -1000, 0, 1000, 32767], dtype=np.int16)
+
+ actual = LOAD_UTILS.load_bytes(_rifx_bytes(samples, 16000))
+
+ expected = samples.astype(np.float32) / 32768.0
+ np.testing.assert_array_equal(actual, expected)
+
+ def test_recognizes_large_wave_container_variants(self):
+ decoded = np.array([0.25, -0.25], dtype=np.float32)
+ for marker in (b"RF64", b"BW64"):
+ container = marker + b"\xff\xff\xff\xffWAVEplaceholder"
+ with self.subTest(marker=marker):
+ with mock.patch.object(
+ LOAD_UTILS, "load_audio_text_image_video", return_value=decoded
+ ):
+ actual = LOAD_UTILS.load_bytes(container)
+ np.testing.assert_array_equal(actual, decoded)
+
+ def test_preserves_raw_int16_pcm_bytes(self):
+ samples = np.array([-32768, -12345, 0, 12345, 32767], dtype=np.int16)
+
+ actual = LOAD_UTILS.load_bytes(samples.tobytes())
+
+ expected = samples.astype(np.float32) / 32768.0
+ np.testing.assert_array_equal(actual, expected)
+
+ def test_preserves_raw_pcm_with_mp3_sync_like_first_sample(self):
+ raw_pcm = b"\xff\xfb\x00\x00\x39\x30\xc7\xcf"
+ samples = np.frombuffer(raw_pcm, dtype=np.int16)
+
+ actual = LOAD_UTILS.load_bytes(raw_pcm)
+
+ expected = samples.astype(np.float32) / 32768.0
+ np.testing.assert_array_equal(actual, expected)
+
+ def test_preserves_raw_pcm_with_inconsistent_free_format_sync_headers(self):
+ raw_pcm = bytearray(np.arange(160, dtype=np.int16).tobytes())
+ for offset in (0, 100, 210):
+ raw_pcm[offset : offset + 4] = b"\xff\xfb\x00\x00"
+ raw_pcm = bytes(raw_pcm)
+ samples = np.frombuffer(raw_pcm, dtype=np.int16)
+
+ actual = LOAD_UTILS.load_bytes(raw_pcm)
+
+ expected = samples.astype(np.float32) / 32768.0
+ np.testing.assert_array_equal(actual, expected)
+
+ def test_preserves_raw_pcm_with_non_wave_riff_prefix(self):
+ raw_pcm = b"RIFF\x00\x00\x00\x00NOPE\x00\x00\x00\x00"
+ samples = np.frombuffer(raw_pcm, dtype=np.int16)
+
+ actual = LOAD_UTILS.load_bytes(raw_pcm)
+
+ expected = samples.astype(np.float32) / 32768.0
+ np.testing.assert_array_equal(actual, expected)
+
+ def test_no_id3_mp3_never_falls_back_to_raw_pcm(self):
+ with mock.patch.object(
+ LOAD_UTILS,
+ "load_audio_text_image_video",
+ side_effect=RuntimeError("decoder unavailable"),
+ ):
+ with self.assertRaisesRegex(RuntimeError, "complete supported audio file"):
+ LOAD_UTILS.load_bytes(NO_ID3_MP3)
+
+ def test_free_format_mp3_never_falls_back_to_raw_pcm(self):
+ with mock.patch.object(
+ LOAD_UTILS,
+ "load_audio_text_image_video",
+ side_effect=RuntimeError("decoder unavailable"),
+ ):
+ with self.assertRaisesRegex(RuntimeError, "complete supported audio file"):
+ LOAD_UTILS.load_bytes(_free_format_mp3_bytes())
+
+ def test_decodes_no_id3_mp3_consistently_with_file_path(self):
+ with tempfile.NamedTemporaryFile(suffix=".mp3") as mp3_file:
+ mp3_file.write(NO_ID3_MP3)
+ mp3_file.flush()
+ expected = LOAD_UTILS.load_audio_text_image_video(mp3_file.name, fs=16000)
+
+ actual = LOAD_UTILS.load_bytes(NO_ID3_MP3)
+
+ if hasattr(expected, "detach"):
+ expected = expected.detach().cpu().numpy()
+ np.testing.assert_allclose(actual, expected, atol=1e-6, rtol=0)
+
+ def test_rewinds_file_like_audio_between_decoders(self):
+ wav_data = _wav_bytes(_sine_pcm(16000), 16000)
+
+ def consume_then_fail(stream):
+ stream.read()
+ raise RuntimeError("first decoder failed")
+
+ with mock.patch.object(
+ LOAD_UTILS.torchaudio, "load", side_effect=consume_then_fail
+ ):
+ actual = LOAD_UTILS.load_audio_text_image_video(
+ io.BytesIO(wav_data), fs=16000
+ )
+
+ expected = _sine_pcm(16000).astype(np.float32) / 32768.0
+ np.testing.assert_allclose(actual, expected, atol=1e-6, rtol=0)
+
+ @unittest.skipUnless(LOAD_UTILS.is_ffmpeg_installed(), "ffmpeg is required")
+ def test_ffmpeg_decodes_file_like_audio(self):
+ samples = _sine_pcm(16000)
+
+ actual = LOAD_UTILS._load_audio_ffmpeg(
+ io.BytesIO(_wav_bytes(samples, 16000)), sr=16000
+ )
+
+ expected = samples.astype(np.float32) / 32768.0
+ np.testing.assert_allclose(actual, expected, atol=1e-6, rtol=0)
+
+ def test_corrupt_container_error_is_actionable(self):
+ corrupt_wav = b"RIFF\x10\x00\x00\x00WAVEbroken"
+
+ with self.assertRaisesRegex(RuntimeError, "complete supported audio file"):
+ LOAD_UTILS.load_bytes(corrupt_wav)
+
+
+if __name__ == "__main__":
+ unittest.main()