Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
Decode container-formatted audio bytes before inference (#3435)
zhifu gao763ee06e3710
diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.pyindex e30559a2a..585187d4a 100644--- a/funasr/utils/load_utils.py+++ b/funasr/utils/load_utils.py@@ -127,12 +127,16 @@ def load_audio_text_image_video(data_or_path_or_list = data_or_path_or_list.mean(0)except:try:+ if hasattr(data_or_path_or_list, "seek"):+ data_or_path_or_list.seek(0)import soundfile as sfdata_np, audio_fs = sf.read(data_or_path_or_list, dtype="float32")data_or_path_or_list = torch.from_numpy(data_np).squeeze()if data_or_path_or_list.ndim > 1 and kwargs.get("reduce_channels", True):data_or_path_or_list = data_or_path_or_list.mean(-1)except:+ if hasattr(data_or_path_or_list, "seek"):+ data_or_path_or_list.seek(0)data_or_path_or_list = _load_audio_ffmpeg(data_or_path_or_list, sr=fs)data_or_path_or_list = torch.from_numpy(data_or_path_or_list@@ -175,6 +179,96 @@ def load_audio_text_image_video(return data_or_path_or_list+def _mp3_header_fields(data: bytes, offset: int):+ """Return MPEG audio header fields, including free-format bitrate index zero."""+ if offset + 4 > len(data):+ return None++ header = int.from_bytes(data[offset : offset + 4], "big")+ if header & 0xFFE00000 != 0xFFE00000:+ return None++ version_id = (header >> 19) & 0x3+ layer_id = (header >> 17) & 0x3+ bitrate_index = (header >> 12) & 0xF+ sample_rate_index = (header >> 10) & 0x3+ padding = (header >> 9) & 0x1+ if (+ version_id == 1+ or layer_id == 0+ or bitrate_index == 15+ or sample_rate_index == 3+ ):+ return None+ return version_id, layer_id, bitrate_index, sample_rate_index, padding+++def _mp3_frame_length(data: bytes, offset: int) -> int:+ """Return a fixed-bitrate MPEG audio frame length, or zero if unavailable."""+ fields = _mp3_header_fields(data, offset)+ if fields is None:+ return 0+ version_id, layer_id, bitrate_index, sample_rate_index, padding = fields+ if bitrate_index == 0:+ return 0++ mpeg1_bitrates = {+ 3: (32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448),+ 2: (32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384),+ 1: (32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320),+ }+ mpeg2_bitrates = {+ 3: (32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256),+ 2: (8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160),+ 1: (8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160),+ }+ sample_rates = {+ 3: (44100, 48000, 32000),+ 2: (22050, 24000, 16000),+ 0: (11025, 12000, 8000),+ }+ bitrate_table = mpeg1_bitrates if version_id == 3 else mpeg2_bitrates+ bitrate = bitrate_table[layer_id][bitrate_index - 1] * 1000+ sample_rate = sample_rates[version_id][sample_rate_index]++ if layer_id == 3:+ return (12 * bitrate // sample_rate + padding) * 4+ coefficient = 144 if version_id == 3 or layer_id == 2 else 72+ return coefficient * bitrate // sample_rate + padding+++def _has_consecutive_mp3_frames(data: bytes) -> bool:+ """Avoid mistaking raw PCM that starts with one sync-like sample for MP3."""+ fields = _mp3_header_fields(data, 0)+ if fields is None:+ return False+ version_id, layer_id, bitrate_index, sample_rate_index, _ = fields+ if bitrate_index == 0:+ signature = version_id, layer_id, sample_rate_index+ padding_slot = 4 if layer_id == 3 else 1+ first_padding = fields[4] * padding_slot+ for second_offset in range(24, min(len(data) - 3, 8192)):+ next_fields = _mp3_header_fields(data, second_offset)+ if next_fields is not None and (+ next_fields[0], next_fields[1], next_fields[3]+ ) == signature and next_fields[2] == 0:+ base_frame_length = second_offset - first_padding+ third_offset = (+ second_offset+ + base_frame_length+ + next_fields[4] * padding_slot+ )+ third_fields = _mp3_header_fields(data, third_offset)+ if third_fields is not None and (+ third_fields[0], third_fields[1], third_fields[3]+ ) == signature and third_fields[2] == 0:+ return True+ return False++ first_frame_length = _mp3_frame_length(data, 0)+ return first_frame_length > 0 and _mp3_frame_length(data, first_frame_length) > 0++def _is_audio_container(data: bytes) -> bool:"""Return True if *data* starts with a recognised container-format magic header.@@ -184,10 +278,15 @@ def _is_audio_container(data: bytes) -> bool:if len(data) < 4:return False# WAV – RIFF....WAVE- if data[:4] == b"RIFF":+ if (+ len(data) >= 12+ and data[:4] in (b"RIFF", b"RIFX", b"RF64", b"BW64")+ and data[8:12] == b"WAVE"+ ):return True- # MP3 – ID3 tag or sync word (0xFF 0xEx)- if data[:3] == b"ID3" or (data[0] == 0xFF and (data[1] & 0xE0) == 0xE0):+ # MP3 – ID3 tag or at least two structurally valid MPEG audio frames+ has_mpeg_sync = data[0] == 0xFF and (data[1] & 0xE0) == 0xE0+ if data[:3] == b"ID3" or (has_mpeg_sync and _has_consecutive_mp3_frames(data)):return True# OGGif data[:4] == b"OggS":@@ -205,23 +304,28 @@ def _is_audio_container(data: bytes) -> bool:def load_bytes(input):- """Convert audio bytes to numpy array.+ """Convert raw PCM or container-formatted audio bytes to a waveform.Args:- input (bytes): Raw audio bytes.+ input (bytes): Raw int16 PCM or encoded audio-file bytes.Returns:- numpy.ndarray: Decoded audio samples.+ numpy.ndarray: Mono float32 samples at 16 kHz."""- # Only run the (expensive) frame-rate validation when the payload is an- # actual audio container (WAV, MP3, OGG, …). Raw PCM buffers have no- # recognisable header and would cause pydub to spend ~200 ms before- # raising an exception that is then silently swallowed anyway.if _is_audio_container(input):try:- input = validate_frame_rate(input)- except Exception:- pass+ waveform = load_audio_text_image_video(BytesIO(input), fs=16000)+ except Exception as exc:+ raise RuntimeError(+ "Failed to decode container-formatted audio bytes. Verify that the input is "+ "a complete supported audio file and that torchaudio, soundfile, or ffmpeg "+ "is available."+ ) from exc+ else:+ if isinstance(waveform, torch.Tensor):+ waveform = waveform.detach().cpu().numpy()+ return np.asarray(waveform, dtype=np.float32)+middle_data = np.frombuffer(input, dtype=np.int16)middle_data = np.asarray(middle_data)if middle_data.dtype.kind not in "iu":@@ -315,14 +419,14 @@ def extract_fbank(data, data_len=None, data_type: str = "sound", frontend=None,return data.to(torch.float32), data_len.to(torch.int32)-def _load_audio_ffmpeg(file: str, sr: int = 16000):+def _load_audio_ffmpeg(file, sr: int = 16000):"""Open an audio file and read as mono waveform, resampling as necessaryParameters----------- file: str- The audio file to open+ file: str or file-like object+ The audio file or byte stream to opensr: intThe sample rate to resample the audio if necessary@@ -336,7 +440,15 @@ def _load_audio_ffmpeg(file: str, sr: int = 16000):# and resampling as necessary. Requires the ffmpeg CLI in PATH.# fmt: offpcm_params = []- if file.lower().endswith('.pcm'):+ stdin_data = None+ if hasattr(file, "read"):+ if hasattr(file, "seek"):+ file.seek(0)+ stdin_data = file.read()+ input_source = "pipe:0"+ else:+ input_source = os.fspath(file)+ if isinstance(input_source, str) and input_source.lower().endswith('.pcm'):pcm_params = ["-f", "s16le","-ar", str(sr),@@ -348,7 +460,7 @@ def _load_audio_ffmpeg(file: str, sr: int = 16000):"-nostdin","-threads", "0",*pcm_params, # PCM files need input format specified before -i since PCM is raw data without headers- "-i", file,+ "-i", input_source,"-f", "s16le","-ac", "1","-acodec", "pcm_s16le",@@ -357,7 +469,7 @@ def _load_audio_ffmpeg(file: str, sr: int = 16000):]# fmt: ontry:- out = run(cmd, capture_output=True, check=True).stdout+ out = run(cmd, input=stdin_data, capture_output=True, check=True).stdoutexcept CalledProcessError as e:raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from ediff --git a/tests/test_load_audio_bytes.py b/tests/test_load_audio_bytes.pynew file mode 100644index 000000000..d23ae04a7--- /dev/null+++ b/tests/test_load_audio_bytes.py@@ -0,0 +1,235 @@+import base64+import importlib.util+import io+from pathlib import Path+import struct+import sys+import tempfile+import types+import unittest+from unittest import mock+import wave++import numpy as np+++ROOT = Path(__file__).resolve().parents[1]+++def _load_utils_module():+ module_names = ["funasr", "funasr.download", "funasr.download.file"]+ previous = {name: sys.modules.get(name) for name in module_names}++ funasr_package = types.ModuleType("funasr")+ funasr_package.__path__ = []+ download_package = types.ModuleType("funasr.download")+ download_package.__path__ = []+ download_file = types.ModuleType("funasr.download.file")+ download_file.download_from_url = lambda url: url++ sys.modules["funasr"] = funasr_package+ sys.modules["funasr.download"] = download_package+ sys.modules["funasr.download.file"] = download_file+ try:+ spec = importlib.util.spec_from_file_location(+ "load_utils_audio_bytes_test", ROOT / "funasr/utils/load_utils.py"+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+ finally:+ for name, original in previous.items():+ if original is None:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = original+++LOAD_UTILS = _load_utils_module()++# 80 ms, 16 kHz mono, generated without ID3 or Xing metadata by ffmpeg 6.1.1.+NO_ID3_MP3 = base64.b64decode(+ "//NIxAAa0LJcBVlIABsLTlpy05ZMsmWnLloB0Uy2BhEGUUDiEXzRhOvE88TrdOJY1lDSQMoYBCIAFBGIO4CYJgHAGBsEw2TtwUQIECBBBwnB8H8oGJTlA/wcOYgB/WDhzIA/wI7n+jhgHz+BDnfg+BAQBDB//B8H1QJghAAQSMFwa3j/mFYgGAQU1hUCfLMm//NIxBcg+gJkDZ2gADIa5vcxhhUcxwIo5huQh2CQ4GXugbI6BkioGWkgiPkQAyWIBoMXQsSBs0DYYRz/hlkMijHCghC3/kNFyi5SaHOHO//KJFSKmReJox//yKkVMi8XjEul1L//8mi8Yl0upF42EoS//Pf///XYuxb//41DPjII3hgHzBwACWYI5Npk2mcG//NIxBYiUm5MAZ6oABXAdm1MCaSE6GJWZaBm8zAWVAg8DDsXAAA46gGGABlEABoH+F1A6xxid//IwiAuAskT//HGWCcIIdJ///J84VCcOm5P///nDQuHTcvqNC5///+s3N1Ghos3N1Ghz//WD4gCIPiAIg////gwEQuQYFyiD4Eqijt1s/+gX9S2f8CFjqrn//NIxA8hCpaUAZqYAPlgf8AdQXUCtxZpIEW4sIuMgA7jJMuo+OAmyIkTLikklo/KxOF8qk+YIqSUtH8rE4ZlUvoGy6SqK/5cTPF9R8uLPUlUV0lf8vqPmizyaj6Cz1FdJVFdJX/6c+hPJz5wBpK2LsX/hgBmwwCYHDAJgcMKtVaq1VVMQU1FMy4xMDBVVVVV//NIxA0AAANIAcAAAFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV"+)[:720]+++def _sine_pcm(sample_rate, duration=0.1):+ sample_count = round(sample_rate * duration)+ times = np.arange(sample_count, dtype=np.float64) / sample_rate+ return np.round(np.sin(2 * np.pi * 440 * times) * 12000).astype(np.int16)+++def _wav_bytes(samples, sample_rate):+ output = io.BytesIO()+ with wave.open(output, "wb") as wav_file:+ wav_file.setnchannels(1)+ wav_file.setsampwidth(2)+ wav_file.setframerate(sample_rate)+ wav_file.writeframes(samples.tobytes())+ return output.getvalue()+++def _rifx_bytes(samples, sample_rate):+ big_endian_samples = samples.astype(">i2").tobytes()+ return (+ b"RIFX"+ + struct.pack(">I", 36 + len(big_endian_samples))+ + b"WAVEfmt "+ + struct.pack(">IHHIIHH", 16, 1, 1, sample_rate, sample_rate * 2, 2, 16)+ + b"data"+ + struct.pack(">I", len(big_endian_samples))+ + big_endian_samples+ )+++def _free_format_mp3_bytes():+ data = bytearray(NO_ID3_MP3)+ for offset in (0, 144, 288, 432, 576):+ data[offset + 2] &= 0x0F+ return bytes(data)+++class TestLoadAudioBytes(unittest.TestCase):+ def test_decodes_wav_container_without_treating_header_as_pcm(self):+ samples = _sine_pcm(16000)++ actual = LOAD_UTILS.load_bytes(_wav_bytes(samples, 16000))++ expected = samples.astype(np.float32) / 32768.0+ self.assertEqual(actual.dtype, np.float32)+ np.testing.assert_allclose(actual, expected, atol=1e-6, rtol=0)++ def test_resamples_wav_container_to_16khz(self):+ samples = _sine_pcm(8000)++ actual = LOAD_UTILS.load_bytes(_wav_bytes(samples, 8000))++ self.assertEqual(actual.dtype, np.float32)+ self.assertEqual(len(actual), 1600)+ self.assertTrue(np.isfinite(actual).all())+ self.assertGreater(float(np.max(np.abs(actual))), 0.1)++ def test_decodes_big_endian_rifx_wav(self):+ samples = np.array([-32768, -1000, 0, 1000, 32767], dtype=np.int16)++ actual = LOAD_UTILS.load_bytes(_rifx_bytes(samples, 16000))++ expected = samples.astype(np.float32) / 32768.0+ np.testing.assert_array_equal(actual, expected)++ def test_recognizes_large_wave_container_variants(self):+ decoded = np.array([0.25, -0.25], dtype=np.float32)+ for marker in (b"RF64", b"BW64"):+ container = marker + b"\xff\xff\xff\xffWAVEplaceholder"+ with self.subTest(marker=marker):+ with mock.patch.object(+ LOAD_UTILS, "load_audio_text_image_video", return_value=decoded+ ):+ actual = LOAD_UTILS.load_bytes(container)+ np.testing.assert_array_equal(actual, decoded)++ def test_preserves_raw_int16_pcm_bytes(self):+ samples = np.array([-32768, -12345, 0, 12345, 32767], dtype=np.int16)++ actual = LOAD_UTILS.load_bytes(samples.tobytes())++ expected = samples.astype(np.float32) / 32768.0+ np.testing.assert_array_equal(actual, expected)++ def test_preserves_raw_pcm_with_mp3_sync_like_first_sample(self):+ raw_pcm = b"\xff\xfb\x00\x00\x39\x30\xc7\xcf"+ samples = np.frombuffer(raw_pcm, dtype=np.int16)++ actual = LOAD_UTILS.load_bytes(raw_pcm)++ expected = samples.astype(np.float32) / 32768.0+ np.testing.assert_array_equal(actual, expected)++ def test_preserves_raw_pcm_with_inconsistent_free_format_sync_headers(self):+ raw_pcm = bytearray(np.arange(160, dtype=np.int16).tobytes())+ for offset in (0, 100, 210):+ raw_pcm[offset : offset + 4] = b"\xff\xfb\x00\x00"+ raw_pcm = bytes(raw_pcm)+ samples = np.frombuffer(raw_pcm, dtype=np.int16)++ actual = LOAD_UTILS.load_bytes(raw_pcm)++ expected = samples.astype(np.float32) / 32768.0+ np.testing.assert_array_equal(actual, expected)++ def test_preserves_raw_pcm_with_non_wave_riff_prefix(self):+ raw_pcm = b"RIFF\x00\x00\x00\x00NOPE\x00\x00\x00\x00"+ samples = np.frombuffer(raw_pcm, dtype=np.int16)++ actual = LOAD_UTILS.load_bytes(raw_pcm)++ expected = samples.astype(np.float32) / 32768.0+ np.testing.assert_array_equal(actual, expected)++ def test_no_id3_mp3_never_falls_back_to_raw_pcm(self):+ with mock.patch.object(+ LOAD_UTILS,+ "load_audio_text_image_video",+ side_effect=RuntimeError("decoder unavailable"),+ ):+ with self.assertRaisesRegex(RuntimeError, "complete supported audio file"):+ LOAD_UTILS.load_bytes(NO_ID3_MP3)++ def test_free_format_mp3_never_falls_back_to_raw_pcm(self):+ with mock.patch.object(+ LOAD_UTILS,+ "load_audio_text_image_video",+ side_effect=RuntimeError("decoder unavailable"),+ ):+ with self.assertRaisesRegex(RuntimeError, "complete supported audio file"):+ LOAD_UTILS.load_bytes(_free_format_mp3_bytes())++ def test_decodes_no_id3_mp3_consistently_with_file_path(self):+ with tempfile.NamedTemporaryFile(suffix=".mp3") as mp3_file:+ mp3_file.write(NO_ID3_MP3)+ mp3_file.flush()+ expected = LOAD_UTILS.load_audio_text_image_video(mp3_file.name, fs=16000)++ actual = LOAD_UTILS.load_bytes(NO_ID3_MP3)++ if hasattr(expected, "detach"):+ expected = expected.detach().cpu().numpy()+ np.testing.assert_allclose(actual, expected, atol=1e-6, rtol=0)++ def test_rewinds_file_like_audio_between_decoders(self):+ wav_data = _wav_bytes(_sine_pcm(16000), 16000)++ def consume_then_fail(stream):+ stream.read()+ raise RuntimeError("first decoder failed")++ with mock.patch.object(+ LOAD_UTILS.torchaudio, "load", side_effect=consume_then_fail+ ):+ actual = LOAD_UTILS.load_audio_text_image_video(+ io.BytesIO(wav_data), fs=16000+ )++ expected = _sine_pcm(16000).astype(np.float32) / 32768.0+ np.testing.assert_allclose(actual, expected, atol=1e-6, rtol=0)++ @unittest.skipUnless(LOAD_UTILS.is_ffmpeg_installed(), "ffmpeg is required")+ def test_ffmpeg_decodes_file_like_audio(self):+ samples = _sine_pcm(16000)++ actual = LOAD_UTILS._load_audio_ffmpeg(+ io.BytesIO(_wav_bytes(samples, 16000)), sr=16000+ )++ expected = samples.astype(np.float32) / 32768.0+ np.testing.assert_allclose(actual, expected, atol=1e-6, rtol=0)++ def test_corrupt_container_error_is_actionable(self):+ corrupt_wav = b"RIFF\x10\x00\x00\x00WAVEbroken"++ with self.assertRaisesRegex(RuntimeError, "complete supported audio file"):+ LOAD_UTILS.load_bytes(corrupt_wav)+++if __name__ == "__main__":+ unittest.main()