Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(amd:audio): decode audio through PyAV when torchcodec installs but cannot load (#8642)

Leo Borcherdingfe90d0da0251
diff --git a/pyproject.toml b/pyproject.toml
index f4b87cfd06d..bf8d4404bde 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -167,7 +167,7 @@ studio = [
"fastmcp>=3.0.2",
"gguf==0.19.0 ; python_version >= '3.10'",
"gguf==0.18.0 ; python_version < '3.10'",
- "av>=12.0.0",
+ "av>=12.3.0",
"sqlite-vec==0.1.9",
"pymupdf==1.27.2.3",
"pymupdf4llm==0.3.4",
@@ -220,17 +220,28 @@ huggingfacenotorch = [
# The only audio-torch2xx whose line has a Linux aarch64 wheel: torchcodec added
# manylinux_2_28_aarch64 at 0.11.0, so this extra carries it and the older ones below,
# which top out at 0.10, still cannot. Mirrors _torchcodec_platform_floor.
+# soundfile and PyAV are what decodes when torchcodec is installed but cannot load
+# (import_fixes.patch_datasets_audio_decoding_without_torchcodec); PyAV < 12.3 needs numpy < 2.
+# They carry torchcodec's marker so a host without a torchcodec wheel resolves nothing here.
audio-torch211 = [
"torchcodec>=0.11.0,<0.12.0 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'linux' and platform_machine == 'aarch64') or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
+ "soundfile>=0.12.1 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'linux' and platform_machine == 'aarch64') or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
+ "av>=12.3.0 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'linux' and platform_machine == 'aarch64') or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
]
audio-torch210 = [
"torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
+ "soundfile>=0.12.1 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
+ "av>=12.3.0 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
]
audio-torch290 = [
"torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
+ "soundfile>=0.12.1 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
+ "av>=12.3.0 ; python_version >= '3.10' and python_version < '3.15' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
]
audio-torch280 = [
"torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9' and python_version < '3.14' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
+ "soundfile>=0.12.1 ; python_version >= '3.9' and python_version < '3.14' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
+ "av>=12.3.0 ; python_version >= '3.9' and python_version < '3.14' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))",
]
huggingface = [
"unsloth[huggingfacenotorch]",
diff --git a/studio/backend/hub/services/datasets/formatting.py b/studio/backend/hub/services/datasets/formatting.py
index 9593d1913f8..4be463b7c7b 100644
--- a/studio/backend/hub/services/datasets/formatting.py
+++ b/studio/backend/hub/services/datasets/formatting.py
@@ -128,9 +128,18 @@ def _serialize_binary_value(data):
return f"<binary data, {len(data)} bytes>"
+def _is_sample_sequence(samples) -> bool:
+ # A list from the JSON path or a numpy array straight from the decoder; never text, bytes or a nested cell.
+ return hasattr(samples, "__len__") and not isinstance(
+ samples, (str, bytes, bytearray, memoryview, dict)
+ )
+
+
def _serialize_decoded_audio(value):
"""Summarise a decoded Audio cell the way binary cells are summarised."""
- samples = value.get("array") or []
+ samples = value.get("array")
+ if samples is None:
+ samples = []
rate = value.get("sampling_rate")
try:
seconds = len(samples) / rate if rate else None
@@ -167,7 +176,7 @@ def _serialize_preview_value(value):
return _serialize_binary_value(raw)
# A decoded Audio cell becomes one float per sample under the soundfile fallback, so ten preview
# rows of a few seconds each are tens of MB of JSON and the client dies rendering it.
- if "sampling_rate" in value and isinstance(value.get("array"), (list, tuple)):
+ if "sampling_rate" in value and _is_sample_sequence(value.get("array")):
return _serialize_decoded_audio(value)
return {str(key): _serialize_preview_value(item) for key, item in value.items()}
diff --git a/studio/backend/requirements/studio.txt b/studio/backend/requirements/studio.txt
index d47c10e5dd6..99b7335f34b 100644
--- a/studio/backend/requirements/studio.txt
+++ b/studio/backend/requirements/studio.txt
@@ -50,7 +50,7 @@ gguf==0.18.0; python_version < "3.10"
# synchronized audio) runs through PyAV, which bundles its own ffmpeg libs. The
# video load path fails fast with a clear error when this is missing, so keep it
# in the base studio set rather than an extra.
-av>=12.0.0
+av>=12.3.0
# RAG (knowledge bases, hybrid retrieval). sentence-transformers lives in
# extras-no-deps.txt; these add the lexical+dense store and document parsing.
sqlite-vec==0.1.9
diff --git a/studio/backend/tests/test_audio_dataset_decode.py b/studio/backend/tests/test_audio_dataset_decode.py
index d4306a05e70..5bbc6af62ff 100644
--- a/studio/backend/tests/test_audio_dataset_decode.py
+++ b/studio/backend/tests/test_audio_dataset_decode.py
@@ -68,6 +68,177 @@ def test_the_soundfile_decoder_resamples_to_the_cast_rate(broken_torchcodec):
assert decoded["path"] == "a.wav"
+def _m4a_bytes(seconds = 1.0, sampling_rate = 22050):
+ """An AAC file in an MP4 container: what torchcodec decoded and libsndfile cannot."""
+ av = pytest.importorskip("av")
+ t = np.arange(int(seconds * sampling_rate)) / sampling_rate
+ tone = (0.5 * np.sin(2 * np.pi * 440 * t)).astype("float32")
+ buf = io.BytesIO()
+ try:
+ with av.open(buf, "w", format = "mp4") as container:
+ stream = container.add_stream("aac", rate = sampling_rate)
+ stream.layout = "mono"
+ frame = av.AudioFrame.from_ndarray(tone[np.newaxis, :], format = "flt", layout = "mono")
+ frame.sample_rate = sampling_rate
+ for packet in stream.encode(frame):
+ container.mux(packet)
+ for packet in stream.encode(None):
+ container.mux(packet)
+ except Exception as exc: # noqa: BLE001
+ pytest.skip(f"this PyAV cannot encode AAC: {exc}")
+ return buf.getvalue()
+
+
+def test_an_m4a_row_decodes_through_pyav_when_torchcodec_is_broken(broken_torchcodec):
+ # soundfile refuses the container, so before this the row raised on every host whose
+ # torchcodec cannot load, and the installer told people to install FFmpeg for a
+ # format PyAV's bundled FFmpeg already reads.
+ from datasets import Audio, Dataset
+
+ raw = _m4a_bytes()
+ with pytest.raises(Exception):
+ sf.read(io.BytesIO(raw))
+ assert audio_decode.ensure_audio_decoding() is True
+ ds = Dataset.from_dict({"audio": [{"path": "tone.m4a", "bytes": raw}]})
+ ds = ds.cast_column("audio", Audio(sampling_rate = 16000))
+ decoded = ds[0]["audio"]
+
+ assert decoded["sampling_rate"] == 16000
+ array = np.asarray(decoded["array"])
+ assert array.ndim == 1 and array.dtype == np.float32
+ # One second of a 440 Hz tone at 0.5 amplitude, resampled to 16 kHz; AAC adds priming
+ # samples and the encoder rounds to frame boundaries, so bound rather than pin.
+ assert 15000 <= len(array) <= 17500
+ assert 0.4 <= float(np.abs(array).max()) <= 0.6
+ assert decoded["path"] == "tone.m4a"
+
+
+def _two_stream_m4a_bytes(rate = 22050, freqs = (440, 880)):
+ """One MP4 holding two AAC tracks, a tone per stream; the second is the one a stream_index must reach."""
+ av = pytest.importorskip("av")
+ t = np.arange(rate) / rate
+ buf = io.BytesIO()
+ try:
+ with av.open(buf, "w", format = "mp4") as container:
+ streams = []
+ for hz in freqs:
+ stream = container.add_stream("aac", rate = rate)
+ stream.layout = "mono"
+ streams.append(stream)
+ for stream, hz in zip(streams, freqs):
+ tone = (0.5 * np.sin(2 * np.pi * hz * t)).astype("float32")
+ frame = av.AudioFrame.from_ndarray(tone[np.newaxis, :], format = "flt", layout = "mono")
+ frame.sample_rate = rate
+ for packet in stream.encode(frame):
+ container.mux(packet)
+ for packet in stream.encode(None):
+ container.mux(packet)
+ except Exception as exc: # noqa: BLE001
+ pytest.skip(f"this PyAV cannot encode AAC: {exc}")
+ return buf.getvalue()
+
+
+def _dominant_hz(array, rate):
+ spectrum = np.abs(np.fft.rfft(np.asarray(array, dtype = "float64")))
+ return float(np.fft.rfftfreq(len(array), 1.0 / rate)[int(np.argmax(spectrum))])
+
+
+def test_the_stream_index_selects_the_track(broken_torchcodec):
+ # datasets.Audio(stream_index=1) must reach the second track, as torchcodec would.
+ from datasets import Audio, Dataset
+
+ raw = _two_stream_m4a_bytes()
+ assert audio_decode.ensure_audio_decoding() is True
+ rows = {"audio": [{"path": "two.m4a", "bytes": raw}]}
+ first = Dataset.from_dict(rows).cast_column("audio", Audio())[0]["audio"]
+ second = Dataset.from_dict(rows).cast_column("audio", Audio(stream_index = 1))[0]["audio"]
+ assert abs(_dominant_hz(first["array"], first["sampling_rate"]) - 440) < 20
+ assert abs(_dominant_hz(second["array"], second["sampling_rate"]) - 880) < 20
+ with pytest.raises(Exception, match = "stream"):
+ audio_decode._read_mono(io.BytesIO(raw), stream_index = 5)
+
+
+def test_a_missing_index_picks_the_default_track_not_the_first(broken_torchcodec, tmp_path):
+ # torchcodec resolves stream_index=None through av_find_best_stream, where a default
+ # disposition beats position; a file whose second track is the default must decode it.
+ import shutil
+ import subprocess
+
+ av = pytest.importorskip("av")
+ if shutil.which("ffmpeg") is None or not hasattr(av.container.streams.StreamContainer, "best"):
+ pytest.skip("needs the ffmpeg CLI to set dispositions and PyAV >= 13 for streams.best")
+ path = tmp_path / "second_is_default.m4a"
+ subprocess.run(
+ [
+ "ffmpeg",
+ "-y",
+ "-loglevel",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=440:duration=1:sample_rate=22050",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=880:duration=1:sample_rate=22050",
+ "-map",
+ "0:a",
+ "-map",
+ "1:a",
+ "-c:a",
+ "aac",
+ "-disposition:a:0",
+ "0",
+ "-disposition:a:1",
+ "default",
+ str(path),
+ ],
+ check = True,
+ )
+ assert audio_decode.ensure_audio_decoding() is True
+ array, rate = audio_decode._read_mono(str(path))
+ assert abs(_dominant_hz(array, rate) - 880) < 20
+ array, rate = audio_decode._read_mono(str(path), stream_index = 0)
+ assert abs(_dominant_hz(array, rate) - 440) < 20
+
+
+def test_a_channel_first_array_round_trips_through_the_encoder(broken_torchcodec):
+ # torchcodec hands decoded audio out as (channels, samples); libsndfile writes (frames, channels).
+ # Written as is, a (2, 1600) clip became two frames of 1600 channels, or failed outright.
+ from datasets import Audio, Dataset
+
+ assert audio_decode.ensure_audio_decoding() is True
+ stereo = np.stack([np.linspace(-0.5, 0.5, 1600, dtype = "float32")] * 2)
+ assert stereo.shape == (2, 1600)
+ ds = Dataset.from_dict({"audio": [{"array": stereo, "sampling_rate": 16000, "path": "s.wav"}]})
+ decoded = ds.cast_column("audio", Audio(sampling_rate = 16000))[0]["audio"]
+ assert len(decoded["array"]) == 1600 and decoded["sampling_rate"] == 16000
+
+
+def test_an_m4a_path_decodes_through_pyav(broken_torchcodec, tmp_path):
+ # The path form goes to av.open as a filename, the bytes form as a buffer.
+ from datasets import Audio, Dataset
+
+ path = tmp_path / "tone.m4a"
+ path.write_bytes(_m4a_bytes())
+ audio_decode.ensure_audio_decoding()
+ ds = Dataset.from_dict({"audio": [str(path)]}).cast_column("audio", Audio())
+ decoded = ds[0]["audio"]
+ assert decoded["sampling_rate"] == 22050
+ assert 20000 <= len(np.asarray(decoded["array"])) <= 24000
+
+
+def test_an_undecodable_row_names_both_decoders(broken_torchcodec):
+ from datasets import Audio, Dataset
+
+ audio_decode.ensure_audio_decoding()
+ ds = Dataset.from_dict({"audio": [{"path": "junk.bin", "bytes": b"not audio at all"}]})
+ ds = ds.cast_column("audio", Audio())
+ with pytest.raises(RuntimeError, match = "soundfile .* PyAV"):
+ ds[0]["audio"]
+
+
def test_a_stereo_source_is_averaged_to_mono(broken_torchcodec):
from datasets import Audio, Dataset
diff --git a/studio/backend/tests/test_audio_decode_load_order.py b/studio/backend/tests/test_audio_decode_load_order.py
index 3ce782266fb..46faa03114d 100644
--- a/studio/backend/tests/test_audio_decode_load_order.py
+++ b/studio/backend/tests/test_audio_decode_load_order.py
@@ -99,3 +99,69 @@ def test_a_datasets_without_the_torchcodec_flag_returns_a_bool():
sys.modules.pop(k, None)
else:
sys.modules[k] = v
+
+
+def test_a_torchcodec_that_raises_anything_at_import_is_marked_unusable():
+ # A damaged wheel can raise AttributeError or SyntaxError rather than ImportError. The
+ # installer's probe reports every import failure as "broken, soundfile takes over", so
+ # the runtime must treat every shape the same way or that promise is false.
+ import importlib.abc
+ import importlib.machinery
+ import sys
+ import types
+
+ from utils.datasets import audio_decode
+
+ class _Raising(importlib.abc.Loader):
+ def create_module(self, spec):
+ return None
+
+ def exec_module(self, module):
+ raise AttributeError("module 'torchcodec' has no attribute '_core'")
+
+ class _Finder(importlib.abc.MetaPathFinder):
+ def find_spec(
+ self,
+ name,
+ path,
+ target = None,
+ ):
+ if name == "datasets.features._torchcodec":
+ return importlib.machinery.ModuleSpec(name, _Raising())
+ return None
+
+ fake_config = types.SimpleNamespace(TORCHCODEC_AVAILABLE = True)
+ fake_audio = types.ModuleType("datasets.features.audio")
+ fake_audio.Audio = type("Audio", (), {"decode_example": None, "encode_example": None})
+ fake_datasets = types.ModuleType("datasets")
+ fake_datasets.config = fake_config
+ fake_features = types.ModuleType("datasets.features")
+ fake_features.__path__ = []
+
+ keys = (
+ "datasets",
+ "datasets.features",
+ "datasets.features.audio",
+ "datasets.features._torchcodec",
+ )
+ saved = {k: sys.modules.get(k) for k in keys}
+ sys.modules["datasets"] = fake_datasets
+ sys.modules["datasets.features"] = fake_features
+ sys.modules["datasets.features.audio"] = fake_audio
+ sys.modules.pop("datasets.features._torchcodec", None)
+ finder = _Finder()
+ sys.meta_path.insert(0, finder)
+ installed_before = audio_decode._installed
+ try:
+ audio_decode.ensure_audio_decoding()
+ assert (
+ fake_config.TORCHCODEC_AVAILABLE is False
+ ), "an AttributeError at import left torchcodec marked usable"
+ finally:
+ sys.meta_path.remove(finder)
+ audio_decode._installed = installed_before
+ for k, v in saved.items():
+ if v is None:
+ sys.modules.pop(k, None)
+ else:
+ sys.modules[k] = v
diff --git a/studio/backend/tests/test_dataset_preview_audio_cells.py b/studio/backend/tests/test_dataset_preview_audio_cells.py
index 58d5db2aa73..e526fe9f115 100644
--- a/studio/backend/tests/test_dataset_preview_audio_cells.py
+++ b/studio/backend/tests/test_dataset_preview_audio_cells.py
@@ -13,6 +13,8 @@ before it could POST /api/train/start.
from __future__ import annotations
+import pytest
+
from hub.services.datasets.formatting import _serialize_preview_value
@@ -43,3 +45,12 @@ def test_an_ordinary_list_column_is_untouched():
def test_a_missing_rate_does_not_raise():
out = _serialize_preview_value({"array": [0.0, 0.1], "sampling_rate": None})
assert out.startswith("<audio, 2 samples")
+
+
+def test_a_numpy_array_cell_is_summarised_too():
+ # The decoder hands back a numpy array, not a list; the raw repr was landing in the table.
+ np = pytest.importorskip("numpy")
+ cell = {"path": "a.m4a", "array": np.zeros(22050, dtype = "float32"), "sampling_rate": 22050}
+ assert _serialize_preview_value({"audio": cell}) == {
+ "audio": "<audio, 22050 samples @ 22050 Hz, 1.0s>"
+ }
diff --git a/studio/backend/utils/datasets/audio_decode.py b/studio/backend/utils/datasets/audio_decode.py
index 857be8d4abd..4ab49197d63 100644
--- a/studio/backend/utils/datasets/audio_decode.py
+++ b/studio/backend/utils/datasets/audio_decode.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Decode `datasets` Audio columns with soundfile when torchcodec cannot load. `datasets` 4.x decodes audio only through torchcodec, which needs an FFmpeg full-shared install to dlopen its native libraries; Windows has none by default, so `disable_torchcodec_if_broken` clears `datasets.config.TORCHCODEC_AVAILABLE` and every audio column raises, blocking the dataset format check and all six audio trainer paths on an otherwise working host. A soundfile decoder restores the pre-4.0 output contract, `{"path", "array", "sampling_rate"}`, which is what those callers already read."""
+"""Decode `datasets` Audio columns with soundfile, then PyAV, when torchcodec cannot load. `datasets` 4.x decodes audio only through torchcodec, which needs an FFmpeg full-shared install to dlopen its native libraries; Windows has none by default, so `disable_torchcodec_if_broken` clears `datasets.config.TORCHCODEC_AVAILABLE` and every audio column raises, blocking the dataset format check and all six audio trainer paths on an otherwise working host. A soundfile decoder restores the pre-4.0 output contract, `{"path", "array", "sampling_rate"}`, which is what those callers already read."""
from __future__ import annotations
@@ -44,16 +44,84 @@ def _token_for_url(path: str, token_per_repo_id: Optional[dict]) -> Any:
return token_per_repo_id.get(fields["repo_id"])
+def _decode_with_av(source: Any, stream_index: Optional[int] = None) -> "tuple[Any, int]":
+ """Mono float32 at the native rate through PyAV's bundled FFmpeg: every container torchcodec would have read (m4a, aac, webm, wma, amr) without a system FFmpeg. Same shape as routes/inference.py's upload decoder, minus its upload ceilings: a dataset row is not an upload."""
+ import av
+ import numpy as np
+
+ chunks = []
+ rate = 0
+ resampler = None
+ with av.open(source, mode = "r", metadata_errors = "ignore") as container:
+ if not container.streams.audio:
+ raise ValueError("audio container has no audio stream")
+ # datasets.Audio(stream_index=...) is the container's absolute stream index, as torchcodec reads it; None is the best audio stream.
+ try:
+ if stream_index is None:
+ # av_find_best_stream, which torchcodec uses: the default-disposition track wins over the first. PyAV < 13 has no wrapper, so take the first audio track there.
+ best = getattr(container.streams, "best", None)
+ stream = best("audio") if best is not None else container.streams.audio[0]
+ else:
+ stream = container.streams[stream_index]
+ except IndexError:
+ raise ValueError(
+ f"stream {stream_index} is not in the container, which has {len(container.streams)} streams"
+ ) from None
+ if stream.type != "audio":
+ raise ValueError(f"stream {stream_index} is not an audio stream")
+ for frame in container.decode(stream):
+ if resampler is None:
+ rate = int(frame.sample_rate or 0)
+ if rate <= 0:
+ raise ValueError("decoded audio has an invalid sample rate")
+ resampler = av.AudioResampler(format = "flt", layout = "mono", rate = rate)
+ for out in resampler.resample(frame):
+ chunks.append(out.to_ndarray().reshape(-1))
+ if resampler is not None:
+ for out in resampler.resample(None):
+ chunks.append(out.to_ndarray().reshape(-1))
+ if not chunks:
+ raise ValueError("audio container decoded to no samples")
+ return np.concatenate(chunks).astype(np.float32, copy = False), rate
+
+
+def _read_mono(source: Any, stream_index: Optional[int] = None) -> "tuple[Any, int]":
+ """soundfile first (wav, flac, mp3, ogg), PyAV for the rest. `source` is a path, a bytes buffer or an open file."""
+ import numpy as np
+ import soundfile as sf
+
+ if stream_index not in (None, 0):
+ # libsndfile only knows single-stream files, so an explicit other stream is PyAV's alone.
+ return _decode_with_av(source, stream_index)
+ try:
+ array, rate = sf.read(source, dtype = "float32", always_2d = False)
+ except Exception as sf_error: # noqa: BLE001 libsndfile raises its own hierarchy
+ try:
+ import av # noqa: F401
+ except ImportError:
+ raise sf_error
+ if hasattr(source, "seek"):
+ source.seek(0)
+ try:
+ return _decode_with_av(source, stream_index)
+ except Exception as av_error: # noqa: BLE001
+ raise RuntimeError(
+ f"audio could not be decoded by soundfile ({sf_error}) or PyAV ({av_error})"
+ ) from av_error
+ if array.ndim > 1:
+ # soundfile returns (frames, channels); torchcodec returns (channels, frames).
+ array = np.mean(array, axis = -1)
+ return array, rate
+
+
def _decode_with_soundfile(
self,
value: dict,
token_per_repo_id: Optional[dict] = None,
) -> dict:
- """Stand-in for `datasets.Audio.decode_example` that never needs FFmpeg."""
+ """Stand-in for `datasets.Audio.decode_example` that never needs a system FFmpeg."""
import io
- import numpy as np
- import soundfile as sf
from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import is_local_path, xopen
@@ -78,10 +146,7 @@ def _decode_with_soundfile(
download_config = DownloadConfig(token = _token_for_url(path, token_per_repo_id)),
)
- array, sampling_rate = sf.read(source, dtype = "float32", always_2d = False)
- if array.ndim > 1:
- # soundfile returns (frames, channels); torchcodec returns (channels, frames).
- array = np.mean(array, axis = -1)
+ array, sampling_rate = _read_mono(source, getattr(self, "stream_index", None))
target = self.sampling_rate
if target and sampling_rate != target:
import librosa
@@ -104,8 +169,19 @@ def _encode_with_soundfile(self, value) -> dict:
if isinstance(value, (bytes, bytearray)):
return {"bytes": bytes(value), "path": None}
if isinstance(value, dict) and value.get("array") is not None:
+ import numpy as np
+
+ array = np.asarray(value["array"])
+ if array.dtype == object:
+ array = np.asarray(
+ array.tolist(), dtype = "float32"
+ ) # a nested list back from Arrow arrives as an object array
+ if array.ndim == 2 and array.shape[0] < array.shape[1]:
+ array = (
+ array.T
+ ) # torchcodec hands out (channels, samples); libsndfile writes (frames, channels)
buf = io.BytesIO()
- sf.write(buf, value["array"], value["sampling_rate"], format = "WAV")
+ sf.write(buf, array, value["sampling_rate"], format = "WAV")
return {"bytes": buf.getvalue(), "path": value.get("path")}
if isinstance(value, dict) and ("bytes" in value or "path" in value):
return {"bytes": value.get("bytes"), "path": value.get("path")}
@@ -127,7 +203,7 @@ def ensure_audio_decoding() -> bool:
try:
# config only ran find_spec, and an installed torchcodec whose native libraries cannot dlopen still passes that. The API process never imports unsloth, so disable_torchcodec_if_broken has not corrected the flag here.
from datasets.features._torchcodec import AudioDecoder # noqa: F401
- except (ImportError, OSError, RuntimeError) as exc:
+ except Exception as exc: # noqa: BLE001 a damaged wheel can raise anything at import; every shape means unusable
logger.info("torchcodec is installed but unusable (%s)", exc)
config.TORCHCODEC_AVAILABLE = False
if config.TORCHCODEC_AVAILABLE:
@@ -150,5 +226,5 @@ def ensure_audio_decoding() -> bool:
Audio.decode_example = _decode_with_soundfile
Audio.encode_example = _encode_with_soundfile
_installed = True
- logger.info("torchcodec is unusable; decoding dataset audio with soundfile")
+ logger.info("torchcodec is unusable; decoding dataset audio with soundfile and PyAV")
return True
diff --git a/tests/test_datasets_audio_fallback.py b/tests/test_datasets_audio_fallback.py
new file mode 100644
index 00000000000..10e6c85d30e
--- /dev/null
+++ b/tests/test_datasets_audio_fallback.py
@@ -0,0 +1,349 @@
+# Unsloth - 2x faster, 60% less VRAM LLM training and finetuning
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+"""`datasets` Audio columns keep decoding after a broken torchcodec is disabled (#8642):
+soundfile for what libsndfile reads, PyAV's bundled FFmpeg for the rest, no system FFmpeg.
+GPU-free; the decoder is installed straight from import_fixes."""
+
+from __future__ import annotations
+
+import ast
+import io
+import re
+from pathlib import Path
+
+import pytest
+
+np = pytest.importorskip("numpy")
+sf = pytest.importorskip("soundfile")
+datasets = pytest.importorskip("datasets")
+pytest.importorskip(
+ "torch"
+) # importing unsloth needs torch; Studio's no-torch venvs skip this file
+
+from unsloth import import_fixes # noqa: E402
+
+_REPO = Path(__file__).resolve().parents[1]
+_STUDIO_SHIM = _REPO / "studio" / "backend" / "utils" / "datasets" / "audio_decode.py"
+
+
+def _wav_bytes(samples = 1600, rate = 16000):
+ buf = io.BytesIO()
+ sf.write(buf, np.linspace(-0.5, 0.5, samples, dtype = "float32"), rate, format = "WAV")
+ return buf.getvalue()
+
+
+def _m4a_bytes(seconds = 1.0, rate = 22050):
+ av = pytest.importorskip("av")
+ t = np.arange(int(seconds * rate)) / rate
+ tone = (0.5 * np.sin(2 * np.pi * 440 * t)).astype("float32")
+ buf = io.BytesIO()
+ try:
+ with av.open(buf, "w", format = "mp4") as container:
+ stream = container.add_stream("aac", rate = rate)
+ stream.layout = "mono"
+ frame = av.AudioFrame.from_ndarray(tone[np.newaxis, :], format = "flt", layout = "mono")
+ frame.sample_rate = rate
+ for packet in stream.encode(frame):
+ container.mux(packet)
+ for packet in stream.encode(None):
+ container.mux(packet)
+ except Exception as exc: # noqa: BLE001
+ pytest.skip(f"this PyAV cannot encode AAC: {exc}")
+ return buf.getvalue()
+
+
+def broken_torchcodec(monkeypatch):
+ """What disable_torchcodec_if_broken leaves behind, with the class restored afterwards."""
+ from datasets import config
+ from datasets.features.audio import Audio
+
+ if not hasattr(config, "TORCHCODEC_AVAILABLE"):
+ pytest.skip("datasets < 4 decodes through soundfile itself")
+ monkeypatch.setattr(config, "TORCHCODEC_AVAILABLE", False)
+ monkeypatch.setattr(Audio, "decode_example", Audio.decode_example)
+ monkeypatch.setattr(Audio, "encode_example", Audio.encode_example)
+ monkeypatch.setattr(Audio, "_unsloth_audio_fallback", False, raising = False)
+
+
+def test_a_wav_row_decodes_and_resamples(broken_torchcodec):
+ from datasets import Audio, Dataset
+
+ assert import_fixes.patch_datasets_audio_decoding_without_torchcodec() is True
+ ds = Dataset.from_dict({"audio": [{"path": "a.wav", "bytes": _wav_bytes()}]})
+ ds = ds.cast_column("audio", Audio(sampling_rate = 24000))
+ decoded = ds[0]["audio"]
+ assert decoded["sampling_rate"] == 24000
+ assert len(decoded["array"]) == pytest.approx(2400, abs = 4)
+ assert decoded["path"] == "a.wav"
+
+
+def test_an_m4a_row_decodes_through_pyav(broken_torchcodec):
+ from datasets import Audio, Dataset
+
+ raw = _m4a_bytes()
+ with pytest.raises(Exception):
+ sf.read(io.BytesIO(raw)) # libsndfile cannot, so this row needs the PyAV leg
+ import_fixes.patch_datasets_audio_decoding_without_torchcodec()
+ ds = Dataset.from_dict({"audio": [{"path": "tone.m4a", "bytes": raw}]})
+ ds = ds.cast_column("audio", Audio(sampling_rate = 16000))
+ decoded = ds[0]["audio"]
+ array = np.asarray(decoded["array"])
+ assert decoded["sampling_rate"] == 16000 and array.ndim == 1
+ assert 15000 <= len(array) <= 17500
+ assert 0.4 <= float(np.abs(array).max()) <= 0.6
+
+
[email protected]("shape", ["absent", "broken"])
+def test_resampling_works_without_a_usable_librosa(broken_torchcodec, monkeypatch, tmp_path, shape):
+ import sys
+
+ pytest.importorskip("av")
+ if shape == "absent":
+ monkeypatch.setitem(sys.modules, "librosa", None) # `import librosa` now raises ImportError
+ else:
+ # An old librosa beside numpy 2 raises AttributeError at import; that must reach PyAV too.
+ (tmp_path / "librosa.py").write_text(
+ "raise AttributeError('np.complex was removed')\n", encoding = "utf-8"
+ )
+ monkeypatch.delitem(sys.modules, "librosa", raising = False)
+ monkeypatch.syspath_prepend(str(tmp_path))
+ out = import_fixes._audio_resample(np.zeros(1600, dtype = np.float32), 16000, 24000)
+ assert len(out) == pytest.approx(2400, abs = 8)
+
+
+def test_a_working_torchcodec_is_left_alone(monkeypatch):
+ from datasets import config
+ from datasets.features.audio import Audio
+
+ if not hasattr(config, "TORCHCODEC_AVAILABLE"):
+ pytest.skip("datasets < 4")
+ monkeypatch.setattr(config, "TORCHCODEC_AVAILABLE", True)
+ before = Audio.decode_example
+ assert import_fixes.patch_datasets_audio_decoding_without_torchcodec() is False
+ assert Audio.decode_example is before
+
+
+def test_the_disabler_installs_the_decoder():
+ # Read the source: importing a real torchcodec would decide this by the host, not the code.
+ src = ast.parse((_REPO / "unsloth" / "import_fixes.py").read_text(encoding = "utf-8"))
+ fn = next(
+ n
+ for n in ast.walk(src)
+ if isinstance(n, ast.FunctionDef) and n.name == "disable_torchcodec_if_broken"
+ )
+ calls = [
+ n.func.id for n in ast.walk(fn) if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
+ ]
+ assert "patch_datasets_audio_decoding_without_torchcodec" in calls
+
+
+def _two_stream_m4a_bytes(rate = 22050, freqs = (440, 880)):
+ """One MP4 holding two AAC tracks, a tone per stream; the second is the one a stream_index must reach."""
+ av = pytest.importorskip("av")
+ t = np.arange(rate) / rate
+ buf = io.BytesIO()
+ try:
+ with av.open(buf, "w", format = "mp4") as container:
+ streams = []
+ for hz in freqs:
+ stream = container.add_stream("aac", rate = rate)
+ stream.layout = "mono"
+ streams.append(stream)
+ for stream, hz in zip(streams, freqs):
+ tone = (0.5 * np.sin(2 * np.pi * hz * t)).astype("float32")
+ frame = av.AudioFrame.from_ndarray(tone[np.newaxis, :], format = "flt", layout = "mono")
+ frame.sample_rate = rate
+ for packet in stream.encode(frame):
+ container.mux(packet)
+ for packet in stream.encode(None):
+ container.mux(packet)
+ except Exception as exc: # noqa: BLE001
+ pytest.skip(f"this PyAV cannot encode AAC: {exc}")
+ return buf.getvalue()
+
+
+def _dominant_hz(array, rate):
+ spectrum = np.abs(np.fft.rfft(np.asarray(array, dtype = "float64")))
+ return float(np.fft.rfftfreq(len(array), 1.0 / rate)[int(np.argmax(spectrum))])
+
+
+def test_the_stream_index_selects_the_track(broken_torchcodec):
+ # datasets.Audio(stream_index=1) must reach the second track, as torchcodec would.
+ from datasets import Audio, Dataset
+
+ raw = _two_stream_m4a_bytes()
+ assert import_fixes.patch_datasets_audio_decoding_without_torchcodec() is True
+ rows = {"audio": [{"path": "two.m4a", "bytes": raw}]}
+ first = Dataset.from_dict(rows).cast_column("audio", Audio())[0]["audio"]
+ second = Dataset.from_dict(rows).cast_column("audio", Audio(stream_index = 1))[0]["audio"]
+ assert abs(_dominant_hz(first["array"], first["sampling_rate"]) - 440) < 20
+ assert abs(_dominant_hz(second["array"], second["sampling_rate"]) - 880) < 20
+ with pytest.raises(Exception, match = "stream"):
+ import_fixes._audio_read_mono(io.BytesIO(raw), stream_index = 5)
+
+
+def test_a_wheel_that_raises_anything_at_import_is_disabled(
+ broken_torchcodec, monkeypatch, tmp_path
+):
+ # A damaged torchcodec need not raise ImportError or RuntimeError; the installer already calls
+ # every failure "broken", so the library must disable it and seat the fallback the same way.
+ import sys
+
+ pkg = tmp_path / "torchcodec"
+ pkg.mkdir()
+ (pkg / "__init__.py").write_text("", encoding = "utf-8")
+ (pkg / "decoders.py").write_text("raise AttributeError('damaged wheel')\n", encoding = "utf-8")
+ for name in [n for n in sys.modules if n == "torchcodec" or n.startswith("torchcodec.")]:
+ monkeypatch.delitem(sys.modules, name)
+ monkeypatch.syspath_prepend(str(tmp_path))
+ monkeypatch.setattr(import_fixes, "_torchcodec_version_mismatch_hint", lambda: None)
+ monkeypatch.setattr(import_fixes, "_torchcodec_provenance_hint", lambda: None)
+ import warnings
+
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ import_fixes.disable_torchcodec_if_broken()
+ from datasets.features.audio import Audio
+
+ assert sys.modules["torchcodec"] is None
+ assert getattr(Audio, "_unsloth_audio_fallback", False) is True
+ # The one report a user gets, now that the installers say nothing: what failed and what decodes.
+ said = [str(w.message) for w in caught if "torchcodec is installed but" in str(w.message)]
+ assert said and "soundfile and PyAV" in said[0] and "reinstall torchcodec" in said[0]
+
+
+def test_the_audio_extras_carry_the_fallback_decoders():
+ # `unsloth[audio-torch2xx]` must install what decodes when torchcodec cannot load, or the
+ # fallback re-raises libsndfile's error on exactly the containers it exists for.
+ tomllib = pytest.importorskip("tomllib")
+ with open(_REPO / "pyproject.toml", "rb") as fh:
+ extras = tomllib.load(fh)["project"]["optional-dependencies"]
+ audio = {k: v for k, v in extras.items() if k.startswith("audio-torch")}
+ assert audio
+ for name, specs in audio.items():
+ names = {re.split(r"[ ;<>=!~\[]", spec, 1)[0] for spec in specs}
+ assert {"torchcodec", "soundfile", "av"} <= names, name
+
+
+def test_the_load_failure_is_classified_for_the_warning(monkeypatch):
+ # The import-time warning names the cause it can establish: FFmpeg off the loader path,
+ # FFmpeg present so the cause is elsewhere, or a failure that never reached libtorchcodec.
+ def raised(msg, cls = RuntimeError):
+ try:
+ raise cls(msg)
+ except cls as exc:
+ return exc
+
+ ffmpeg_gone = raised(
+ "Could not load libtorchcodec. Likely causes: 1. FFmpeg is not properly installed"
+ )
+ monkeypatch.setattr(import_fixes, "_ffmpeg_on_loader_path", lambda: False)
+ assert import_fixes._torchcodec_load_failure(ffmpeg_gone) == "ffmpeg"
+ monkeypatch.setattr(import_fixes, "_ffmpeg_on_loader_path", lambda: True)
+ assert import_fixes._torchcodec_load_failure(ffmpeg_gone) == "native"
+ assert (
+ import_fixes._torchcodec_load_failure(
+ raised("DLL load failed while importing _core", ImportError)
+ )
+ == "broken"
+ )
+ assert set(import_fixes._TORCHCODEC_FALLBACK_NOTES) == {"ffmpeg", "native", "broken"}
+
+
+def test_a_missing_index_picks_the_default_track_not_the_first(broken_torchcodec, tmp_path):
+ # torchcodec resolves stream_index=None through av_find_best_stream, where a default
+ # disposition beats position; a file whose second track is the default must decode it.
+ import shutil
+ import subprocess
+
+ av = pytest.importorskip("av")
+ if shutil.which("ffmpeg") is None or not hasattr(av.container.streams.StreamContainer, "best"):
+ pytest.skip("needs the ffmpeg CLI to set dispositions and PyAV >= 13 for streams.best")
+ path = tmp_path / "second_is_default.m4a"
+ subprocess.run(
+ [
+ "ffmpeg",
+ "-y",
+ "-loglevel",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=440:duration=1:sample_rate=22050",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=880:duration=1:sample_rate=22050",
+ "-map",
+ "0:a",
+ "-map",
+ "1:a",
+ "-c:a",
+ "aac",
+ "-disposition:a:0",
+ "0",
+ "-disposition:a:1",
+ "default",
+ str(path),
+ ],
+ check = True,
+ )
+ assert import_fixes.patch_datasets_audio_decoding_without_torchcodec() is True
+ array, rate = import_fixes._audio_read_mono(str(path))
+ assert abs(_dominant_hz(array, rate) - 880) < 20
+ array, rate = import_fixes._audio_read_mono(str(path), stream_index = 0)
+ assert abs(_dominant_hz(array, rate) - 440) < 20
+
+
+def test_a_channel_first_array_round_trips_through_the_encoder(broken_torchcodec):
+ # torchcodec hands decoded audio out as (channels, samples); libsndfile writes (frames, channels).
+ # Written as is, a (2, 1600) clip became two frames of 1600 channels, or failed outright.
+ from datasets import Audio, Dataset
+
+ assert import_fixes.patch_datasets_audio_decoding_without_torchcodec() is True
+ stereo = np.stack([np.linspace(-0.5, 0.5, 1600, dtype = "float32")] * 2)
+ assert stereo.shape == (2, 1600)
+ ds = Dataset.from_dict({"audio": [{"array": stereo, "sampling_rate": 16000, "path": "s.wav"}]})
+ decoded = ds.cast_column("audio", Audio(sampling_rate = 16000))[0]["audio"]
+ assert len(decoded["array"]) == 1600 and decoded["sampling_rate"] == 16000
+
+
+def _normalized(path: Path, name: str, rename: dict) -> str:
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ fn = next(n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == name)
+ fn.name = "f"
+ fn.returns = None # Studio annotates, the library does not; the bodies are what must match
+ for arg in fn.args.args:
+ arg.annotation = None
+ fn.body = [
+ b for b in fn.body if not (isinstance(b, ast.Expr) and isinstance(b.value, ast.Constant))
+ ]
+ for n in ast.walk(fn):
+ if isinstance(n, ast.Name) and n.id in rename:
+ n.id = rename[n.id]
+ return ast.dump(fn)
+
+
+ ("library", "studio"),
+ [("_audio_decode_with_av", "_decode_with_av"), ("_audio_read_mono", "_read_mono")],
+)
+def test_the_library_and_studio_decoders_do_not_drift(library, studio):
+ # Studio's API process never imports unsloth, so it carries its own copy of the decoder.
+ if not _STUDIO_SHIM.exists():
+ pytest.skip("no studio checkout")
+ rename = {"_audio_decode_with_av": "_decode_with_av"}
+ assert _normalized(_REPO / "unsloth" / "import_fixes.py", library, rename) == _normalized(
+ _STUDIO_SHIM, studio, {}
+ )
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 120dce37047..b5184a6f704 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -2665,6 +2665,55 @@ def _torchcodec_provenance_hint() -> "str | None":
)
+def _ffmpeg_on_loader_path():
+ """Every library torchcodec links (per the shipped libtorchcodec_core*.so NEEDED entries) resolvable by the dynamic loader. Distros package them separately, so a host missing only libswscale cannot load the codec; calling that present sends the user at a torch ABI bug."""
+ import ctypes.util
+ import glob
+ import os
+
+ dirs = [d for d in os.environ.get("PATH", "").split(os.pathsep) if d]
+ # A prefix on LD_LIBRARY_PATH may ship only versioned files (libavcodec.so.61), which the loader resolves but find_library never sees: it reads the ld cache and linker names.
+ libdirs = [
+ d
+ for v in ("LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH")
+ for d in os.environ.get(v, "").split(os.pathsep)
+ if d
+ ]
+ for name in ("avutil", "avcodec", "avformat", "avdevice", "avfilter", "swscale", "swresample"):
+ if ctypes.util.find_library(name):
+ continue
+ # find_library does not glob, so walk PATH for Windows names like avutil-59.dll. Windows only: WSL puts the Windows PATH on the Linux one, and those DLLs cannot load here.
+ if os.name == "nt" and any(glob.glob(os.path.join(d, name + "-*.dll")) for d in dirs):
+ continue
+ if os.name != "nt" and any(
+ glob.glob(os.path.join(d, "lib" + name + ".so*"))
+ or glob.glob(os.path.join(d, "lib" + name + ".*dylib"))
+ for d in libdirs
+ ):
+ continue
+ return False
+ return True
+
+
+def _torchcodec_load_failure(exc):
+ """Why an installed torchcodec did not import: "ffmpeg" (its FFmpeg libraries are not on the loader path), "native" (they are, so the cause is elsewhere) or "broken" (a failure that does not involve libtorchcodec at all)."""
+ import traceback
+
+ # One libtorchcodec message covers a missing FFmpeg, a torch mismatch and other runtime deps, so the text cannot pick between them. Ask the system: FFmpeg missing from the loader path is the one cause establishable here.
+ if "libtorchcodec" not in "".join(
+ traceback.format_exception(type(exc), exc, exc.__traceback__)
+ ):
+ return "broken"
+ return "native" if _ffmpeg_on_loader_path() else "ffmpeg"
+
+
+_TORCHCODEC_FALLBACK_NOTES = {
+ "ffmpeg": "cannot load its FFmpeg libraries; install an FFmpeg full-shared build only if you need torchcodec itself",
+ "native": "cannot load its native libraries although FFmpeg is on the loader path; likely an FFmpeg major it does not support (it takes 4 to 8), a missing CUDA NPP runtime (nvidia-npp), or a build that does not match this torch",
+ "broken": "fails to import for a reason other than its FFmpeg libraries; reinstall torchcodec against this torch build",
+}
+
+
def disable_torchcodec_if_broken():
"""Make broken torchcodec behave as if uninstalled (#5446).
@@ -2687,9 +2736,10 @@ def disable_torchcodec_if_broken():
if importlib.util.find_spec("torchcodec") is None:
return # absent or already disabled
- # RuntimeError on dlopen failure; OSError covers chained libavutil.so misses.
+ # RuntimeError on dlopen failure, OSError on chained libavutil.so misses, and a damaged or
+ # version-skewed wheel can raise anything else; the package is present, so every shape is "broken".
from torchcodec.decoders import AudioDecoder
- except (ImportError, RuntimeError, OSError):
+ except Exception as load_error:
if mismatch_hint is None:
# Versions agree, so the load failed for another reason. A mismatched accelerator
# build is the one this can still name, and the one pinning the index repairs.
@@ -2738,6 +2788,230 @@ def disable_torchcodec_if_broken():
]:
sys.modules.pop(_stale, None)
sys.modules["torchcodec"] = None
+ decodes = patch_datasets_audio_decoding_without_torchcodec()
+ try:
+ import warnings
+
+ note = _TORCHCODEC_FALLBACK_NOTES[_torchcodec_load_failure(load_error)]
+ tail = (
+ "audio datasets decode through soundfile and PyAV instead (wav/flac/mp3/ogg, m4a/aac/webm)"
+ if decodes
+ else "audio datasets will not decode until soundfile and PyAV are installed (pip install soundfile av)"
+ )
+ warnings.warn(f"Unsloth: torchcodec is installed but {note}; {tail}.", stacklevel = 2)
+ except Exception:
+ pass # a report must never abort the disable fallback above
+
+
+def _audio_decode_with_av(source, stream_index = None):
+ """Mono float32 at the native rate through PyAV's bundled FFmpeg: every container torchcodec would have read (m4a, aac, webm, wma, amr) without a system FFmpeg. Kept identical to studio/backend/utils/datasets/audio_decode.py; a test holds the two together."""
+ import av
+ import numpy as np
+
+ chunks = []
+ rate = 0
+ resampler = None
+ with av.open(source, mode = "r", metadata_errors = "ignore") as container:
+ if not container.streams.audio:
+ raise ValueError("audio container has no audio stream")
+ # datasets.Audio(stream_index=...) is the container's absolute stream index, as torchcodec reads it; None is the best audio stream.
+ try:
+ if stream_index is None:
+ # av_find_best_stream, which torchcodec uses: the default-disposition track wins over the first. PyAV < 13 has no wrapper, so take the first audio track there.
+ best = getattr(container.streams, "best", None)
+ stream = best("audio") if best is not None else container.streams.audio[0]
+ else:
+ stream = container.streams[stream_index]
+ except IndexError:
+ raise ValueError(
+ f"stream {stream_index} is not in the container, which has {len(container.streams)} streams"
+ ) from None
+ if stream.type != "audio":
+ raise ValueError(f"stream {stream_index} is not an audio stream")
+ for frame in container.decode(stream):
+ if resampler is None:
+ rate = int(frame.sample_rate or 0)
+ if rate <= 0:
+ raise ValueError("decoded audio has an invalid sample rate")
+ resampler = av.AudioResampler(format = "flt", layout = "mono", rate = rate)
+ for out in resampler.resample(frame):
+ chunks.append(out.to_ndarray().reshape(-1))
+ if resampler is not None:
+ for out in resampler.resample(None):
+ chunks.append(out.to_ndarray().reshape(-1))
+ if not chunks:
+ raise ValueError("audio container decoded to no samples")
+ return np.concatenate(chunks).astype(np.float32, copy = False), rate
+
+
+def _audio_read_mono(source, stream_index = None):
+ """soundfile first (wav, flac, mp3, ogg), PyAV for the rest. `source` is a path, a bytes buffer or an open file."""
+ import numpy as np
+ import soundfile as sf
+
+ if stream_index not in (None, 0):
+ # libsndfile only knows single-stream files, so an explicit other stream is PyAV's alone.
+ return _audio_decode_with_av(source, stream_index)
+ try:
+ array, rate = sf.read(source, dtype = "float32", always_2d = False)
+ except Exception as sf_error: # noqa: BLE001 libsndfile raises its own hierarchy
+ try:
+ import av # noqa: F401
+ except ImportError:
+ raise sf_error
+ if hasattr(source, "seek"):
+ source.seek(0)
+ try:
+ return _audio_decode_with_av(source, stream_index)
+ except Exception as av_error: # noqa: BLE001
+ raise RuntimeError(
+ f"audio could not be decoded by soundfile ({sf_error}) or PyAV ({av_error})"
+ ) from av_error
+ if array.ndim > 1:
+ # soundfile returns (frames, channels); torchcodec returns (channels, frames).
+ array = np.mean(array, axis = -1)
+ return array, rate
+
+
+def _audio_resample(array, rate, target):
+ """librosa when installed, else swresample through PyAV; both are already on the audio extras."""
+ try:
+ import librosa
+ except Exception: # noqa: BLE001 an old librosa beside numpy 2 raises AttributeError at import, not ImportError
+ librosa = None
+ if librosa is not None:
+ return librosa.resample(array, orig_sr = rate, target_sr = target)
+ import av
+ import numpy as np
+
+ frame = av.AudioFrame.from_ndarray(
+ np.ascontiguousarray(array, dtype = np.float32)[np.newaxis, :], format = "flt", layout = "mono"
+ )
+ frame.sample_rate = rate
+ resampler = av.AudioResampler(format = "flt", layout = "mono", rate = target)
+ chunks = [out.to_ndarray().reshape(-1) for out in resampler.resample(frame)]
+ chunks += [out.to_ndarray().reshape(-1) for out in resampler.resample(None)]
+ return np.concatenate(chunks)
+
+
+def patch_datasets_audio_decoding_without_torchcodec():
+ """Keep `datasets` Audio columns decodable when torchcodec is unusable (#8642).
+
+ `datasets` >= 4 decodes audio only through torchcodec, which needs an FFmpeg full-shared
+ install to dlopen its native libraries; with `TORCHCODEC_AVAILABLE` cleared above every
+ audio row raises "please install torchcodec" for a package that is installed. This seats
+ a decoder on `datasets.features.audio.Audio` that reads through soundfile, then PyAV's
+ bundled FFmpeg, and resamples to the cast rate, returning the pre-4.0 dict contract
+ `{"path", "array", "sampling_rate"}`. Studio installs the same decoder from its own copy
+ (utils/datasets/audio_decode.py) because its API process never imports unsloth.
+ No-op on `datasets` < 4, on a working torchcodec, and without soundfile. Idempotent.
+ """
+ try:
+ from datasets import config
+ from datasets.features.audio import Audio
+ except ImportError:
+ return False
+ if not hasattr(config, "TORCHCODEC_AVAILABLE") or config.TORCHCODEC_AVAILABLE:
+ return False
+ if getattr(Audio, "_unsloth_audio_fallback", False):
+ return True
+ try:
+ import soundfile # noqa: F401
+ except Exception: # noqa: BLE001 libsndfile absent raises OSError, not ImportError
+ return False
+ original_encode = Audio.encode_example
+
+ def _token_for_url(path, token_per_repo_id):
+ if not token_per_repo_id:
+ return None
+ try:
+ from datasets.utils.py_utils import string_to_dict
+
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL
+ if source_url.startswith(config.HF_ENDPOINT)
+ else config.HUB_DATASETS_HFFS_URL
+ )
+ fields = string_to_dict(source_url, pattern)
+ except Exception: # noqa: BLE001
+ fields = None
+ if fields is None:
+ values = list(token_per_repo_id.values())
+ return values[0] if len(values) == 1 else None
+ return token_per_repo_id.get(fields["repo_id"])
+
+ def decode_example(
+ self,
+ value,
+ token_per_repo_id = None,
+ ):
+ import io
+
+ from datasets.download.download_config import DownloadConfig
+ from datasets.utils.file_utils import is_local_path, xopen
+
+ if not self.decode:
+ raise RuntimeError(
+ "Decoding is disabled for this feature. Please use Audio(decode=True) instead."
+ )
+ path, raw = value["path"], value["bytes"]
+ if path is None and raw is None:
+ raise ValueError(
+ f"An audio sample should have one of 'path' or 'bytes' but both are None in {value}."
+ )
+ if raw is not None:
+ source = io.BytesIO(raw)
+ elif is_local_path(path):
+ source = path
+ else:
+ source = xopen(
+ path,
+ "rb",
+ download_config = DownloadConfig(token = _token_for_url(path, token_per_repo_id)),
+ )
+ array, sampling_rate = _audio_read_mono(source, getattr(self, "stream_index", None))
+ target = self.sampling_rate
+ if target and sampling_rate != target:
+ array = _audio_resample(array, sampling_rate, target)
+ sampling_rate = target
+ return {"path": path, "array": array, "sampling_rate": sampling_rate}
+
+ def encode_example(self, value):
+ import io
+ from pathlib import Path
+
+ import soundfile as sf
+
+ if isinstance(value, str):
+ return {"bytes": None, "path": value}
+ if isinstance(value, Path):
+ return {"bytes": None, "path": str(value.absolute())}
+ if isinstance(value, (bytes, bytearray)):
+ return {"bytes": bytes(value), "path": None}
+ if isinstance(value, dict) and value.get("array") is not None:
+ import numpy as np
+
+ array = np.asarray(value["array"])
+ if array.dtype == object:
+ array = np.asarray(
+ array.tolist(), dtype = "float32"
+ ) # a nested list back from Arrow arrives as an object array
+ if array.ndim == 2 and array.shape[0] < array.shape[1]:
+ array = (
+ array.T
+ ) # torchcodec hands out (channels, samples); libsndfile writes (frames, channels)
+ buf = io.BytesIO()
+ sf.write(buf, array, value["sampling_rate"], format = "WAV")
+ return {"bytes": buf.getvalue(), "path": value.get("path")}
+ if isinstance(value, dict) and ("bytes" in value or "path" in value):
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ return original_encode(self, value)
+
+ Audio.decode_example = decode_example
+ Audio.encode_example = encode_example
+ Audio._unsloth_audio_fallback = True
+ return True
def disable_torchaudio_if_cuda_mismatched():