Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

5.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Training worker installs the soundfile decoder before reading any row; it needs no optional deps."""

from __future__ import annotations

import ast
from pathlib import Path

_BACKEND = Path(__file__).resolve().parents[1]
_TRAINER = _BACKEND / "core/training/trainer.py"


def _load_and_format_dataset_body() -> str:
    text = _TRAINER.read_text(encoding = "utf-8")
    return text[text.index("    def load_and_format_dataset(") :]


def _calls_the_shim(node: ast.AST) -> bool:
    return any(
        isinstance(n, ast.Call) and getattr(n.func, "id", "") == "ensure_audio_decoding"
        for n in ast.walk(node)
    )


def test_the_shim_is_installed_before_the_first_row_is_read():
    body = _load_and_format_dataset_body()
    assert body.index("ensure_audio_decoding()") < body.index("= load_dataset(")


def test_the_audio_branches_are_still_covered():
    body = _load_and_format_dataset_body()
    assert body.index("ensure_audio_decoding()") < body.index(
        "# ========== AUDIO MODELS: custom preprocessing =========="
    )


def test_the_import_is_module_level():
    text = _TRAINER.read_text(encoding = "utf-8")
    assert "\nfrom utils.datasets.audio_decode import ensure_audio_decoding\n" in text


def test_the_early_call_cannot_stop_a_text_run():
    # Above the method's try, so anything uncaught would fail every run.
    fn = next(
        node
        for node in ast.walk(ast.parse(_TRAINER.read_text(encoding = "utf-8")))
        if isinstance(node, ast.FunctionDef) and node.name == "load_and_format_dataset"
    )
    first = next(stmt for stmt in fn.body if _calls_the_shim(stmt))
    assert isinstance(first, ast.Try), "the early call is not wrapped"
    assert any(isinstance(b, ast.Expr) and _calls_the_shim(b) for b in first.body)
    assert any(getattr(h.type, "id", "") == "Exception" for h in first.handlers)


def test_a_datasets_without_the_torchcodec_flag_returns_a_bool():
    # datasets < 4 has no TORCHCODEC_AVAILABLE; it decodes via soundfile.
    import sys
    import types

    from utils.datasets import audio_decode

    fake_config = types.SimpleNamespace()
    fake_audio = types.ModuleType("datasets.features.audio")
    fake_audio.Audio = type("Audio", (), {"decode_example": None, "encode_example": None})
    fake_datasets = types.ModuleType("datasets")
    fake_datasets.config = fake_config
    fake_features = types.ModuleType("datasets.features")

    saved = {
        k: sys.modules.get(k) for k in ("datasets", "datasets.features", "datasets.features.audio")
    }
    sys.modules["datasets"] = fake_datasets
    sys.modules["datasets.features"] = fake_features
    sys.modules["datasets.features.audio"] = fake_audio
    installed_before = audio_decode._installed
    try:
        assert audio_decode.ensure_audio_decoding() is True
        assert audio_decode._installed == installed_before, "patched a version that works"
        assert fake_audio.Audio.decode_example is None, "patched datasets<4"
    finally:
        for k, v in saved.items():
            if v is None:
                sys.modules.pop(k, None)
            else:
                sys.modules[k] = v


def test_a_torchcodec_that_raises_anything_at_import_is_marked_unusable():
    # Damaged wheels raise non-ImportError; treat every failure the same.
    import importlib.abc
    import importlib.machinery
    import sys
    import types

    from utils.datasets import audio_decode

    class _Raising(importlib.abc.Loader):
        def create_module(self, spec):
            return None

        def exec_module(self, module):
            raise AttributeError("module 'torchcodec' has no attribute '_core'")

    class _Finder(importlib.abc.MetaPathFinder):
        def find_spec(
            self,
            name,
            path,
            target = None,
        ):
            if name == "datasets.features._torchcodec":
                return importlib.machinery.ModuleSpec(name, _Raising())
            return None

    fake_config = types.SimpleNamespace(TORCHCODEC_AVAILABLE = True)
    fake_audio = types.ModuleType("datasets.features.audio")
    fake_audio.Audio = type("Audio", (), {"decode_example": None, "encode_example": None})
    fake_datasets = types.ModuleType("datasets")
    fake_datasets.config = fake_config
    fake_features = types.ModuleType("datasets.features")
    fake_features.__path__ = []

    keys = (
        "datasets",
        "datasets.features",
        "datasets.features.audio",
        "datasets.features._torchcodec",
    )
    saved = {k: sys.modules.get(k) for k in keys}
    sys.modules["datasets"] = fake_datasets
    sys.modules["datasets.features"] = fake_features
    sys.modules["datasets.features.audio"] = fake_audio
    sys.modules.pop("datasets.features._torchcodec", None)
    finder = _Finder()
    sys.meta_path.insert(0, finder)
    installed_before = audio_decode._installed
    try:
        audio_decode.ensure_audio_decoding()
        assert (
            fake_config.TORCHCODEC_AVAILABLE is False
        ), "an AttributeError at import left torchcodec marked usable"
    finally:
        sys.meta_path.remove(finder)
        audio_decode._installed = installed_before
        for k, v in saved.items():
            if v is None:
                sys.modules.pop(k, None)
            else:
                sys.modules[k] = v