Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

15.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Known audio.cpp failures, named so clients can offer a fix; raw runtime text is only classified,
never returned, and details go through sanitize_runtime_tail."""

from __future__ import annotations

import re
from dataclasses import asdict, dataclass, field
from typing import Any, Optional

from core.inference.audio_errors import sanitize_runtime_tail

# Ordered: the first code with a matching pattern wins. Lowercased text is matched.
_PATTERNS = tuple(
    (code, re.compile(pattern))
    for code, pattern in (
        # Missing library or data only: a request the runtime refuses can mention eSpeak too.
        (
            "espeak_missing",
            r"could not load espeak|espeak-ng failed to initialize|espeak data package missing"
            r"|cannot (?:read|open) espeak data|needs an audio runtime built with espeak",
        ),
        ("runtime_missing", r"the audio runtime is not installed"),
        (
            "family_unknown_to_runtime",
            r"unsupported model family hint:|no registered model loader can load"
            r"|needs a newer audio runtime|update the audio runtime",
        ),
        (
            "gpu_unavailable",
            r"backend requested but|failed to initialize \w+ backend on device"
            r"|failed to initialize (?:cuda|rocm|musa)\b|ggml_vulkan: no devices found|has no device",
        ),
        (
            "out_of_memory",
            r"backend weight buffer|failed to allocate \S+ buffer of size|cudamalloc failed"
            r"|cuda error: out of memory|memory allocation of size \d+ failed"
            r"|does not fit in device memory|insufficient_memory|headroom exceeds available"
            r"|no single gpu has enough free memory",
        ),
        (
            "file_incomplete",
            r"failed to read (?:magic|header|tensor info|tensor data)|invalid magic characters"
            r"|failed to read gguf (?:metadata|tensor source)|tensor data range is out of bounds"
            r"|model path does not exist",
        ),
    )
)
_CODES = frozenset(code for code, _ in _PATTERNS) | {"runtime_stopped"}
_STATUS = {"file_incomplete": 422}  # else 503

# Messages that already come from Unsloth Studio, so no runtime line is appended.
_STUDIO_CODES = frozenset({"runtime_missing", "family_unknown_to_runtime", "espeak_missing"})

_UPDATE = "Update it from the update banner in Unsloth Studio, or run `unsloth studio update`."
_MESSAGES = {
    "runtime_missing": "The audio runtime isn't installed. Install it from the update banner in "
    "Unsloth Studio, or run `unsloth studio update`.",
    "family_unknown_to_runtime": "{name} needs a newer audio runtime than the one installed. "
    + _UPDATE,
    "espeak_missing": "{name} needs an audio runtime built with eSpeak-ng. " + _UPDATE,
    "gpu_unavailable": "The audio runtime could not use the GPU. Set the audio device to CPU in "
    "Unsloth Studio, or set UNSLOTH_AUDIO_CPP_BACKEND=cpu.",
    "out_of_memory": "{name} ran out of memory. Free GPU memory, choose a smaller variant, or set "
    "the audio device to CPU.",
    "out_of_memory_cpu": "{name} ran out of memory. Close other apps to free RAM, or choose a "
    "smaller variant.",
    "file_incomplete": "The model file is incomplete or damaged. Delete it and download it again, "
    "or choose another variant.",
    # Runtime 4xx: the input is too big for the device (upstream CapacityError).
    "capacity": "This input does not fit in device memory. Try a shorter input, or set the audio "
    "device to CPU.",
    "capacity_cpu": "This input does not fit in memory. Try a shorter input.",
    # Unsloth Studio's own refusal before launch.
    "placement": "No single GPU has enough free memory for {name}.{need} Free GPU memory, choose a "
    "smaller variant, or set the audio device to CPU.",
}


@dataclass(frozen = True)
class AudioCppFailure:
    code: str
    message: str
    status: int
    # smaller_variants / offload_options / device only: never a path or raw runtime text.
    extras: dict = field(default_factory = dict)

    def detail(self) -> dict:
        return {"code": self.code, "message": self.message, **self.extras}

    def payload(self) -> dict:
        return asdict(self)

    @classmethod
    def from_payload(cls, data: Any) -> Optional["AudioCppFailure"]:
        if not isinstance(data, dict) or data.get("code") not in _CODES:
            return None
        code = str(data["code"])
        extras = data.get("extras") or {}
        extras = {
            k: v
            for k, v in extras.items()
            if k in ("smaller_variants", "offload_options", "device")
        }
        return cls(
            code,
            str(data.get("message") or ""),
            int(data.get("status") or _STATUS.get(code, 503)),
            extras,
        )


class AudioCppFailureError(RuntimeError):
    """A worker's classified failure, rebuilt in the parent process."""

    def __init__(self, message: str, failure: AudioCppFailure) -> None:
        super().__init__(message)
        self.audio_cpp_failure = failure


def failure_of(exc: Any) -> Optional[AudioCppFailure]:
    failure = getattr(exc, "audio_cpp_failure", None)
    return failure if isinstance(failure, AudioCppFailure) else None


def attach_failure(exc: BaseException, failure: Optional[AudioCppFailure]) -> BaseException:
    if failure is not None and failure_of(exc) is None:
        try:
            exc.audio_cpp_failure = failure
        except AttributeError:
            pass
    return exc


# GiB; resident = weights * per_weight + fixed. The generic lines are per-backend fallbacks for
# unmeasured families; a lower CUDA line would run a held-out measured family out of memory.
_GENERIC_PROFILE = (2.5, 1.5)
_VULKAN_GENERIC_PROFILE = (1.85, 4.5)
# Peak keeps growing with the input, so no line fit to peaks holds: the CUDA fallback everywhere.
_UNBOUNDED_FAMILIES = frozenset({"vibevoice"})
# family -> (per_weight, fixed, largest package measured): audio.cpp v0.9.3 CUDA peak plus at least
# 0.9 GiB, at default options and each family's generation cap; a raised token cap can exceed it.
_MEASURED_PROFILES = {
    "ace_step": (0.98, 6.46, 9.4),
    "breeze_tts": (0.55, 5.18, 6.9),
    # Covers convert sources up to 30 s; a 60 s source peaks at 12.7 GiB.
    "chatterbox": (0.05, 5.98, 3.5),
    "cosyvoice3": (0.6, 2.82, 6.6),
    "heartmula": (2.5, 1.48, 7.2),
    # Measured at the default 2048-token chunk cap; a higher max_tokens grows the cache.
    "higgs_audio_tts": (1.0, 2.4, 8.0),
    # Also covers the 2.74 GiB #16514 measured for Q8_0 (whole device).
    "kokoro_tts": (0.0, 3.65, 0.9),
    "midashenglm_gen": (1.0, 1.69, 3.2),
    "minimax_music3": (0.7, 8.49, 12.7),
    "moss_tts_v15": (0.95, 0.74, 19.2),
    "omnivoice": (0.05, 5.29, 1.6),
    # Steeper than the tightest fit so 0.6B Q8_0 (7.2 GB) stays loadable on an 8 GB card.
    "qwen3_tts": (1.5, 4.44, 4.3),
    "stable_audio": (1.5, 1.17, 5.5),
    # Kept from #16514 (whole-device peaks above these runs); also covers Vulkan, so no entry there.
    "voxcpm2": (2.2, 4.7, 8.9),
    "voxtral_tts": (1.0, 2.0, 7.5),
    "yue2": (1.03, 4.88, 7.1),
}
# The same inputs on audio.cpp's Vulkan build (AMD hosts); a family absent here uses its CUDA line.
_VULKAN_PROFILES = {
    "ace_step": (1.15, 5.55, 9.4),
    "breeze_tts": (0.0, 12.1, 6.9),
    "chatterbox": (0.25, 5.25, 3.5),
    "cosyvoice3": (0.6, 4.38, 6.6),
    "heartmula": (2.5, 2.7, 7.2),
    "higgs_audio_tts": (0.95, 2.94, 8.0),
    "kokoro_tts": (0.1, 3.89, 0.9),
    "midashenglm_gen": (1.0, 2.25, 3.2),
    "minimax_music3": (0.7, 9.41, 12.7),
    "moss_tts_v15": (1.0, 2.63, 19.2),
    "omnivoice": (0.05, 7.67, 1.6),
    "qwen3_tts": (1.1, 8.87, 4.3),
    "stable_audio": (1.8, 0.65, 5.5),
    "voxtral_tts": (1.7, -0.64, 7.5),
    "yue2": (1.2, 4.45, 7.1),
}
# Families measured far higher on pre-Ampere NVIDIA cards (YuE2 on a T4): the generic formula there.
_NOT_BEFORE_AMPERE = frozenset({"yue2"})


def _runtime_backend() -> Optional[str]:
    """The installed audio.cpp's backend ("cuda", "vulkan", ...), or None when none is installed."""
    from core.inference.audio_cpp_server import find_audio_cpp_server_binary, select_backend

    binary = find_audio_cpp_server_binary()
    return select_backend(binary, False) if binary else None


def _may_have_pre_ampere_nvidia_card() -> bool:
    """Uncached (gpu_query caches); unreadable NVIDIA capabilities count as pre-Ampere."""
    from utils.hardware import nvidia

    capabilities = nvidia.get_compute_capabilities()
    if not capabilities:
        return nvidia.has_nvidia_driver()
    return any(capability < (8, 0) for capability in capabilities.values())


def gpu_estimate_applies(audio_type: Optional[str]) -> bool:
    """Whether the placement check sizes an Auto load of ``audio_type`` with its estimate."""
    import utils.hardware as hardware
    from core.inference.audio_cpp_models import AUDIO_CPP_AUDIO_TYPES
    from core.inference.audio_device import audio_load_runs_on_cpu

    return (
        audio_type in AUDIO_CPP_AUDIO_TYPES
        and hardware.get_device() in (hardware.DeviceType.CUDA, hardware.DeviceType.XPU)
        and not audio_load_runs_on_cpu(audio_type, "auto")
    )


def estimate_resident_gb(size_bytes: int, family: Optional[str] = None) -> float:
    """Resident GPU memory for an audio.cpp package: weights plus graph and cache buffers.
    A measured family uses its own profile; a package past the sizes measured for it gets the
    larger of that profile and the generic one."""
    weights_gb = size_bytes / (1024**3)
    backend = _runtime_backend()

    def pick(cuda, vulkan):
        # Not installed yet, or a build measured on neither (HIP, CPU): both, and the larger wins.
        return {"cuda": [cuda], "vulkan": [vulkan]}.get(backend, [cuda, vulkan])

    generics = (
        [_GENERIC_PROFILE]
        if family in _UNBOUNDED_FAMILIES
        else pick(_GENERIC_PROFILE, _VULKAN_GENERIC_PROFILE)
    )
    generic = max(weights_gb * per_weight + fixed for per_weight, fixed in generics)
    profile = _MEASURED_PROFILES.get(family or "")
    if profile is None or (family in _NOT_BEFORE_AMPERE and _may_have_pre_ampere_nvidia_card()):
        return round(generic, 2)
    profiles = pick(profile, _VULKAN_PROFILES[family]) if family in _VULKAN_PROFILES else [profile]
    measured = max(weights_gb * per_weight + fixed for per_weight, fixed, _largest in profiles)
    measured_up_to_gb = min(largest for _per_weight, _fixed, largest in profiles)
    return round(measured if weights_gb <= measured_up_to_gb else max(measured, generic), 2)


def smaller_variants(
    model,
    *,
    free_gb: Optional[float] = None,
    limit: int = 3,
) -> list[dict]:
    """Variants with a lower estimate than the loaded one, largest first; only those that fit
    ``free_gb``. Byte size alone is not enough: ACE-Step base is a few bytes under turbo."""
    if model is None:
        return []
    from core.inference.audio_cpp_models import quant_label, resolve

    extra = 0
    for companion in getattr(model, "companions", ()) or ():
        try:
            found = resolve(companion.id, companion.variant, network = False)
        except Exception:  # noqa: BLE001 - an unknown companion counts as nothing
            found = None
        extra += found.size_bytes if found is not None else 0
    family = getattr(model, "family", None)
    current = estimate_resident_gb(model.size_bytes + extra, family)
    rows = []
    for variant in sorted(model.variants, key = lambda v: v.size_bytes, reverse = True):
        estimate = estimate_resident_gb(variant.size_bytes + extra, family)
        if (
            not 0 < variant.size_bytes < model.size_bytes
            or estimate >= current
            or (free_gb is not None and estimate > free_gb)
        ):
            continue
        label = variant.label or quant_label(variant.main_file)
        rows.append(
            {
                # The quant /api/models/gguf-variants lists for this row.
                "variant": variant.key,
                "label": label,
                "size_bytes": variant.size_bytes,
                "estimate_gb": estimate,
            }
        )
    return rows[:limit]


def offload_options(model) -> list[str]:
    """Session options of the installed runtime's spec that move experts off the GPU."""
    if model is None:
        return []
    from core.inference.audio_cpp_models import runtime_spec

    spec = runtime_spec(model.family) or {}
    session = (spec.get("options") or {}).get("session") if isinstance(spec, dict) else None
    return [
        str(option["name"])
        for option in session or ()
        if isinstance(option, dict) and "expert" in str(option.get("name") or "")
    ]


def classify_audio_cpp_failure(
    text: str,
    *,
    backend: Optional[str] = None,
    model = None,
    runtime_status: Optional[int] = None,
    free_gb: Optional[float] = None,
    required_gb: Optional[float] = None,
) -> Optional[AudioCppFailure]:
    """The known failure in ``text`` (a runtime log tail or error), or None."""
    raw = str(text or "")
    lowered = raw.lower()
    for code, pattern in _PATTERNS:
        # A CUDA build prints its CUDA init failure on CPU launches too.
        if code == "gpu_unavailable" and backend in (None, "cpu"):
            continue
        match = pattern.search(lowered)
        if match is None:
            continue
        capacity = (
            code == "out_of_memory" and runtime_status is not None and 400 <= runtime_status < 500
        )
        placement = "no single gpu" in lowered
        on_cpu = code == "out_of_memory" and backend == "cpu"
        key = "placement" if placement else "capacity" if capacity else code
        key = f"{key}_cpu" if on_cpu and key in ("out_of_memory", "capacity") else key
        need = ""
        if placement and required_gb is not None:
            digits = 2 if free_gb is not None and f"{required_gb:.1f}" == f"{free_gb:.1f}" else 1
            need = f" It needs about {required_gb:.{digits}f} GB of GPU memory while running"
            need += (
                f", and the most free on one GPU is {free_gb:.{digits}f} GB."
                if free_gb is not None
                else "."
            )
        message = _MESSAGES[key].format(
            name = getattr(model, "display_name", None) or "This model", need = need
        )
        # "device": the run was already on the CPU, so "set the audio device to CPU" can't help.
        extras: dict = {"device": "cpu"} if on_cpu else {}
        if code == "out_of_memory" and not capacity:
            if variants := smaller_variants(model, free_gb = free_gb):
                extras["smaller_variants"] = variants
                listed = ", ".join(
                    f"{row['label']} (~{row['estimate_gb']:.1f} GB)" for row in variants
                )
                message += f" Smaller variants: {listed}."
            if offload := offload_options(model):
                extras["offload_options"] = offload
                message += f" This model can offload experts: set {' or '.join(offload[:2])}."
        if code not in _STUDIO_CODES and not placement:
            start = raw.rfind("\n", 0, match.start()) + 1
            end = raw.find("\n", match.end())
            line = sanitize_runtime_tail(raw[start : end if end >= 0 else len(raw)], 200)
            if line:
                message += f" Details: {line}"
        return AudioCppFailure(code, message, 400 if capacity else _STATUS.get(code, 503), extras)
    return None


def problem_http_exception(problem: str, model = None):
    """A preflight refusal as an HTTPException: the classified shape, else today's 501 string."""
    from fastapi import HTTPException

    failure = classify_audio_cpp_failure(problem, model = model)
    if failure is None:
        return HTTPException(status_code = 501, detail = problem)
    return HTTPException(status_code = failure.status, detail = failure.detail())