Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Known audio.cpp failures, named so clients can offer a fix; raw runtime text is only classified,
never returned, and details go through sanitize_runtime_tail."""
from __future__ import annotations
import re
from dataclasses import asdict, dataclass, field
from typing import Any, Optional
from core.inference.audio_errors import sanitize_runtime_tail
# Ordered: the first code with a matching pattern wins. Lowercased text is matched.
_PATTERNS = tuple(
(code, re.compile(pattern))
for code, pattern in (
# Missing library or data only: a request the runtime refuses can mention eSpeak too.
(
"espeak_missing",
r"could not load espeak|espeak-ng failed to initialize|espeak data package missing"
r"|cannot (?:read|open) espeak data|needs an audio runtime built with espeak",
),
("runtime_missing", r"the audio runtime is not installed"),
(
"family_unknown_to_runtime",
r"unsupported model family hint:|no registered model loader can load"
r"|needs a newer audio runtime|update the audio runtime",
),
(
"gpu_unavailable",
r"backend requested but|failed to initialize \w+ backend on device"
r"|failed to initialize (?:cuda|rocm|musa)\b|ggml_vulkan: no devices found|has no device",
),
(
"out_of_memory",
r"backend weight buffer|failed to allocate \S+ buffer of size|cudamalloc failed"
r"|cuda error: out of memory|memory allocation of size \d+ failed"
r"|does not fit in device memory|insufficient_memory|headroom exceeds available"
r"|no single gpu has enough free memory",
),
(
"file_incomplete",
r"failed to read (?:magic|header|tensor info|tensor data)|invalid magic characters"
r"|failed to read gguf (?:metadata|tensor source)|tensor data range is out of bounds"
r"|model path does not exist",
),
)
)
_CODES = frozenset(code for code, _ in _PATTERNS) | {"runtime_stopped"}
_STATUS = {"file_incomplete": 422} # else 503
# Messages that already come from Unsloth Studio, so no runtime line is appended.
_STUDIO_CODES = frozenset({"runtime_missing", "family_unknown_to_runtime", "espeak_missing"})
_UPDATE = "Update it from the update banner in Unsloth Studio, or run `unsloth studio update`."
_MESSAGES = {
"runtime_missing": "The audio runtime isn't installed. Install it from the update banner in "
"Unsloth Studio, or run `unsloth studio update`.",
"family_unknown_to_runtime": "{name} needs a newer audio runtime than the one installed. "
+ _UPDATE,
"espeak_missing": "{name} needs an audio runtime built with eSpeak-ng. " + _UPDATE,
"gpu_unavailable": "The audio runtime could not use the GPU. Set the audio device to CPU in "
"Unsloth Studio, or set UNSLOTH_AUDIO_CPP_BACKEND=cpu.",
"out_of_memory": "{name} ran out of memory. Free GPU memory, choose a smaller variant, or set "
"the audio device to CPU.",
"out_of_memory_cpu": "{name} ran out of memory. Close other apps to free RAM, or choose a "
"smaller variant.",
"file_incomplete": "The model file is incomplete or damaged. Delete it and download it again, "
"or choose another variant.",
# Runtime 4xx: the input is too big for the device (upstream CapacityError).
"capacity": "This input does not fit in device memory. Try a shorter input, or set the audio "
"device to CPU.",
"capacity_cpu": "This input does not fit in memory. Try a shorter input.",
# Unsloth Studio's own refusal before launch.
"placement": "No single GPU has enough free memory for {name}.{need} Free GPU memory, choose a "
"smaller variant, or set the audio device to CPU.",
}
@dataclass(frozen = True)
class AudioCppFailure:
code: str
message: str
status: int
# smaller_variants / offload_options / device only: never a path or raw runtime text.
extras: dict = field(default_factory = dict)
def detail(self) -> dict:
return {"code": self.code, "message": self.message, **self.extras}
def payload(self) -> dict:
return asdict(self)
@classmethod
def from_payload(cls, data: Any) -> Optional["AudioCppFailure"]:
if not isinstance(data, dict) or data.get("code") not in _CODES:
return None
code = str(data["code"])
extras = data.get("extras") or {}
extras = {
k: v
for k, v in extras.items()
if k in ("smaller_variants", "offload_options", "device")
}
return cls(
code,
str(data.get("message") or ""),
int(data.get("status") or _STATUS.get(code, 503)),
extras,
)
class AudioCppFailureError(RuntimeError):
"""A worker's classified failure, rebuilt in the parent process."""
def __init__(self, message: str, failure: AudioCppFailure) -> None:
super().__init__(message)
self.audio_cpp_failure = failure
def failure_of(exc: Any) -> Optional[AudioCppFailure]:
failure = getattr(exc, "audio_cpp_failure", None)
return failure if isinstance(failure, AudioCppFailure) else None
def attach_failure(exc: BaseException, failure: Optional[AudioCppFailure]) -> BaseException:
if failure is not None and failure_of(exc) is None:
try:
exc.audio_cpp_failure = failure
except AttributeError:
pass
return exc
# GiB; resident = weights * per_weight + fixed. The generic lines are per-backend fallbacks for
# unmeasured families; a lower CUDA line would run a held-out measured family out of memory.
_GENERIC_PROFILE = (2.5, 1.5)
_VULKAN_GENERIC_PROFILE = (1.85, 4.5)
# Peak keeps growing with the input, so no line fit to peaks holds: the CUDA fallback everywhere.
_UNBOUNDED_FAMILIES = frozenset({"vibevoice"})
# family -> (per_weight, fixed, largest package measured): audio.cpp v0.9.3 CUDA peak plus at least
# 0.9 GiB, at default options and each family's generation cap; a raised token cap can exceed it.
_MEASURED_PROFILES = {
"ace_step": (0.98, 6.46, 9.4),
"breeze_tts": (0.55, 5.18, 6.9),
# Covers convert sources up to 30 s; a 60 s source peaks at 12.7 GiB.
"chatterbox": (0.05, 5.98, 3.5),
"cosyvoice3": (0.6, 2.82, 6.6),
"heartmula": (2.5, 1.48, 7.2),
# Measured at the default 2048-token chunk cap; a higher max_tokens grows the cache.
"higgs_audio_tts": (1.0, 2.4, 8.0),
# Also covers the 2.74 GiB #16514 measured for Q8_0 (whole device).
"kokoro_tts": (0.0, 3.65, 0.9),
"midashenglm_gen": (1.0, 1.69, 3.2),
"minimax_music3": (0.7, 8.49, 12.7),
"moss_tts_v15": (0.95, 0.74, 19.2),
"omnivoice": (0.05, 5.29, 1.6),
# Steeper than the tightest fit so 0.6B Q8_0 (7.2 GB) stays loadable on an 8 GB card.
"qwen3_tts": (1.5, 4.44, 4.3),
"stable_audio": (1.5, 1.17, 5.5),
# Kept from #16514 (whole-device peaks above these runs); also covers Vulkan, so no entry there.
"voxcpm2": (2.2, 4.7, 8.9),
"voxtral_tts": (1.0, 2.0, 7.5),
"yue2": (1.03, 4.88, 7.1),
}
# The same inputs on audio.cpp's Vulkan build (AMD hosts); a family absent here uses its CUDA line.
_VULKAN_PROFILES = {
"ace_step": (1.15, 5.55, 9.4),
"breeze_tts": (0.0, 12.1, 6.9),
"chatterbox": (0.25, 5.25, 3.5),
"cosyvoice3": (0.6, 4.38, 6.6),
"heartmula": (2.5, 2.7, 7.2),
"higgs_audio_tts": (0.95, 2.94, 8.0),
"kokoro_tts": (0.1, 3.89, 0.9),
"midashenglm_gen": (1.0, 2.25, 3.2),
"minimax_music3": (0.7, 9.41, 12.7),
"moss_tts_v15": (1.0, 2.63, 19.2),
"omnivoice": (0.05, 7.67, 1.6),
"qwen3_tts": (1.1, 8.87, 4.3),
"stable_audio": (1.8, 0.65, 5.5),
"voxtral_tts": (1.7, -0.64, 7.5),
"yue2": (1.2, 4.45, 7.1),
}
# Families measured far higher on pre-Ampere NVIDIA cards (YuE2 on a T4): the generic formula there.
_NOT_BEFORE_AMPERE = frozenset({"yue2"})
def _runtime_backend() -> Optional[str]:
"""The installed audio.cpp's backend ("cuda", "vulkan", ...), or None when none is installed."""
from core.inference.audio_cpp_server import find_audio_cpp_server_binary, select_backend
binary = find_audio_cpp_server_binary()
return select_backend(binary, False) if binary else None
def _may_have_pre_ampere_nvidia_card() -> bool:
"""Uncached (gpu_query caches); unreadable NVIDIA capabilities count as pre-Ampere."""
from utils.hardware import nvidia
capabilities = nvidia.get_compute_capabilities()
if not capabilities:
return nvidia.has_nvidia_driver()
return any(capability < (8, 0) for capability in capabilities.values())
def gpu_estimate_applies(audio_type: Optional[str]) -> bool:
"""Whether the placement check sizes an Auto load of ``audio_type`` with its estimate."""
import utils.hardware as hardware
from core.inference.audio_cpp_models import AUDIO_CPP_AUDIO_TYPES
from core.inference.audio_device import audio_load_runs_on_cpu
return (
audio_type in AUDIO_CPP_AUDIO_TYPES
and hardware.get_device() in (hardware.DeviceType.CUDA, hardware.DeviceType.XPU)
and not audio_load_runs_on_cpu(audio_type, "auto")
)
def estimate_resident_gb(size_bytes: int, family: Optional[str] = None) -> float:
"""Resident GPU memory for an audio.cpp package: weights plus graph and cache buffers.
A measured family uses its own profile; a package past the sizes measured for it gets the
larger of that profile and the generic one."""
weights_gb = size_bytes / (1024**3)
backend = _runtime_backend()
def pick(cuda, vulkan):
# Not installed yet, or a build measured on neither (HIP, CPU): both, and the larger wins.
return {"cuda": [cuda], "vulkan": [vulkan]}.get(backend, [cuda, vulkan])
generics = (
[_GENERIC_PROFILE]
if family in _UNBOUNDED_FAMILIES
else pick(_GENERIC_PROFILE, _VULKAN_GENERIC_PROFILE)
)
generic = max(weights_gb * per_weight + fixed for per_weight, fixed in generics)
profile = _MEASURED_PROFILES.get(family or "")
if profile is None or (family in _NOT_BEFORE_AMPERE and _may_have_pre_ampere_nvidia_card()):
return round(generic, 2)
profiles = pick(profile, _VULKAN_PROFILES[family]) if family in _VULKAN_PROFILES else [profile]
measured = max(weights_gb * per_weight + fixed for per_weight, fixed, _largest in profiles)
measured_up_to_gb = min(largest for _per_weight, _fixed, largest in profiles)
return round(measured if weights_gb <= measured_up_to_gb else max(measured, generic), 2)
def smaller_variants(
model,
*,
free_gb: Optional[float] = None,
limit: int = 3,
) -> list[dict]:
"""Variants with a lower estimate than the loaded one, largest first; only those that fit
``free_gb``. Byte size alone is not enough: ACE-Step base is a few bytes under turbo."""
if model is None:
return []
from core.inference.audio_cpp_models import quant_label, resolve
extra = 0
for companion in getattr(model, "companions", ()) or ():
try:
found = resolve(companion.id, companion.variant, network = False)
except Exception: # noqa: BLE001 - an unknown companion counts as nothing
found = None
extra += found.size_bytes if found is not None else 0
family = getattr(model, "family", None)
current = estimate_resident_gb(model.size_bytes + extra, family)
rows = []
for variant in sorted(model.variants, key = lambda v: v.size_bytes, reverse = True):
estimate = estimate_resident_gb(variant.size_bytes + extra, family)
if (
not 0 < variant.size_bytes < model.size_bytes
or estimate >= current
or (free_gb is not None and estimate > free_gb)
):
continue
label = variant.label or quant_label(variant.main_file)
rows.append(
{
# The quant /api/models/gguf-variants lists for this row.
"variant": variant.key,
"label": label,
"size_bytes": variant.size_bytes,
"estimate_gb": estimate,
}
)
return rows[:limit]
def offload_options(model) -> list[str]:
"""Session options of the installed runtime's spec that move experts off the GPU."""
if model is None:
return []
from core.inference.audio_cpp_models import runtime_spec
spec = runtime_spec(model.family) or {}
session = (spec.get("options") or {}).get("session") if isinstance(spec, dict) else None
return [
str(option["name"])
for option in session or ()
if isinstance(option, dict) and "expert" in str(option.get("name") or "")
]
def classify_audio_cpp_failure(
text: str,
*,
backend: Optional[str] = None,
model = None,
runtime_status: Optional[int] = None,
free_gb: Optional[float] = None,
required_gb: Optional[float] = None,
) -> Optional[AudioCppFailure]:
"""The known failure in ``text`` (a runtime log tail or error), or None."""
raw = str(text or "")
lowered = raw.lower()
for code, pattern in _PATTERNS:
# A CUDA build prints its CUDA init failure on CPU launches too.
if code == "gpu_unavailable" and backend in (None, "cpu"):
continue
match = pattern.search(lowered)
if match is None:
continue
capacity = (
code == "out_of_memory" and runtime_status is not None and 400 <= runtime_status < 500
)
placement = "no single gpu" in lowered
on_cpu = code == "out_of_memory" and backend == "cpu"
key = "placement" if placement else "capacity" if capacity else code
key = f"{key}_cpu" if on_cpu and key in ("out_of_memory", "capacity") else key
need = ""
if placement and required_gb is not None:
digits = 2 if free_gb is not None and f"{required_gb:.1f}" == f"{free_gb:.1f}" else 1
need = f" It needs about {required_gb:.{digits}f} GB of GPU memory while running"
need += (
f", and the most free on one GPU is {free_gb:.{digits}f} GB."
if free_gb is not None
else "."
)
message = _MESSAGES[key].format(
name = getattr(model, "display_name", None) or "This model", need = need
)
# "device": the run was already on the CPU, so "set the audio device to CPU" can't help.
extras: dict = {"device": "cpu"} if on_cpu else {}
if code == "out_of_memory" and not capacity:
if variants := smaller_variants(model, free_gb = free_gb):
extras["smaller_variants"] = variants
listed = ", ".join(
f"{row['label']} (~{row['estimate_gb']:.1f} GB)" for row in variants
)
message += f" Smaller variants: {listed}."
if offload := offload_options(model):
extras["offload_options"] = offload
message += f" This model can offload experts: set {' or '.join(offload[:2])}."
if code not in _STUDIO_CODES and not placement:
start = raw.rfind("\n", 0, match.start()) + 1
end = raw.find("\n", match.end())
line = sanitize_runtime_tail(raw[start : end if end >= 0 else len(raw)], 200)
if line:
message += f" Details: {line}"
return AudioCppFailure(code, message, 400 if capacity else _STATUS.get(code, 503), extras)
return None
def problem_http_exception(problem: str, model = None):
"""A preflight refusal as an HTTPException: the classified shape, else today's 501 string."""
from fastapi import HTTPException
failure = classify_audio_cpp_failure(problem, model = model)
if failure is None:
return HTTPException(status_code = 501, detail = problem)
return HTTPException(status_code = failure.status, detail = failure.detail())