Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""A classified failure reaches the client as {"detail": {"code", "message", ...}}; anything
unclassified keeps today's status and text."""
from __future__ import annotations
import asyncio
import dataclasses
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
import routes.inference as ri
from core.inference import audio_cpp_failures as af
from core.inference import audio_cpp_server as srv
from models.inference import ChatCompletionRequest, LoadRequest
from .test_audio_cpp_failures import _model
@pytest.fixture(autouse = True)
def _cuda_audio_runtime(monkeypatch):
monkeypatch.setattr(af, "_runtime_backend", lambda: "cuda")
_MISSING = "The audio runtime is not installed. Run `unsloth studio update` to install it."
def _oom() -> af.AudioCppFailure:
return af.classify_audio_cpp_failure("cudaMalloc failed: out of memory", backend = "cuda")
def _load(side_effect) -> HTTPException:
request = LoadRequest(model_path = "audio-cpp/audio.cpp-gguf/M-GGUF")
with (
patch.object(
ri,
"_resolve_model_identifier_for_request",
return_value = (request.model_path,) * 2 + (False,),
),
patch.object(ri, "resolve_effective_chat_template_override", return_value = None),
patch.object(ri, "get_inference_backend", return_value = MagicMock(active_model_name = None)),
patch.object(ri, "get_llama_cpp_backend", return_value = MagicMock()),
patch.object(ri.ModelConfig, "from_identifier", side_effect = side_effect),
):
with pytest.raises(HTTPException) as raised:
asyncio.run(ri.load_model(request, MagicMock(), current_subject = "test-user"))
return raised.value
def test_load_detail_is_structured_when_classified_else_500_prefix():
error = _load(af.AudioCppFailureError("Failed to load model", _oom()))
assert error.status_code == 503 and error.detail["code"] == "out_of_memory"
assert "ran out of memory" in error.detail["message"]
error = _load(RuntimeError("totally different failure"))
assert error.status_code == 500 and error.detail.startswith("Failed to load model: ")
def test_deferred_load_error_keeps_dict_detail(monkeypatch):
monkeypatch.setattr(ri, "_TUNNEL_KEEPALIVE_AFTER_S", 0.05)
monkeypatch.setattr(ri, "_TUNNEL_KEEPALIVE_EVERY_S", 0.02)
detail = _oom().detail()
async def slow_boom():
await asyncio.sleep(0.2)
raise HTTPException(status_code = 503, detail = detail)
async def run():
response = await ri._tunnel_safe_json(slow_boom(), label = "t")
return b"".join([chunk async for chunk in response.body_iterator])
body = json.loads(asyncio.run(run()))
assert body["_deferred_error"] == {"status_code": 503, "detail": detail}
def test_v1_path_maps_detail_code_to_openai_error_code():
from utils.api_errors import install_api_error_handlers
app = FastAPI()
install_api_error_handlers(app)
@app.post("/v1/audio/speech")
async def speech():
raise af.problem_http_exception(_MISSING)
response = TestClient(app).post("/v1/audio/speech")
assert response.status_code == 503
error = response.json()["error"]
assert error["code"] == "runtime_missing" and "isn't installed" in error["message"]
def _preflight(
problem,
*,
availability = None,
prepared = None,
model = None,
):
import utils.hardware as hardware
model = model or _model()
config = SimpleNamespace(identifier = model.id, audio_type = "audiocpp_tts", audio_cpp = model)
request = SimpleNamespace(gpu_ids = None, hf_token = None, max_seq_length = 2048, audio_device = None)
placement = ri._LoadPlacement(None, None, False, False)
with (
patch.object(srv, "model_runtime_problem", lambda _model: problem),
patch.object(ri, "_native_audio_cpu_load", lambda *_a: False),
patch.object(ri, "_native_audio_post_handoff_free_gb", lambda: availability),
patch("utils.hardware.get_device", return_value = hardware.DeviceType.CUDA),
patch.object(hardware, "IS_ROCM", False),
patch("core.inference.native_audio.native_audio_security_targets", return_value = [model.id]),
patch("utils.hardware.prepare_gpu_selection", return_value = prepared or ([0], {})),
):
return asyncio.run(ri._preflight_native_audio_placement(config, request, placement))
@pytest.mark.parametrize(
"problem, status, detail",
[
(_MISSING, 503, "runtime_missing"),
("The audio runtime is being updated. Try again in a moment.", 501, None),
],
)
def test_preflight_problem_is_503_with_code_else_501_string(problem, status, detail):
with pytest.raises(HTTPException) as raised:
_preflight(problem)
assert raised.value.status_code == status
assert (
raised.value.detail == problem if detail is None else raised.value.detail["code"] == detail
)
def test_placement_refusal_lists_variants_that_fit(monkeypatch):
monkeypatch.setattr(af, "offload_options", lambda model: [])
availability = ri._NativeAudioAvailability({0: 7.0, 1: 5.0}, {0: 0.0, 1: 0.0})
with pytest.raises(HTTPException) as raised:
_preflight(
None, availability = availability, prepared = ([0, 1], {"selection_mode": "fallback_all"})
)
detail = raised.value.detail
assert ri.audio_cpp_resident_gb is af.estimate_resident_gb
assert raised.value.status_code == 400 and detail["code"] == "out_of_memory"
assert [row["label"] for row in detail["smaller_variants"]] == ["Q2", "Q1"]
assert detail["message"].startswith(
"No single GPU has enough free memory for Model M. It needs about 21.5 GB of GPU memory "
"while running, and the most free on one GPU is 7.0 GB."
)
def test_with_nothing_to_unload_the_refusal_reads_the_cards_free_memory(monkeypatch):
monkeypatch.setattr(af, "offload_options", lambda model: [])
monkeypatch.setattr(ri, "_visible_gpu_memory_gb", lambda: {0: (7.0, 8.0), 1: (5.0, 8.0)})
with pytest.raises(HTTPException) as raised:
_preflight(None, availability = None, prepared = ([0, 1], {"selection_mode": "fallback_all"}))
detail = raised.value.detail
assert [row["label"] for row in detail["smaller_variants"]] == ["Q2", "Q1"]
assert "and the most free on one GPU is 7.0 GB." in detail["message"]
@pytest.mark.parametrize(
"family, size_gb, free_gb, fits",
[
# 7.3 GB free is an 8 GB card under a desktop session.
("voxtral_tts", 3, 7.3, True),
("chatterbox", 3, 7.3, True),
("neutts", 3, 7.3, False),
("vibevoice", 3, 7.3, False),
# Families whose measured peak is above the generic formula are refused where it fit.
("voxcpm2", 2.75, 9.5, False),
("neutts", 2.75, 9.5, True),
("kokoro_tts", 0.18, 3.0, False),
# ACE-Step1.5 Q8_0 on a 16 GB card: measured 11.2 GB, the generic formula said 15.9.
("ace_step", 5.76, 15.0, True),
("ace_step", 5.76, 9.2, False),
],
)
def test_placement_uses_the_family_estimate(monkeypatch, family, size_gb, free_gb, fits):
monkeypatch.setattr(af, "offload_options", lambda model: [])
size = int(size_gb * 1024**3)
base = _model(sizes = (3,))
variant = dataclasses.replace(
base.variant, files = (dataclasses.replace(base.variant.files[0], size = size),)
)
model = dataclasses.replace(base, family = family, variant = variant, variants = (variant,))
availability = ri._NativeAudioAvailability({0: free_gb}, {0: 0.0})
prepared = ([0], {"selection_mode": "fallback_all"})
if not fits:
with pytest.raises(HTTPException) as raised:
_preflight(None, availability = availability, prepared = prepared, model = model)
assert raised.value.detail["code"] == "out_of_memory"
return
placement = _preflight(None, availability = availability, prepared = prepared, model = model)
assert placement.resolved_gpu_ids == [0]
assert placement.native_required_gb == af.estimate_resident_gb(size, family) < free_gb
@pytest.mark.parametrize(
"backend, capabilities, driver, free_gb, required",
[
# #16627: a 10 GB card with about 9.4 GiB free.
("cuda", {0: (8, 6)}, True, 9.4, 9.23),
("cuda", {0: (8, 6)}, True, 9.2, None),
("vulkan", {0: (8, 6)}, True, 9.6, 9.51),
("vulkan", {0: (8, 6)}, True, 9.4, None),
(None, {}, False, 9.6, 9.51),
(None, {0: (8, 6)}, True, 9.4, None),
("cuda", {0: (7, 5)}, True, 9.6, None),
("cuda", {}, True, 9.6, None),
],
)
def test_yue2_q8_on_a_10gb_card(monkeypatch, backend, capabilities, driver, free_gb, required):
from utils.hardware import nvidia
monkeypatch.setattr(af, "_runtime_backend", lambda: backend)
monkeypatch.setattr(af, "offload_options", lambda model: [])
monkeypatch.setattr(nvidia, "get_compute_capabilities", lambda: dict(capabilities))
monkeypatch.setattr(nvidia, "has_nvidia_driver", lambda: driver)
base = _model(sizes = (3,))
variant = dataclasses.replace(
base.variant, files = (dataclasses.replace(base.variant.files[0], size = 4531978293),)
)
model = dataclasses.replace(base, family = "yue2", variant = variant, variants = (variant,))
availability = ri._NativeAudioAvailability({0: free_gb}, {0: 0.0})
prepared = ([0], {"selection_mode": "fallback_all"})
if required is None:
with pytest.raises(HTTPException) as raised:
_preflight(None, availability = availability, prepared = prepared, model = model)
assert raised.value.detail["code"] == "out_of_memory"
return
placement = _preflight(None, availability = availability, prepared = prepared, model = model)
assert placement.resolved_gpu_ids == [0] and placement.native_required_gb == required
def test_generation_crash_oom_maps_to_out_of_memory(monkeypatch):
tail = "ggml_gallocr_reserve_n: failed to allocate CUDA0 buffer of size 9663676416"
crash = af.attach_failure(
srv.AudioCppUnavailableError("The audio runtime stopped while serving the request."),
af.classify_audio_cpp_failure(tail, backend = "cuda"),
)
model = "audio-cpp/audio.cpp-gguf/M-GGUF"
backend = SimpleNamespace(
active_model_name = model,
models = {model: {"is_audio": True, "audio_type": "audiocpp_tts"}},
generate_audio_response = MagicMock(side_effect = crash),
)
monkeypatch.setattr(
ri, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False, _is_audio = False)
)
monkeypatch.setattr(ri, "get_inference_backend", lambda: backend)
monkeypatch.setattr(ri, "_maybe_auto_switch_model", AsyncMock(return_value = None))
payload = ChatCompletionRequest(model = model, messages = [{"role": "user", "content": "hello"}])
with pytest.raises(HTTPException) as raised:
asyncio.run(ri._generate_tts_wav("hello", payload, request = None, current_subject = "t"))
assert raised.value.status_code == 503 and raised.value.detail["code"] == "out_of_memory"
def test_oom_retry_and_monitor_read_the_structured_detail(monkeypatch):
oom = HTTPException(status_code = 503, detail = _oom().detail())
assert ri._ran_out_of_memory(oom)
other = af.classify_audio_cpp_failure("model path does not exist: /m", backend = "cuda")
assert not ri._ran_out_of_memory(HTTPException(status_code = 422, detail = other.detail()))
# Out of CPU RAM: unloading kept GPU models cannot make room, so no eviction retry.
cpu = af.classify_audio_cpp_failure(
"failed to allocate CPU buffer of size 9663676416", backend = "cpu"
)
assert cpu.extras == {"device": "cpu"}
assert not ri._ran_out_of_memory(HTTPException(status_code = 503, detail = cpu.detail()))
monitor = MagicMock()
monkeypatch.setattr(ri, "api_monitor", monitor)
monkeypatch.setattr(ri, "_request_used_api_key", lambda request: False)
request = SimpleNamespace(
state = SimpleNamespace(), url = SimpleNamespace(path = "/x"), method = "POST"
)
async def run():
async with ri._monitored_media_request(request, model = "m", prompt = "", subject = None):
raise oom
with pytest.raises(HTTPException):
asyncio.run(run())
assert monitor.fail.call_args.args[1] == oom.detail["message"]