Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""A CPU-placed audio load must not be charged VRAM it never takes, or the GPU checks refuse it."""
import asyncio
import sys
import types
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from fastapi import HTTPException # noqa: E402
import routes.inference as ri # noqa: E402
from routes.training_vram import _stt_sidecar_holds_no_vram # noqa: E402
@pytest.fixture(autouse = True)
def _neutral_audio_device_env(monkeypatch):
"""Clears UNSLOTH_AUDIO_DEVICE so a host-wide placement default cannot change these outcomes."""
monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
def _audio(audio_type = "higgs_tts2", **kwargs):
return types.SimpleNamespace(audio_type = audio_type, is_lora = False, identifier = "x/y", **kwargs)
def _request(audio_device = None):
return types.SimpleNamespace(audio_device = audio_device)
def test_only_a_native_audio_model_counts_as_a_cpu_audio_load():
assert ri._native_audio_cpu_load(_audio(), _request("cpu"))
assert not ri._native_audio_cpu_load(_audio(), _request("auto"))
assert not ri._native_audio_cpu_load(_audio(), _request(None))
def test_an_auto_gguf_audio_load_on_a_cpu_only_runtime_counts_as_cpu(monkeypatch):
from core.inference import audio_cpp_server
gguf_tts = _audio(audio_type = "audiocpp_tts")
monkeypatch.setattr(audio_cpp_server, "runtime_runs_on_cpu", lambda: True)
assert ri._native_audio_cpu_load(gguf_tts, _request("auto"))
assert not ri._native_audio_cpu_load(_audio(), _request("auto"))
monkeypatch.setattr(audio_cpp_server, "runtime_runs_on_cpu", lambda: False)
assert not ri._native_audio_cpu_load(gguf_tts, _request("auto"))
def test_a_chat_model_cannot_skip_the_guards_by_sending_audio_device():
"""Ungated, any load could set audio_device and skip the training guard."""
assert not ri._native_audio_cpu_load(_audio(audio_type = None), _request("cpu"))
assert not ri._native_audio_cpu_load(_audio(audio_type = "whisper"), _request("cpu"))
def test_a_cpu_audio_load_is_not_refused_while_training_runs(monkeypatch):
"""Neither the 409 for unsized loads nor the training refusal applies to CPU weights."""
monkeypatch.setattr(ri, "_diffusion_training_active", lambda: True)
assert (
ri._guard_chat_load_against_training(
_audio(is_gguf = False),
types.SimpleNamespace(
audio_device = "cpu",
gpu_memory_mode = "auto",
gpu_layers = -1,
tensor_parallel = False,
),
load_in_4bit = False,
placement = types.SimpleNamespace(
requested_gpu_ids = None,
gpu_ids_are_vulkan_ordinals = False,
diffusion_kind = None,
),
)
is None
)
def test_a_gpu_audio_load_is_still_refused_during_diffusion_training(monkeypatch):
"""The exemption must be the CPU placement, not the audio type."""
monkeypatch.setattr(ri, "_diffusion_training_active", lambda: True)
with pytest.raises(HTTPException) as excinfo:
ri._guard_chat_load_against_training(
_audio(is_gguf = False),
types.SimpleNamespace(
audio_device = "auto",
gpu_memory_mode = "auto",
gpu_layers = -1,
tensor_parallel = False,
),
load_in_4bit = False,
placement = types.SimpleNamespace(
requested_gpu_ids = None,
gpu_ids_are_vulkan_ordinals = False,
diffusion_kind = None,
),
)
assert excinfo.value.status_code == 409
def test_a_cpu_load_skips_the_vram_preflight_entirely(monkeypatch):
"""Sizing would refuse a full-GPU load, the case the option is for; the probe must not run."""
def _never():
raise AssertionError("a CPU load must not size GPU memory")
monkeypatch.setattr(ri, "_native_audio_post_handoff_free_gb", _never)
placement = types.SimpleNamespace(requested_gpu_ids = None)
result = asyncio.run(ri._preflight_native_audio_placement(_audio(), _request("cpu"), placement))
assert result is placement
def test_minimax_on_cpu_is_refused_before_the_resident_model_is_evicted():
"""Its runtime needs CUDA; refusing late would evict the resident model before failing."""
with pytest.raises(HTTPException) as excinfo:
asyncio.run(
ri._preflight_native_audio_placement(
_audio(audio_type = "minimax_music3"),
_request("cpu"),
types.SimpleNamespace(requested_gpu_ids = None),
)
)
assert excinfo.value.status_code == 400
assert "CPU RAM" in excinfo.value.detail
def _backend(audio_cpu, audio_type = "higgs_tts2"):
entry = {"is_audio": True, "audio_type": audio_type}
if audio_cpu is not None:
entry["audio_cpu"] = audio_cpu
return types.SimpleNamespace(active_model_name = "x/y", models = {"x/y": entry})
def test_a_resident_gpu_audio_model_does_not_satisfy_a_cpu_request():
assert not ri._resident_audio_placement_matches(_backend(audio_cpu = False), _request("cpu"))
def test_a_resident_cpu_audio_model_satisfies_the_same_request_again():
assert ri._resident_audio_placement_matches(_backend(audio_cpu = True), _request("cpu"))
def test_a_model_loaded_before_this_existed_is_read_as_gpu():
"""No recorded key means the load predates the option, which placed on GPU."""
assert ri._resident_audio_placement_matches(_backend(audio_cpu = None), _request("auto"))
assert not ri._resident_audio_placement_matches(_backend(audio_cpu = None), _request("cpu"))
def test_a_gguf_audio_model_on_a_cpu_only_runtime_is_resident_in_cpu_ram(monkeypatch):
"""Auto on a CPU-only runtime lands in CPU RAM: the orchestrator must record that, and a repeat
Auto load must neither reload it nor take the GPU from a running Images/Video pipeline."""
import inspect
from core.inference import audio_cpp_server, orchestrator
monkeypatch.setattr(audio_cpp_server, "runtime_runs_on_cpu", lambda: True)
resident = _backend(audio_cpu = True, audio_type = "audiocpp_tts")
assert ri._resident_audio_placement_matches(resident, _request("auto"))
assert ri._resident_audio_holds_no_gpu(resident)
assert "audio_load_runs_on_cpu(_audio_type, audio_device)" in inspect.getsource(orchestrator)
monkeypatch.setattr(audio_cpp_server, "runtime_runs_on_cpu", lambda: False)
assert not ri._resident_audio_placement_matches(resident, _request("auto"))
def test_an_auto_load_on_a_cpu_only_runtime_reaches_the_worker_as_cpu(monkeypatch):
"""The worker hides the accelerators and no card is sized only for an explicit CPU load, so the
orchestrator must turn Auto into CPU when the runtime can only run there."""
import threading
from core.inference import audio_cpp_server
from core.inference import orchestrator as orch_mod
seen = {}
class _StopAfterSelection(threading.Event):
def is_set(self):
return "selection" in seen
def _gpu_selection(*args, **kwargs):
seen["selection"] = "gpu"
return [0], {"selection_mode": "auto"}
monkeypatch.setattr(orch_mod, "prepare_gpu_selection", _gpu_selection)
orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)
orch.loading_models = set()
config = types.SimpleNamespace(
identifier = "unsloth/Kokoro-82M-GGUF",
audio_type = "audiocpp_tts",
audio_cpp = object(),
gguf_variant = None,
)
def _load():
seen.clear()
orig = orch_mod.audio_device_forces_cpu
def _forces(value):
if "selection" not in seen and orig(value):
seen["selection"] = "cpu"
return orig(value)
monkeypatch.setattr(orch_mod, "audio_device_forces_cpu", _forces)
assert (
orch.load_model(config, audio_device = "auto", load_cancel_event = _StopAfterSelection())
is False
)
return seen["selection"]
monkeypatch.setattr(audio_cpp_server, "runtime_runs_on_cpu", lambda: True)
assert _load() == "cpu"
monkeypatch.setattr(audio_cpp_server, "runtime_runs_on_cpu", lambda: False)
assert _load() == "gpu"
def test_a_non_audio_model_keeps_the_shortcut():
assert ri._resident_audio_placement_matches(
_backend(audio_cpu = None, audio_type = None), _request("cpu")
)
def test_a_cpu_placed_sidecar_is_left_alone_when_training_claims_vram():
assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "cpu"))
assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = True))
assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "llama.cpp", _gpu_disabled = True))
def test_anything_that_might_hold_vram_is_still_evicted():
"""Default-deny: starving the run this makes room for is the worse failure."""
assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "cuda"))
assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "mps"))
assert not _stt_sidecar_holds_no_vram(
types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = False)
)
assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace())
class _Raises:
@property
def device(self):
raise RuntimeError("unreadable")
assert not _stt_sidecar_holds_no_vram(_Raises())
def test_the_shortcut_reads_the_resident_model_not_the_requested_config():
"""Runs before config resolution; reading a config there gave UnboundLocalError and 500s."""
import inspect
assert list(inspect.signature(ri._resident_audio_placement_matches).parameters) == [
"backend",
"request",
]
def test_a_cpu_placed_audio_model_never_takes_the_arbiter():
"""It holds no GPU, so acquiring would cancel an image or video run for nothing."""
assert ri._resident_audio_holds_no_gpu(_backend(audio_cpu = True))
assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = False))
assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = None))
def test_a_chat_model_cannot_reach_the_arbiter_skip():
"""Same audio-type gate as the writer, so a stray marker cannot skip it."""
assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = True, audio_type = None))
def test_nothing_resident_reads_as_holding_the_gpu():
empty = types.SimpleNamespace(active_model_name = None, models = {})
assert not ri._resident_audio_holds_no_gpu(empty)
def _inference_source() -> str:
import pathlib
# Explicit encoding: read_text() defaults to the locale one, cp1252 on Windows,
return pathlib.Path(ri.__file__).read_text(encoding = "utf-8")
def test_the_already_loaded_branch_guards_its_acquire():
src = _inference_source()
assert "if not _resident_audio_holds_no_gpu(backend):\n" in src
def test_the_post_load_ownership_check_is_gated_like_the_gguf_one():
"""Ungated, the CPU load unloads its model and returns 409, since acquire_for is skipped."""
src = _inference_source()
assert "if chat_load_needs_gpu and not holds(CHAT):" in src
assert "\n if not holds(CHAT):" not in src
assert "chat_load_needs_gpu and current_owner() != CHAT" not in src
def test_a_cpu_audio_worker_hides_cuda_and_hip():
from core.inference.audio_device import mask_accelerators_for_cpu_audio
env = {"CUDA_VISIBLE_DEVICES": "0,1", "HIP_VISIBLE_DEVICES": "0"}
mask_accelerators_for_cpu_audio(env)
assert env["CUDA_VISIBLE_DEVICES"] == ""
# HIP reads the CUDA var only when its own is unset; -1 is the CPU sentinel.
assert env["HIP_VISIBLE_DEVICES"] == "-1"
def test_an_inherited_rocr_mask_is_left_alone():
"""Clearing it exposes more agents to HSA enumeration, not fewer."""
from core.inference.audio_device import mask_accelerators_for_cpu_audio
env = {"ROCR_VISIBLE_DEVICES": "0"}
mask_accelerators_for_cpu_audio(env)
assert env["ROCR_VISIBLE_DEVICES"] == "0"
def test_the_mask_runs_before_hardware_detection():
"""get_device_properties creates the context, so the mask must precede detection."""
import pathlib
from core.inference import worker
src = pathlib.Path(worker.__file__).read_text(encoding = "utf-8")
assert src.index("mask_accelerators_for_cpu_audio(os.environ)") < src.index(
"_hw.detect_hardware()"
)
def test_a_zero_gpu_standard_load_drops_the_stale_chat_claim():
"""Leaving the CHAT claim after a load makes the next Images/Video acquire evict a CPU audio model."""
src = _inference_source()
assert src.count("await asyncio.to_thread(_release_chat_for_zero_vram_primary)") == 2
assert (
src.count(
"_release_chat_for_zero_vram_primary if replacing and not chat_load_needs_gpu else None"
)
== 1
)
def test_the_release_is_gated_on_the_same_flag_as_the_409():
"""Two if-not sites plus the callback's inline gate, which spells the flag the same way."""
src = _inference_source()
assert src.count("if replacing and not chat_load_needs_gpu:") == 2
assert src.count("if replacing and not chat_load_needs_gpu else None") == 1
def test_every_http_device_field_pins_the_three_canonical_values():
"""A misspelled cpu would fall to auto and silently move to the GPU; 422 rejects it."""
import inspect
import typing
from models.inference import (
LoadRequest,
SttLoadRequest,
TranscribeRequest,
ValidateModelRequest,
)
expected = typing.Optional[typing.Literal["auto", "cpu", "gpu"]]
for model, field in (
(LoadRequest, "audio_device"),
(ValidateModelRequest, "audio_device"),
(SttLoadRequest, "device"),
(TranscribeRequest, "device"),
):
assert model.model_fields[field].annotation == expected, f"{model.__name__}.{field}"
assert inspect.signature(ri.transcribe_audio_raw).parameters["device"].annotation == expected
def test_a_gpu_resident_mtmd_server_is_never_reported_as_holding_no_vram():
"""_forced_cpu may read CPU while a kept llama-server still holds the model in VRAM."""
resident_on_gpu = types.SimpleNamespace(
device = "llama.cpp",
_gpu_disabled = False,
_forced_cpu = True,
)
assert _stt_sidecar_holds_no_vram(resident_on_gpu) is False
really_cpu = types.SimpleNamespace(
device = "llama.cpp",
_gpu_disabled = True,
_forced_cpu = True,
)
assert _stt_sidecar_holds_no_vram(really_cpu) is True
def test_whisper_cpp_still_exempts_a_server_started_with_no_gpu():
"""ggml has no separate wish: _forced_cpu sits next to the spawned --no-gpu."""
assert (
_stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = True))
is True
)
assert (
_stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = False))
is False
)
def test_a_forced_cpu_native_audio_load_selects_no_gpu():
"""Selection returns every needed card, which the backend rejects when more than one;
required_gb also feeds expected_free_gb, whose settle wait raises if Images holds a card."""
import inspect
from core.inference.orchestrator import InferenceOrchestrator
src = inspect.getsource(InferenceOrchestrator.load_model)
head = src[: src.index("prepare_gpu_selection(")]
assert "audio_device_forces_cpu" in head
assert "is_native_audio_model" in head
def test_the_stale_chat_claim_is_dropped_before_the_load_not_only_after():
"""A chat claim held across a download and load makes an Images acquire cancel that load."""
import inspect
src = inspect.getsource(ri._load_model_impl)
before_load = src[: src.index("backend.load_model,")]
assert before_load.count("_release_chat_for_zero_vram_primary") >= 1
def test_the_gguf_audio_codec_follows_the_servers_own_placement():
"""The codec follows the server's placement, so a CPU server's codec never takes VRAM."""
import inspect
from core.inference.llama_cpp import LlamaCppBackend
src = inspect.getsource(LlamaCppBackend.init_audio_codec)
device_line = next(l for l in src.splitlines() if l.strip().startswith("device ="))
assert "holds_no_vram" in device_line, device_line
def test_decoding_happens_where_the_codec_actually_is():
"""Decode on the codec's recorded device, not the requested one, or SNAC and DAC fail on mismatch."""
import torch
from core.inference.audio_codecs import AudioCodecManager
mgr = AudioCodecManager()
seen = {}
class _FakeSnac:
def decode(self, codes):
seen["device"] = codes[0].device.type
return torch.zeros(1, 8)
mgr._snac_model = _FakeSnac()
mgr._codec_devices["snac"] = "cpu"
token_ids = [128257] + [128266 + i for i in range(7)]
mgr.decode("snac", "cuda", token_ids = token_ids)
assert seen["device"] == "cpu"
def test_a_codec_with_no_recorded_placement_still_honours_the_caller():
"""Nothing is recorded until a codec is actually loaded, and every pre-existing
caller passes the device it wants. An unknown codec must keep deferring to it."""
from core.inference.audio_codecs import AudioCodecManager
mgr = AudioCodecManager()
assert mgr._codec_devices == {}
def test_a_cpu_load_leaves_a_running_export_alone():
"""The export teardown frees VRAM for the incoming model; a load that masks the accelerators
needs none of it, so killing the export only costs the user the job."""
import inspect
src = inspect.getsource(ri._load_model_impl)
guard = next(
l for l in src.splitlines() if "exp_backend.current_checkpoint" in l and "if " in l
)
assert "chat_load_needs_gpu" in guard, guard
def test_the_chat_claim_outlives_the_worker_that_earned_it():
"""Release the chat claim after the old worker exits, or an Images/Video load allocates over it."""
import inspect
from core.inference.orchestrator import InferenceOrchestrator
src = inspect.getsource(ri._load_model_impl)
assert "on_prior_worker_released = _release_chat_after_teardown" in src
sig = inspect.signature(InferenceOrchestrator.load_model)
assert "on_prior_worker_released" in sig.parameters
load_src = inspect.getsource(InferenceOrchestrator.load_model)
# The fresh path's call; a context narrowed in the held worker has no teardown to wait for.
hook = load_src.rindex("on_prior_worker_released()")
assert load_src.index("did not exit and still holds GPU") < hook
assert load_src.index("if expected_free_gb is not None and not settled:") < hook
def test_whisper_cpp_publishes_the_load_before_choosing_its_placement():
"""Publish the load before choosing placement, since training reads is_loading() without the lock."""
import inspect
from core.inference.stt_ggml_sidecar import GgmlSttSidecar
src = inspect.getsource(GgmlSttSidecar.load)
assert src.index("self._loading = True") < src.index("elif _training_active():")
def test_a_model_that_fell_back_to_cpu_is_recorded_as_holding_no_gpu(monkeypatch):
"""No VRAM counted for it, and a repeat Auto load does not retry the GPU."""
import inspect
from core.inference import audio_cpp_server, orchestrator
assert 'model_info.get("audio_cpp_backend") == "cpu"' in inspect.getsource(orchestrator)
monkeypatch.setattr(audio_cpp_server, "runtime_runs_on_cpu", lambda: False)
monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
resident = _backend(audio_cpu = True, audio_type = "audiocpp_tts")
resident.models["x/y"]["audio_device_fallback"] = "cpu"
assert ri._resident_audio_holds_no_gpu(resident)
assert ri._resident_audio_placement_matches(resident, _request("auto"))
assert ri._resident_audio_placement_matches(resident, _request(None))
assert not ri._resident_audio_placement_matches(resident, _request("gpu"))