Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Audio placement preference: CPU RAM beats a working GPU; a changed preference reloads the model."""
import sys
import threading
import types
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from core.inference.audio_device import ( # noqa: E402
audio_device_default,
audio_device_forces_cpu,
normalize_audio_device,
)
@pytest.fixture(autouse = True)
def _neutral_audio_device_env(monkeypatch):
"""Clears UNSLOTH_AUDIO_DEVICE so a host-wide placement default cannot change these outcomes."""
monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
@pytest.mark.parametrize(
"value, expected",
[
("cpu", "cpu"),
("CPU", "cpu"),
(" cpu ", "cpu"),
("ram", "cpu"),
("gpu", "gpu"),
("cuda", "gpu"),
("mps", "gpu"),
("rocm", "gpu"),
("auto", "auto"),
("", "auto"),
(None, "auto"),
],
)
def test_every_accepted_spelling_maps_onto_one_of_three_values(value, expected):
assert normalize_audio_device(value) == expected
def test_an_unknown_preference_detects_rather_than_failing_the_load():
"""Detection is what the caller would have done without the option at all."""
assert normalize_audio_device("gpu-2") == "auto"
assert normalize_audio_device("nvidia rtx 4090") == "auto"
assert not audio_device_forces_cpu("gpu-2")
def test_the_environment_supplies_the_default_for_a_request_that_names_none(monkeypatch):
"""A headless or CLI Studio sets this once instead of sending it per request."""
monkeypatch.setenv("UNSLOTH_AUDIO_DEVICE", "cpu")
assert audio_device_default() == "cpu"
assert audio_device_forces_cpu(None)
assert not audio_device_forces_cpu("auto")
assert audio_device_forces_cpu("cpu")
def test_without_the_environment_variable_nothing_is_forced_to_cpu(monkeypatch):
monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
assert audio_device_default() == "auto"
assert not audio_device_forces_cpu(None)
def test_saved_voice_settings_device_is_the_stt_default_when_env_is_unset(monkeypatch, tmp_path):
"""Cold /v1/audio/transcriptions must honour Voice Settings "Load into" (#15280)."""
from core.inference.audio_device import stt_device_default, stt_device_forces_cpu
from storage import studio_db
from utils import stt_device_settings as sds
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
monkeypatch.setattr(studio_db, "_schema_ready", set())
monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
sds.forget_cached_stt_device()
assert stt_device_default() == "auto"
sds.set_stt_device("cpu")
assert stt_device_default() == "cpu"
assert stt_device_forces_cpu(None)
assert not stt_device_forces_cpu("gpu")
# The STT choice does not move TTS / native audio, which read the env-only default.
assert audio_device_default() == "auto"
assert not audio_device_forces_cpu(None)
# Env still outranks the saved choice (headless/CLI).
monkeypatch.setenv("UNSLOTH_AUDIO_DEVICE", "auto")
assert stt_device_default() == "auto"
assert not stt_device_forces_cpu(None)
def _torch_with_cuda(monkeypatch):
"""A torch whose CUDA is available, so anything but CPU is a real choice."""
torch = types.SimpleNamespace(
float16 = "float16",
float32 = "float32",
cuda = types.SimpleNamespace(is_available = lambda: True),
backends = types.SimpleNamespace(mps = types.SimpleNamespace(is_available = lambda: False)),
)
monkeypatch.setitem(sys.modules, "torch", torch)
return torch
def test_cpu_is_honoured_on_a_machine_with_a_working_gpu(monkeypatch):
"""The whole point of the option: detection would have said cuda."""
from core.inference import stt_sidecar
_torch_with_cuda(monkeypatch)
monkeypatch.setattr(stt_sidecar, "_training_active", lambda: False)
assert stt_sidecar._pick_device("cpu") == ("cpu", "float32")
def test_auto_and_gpu_both_still_detect_the_accelerator(monkeypatch):
""" "gpu" is not a separate placement: detection already prefers the card."""
from core.inference import stt_sidecar
_torch_with_cuda(monkeypatch)
monkeypatch.setattr(stt_sidecar, "_training_active", lambda: False)
assert stt_sidecar._pick_device("auto") == ("cuda", "float16")
assert stt_sidecar._pick_device("gpu") == ("cuda", "float16")
assert stt_sidecar._pick_device(None) == ("cuda", "float16")
def test_a_resident_model_on_the_other_device_is_reloaded_not_reused(monkeypatch):
"""A preference is a request about placement. Reusing the old one ignores it."""
from core.inference import stt_sidecar
monkeypatch.setattr(stt_sidecar, "_pick_device", lambda _preference = None: ("cpu", "float32"))
monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
sidecar._engine = object()
sidecar._model_id = "small"
sidecar._device = "cuda"
sidecar._device_preference = "auto"
builds: list[str] = []
def _build(snapshot_path, device, dtype, cancel_event):
builds.append(device)
return object()
monkeypatch.setattr(sidecar, "_build_model", _build)
monkeypatch.setattr(
sidecar,
"_ensure_model_downloaded",
lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(
path = None if use_resident else "/snapshots/small",
is_multilingual = True,
),
)
monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)
sidecar.load("small", device = "cpu")
assert builds == ["cpu"], "the CPU preference must have driven a fresh load"
assert sidecar._device == "cpu"
assert sidecar._device_preference == "cpu"
def test_the_same_preference_reuses_the_resident_model(monkeypatch):
"""Unchanged placement must stay a residency check, not a reload."""
from core.inference import stt_sidecar
monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
resident = object()
sidecar._engine = resident
sidecar._model_id = "small"
sidecar._device = "cpu"
sidecar._device_preference = "cpu"
def _never(*args, **kwargs):
raise AssertionError("a matching preference must not rebuild the model")
monkeypatch.setattr(sidecar, "_build_model", _never)
assert sidecar.load("small", device = "cpu") is resident
def test_a_model_loaded_before_the_option_existed_is_not_reloaded(monkeypatch):
"""No recorded preference means the engine predates the choice; reusing it is
what every caller got before, and a forced reload would cost a load for nothing."""
from core.inference import stt_sidecar
monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
resident = object()
sidecar._engine = resident
sidecar._model_id = "small"
sidecar._device = "cuda"
sidecar._device_preference = None
monkeypatch.setattr(
sidecar,
"_build_model",
lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not rebuild")),
)
assert sidecar.load("small", device = "auto") is resident
def test_the_registry_hands_the_preference_to_the_engines_sidecar(monkeypatch):
from core.inference import stt_registry
seen: dict = {}
class _Sidecar:
def load(
self,
model,
request_cancel_event = None,
device = None,
):
seen["model"] = model
seen["device"] = device
monkeypatch.setattr(stt_registry, "sidecar_for", lambda _engine: _Sidecar())
monkeypatch.setattr(stt_registry, "_model_is_downloaded", lambda _e, _m: True)
monkeypatch.setattr(stt_registry, "unload", lambda *a, **k: [])
stt_registry.load("small", "transformers", threading.Event(), device = "cpu")
assert seen == {"model": "small", "device": "cpu"}
def test_native_audio_holds_tts_weights_in_cpu_ram_when_asked(monkeypatch):
from core.inference import native_audio
_torch_with_cuda(monkeypatch)
assert native_audio.NativeAudioBackend(device_preference = "cpu").device == "cpu"
assert native_audio.NativeAudioBackend(device_preference = "auto").device == "cuda"
assert native_audio.NativeAudioBackend().device == "cuda"
def test_minimax_music_explains_that_cpu_was_chosen_rather_than_missing(monkeypatch):
"""The generic refusal reads like "your hardware cannot do this" and sends a
user with a perfectly good card looking for one."""
from core.inference import native_audio
_torch_with_cuda(monkeypatch)
backend = native_audio.NativeAudioBackend(device_preference = "cpu")
config = types.SimpleNamespace(
identifier = "MiniMaxAI/MiniMax-Music3",
audio_type = "minimax_music3",
path = None,
)
with pytest.raises(RuntimeError, match = "cannot be loaded into CPU RAM"):
backend.load_model(config)
def test_a_caller_that_sends_no_device_leaves_the_placement_alone(monkeypatch):
"""``/v1/audio/transcriptions`` sends none. Treating that as "auto" pulled a
CPU model back onto the GPU, and the next dictation pulled it off again:
two full reloads per alternation, and VRAM the user asked us not to take."""
from core.inference import stt_sidecar
monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
monkeypatch.setattr(
stt_sidecar,
"_pick_device",
lambda preference = None: ("cpu", "float32") if preference == "cpu" else ("cuda", "float16"),
)
sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
builds: list[str] = []
monkeypatch.setattr(
sidecar, "_build_model", lambda p, device, dtype, c: builds.append(device) or object()
)
monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)
monkeypatch.setattr(
sidecar,
"_ensure_model_downloaded",
lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(
path = None if (sidecar._engine is not None and use_resident) else "/snapshots/small",
is_multilingual = True,
),
)
sidecar.load("small", device = "cpu")
sidecar.load("small", device = None)
sidecar.load("small", device = "cpu")
assert builds == ["cpu"], "only the first load should have built anything"
assert sidecar._device == "cpu"
def test_an_explicit_change_still_reloads_after_a_no_opinion_call(monkeypatch):
"""No opinion must not also freeze the placement: the setting still moves it."""
from core.inference import stt_sidecar
monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
monkeypatch.setattr(
stt_sidecar,
"_pick_device",
lambda preference = None: ("cpu", "float32") if preference == "cpu" else ("cuda", "float16"),
)
sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
builds: list[str] = []
monkeypatch.setattr(
sidecar, "_build_model", lambda p, device, dtype, c: builds.append(device) or object()
)
monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)
monkeypatch.setattr(
sidecar,
"_ensure_model_downloaded",
lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(
path = None if (sidecar._engine is not None and use_resident) else "/snapshots/small",
is_multilingual = True,
),
)
sidecar.load("small", device = "cpu")
sidecar.load("small", device = None)
sidecar.load("small", device = "auto")
assert builds == ["cpu", "cuda"]
def test_the_mtmd_reuse_branch_still_records_the_choice(monkeypatch):
"""Training makes an explicit "cpu" indistinguishable from the running
server, so the early return has to keep the preference or the next
device-less load sends the model back to the GPU once training ends."""
from core.inference import stt_mtmd_sidecar
sidecar = stt_mtmd_sidecar.MtmdSttSidecar.__new__(stt_mtmd_sidecar.MtmdSttSidecar)
sidecar._lock = threading.RLock()
sidecar._forced_cpu = False
sidecar._gpu_disabled = True
sidecar._model_id = "m"
sidecar._binary_path_revision = 1
sidecar._active_requests = 0
sidecar._context_preference = 0
sidecar._context_length = 4096
monkeypatch.setattr(sidecar, "_process_alive", lambda: True)
monkeypatch.setattr(sidecar, "_schedule_idle_unload_locked", lambda: None)
monkeypatch.setattr(stt_mtmd_sidecar, "_training_active", lambda: True)
sidecar._load_locked("m", "whisper-server", path_revision = 1, device = "cpu")
assert sidecar._forced_cpu is True
def test_stt_unload_can_skip_a_sidecar_that_is_mid_transcription():
"""Early release, not reclaim: killing a mid-transcription sidecar loses the recording."""
import inspect
from core.inference import stt_registry
from core.inference.orchestrator import InferenceOrchestrator
from routes import inference as ri
assert "wait" in inspect.signature(ri.stt_unload).parameters
assert inspect.signature(ri.stt_unload).parameters["wait"].default is True
assert "wait" in inspect.signature(InferenceOrchestrator.unload_stt_model).parameters
assert "wait" in inspect.signature(stt_registry.unload).parameters