Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Let audio models load into CPU RAM instead of the GPU (#10054)
Michael Han03c849f586f6
diff --git a/studio/backend/core/inference/audio_codecs.py b/studio/backend/core/inference/audio_codecs.pyindex 0a4aa78d061..89ba21284f8 100644--- a/studio/backend/core/inference/audio_codecs.py+++ b/studio/backend/core/inference/audio_codecs.py@@ -56,6 +56,9 @@ class AudioCodecManager:self._bicodec_code_dir = Noneself._dac_audio_codec = Noneself._outetts_code_dir = None+ # The loaders reuse a resident codec, so a later request gets the first+ # placement rather than the one it asked for.+ self._codec_devices: dict = {}def load_codec(self,@@ -89,6 +92,7 @@ class AudioCodecManager:.to(device).eval())+ self._codec_devices["snac"] = devicelogger.info("Loaded SNAC codec (24kHz)")def _load_bicodec(@@ -109,6 +113,7 @@ class AudioCodecManager:tokenizer_path = model_repo_path or spark_code_dirself._bicodec_repo_path = tokenizer_pathself._bicodec_tokenizer = BiCodecTokenizer(tokenizer_path, device)+ self._codec_devices["bicodec"] = devicelogger.info(f"Loaded BiCodec tokenizer from {tokenizer_path}")def _load_dac(self, device: str) -> None:@@ -133,6 +138,7 @@ class AudioCodecManager:)processor = AudioProcessor(config = dummy_config)self._dac_audio_codec = processor.audio_codec+ self._codec_devices["dac"] = devicelogger.info("Loaded DAC audio codec")# ── Decoders ─────────────────────────────────────────────────@@ -258,7 +264,14 @@ class AudioCodecManager:token_ids: Optional[list] = None,text: Optional[str] = None,) -> Tuple[bytes, int]:- """Unified decode — dispatches to the right codec decoder."""+ """Unified decode — dispatches to the right codec decoder.++ ``device`` is what the caller would like. Where the codec is actually+ resident wins: input tensors built on another device fail outright for SNAC+ and DAC, and BiCodec would move a CPU-resident codec onto the card, taking+ the VRAM a CPU RAM load promised not to take.+ """+ device = self._codec_devices.get(audio_type, device)if audio_type == "snac":if not token_ids:raise ValueError("SNAC decoding requires token_ids")@@ -293,4 +306,5 @@ class AudioCodecManager:if self._outetts_code_dir is not None:deactivate_pinned_package("outetts", self._outetts_code_dir)self._outetts_code_dir = None+ self._codec_devices.clear()logger.info("Unloaded all audio codecs")diff --git a/studio/backend/core/inference/audio_device.py b/studio/backend/core/inference/audio_device.pynew file mode 100644index 00000000000..86c9f7226d9--- /dev/null+++ b/studio/backend/core/inference/audio_device.py@@ -0,0 +1,107 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Where an audio model's weights go: the accelerator, or plain CPU RAM.++Audio loads take an accelerator whenever one exists. That is right until the GPU+is the scarce resource (a resident chat model, a training run, a card too small+for the checkpoint), and Whisper and the smaller TTS models run fine on CPU.++Values match ``RAG_EMBED_DEVICE`` (``core/rag/config.py``):++``auto`` detect as before.+``cpu`` force CPU RAM, even with a working accelerator.+``gpu`` prefer the accelerator. The existing CPU retry after a failed load+ still applies, so this is a preference and not a guarantee.++``UNSLOTH_AUDIO_DEVICE`` supplies the default for a request that names none.+"""++from __future__ import annotations++import os+from typing import Optional++__all__ = [+ "AUDIO_DEVICE_CHOICES",+ "audio_device_default",+ "audio_device_forces_cpu",+ "mask_accelerators_for_cpu_audio",+ "normalize_audio_device",+]++AUDIO_DEVICE_CHOICES = ("auto", "cpu", "gpu")++# Spellings other Studio surfaces already use; names arrive from a status echo.+_CPU_ALIASES = frozenset({"cpu", "ram", "cpu_ram", "system", "system_ram"})+_GPU_ALIASES = frozenset(+ {"gpu", "cuda", "rocm", "hip", "xpu", "mps", "metal", "accelerator", "accel"}+)+++def normalize_audio_device(value: Optional[str]) -> str:+ """Map any accepted spelling onto ``auto``/``cpu``/``gpu``.++ Anything unrecognised becomes ``auto``: an unknown preference must not fail+ a load, and detection is what the caller would have done regardless.++ That fallback is for values the user did not type: ``UNSLOTH_AUDIO_DEVICE``,+ and device names read back off a status payload. The HTTP models pin the+ three canonical values instead, so a misspelled ``cpu`` is a 422 rather than+ a silent placement back on the GPU.+ """+ text = str(value or "").strip().lower()+ if not text:+ return "auto"+ if text in _CPU_ALIASES:+ return "cpu"+ if text in _GPU_ALIASES:+ return "gpu"+ if text == "auto":+ return "auto"+ return "auto"+++def audio_device_default() -> str:+ """The preference for a request that carries none (``UNSLOTH_AUDIO_DEVICE``).++ Scope: the native audio backend and the three STT sidecars. It does NOT reach a+ GGUF TTS model. llama.cpp placement is decided from ``gpu_memory_mode`` and+ ``gpu_layers`` at request time, but nothing knows a GGUF is audio until+ llama-server reports its ``_audio_type`` after the load, so there is no point+ early enough to translate the default into zero offload. The Audio page does it+ from the catalog it already has; a headless caller must send the GGUF placement+ fields itself.+ """+ return normalize_audio_device(os.environ.get("UNSLOTH_AUDIO_DEVICE"))+++def audio_device_forces_cpu(value: Optional[str]) -> bool:+ """True when this preference means "load into CPU RAM".++ ``None`` falls back to the environment default, so an older caller still+ honours a server-wide setting.+ """+ if value is None:+ return audio_device_default() == "cpu"+ return normalize_audio_device(value) == "cpu"+++def mask_accelerators_for_cpu_audio(env: dict) -> None:+ """Hide CUDA/ROCm from a worker whose weights stay in CPU RAM.++ Placing the weights on CPU is not enough on its own: the worker runs+ ``detect_hardware()`` first, and that calls ``torch.cuda.get_device_properties``,+ which creates a context worth a few hundred MB. Masking first is what makes+ "this load holds no VRAM" true.++ Same values as the CPU embed server (``core/rag/embed_llama_server.py``):+ blank for CUDA, ``-1`` for HIP because it reads the CUDA variable only when+ its own is unset. An inherited ROCR mask is left alone, since clearing it+ exposes more agents rather than fewer. XPU is not masked: its probe is+ ``torch.xpu.is_available()`` and takes no context.++ Call before importing torch. Mutates ``env`` in place.+ """+ env["CUDA_VISIBLE_DEVICES"] = ""+ env["HIP_VISIBLE_DEVICES"] = "-1"diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex e1e7f330a70..fbea7138785 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -32520,7 +32520,10 @@ class LlamaCppBackend:if LlamaCppBackend._codec_mgr is None:LlamaCppBackend._codec_mgr = AudioCodecManager()- device = "cuda" if torch.cuda.is_available() else "cpu"+ # A second allocation: on CUDA for a zero-offload server it would hold VRAM the+ # load is classified as not holding, which is what lets the route skip+ # arbitration and survive training.+ device = "cuda" if torch.cuda.is_available() and not self.holds_no_vram else "cpu"model_repo_path = None# BiCodec needs a repo with BiCodec/ weights -- download canonical SparkTTSdiff --git a/studio/backend/core/inference/native_audio.py b/studio/backend/core/inference/native_audio.pyindex 602e21c23f3..2c7980459c3 100644--- a/studio/backend/core/inference/native_audio.py+++ b/studio/backend/core/inference/native_audio.py@@ -671,10 +671,15 @@ def _as_wav_bytes(audio, sample_rate: int) -> bytes:class NativeAudioBackend:"""One-model backend for the five curated native audio architectures."""- def __init__(self) -> None:+ def __init__(self, device_preference: Optional[str] = None) -> None:import torch- if torch.cuda.is_available():+ from core.inference.audio_device import audio_device_forces_cpu++ self.device_preference = device_preference+ if audio_device_forces_cpu(device_preference):+ self.device = "cpu"+ elif torch.cuda.is_available():self.device = "cuda"elif hasattr(torch, "xpu") and torch.xpu.is_available():self.device = "xpu"@@ -798,6 +803,14 @@ class NativeAudioBackend:"multi-GPU sharding is not supported yet.")if audio_type == "minimax_music3" and self.device != "cuda":+ # Chosen, not missing: the generic message sends users hunting for a card.+ from core.inference.audio_device import audio_device_forces_cpu+ if audio_device_forces_cpu(self.device_preference):+ raise RuntimeError(+ "MiniMax Music 3 cannot be loaded into CPU RAM: its official local "+ "runtime requires an NVIDIA CUDA GPU. Set the audio device back to "+ "Auto (or GPU) to load this model."+ )raise RuntimeError("MiniMax Music 3 currently requires an NVIDIA CUDA GPU in its official ""local runtime. It is not available on CPU, Apple Silicon, AMD, or Intel XPU."diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex a2703466d50..e941a3ba026 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -26,7 +26,9 @@ import timeimport uuidfrom io import BytesIOfrom pathlib import Path-from typing import Any, Generator, Optional, Sequence, Tuple, Union+from typing import Any, Callable, Generator, Optional, Sequence, Tuple, Union+from core.inference.audio_device import audio_device_forces_cpu+from core.inference.native_audio import NATIVE_AUDIO_TYPES, is_native_audio_modelfrom core.inference.audio_errors import (AUDIO_UNSUPPORTED_CODE,AudioBackendUnsupportedError,@@ -1495,6 +1497,8 @@ class InferenceOrchestrator:chat_template_override: Optional[str] = None,load_cancel_event: Optional[threading.Event] = None,post_handoff_expected_free_gb: Optional[dict[int, float]] = None,+ audio_device: Optional[str] = None,+ on_prior_worker_released: Optional[Callable[[], None]] = None,) -> bool:"""Load a model for inference.@@ -1532,13 +1536,21 @@ class InferenceOrchestrator:else None,"mlx_kv_bits": mlx_kv_bits,"chat_template_override": chat_template_override,+ # Read in the worker, which hides the accelerators before detection.+ "audio_device": audio_device,}- resolved_gpu_ids, gpu_selection = prepare_gpu_selection(- gpu_ids,- model_name = model_name,- hf_token = hf_token,- load_in_4bit = load_in_4bit,- )+ if audio_device_forces_cpu(audio_device) and is_native_audio_model(model_name):+ # Choosing a card for a load that takes none harms it twice: several+ # GPUs are rejected as unsupported sharding, and required_gb becomes+ # expected_free_gb, so the settle wait raises on a busy card.+ resolved_gpu_ids, gpu_selection = None, {"selection_mode": "cpu_audio"}+ else:+ resolved_gpu_ids, gpu_selection = prepare_gpu_selection(+ gpu_ids,+ model_name = model_name,+ hf_token = hf_token,+ load_in_4bit = load_in_4bit,+ )sub_config["resolved_gpu_ids"] = resolved_gpu_idssub_config["gpu_selection"] = gpu_selection# Parent-detected backend for the worker's apply_gpu_ids().@@ -1604,6 +1616,10 @@ class InferenceOrchestrator:"not starting the replacement model. Retry shortly.")+ # Previous worker gone, VRAM back: the last moment before a long download.+ if on_prior_worker_released is not None:+ on_prior_worker_released()+disable_xet = sub_config.get("disable_xet", False) or (os.environ.get("HF_HUB_DISABLE_XET") == "1")@@ -1708,6 +1724,12 @@ class InferenceOrchestrator:self.models[self.active_model_name] = _mirrored_model_entry(model_info, model_name)+ # Lets the already-loaded shortcut tell a CPU request from the GPU+ # model it would otherwise report as satisfied. Native audio only:+ # marking anything else tells training a GPU model holds no VRAM.+ self.models[self.active_model_name]["audio_cpu"] = model_info.get(+ "audio_type"+ ) in NATIVE_AUDIO_TYPES and audio_device_forces_cpu(audio_device)self.models[self.active_model_name].update(_mlx_runtime_mirror_fields(model_info))@@ -1796,22 +1818,29 @@ class InferenceOrchestrator:model: Optional[str],engine: str,request_cancel_event: Optional[threading.Event] = None,+ device: Optional[str] = None,) -> None:- """Make a dictation model resident on its sidecar."""+ """Make a dictation model resident on its sidecar.++ ``device`` is the user's audio device preference (``auto``/``cpu``/``gpu``).+ """from core.inference import stt_registry- stt_registry.load(model, engine, request_cancel_event)+ stt_registry.load(model, engine, request_cancel_event, device = device)def unload_stt_model(self,engines: Optional[Sequence[str]] = None,expected_model: Optional[str] = None,+ wait: bool = True,) -> list:"""Release dictation models (all engines by default); returns refusals.``expected_model`` scopes the release to a sidecar still holding that model.+ ``wait=False`` leaves a sidecar that is mid-request alone, for a caller+ freeing memory opportunistically rather than to reclaim it now."""from core.inference import stt_registry- return stt_registry.unload(engines, expected_model = expected_model)+ return stt_registry.unload(engines, wait = wait, expected_model = expected_model)def resident_stt_model(self) -> dict:"""What dictation holds, alongside active_model_name for chat."""diff --git a/studio/backend/core/inference/stt_ggml_sidecar.py b/studio/backend/core/inference/stt_ggml_sidecar.pyindex a392f026ae8..f2a63212195 100644--- a/studio/backend/core/inference/stt_ggml_sidecar.py+++ b/studio/backend/core/inference/stt_ggml_sidecar.py@@ -716,6 +716,8 @@ class GgmlSttSidecar:self._process: Optional[subprocess.Popen] = Noneself._port: Optional[int] = Noneself._model_id: Optional[str] = None+ # --no-gpu at the user's request, tracked because it varies per request.+ self._forced_cpu = Falseself._idle_timer: Optional[threading.Timer] = Noneself._idle_generation = 0self._keep_alive_seconds = keep_alive_seconds@@ -791,6 +793,7 @@ class GgmlSttSidecar:self._process = Noneself._port = Noneself._model_id = None+ self._forced_cpu = Falseif process is not None and process.poll() is None:process.terminate()try:@@ -929,8 +932,17 @@ class GgmlSttSidecar:self,model: Optional[str] = None,request_cancel_event: Optional[threading.Event] = None,+ device: Optional[str] = None,) -> None:- """Start (or switch) whisper-server for the requested curated model."""+ """Start (or switch) whisper-server for the requested curated model.++ ``device`` is the user's audio device preference; ``cpu`` starts the server+ with ``--no-gpu``, the same flag training and a CPU-only install already use.+ ``None`` is no opinion: it keeps the running server's placement, so a+ caller that never sends one cannot restart it onto the other device.+ """+ from core.inference.audio_device import audio_device_forces_cpu+if request_cancel_event is not None and request_cancel_event.is_set():raise SttTranscriptionCancelledError("Transcription cancelled.")self._raise_if_update_in_progress()@@ -940,26 +952,22 @@ class GgmlSttSidecar:raise SttTranscriptionCancelledError("Transcription cancelled.")self._raise_if_update_in_progress()binary = ensure_engine_available()- if self._process_alive() and self._model_id == model_id:+ force_cpu = (+ self._forced_cpu+ if device is None and self._process_alive()+ else audio_device_forces_cpu(device)+ )+ if (+ self._process_alive()+ and self._model_id == model_id+ and self._forced_cpu == force_cpu+ ):self._schedule_idle_unload_locked()returnmodel_path = self._ensure_model_downloaded(model_id)reservation, port = self._reserve_free_port()command = [binary, "-m", model_path, "--host", "127.0.0.1", "--port", str(port)]marker = _whisper_install_marker(binary)- if _training_active():- # Keep whisper.cpp off the accelerator during training (like the Transformers sidecar's CPU choice) so a- # mid-training dictation cannot reclaim the VRAM training just freed.- command.append("--no-gpu")- elif marker is not None and marker.get("backend") == "cpu":- # A deliberate CPU install must stay CPU: the slim wiring links every llama ggml backend (including- # CUDA/ROCm), so without this flag a cpu-selected install would still grab the GPU.- command.append("--no-gpu")- logger.info(- "Starting whisper-server for STT model %s on 127.0.0.1:%s",- model_id,- port,- )cancel_event = (request_cancel_event if request_cancel_event is not None else threading.Event())@@ -970,6 +978,25 @@ class GgmlSttSidecar:try:if cancel_event.is_set():raise SttLoadCancelledError("GGUF STT model loading was cancelled.")+ # Decided under the loading flag, never before it: the training hook+ # reads is_loading() without this lock, so an earlier decision is+ # invisible and it preserves the outgoing model as holding no VRAM.+ if force_cpu:+ # Tracked, so the reuse check above restarts on a change.+ command.append("--no-gpu")+ elif _training_active():+ # Off the accelerator during training, so a mid-training dictation+ # cannot reclaim the VRAM training just freed.+ command.append("--no-gpu")+ elif marker is not None and marker.get("backend") == "cpu":+ # The slim wiring links every ggml backend, so without this flag a+ # cpu-selected install would still grab the GPU.+ command.append("--no-gpu")+ logger.info(+ "Starting whisper-server for STT model %s on 127.0.0.1:%s",+ model_id,+ port,+ )self._release_locked()# release the reservation as late as possible: whisper-server binds the port moments after this closereservation.close()@@ -999,6 +1026,7 @@ class GgmlSttSidecar:self._process = processself._port = portself._model_id = model_id+ self._forced_cpu = force_cpuself._schedule_idle_unload_locked()finally:reservation.close() # no-op when already released before spawndiff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.pyindex 059c2211b36..8c6c8396a72 100644--- a/studio/backend/core/inference/stt_mtmd_sidecar.py+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py@@ -557,6 +557,8 @@ class MtmdSttSidecar:# Whether the resident server was launched with the GPU pinned off for training. Kept so a dictation after the# run does not stay on CPU.self._gpu_disabled = False+ # Apart from _gpu_disabled, so training restarts while the preference survives.+ self._forced_cpu = Falseself._load_cancel_event: Optional[threading.Event] = Noneself._load_owner_cancel_event: Optional[threading.Event] = Noneself._update_in_progress = False@@ -794,7 +796,12 @@ class MtmdSttSidecar:self,model: Optional[str] = None,request_cancel_event: Optional[threading.Event] = None,+ device: Optional[str] = None,) -> None:+ """``device`` is the user's audio device preference; ``cpu`` starts+ llama-server at ``-ngl 0``, the same offload training already forces."""+ from core.inference.audio_device import audio_device_forces_cpu+if request_cancel_event is not None and request_cancel_event.is_set():raise SttTranscriptionCancelledError("Transcription cancelled.")self._raise_if_update_in_progress()@@ -813,6 +820,7 @@ class MtmdSttSidecar:binary,request_cancel_event,path_revision = path_revision,+ device = device,)def _load_locked(@@ -822,12 +830,23 @@ class MtmdSttSidecar:request_cancel_event: Optional[threading.Event] = None,*,path_revision: Optional[int] = None,+ device: Optional[str] = None,) -> None:+ from core.inference.audio_device import audio_device_forces_cpu+if path_revision is None:from utils.llama_cpp_path_settings import custom_llama_cpp_path_revisionpath_revision = custom_llama_cpp_path_revision()with self._lock:- training = _training_active()+ # None is no opinion: a caller sending none cannot move the server.+ if device is None:+ forced_cpu = (+ self._forced_cpu if self._process_alive() else audio_device_forces_cpu(None)+ )+ else:+ forced_cpu = audio_device_forces_cpu(device)+ # Both mean "no offload", which _gpu_disabled already tracks.+ training = _training_active() or forced_cpuif (self._process_alive()and self._model_id == model_id@@ -838,6 +857,9 @@ class MtmdSttSidecar:# killing a running transcription for, so an in-flight request keeps the server it has and the next idle# load picks the GPU back up.if self._gpu_disabled == training or self._active_requests:+ # Recorded though nothing restarts: dropping it sends the next+ # device-less load back to the GPU once training ends.+ self._forced_cpu = forced_cpuself._schedule_idle_unload_locked()return# Announced before the slow probe and reap: is_loading() is read lock-free, so a training start would@@ -873,7 +895,7 @@ class MtmdSttSidecar:# Re-read last: _release_locked() reaps the old server, which can take seconds, and training admission that# already passed its own check cannot come back to cancel this load. Publishing _loading first covers the# other order, so between them every training start either cancels this load or is seen by it.- training = _training_active()+ training = _training_active() or forced_cputry:sock, port = self._reserve_free_port()cmd = [@@ -933,6 +955,7 @@ class MtmdSttSidecar:self._model_id = model_idself._binary_path_revision = path_revisionself._gpu_disabled = training+ self._forced_cpu = forced_cpuself._generation += 1self._schedule_idle_unload_locked()finally:diff --git a/studio/backend/core/inference/stt_registry.py b/studio/backend/core/inference/stt_registry.pyindex fee8f1c9af1..39618cb005c 100644--- a/studio/backend/core/inference/stt_registry.py+++ b/studio/backend/core/inference/stt_registry.py@@ -44,9 +44,14 @@ def load(model: Optional[str],engine: str,request_cancel_event: Optional[threading.Event] = None,+ device: Optional[str] = None,) -> None:"""Make ``model`` resident on ``engine``, then release every idle other engine.+ ``device`` is the user's audio device preference (``auto``/``cpu``/``gpu``);+ every engine honours it, on CPU by holding the weights in system RAM instead+ of the accelerator.+Dictation is one user-visible choice, so engines are alternatives, not slots:holding two at once doubles VRAM for the whole keep-alive window. An engine servinga request keeps its model and releases it on its own idle timer. Raises what the@@ -62,9 +67,13 @@ def load(# answer is not certain, keep the old order and accept the peak.if _model_is_downloaded(engine, model):unload(others, wait = False)- sidecar_for(engine).load(model, request_cancel_event = request_cancel_event)+ sidecar_for(engine).load(+ model, request_cancel_event = request_cancel_event, device = device+ )else:- sidecar_for(engine).load(model, request_cancel_event = request_cancel_event)+ sidecar_for(engine).load(+ model, request_cancel_event = request_cancel_event, device = device+ )unload(others, wait = False)diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.pyindex 76c9ea1a100..53930dfff4c 100644--- a/studio/backend/core/inference/stt_sidecar.py+++ b/studio/backend/core/inference/stt_sidecar.py@@ -967,15 +967,24 @@ def _reported_device(device: Optional[str]) -> Optional[str]:return device-def _pick_device():+def _pick_device(preference: Optional[str] = None):"""Return (device, torch_dtype) for the Whisper model.CUDA uses float16. MPS and CPU use float32: Whisper's decoder is unstable infloat16 on MPS and degenerates into repeated tokens.++ ``preference`` is the user's choice from ``core.inference.audio_device``.+ ``cpu`` short-circuits detection entirely -- that is the whole point of the+ option, so it must hold even on a machine with a working accelerator. ``gpu``+ and ``auto`` both detect: preferring the GPU is what detection already does,+ and the caller's CPU retry still covers a card that cannot take the model."""+ from core.inference.audio_device import audio_device_forces_cputry:import torch+ if audio_device_forces_cpu(preference):+ return "cpu", torch.float32# new loads use CPU during training; a resident GPU model may stay put when the admission check confirms# headroomtraining_active = _training_active()@@ -1144,6 +1153,7 @@ class WhisperSttSidecar:self._engine = Noneself._model_id: Optional[str] = Noneself._device: Optional[str] = None+ self._device_preference: Optional[str] = Noneself._lock = threading.RLock()self._load_state_lock = threading.Lock()self._loading = False@@ -1285,6 +1295,7 @@ class WhisperSttSidecar:self._engine = Noneself._model_id = Noneself._device = None+ self._device_preference = Noneself._survivor = Falseelse:self._survivor = True@@ -1393,12 +1404,22 @@ class WhisperSttSidecar:raisereturn worker- def _ensure_model_downloaded(self, model_id: str) -> _CachedSttSnapshot:+ def _ensure_model_downloaded(+ self,+ model_id: str,+ use_resident: bool = True,+ ) -> _CachedSttSnapshot:"""Validate the local snapshot before decode or model replacement.Returns the checkpoint's multilingual flag when local metadata providesit. Curated defaults are known multilingual.+ ``use_resident = False`` skips the resident-model shortcut and resolves the+ path on disk. The shortcut answers from the loaded model and returns no+ path, which is right when that model is about to be reused and wrong when+ the caller is replacing it with one on another device: the same model id is+ resident, but the load still needs somewhere to read the weights from.+A survivor is held for its memory alone, so it does not answer for themodel the way a resident one does: the snapshot is looked up on disk, orthe load it precedes would be turned away as a checkpoint that is not@@ -1406,7 +1427,7 @@ class WhisperSttSidecar:"""model_id = resolve_model_id(model_id)with self._lock:- reusable = self._engine is not None and self._model_id == model_id+ reusable = use_resident and self._engine is not None and self._model_id == model_idif reusable and not self._is_survivor_locked():resident_model = (self._engine[0] if isinstance(self._engine, (tuple, list)) else self._engine@@ -1443,20 +1464,39 @@ class WhisperSttSidecar:self,model: Optional[str] = None,request_cancel_event: Optional[threading.Event] = None,+ device: Optional[str] = None,):"""Load (or switch to) a model, reusing it if already resident.+ ``device`` is the user's device preference (``auto``/``cpu``/``gpu``, see+ ``core.inference.audio_device``). An explicit one that differs from the+ resident model's reloads it, or the setting would be silently ignored.+ ``None`` means no opinion: it takes the server default for a fresh load+ and reuses whatever is resident, so a caller that never sends one cannot+ move a model another surface placed.+Returns a ``(model, processor)`` pair."""+ from core.inference.audio_device import audio_device_default, normalize_audio_device+if request_cancel_event is not None and request_cancel_event.is_set():raise SttTranscriptionCancelledError("Transcription cancelled.")model_id = resolve_model_id(model)+ # None is "no opinion", not "auto": a caller sending none must not drag a model+ # off the device another surface asked for.+ requested = None if device is None else normalize_audio_device(device)+ preference = audio_device_default() if requested is None else requestedwith self._lock:if request_cancel_event is not None and request_cancel_event.is_set():raise SttTranscriptionCancelledError("Transcription cancelled.")ensure_stt_available()self._release_dead_engine_locked()- reusable = self._engine is not None and self._model_id == model_id+ placement_matches = (+ requested is None+ or self._device_preference is None+ or self._device_preference == requested+ )+ reusable = self._engine is not None and self._model_id == model_id and placement_matchesif reusable and not self._is_survivor_locked():self._schedule_idle_unload_locked()return self._engine@@ -1469,7 +1509,8 @@ class WhisperSttSidecar:self._start_survivor = Nonetry:self._raise_if_load_cancelled(cancel_event)- cached = self._ensure_model_downloaded(model_id)+ # The resident shortcut returns no path, and this load replaces it.+ cached = self._ensure_model_downloaded(model_id, use_resident = placement_matches)snapshot_path = cached.pathif snapshot_path is None:raise SttModelNotDownloadedError(@@ -1477,7 +1518,7 @@ class WhisperSttSidecar:"Download it in Settings, then Voice, before loading it.")self._raise_if_load_cancelled(cancel_event)- device, dtype = _pick_device()+ device, dtype = _pick_device(preference)if not self._release_engine_locked():# starting a second child over one that never exited doubles the memory this release was meant to# give back@@ -1545,6 +1586,7 @@ class WhisperSttSidecar:self._engine = candidateself._model_id = model_idself._device = device+ self._device_preference = preferenceself._survivor = Falseself._load_cancel_event = Noneself._load_owner_cancel_event = Nonediff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 08a4a7a8080..f4050f07351 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1114,6 +1114,17 @@ def run_inference_process(_native_audio_worker = is_native_audio_model(model_name)+ # Before detect_hardware(), whose probe would leave a CUDA context here; the route+ # skips the arbiter on the basis that this load reserves none.+ if _native_audio_worker:+ from core.inference.audio_device import (+ audio_device_forces_cpu,+ mask_accelerators_for_cpu_audio,+ )+ if audio_device_forces_cpu(config.get("audio_device")):+ mask_accelerators_for_cpu_audio(os.environ)+ logger.info("Audio model '%s' pinned to CPU RAM; accelerators hidden", model_name)+# ── 0. MLX fast-path — skip torch/transformers ──_ensure_backend_on_path()@@ -1424,7 +1435,12 @@ def run_inference_process(# ── 3. Create inference backend and load initial model ──try:- backend = InferenceBackend()+ # Native audio picks its device in __init__, so the preference goes there.+ backend = (+ InferenceBackend(device_preference = config.get("audio_device"))+ if _native_audio_worker+ else InferenceBackend()+ )_send_response(resp_queue,diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex e114f9a6a2d..34ccb599d7a 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -257,6 +257,19 @@ class LoadRequest(BaseModel):"non-GGUF models."),)+ audio_device: Optional[Literal["auto", "cpu", "gpu"]] = Field(+ None,+ description = (+ "Native audio (TTS / music) models only: where to hold the weights. "+ "'cpu' keeps them in CPU RAM rather than the GPU -- slower generation, "+ "but it leaves VRAM for other models and runs checkpoints too large for "+ "the card. 'gpu' prefers the accelerator; 'auto' (default) detects. "+ "Ignored for every non-audio model, and for GGUF audio: llama.cpp "+ "placement comes from gpu_memory_mode and gpu_layers, which are decided "+ "before anything can know a GGUF is audio. Send gpu_memory_mode='manual' "+ "with gpu_layers=0 and speculative_type='off' to hold a GGUF TTS in RAM."+ ),+ )gpu_memory_mode: Literal["auto", "manual"] = Field("auto",description = (@@ -397,6 +410,14 @@ class TranscribeRequest(BaseModel):None,description = "STT engine: 'transformers' (default) or 'gguf' (whisper.cpp)",)+ device: Optional[Literal["auto", "cpu", "gpu"]] = Field(+ None,+ description = (+ "Where to hold the model: 'cpu' keeps it in CPU RAM, 'gpu' prefers the "+ "accelerator, 'auto' (default) detects. Applies when this request has to "+ "load the model; a resident model on another device is reloaded to honour it."+ ),+ )class SttLoadRequest(BaseModel):@@ -407,6 +428,13 @@ class SttLoadRequest(BaseModel):None,description = "STT engine: 'transformers' (default) or 'gguf' (whisper.cpp)",)+ device: Optional[Literal["auto", "cpu", "gpu"]] = Field(+ None,+ description = (+ "Where to hold the model: 'cpu' keeps it in CPU RAM rather than the GPU, "+ "'gpu' prefers the accelerator, 'auto' (default) detects."+ ),+ )class ValidateModelRequest(BaseModel):@@ -440,6 +468,14 @@ class ValidateModelRequest(BaseModel):# refuse a load that then fits.disable_vision: bool = Field(False)gpu_ids: Optional[List[int]] = Field(None)+ # Sized with too: preflighting a CPU load would refuse one that takes no VRAM.+ audio_device: Optional[Literal["auto", "cpu", "gpu"]] = Field(+ None,+ description = (+ "Native audio placement intended for the follow-up load. 'cpu' skips "+ "the GPU-memory preflight, which that load would not use."+ ),+ )gpu_memory_mode: Literal["auto", "manual"] = Field("auto",description = (diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex d535131d3e6..553439cf7ac 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -28,7 +28,17 @@ from fastapi import (from fastapi.encoders import jsonable_encoderfrom fastapi.responses import StreamingResponse, JSONResponse, PlainTextResponse, Responsefrom starlette.requests import ClientDisconnect-from typing import Any, Callable, Collection, List, NamedTuple, Optional, TYPE_CHECKING, Union+from typing import (+ Any,+ Callable,+ Collection,+ List,+ Literal,+ NamedTuple,+ Optional,+ TYPE_CHECKING,+ Union,+)import functoolsimport jsonimport httpx@@ -11953,6 +11963,55 @@ async def _prepare_load_placement(return _LoadPlacement(requested, resolved, is_vulkan, diffusion_kind)+def _native_audio_cpu_load(config, request) -> bool:+ """True when this load is a native audio model the user placed in CPU RAM.++ Gated on the audio type on purpose: audio_device is documented as ignored+ for everything else, so a chat model cannot send it to skip the VRAM guards+ that read this.+ """+ from core.inference.audio_device import audio_device_forces_cpu+ from core.inference.native_audio import NATIVE_AUDIO_TYPES++ return getattr(config, "audio_type", None) in NATIVE_AUDIO_TYPES and audio_device_forces_cpu(+ getattr(request, "audio_device", None)+ )+++def _resident_audio_placement_matches(backend, request) -> bool:+ """False when the resident audio model is not where this request wants it.++ Reads the resident entry, not the requested config: this runs ahead of config+ resolution, and the question is about the model already loaded. Only native+ audio records a placement, so everything else keeps the shortcut. An entry+ from before this existed has no key and is read as GPU, which is what those+ loads did.+ """+ from core.inference.audio_device import audio_device_forces_cpu+ from core.inference.native_audio import NATIVE_AUDIO_TYPES++ resident = backend.models.get(backend.active_model_name, {})+ if resident.get("audio_type") not in NATIVE_AUDIO_TYPES:+ return True+ return bool(resident.get("audio_cpu", False)) == audio_device_forces_cpu(+ getattr(request, "audio_device", None)+ )+++def _resident_audio_holds_no_gpu(backend) -> bool:+ """True when the resident model is a native audio model placed in CPU RAM.++ Same audio-type gate as the writer, so only an entry that really recorded a+ CPU placement can skip the arbiter.+ """+ from core.inference.native_audio import NATIVE_AUDIO_TYPES++ resident = backend.models.get(backend.active_model_name, {})+ return resident.get("audio_type") in NATIVE_AUDIO_TYPES and bool(+ resident.get("audio_cpu", False)+ )++async def _preflight_native_audio_placement(config: ModelConfig, request: LoadRequest | ValidateModelRequest, placement: _LoadPlacement) -> _LoadPlacement:@@ -11976,6 +12035,20 @@ async def _preflight_native_audio_placement(detail = "Higgs TTS requires Python 3.10 or newer in Studio.",)+ # Before every VRAM question: sizing a CPU load refuses it on a full GPU.+ if _native_audio_cpu_load(config, request):+ if audio_type == "minimax_music3":+ # The switch already evicted the resident model, so a doomed load costs it.+ raise HTTPException(+ status_code = 400,+ detail = (+ "MiniMax Music 3 cannot be loaded into CPU RAM: its official local "+ "runtime requires an NVIDIA CUDA GPU. Set the audio device back to "+ "Auto to load this model."+ ),+ )+ return placement+automatic = not placement.requested_gpu_idsavailability = (await asyncio.to_thread(_native_audio_post_handoff_free_gb) if automatic else None@@ -12236,6 +12309,10 @@ def _guard_chat_load_against_training(from core.training import get_training_backendfrom routes.training_vram import can_load_chat_during_training+ # Ahead of the diffusion branch, which would 409 a load that takes no VRAM.+ if _native_audio_cpu_load(config, request):+ return+requested_gpu_ids = placement.requested_gpu_idsgpu_ids_are_vulkan_ordinals = placement.gpu_ids_are_vulkan_ordinalsdiffusion_kind = placement.diffusion_kind@@ -13419,6 +13496,8 @@ async def _load_model_impl(backend.models.get(backend.active_model_name) or {},request.max_seq_length,)+ # Without this the route reports already_loaded and nothing moves.+ and _resident_audio_placement_matches(backend, request)):api_monitor.discard(_load_event) # nothing loaded, no monitor rowlogger.info(f"Model already loaded (Unsloth): {model_log_label}, skipping reload")@@ -13439,7 +13518,9 @@ async def _load_model_impl(_sf_supports_reasoning = _sf_flags["supports_reasoning"]_sf_reasoning_style = _sf_flags["reasoning_style"]# Requested chat model already resident: assert CHAT ownership (no-op when held) to correct a drifted owner.- await asyncio.to_thread(acquire_for, CHAT)+ # Owns no GPU, so the arbiter would cancel a generation for nothing.+ if not _resident_audio_holds_no_gpu(backend):+ await asyncio.to_thread(acquire_for, CHAT)return LoadResponse(status = "already_loaded",model = model_log_label if native_grant_backed else backend.active_model_name,@@ -13701,7 +13782,9 @@ async def _load_model_impl(# ...but only when this load will actually use the GPU, exactly as the image and video loaders gate on their device:# a manual gpu_layers=0 load runs on CPU, so taking the arbiter would cancel an image/video generation for nothing.chat_load_needs_gpu = not (- config.is_gguf+ # The non-GGUF case of the same thing: no VRAM, so no arbiter.+ _native_audio_cpu_load(config, request)+ or config.is_ggufand await asyncio.to_thread(zero_vram_chat_load,request.gpu_memory_mode,@@ -14048,11 +14131,12 @@ async def _load_model_impl(_restore_marker_if_prior_preview_still_resident()raise- # Shut down any export subprocess to free VRAM+ # Free the export's VRAM, but only when this load wants VRAM: a CPU load masks+ # the accelerators, so killing the user's export frees nothing it needs.try:from core.export import get_export_backendexp_backend = get_export_backend()- if exp_backend.current_checkpoint:+ if chat_load_needs_gpu and exp_backend.current_checkpoint:logger.info("Shutting down export subprocess to free GPU memory for inference")exp_backend._shutdown_subprocess()exp_backend.current_checkpoint = None@@ -14072,6 +14156,11 @@ async def _load_model_impl(_prior_alias = getattr(backend, "_openai_advertised_id", None)_prior_active = getattr(backend, "active_model_name", None)backend._openai_advertised_id = None+ # Not after the load (held across a long download, the chat evictor cancels+ # this very load) and not before it (until the previous worker exits, this+ # claim is all that stops a second pipeline allocating over a resident model).+ # load_model fires it in between; the post-load release covers a re-taken claim.+ _release_chat_after_teardown = (lambda: release(CHAT)) if not chat_load_needs_gpu else Nonetry:success = await asyncio.to_thread(backend.load_model,@@ -14090,7 +14179,9 @@ async def _load_model_impl(mlx_kv_bits = request.mlx_kv_bits,chat_template_override = request.chat_template_override,load_cancel_event = load_cancel_event,+ on_prior_worker_released = _release_chat_after_teardown,post_handoff_expected_free_gb = post_chat_handoff_expected_free_gb,+ audio_device = request.audio_device,)except Exception:_restore_marker_if_prior_preview_still_resident()@@ -14118,7 +14209,8 @@ async def _load_model_impl()# Same guard the GGUF branch runs above: an Images/Video acquire can land between this load's cancellation and its publish, so this load undoes itself.- if current_owner() != CHAT:+ # On a clean server the owner is None, so ungated this unloads what it loaded.+ if chat_load_needs_gpu and current_owner() != CHAT:await asyncio.to_thread(backend.unload_model, config.identifier)# The worker's base CUDA context outlives the model unload, so kill it too.await asyncio.to_thread(backend._shutdown_subprocess, 5.0)@@ -14129,6 +14221,10 @@ async def _load_model_impl("so the load was cancelled. Unload that model, then try again."),)+ if not chat_load_needs_gpu:+ # This load replaced whatever held CHAT; leaving the claim makes the next+ # Images/Video acquire evict a model that never used the GPU.+ await asyncio.to_thread(release, CHAT)# Stamped here, not in backend.load_model: that entry is built in the load# subprocess and only a fixed model_info mirror crosses back, so it would@@ -17436,7 +17532,9 @@ async def stt_load(_await_stt_disconnect_then_cancel(request, sidecar, cancel_event))try:- await asyncio.to_thread(load_stt, payload.model, engine, cancel_event)+ await asyncio.to_thread(+ functools.partial(load_stt, payload.model, engine, cancel_event, device = payload.device)+ )except SttModelNotDownloadedError as e:raise HTTPException(status_code = 409, detail = str(e))except SttUnavailableError as e:@@ -17481,6 +17579,7 @@ async def stt_validate(async def stt_unload(engine: Optional[str] = None,model: Optional[str] = None,+ wait: bool = True,current_subject: str = Depends(get_current_subject),):"""Release the local STT model when dictation is idle.@@ -17490,7 +17589,9 @@ async def stt_unload(release to the model the caller claims: a surface owns one model, and anothercan switch the same engine between that ownership check and this requestarriving, so the sidecar re-checks under its own lock rather than releasing- whatever happens to be resident by then.+ whatever happens to be resident by then. ``wait=false`` skips a sidecar that+ is mid-transcription instead of draining it: a caller freeing memory as a+ convenience must not cost the user a recording that is still being decoded."""if engine is None:engines = None@@ -17507,7 +17608,9 @@ async def stt_unload(# is not, and only the former takes it positionally. Registry-side it sits# behind a `*`, so passing it positionally raised TypeError, which is the# state a fresh process is in before anything has loaded.- failed: list[str] = await asyncio.to_thread(unload_stt, engines, expected_model = model)+ failed: list[str] = await asyncio.to_thread(+ unload_stt, engines, expected_model = model, wait = wait+ )if failed:raise HTTPException(status_code = 500,@@ -17523,10 +17626,11 @@ async def _transcribe_audio_bytes(fast: bool,engine: Optional[str] = None,request: Optional[Request] = None,+ device: Optional[str] = None,) -> JSONResponse:"""Run STT for already-decoded request bytes."""return JSONResponse(- content = await _transcribe_audio_result(raw, model, language, fast, engine, request)+ content = await _transcribe_audio_result(raw, model, language, fast, engine, request, device))@@ -17537,6 +17641,7 @@ async def _transcribe_audio_result(fast: bool,engine: Optional[str] = None,request: Optional[Request] = None,+ device: Optional[str] = None,) -> dict:"""STT for already-decoded bytes, sidecar errors mapped to HTTP statuses.Returns the sidecar's result dict so callers own the response shape."""@@ -17580,7 +17685,9 @@ async def _transcribe_audio_result(# timers fired, which is what OOMs a device that fits either alone. A no-op once# the model is resident, so the steady state costs a residency check.load_stt, _ = _stt_lifecycle()- await asyncio.to_thread(load_stt, model, serving_engine, cancel_event)+ await asyncio.to_thread(+ functools.partial(load_stt, model, serving_engine, cancel_event, device = device)+ )if cancel_event is None:result = await asyncio.to_thread(sidecar.transcribe, raw, model, language, fast)else:@@ -17655,7 +17762,13 @@ async def transcribe_audio(# Same disconnect cancellation as the raw and OpenAI routes: without the request# a client that goes away leaves the sidecar transcribing under its lock.return await _transcribe_audio_bytes(- raw, payload.model, payload.language, payload.fast, payload.engine, request+ raw,+ payload.model,+ payload.language,+ payload.fast,+ payload.engine,+ request,+ payload.device,)@@ -17666,6 +17779,9 @@ async def transcribe_audio_raw(language: Optional[str] = None,fast: bool = False,engine: Optional[str] = None,+ # Same literal as the JSON models: a misspelled "cpu" falling through to auto+ # would silently put the model back on the GPU. 422 instead.+ device: Optional[Literal["auto", "cpu", "gpu"]] = None,current_subject: str = Depends(get_current_subject),):"""Transcribe a raw audio body without base64 or JSON conversion overhead."""@@ -17678,7 +17794,7 @@ async def transcribe_audio_raw(chunks.append(chunk)return JSONResponse(content = await _transcribe_audio_result(- b"".join(chunks), model, language, fast, engine, request+ b"".join(chunks), model, language, fast, engine, request, device))diff --git a/studio/backend/routes/training_vram.py b/studio/backend/routes/training_vram.pyindex 9720891869d..bd4a43524bf 100644--- a/studio/backend/routes/training_vram.py+++ b/studio/backend/routes/training_vram.py@@ -418,6 +418,10 @@ def free_chat_models_for_training(reason: str) -> List[str]:try:from core.inference import get_inference_backendinf = get_inference_backend()+ # No CPU exemption here, unlike the GGUF branch and the STT sidecars: it would+ # key off a marker the orchestrator writes rather than the worker that masked,+ # and a marker that disagreed is an OOM mid-training. Freeing a model that held+ # no VRAM only costs a reload.if inf.active_model_name or inf.loading_models:name = inf.active_model_name or next(iter(inf.loading_models), None)logger.info(@@ -452,6 +456,33 @@ def free_chat_models_for_training(reason: str) -> List[str]:return freed+def _stt_sidecar_holds_no_vram(sidecar) -> bool:+ """True only when the resident dictation model is provably in CPU RAM.++ Conservative on purpose: anything unreadable answers False and the sidecar is+ freed as before. Skipping one that does hold VRAM would starve the run this+ is making room for, which is far worse than a needless reload.+ """+ try:+ device = getattr(sidecar, "device", None)+ if isinstance(device, str) and device.strip().lower() == "cpu":+ return True+ # whisper.cpp and llama.cpp report a runtime name rather than a device, so read+ # the flag each sets when it started without the GPU. Prefer the fact over the+ # wish: mtmd's _gpu_disabled is what the live server was started with, while+ # _forced_cpu is the standing preference, recorded even on the branch that does+ # NOT restart a server with a request in flight, so reading it reports a server+ # still at -ngl 99 as holding no VRAM.+ gpu_disabled = getattr(sidecar, "_gpu_disabled", None)+ if gpu_disabled is not None:+ return gpu_disabled is True+ # whisper.cpp sets _forced_cpu only alongside a spawned --no-gpu and clears it+ # on release, so there it is the fact.+ return getattr(sidecar, "_forced_cpu", False) is True+ except Exception: # noqa: BLE001 - a probe must never fail the release it precedes+ return False++def free_stt_model_for_training(reason: str) -> List[str]:"""Unload the dictation model(s) before training. Never raises.@@ -475,7 +506,11 @@ def free_stt_model_for_training(reason: str) -> List[str]:freed.append("stt:loading")else:model = sidecar.loaded_model- if model:+ if model and _stt_sidecar_holds_no_vram(sidecar):+ logger.info(+ "Keeping CPU-placed STT model '%s' through training (%s)", model, reason+ )+ elif model:logger.info("Unloading STT model '%s' for training (%s)", model, reason)sidecar.unload()freed.append(f"stt:{model}")@@ -497,7 +532,13 @@ def free_stt_model_for_training(reason: str) -> List[str]:freed.append("stt:gguf-loading")else:ggml_model = ggml.loaded_model- if ggml_model:+ if ggml_model and _stt_sidecar_holds_no_vram(ggml):+ logger.info(+ "Keeping CPU-placed GGUF STT model '%s' through training (%s)",+ ggml_model,+ reason,+ )+ elif ggml_model:logger.info("Unloading GGUF STT model '%s' for training (%s)", ggml_model, reason)ggml.unload()freed.append(f"stt:{ggml_model}")@@ -518,7 +559,13 @@ def free_stt_model_for_training(reason: str) -> List[str]:freed.append("stt:mtmd-loading")else:mtmd_model = mtmd.loaded_model- if mtmd_model:+ if mtmd_model and _stt_sidecar_holds_no_vram(mtmd):+ logger.info(+ "Keeping CPU-placed mtmd STT model '%s' through training (%s)",+ mtmd_model,+ reason,+ )+ elif mtmd_model:logger.info("Unloading mtmd STT model '%s' for training (%s)", mtmd_model, reason)mtmd.unload()freed.append(f"stt:{mtmd_model}")diff --git a/studio/backend/tests/test_audio_device_guards.py b/studio/backend/tests/test_audio_device_guards.pynew file mode 100644index 00000000000..7909afb7065--- /dev/null+++ b/studio/backend/tests/test_audio_device_guards.py@@ -0,0 +1,471 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Guards that must not charge a CPU-placed audio load for VRAM it never takes.++A load held in system RAM still passed the training coexistence check, the GPU+arbiter and the memory preflight, so it could be refused on a full card, evict an+image or video pipeline, or be reported already-loaded while sitting on the GPU.+"""++import asyncio+import sys+import types+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parent.parent))++from fastapi import HTTPException # noqa: E402++import routes.inference as ri # noqa: E402+from routes.training_vram import _stt_sidecar_holds_no_vram # noqa: E402++[email protected](autouse = True)+def _neutral_audio_device_env(monkeypatch):+ """A server-wide default must not decide the outcome of these tests.++ Placement here is asserted against no opinion, so a host that sets+ UNSLOTH_AUDIO_DEVICE would fail these on correct behaviour, and that host is+ exactly the one most likely to run them.+ """+ monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)+++def _audio(audio_type = "higgs_tts2", **kwargs):+ return types.SimpleNamespace(audio_type = audio_type, is_lora = False, identifier = "x/y", **kwargs)+++def _request(audio_device = None):+ return types.SimpleNamespace(audio_device = audio_device)+++def test_only_a_native_audio_model_counts_as_a_cpu_audio_load():+ assert ri._native_audio_cpu_load(_audio(), _request("cpu"))+ assert not ri._native_audio_cpu_load(_audio(), _request("auto"))+ assert not ri._native_audio_cpu_load(_audio(), _request(None))+++def test_a_chat_model_cannot_skip_the_guards_by_sending_audio_device():+ """audio_device is documented as ignored off the audio path. If it were not+ gated here, any load could set it and walk past the training guard."""+ assert not ri._native_audio_cpu_load(_audio(audio_type = None), _request("cpu"))+ assert not ri._native_audio_cpu_load(_audio(audio_type = "whisper"), _request("cpu"))+++def test_a_cpu_audio_load_is_not_refused_while_training_runs(monkeypatch):+ """The guard 409s an unsized load, and refuses everything outright during+ diffusion training. Neither applies to weights that never reach the card."""+ monkeypatch.setattr(ri, "_diffusion_training_active", lambda: True)++ assert (+ ri._guard_chat_load_against_training(+ _audio(is_gguf = False),+ types.SimpleNamespace(+ audio_device = "cpu",+ gpu_memory_mode = "auto",+ gpu_layers = -1,+ tensor_parallel = False,+ ),+ load_in_4bit = False,+ placement = types.SimpleNamespace(+ requested_gpu_ids = None,+ gpu_ids_are_vulkan_ordinals = False,+ diffusion_kind = None,+ ),+ )+ is None+ )+++def test_a_gpu_audio_load_is_still_refused_during_diffusion_training(monkeypatch):+ """The exemption must be the CPU placement, not the audio type."""+ monkeypatch.setattr(ri, "_diffusion_training_active", lambda: True)++ with pytest.raises(HTTPException) as excinfo:+ ri._guard_chat_load_against_training(+ _audio(is_gguf = False),+ types.SimpleNamespace(+ audio_device = "auto",+ gpu_memory_mode = "auto",+ gpu_layers = -1,+ tensor_parallel = False,+ ),+ load_in_4bit = False,+ placement = types.SimpleNamespace(+ requested_gpu_ids = None,+ gpu_ids_are_vulkan_ordinals = False,+ diffusion_kind = None,+ ),+ )+ assert excinfo.value.status_code == 409+++def test_a_cpu_load_skips_the_vram_preflight_entirely(monkeypatch):+ """Sizing it would refuse the load on a full GPU, which is the case the+ option exists for. The probe must not even be reached."""++ def _never():+ raise AssertionError("a CPU load must not size GPU memory")++ monkeypatch.setattr(ri, "_native_audio_post_handoff_free_gb", _never)+ placement = types.SimpleNamespace(requested_gpu_ids = None)++ result = asyncio.run(ri._preflight_native_audio_placement(_audio(), _request("cpu"), placement))+ assert result is placement+++def test_minimax_on_cpu_is_refused_before_the_resident_model_is_evicted():+ """Its runtime needs CUDA. Failing later in the worker would cost the user+ the model they already had, since the switch evicts before the load runs."""+ with pytest.raises(HTTPException) as excinfo:+ asyncio.run(+ ri._preflight_native_audio_placement(+ _audio(audio_type = "minimax_music3"),+ _request("cpu"),+ types.SimpleNamespace(requested_gpu_ids = None),+ )+ )+ assert excinfo.value.status_code == 400+ assert "CPU RAM" in excinfo.value.detail+++def _backend(audio_cpu, audio_type = "higgs_tts2"):+ entry = {"is_audio": True, "audio_type": audio_type}+ if audio_cpu is not None:+ entry["audio_cpu"] = audio_cpu+ return types.SimpleNamespace(active_model_name = "x/y", models = {"x/y": entry})+++def test_a_resident_gpu_audio_model_does_not_satisfy_a_cpu_request():+ assert not ri._resident_audio_placement_matches(_backend(audio_cpu = False), _request("cpu"))+++def test_a_resident_cpu_audio_model_satisfies_the_same_request_again():+ assert ri._resident_audio_placement_matches(_backend(audio_cpu = True), _request("cpu"))+++def test_a_model_loaded_before_this_existed_is_read_as_gpu():+ """No recorded key means the load predates the option, which placed on GPU."""+ assert ri._resident_audio_placement_matches(_backend(audio_cpu = None), _request("auto"))+ assert not ri._resident_audio_placement_matches(_backend(audio_cpu = None), _request("cpu"))+++def test_a_non_audio_model_keeps_the_shortcut():+ assert ri._resident_audio_placement_matches(+ _backend(audio_cpu = None, audio_type = None), _request("cpu")+ )+++def test_a_cpu_placed_sidecar_is_left_alone_when_training_claims_vram():+ assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "cpu"))+ assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = True))+ assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "llama.cpp", _gpu_disabled = True))+++def test_anything_that_might_hold_vram_is_still_evicted():+ """Default-deny: starving the run this makes room for is the worse failure."""+ assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "cuda"))+ assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "mps"))+ assert not _stt_sidecar_holds_no_vram(+ types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = False)+ )+ assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace())++ class _Raises:+ @property+ def device(self):+ raise RuntimeError("unreadable")++ assert not _stt_sidecar_holds_no_vram(_Raises())+++def test_the_shortcut_reads_the_resident_model_not_the_requested_config():+ """It runs ahead of config resolution, so reading a config there raised+ UnboundLocalError and turned every repeat safetensors load into a 500."""+ import inspect+ assert list(inspect.signature(ri._resident_audio_placement_matches).parameters) == [+ "backend",+ "request",+ ]+++def test_a_cpu_placed_audio_model_never_takes_the_arbiter():+ """It holds no GPU, so acquiring would cancel an image or video run for nothing."""+ assert ri._resident_audio_holds_no_gpu(_backend(audio_cpu = True))+ assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = False))+ assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = None))+++def test_a_chat_model_cannot_reach_the_arbiter_skip():+ """Same audio-type gate as the writer, so a stray marker cannot skip it."""+ assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = True, audio_type = None))+++def test_nothing_resident_reads_as_holding_the_gpu():+ empty = types.SimpleNamespace(active_model_name = None, models = {})+ assert not ri._resident_audio_holds_no_gpu(empty)+++def _inference_source() -> str:+ import pathlib++ # Explicit encoding: read_text() defaults to the locale one, cp1252 on Windows,+ # and this file is UTF-8.+ return pathlib.Path(ri.__file__).read_text(encoding = "utf-8")+++def test_the_already_loaded_branch_guards_its_acquire():+ src = _inference_source()+ assert "if not _resident_audio_holds_no_gpu(backend):\n" in src+++def test_the_post_load_ownership_check_is_gated_like_the_gguf_one():+ """Ungated, it unloads the CPU audio model it just loaded and returns 409:+ the load skips acquire_for, so on a clean server the owner is None."""+ src = _inference_source()+ assert "if chat_load_needs_gpu and current_owner() != CHAT:" in src+ assert "\n if current_owner() != CHAT:" not in src+++def test_a_cpu_audio_worker_hides_cuda_and_hip():+ from core.inference.audio_device import mask_accelerators_for_cpu_audio++ env = {"CUDA_VISIBLE_DEVICES": "0,1", "HIP_VISIBLE_DEVICES": "0"}+ mask_accelerators_for_cpu_audio(env)+ assert env["CUDA_VISIBLE_DEVICES"] == ""+ # HIP reads the CUDA variable only when its own is unset, so blanking one is+ # not enough; -1 is the sentinel the CPU embed server already uses.+ assert env["HIP_VISIBLE_DEVICES"] == "-1"+++def test_an_inherited_rocr_mask_is_left_alone():+ """Clearing it exposes more agents to HSA enumeration, not fewer."""+ from core.inference.audio_device import mask_accelerators_for_cpu_audio++ env = {"ROCR_VISIBLE_DEVICES": "0"}+ mask_accelerators_for_cpu_audio(env)+ assert env["ROCR_VISIBLE_DEVICES"] == "0"+++def test_the_mask_runs_before_hardware_detection():+ """detect_hardware() calls get_device_properties, which creates the context+ this load is supposed not to hold, so masking after it is too late."""+ import pathlib+ from core.inference import worker++ src = pathlib.Path(worker.__file__).read_text(encoding = "utf-8")+ assert src.index("mask_accelerators_for_cpu_audio(os.environ)") < src.index(+ "_hw.detect_hardware()"+ )+++def test_a_zero_gpu_standard_load_drops_the_stale_chat_claim():+ """The load replaced whatever held CHAT. Leaving the claim makes the next+ Images/Video acquire run the CHAT evictor and unload a CPU audio model that+ was never on the GPU. The GGUF branch already releases; both do now.++ Two awaited sites: the GGUF branch, and the standard branch after its load+ (which cannot cover a claim re-taken while it ran). The standard branch's+ during-load release is the third, handed to load_model as a callback so it+ fires once the previous worker is gone rather than before it."""+ src = _inference_source()+ assert src.count("await asyncio.to_thread(release, CHAT)") == 2+ assert src.count("(lambda: release(CHAT)) if not chat_load_needs_gpu else None") == 1+++def test_the_release_is_gated_on_the_same_flag_as_the_409():+ """Two `if not` sites plus the callback's own inline gate, which spells the+ flag the same way."""+ src = _inference_source()+ assert src.count("if not chat_load_needs_gpu:") == 2+ assert src.count("if not chat_load_needs_gpu else None") == 1+++def test_every_http_device_field_pins_the_three_canonical_values():+ """A misspelled "cpu" that fell through to auto would put the model back on+ the GPU without saying so. 422 is the honest answer at the boundary."""+ import inspect+ import typing++ from models.inference import (+ LoadRequest,+ SttLoadRequest,+ TranscribeRequest,+ ValidateModelRequest,+ )++ expected = typing.Optional[typing.Literal["auto", "cpu", "gpu"]]+ for model, field in (+ (LoadRequest, "audio_device"),+ (ValidateModelRequest, "audio_device"),+ (SttLoadRequest, "device"),+ (TranscribeRequest, "device"),+ ):+ assert model.model_fields[field].annotation == expected, f"{model.__name__}.{field}"+ # The raw endpoint takes it as a query param, so it is annotated rather than+ # declared on a model; it must not be the odd one out.+ assert inspect.signature(ri.transcribe_audio_raw).parameters["device"].annotation == expected+++def test_a_gpu_resident_mtmd_server_is_never_reported_as_holding_no_vram():+ """mtmd records the user's wish on the branch that does not restart the server.++ _load_locked writes _forced_cpu even when an in-flight request keeps the running+ server, so a llama-server still at -ngl 99 carries a CPU wish. Trusting it would+ let training start beside a model that holds the whole checkpoint in VRAM.+ """+ resident_on_gpu = types.SimpleNamespace(+ device = "llama.cpp",+ _gpu_disabled = False,+ _forced_cpu = True,+ )+ assert _stt_sidecar_holds_no_vram(resident_on_gpu) is False++ really_cpu = types.SimpleNamespace(+ device = "llama.cpp",+ _gpu_disabled = True,+ _forced_cpu = True,+ )+ assert _stt_sidecar_holds_no_vram(really_cpu) is True+++def test_whisper_cpp_still_exempts_a_server_started_with_no_gpu():+ """ggml has no separate wish: _forced_cpu sits next to the spawned --no-gpu."""+ assert (+ _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = True))+ is True+ )+ assert (+ _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = False))+ is False+ )+++def test_a_forced_cpu_native_audio_load_selects_no_gpu():+ """Selection returns every card needed to hold the checkpoint, and the worker+ forwards that list to a backend that rejects more than one. Its required_gb also+ becomes expected_free_gb, whose settle wait raises when Images holds the card."""+ import inspect++ from core.inference.orchestrator import InferenceOrchestrator++ src = inspect.getsource(InferenceOrchestrator.load_model)+ head = src[: src.index("prepare_gpu_selection(")]+ assert "audio_device_forces_cpu" in head+ assert "is_native_audio_model" in head+++def test_the_stale_chat_claim_is_dropped_before_the_load_not_only_after():+ """A download and load can run for minutes. Held across that window the claim+ makes an Images acquire run the chat evictor, which cancels this load."""+ import inspect++ src = inspect.getsource(ri._load_model_impl)+ before_load = src[: src.index("backend.load_model,")]+ assert before_load.count("release, CHAT") >= 1+++def test_the_gguf_audio_codec_follows_the_servers_own_placement():+ """The codec is a second allocation. A server launched at zero offload is+ classified holds_no_vram, which lets the route skip GPU arbitration and leaves+ it resident when training reclaims memory; a codec on CUDA would hold VRAM+ under both of those promises."""+ import inspect++ from core.inference.llama_cpp import LlamaCppBackend++ src = inspect.getsource(LlamaCppBackend.init_audio_codec)+ device_line = next(l for l in src.splitlines() if l.strip().startswith("device ="))+ assert "holds_no_vram" in device_line, device_line+++def test_decoding_happens_where_the_codec_actually_is():+ """Loading the codec on CPU is only half of it.++ The decoders build their input tensors on the device they are handed. Handed+ CUDA for a CPU-resident codec, SNAC and DAC fail outright on a device mismatch+ and BiCodec moves the codec onto the card, taking the VRAM a CPU RAM load+ promised not to take. The recorded placement wins over the caller's request.+ """+ import torch++ from core.inference.audio_codecs import AudioCodecManager++ mgr = AudioCodecManager()+ seen = {}++ class _FakeSnac:+ def decode(self, codes):+ seen["device"] = codes[0].device.type+ return torch.zeros(1, 8)++ mgr._snac_model = _FakeSnac()+ mgr._codec_devices["snac"] = "cpu"++ # 128257 opens the speech section; the seven codes after it make one frame.+ token_ids = [128257] + [128266 + i for i in range(7)]+ mgr.decode("snac", "cuda", token_ids = token_ids)++ assert seen["device"] == "cpu"+++def test_a_codec_with_no_recorded_placement_still_honours_the_caller():+ """Nothing is recorded until a codec is actually loaded, and every pre-existing+ caller passes the device it wants. An unknown codec must keep deferring to it."""+ from core.inference.audio_codecs import AudioCodecManager++ mgr = AudioCodecManager()+ assert mgr._codec_devices == {}+++def test_a_cpu_load_leaves_a_running_export_alone():+ """The export teardown exists to free VRAM for the incoming model. A load that+ masks the accelerators wants none of it, so killing the user's export buys+ nothing and costs them the job."""+ import inspect++ src = inspect.getsource(ri._load_model_impl)+ guard = next(+ l for l in src.splitlines() if "exp_backend.current_checkpoint" in l and "if " in l+ )+ assert "chat_load_needs_gpu" in guard, guard+++def test_the_chat_claim_outlives_the_worker_that_earned_it():+ """Released before the load, the claim leaves a still-resident GPU model+ unowned, and acquire_for evicts nobody when the arbiter has no owner: an+ Images or Video load then allocates straight over it. The release is handed to+ load_model, which runs it once the previous worker is gone and its memory is+ back, still ahead of the download."""+ import inspect++ from core.inference.orchestrator import InferenceOrchestrator++ src = inspect.getsource(ri._load_model_impl)+ assert "on_prior_worker_released = _release_chat_after_teardown" in src++ sig = inspect.signature(InferenceOrchestrator.load_model)+ assert "on_prior_worker_released" in sig.parameters++ # After both bail-outs: either one means the card is still busy, so the claim is+ # still true.+ load_src = inspect.getsource(InferenceOrchestrator.load_model)+ hook = load_src.index("on_prior_worker_released()")+ assert load_src.index("did not exit and still holds GPU") < hook+ assert load_src.index("was not released; ") < hook+++def test_whisper_cpp_publishes_the_load_before_choosing_its_placement():+ """The training hook reads is_loading() without this method's lock. A placement+ decided ahead of that flag is invisible to training: it sees no load in flight,+ reads the outgoing model's CPU placement, preserves it as holding no VRAM, and+ the command already built then starts a GPU-backed server beside the run."""+ import inspect++ from core.inference.stt_ggml_sidecar import GgmlSttSidecar++ src = inspect.getsource(GgmlSttSidecar.load)+ assert src.index("self._loading = True") < src.index("elif _training_active():")diff --git a/studio/backend/tests/test_audio_device_preference.py b/studio/backend/tests/test_audio_device_preference.pynew file mode 100644index 00000000000..e811899cc21--- /dev/null+++ b/studio/backend/tests/test_audio_device_preference.py@@ -0,0 +1,360 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The user's choice of where an audio model's weights go.++Audio loads pick an accelerator on their own. These cover the option that+overrides that: CPU RAM must win over a working GPU, "auto" must still detect,+and a resident model loaded under the other preference must be reloaded rather+than reused where it is.+"""++import sys+import threading+import types+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parent.parent))++from core.inference.audio_device import ( # noqa: E402+ audio_device_default,+ audio_device_forces_cpu,+ normalize_audio_device,+)++[email protected](autouse = True)+def _neutral_audio_device_env(monkeypatch):+ """A server-wide default must not decide the outcome of these tests.++ Placement here is asserted against no opinion, so a host that sets+ UNSLOTH_AUDIO_DEVICE would fail these on correct behaviour, and that host is+ exactly the one most likely to run them.+ """+ monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)+++ "value, expected",+ [+ ("cpu", "cpu"),+ ("CPU", "cpu"),+ (" cpu ", "cpu"),+ ("ram", "cpu"),+ ("gpu", "gpu"),+ ("cuda", "gpu"),+ ("mps", "gpu"),+ ("rocm", "gpu"),+ ("auto", "auto"),+ ("", "auto"),+ (None, "auto"),+ ],+)+def test_every_accepted_spelling_maps_onto_one_of_three_values(value, expected):+ assert normalize_audio_device(value) == expected+++def test_an_unknown_preference_detects_rather_than_failing_the_load():+ """Detection is what the caller would have done without the option at all."""+ assert normalize_audio_device("gpu-2") == "auto"+ assert normalize_audio_device("nvidia rtx 4090") == "auto"+ assert not audio_device_forces_cpu("gpu-2")+++def test_the_environment_supplies_the_default_for_a_request_that_names_none(monkeypatch):+ """A headless or CLI Studio sets this once instead of sending it per request."""+ monkeypatch.setenv("UNSLOTH_AUDIO_DEVICE", "cpu")+ assert audio_device_default() == "cpu"+ assert audio_device_forces_cpu(None)++ # An explicit request still outranks it, in both directions.+ assert not audio_device_forces_cpu("auto")+ assert audio_device_forces_cpu("cpu")+++def test_without_the_environment_variable_nothing_is_forced_to_cpu(monkeypatch):+ monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)+ assert audio_device_default() == "auto"+ assert not audio_device_forces_cpu(None)+++def _torch_with_cuda(monkeypatch):+ """A torch whose CUDA is available, so anything but CPU is a real choice."""+ torch = types.SimpleNamespace(+ float16 = "float16",+ float32 = "float32",+ cuda = types.SimpleNamespace(is_available = lambda: True),+ backends = types.SimpleNamespace(mps = types.SimpleNamespace(is_available = lambda: False)),+ )+ monkeypatch.setitem(sys.modules, "torch", torch)+ return torch+++def test_cpu_is_honoured_on_a_machine_with_a_working_gpu(monkeypatch):+ """The whole point of the option: detection would have said cuda."""+ from core.inference import stt_sidecar++ _torch_with_cuda(monkeypatch)+ monkeypatch.setattr(stt_sidecar, "_training_active", lambda: False)++ assert stt_sidecar._pick_device("cpu") == ("cpu", "float32")+++def test_auto_and_gpu_both_still_detect_the_accelerator(monkeypatch):+ """ "gpu" is not a separate placement: detection already prefers the card."""+ from core.inference import stt_sidecar++ _torch_with_cuda(monkeypatch)+ monkeypatch.setattr(stt_sidecar, "_training_active", lambda: False)++ assert stt_sidecar._pick_device("auto") == ("cuda", "float16")+ assert stt_sidecar._pick_device("gpu") == ("cuda", "float16")+ assert stt_sidecar._pick_device(None) == ("cuda", "float16")+++def test_a_resident_model_on_the_other_device_is_reloaded_not_reused(monkeypatch):+ """A preference is a request about placement. Reusing the old one ignores it."""+ from core.inference import stt_sidecar++ monkeypatch.setattr(stt_sidecar, "_pick_device", lambda _preference = None: ("cpu", "float32"))+ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")++ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)+ sidecar._engine = object()+ sidecar._model_id = "small"+ sidecar._device = "cuda"+ sidecar._device_preference = "auto"++ builds: list[str] = []++ def _build(snapshot_path, device, dtype, cancel_event):+ builds.append(device)+ return object()++ monkeypatch.setattr(sidecar, "_build_model", _build)+ monkeypatch.setattr(+ sidecar,+ "_ensure_model_downloaded",+ lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(+ # The resident shortcut answers with no path; a replacement load needs one.+ path = None if use_resident else "/snapshots/small",+ is_multilingual = True,+ ),+ )+ monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)++ sidecar.load("small", device = "cpu")++ assert builds == ["cpu"], "the CPU preference must have driven a fresh load"+ assert sidecar._device == "cpu"+ assert sidecar._device_preference == "cpu"+++def test_the_same_preference_reuses_the_resident_model(monkeypatch):+ """Unchanged placement must stay a residency check, not a reload."""+ from core.inference import stt_sidecar++ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")++ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)+ resident = object()+ sidecar._engine = resident+ sidecar._model_id = "small"+ sidecar._device = "cpu"+ sidecar._device_preference = "cpu"++ def _never(*args, **kwargs):+ raise AssertionError("a matching preference must not rebuild the model")++ monkeypatch.setattr(sidecar, "_build_model", _never)++ assert sidecar.load("small", device = "cpu") is resident+++def test_a_model_loaded_before_the_option_existed_is_not_reloaded(monkeypatch):+ """No recorded preference means the engine predates the choice; reusing it is+ what every caller got before, and a forced reload would cost a load for nothing."""+ from core.inference import stt_sidecar++ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")++ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)+ resident = object()+ sidecar._engine = resident+ sidecar._model_id = "small"+ sidecar._device = "cuda"+ sidecar._device_preference = None++ monkeypatch.setattr(+ sidecar,+ "_build_model",+ lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not rebuild")),+ )++ assert sidecar.load("small", device = "auto") is resident+++def test_the_registry_hands_the_preference_to_the_engines_sidecar(monkeypatch):+ from core.inference import stt_registry++ seen: dict = {}++ class _Sidecar:+ def load(+ self,+ model,+ request_cancel_event = None,+ device = None,+ ):+ seen["model"] = model+ seen["device"] = device++ monkeypatch.setattr(stt_registry, "sidecar_for", lambda _engine: _Sidecar())+ monkeypatch.setattr(stt_registry, "_model_is_downloaded", lambda _e, _m: True)+ monkeypatch.setattr(stt_registry, "unload", lambda *a, **k: [])++ stt_registry.load("small", "transformers", threading.Event(), device = "cpu")++ assert seen == {"model": "small", "device": "cpu"}+++def test_native_audio_holds_tts_weights_in_cpu_ram_when_asked(monkeypatch):+ from core.inference import native_audio++ _torch_with_cuda(monkeypatch)++ assert native_audio.NativeAudioBackend(device_preference = "cpu").device == "cpu"+ assert native_audio.NativeAudioBackend(device_preference = "auto").device == "cuda"+ assert native_audio.NativeAudioBackend().device == "cuda"+++def test_minimax_music_explains_that_cpu_was_chosen_rather_than_missing(monkeypatch):+ """The generic refusal reads like "your hardware cannot do this" and sends a+ user with a perfectly good card looking for one."""+ from core.inference import native_audio++ _torch_with_cuda(monkeypatch)+ backend = native_audio.NativeAudioBackend(device_preference = "cpu")+ config = types.SimpleNamespace(+ identifier = "MiniMaxAI/MiniMax-Music3",+ audio_type = "minimax_music3",+ path = None,+ )++ with pytest.raises(RuntimeError, match = "cannot be loaded into CPU RAM"):+ backend.load_model(config)+++def test_a_caller_that_sends_no_device_leaves_the_placement_alone(monkeypatch):+ """``/v1/audio/transcriptions`` sends none. Treating that as "auto" pulled a+ CPU model back onto the GPU, and the next dictation pulled it off again:+ two full reloads per alternation, and VRAM the user asked us not to take."""+ from core.inference import stt_sidecar++ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")+ monkeypatch.setattr(+ stt_sidecar,+ "_pick_device",+ lambda preference = None: ("cpu", "float32") if preference == "cpu" else ("cuda", "float16"),+ )++ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)+ builds: list[str] = []+ monkeypatch.setattr(+ sidecar, "_build_model", lambda p, device, dtype, c: builds.append(device) or object()+ )+ monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)+ monkeypatch.setattr(+ sidecar,+ "_ensure_model_downloaded",+ lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(+ path = None if (sidecar._engine is not None and use_resident) else "/snapshots/small",+ is_multilingual = True,+ ),+ )++ sidecar.load("small", device = "cpu") # Voice settings: the user picked CPU+ sidecar.load("small", device = None) # OpenAI-compatible route: no opinion+ sidecar.load("small", device = "cpu") # the next dictation++ assert builds == ["cpu"], "only the first load should have built anything"+ assert sidecar._device == "cpu"+++def test_an_explicit_change_still_reloads_after_a_no_opinion_call(monkeypatch):+ """No opinion must not also freeze the placement: the setting still moves it."""+ from core.inference import stt_sidecar++ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")+ monkeypatch.setattr(+ stt_sidecar,+ "_pick_device",+ lambda preference = None: ("cpu", "float32") if preference == "cpu" else ("cuda", "float16"),+ )++ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)+ builds: list[str] = []+ monkeypatch.setattr(+ sidecar, "_build_model", lambda p, device, dtype, c: builds.append(device) or object()+ )+ monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)+ monkeypatch.setattr(+ sidecar,+ "_ensure_model_downloaded",+ lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(+ path = None if (sidecar._engine is not None and use_resident) else "/snapshots/small",+ is_multilingual = True,+ ),+ )++ sidecar.load("small", device = "cpu")+ sidecar.load("small", device = None)+ sidecar.load("small", device = "auto")++ assert builds == ["cpu", "cuda"]+++def test_the_mtmd_reuse_branch_still_records_the_choice(monkeypatch):+ """Training makes an explicit "cpu" indistinguishable from the running+ server, so the early return has to keep the preference or the next+ device-less load sends the model back to the GPU once training ends."""+ from core.inference import stt_mtmd_sidecar++ sidecar = stt_mtmd_sidecar.MtmdSttSidecar.__new__(stt_mtmd_sidecar.MtmdSttSidecar)+ sidecar._lock = threading.RLock()+ sidecar._forced_cpu = False+ sidecar._gpu_disabled = True+ sidecar._model_id = "m"+ sidecar._binary_path_revision = 1+ sidecar._active_requests = 0+ monkeypatch.setattr(sidecar, "_process_alive", lambda: True)+ monkeypatch.setattr(sidecar, "_schedule_idle_unload_locked", lambda: None)+ monkeypatch.setattr(stt_mtmd_sidecar, "_training_active", lambda: True)++ sidecar._load_locked("m", "whisper-server", path_revision = 1, device = "cpu")++ assert sidecar._forced_cpu is True+++def test_stt_unload_can_skip_a_sidecar_that_is_mid_transcription():+ """The Voice device switch is an early release, not a reclaim: draining a+ live transcription for 30s and then killing it loses the recording."""+ import inspect++ from core.inference import stt_registry+ from core.inference.orchestrator import InferenceOrchestrator+ from routes import inference as ri++ assert "wait" in inspect.signature(ri.stt_unload).parameters+ assert inspect.signature(ri.stt_unload).parameters["wait"].default is True+ # Both halves of _stt_lifecycle have to take it, or the route raises TypeError+ # for whichever one is live.+ assert "wait" in inspect.signature(InferenceOrchestrator.unload_stt_model).parameters+ assert "wait" in inspect.signature(stt_registry.unload).parametersdiff --git a/studio/backend/tests/test_gpu_arbiter.py b/studio/backend/tests/test_gpu_arbiter.pyindex de553b9de41..c2b06e44a65 100644--- a/studio/backend/tests/test_gpu_arbiter.py+++ b/studio/backend/tests/test_gpu_arbiter.py@@ -343,6 +343,9 @@ def test_the_safetensors_load_yields_a_gpu_it_lost_while_loading():load_impl = route_src[route_src.index("async def _load_model_impl") :]unsloth_load = load_impl.index("success = await asyncio.to_thread(")tail = load_impl[unsloth_load:]- guard = tail.index("if current_owner() != CHAT:")+ # Gated on chat_load_needs_gpu like the GGUF branch: a load that never took the+ # arbiter has no ownership to lose, and checking against a None owner would 409+ # every CPU-placed audio load.+ guard = tail.index("if chat_load_needs_gpu and current_owner() != CHAT:")assert "await asyncio.to_thread(backend.unload_model, config.identifier)" in tail[guard:]assert tail.index("status_code = 409", guard) > guarddiff --git a/studio/backend/tests/test_openai_audio_transcriptions_route.py b/studio/backend/tests/test_openai_audio_transcriptions_route.pyindex b6681b4afcb..78ee761ed3f 100644--- a/studio/backend/tests/test_openai_audio_transcriptions_route.py+++ b/studio/backend/tests/test_openai_audio_transcriptions_route.py@@ -31,6 +31,7 @@ def _make_client(monkeypatch, transcribe = None):fast,engine = None,request = None,+ device = None,):calls.append({diff --git a/studio/backend/tests/test_stt_registry.py b/studio/backend/tests/test_stt_registry.pyindex 483a729cb2b..7d3b3e4a983 100644--- a/studio/backend/tests/test_stt_registry.py+++ b/studio/backend/tests/test_stt_registry.py@@ -27,6 +27,7 @@ class _Sidecar:self.unload_expected = []self.loaded_with = Noneself.load_cancel_event = None+ self.loaded_device = Nonedef is_loading(self):return self._loading@@ -35,9 +36,11 @@ class _Sidecar:self,model,request_cancel_event = None,+ device = None,):self.loaded_with = modelself.load_cancel_event = request_cancel_event+ self.loaded_device = devicedef unload(self,@@ -85,7 +88,7 @@ def test_load_releases_the_other_engines_after_the_target_loads(monkeypatch):f"unload:{name}")target = sidecars["mtmd"]- target.load = lambda model, request_cancel_event = None: order.append("load:mtmd")+ target.load = lambda model, request_cancel_event = None, device = None: order.append("load:mtmd")monkeypatch.setattr(stt_registry, "sidecar_for", lambda name: sidecars[name])stt_registry.load("qwen3-asr-0.6b", "mtmd")@@ -231,7 +234,11 @@ def test_a_failed_load_leaves_the_engine_the_user_was_using(monkeypatch):"""The sidecars order preflight before release for this reason; so does the registry."""sidecars = {name: _Sidecar(name) for name in stt_registry.STT_ENGINES}- def refuse(model, request_cancel_event = None):+ def refuse(+ model,+ request_cancel_event = None,+ device = None,+ ):raise RuntimeError("STT model 'x' is not downloaded.")sidecars["mtmd"].load = refuse@@ -314,6 +321,7 @@ def test_a_downloaded_switch_releases_the_old_engine_before_allocating(monkeypatself,model,request_cancel_event = None,+ device = None,):order.append(("load", model))@@ -340,6 +348,7 @@ def test_an_undownloaded_switch_keeps_the_resident_engine_until_the_load_succeedself,model,request_cancel_event = None,+ device = None,):order.append(("load", model))@@ -442,7 +451,10 @@ def test_an_implicit_transcribe_load_releases_the_other_engines(monkeypatch):monkeypatch.setattr(ri,"_stt_lifecycle",- lambda: (lambda model, engine, cancel = None: loaded.append((model, engine)), lambda *a: []),+ lambda: (+ lambda model, engine, cancel = None, device = None: loaded.append((model, engine)),+ lambda *a: [],+ ),)result = asyncio.run(diff --git a/studio/backend/tests/test_stt_sidecar.py b/studio/backend/tests/test_stt_sidecar.pyindex 704037fc40e..79134626c1e 100644--- a/studio/backend/tests/test_stt_sidecar.py+++ b/studio/backend/tests/test_stt_sidecar.py@@ -35,6 +35,18 @@ from core.inference.stt_sidecar import (validate_remote_model,)+[email protected](autouse = True)+def _neutral_audio_device_env(monkeypatch):+ """A server-wide default must not decide the outcome of these tests.++ Placement here is asserted against no opinion, so a host that sets+ UNSLOTH_AUDIO_DEVICE would fail these on correct behaviour, and that host is+ exactly the one most likely to run them.+ """+ monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)++_REAL_DECODE_AUDIO_BOUNDED = stt_sidecar_module._decode_audio_bounded_REAL_ENSURE_STT_AVAILABLE = stt_sidecar_module.ensure_stt_available_REAL_SNAPSHOT_IS_COMPLETE = stt_sidecar_module._snapshot_is_complete@@ -479,7 +491,9 @@ def test_load_hands_the_cached_snapshot_to_the_worker_process(monkeypatch):# never given back, which is the whole reason the engine is out of process._install_fake_torch(monkeypatch)started = _install_fake_worker(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )sidecar = WhisperSttSidecar(keep_alive_seconds = 0)sidecar.load("small")@@ -493,7 +507,9 @@ def test_load_sends_the_dtype_by_name_so_torch_stays_out_of_the_command(monkeypafake_torch = _install_fake_torch(monkeypatch)fake_torch.float16 = "torch.float16"started = _install_fake_worker(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", fake_torch.float16))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cuda", fake_torch.float16)+ )WhisperSttSidecar(keep_alive_seconds = 0).load("small")@@ -503,7 +519,9 @@ def test_load_sends_the_dtype_by_name_so_torch_stays_out_of_the_command(monkeypadef test_a_worker_that_died_is_not_resident_and_is_replaced_on_the_next_load(monkeypatch):_install_fake_torch(monkeypatch)started = _install_fake_worker(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )sidecar = WhisperSttSidecar(keep_alive_seconds = 0)worker = sidecar.load("small")@@ -522,7 +540,9 @@ def test_unload_stops_the_worker_process(monkeypatch):# empty_cache cannot return the context; ending the process is what does._install_fake_torch(monkeypatch)_install_fake_worker(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )sidecar = WhisperSttSidecar(keep_alive_seconds = 0)worker = sidecar.load("small")@@ -537,7 +557,9 @@ def test_a_worker_rejected_by_a_late_cancel_is_stopped_rather_than_leaked(monkey# start() came back with a live child. Nothing installed that child, and dropping# the handle does not end the process holding the context training waits for._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )sidecar = WhisperSttSidecar(keep_alive_seconds = 0)workers = []@@ -583,7 +605,9 @@ def test_a_late_cancelled_worker_that_outlived_the_kill_stays_resident(monkeypat# and kill and still holds its accelerator memory, so reporting nothing resident# would let training be admitted against memory that is not free._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )sidecar = WhisperSttSidecar(keep_alive_seconds = 0)workers = []@@ -667,7 +691,9 @@ def test_a_worker_that_outlived_the_kill_stays_resident_rather_than_reported_unl# still holds its memory, so reporting the model unloaded would let training be# admitted against memory that is not free._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )workers = _install_unkillable_worker(monkeypatch)sidecar = WhisperSttSidecar(keep_alive_seconds = 0)@@ -683,7 +709,9 @@ def test_a_new_load_is_refused_while_the_previous_worker_still_holds_its_memory(# Starting a second child over one that never exited doubles the memory the# first was unloaded to release, so refuse and let the caller retry._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )workers = _install_unkillable_worker(monkeypatch)sidecar = WhisperSttSidecar(keep_alive_seconds = 0)@@ -702,7 +730,9 @@ def test_a_surviving_worker_is_not_handed_to_the_next_dictation(monkeypatch):# terminate and a kill, so handing it to a transcription costs the caller the whole# command timeout under the model lock. Refuse, and let the retry kill it again._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )workers = _install_unkillable_worker(monkeypatch)sidecar = WhisperSttSidecar(keep_alive_seconds = 0)@@ -723,7 +753,9 @@ def test_a_worker_wedged_by_a_cancelled_transcription_is_not_handed_to_the_next_# close() answers False to nobody and the sidecar never learns the child outlived# both signals. Handing it on spends the whole command timeout under the model lock._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )workers = []class WedgedByCancelWorker:@@ -822,7 +854,9 @@ def test_a_child_that_outlived_the_kill_inside_start_stays_accounted(monkeypatch# installed that handle and it is the only one on the process: dropping it reports# nothing resident while the context is taken, admitting training against it._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )workers = _install_worker_that_survives_its_own_start(monkeypatch, SttLoadCancelledError("STT model loading was cancelled so training can start."))@@ -842,7 +876,9 @@ def test_a_load_whose_child_outlived_the_kill_is_not_retried_onto_a_second_child# The accelerator attempt failed leaving a live child, so the CPU retry would# put a second child beside it and, installing that one, forget the first._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", "float16"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cuda", "float16")+ )workers = _install_worker_that_survives_its_own_start(monkeypatch, RuntimeError("the dictation worker stopped responding"))@@ -861,7 +897,9 @@ def test_an_idle_unload_retries_a_worker_that_outlived_the_kill(monkeypatch):# Keeping the survivor resident must not strand it: the idle timer is rearmed# so the release is tried again once the child finally goes._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )monkeypatch.setattr(stt_sidecar_module.threading, "Timer", _FakeTimer)workers = _install_unkillable_worker(monkeypatch)@@ -917,7 +955,9 @@ def test_a_worker_whose_liveness_cannot_be_read_stays_resident(monkeypatch):# An unanswerable probe is not evidence that the context was released, so reporting# nothing resident would let training be admitted against memory still held._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )def unreadable(_worker):raise OSError("the process handle cannot be inspected")@@ -936,7 +976,9 @@ def test_a_close_that_raises_is_a_failed_release(monkeypatch):# close() raising out of join/terminate/kill confirms nothing dead, so discarding# the only handle would advertise memory the child is still holding as free._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )def raising(_worker):raise OSError("the worker could not be joined")@@ -956,7 +998,9 @@ def test_a_close_that_raises_over_a_child_already_gone_still_releases(monkeypatc# The other direction: bookkeeping that raised over a confirmed dead child# holds no memory, and keeping it would refuse every later load._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )def raising_after_death(worker):worker.alive = False@@ -980,7 +1024,9 @@ def test_a_worker_being_reaped_stays_visible_to_training_admission(monkeypatch):# lock, so clearing them before the reap would report nothing resident for the# whole close, and training would start into the child's accelerator memory._install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )release = threading.Event()def slow_close(worker):@@ -1018,7 +1064,9 @@ def test_a_host_that_cannot_spawn_keeps_dictation_in_process_on_cpu(monkeypatch)import core.inference.stt_transformers_worker as worker_module_install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", "float16"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cuda", "float16")+ )spawned = []loaded = []@@ -1084,7 +1132,9 @@ def test_load_reports_model_hub_cache_miss(monkeypatch):raise SttModelNotDownloadedError("The dictation model is not downloaded.")_install_fake_worker(monkeypatch, start = missing)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )with pytest.raises(SttModelNotDownloadedError, match = "large-v3"):WhisperSttSidecar(keep_alive_seconds = 0).load("large-v3")@@ -1109,7 +1159,7 @@ def test_unavailable_runtime_is_rejected_before_audio_decode(monkeypatch):def test_missing_model_is_rejected_before_audio_decode(monkeypatch):sidecar = WhisperSttSidecar(keep_alive_seconds = 0)- def missing(_model_id):+ def missing(_model_id, use_resident = True):raise SttModelNotDownloadedError("not downloaded")def should_not_decode(_audio):@@ -1129,7 +1179,7 @@ def test_missing_model_switch_keeps_resident_model(monkeypatch):sidecar._model_id = "small"sidecar._device = "cpu"- def missing(_model_id):+ def missing(_model_id, use_resident = True):raise SttModelNotDownloadedError("not downloaded")monkeypatch.setattr(sidecar, "_ensure_model_downloaded", missing, raising = False)@@ -1184,7 +1234,9 @@ def test_loaded_model_stays_warm_until_idle_timer_fires(monkeypatch):monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None)monkeypatch.setattr(stt_sidecar_module.threading, "Timer", make_timer)monkeypatch.setattr(sidecar, "_build_model", lambda *_args: (object(), object()))- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )sidecar.load("small")@@ -1210,7 +1262,9 @@ def test_reusing_loaded_model_refreshes_idle_timer(monkeypatch):monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None)monkeypatch.setattr(stt_sidecar_module.threading, "Timer", make_timer)monkeypatch.setattr(sidecar, "_build_model", lambda *_args: (object(), object()))- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )sidecar.load("small")first = timers[-1]@@ -1316,7 +1370,9 @@ def test_accelerator_load_failure_retries_on_cpu(monkeypatch):raise RuntimeError("accelerator allocation failed")return object(), object()- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", "float16"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cuda", "float16")+ )monkeypatch.setattr(sidecar, "_build_model", build)sidecar.load("small")@@ -1352,7 +1408,9 @@ def test_pending_load_can_be_cancelled_without_waiting_for_model_lock(monkeypatcerrors.append(exc)monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")+ )monkeypatch.setattr(sidecar, "_build_model", build)load_thread = threading.Thread(target = run_load)@@ -1873,7 +1931,9 @@ def test_preflight_rejects_partial_snapshot(monkeypatch, tmp_path, model_id):def test_cpu_retry_releases_failed_accelerator_load(monkeypatch):_install_fake_torch(monkeypatch)- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("mps", "float16"))+ monkeypatch.setattr(+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("mps", "float16")+ )class Marker:passdiff --git a/studio/backend/tests/test_stt_transcription_cancellation.py b/studio/backend/tests/test_stt_transcription_cancellation.pyindex 89657739673..fb792ed2b13 100644--- a/studio/backend/tests/test_stt_transcription_cancellation.py+++ b/studio/backend/tests/test_stt_transcription_cancellation.py@@ -121,7 +121,12 @@ def test_disconnected_load_cancels_only_its_request_event(monkeypatch):sidecar._load_owner_cancel_event = sibling_ownercancelled_request = None- def load(_model, _engine, request_cancel_event):+ def load(+ _model,+ _engine,+ request_cancel_event,+ device = None,+ ):nonlocal cancelled_requestcancelled_request = request_cancel_eventassert request_cancel_event.wait(1), "disconnect must cancel this load request"@@ -169,7 +174,12 @@ def test_disconnected_raw_transcription_cancels_its_sidecar(monkeypatch):loaded = []- def load(model, _engine, _request_cancel_event):+ def load(+ model,+ _engine,+ _request_cancel_event,+ device = None,+ ):# Stubbed for the same reason the sibling test above stubs it: the real# implicit load goes through the registry, which refuses with# SttModelNotDownloadedError (409) unless a snapshot happens to be on disk.diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.tsindex fe10bcb1a6f..53b5dfa3faf 100644--- a/studio/frontend/scripts/check-bundle-budget.ts+++ b/studio/frontend/scripts/check-bundle-budget.ts@@ -38,7 +38,10 @@ const DIST = resolve(HERE, "..", "dist");*/export const BUDGET = {// Measured 1,496.2 KB transfer / 5,207.2 KB raw at 17363f8a2.- transferBytes: 1_600_000,+ // Raised for the audio placement control: same build both sides, merge base+ // 1,560.9 KB transfer against branch 1,562.6 KB, so it crossed the old 1,562.5 KB+ // ceiling by a tenth of a kilobyte.+ transferBytes: 1_620_000,rawBytes: 5_500_000,};diff --git a/studio/frontend/src/features/audio/audio-page-policy.ts b/studio/frontend/src/features/audio/audio-page-policy.tsindex ab1eb341202..b1f489caf77 100644--- a/studio/frontend/src/features/audio/audio-page-policy.ts+++ b/studio/frontend/src/features/audio/audio-page-policy.ts@@ -267,6 +267,35 @@ export function exactGgufLoadSelector(return meta.ggufFilename ?? meta.ggufVariant ?? null;}+/** Whether a TTS pick loads through llama.cpp.+ *+ * A direct .gguf file and a GGUF repo id carry no variant filename, so the selector+ * alone misses both. `meta.isGguf` wins where a caller has it; this covers the rest.+ */+export function isGgufTtsTarget({+ repoId,+ ggufFilename,+ loadId,+ isGguf,+}: {+ repoId: string;+ ggufFilename?: string | null;+ loadId?: string | null;+ /** The catalog's own answer, when the caller has one. The tests below are+ * name heuristics, blind to a GGUF repo whose ids do not spell it. */+ isGguf?: boolean | null;+}): boolean {+ const endsWithGguf = (value: string | null | undefined): boolean =>+ Boolean(value?.toLowerCase().endsWith(".gguf"));+ return Boolean(+ isGguf ||+ ggufFilename ||+ /(?:^|[-/])gguf(?:$|[-/])/i.test(repoId) ||+ endsWithGguf(repoId) ||+ endsWithGguf(loadId),+ );+}+export type MacTtsPickAction = "allow" | "use-gguf-sibling" | "reject";/** MLX has no codec TTS decoder. Curated native PyTorch audio models bypass MLX;diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsxindex c8c5b21c700..37634e0f283 100644--- a/studio/frontend/src/features/audio/audio-page.tsx+++ b/studio/frontend/src/features/audio/audio-page.tsx@@ -86,6 +86,7 @@ import {trackSttDownload,} from "@/features/settings/lib/stt-download-mirror";import { sttModelSize } from "@/features/settings/stores/stt-model-catalog";+import { usePersistedChoice } from "@/hooks/use-persisted-choice";import { usePersistedToggle } from "@/hooks/use-persisted-toggle";import { useScrollFades } from "@/hooks/use-scroll-fades";import { fetchSystemInfo } from "@/hooks/use-system";@@ -120,6 +121,7 @@ import {canTransitionAudioMode,exactGgufLoadSelector,expectedGgufDownloadBytes,+ isGgufTtsTarget,isTtsAudioType,macTtsPickAction,mergeGalleryPage,@@ -379,6 +381,7 @@ export function AudioPage({loadId?: string | null;audioType?: string | null;remoteCodeApproval?: RemoteCodeApproval;+ isGguf?: boolean | null;} | null>(null);const ttsStatusRefreshGeneration = useRef(0);const ttsLoadGeneration = useRef(0);@@ -463,6 +466,7 @@ export function AudioPage({loadId?: string | null;audioType?: string | null;remoteCodeApproval?: RemoteCodeApproval;+ isGguf?: boolean | null;generation: number;} | null>(null);const stagedTtsLoadDeferred = useRef(false);@@ -526,6 +530,11 @@ export function AudioPage({const [advancedOpen, setAdvancedOpen] = usePersistedToggle("unsloth_audio_advanced_open",);+ // Read at load time; the handler below ejects so a change takes effect.+ const [audioDevice, setAudioDeviceState] = usePersistedChoice(+ "unsloth_audio_device",+ "auto",+ );const refreshStatus = useCallback(async () => {const generation = ++ttsStatusRefreshGeneration.current;@@ -894,6 +903,8 @@ export function AudioPage({loadId?: string | null,audioType?: string | null,remoteCodeApproval?: RemoteCodeApproval,+ // The catalog's answer: the ids alone miss a GGUF repo that does not spell it.+ isGguf?: boolean | null,) => {// A routed pick that arrives while a previous load is still tearing down would// otherwise be dropped here, and the route effect has already cleared ?model=, so@@ -905,6 +916,7 @@ export function AudioPage({loadId,audioType,remoteCodeApproval,+ isGguf,};return;}@@ -944,6 +956,7 @@ export function AudioPage({loadId,audioType,remoteCodeApproval,+ isGguf,};return;}@@ -993,6 +1006,8 @@ export function AudioPage({);if (controller.signal.aborted || !isCurrent()) return;}+ const wantsCpu = audioDevice === "cpu";+ const isGgufLoad = isGgufTtsTarget({ repoId, ggufFilename, loadId, isGguf });const res = await loadModel({model_path: loadId || repoId,@@ -1005,6 +1020,18 @@ export function AudioPage({gguf_variant: ggufFilename ?? null,trust_remote_code: trustRemoteCode,approved_remote_code_fingerprint: approvedRemoteCodeFingerprint,+ audio_device: wantsCpu ? "cpu" : "auto",+ // GGUF ignores audio_device: llama.cpp offloads unless told not to.+ // An absent speculative_type resolves to "auto", which may attach a GPU+ // drafter, and the backend then evicts image/video for a CPU load.+ ...(wantsCpu && isGgufLoad+ ? // biome-ignore lint/style/useNamingConvention: API schema+ {+ gpu_memory_mode: "manual" as const,+ gpu_layers: 0,+ speculative_type: "off" as const,+ }+ : {}),},{signal: controller.signal,@@ -1026,6 +1053,17 @@ export function AudioPage({toast.success(`Model loaded (${res.audio_type ?? "audio"})`, {id: toastId,});+ // Only the native runtime and GGUF can be held in RAM.+ if (+ wantsCpu &&+ !isGgufLoad &&+ !usesNativeAudioRuntime(repoId, res.audio_type)+ ) {+ toast.info(+ "This model does not support CPU RAM yet, so it loaded on the GPU.",+ { duration: 6000 },+ );+ }} else {toast.error(`${repoId} loaded but is not a supported TTS model.`, {id: toastId,@@ -1056,7 +1094,7 @@ export function AudioPage({if (activeRef.current) replayQueuedTtsPick();}},- [refreshStatus],+ [refreshStatus, audioDevice],);// Stage uncached Hub GGUFs through the shared manager so Audio gets the same@@ -1075,6 +1113,7 @@ export function AudioPage({queued.loadId,queued.audioType,queued.remoteCodeApproval,+ queued.isGguf,);}, []);const invalidatePendingStagedTts = useCallback(() => {@@ -1173,6 +1212,7 @@ export function AudioPage({pending.loadId,pending.audioType,pending.remoteCodeApproval,+ pending.isGguf,);},});@@ -1199,6 +1239,7 @@ export function AudioPage({pending.loadId,pending.audioType,pending.remoteCodeApproval,+ pending.isGguf,);}, [active, busy]);@@ -1268,6 +1309,7 @@ export function AudioPage({meta.loadId,meta.audioType,remoteCodeApproval,+ meta.isGguf,);return;}@@ -1287,6 +1329,7 @@ export function AudioPage({loadId: meta.loadId,audioType: meta.audioType,remoteCodeApproval,+ isGguf: meta.isGguf,generation,};stageTtsDownload(@@ -1313,6 +1356,7 @@ export function AudioPage({loadId: meta.loadId,audioType: meta.audioType,remoteCodeApproval,+ isGguf: meta.isGguf,generation,};stageTtsDownload([@@ -1334,6 +1378,7 @@ export function AudioPage({meta.loadId,meta.audioType,remoteCodeApproval,+ meta.isGguf,);},[stageTtsDownload],@@ -1576,10 +1621,7 @@ export function AudioPage({if (ttsPickGeneration.current !== selectionGeneration) return;const exactGguf = exactGgufLoadSelector(meta);const isGguf = Boolean(- meta.isGguf ||- exactGguf ||- /(?:^|[-/])gguf(?:$|[-/])/i.test(id) ||- id.toLowerCase().endsWith(".gguf"),+ meta.isGguf || isGgufTtsTarget({ repoId: id, ggufFilename: exactGguf }),);const ggufSibling = isGguf ? null : ggufSiblingFor(id);const nativeRuntime =@@ -2629,6 +2671,47 @@ export function AudioPage({/></Field>) : null}+ {/* Field inlined: its label needs a form control to point+ at, and PillTabs is a tablist with its own name. */}+ <div className="grid gap-1.5">+ <span className="text-ui-13 font-medium text-foreground">+ Load model into+ </span>+ <PillTabs+ ariaLabel="Load model into"+ value={audioDevice === "cpu" ? "cpu" : "auto"}+ // The eject below applies the change and cannot interrupt a load.+ disabled={busy !== null || isRecording}+ onValueChange={(value) => {+ const next = value === "cpu" ? "cpu" : "auto";+ if (next === audioDevice) return;+ // MiniMax needs CUDA, and the backend's refusal cannot save a+ // model already ejected here.+ if (next === "cpu" && status?.audio_type === "minimax_music3") {+ toast.info(+ "MiniMax Music 3 needs a GPU, so it cannot be held in CPU RAM.",+ );+ return;+ }+ setAudioDeviceState(next);+ if (ttsLoaded) handleEject();+ }}+ fit={true}+ className="h-[30px] self-start [&>button]:h-[30px] [&>button]:px-6"+ tabs={[+ { value: "auto", label: "GPU when available" },+ { value: "cpu", label: "CPU RAM" },+ ]}+ />+ {/* Phrased as what the next load will do, not as the resident+ model's state: a model loaded by another tab or client can+ be on the other device, and status does not report it. */}+ <p className="text-ui-11p5 leading-snug text-muted-foreground">+ {audioDevice === "cpu"+ ? "New loads go into system RAM instead of the GPU. Slower to generate, and no GPU memory is used."+ : "New loads use the GPU when there is one, and the CPU otherwise."}+ </p>+ </div><AdvancedDisclosureopen={advancedOpen}onOpenChange={setAdvancedOpen}diff --git a/studio/frontend/src/features/chat/adapters/studio-model-dictation-adapter.ts b/studio/frontend/src/features/chat/adapters/studio-model-dictation-adapter.tsindex dfdc2c7b472..c612f5d196a 100644--- a/studio/frontend/src/features/chat/adapters/studio-model-dictation-adapter.ts+++ b/studio/frontend/src/features/chat/adapters/studio-model-dictation-adapter.ts@@ -8,6 +8,7 @@ import { useSettingsDialogStore } from "@/features/settings/stores/settings-dialimport { requestSttDownload } from "@/features/settings/stores/stt-download-prompt-store";import {MTMD_STT_MODELS,+ type SttDevice,applyDictationDictionary,isCuratedSttModel,recordRecentDictation,@@ -181,6 +182,7 @@ export async function transcribeAudioBlob(const engine = options.engine ?? sttEngineFor(model);const params = new URLSearchParams({ model, fast: "true", engine });if (language) params.set("language", language);+ params.set("device", settings.sttDevice);const response = await authFetch(`/api/inference/audio/transcribe/raw?${params.toString()}`,{@@ -309,15 +311,21 @@ export function loadSttModel(model: string,engine?: SttEngine,signal?: AbortSignal,+ device?: SttDevice,): Promise<void> {const resolvedEngine = engine ?? sttEngineFor(model);+ const resolvedDevice = device ?? useVoiceSettingsStore.getState().sttDevice;// Announced so the indicator shows the load immediately, as the toast does.return queueSttLifecycle(() =>withModelLoadNotice("stt", model, async () => {const response = await authFetch("/api/inference/audio/stt/load", {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ model, engine: resolvedEngine }),+ body: JSON.stringify({+ model,+ engine: resolvedEngine,+ device: resolvedDevice,+ }),signal,});if (!response.ok) {@@ -380,11 +388,15 @@ export async function cancelSttDownload(export function unloadSttModel(engine?: SttEngine,model?: string,+ options?: { wait?: boolean },): Promise<void> {return queueSttLifecycle(async () => {const params = new URLSearchParams();if (engine) params.set("engine", engine);if (model) params.set("model", model);+ // Opt-out only: the default drains an in-flight transcription, right when the+ // caller needs the memory back now.+ if (options?.wait === false) params.set("wait", "false");const query = params.size ? `?${params}` : "";const response = await authFetch(`/api/inference/audio/stt/unload${query}`,diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.tsindex 7f6956af277..caa70a7e5d6 100644--- a/studio/frontend/src/features/chat/api/chat-api.ts+++ b/studio/frontend/src/features/chat/api/chat-api.ts@@ -370,6 +370,8 @@ export async function validateModel(tensor_parallel: payload.tensor_parallel ?? false,disable_vision: payload.disable_vision ?? false,gpu_ids: payload.gpu_ids,+ // Takes no VRAM, so validate must not preflight it and refuse what /load takes.+ audio_device: payload.audio_device ?? null,// Manual placement is an explicit override: Auto layers use llama.cpp --fit, a pinned// layer count is owned by the user. Tell validate so it applies the same policy as /load.gpu_memory_mode: payload.gpu_memory_mode,diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 65d464601b2..deb34e9bbc8 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -136,6 +136,9 @@ export interface LoadModelRequest {tensor_split?: number[] | null;/** Picked CUDA/ROCm physical IDs or Vulkan ordinals (omit/empty = automatic). */gpu_ids?: number[];+ /** Native audio (TTS / music) only: "cpu" holds the weights in system RAM+ * rather than the GPU. Ignored for every non-audio model. */+ audio_device?: "auto" | "cpu" | "gpu";}export interface ValidateModelResponse {diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsxindex 472e661a4da..81c5e660eae 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx+++ b/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx@@ -21,6 +21,7 @@ export function PillTabs({className,compact = false,fit = false,+ disabled = false,}: {tabs: PillTab[];value: string;@@ -28,6 +29,8 @@ export function PillTabs({ariaLabel: string;className?: string;compact?: boolean;+ /** Block every tab, for a choice that cannot be applied right now. */+ disabled?: boolean;/** Size each tab to its label instead of equal widths. The active tab carries* the pill background directly (the toggle never animates). Tabs only shrink* when their combined intrinsic width exceeds the available space. */@@ -70,6 +73,7 @@ export function PillTabs({// Left/Right move between tabs (WAI-ARIA tablist pattern). ArrowDown// is left to bubble so the picker's "enter the list" handler still runs.tabIndex={value === tab.value ? 0 : -1}+ disabled={disabled}onKeyDown={(e) => {if (e.key !== "ArrowRight" && e.key !== "ArrowLeft") return;e.preventDefault();@@ -85,6 +89,7 @@ export function PillTabs({onClick={() => onValueChange(tab.value)}className={cn("relative z-10 inline-flex items-center justify-center gap-1.5 rounded-full transition-colors",+ disabled && "cursor-not-allowed opacity-50",fit ? "min-w-0 shrink" : "min-w-0 flex-1",compact? "h-7 px-2.5 text-ui-11"diff --git a/studio/frontend/src/features/settings/stores/voice-settings-store.ts b/studio/frontend/src/features/settings/stores/voice-settings-store.tsindex 3e1845d0d20..28ec3a7c6ce 100644--- a/studio/frontend/src/features/settings/stores/voice-settings-store.ts+++ b/studio/frontend/src/features/settings/stores/voice-settings-store.ts@@ -80,6 +80,16 @@ export function isSttModelLanguageCompatible(export type DictationEngine = "browser" | "model" | "custom";+/** Mirrors the backend's `SttLoadRequest.device`. "gpu" is not offered in the+ * UI: it differs from "auto" only where it cannot be honoured anyway. */+export type SttDevice = "auto" | "cpu";++export const DEFAULT_STT_DEVICE: SttDevice = "auto";++function normalizeSttDevice(value: unknown): SttDevice {+ return value === "cpu" ? "cpu" : DEFAULT_STT_DEVICE;+}+export type TtsEngine = "system" | "studio" | "custom";/**@@ -107,6 +117,11 @@ export interface VoiceSettingsState {sttModel: SttModel;setSttModel: (value: SttModel) => void;+ /** "cpu" holds the dictation model in system RAM instead of the GPU. Sent+ * with every load and transcribe, so a change applies on the next load. */+ sttDevice: SttDevice;+ setSttDevice: (value: SttDevice) => void;+sttProviderId: string;setSttProviderId: (value: string) => void;sttProviderModel: string;@@ -225,6 +240,9 @@ export const useVoiceSettingsStore = create<VoiceSettingsState>()(};}),+ sttDevice: DEFAULT_STT_DEVICE,+ setSttDevice: (value) => set({ sttDevice: normalizeSttDevice(value) }),+sttProviderId: "",setSttProviderId: (sttProviderId) => set({ sttProviderId }),sttProviderModel: "",@@ -362,6 +380,7 @@ export const useVoiceSettingsStore = create<VoiceSettingsState>()(micDeviceId: asString(saved?.micDeviceId, "default"),dictationEngine,sttModel,+ sttDevice: normalizeSttDevice(saved?.sttDevice),sttProviderId: asString(saved?.sttProviderId, ""),sttProviderModel: asString(saved?.sttProviderModel, ""),dictationLanguage,diff --git a/studio/frontend/src/features/settings/tabs/voice-tab.tsx b/studio/frontend/src/features/settings/tabs/voice-tab.tsxindex 278ec78d985..ce93f44cba7 100644--- a/studio/frontend/src/features/settings/tabs/voice-tab.tsx+++ b/studio/frontend/src/features/settings/tabs/voice-tab.tsx@@ -30,6 +30,7 @@ import {loadSttModel,releaseTtsAudioUrl,startSttDownload,+ sttEngineFor,unloadSttModel,useExternalProvidersStore,validateSttModel,@@ -406,6 +407,9 @@ export function VoiceTab() {const setDictationEngine = useVoiceSettingsStore((s) => s.setDictationEngine);const sttModel = useVoiceSettingsStore((s) => s.sttModel);const setSttModel = useVoiceSettingsStore((s) => s.setSttModel);+ // Named apart from the `sttDevice` state below, which the sidecar reports back.+ const sttDevicePreference = useVoiceSettingsStore((s) => s.sttDevice);+ const setSttDevicePreference = useVoiceSettingsStore((s) => s.setSttDevice);const sttProviderId = useVoiceSettingsStore((s) => s.sttProviderId);const setSttProviderId = useVoiceSettingsStore((s) => s.setSttProviderId);const sttProviderModel = useVoiceSettingsStore((s) => s.sttProviderModel);@@ -1130,6 +1134,51 @@ export function VoiceTab() {)) : null}+ {isLocalEngine && modelSttSupported ? (+ <SettingsRow+ label={t("settings.voice.dictation.sttDeviceLabel")}+ description={+ sttDevicePreference === "cpu"+ ? t("settings.voice.dictation.sttDeviceCpuDescription")+ : t("settings.voice.dictation.sttDeviceAutoDescription")+ }+ >+ <Select+ value={sttDevicePreference}+ onValueChange={(value) => {+ const next = value === "cpu" ? "cpu" : "auto";+ if (next === sttDevicePreference) return;+ // Scoped, so moving our placement cannot evict a model another surface+ // swapped in. wait:false so a decoding dictation is not killed for a+ // setting the next load applies anyway.+ void unloadSttModel(sttEngineFor(sttModel), sttModel, {+ wait: false,+ }).catch(() => {});+ setSttPhase("on-demand");+ setSttDevice(null);+ setSttDevicePreference(next);+ }}+ >+ <SelectTrigger+ data-testid="stt-device-trigger"+ aria-label={t("settings.voice.dictation.sttDeviceLabel")}+ className="w-56"+ size="sm"+ >+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value="auto" data-testid="stt-device-auto">+ {t("settings.voice.dictation.sttDeviceAuto")}+ </SelectItem>+ <SelectItem value="cpu" data-testid="stt-device-cpu">+ {t("settings.voice.dictation.sttDeviceCpu")}+ </SelectItem>+ </SelectContent>+ </Select>+ </SettingsRow>+ ) : null}+<SettingsRowlabel={t("settings.voice.dictation.microphoneLabel")}description={diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.tsindex 609c63fab76..8347defe5f9 100644--- a/studio/frontend/src/i18n/locales/ar.ts+++ b/studio/frontend/src/i18n/locales/ar.ts@@ -491,6 +491,13 @@ export const ar = {customModelDescription: "اسم النموذج المرسل إلى /v1/audio/transcriptions.",sttModelLabel: "نموذج التعرّف على الكلام",sttModelDescription: "اختر نموذج STT أو ابحث عنه لتشغيله محليًا.",+ sttDeviceLabel: "التحميل إلى",+ sttDeviceAuto: "وحدة معالجة الرسومات عند توفرها",+ sttDeviceCpu: "ذاكرة المعالج",+ sttDeviceAutoDescription:+ "استخدم وحدة معالجة الرسومات إن وُجدت، وإلا فالمعالج.",+ sttDeviceCpuDescription:+ "احتفظ بالنموذج في ذاكرة النظام. النسخ أبطأ، لكن دون استخدام ذاكرة الرسومات.",sttModelSearchPlaceholder: "ابحث عن نموذج",sttModelSearching: "جارٍ البحث في Hugging Face…",sttModelValidating: "جارٍ التحقق من التوافق مع Whisper…",diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.tsindex 4d064dd3f4e..fbc32a2da04 100644--- a/studio/frontend/src/i18n/locales/de.ts+++ b/studio/frontend/src/i18n/locales/de.ts@@ -497,6 +497,13 @@ export const de = {sttModelLabel: "Spracherkennungsmodell",sttModelDescription:"Wählen oder suchen Sie ein STT-Modell für die lokale Ausführung.",+ sttDeviceLabel: "Laden in",+ sttDeviceAuto: "GPU, wenn verfügbar",+ sttDeviceCpu: "CPU-RAM",+ sttDeviceAutoDescription:+ "Die GPU verwenden, sofern vorhanden, sonst die CPU.",+ sttDeviceCpuDescription:+ "Das Modell im Arbeitsspeicher halten. Die Transkription ist langsamer, belegt aber keinen GPU-Speicher.",sttModelSearchPlaceholder: "Modell suchen",sttModelSearching: "Hugging Face wird durchsucht…",sttModelValidating: "Whisper-Kompatibilität wird geprüft…",diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.tsindex 956f6990fa5..915435cce37 100644--- a/studio/frontend/src/i18n/locales/en.ts+++ b/studio/frontend/src/i18n/locales/en.ts@@ -493,6 +493,13 @@ export const en = {customModelDescription: "Model name sent to /v1/audio/transcriptions.",sttModelLabel: "Speech recognition model",sttModelDescription: "Choose or search a STT model to run locally.",+ sttDeviceLabel: "Load into",+ sttDeviceAuto: "GPU when available",+ sttDeviceCpu: "CPU RAM",+ sttDeviceAutoDescription:+ "Use the GPU when there is one, and the CPU otherwise.",+ sttDeviceCpuDescription:+ "Keep the model in system RAM. Transcription is slower, but no GPU memory is used.",sttModelSearchPlaceholder: "Search any model on HF",sttModelSearching: "Searching Hugging Face…",sttModelValidating: "Checking Whisper compatibility…",diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.tsindex 6120f03d59a..5b9f607eb33 100644--- a/studio/frontend/src/i18n/locales/es.ts+++ b/studio/frontend/src/i18n/locales/es.ts@@ -496,6 +496,13 @@ export const es = {sttModelLabel: "Modelo de reconocimiento de voz",sttModelDescription:"Elige o busca un modelo STT para ejecutarlo en local.",+ sttDeviceLabel: "Cargar en",+ sttDeviceAuto: "GPU cuando esté disponible",+ sttDeviceCpu: "RAM de la CPU",+ sttDeviceAutoDescription:+ "Usar la GPU cuando haya una y, si no, la CPU.",+ sttDeviceCpuDescription:+ "Mantener el modelo en la RAM del sistema. La transcripción es más lenta, pero no usa memoria de la GPU.",sttModelSearchPlaceholder: "Buscar modelo",sttModelSearching: "Buscando en Hugging Face…",sttModelValidating: "Comprobando la compatibilidad con Whisper…",diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.tsindex 3222489f5cc..ce5ee71a352 100644--- a/studio/frontend/src/i18n/locales/fr.ts+++ b/studio/frontend/src/i18n/locales/fr.ts@@ -497,6 +497,13 @@ export const fr = {sttModelLabel: "Modèle de reconnaissance vocale",sttModelDescription:"Choisissez ou recherchez un modèle STT à exécuter en local.",+ sttDeviceLabel: "Charger dans",+ sttDeviceAuto: "GPU si disponible",+ sttDeviceCpu: "RAM du CPU",+ sttDeviceAutoDescription:+ "Utiliser le GPU quand il y en a un, sinon le CPU.",+ sttDeviceCpuDescription:+ "Garder le modèle dans la RAM système. La transcription est plus lente, mais n'utilise pas de mémoire GPU.",sttModelSearchPlaceholder: "Rechercher un modèle",sttModelSearching: "Recherche sur Hugging Face…",sttModelValidating: "Vérification de la compatibilité Whisper…",diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.tsindex 3f2734b2ed0..22646e2e262 100644--- a/studio/frontend/src/i18n/locales/hi.ts+++ b/studio/frontend/src/i18n/locales/hi.ts@@ -495,6 +495,13 @@ export const hi = {"/v1/audio/transcriptions को भेजा जाने वाला मॉडल नाम।",sttModelLabel: "स्पीच रिकग्निशन मॉडल",sttModelDescription: "लोकली चलाने के लिए STT मॉडल चुनें या खोजें।",+ sttDeviceLabel: "इसमें लोड करें",+ sttDeviceAuto: "उपलब्ध होने पर GPU",+ sttDeviceCpu: "CPU RAM",+ sttDeviceAutoDescription:+ "GPU उपलब्ध हो तो उसका उपयोग करें, अन्यथा CPU का।",+ sttDeviceCpuDescription:+ "मॉडल को सिस्टम RAM में रखें। ट्रांसक्रिप्शन धीमा होगा, पर GPU मेमोरी नहीं लगेगी।",sttModelSearchPlaceholder: "मॉडल खोजें",sttModelSearching: "Hugging Face पर खोजा जा रहा है…",sttModelValidating: "Whisper संगतता जाँची जा रही है…",diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.tsindex ef8a5eb2e36..bb3aa7d2c2d 100644--- a/studio/frontend/src/i18n/locales/it.ts+++ b/studio/frontend/src/i18n/locales/it.ts@@ -462,6 +462,13 @@ export const it = {sttModelLabel: "Modello di riconoscimento vocale",sttModelDescription:"Scegli o cerca un modello STT da eseguire in locale.",+ sttDeviceLabel: "Carica in",+ sttDeviceAuto: "GPU quando disponibile",+ sttDeviceCpu: "RAM della CPU",+ sttDeviceAutoDescription:+ "Usa la GPU quando c'è, altrimenti la CPU.",+ sttDeviceCpuDescription:+ "Tieni il modello nella RAM di sistema. La trascrizione è più lenta, ma non usa memoria GPU.",sttModelSearchPlaceholder: "Cerca un modello",sttModelSearching: "Ricerca su Hugging Face…",sttModelValidating: "Verifica della compatibilità con Whisper…",diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.tsindex 4ef4cb463e9..f5b12fc7eb8 100644--- a/studio/frontend/src/i18n/locales/ja.ts+++ b/studio/frontend/src/i18n/locales/ja.ts@@ -495,6 +495,13 @@ export const ja = {sttModelLabel: "音声認識モデル",sttModelDescription:"ローカルで実行する STT モデルを選択または検索します。",+ sttDeviceLabel: "読み込み先",+ sttDeviceAuto: "GPU(利用できる場合)",+ sttDeviceCpu: "CPU メモリ",+ sttDeviceAutoDescription:+ "GPU があれば GPU を、なければ CPU を使用します。",+ sttDeviceCpuDescription:+ "モデルをシステムメモリに保持します。文字起こしは遅くなりますが、GPU メモリを使いません。",sttModelSearchPlaceholder: "モデルを検索",sttModelSearching: "Hugging Face を検索中…",sttModelValidating: "Whisper との互換性を確認中…",diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.tsindex 24bef3b8ec9..1a58caa866e 100644--- a/studio/frontend/src/i18n/locales/ko.ts+++ b/studio/frontend/src/i18n/locales/ko.ts@@ -492,6 +492,13 @@ export const ko = {"/v1/audio/transcriptions에 전송할 모델 이름입니다.",sttModelLabel: "음성 인식 모델",sttModelDescription: "로컬에서 실행할 STT 모델을 선택하거나 검색하세요.",+ sttDeviceLabel: "로드 위치",+ sttDeviceAuto: "GPU(사용 가능할 때)",+ sttDeviceCpu: "CPU RAM",+ sttDeviceAutoDescription:+ "GPU가 있으면 GPU를, 없으면 CPU를 사용합니다.",+ sttDeviceCpuDescription:+ "모델을 시스템 RAM에 유지합니다. 변환은 느리지만 GPU 메모리를 쓰지 않습니다.",sttModelSearchPlaceholder: "모델 검색",sttModelSearching: "Hugging Face에서 검색 중…",sttModelValidating: "Whisper 호환성 확인 중…",diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.tsindex 9c9bb570c3a..4e8cb7aefc3 100644--- a/studio/frontend/src/i18n/locales/pt-br.ts+++ b/studio/frontend/src/i18n/locales/pt-br.ts@@ -495,6 +495,13 @@ export const ptBR = {sttModelLabel: "Modelo de reconhecimento de fala",sttModelDescription:"Escolha ou busque um modelo STT para executar localmente.",+ sttDeviceLabel: "Carregar em",+ sttDeviceAuto: "GPU quando disponível",+ sttDeviceCpu: "RAM da CPU",+ sttDeviceAutoDescription:+ "Usar a GPU quando houver uma e, caso contrário, a CPU.",+ sttDeviceCpuDescription:+ "Manter o modelo na RAM do sistema. A transcrição fica mais lenta, mas não usa memória da GPU.",sttModelSearchPlaceholder: "Buscar modelo",sttModelSearching: "Buscando no Hugging Face…",sttModelValidating: "Verificando a compatibilidade com o Whisper…",diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.tsindex ff30d75409d..65703730ca1 100644--- a/studio/frontend/src/i18n/locales/ru.ts+++ b/studio/frontend/src/i18n/locales/ru.ts@@ -495,6 +495,13 @@ export const ru = {sttModelLabel: "Модель распознавания речи",sttModelDescription:"Выберите или найдите модель STT для локального запуска.",+ sttDeviceLabel: "Загружать в",+ sttDeviceAuto: "GPU, если доступен",+ sttDeviceCpu: "ОЗУ процессора",+ sttDeviceAutoDescription:+ "Использовать GPU, когда он есть, иначе CPU.",+ sttDeviceCpuDescription:+ "Держать модель в системной памяти. Расшифровка медленнее, но память GPU не занимается.",sttModelSearchPlaceholder: "Поиск модели",sttModelSearching: "Поиск на Hugging Face…",sttModelValidating: "Проверка совместимости с Whisper…",diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.tsindex 16f07e270a2..83a452eff4c 100644--- a/studio/frontend/src/i18n/locales/zh-CN.ts+++ b/studio/frontend/src/i18n/locales/zh-CN.ts@@ -490,6 +490,12 @@ export const zhCN = {"发送到 /v1/audio/transcriptions 的模型名称。",sttModelLabel: "语音识别模型",sttModelDescription: "选择或搜索要在本地运行的 STT 模型。",+ sttDeviceLabel: "加载到",+ sttDeviceAuto: "有 GPU 时使用 GPU",+ sttDeviceCpu: "CPU 内存",+ sttDeviceAutoDescription: "有 GPU 时使用 GPU,否则使用 CPU。",+ sttDeviceCpuDescription:+ "将模型保留在系统内存中。转写速度较慢,但不占用显存。",sttModelSearchPlaceholder: "搜索模型",sttModelSearching: "正在搜索 Hugging Face…",sttModelValidating: "正在检查 Whisper 兼容性…",diff --git a/studio/frontend/tests/audio-model-eject.test.ts b/studio/frontend/tests/audio-model-eject.test.tsindex 115ef887c9e..ef6b1f542c2 100644--- a/studio/frontend/tests/audio-model-eject.test.ts+++ b/studio/frontend/tests/audio-model-eject.test.ts@@ -67,7 +67,7 @@ test("a Speak load asks the same question and forces from the answer", () => {// The slot is claimed before the await, so a routed pick arriving mid-dialog queues.assert.match(source,- /if \(ttsLoadInFlight\.current \|\| busyRef\.current === "generating"\) \{\s*pendingRoutedTtsPick\.current = \{\s*repoId,\s*ggufFilename,\s*loadId,\s*audioType,\s*remoteCodeApproval,\s*\};\s*return;\s*\}[\s\S]{0,400}?ttsLoadInFlight\.current = true;/,+ /if \(ttsLoadInFlight\.current \|\| busyRef\.current === "generating"\) \{\s*pendingRoutedTtsPick\.current = \{\s*repoId,\s*ggufFilename,\s*loadId,\s*audioType,\s*remoteCodeApproval,\s*isGguf,\s*\};\s*return;\s*\}[\s\S]{0,400}?ttsLoadInFlight\.current = true;/,);// Declining releases the slot and drops the queued pick, which would else re-ask.assert.match(@@ -124,7 +124,7 @@ test("a load confirmed after Audio is hidden is deferred, not sent", () => {// nothing to abort. Sending anyway let a hidden page replace the visible page's model.assert.match(source,- /if \(!activeRef\.current\) \{\s*releaseLifecycle\(\);\s*ttsLoadInFlight\.current = false;\s*pendingRoutedTtsPick\.current = \{\s*repoId,\s*ggufFilename,\s*loadId,\s*audioType,\s*remoteCodeApproval,\s*\};\s*return;\s*\}/,+ /if \(!activeRef\.current\) \{\s*releaseLifecycle\(\);\s*ttsLoadInFlight\.current = false;\s*pendingRoutedTtsPick\.current = \{\s*repoId,\s*ggufFilename,\s*loadId,\s*audioType,\s*remoteCodeApproval,\s*isGguf,\s*\};\s*return;\s*\}/,);// The activation effect replays exactly that queue, so the pick is not lost.assert.match(diff --git a/studio/frontend/tests/audio-page-policy.test.ts b/studio/frontend/tests/audio-page-policy.test.tsindex 6d6f6883aad..49dcf2de7f3 100644--- a/studio/frontend/tests/audio-page-policy.test.ts+++ b/studio/frontend/tests/audio-page-policy.test.ts@@ -14,6 +14,7 @@ import {canTransitionAudioMode,exactGgufLoadSelector,expectedGgufDownloadBytes,+ isGgufTtsTarget,isTtsAudioType,macTtsPickAction,mergeGalleryPage,@@ -735,3 +736,65 @@ test("a capped restore refresh invalidates a page fetched from the older cursor"/const cursor = galleryCache\.nextCursor;\s*try\s*{\s*const page = await listAudioGallery\(\s*0,\s*PAGE_SIZE,\s*cursor,?\s*\);\s*if \(\s*refreshGeneration !== galleryRefreshGeneration\.current \|\|\s*cursor !== galleryCache\.nextCursor\s*\)\s*return;/,);});++test("a direct .gguf pick is a GGUF target even without a variant filename", () => {+ // Local direct rows supply neither ggufFilename nor ggufVariant, so a check on the+ // selector alone left them on GPU offload.+ assert.equal(+ isGgufTtsTarget({ repoId: "/models/orpheus-3b-Q4_K_M.gguf" }),+ true,+ );+ assert.equal(+ isGgufTtsTarget({ repoId: "Orpheus TTS", loadId: "/m/x.GGUF" }),+ true,+ );+ assert.equal(isGgufTtsTarget({ repoId: "unsloth/orpheus-3b-0.1-ft-GGUF" }), true);+ assert.equal(+ isGgufTtsTarget({ repoId: "unsloth/orpheus", ggufFilename: "x-Q4.gguf" }),+ true,+ );+});++test("the catalog's own answer outranks the name heuristics", () => {+ // Invisible to every test below; only the catalog knows. Losing it dropped offload.+ assert.equal(+ isGgufTtsTarget({ repoId: "acme/voicebox", isGguf: true }),+ true,+ );+ assert.equal(+ isGgufTtsTarget({ repoId: "acme/voicebox-GGUF", isGguf: false }),+ true,+ );+ assert.equal(+ isGgufTtsTarget({ repoId: "acme/voicebox", isGguf: null }),+ false,+ );+});++test("a safetensors pick is not a GGUF target", () => {+ assert.equal(isGgufTtsTarget({ repoId: "unsloth/orpheus-3b-0.1-ft" }), false);+ assert.equal(isGgufTtsTarget({ repoId: "bosonai/higgs-tts-2-3b-base" }), false);+ // "gguf" only as a bare path segment, so a name merely containing it does not match.+ assert.equal(isGgufTtsTarget({ repoId: "acme/ggufology" }), false);+});++test("a CPU GGUF audio load declares speculation off", () => {+ // An absent speculative_type resolves to "auto", which can attach a GPU drafter;+ // zero_vram_chat_load then takes the arbiter and cancels an image or video job.+ assert.match(+ audioPageSource,+ /gpu_memory_mode: "manual" as const,\s*gpu_layers: 0,\s*speculative_type: "off" as const,/,+ );+});++test("selecting CPU never ejects a resident MiniMax, which cannot load on CPU", () => {+ // The backend's refusal cannot help once ejected: recovery needs the refused load.+ const handler = audioPageSource.slice(+ audioPageSource.indexOf('const next = value === "cpu" ? "cpu" : "auto";'),+ );+ const guard = handler.indexOf('status?.audio_type === "minimax_music3"');+ const eject = handler.indexOf("handleEject()");+ assert.ok(guard > -1, "no MiniMax guard on the placement control");+ assert.ok(guard < eject, "the guard must return before the eject");+ assert.match(handler.slice(guard, eject), /return;/);+});diff --git a/studio/frontend/tests/voice-stt-device-control.test.ts b/studio/frontend/tests/voice-stt-device-control.test.tsnew file mode 100644index 00000000000..9903ed14baf--- /dev/null+++ b/studio/frontend/tests/voice-stt-device-control.test.ts@@ -0,0 +1,46 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import test from "node:test";++const voiceTab = readFileSync(+ new URL("../src/features/settings/tabs/voice-tab.tsx", import.meta.url),+ "utf8",+);+const adapter = readFileSync(+ new URL(+ "../src/features/chat/adapters/studio-model-dictation-adapter.ts",+ import.meta.url,+ ),+ "utf8",+);++test("changing the dictation device releases only this tab's model", () => {+ // Unscoped, the unload can tear down a model this tab never owned.+ assert.match(+ voiceTab,+ /void unloadSttModel\(sttEngineFor\(sttModel\), sttModel, \{\s*wait: false,\s*\}\)/,+ );+});++test("switching device never kills a transcription being decoded", () => {+ // The default drains for 30s and releases anyway, throwing the recording away.+ assert.match(voiceTab, /wait: false/);+ assert.match(adapter, /if \(options\?\.wait === false\) params\.set\("wait", "false"\)/);+});++test("the unload API still takes the engine and model that scoping needs", () => {+ assert.match(+ adapter,+ /export function unloadSttModel\(\s*engine\?: SttEngine,\s*model\?: string,\s*options\?: \{ wait\?: boolean \},\s*\)/,+ );+ assert.match(adapter, /if \(model\) params\.set\("model", model\)/);+});++test("the device preference travels with every load and transcribe", () => {+ // A load that omits it reads as "no opinion", so the setting never applies.+ assert.match(adapter, /device: resolvedDevice/);+ assert.match(adapter, /params\.set\("device", settings\.sttDevice\)/);+});