Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Let audio models load into CPU RAM instead of the GPU (#10054)

Michael Han03c849f586f6
diff --git a/studio/backend/core/inference/audio_codecs.py b/studio/backend/core/inference/audio_codecs.py
index 0a4aa78d061..89ba21284f8 100644
--- a/studio/backend/core/inference/audio_codecs.py
+++ b/studio/backend/core/inference/audio_codecs.py
@@ -56,6 +56,9 @@ class AudioCodecManager:
self._bicodec_code_dir = None
self._dac_audio_codec = None
self._outetts_code_dir = None
+ # The loaders reuse a resident codec, so a later request gets the first
+ # placement rather than the one it asked for.
+ self._codec_devices: dict = {}
def load_codec(
self,
@@ -89,6 +92,7 @@ class AudioCodecManager:
.to(device)
.eval()
)
+ self._codec_devices["snac"] = device
logger.info("Loaded SNAC codec (24kHz)")
def _load_bicodec(
@@ -109,6 +113,7 @@ class AudioCodecManager:
tokenizer_path = model_repo_path or spark_code_dir
self._bicodec_repo_path = tokenizer_path
self._bicodec_tokenizer = BiCodecTokenizer(tokenizer_path, device)
+ self._codec_devices["bicodec"] = device
logger.info(f"Loaded BiCodec tokenizer from {tokenizer_path}")
def _load_dac(self, device: str) -> None:
@@ -133,6 +138,7 @@ class AudioCodecManager:
)
processor = AudioProcessor(config = dummy_config)
self._dac_audio_codec = processor.audio_codec
+ self._codec_devices["dac"] = device
logger.info("Loaded DAC audio codec")
# ── Decoders ─────────────────────────────────────────────────
@@ -258,7 +264,14 @@ class AudioCodecManager:
token_ids: Optional[list] = None,
text: Optional[str] = None,
) -> Tuple[bytes, int]:
- """Unified decode — dispatches to the right codec decoder."""
+ """Unified decode — dispatches to the right codec decoder.
+
+ ``device`` is what the caller would like. Where the codec is actually
+ resident wins: input tensors built on another device fail outright for SNAC
+ and DAC, and BiCodec would move a CPU-resident codec onto the card, taking
+ the VRAM a CPU RAM load promised not to take.
+ """
+ device = self._codec_devices.get(audio_type, device)
if audio_type == "snac":
if not token_ids:
raise ValueError("SNAC decoding requires token_ids")
@@ -293,4 +306,5 @@ class AudioCodecManager:
if self._outetts_code_dir is not None:
deactivate_pinned_package("outetts", self._outetts_code_dir)
self._outetts_code_dir = None
+ self._codec_devices.clear()
logger.info("Unloaded all audio codecs")
diff --git a/studio/backend/core/inference/audio_device.py b/studio/backend/core/inference/audio_device.py
new file mode 100644
index 00000000000..86c9f7226d9
--- /dev/null
+++ b/studio/backend/core/inference/audio_device.py
@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Where an audio model's weights go: the accelerator, or plain CPU RAM.
+
+Audio loads take an accelerator whenever one exists. That is right until the GPU
+is the scarce resource (a resident chat model, a training run, a card too small
+for the checkpoint), and Whisper and the smaller TTS models run fine on CPU.
+
+Values match ``RAG_EMBED_DEVICE`` (``core/rag/config.py``):
+
+``auto`` detect as before.
+``cpu`` force CPU RAM, even with a working accelerator.
+``gpu`` prefer the accelerator. The existing CPU retry after a failed load
+ still applies, so this is a preference and not a guarantee.
+
+``UNSLOTH_AUDIO_DEVICE`` supplies the default for a request that names none.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Optional
+
+__all__ = [
+ "AUDIO_DEVICE_CHOICES",
+ "audio_device_default",
+ "audio_device_forces_cpu",
+ "mask_accelerators_for_cpu_audio",
+ "normalize_audio_device",
+]
+
+AUDIO_DEVICE_CHOICES = ("auto", "cpu", "gpu")
+
+# Spellings other Studio surfaces already use; names arrive from a status echo.
+_CPU_ALIASES = frozenset({"cpu", "ram", "cpu_ram", "system", "system_ram"})
+_GPU_ALIASES = frozenset(
+ {"gpu", "cuda", "rocm", "hip", "xpu", "mps", "metal", "accelerator", "accel"}
+)
+
+
+def normalize_audio_device(value: Optional[str]) -> str:
+ """Map any accepted spelling onto ``auto``/``cpu``/``gpu``.
+
+ Anything unrecognised becomes ``auto``: an unknown preference must not fail
+ a load, and detection is what the caller would have done regardless.
+
+ That fallback is for values the user did not type: ``UNSLOTH_AUDIO_DEVICE``,
+ and device names read back off a status payload. The HTTP models pin the
+ three canonical values instead, so a misspelled ``cpu`` is a 422 rather than
+ a silent placement back on the GPU.
+ """
+ text = str(value or "").strip().lower()
+ if not text:
+ return "auto"
+ if text in _CPU_ALIASES:
+ return "cpu"
+ if text in _GPU_ALIASES:
+ return "gpu"
+ if text == "auto":
+ return "auto"
+ return "auto"
+
+
+def audio_device_default() -> str:
+ """The preference for a request that carries none (``UNSLOTH_AUDIO_DEVICE``).
+
+ Scope: the native audio backend and the three STT sidecars. It does NOT reach a
+ GGUF TTS model. llama.cpp placement is decided from ``gpu_memory_mode`` and
+ ``gpu_layers`` at request time, but nothing knows a GGUF is audio until
+ llama-server reports its ``_audio_type`` after the load, so there is no point
+ early enough to translate the default into zero offload. The Audio page does it
+ from the catalog it already has; a headless caller must send the GGUF placement
+ fields itself.
+ """
+ return normalize_audio_device(os.environ.get("UNSLOTH_AUDIO_DEVICE"))
+
+
+def audio_device_forces_cpu(value: Optional[str]) -> bool:
+ """True when this preference means "load into CPU RAM".
+
+ ``None`` falls back to the environment default, so an older caller still
+ honours a server-wide setting.
+ """
+ if value is None:
+ return audio_device_default() == "cpu"
+ return normalize_audio_device(value) == "cpu"
+
+
+def mask_accelerators_for_cpu_audio(env: dict) -> None:
+ """Hide CUDA/ROCm from a worker whose weights stay in CPU RAM.
+
+ Placing the weights on CPU is not enough on its own: the worker runs
+ ``detect_hardware()`` first, and that calls ``torch.cuda.get_device_properties``,
+ which creates a context worth a few hundred MB. Masking first is what makes
+ "this load holds no VRAM" true.
+
+ Same values as the CPU embed server (``core/rag/embed_llama_server.py``):
+ blank for CUDA, ``-1`` for HIP because it reads the CUDA variable only when
+ its own is unset. An inherited ROCR mask is left alone, since clearing it
+ exposes more agents rather than fewer. XPU is not masked: its probe is
+ ``torch.xpu.is_available()`` and takes no context.
+
+ Call before importing torch. Mutates ``env`` in place.
+ """
+ env["CUDA_VISIBLE_DEVICES"] = ""
+ env["HIP_VISIBLE_DEVICES"] = "-1"
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index e1e7f330a70..fbea7138785 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -32520,7 +32520,10 @@ class LlamaCppBackend:
if LlamaCppBackend._codec_mgr is None:
LlamaCppBackend._codec_mgr = AudioCodecManager()
- device = "cuda" if torch.cuda.is_available() else "cpu"
+ # A second allocation: on CUDA for a zero-offload server it would hold VRAM the
+ # load is classified as not holding, which is what lets the route skip
+ # arbitration and survive training.
+ device = "cuda" if torch.cuda.is_available() and not self.holds_no_vram else "cpu"
model_repo_path = None
# BiCodec needs a repo with BiCodec/ weights -- download canonical SparkTTS
diff --git a/studio/backend/core/inference/native_audio.py b/studio/backend/core/inference/native_audio.py
index 602e21c23f3..2c7980459c3 100644
--- a/studio/backend/core/inference/native_audio.py
+++ b/studio/backend/core/inference/native_audio.py
@@ -671,10 +671,15 @@ def _as_wav_bytes(audio, sample_rate: int) -> bytes:
class NativeAudioBackend:
"""One-model backend for the five curated native audio architectures."""
- def __init__(self) -> None:
+ def __init__(self, device_preference: Optional[str] = None) -> None:
import torch
- if torch.cuda.is_available():
+ from core.inference.audio_device import audio_device_forces_cpu
+
+ self.device_preference = device_preference
+ if audio_device_forces_cpu(device_preference):
+ self.device = "cpu"
+ elif torch.cuda.is_available():
self.device = "cuda"
elif hasattr(torch, "xpu") and torch.xpu.is_available():
self.device = "xpu"
@@ -798,6 +803,14 @@ class NativeAudioBackend:
"multi-GPU sharding is not supported yet."
)
if audio_type == "minimax_music3" and self.device != "cuda":
+ # Chosen, not missing: the generic message sends users hunting for a card.
+ from core.inference.audio_device import audio_device_forces_cpu
+ if audio_device_forces_cpu(self.device_preference):
+ raise RuntimeError(
+ "MiniMax Music 3 cannot be loaded into CPU RAM: its official local "
+ "runtime requires an NVIDIA CUDA GPU. Set the audio device back to "
+ "Auto (or GPU) to load this model."
+ )
raise RuntimeError(
"MiniMax Music 3 currently requires an NVIDIA CUDA GPU in its official "
"local runtime. It is not available on CPU, Apple Silicon, AMD, or Intel XPU."
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index a2703466d50..e941a3ba026 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -26,7 +26,9 @@ import time
import uuid
from io import BytesIO
from pathlib import Path
-from typing import Any, Generator, Optional, Sequence, Tuple, Union
+from typing import Any, Callable, Generator, Optional, Sequence, Tuple, Union
+from core.inference.audio_device import audio_device_forces_cpu
+from core.inference.native_audio import NATIVE_AUDIO_TYPES, is_native_audio_model
from core.inference.audio_errors import (
AUDIO_UNSUPPORTED_CODE,
AudioBackendUnsupportedError,
@@ -1495,6 +1497,8 @@ class InferenceOrchestrator:
chat_template_override: Optional[str] = None,
load_cancel_event: Optional[threading.Event] = None,
post_handoff_expected_free_gb: Optional[dict[int, float]] = None,
+ audio_device: Optional[str] = None,
+ on_prior_worker_released: Optional[Callable[[], None]] = None,
) -> bool:
"""Load a model for inference.
@@ -1532,13 +1536,21 @@ class InferenceOrchestrator:
else None,
"mlx_kv_bits": mlx_kv_bits,
"chat_template_override": chat_template_override,
+ # Read in the worker, which hides the accelerators before detection.
+ "audio_device": audio_device,
}
- resolved_gpu_ids, gpu_selection = prepare_gpu_selection(
- gpu_ids,
- model_name = model_name,
- hf_token = hf_token,
- load_in_4bit = load_in_4bit,
- )
+ if audio_device_forces_cpu(audio_device) and is_native_audio_model(model_name):
+ # Choosing a card for a load that takes none harms it twice: several
+ # GPUs are rejected as unsupported sharding, and required_gb becomes
+ # expected_free_gb, so the settle wait raises on a busy card.
+ resolved_gpu_ids, gpu_selection = None, {"selection_mode": "cpu_audio"}
+ else:
+ resolved_gpu_ids, gpu_selection = prepare_gpu_selection(
+ gpu_ids,
+ model_name = model_name,
+ hf_token = hf_token,
+ load_in_4bit = load_in_4bit,
+ )
sub_config["resolved_gpu_ids"] = resolved_gpu_ids
sub_config["gpu_selection"] = gpu_selection
# Parent-detected backend for the worker's apply_gpu_ids().
@@ -1604,6 +1616,10 @@ class InferenceOrchestrator:
"not starting the replacement model. Retry shortly."
)
+ # Previous worker gone, VRAM back: the last moment before a long download.
+ if on_prior_worker_released is not None:
+ on_prior_worker_released()
+
disable_xet = sub_config.get("disable_xet", False) or (
os.environ.get("HF_HUB_DISABLE_XET") == "1"
)
@@ -1708,6 +1724,12 @@ class InferenceOrchestrator:
self.models[self.active_model_name] = _mirrored_model_entry(
model_info, model_name
)
+ # Lets the already-loaded shortcut tell a CPU request from the GPU
+ # model it would otherwise report as satisfied. Native audio only:
+ # marking anything else tells training a GPU model holds no VRAM.
+ self.models[self.active_model_name]["audio_cpu"] = model_info.get(
+ "audio_type"
+ ) in NATIVE_AUDIO_TYPES and audio_device_forces_cpu(audio_device)
self.models[self.active_model_name].update(
_mlx_runtime_mirror_fields(model_info)
)
@@ -1796,22 +1818,29 @@ class InferenceOrchestrator:
model: Optional[str],
engine: str,
request_cancel_event: Optional[threading.Event] = None,
+ device: Optional[str] = None,
) -> None:
- """Make a dictation model resident on its sidecar."""
+ """Make a dictation model resident on its sidecar.
+
+ ``device`` is the user's audio device preference (``auto``/``cpu``/``gpu``).
+ """
from core.inference import stt_registry
- stt_registry.load(model, engine, request_cancel_event)
+ stt_registry.load(model, engine, request_cancel_event, device = device)
def unload_stt_model(
self,
engines: Optional[Sequence[str]] = None,
expected_model: Optional[str] = None,
+ wait: bool = True,
) -> list:
"""Release dictation models (all engines by default); returns refusals.
``expected_model`` scopes the release to a sidecar still holding that model.
+ ``wait=False`` leaves a sidecar that is mid-request alone, for a caller
+ freeing memory opportunistically rather than to reclaim it now.
"""
from core.inference import stt_registry
- return stt_registry.unload(engines, expected_model = expected_model)
+ return stt_registry.unload(engines, wait = wait, expected_model = expected_model)
def resident_stt_model(self) -> dict:
"""What dictation holds, alongside active_model_name for chat."""
diff --git a/studio/backend/core/inference/stt_ggml_sidecar.py b/studio/backend/core/inference/stt_ggml_sidecar.py
index a392f026ae8..f2a63212195 100644
--- a/studio/backend/core/inference/stt_ggml_sidecar.py
+++ b/studio/backend/core/inference/stt_ggml_sidecar.py
@@ -716,6 +716,8 @@ class GgmlSttSidecar:
self._process: Optional[subprocess.Popen] = None
self._port: Optional[int] = None
self._model_id: Optional[str] = None
+ # --no-gpu at the user's request, tracked because it varies per request.
+ self._forced_cpu = False
self._idle_timer: Optional[threading.Timer] = None
self._idle_generation = 0
self._keep_alive_seconds = keep_alive_seconds
@@ -791,6 +793,7 @@ class GgmlSttSidecar:
self._process = None
self._port = None
self._model_id = None
+ self._forced_cpu = False
if process is not None and process.poll() is None:
process.terminate()
try:
@@ -929,8 +932,17 @@ class GgmlSttSidecar:
self,
model: Optional[str] = None,
request_cancel_event: Optional[threading.Event] = None,
+ device: Optional[str] = None,
) -> None:
- """Start (or switch) whisper-server for the requested curated model."""
+ """Start (or switch) whisper-server for the requested curated model.
+
+ ``device`` is the user's audio device preference; ``cpu`` starts the server
+ with ``--no-gpu``, the same flag training and a CPU-only install already use.
+ ``None`` is no opinion: it keeps the running server's placement, so a
+ caller that never sends one cannot restart it onto the other device.
+ """
+ from core.inference.audio_device import audio_device_forces_cpu
+
if request_cancel_event is not None and request_cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
self._raise_if_update_in_progress()
@@ -940,26 +952,22 @@ class GgmlSttSidecar:
raise SttTranscriptionCancelledError("Transcription cancelled.")
self._raise_if_update_in_progress()
binary = ensure_engine_available()
- if self._process_alive() and self._model_id == model_id:
+ force_cpu = (
+ self._forced_cpu
+ if device is None and self._process_alive()
+ else audio_device_forces_cpu(device)
+ )
+ if (
+ self._process_alive()
+ and self._model_id == model_id
+ and self._forced_cpu == force_cpu
+ ):
self._schedule_idle_unload_locked()
return
model_path = self._ensure_model_downloaded(model_id)
reservation, port = self._reserve_free_port()
command = [binary, "-m", model_path, "--host", "127.0.0.1", "--port", str(port)]
marker = _whisper_install_marker(binary)
- if _training_active():
- # Keep whisper.cpp off the accelerator during training (like the Transformers sidecar's CPU choice) so a
- # mid-training dictation cannot reclaim the VRAM training just freed.
- command.append("--no-gpu")
- elif marker is not None and marker.get("backend") == "cpu":
- # A deliberate CPU install must stay CPU: the slim wiring links every llama ggml backend (including
- # CUDA/ROCm), so without this flag a cpu-selected install would still grab the GPU.
- command.append("--no-gpu")
- logger.info(
- "Starting whisper-server for STT model %s on 127.0.0.1:%s",
- model_id,
- port,
- )
cancel_event = (
request_cancel_event if request_cancel_event is not None else threading.Event()
)
@@ -970,6 +978,25 @@ class GgmlSttSidecar:
try:
if cancel_event.is_set():
raise SttLoadCancelledError("GGUF STT model loading was cancelled.")
+ # Decided under the loading flag, never before it: the training hook
+ # reads is_loading() without this lock, so an earlier decision is
+ # invisible and it preserves the outgoing model as holding no VRAM.
+ if force_cpu:
+ # Tracked, so the reuse check above restarts on a change.
+ command.append("--no-gpu")
+ elif _training_active():
+ # Off the accelerator during training, so a mid-training dictation
+ # cannot reclaim the VRAM training just freed.
+ command.append("--no-gpu")
+ elif marker is not None and marker.get("backend") == "cpu":
+ # The slim wiring links every ggml backend, so without this flag a
+ # cpu-selected install would still grab the GPU.
+ command.append("--no-gpu")
+ logger.info(
+ "Starting whisper-server for STT model %s on 127.0.0.1:%s",
+ model_id,
+ port,
+ )
self._release_locked()
# release the reservation as late as possible: whisper-server binds the port moments after this close
reservation.close()
@@ -999,6 +1026,7 @@ class GgmlSttSidecar:
self._process = process
self._port = port
self._model_id = model_id
+ self._forced_cpu = force_cpu
self._schedule_idle_unload_locked()
finally:
reservation.close() # no-op when already released before spawn
diff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.py
index 059c2211b36..8c6c8396a72 100644
--- a/studio/backend/core/inference/stt_mtmd_sidecar.py
+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py
@@ -557,6 +557,8 @@ class MtmdSttSidecar:
# Whether the resident server was launched with the GPU pinned off for training. Kept so a dictation after the
# run does not stay on CPU.
self._gpu_disabled = False
+ # Apart from _gpu_disabled, so training restarts while the preference survives.
+ self._forced_cpu = False
self._load_cancel_event: Optional[threading.Event] = None
self._load_owner_cancel_event: Optional[threading.Event] = None
self._update_in_progress = False
@@ -794,7 +796,12 @@ class MtmdSttSidecar:
self,
model: Optional[str] = None,
request_cancel_event: Optional[threading.Event] = None,
+ device: Optional[str] = None,
) -> None:
+ """``device`` is the user's audio device preference; ``cpu`` starts
+ llama-server at ``-ngl 0``, the same offload training already forces."""
+ from core.inference.audio_device import audio_device_forces_cpu
+
if request_cancel_event is not None and request_cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
self._raise_if_update_in_progress()
@@ -813,6 +820,7 @@ class MtmdSttSidecar:
binary,
request_cancel_event,
path_revision = path_revision,
+ device = device,
)
def _load_locked(
@@ -822,12 +830,23 @@ class MtmdSttSidecar:
request_cancel_event: Optional[threading.Event] = None,
*,
path_revision: Optional[int] = None,
+ device: Optional[str] = None,
) -> None:
+ from core.inference.audio_device import audio_device_forces_cpu
+
if path_revision is None:
from utils.llama_cpp_path_settings import custom_llama_cpp_path_revision
path_revision = custom_llama_cpp_path_revision()
with self._lock:
- training = _training_active()
+ # None is no opinion: a caller sending none cannot move the server.
+ if device is None:
+ forced_cpu = (
+ self._forced_cpu if self._process_alive() else audio_device_forces_cpu(None)
+ )
+ else:
+ forced_cpu = audio_device_forces_cpu(device)
+ # Both mean "no offload", which _gpu_disabled already tracks.
+ training = _training_active() or forced_cpu
if (
self._process_alive()
and self._model_id == model_id
@@ -838,6 +857,9 @@ class MtmdSttSidecar:
# killing a running transcription for, so an in-flight request keeps the server it has and the next idle
# load picks the GPU back up.
if self._gpu_disabled == training or self._active_requests:
+ # Recorded though nothing restarts: dropping it sends the next
+ # device-less load back to the GPU once training ends.
+ self._forced_cpu = forced_cpu
self._schedule_idle_unload_locked()
return
# Announced before the slow probe and reap: is_loading() is read lock-free, so a training start would
@@ -873,7 +895,7 @@ class MtmdSttSidecar:
# Re-read last: _release_locked() reaps the old server, which can take seconds, and training admission that
# already passed its own check cannot come back to cancel this load. Publishing _loading first covers the
# other order, so between them every training start either cancels this load or is seen by it.
- training = _training_active()
+ training = _training_active() or forced_cpu
try:
sock, port = self._reserve_free_port()
cmd = [
@@ -933,6 +955,7 @@ class MtmdSttSidecar:
self._model_id = model_id
self._binary_path_revision = path_revision
self._gpu_disabled = training
+ self._forced_cpu = forced_cpu
self._generation += 1
self._schedule_idle_unload_locked()
finally:
diff --git a/studio/backend/core/inference/stt_registry.py b/studio/backend/core/inference/stt_registry.py
index fee8f1c9af1..39618cb005c 100644
--- a/studio/backend/core/inference/stt_registry.py
+++ b/studio/backend/core/inference/stt_registry.py
@@ -44,9 +44,14 @@ def load(
model: Optional[str],
engine: str,
request_cancel_event: Optional[threading.Event] = None,
+ device: Optional[str] = None,
) -> None:
"""Make ``model`` resident on ``engine``, then release every idle other engine.
+ ``device`` is the user's audio device preference (``auto``/``cpu``/``gpu``);
+ every engine honours it, on CPU by holding the weights in system RAM instead
+ of the accelerator.
+
Dictation is one user-visible choice, so engines are alternatives, not slots:
holding two at once doubles VRAM for the whole keep-alive window. An engine serving
a request keeps its model and releases it on its own idle timer. Raises what the
@@ -62,9 +67,13 @@ def load(
# answer is not certain, keep the old order and accept the peak.
if _model_is_downloaded(engine, model):
unload(others, wait = False)
- sidecar_for(engine).load(model, request_cancel_event = request_cancel_event)
+ sidecar_for(engine).load(
+ model, request_cancel_event = request_cancel_event, device = device
+ )
else:
- sidecar_for(engine).load(model, request_cancel_event = request_cancel_event)
+ sidecar_for(engine).load(
+ model, request_cancel_event = request_cancel_event, device = device
+ )
unload(others, wait = False)
diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.py
index 76c9ea1a100..53930dfff4c 100644
--- a/studio/backend/core/inference/stt_sidecar.py
+++ b/studio/backend/core/inference/stt_sidecar.py
@@ -967,15 +967,24 @@ def _reported_device(device: Optional[str]) -> Optional[str]:
return device
-def _pick_device():
+def _pick_device(preference: Optional[str] = None):
"""Return (device, torch_dtype) for the Whisper model.
CUDA uses float16. MPS and CPU use float32: Whisper's decoder is unstable in
float16 on MPS and degenerates into repeated tokens.
+
+ ``preference`` is the user's choice from ``core.inference.audio_device``.
+ ``cpu`` short-circuits detection entirely -- that is the whole point of the
+ option, so it must hold even on a machine with a working accelerator. ``gpu``
+ and ``auto`` both detect: preferring the GPU is what detection already does,
+ and the caller's CPU retry still covers a card that cannot take the model.
"""
+ from core.inference.audio_device import audio_device_forces_cpu
try:
import torch
+ if audio_device_forces_cpu(preference):
+ return "cpu", torch.float32
# new loads use CPU during training; a resident GPU model may stay put when the admission check confirms
# headroom
training_active = _training_active()
@@ -1144,6 +1153,7 @@ class WhisperSttSidecar:
self._engine = None
self._model_id: Optional[str] = None
self._device: Optional[str] = None
+ self._device_preference: Optional[str] = None
self._lock = threading.RLock()
self._load_state_lock = threading.Lock()
self._loading = False
@@ -1285,6 +1295,7 @@ class WhisperSttSidecar:
self._engine = None
self._model_id = None
self._device = None
+ self._device_preference = None
self._survivor = False
else:
self._survivor = True
@@ -1393,12 +1404,22 @@ class WhisperSttSidecar:
raise
return worker
- def _ensure_model_downloaded(self, model_id: str) -> _CachedSttSnapshot:
+ def _ensure_model_downloaded(
+ self,
+ model_id: str,
+ use_resident: bool = True,
+ ) -> _CachedSttSnapshot:
"""Validate the local snapshot before decode or model replacement.
Returns the checkpoint's multilingual flag when local metadata provides
it. Curated defaults are known multilingual.
+ ``use_resident = False`` skips the resident-model shortcut and resolves the
+ path on disk. The shortcut answers from the loaded model and returns no
+ path, which is right when that model is about to be reused and wrong when
+ the caller is replacing it with one on another device: the same model id is
+ resident, but the load still needs somewhere to read the weights from.
+
A survivor is held for its memory alone, so it does not answer for the
model the way a resident one does: the snapshot is looked up on disk, or
the load it precedes would be turned away as a checkpoint that is not
@@ -1406,7 +1427,7 @@ class WhisperSttSidecar:
"""
model_id = resolve_model_id(model_id)
with self._lock:
- reusable = self._engine is not None and self._model_id == model_id
+ reusable = use_resident and self._engine is not None and self._model_id == model_id
if reusable and not self._is_survivor_locked():
resident_model = (
self._engine[0] if isinstance(self._engine, (tuple, list)) else self._engine
@@ -1443,20 +1464,39 @@ class WhisperSttSidecar:
self,
model: Optional[str] = None,
request_cancel_event: Optional[threading.Event] = None,
+ device: Optional[str] = None,
):
"""Load (or switch to) a model, reusing it if already resident.
+ ``device`` is the user's device preference (``auto``/``cpu``/``gpu``, see
+ ``core.inference.audio_device``). An explicit one that differs from the
+ resident model's reloads it, or the setting would be silently ignored.
+ ``None`` means no opinion: it takes the server default for a fresh load
+ and reuses whatever is resident, so a caller that never sends one cannot
+ move a model another surface placed.
+
Returns a ``(model, processor)`` pair.
"""
+ from core.inference.audio_device import audio_device_default, normalize_audio_device
+
if request_cancel_event is not None and request_cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
model_id = resolve_model_id(model)
+ # None is "no opinion", not "auto": a caller sending none must not drag a model
+ # off the device another surface asked for.
+ requested = None if device is None else normalize_audio_device(device)
+ preference = audio_device_default() if requested is None else requested
with self._lock:
if request_cancel_event is not None and request_cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
ensure_stt_available()
self._release_dead_engine_locked()
- reusable = self._engine is not None and self._model_id == model_id
+ placement_matches = (
+ requested is None
+ or self._device_preference is None
+ or self._device_preference == requested
+ )
+ reusable = self._engine is not None and self._model_id == model_id and placement_matches
if reusable and not self._is_survivor_locked():
self._schedule_idle_unload_locked()
return self._engine
@@ -1469,7 +1509,8 @@ class WhisperSttSidecar:
self._start_survivor = None
try:
self._raise_if_load_cancelled(cancel_event)
- cached = self._ensure_model_downloaded(model_id)
+ # The resident shortcut returns no path, and this load replaces it.
+ cached = self._ensure_model_downloaded(model_id, use_resident = placement_matches)
snapshot_path = cached.path
if snapshot_path is None:
raise SttModelNotDownloadedError(
@@ -1477,7 +1518,7 @@ class WhisperSttSidecar:
"Download it in Settings, then Voice, before loading it."
)
self._raise_if_load_cancelled(cancel_event)
- device, dtype = _pick_device()
+ device, dtype = _pick_device(preference)
if not self._release_engine_locked():
# starting a second child over one that never exited doubles the memory this release was meant to
# give back
@@ -1545,6 +1586,7 @@ class WhisperSttSidecar:
self._engine = candidate
self._model_id = model_id
self._device = device
+ self._device_preference = preference
self._survivor = False
self._load_cancel_event = None
self._load_owner_cancel_event = None
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 08a4a7a8080..f4050f07351 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1114,6 +1114,17 @@ def run_inference_process(
_native_audio_worker = is_native_audio_model(model_name)
+ # Before detect_hardware(), whose probe would leave a CUDA context here; the route
+ # skips the arbiter on the basis that this load reserves none.
+ if _native_audio_worker:
+ from core.inference.audio_device import (
+ audio_device_forces_cpu,
+ mask_accelerators_for_cpu_audio,
+ )
+ if audio_device_forces_cpu(config.get("audio_device")):
+ mask_accelerators_for_cpu_audio(os.environ)
+ logger.info("Audio model '%s' pinned to CPU RAM; accelerators hidden", model_name)
+
# ── 0. MLX fast-path — skip torch/transformers ──
_ensure_backend_on_path()
@@ -1424,7 +1435,12 @@ def run_inference_process(
# ── 3. Create inference backend and load initial model ──
try:
- backend = InferenceBackend()
+ # Native audio picks its device in __init__, so the preference goes there.
+ backend = (
+ InferenceBackend(device_preference = config.get("audio_device"))
+ if _native_audio_worker
+ else InferenceBackend()
+ )
_send_response(
resp_queue,
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index e114f9a6a2d..34ccb599d7a 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -257,6 +257,19 @@ class LoadRequest(BaseModel):
"non-GGUF models."
),
)
+ audio_device: Optional[Literal["auto", "cpu", "gpu"]] = Field(
+ None,
+ description = (
+ "Native audio (TTS / music) models only: where to hold the weights. "
+ "'cpu' keeps them in CPU RAM rather than the GPU -- slower generation, "
+ "but it leaves VRAM for other models and runs checkpoints too large for "
+ "the card. 'gpu' prefers the accelerator; 'auto' (default) detects. "
+ "Ignored for every non-audio model, and for GGUF audio: llama.cpp "
+ "placement comes from gpu_memory_mode and gpu_layers, which are decided "
+ "before anything can know a GGUF is audio. Send gpu_memory_mode='manual' "
+ "with gpu_layers=0 and speculative_type='off' to hold a GGUF TTS in RAM."
+ ),
+ )
gpu_memory_mode: Literal["auto", "manual"] = Field(
"auto",
description = (
@@ -397,6 +410,14 @@ class TranscribeRequest(BaseModel):
None,
description = "STT engine: 'transformers' (default) or 'gguf' (whisper.cpp)",
)
+ device: Optional[Literal["auto", "cpu", "gpu"]] = Field(
+ None,
+ description = (
+ "Where to hold the model: 'cpu' keeps it in CPU RAM, 'gpu' prefers the "
+ "accelerator, 'auto' (default) detects. Applies when this request has to "
+ "load the model; a resident model on another device is reloaded to honour it."
+ ),
+ )
class SttLoadRequest(BaseModel):
@@ -407,6 +428,13 @@ class SttLoadRequest(BaseModel):
None,
description = "STT engine: 'transformers' (default) or 'gguf' (whisper.cpp)",
)
+ device: Optional[Literal["auto", "cpu", "gpu"]] = Field(
+ None,
+ description = (
+ "Where to hold the model: 'cpu' keeps it in CPU RAM rather than the GPU, "
+ "'gpu' prefers the accelerator, 'auto' (default) detects."
+ ),
+ )
class ValidateModelRequest(BaseModel):
@@ -440,6 +468,14 @@ class ValidateModelRequest(BaseModel):
# refuse a load that then fits.
disable_vision: bool = Field(False)
gpu_ids: Optional[List[int]] = Field(None)
+ # Sized with too: preflighting a CPU load would refuse one that takes no VRAM.
+ audio_device: Optional[Literal["auto", "cpu", "gpu"]] = Field(
+ None,
+ description = (
+ "Native audio placement intended for the follow-up load. 'cpu' skips "
+ "the GPU-memory preflight, which that load would not use."
+ ),
+ )
gpu_memory_mode: Literal["auto", "manual"] = Field(
"auto",
description = (
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index d535131d3e6..553439cf7ac 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -28,7 +28,17 @@ from fastapi import (
from fastapi.encoders import jsonable_encoder
from fastapi.responses import StreamingResponse, JSONResponse, PlainTextResponse, Response
from starlette.requests import ClientDisconnect
-from typing import Any, Callable, Collection, List, NamedTuple, Optional, TYPE_CHECKING, Union
+from typing import (
+ Any,
+ Callable,
+ Collection,
+ List,
+ Literal,
+ NamedTuple,
+ Optional,
+ TYPE_CHECKING,
+ Union,
+)
import functools
import json
import httpx
@@ -11953,6 +11963,55 @@ async def _prepare_load_placement(
return _LoadPlacement(requested, resolved, is_vulkan, diffusion_kind)
+def _native_audio_cpu_load(config, request) -> bool:
+ """True when this load is a native audio model the user placed in CPU RAM.
+
+ Gated on the audio type on purpose: audio_device is documented as ignored
+ for everything else, so a chat model cannot send it to skip the VRAM guards
+ that read this.
+ """
+ from core.inference.audio_device import audio_device_forces_cpu
+ from core.inference.native_audio import NATIVE_AUDIO_TYPES
+
+ return getattr(config, "audio_type", None) in NATIVE_AUDIO_TYPES and audio_device_forces_cpu(
+ getattr(request, "audio_device", None)
+ )
+
+
+def _resident_audio_placement_matches(backend, request) -> bool:
+ """False when the resident audio model is not where this request wants it.
+
+ Reads the resident entry, not the requested config: this runs ahead of config
+ resolution, and the question is about the model already loaded. Only native
+ audio records a placement, so everything else keeps the shortcut. An entry
+ from before this existed has no key and is read as GPU, which is what those
+ loads did.
+ """
+ from core.inference.audio_device import audio_device_forces_cpu
+ from core.inference.native_audio import NATIVE_AUDIO_TYPES
+
+ resident = backend.models.get(backend.active_model_name, {})
+ if resident.get("audio_type") not in NATIVE_AUDIO_TYPES:
+ return True
+ return bool(resident.get("audio_cpu", False)) == audio_device_forces_cpu(
+ getattr(request, "audio_device", None)
+ )
+
+
+def _resident_audio_holds_no_gpu(backend) -> bool:
+ """True when the resident model is a native audio model placed in CPU RAM.
+
+ Same audio-type gate as the writer, so only an entry that really recorded a
+ CPU placement can skip the arbiter.
+ """
+ from core.inference.native_audio import NATIVE_AUDIO_TYPES
+
+ resident = backend.models.get(backend.active_model_name, {})
+ return resident.get("audio_type") in NATIVE_AUDIO_TYPES and bool(
+ resident.get("audio_cpu", False)
+ )
+
+
async def _preflight_native_audio_placement(
config: ModelConfig, request: LoadRequest | ValidateModelRequest, placement: _LoadPlacement
) -> _LoadPlacement:
@@ -11976,6 +12035,20 @@ async def _preflight_native_audio_placement(
detail = "Higgs TTS requires Python 3.10 or newer in Studio.",
)
+ # Before every VRAM question: sizing a CPU load refuses it on a full GPU.
+ if _native_audio_cpu_load(config, request):
+ if audio_type == "minimax_music3":
+ # The switch already evicted the resident model, so a doomed load costs it.
+ raise HTTPException(
+ status_code = 400,
+ detail = (
+ "MiniMax Music 3 cannot be loaded into CPU RAM: its official local "
+ "runtime requires an NVIDIA CUDA GPU. Set the audio device back to "
+ "Auto to load this model."
+ ),
+ )
+ return placement
+
automatic = not placement.requested_gpu_ids
availability = (
await asyncio.to_thread(_native_audio_post_handoff_free_gb) if automatic else None
@@ -12236,6 +12309,10 @@ def _guard_chat_load_against_training(
from core.training import get_training_backend
from routes.training_vram import can_load_chat_during_training
+ # Ahead of the diffusion branch, which would 409 a load that takes no VRAM.
+ if _native_audio_cpu_load(config, request):
+ return
+
requested_gpu_ids = placement.requested_gpu_ids
gpu_ids_are_vulkan_ordinals = placement.gpu_ids_are_vulkan_ordinals
diffusion_kind = placement.diffusion_kind
@@ -13419,6 +13496,8 @@ async def _load_model_impl(
backend.models.get(backend.active_model_name) or {},
request.max_seq_length,
)
+ # Without this the route reports already_loaded and nothing moves.
+ and _resident_audio_placement_matches(backend, request)
):
api_monitor.discard(_load_event) # nothing loaded, no monitor row
logger.info(f"Model already loaded (Unsloth): {model_log_label}, skipping reload")
@@ -13439,7 +13518,9 @@ async def _load_model_impl(
_sf_supports_reasoning = _sf_flags["supports_reasoning"]
_sf_reasoning_style = _sf_flags["reasoning_style"]
# Requested chat model already resident: assert CHAT ownership (no-op when held) to correct a drifted owner.
- await asyncio.to_thread(acquire_for, CHAT)
+ # Owns no GPU, so the arbiter would cancel a generation for nothing.
+ if not _resident_audio_holds_no_gpu(backend):
+ await asyncio.to_thread(acquire_for, CHAT)
return LoadResponse(
status = "already_loaded",
model = model_log_label if native_grant_backed else backend.active_model_name,
@@ -13701,7 +13782,9 @@ async def _load_model_impl(
# ...but only when this load will actually use the GPU, exactly as the image and video loaders gate on their device:
# a manual gpu_layers=0 load runs on CPU, so taking the arbiter would cancel an image/video generation for nothing.
chat_load_needs_gpu = not (
- config.is_gguf
+ # The non-GGUF case of the same thing: no VRAM, so no arbiter.
+ _native_audio_cpu_load(config, request)
+ or config.is_gguf
and await asyncio.to_thread(
zero_vram_chat_load,
request.gpu_memory_mode,
@@ -14048,11 +14131,12 @@ async def _load_model_impl(
_restore_marker_if_prior_preview_still_resident()
raise
- # Shut down any export subprocess to free VRAM
+ # Free the export's VRAM, but only when this load wants VRAM: a CPU load masks
+ # the accelerators, so killing the user's export frees nothing it needs.
try:
from core.export import get_export_backend
exp_backend = get_export_backend()
- if exp_backend.current_checkpoint:
+ if chat_load_needs_gpu and exp_backend.current_checkpoint:
logger.info("Shutting down export subprocess to free GPU memory for inference")
exp_backend._shutdown_subprocess()
exp_backend.current_checkpoint = None
@@ -14072,6 +14156,11 @@ async def _load_model_impl(
_prior_alias = getattr(backend, "_openai_advertised_id", None)
_prior_active = getattr(backend, "active_model_name", None)
backend._openai_advertised_id = None
+ # Not after the load (held across a long download, the chat evictor cancels
+ # this very load) and not before it (until the previous worker exits, this
+ # claim is all that stops a second pipeline allocating over a resident model).
+ # load_model fires it in between; the post-load release covers a re-taken claim.
+ _release_chat_after_teardown = (lambda: release(CHAT)) if not chat_load_needs_gpu else None
try:
success = await asyncio.to_thread(
backend.load_model,
@@ -14090,7 +14179,9 @@ async def _load_model_impl(
mlx_kv_bits = request.mlx_kv_bits,
chat_template_override = request.chat_template_override,
load_cancel_event = load_cancel_event,
+ on_prior_worker_released = _release_chat_after_teardown,
post_handoff_expected_free_gb = post_chat_handoff_expected_free_gb,
+ audio_device = request.audio_device,
)
except Exception:
_restore_marker_if_prior_preview_still_resident()
@@ -14118,7 +14209,8 @@ async def _load_model_impl(
)
# Same guard the GGUF branch runs above: an Images/Video acquire can land between this load's cancellation and its publish, so this load undoes itself.
- if current_owner() != CHAT:
+ # On a clean server the owner is None, so ungated this unloads what it loaded.
+ if chat_load_needs_gpu and current_owner() != CHAT:
await asyncio.to_thread(backend.unload_model, config.identifier)
# The worker's base CUDA context outlives the model unload, so kill it too.
await asyncio.to_thread(backend._shutdown_subprocess, 5.0)
@@ -14129,6 +14221,10 @@ async def _load_model_impl(
"so the load was cancelled. Unload that model, then try again."
),
)
+ if not chat_load_needs_gpu:
+ # This load replaced whatever held CHAT; leaving the claim makes the next
+ # Images/Video acquire evict a model that never used the GPU.
+ await asyncio.to_thread(release, CHAT)
# Stamped here, not in backend.load_model: that entry is built in the load
# subprocess and only a fixed model_info mirror crosses back, so it would
@@ -17436,7 +17532,9 @@ async def stt_load(
_await_stt_disconnect_then_cancel(request, sidecar, cancel_event)
)
try:
- await asyncio.to_thread(load_stt, payload.model, engine, cancel_event)
+ await asyncio.to_thread(
+ functools.partial(load_stt, payload.model, engine, cancel_event, device = payload.device)
+ )
except SttModelNotDownloadedError as e:
raise HTTPException(status_code = 409, detail = str(e))
except SttUnavailableError as e:
@@ -17481,6 +17579,7 @@ async def stt_validate(
async def stt_unload(
engine: Optional[str] = None,
model: Optional[str] = None,
+ wait: bool = True,
current_subject: str = Depends(get_current_subject),
):
"""Release the local STT model when dictation is idle.
@@ -17490,7 +17589,9 @@ async def stt_unload(
release to the model the caller claims: a surface owns one model, and another
can switch the same engine between that ownership check and this request
arriving, so the sidecar re-checks under its own lock rather than releasing
- whatever happens to be resident by then.
+ whatever happens to be resident by then. ``wait=false`` skips a sidecar that
+ is mid-transcription instead of draining it: a caller freeing memory as a
+ convenience must not cost the user a recording that is still being decoded.
"""
if engine is None:
engines = None
@@ -17507,7 +17608,9 @@ async def stt_unload(
# is not, and only the former takes it positionally. Registry-side it sits
# behind a `*`, so passing it positionally raised TypeError, which is the
# state a fresh process is in before anything has loaded.
- failed: list[str] = await asyncio.to_thread(unload_stt, engines, expected_model = model)
+ failed: list[str] = await asyncio.to_thread(
+ unload_stt, engines, expected_model = model, wait = wait
+ )
if failed:
raise HTTPException(
status_code = 500,
@@ -17523,10 +17626,11 @@ async def _transcribe_audio_bytes(
fast: bool,
engine: Optional[str] = None,
request: Optional[Request] = None,
+ device: Optional[str] = None,
) -> JSONResponse:
"""Run STT for already-decoded request bytes."""
return JSONResponse(
- content = await _transcribe_audio_result(raw, model, language, fast, engine, request)
+ content = await _transcribe_audio_result(raw, model, language, fast, engine, request, device)
)
@@ -17537,6 +17641,7 @@ async def _transcribe_audio_result(
fast: bool,
engine: Optional[str] = None,
request: Optional[Request] = None,
+ device: Optional[str] = None,
) -> dict:
"""STT for already-decoded bytes, sidecar errors mapped to HTTP statuses.
Returns the sidecar's result dict so callers own the response shape."""
@@ -17580,7 +17685,9 @@ async def _transcribe_audio_result(
# timers fired, which is what OOMs a device that fits either alone. A no-op once
# the model is resident, so the steady state costs a residency check.
load_stt, _ = _stt_lifecycle()
- await asyncio.to_thread(load_stt, model, serving_engine, cancel_event)
+ await asyncio.to_thread(
+ functools.partial(load_stt, model, serving_engine, cancel_event, device = device)
+ )
if cancel_event is None:
result = await asyncio.to_thread(sidecar.transcribe, raw, model, language, fast)
else:
@@ -17655,7 +17762,13 @@ async def transcribe_audio(
# Same disconnect cancellation as the raw and OpenAI routes: without the request
# a client that goes away leaves the sidecar transcribing under its lock.
return await _transcribe_audio_bytes(
- raw, payload.model, payload.language, payload.fast, payload.engine, request
+ raw,
+ payload.model,
+ payload.language,
+ payload.fast,
+ payload.engine,
+ request,
+ payload.device,
)
@@ -17666,6 +17779,9 @@ async def transcribe_audio_raw(
language: Optional[str] = None,
fast: bool = False,
engine: Optional[str] = None,
+ # Same literal as the JSON models: a misspelled "cpu" falling through to auto
+ # would silently put the model back on the GPU. 422 instead.
+ device: Optional[Literal["auto", "cpu", "gpu"]] = None,
current_subject: str = Depends(get_current_subject),
):
"""Transcribe a raw audio body without base64 or JSON conversion overhead."""
@@ -17678,7 +17794,7 @@ async def transcribe_audio_raw(
chunks.append(chunk)
return JSONResponse(
content = await _transcribe_audio_result(
- b"".join(chunks), model, language, fast, engine, request
+ b"".join(chunks), model, language, fast, engine, request, device
)
)
diff --git a/studio/backend/routes/training_vram.py b/studio/backend/routes/training_vram.py
index 9720891869d..bd4a43524bf 100644
--- a/studio/backend/routes/training_vram.py
+++ b/studio/backend/routes/training_vram.py
@@ -418,6 +418,10 @@ def free_chat_models_for_training(reason: str) -> List[str]:
try:
from core.inference import get_inference_backend
inf = get_inference_backend()
+ # No CPU exemption here, unlike the GGUF branch and the STT sidecars: it would
+ # key off a marker the orchestrator writes rather than the worker that masked,
+ # and a marker that disagreed is an OOM mid-training. Freeing a model that held
+ # no VRAM only costs a reload.
if inf.active_model_name or inf.loading_models:
name = inf.active_model_name or next(iter(inf.loading_models), None)
logger.info(
@@ -452,6 +456,33 @@ def free_chat_models_for_training(reason: str) -> List[str]:
return freed
+def _stt_sidecar_holds_no_vram(sidecar) -> bool:
+ """True only when the resident dictation model is provably in CPU RAM.
+
+ Conservative on purpose: anything unreadable answers False and the sidecar is
+ freed as before. Skipping one that does hold VRAM would starve the run this
+ is making room for, which is far worse than a needless reload.
+ """
+ try:
+ device = getattr(sidecar, "device", None)
+ if isinstance(device, str) and device.strip().lower() == "cpu":
+ return True
+ # whisper.cpp and llama.cpp report a runtime name rather than a device, so read
+ # the flag each sets when it started without the GPU. Prefer the fact over the
+ # wish: mtmd's _gpu_disabled is what the live server was started with, while
+ # _forced_cpu is the standing preference, recorded even on the branch that does
+ # NOT restart a server with a request in flight, so reading it reports a server
+ # still at -ngl 99 as holding no VRAM.
+ gpu_disabled = getattr(sidecar, "_gpu_disabled", None)
+ if gpu_disabled is not None:
+ return gpu_disabled is True
+ # whisper.cpp sets _forced_cpu only alongside a spawned --no-gpu and clears it
+ # on release, so there it is the fact.
+ return getattr(sidecar, "_forced_cpu", False) is True
+ except Exception: # noqa: BLE001 - a probe must never fail the release it precedes
+ return False
+
+
def free_stt_model_for_training(reason: str) -> List[str]:
"""Unload the dictation model(s) before training. Never raises.
@@ -475,7 +506,11 @@ def free_stt_model_for_training(reason: str) -> List[str]:
freed.append("stt:loading")
else:
model = sidecar.loaded_model
- if model:
+ if model and _stt_sidecar_holds_no_vram(sidecar):
+ logger.info(
+ "Keeping CPU-placed STT model '%s' through training (%s)", model, reason
+ )
+ elif model:
logger.info("Unloading STT model '%s' for training (%s)", model, reason)
sidecar.unload()
freed.append(f"stt:{model}")
@@ -497,7 +532,13 @@ def free_stt_model_for_training(reason: str) -> List[str]:
freed.append("stt:gguf-loading")
else:
ggml_model = ggml.loaded_model
- if ggml_model:
+ if ggml_model and _stt_sidecar_holds_no_vram(ggml):
+ logger.info(
+ "Keeping CPU-placed GGUF STT model '%s' through training (%s)",
+ ggml_model,
+ reason,
+ )
+ elif ggml_model:
logger.info("Unloading GGUF STT model '%s' for training (%s)", ggml_model, reason)
ggml.unload()
freed.append(f"stt:{ggml_model}")
@@ -518,7 +559,13 @@ def free_stt_model_for_training(reason: str) -> List[str]:
freed.append("stt:mtmd-loading")
else:
mtmd_model = mtmd.loaded_model
- if mtmd_model:
+ if mtmd_model and _stt_sidecar_holds_no_vram(mtmd):
+ logger.info(
+ "Keeping CPU-placed mtmd STT model '%s' through training (%s)",
+ mtmd_model,
+ reason,
+ )
+ elif mtmd_model:
logger.info("Unloading mtmd STT model '%s' for training (%s)", mtmd_model, reason)
mtmd.unload()
freed.append(f"stt:{mtmd_model}")
diff --git a/studio/backend/tests/test_audio_device_guards.py b/studio/backend/tests/test_audio_device_guards.py
new file mode 100644
index 00000000000..7909afb7065
--- /dev/null
+++ b/studio/backend/tests/test_audio_device_guards.py
@@ -0,0 +1,471 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Guards that must not charge a CPU-placed audio load for VRAM it never takes.
+
+A load held in system RAM still passed the training coexistence check, the GPU
+arbiter and the memory preflight, so it could be refused on a full card, evict an
+image or video pipeline, or be reported already-loaded while sitting on the GPU.
+"""
+
+import asyncio
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from fastapi import HTTPException # noqa: E402
+
+import routes.inference as ri # noqa: E402
+from routes.training_vram import _stt_sidecar_holds_no_vram # noqa: E402
+
+
[email protected](autouse = True)
+def _neutral_audio_device_env(monkeypatch):
+ """A server-wide default must not decide the outcome of these tests.
+
+ Placement here is asserted against no opinion, so a host that sets
+ UNSLOTH_AUDIO_DEVICE would fail these on correct behaviour, and that host is
+ exactly the one most likely to run them.
+ """
+ monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
+
+
+def _audio(audio_type = "higgs_tts2", **kwargs):
+ return types.SimpleNamespace(audio_type = audio_type, is_lora = False, identifier = "x/y", **kwargs)
+
+
+def _request(audio_device = None):
+ return types.SimpleNamespace(audio_device = audio_device)
+
+
+def test_only_a_native_audio_model_counts_as_a_cpu_audio_load():
+ assert ri._native_audio_cpu_load(_audio(), _request("cpu"))
+ assert not ri._native_audio_cpu_load(_audio(), _request("auto"))
+ assert not ri._native_audio_cpu_load(_audio(), _request(None))
+
+
+def test_a_chat_model_cannot_skip_the_guards_by_sending_audio_device():
+ """audio_device is documented as ignored off the audio path. If it were not
+ gated here, any load could set it and walk past the training guard."""
+ assert not ri._native_audio_cpu_load(_audio(audio_type = None), _request("cpu"))
+ assert not ri._native_audio_cpu_load(_audio(audio_type = "whisper"), _request("cpu"))
+
+
+def test_a_cpu_audio_load_is_not_refused_while_training_runs(monkeypatch):
+ """The guard 409s an unsized load, and refuses everything outright during
+ diffusion training. Neither applies to weights that never reach the card."""
+ monkeypatch.setattr(ri, "_diffusion_training_active", lambda: True)
+
+ assert (
+ ri._guard_chat_load_against_training(
+ _audio(is_gguf = False),
+ types.SimpleNamespace(
+ audio_device = "cpu",
+ gpu_memory_mode = "auto",
+ gpu_layers = -1,
+ tensor_parallel = False,
+ ),
+ load_in_4bit = False,
+ placement = types.SimpleNamespace(
+ requested_gpu_ids = None,
+ gpu_ids_are_vulkan_ordinals = False,
+ diffusion_kind = None,
+ ),
+ )
+ is None
+ )
+
+
+def test_a_gpu_audio_load_is_still_refused_during_diffusion_training(monkeypatch):
+ """The exemption must be the CPU placement, not the audio type."""
+ monkeypatch.setattr(ri, "_diffusion_training_active", lambda: True)
+
+ with pytest.raises(HTTPException) as excinfo:
+ ri._guard_chat_load_against_training(
+ _audio(is_gguf = False),
+ types.SimpleNamespace(
+ audio_device = "auto",
+ gpu_memory_mode = "auto",
+ gpu_layers = -1,
+ tensor_parallel = False,
+ ),
+ load_in_4bit = False,
+ placement = types.SimpleNamespace(
+ requested_gpu_ids = None,
+ gpu_ids_are_vulkan_ordinals = False,
+ diffusion_kind = None,
+ ),
+ )
+ assert excinfo.value.status_code == 409
+
+
+def test_a_cpu_load_skips_the_vram_preflight_entirely(monkeypatch):
+ """Sizing it would refuse the load on a full GPU, which is the case the
+ option exists for. The probe must not even be reached."""
+
+ def _never():
+ raise AssertionError("a CPU load must not size GPU memory")
+
+ monkeypatch.setattr(ri, "_native_audio_post_handoff_free_gb", _never)
+ placement = types.SimpleNamespace(requested_gpu_ids = None)
+
+ result = asyncio.run(ri._preflight_native_audio_placement(_audio(), _request("cpu"), placement))
+ assert result is placement
+
+
+def test_minimax_on_cpu_is_refused_before_the_resident_model_is_evicted():
+ """Its runtime needs CUDA. Failing later in the worker would cost the user
+ the model they already had, since the switch evicts before the load runs."""
+ with pytest.raises(HTTPException) as excinfo:
+ asyncio.run(
+ ri._preflight_native_audio_placement(
+ _audio(audio_type = "minimax_music3"),
+ _request("cpu"),
+ types.SimpleNamespace(requested_gpu_ids = None),
+ )
+ )
+ assert excinfo.value.status_code == 400
+ assert "CPU RAM" in excinfo.value.detail
+
+
+def _backend(audio_cpu, audio_type = "higgs_tts2"):
+ entry = {"is_audio": True, "audio_type": audio_type}
+ if audio_cpu is not None:
+ entry["audio_cpu"] = audio_cpu
+ return types.SimpleNamespace(active_model_name = "x/y", models = {"x/y": entry})
+
+
+def test_a_resident_gpu_audio_model_does_not_satisfy_a_cpu_request():
+ assert not ri._resident_audio_placement_matches(_backend(audio_cpu = False), _request("cpu"))
+
+
+def test_a_resident_cpu_audio_model_satisfies_the_same_request_again():
+ assert ri._resident_audio_placement_matches(_backend(audio_cpu = True), _request("cpu"))
+
+
+def test_a_model_loaded_before_this_existed_is_read_as_gpu():
+ """No recorded key means the load predates the option, which placed on GPU."""
+ assert ri._resident_audio_placement_matches(_backend(audio_cpu = None), _request("auto"))
+ assert not ri._resident_audio_placement_matches(_backend(audio_cpu = None), _request("cpu"))
+
+
+def test_a_non_audio_model_keeps_the_shortcut():
+ assert ri._resident_audio_placement_matches(
+ _backend(audio_cpu = None, audio_type = None), _request("cpu")
+ )
+
+
+def test_a_cpu_placed_sidecar_is_left_alone_when_training_claims_vram():
+ assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "cpu"))
+ assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = True))
+ assert _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "llama.cpp", _gpu_disabled = True))
+
+
+def test_anything_that_might_hold_vram_is_still_evicted():
+ """Default-deny: starving the run this makes room for is the worse failure."""
+ assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "cuda"))
+ assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "mps"))
+ assert not _stt_sidecar_holds_no_vram(
+ types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = False)
+ )
+ assert not _stt_sidecar_holds_no_vram(types.SimpleNamespace())
+
+ class _Raises:
+ @property
+ def device(self):
+ raise RuntimeError("unreadable")
+
+ assert not _stt_sidecar_holds_no_vram(_Raises())
+
+
+def test_the_shortcut_reads_the_resident_model_not_the_requested_config():
+ """It runs ahead of config resolution, so reading a config there raised
+ UnboundLocalError and turned every repeat safetensors load into a 500."""
+ import inspect
+ assert list(inspect.signature(ri._resident_audio_placement_matches).parameters) == [
+ "backend",
+ "request",
+ ]
+
+
+def test_a_cpu_placed_audio_model_never_takes_the_arbiter():
+ """It holds no GPU, so acquiring would cancel an image or video run for nothing."""
+ assert ri._resident_audio_holds_no_gpu(_backend(audio_cpu = True))
+ assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = False))
+ assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = None))
+
+
+def test_a_chat_model_cannot_reach_the_arbiter_skip():
+ """Same audio-type gate as the writer, so a stray marker cannot skip it."""
+ assert not ri._resident_audio_holds_no_gpu(_backend(audio_cpu = True, audio_type = None))
+
+
+def test_nothing_resident_reads_as_holding_the_gpu():
+ empty = types.SimpleNamespace(active_model_name = None, models = {})
+ assert not ri._resident_audio_holds_no_gpu(empty)
+
+
+def _inference_source() -> str:
+ import pathlib
+
+ # Explicit encoding: read_text() defaults to the locale one, cp1252 on Windows,
+ # and this file is UTF-8.
+ return pathlib.Path(ri.__file__).read_text(encoding = "utf-8")
+
+
+def test_the_already_loaded_branch_guards_its_acquire():
+ src = _inference_source()
+ assert "if not _resident_audio_holds_no_gpu(backend):\n" in src
+
+
+def test_the_post_load_ownership_check_is_gated_like_the_gguf_one():
+ """Ungated, it unloads the CPU audio model it just loaded and returns 409:
+ the load skips acquire_for, so on a clean server the owner is None."""
+ src = _inference_source()
+ assert "if chat_load_needs_gpu and current_owner() != CHAT:" in src
+ assert "\n if current_owner() != CHAT:" not in src
+
+
+def test_a_cpu_audio_worker_hides_cuda_and_hip():
+ from core.inference.audio_device import mask_accelerators_for_cpu_audio
+
+ env = {"CUDA_VISIBLE_DEVICES": "0,1", "HIP_VISIBLE_DEVICES": "0"}
+ mask_accelerators_for_cpu_audio(env)
+ assert env["CUDA_VISIBLE_DEVICES"] == ""
+ # HIP reads the CUDA variable only when its own is unset, so blanking one is
+ # not enough; -1 is the sentinel the CPU embed server already uses.
+ assert env["HIP_VISIBLE_DEVICES"] == "-1"
+
+
+def test_an_inherited_rocr_mask_is_left_alone():
+ """Clearing it exposes more agents to HSA enumeration, not fewer."""
+ from core.inference.audio_device import mask_accelerators_for_cpu_audio
+
+ env = {"ROCR_VISIBLE_DEVICES": "0"}
+ mask_accelerators_for_cpu_audio(env)
+ assert env["ROCR_VISIBLE_DEVICES"] == "0"
+
+
+def test_the_mask_runs_before_hardware_detection():
+ """detect_hardware() calls get_device_properties, which creates the context
+ this load is supposed not to hold, so masking after it is too late."""
+ import pathlib
+ from core.inference import worker
+
+ src = pathlib.Path(worker.__file__).read_text(encoding = "utf-8")
+ assert src.index("mask_accelerators_for_cpu_audio(os.environ)") < src.index(
+ "_hw.detect_hardware()"
+ )
+
+
+def test_a_zero_gpu_standard_load_drops_the_stale_chat_claim():
+ """The load replaced whatever held CHAT. Leaving the claim makes the next
+ Images/Video acquire run the CHAT evictor and unload a CPU audio model that
+ was never on the GPU. The GGUF branch already releases; both do now.
+
+ Two awaited sites: the GGUF branch, and the standard branch after its load
+ (which cannot cover a claim re-taken while it ran). The standard branch's
+ during-load release is the third, handed to load_model as a callback so it
+ fires once the previous worker is gone rather than before it."""
+ src = _inference_source()
+ assert src.count("await asyncio.to_thread(release, CHAT)") == 2
+ assert src.count("(lambda: release(CHAT)) if not chat_load_needs_gpu else None") == 1
+
+
+def test_the_release_is_gated_on_the_same_flag_as_the_409():
+ """Two `if not` sites plus the callback's own inline gate, which spells the
+ flag the same way."""
+ src = _inference_source()
+ assert src.count("if not chat_load_needs_gpu:") == 2
+ assert src.count("if not chat_load_needs_gpu else None") == 1
+
+
+def test_every_http_device_field_pins_the_three_canonical_values():
+ """A misspelled "cpu" that fell through to auto would put the model back on
+ the GPU without saying so. 422 is the honest answer at the boundary."""
+ import inspect
+ import typing
+
+ from models.inference import (
+ LoadRequest,
+ SttLoadRequest,
+ TranscribeRequest,
+ ValidateModelRequest,
+ )
+
+ expected = typing.Optional[typing.Literal["auto", "cpu", "gpu"]]
+ for model, field in (
+ (LoadRequest, "audio_device"),
+ (ValidateModelRequest, "audio_device"),
+ (SttLoadRequest, "device"),
+ (TranscribeRequest, "device"),
+ ):
+ assert model.model_fields[field].annotation == expected, f"{model.__name__}.{field}"
+ # The raw endpoint takes it as a query param, so it is annotated rather than
+ # declared on a model; it must not be the odd one out.
+ assert inspect.signature(ri.transcribe_audio_raw).parameters["device"].annotation == expected
+
+
+def test_a_gpu_resident_mtmd_server_is_never_reported_as_holding_no_vram():
+ """mtmd records the user's wish on the branch that does not restart the server.
+
+ _load_locked writes _forced_cpu even when an in-flight request keeps the running
+ server, so a llama-server still at -ngl 99 carries a CPU wish. Trusting it would
+ let training start beside a model that holds the whole checkpoint in VRAM.
+ """
+ resident_on_gpu = types.SimpleNamespace(
+ device = "llama.cpp",
+ _gpu_disabled = False,
+ _forced_cpu = True,
+ )
+ assert _stt_sidecar_holds_no_vram(resident_on_gpu) is False
+
+ really_cpu = types.SimpleNamespace(
+ device = "llama.cpp",
+ _gpu_disabled = True,
+ _forced_cpu = True,
+ )
+ assert _stt_sidecar_holds_no_vram(really_cpu) is True
+
+
+def test_whisper_cpp_still_exempts_a_server_started_with_no_gpu():
+ """ggml has no separate wish: _forced_cpu sits next to the spawned --no-gpu."""
+ assert (
+ _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = True))
+ is True
+ )
+ assert (
+ _stt_sidecar_holds_no_vram(types.SimpleNamespace(device = "whisper.cpp", _forced_cpu = False))
+ is False
+ )
+
+
+def test_a_forced_cpu_native_audio_load_selects_no_gpu():
+ """Selection returns every card needed to hold the checkpoint, and the worker
+ forwards that list to a backend that rejects more than one. Its required_gb also
+ becomes expected_free_gb, whose settle wait raises when Images holds the card."""
+ import inspect
+
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ src = inspect.getsource(InferenceOrchestrator.load_model)
+ head = src[: src.index("prepare_gpu_selection(")]
+ assert "audio_device_forces_cpu" in head
+ assert "is_native_audio_model" in head
+
+
+def test_the_stale_chat_claim_is_dropped_before_the_load_not_only_after():
+ """A download and load can run for minutes. Held across that window the claim
+ makes an Images acquire run the chat evictor, which cancels this load."""
+ import inspect
+
+ src = inspect.getsource(ri._load_model_impl)
+ before_load = src[: src.index("backend.load_model,")]
+ assert before_load.count("release, CHAT") >= 1
+
+
+def test_the_gguf_audio_codec_follows_the_servers_own_placement():
+ """The codec is a second allocation. A server launched at zero offload is
+ classified holds_no_vram, which lets the route skip GPU arbitration and leaves
+ it resident when training reclaims memory; a codec on CUDA would hold VRAM
+ under both of those promises."""
+ import inspect
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ src = inspect.getsource(LlamaCppBackend.init_audio_codec)
+ device_line = next(l for l in src.splitlines() if l.strip().startswith("device ="))
+ assert "holds_no_vram" in device_line, device_line
+
+
+def test_decoding_happens_where_the_codec_actually_is():
+ """Loading the codec on CPU is only half of it.
+
+ The decoders build their input tensors on the device they are handed. Handed
+ CUDA for a CPU-resident codec, SNAC and DAC fail outright on a device mismatch
+ and BiCodec moves the codec onto the card, taking the VRAM a CPU RAM load
+ promised not to take. The recorded placement wins over the caller's request.
+ """
+ import torch
+
+ from core.inference.audio_codecs import AudioCodecManager
+
+ mgr = AudioCodecManager()
+ seen = {}
+
+ class _FakeSnac:
+ def decode(self, codes):
+ seen["device"] = codes[0].device.type
+ return torch.zeros(1, 8)
+
+ mgr._snac_model = _FakeSnac()
+ mgr._codec_devices["snac"] = "cpu"
+
+ # 128257 opens the speech section; the seven codes after it make one frame.
+ token_ids = [128257] + [128266 + i for i in range(7)]
+ mgr.decode("snac", "cuda", token_ids = token_ids)
+
+ assert seen["device"] == "cpu"
+
+
+def test_a_codec_with_no_recorded_placement_still_honours_the_caller():
+ """Nothing is recorded until a codec is actually loaded, and every pre-existing
+ caller passes the device it wants. An unknown codec must keep deferring to it."""
+ from core.inference.audio_codecs import AudioCodecManager
+
+ mgr = AudioCodecManager()
+ assert mgr._codec_devices == {}
+
+
+def test_a_cpu_load_leaves_a_running_export_alone():
+ """The export teardown exists to free VRAM for the incoming model. A load that
+ masks the accelerators wants none of it, so killing the user's export buys
+ nothing and costs them the job."""
+ import inspect
+
+ src = inspect.getsource(ri._load_model_impl)
+ guard = next(
+ l for l in src.splitlines() if "exp_backend.current_checkpoint" in l and "if " in l
+ )
+ assert "chat_load_needs_gpu" in guard, guard
+
+
+def test_the_chat_claim_outlives_the_worker_that_earned_it():
+ """Released before the load, the claim leaves a still-resident GPU model
+ unowned, and acquire_for evicts nobody when the arbiter has no owner: an
+ Images or Video load then allocates straight over it. The release is handed to
+ load_model, which runs it once the previous worker is gone and its memory is
+ back, still ahead of the download."""
+ import inspect
+
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ src = inspect.getsource(ri._load_model_impl)
+ assert "on_prior_worker_released = _release_chat_after_teardown" in src
+
+ sig = inspect.signature(InferenceOrchestrator.load_model)
+ assert "on_prior_worker_released" in sig.parameters
+
+ # After both bail-outs: either one means the card is still busy, so the claim is
+ # still true.
+ load_src = inspect.getsource(InferenceOrchestrator.load_model)
+ hook = load_src.index("on_prior_worker_released()")
+ assert load_src.index("did not exit and still holds GPU") < hook
+ assert load_src.index("was not released; ") < hook
+
+
+def test_whisper_cpp_publishes_the_load_before_choosing_its_placement():
+ """The training hook reads is_loading() without this method's lock. A placement
+ decided ahead of that flag is invisible to training: it sees no load in flight,
+ reads the outgoing model's CPU placement, preserves it as holding no VRAM, and
+ the command already built then starts a GPU-backed server beside the run."""
+ import inspect
+
+ from core.inference.stt_ggml_sidecar import GgmlSttSidecar
+
+ src = inspect.getsource(GgmlSttSidecar.load)
+ assert src.index("self._loading = True") < src.index("elif _training_active():")
diff --git a/studio/backend/tests/test_audio_device_preference.py b/studio/backend/tests/test_audio_device_preference.py
new file mode 100644
index 00000000000..e811899cc21
--- /dev/null
+++ b/studio/backend/tests/test_audio_device_preference.py
@@ -0,0 +1,360 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The user's choice of where an audio model's weights go.
+
+Audio loads pick an accelerator on their own. These cover the option that
+overrides that: CPU RAM must win over a working GPU, "auto" must still detect,
+and a resident model loaded under the other preference must be reloaded rather
+than reused where it is.
+"""
+
+import sys
+import threading
+import types
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from core.inference.audio_device import ( # noqa: E402
+ audio_device_default,
+ audio_device_forces_cpu,
+ normalize_audio_device,
+)
+
+
[email protected](autouse = True)
+def _neutral_audio_device_env(monkeypatch):
+ """A server-wide default must not decide the outcome of these tests.
+
+ Placement here is asserted against no opinion, so a host that sets
+ UNSLOTH_AUDIO_DEVICE would fail these on correct behaviour, and that host is
+ exactly the one most likely to run them.
+ """
+ monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
+
+
+ "value, expected",
+ [
+ ("cpu", "cpu"),
+ ("CPU", "cpu"),
+ (" cpu ", "cpu"),
+ ("ram", "cpu"),
+ ("gpu", "gpu"),
+ ("cuda", "gpu"),
+ ("mps", "gpu"),
+ ("rocm", "gpu"),
+ ("auto", "auto"),
+ ("", "auto"),
+ (None, "auto"),
+ ],
+)
+def test_every_accepted_spelling_maps_onto_one_of_three_values(value, expected):
+ assert normalize_audio_device(value) == expected
+
+
+def test_an_unknown_preference_detects_rather_than_failing_the_load():
+ """Detection is what the caller would have done without the option at all."""
+ assert normalize_audio_device("gpu-2") == "auto"
+ assert normalize_audio_device("nvidia rtx 4090") == "auto"
+ assert not audio_device_forces_cpu("gpu-2")
+
+
+def test_the_environment_supplies_the_default_for_a_request_that_names_none(monkeypatch):
+ """A headless or CLI Studio sets this once instead of sending it per request."""
+ monkeypatch.setenv("UNSLOTH_AUDIO_DEVICE", "cpu")
+ assert audio_device_default() == "cpu"
+ assert audio_device_forces_cpu(None)
+
+ # An explicit request still outranks it, in both directions.
+ assert not audio_device_forces_cpu("auto")
+ assert audio_device_forces_cpu("cpu")
+
+
+def test_without_the_environment_variable_nothing_is_forced_to_cpu(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
+ assert audio_device_default() == "auto"
+ assert not audio_device_forces_cpu(None)
+
+
+def _torch_with_cuda(monkeypatch):
+ """A torch whose CUDA is available, so anything but CPU is a real choice."""
+ torch = types.SimpleNamespace(
+ float16 = "float16",
+ float32 = "float32",
+ cuda = types.SimpleNamespace(is_available = lambda: True),
+ backends = types.SimpleNamespace(mps = types.SimpleNamespace(is_available = lambda: False)),
+ )
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ return torch
+
+
+def test_cpu_is_honoured_on_a_machine_with_a_working_gpu(monkeypatch):
+ """The whole point of the option: detection would have said cuda."""
+ from core.inference import stt_sidecar
+
+ _torch_with_cuda(monkeypatch)
+ monkeypatch.setattr(stt_sidecar, "_training_active", lambda: False)
+
+ assert stt_sidecar._pick_device("cpu") == ("cpu", "float32")
+
+
+def test_auto_and_gpu_both_still_detect_the_accelerator(monkeypatch):
+ """ "gpu" is not a separate placement: detection already prefers the card."""
+ from core.inference import stt_sidecar
+
+ _torch_with_cuda(monkeypatch)
+ monkeypatch.setattr(stt_sidecar, "_training_active", lambda: False)
+
+ assert stt_sidecar._pick_device("auto") == ("cuda", "float16")
+ assert stt_sidecar._pick_device("gpu") == ("cuda", "float16")
+ assert stt_sidecar._pick_device(None) == ("cuda", "float16")
+
+
+def test_a_resident_model_on_the_other_device_is_reloaded_not_reused(monkeypatch):
+ """A preference is a request about placement. Reusing the old one ignores it."""
+ from core.inference import stt_sidecar
+
+ monkeypatch.setattr(stt_sidecar, "_pick_device", lambda _preference = None: ("cpu", "float32"))
+ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
+
+ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
+ sidecar._engine = object()
+ sidecar._model_id = "small"
+ sidecar._device = "cuda"
+ sidecar._device_preference = "auto"
+
+ builds: list[str] = []
+
+ def _build(snapshot_path, device, dtype, cancel_event):
+ builds.append(device)
+ return object()
+
+ monkeypatch.setattr(sidecar, "_build_model", _build)
+ monkeypatch.setattr(
+ sidecar,
+ "_ensure_model_downloaded",
+ lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(
+ # The resident shortcut answers with no path; a replacement load needs one.
+ path = None if use_resident else "/snapshots/small",
+ is_multilingual = True,
+ ),
+ )
+ monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)
+
+ sidecar.load("small", device = "cpu")
+
+ assert builds == ["cpu"], "the CPU preference must have driven a fresh load"
+ assert sidecar._device == "cpu"
+ assert sidecar._device_preference == "cpu"
+
+
+def test_the_same_preference_reuses_the_resident_model(monkeypatch):
+ """Unchanged placement must stay a residency check, not a reload."""
+ from core.inference import stt_sidecar
+
+ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
+
+ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
+ resident = object()
+ sidecar._engine = resident
+ sidecar._model_id = "small"
+ sidecar._device = "cpu"
+ sidecar._device_preference = "cpu"
+
+ def _never(*args, **kwargs):
+ raise AssertionError("a matching preference must not rebuild the model")
+
+ monkeypatch.setattr(sidecar, "_build_model", _never)
+
+ assert sidecar.load("small", device = "cpu") is resident
+
+
+def test_a_model_loaded_before_the_option_existed_is_not_reloaded(monkeypatch):
+ """No recorded preference means the engine predates the choice; reusing it is
+ what every caller got before, and a forced reload would cost a load for nothing."""
+ from core.inference import stt_sidecar
+
+ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
+
+ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
+ resident = object()
+ sidecar._engine = resident
+ sidecar._model_id = "small"
+ sidecar._device = "cuda"
+ sidecar._device_preference = None
+
+ monkeypatch.setattr(
+ sidecar,
+ "_build_model",
+ lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not rebuild")),
+ )
+
+ assert sidecar.load("small", device = "auto") is resident
+
+
+def test_the_registry_hands_the_preference_to_the_engines_sidecar(monkeypatch):
+ from core.inference import stt_registry
+
+ seen: dict = {}
+
+ class _Sidecar:
+ def load(
+ self,
+ model,
+ request_cancel_event = None,
+ device = None,
+ ):
+ seen["model"] = model
+ seen["device"] = device
+
+ monkeypatch.setattr(stt_registry, "sidecar_for", lambda _engine: _Sidecar())
+ monkeypatch.setattr(stt_registry, "_model_is_downloaded", lambda _e, _m: True)
+ monkeypatch.setattr(stt_registry, "unload", lambda *a, **k: [])
+
+ stt_registry.load("small", "transformers", threading.Event(), device = "cpu")
+
+ assert seen == {"model": "small", "device": "cpu"}
+
+
+def test_native_audio_holds_tts_weights_in_cpu_ram_when_asked(monkeypatch):
+ from core.inference import native_audio
+
+ _torch_with_cuda(monkeypatch)
+
+ assert native_audio.NativeAudioBackend(device_preference = "cpu").device == "cpu"
+ assert native_audio.NativeAudioBackend(device_preference = "auto").device == "cuda"
+ assert native_audio.NativeAudioBackend().device == "cuda"
+
+
+def test_minimax_music_explains_that_cpu_was_chosen_rather_than_missing(monkeypatch):
+ """The generic refusal reads like "your hardware cannot do this" and sends a
+ user with a perfectly good card looking for one."""
+ from core.inference import native_audio
+
+ _torch_with_cuda(monkeypatch)
+ backend = native_audio.NativeAudioBackend(device_preference = "cpu")
+ config = types.SimpleNamespace(
+ identifier = "MiniMaxAI/MiniMax-Music3",
+ audio_type = "minimax_music3",
+ path = None,
+ )
+
+ with pytest.raises(RuntimeError, match = "cannot be loaded into CPU RAM"):
+ backend.load_model(config)
+
+
+def test_a_caller_that_sends_no_device_leaves_the_placement_alone(monkeypatch):
+ """``/v1/audio/transcriptions`` sends none. Treating that as "auto" pulled a
+ CPU model back onto the GPU, and the next dictation pulled it off again:
+ two full reloads per alternation, and VRAM the user asked us not to take."""
+ from core.inference import stt_sidecar
+
+ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
+ monkeypatch.setattr(
+ stt_sidecar,
+ "_pick_device",
+ lambda preference = None: ("cpu", "float32") if preference == "cpu" else ("cuda", "float16"),
+ )
+
+ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
+ builds: list[str] = []
+ monkeypatch.setattr(
+ sidecar, "_build_model", lambda p, device, dtype, c: builds.append(device) or object()
+ )
+ monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)
+ monkeypatch.setattr(
+ sidecar,
+ "_ensure_model_downloaded",
+ lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(
+ path = None if (sidecar._engine is not None and use_resident) else "/snapshots/small",
+ is_multilingual = True,
+ ),
+ )
+
+ sidecar.load("small", device = "cpu") # Voice settings: the user picked CPU
+ sidecar.load("small", device = None) # OpenAI-compatible route: no opinion
+ sidecar.load("small", device = "cpu") # the next dictation
+
+ assert builds == ["cpu"], "only the first load should have built anything"
+ assert sidecar._device == "cpu"
+
+
+def test_an_explicit_change_still_reloads_after_a_no_opinion_call(monkeypatch):
+ """No opinion must not also freeze the placement: the setting still moves it."""
+ from core.inference import stt_sidecar
+
+ monkeypatch.setattr(stt_sidecar, "ensure_stt_available", lambda: None)
+ monkeypatch.setattr(stt_sidecar, "resolve_model_id", lambda model: model or "small")
+ monkeypatch.setattr(
+ stt_sidecar,
+ "_pick_device",
+ lambda preference = None: ("cpu", "float32") if preference == "cpu" else ("cuda", "float16"),
+ )
+
+ sidecar = stt_sidecar.WhisperSttSidecar(keep_alive_seconds = 0.0)
+ builds: list[str] = []
+ monkeypatch.setattr(
+ sidecar, "_build_model", lambda p, device, dtype, c: builds.append(device) or object()
+ )
+ monkeypatch.setattr(sidecar, "_release_engine_locked", lambda: True)
+ monkeypatch.setattr(
+ sidecar,
+ "_ensure_model_downloaded",
+ lambda model_id, use_resident = True: stt_sidecar._CachedSttSnapshot(
+ path = None if (sidecar._engine is not None and use_resident) else "/snapshots/small",
+ is_multilingual = True,
+ ),
+ )
+
+ sidecar.load("small", device = "cpu")
+ sidecar.load("small", device = None)
+ sidecar.load("small", device = "auto")
+
+ assert builds == ["cpu", "cuda"]
+
+
+def test_the_mtmd_reuse_branch_still_records_the_choice(monkeypatch):
+ """Training makes an explicit "cpu" indistinguishable from the running
+ server, so the early return has to keep the preference or the next
+ device-less load sends the model back to the GPU once training ends."""
+ from core.inference import stt_mtmd_sidecar
+
+ sidecar = stt_mtmd_sidecar.MtmdSttSidecar.__new__(stt_mtmd_sidecar.MtmdSttSidecar)
+ sidecar._lock = threading.RLock()
+ sidecar._forced_cpu = False
+ sidecar._gpu_disabled = True
+ sidecar._model_id = "m"
+ sidecar._binary_path_revision = 1
+ sidecar._active_requests = 0
+ monkeypatch.setattr(sidecar, "_process_alive", lambda: True)
+ monkeypatch.setattr(sidecar, "_schedule_idle_unload_locked", lambda: None)
+ monkeypatch.setattr(stt_mtmd_sidecar, "_training_active", lambda: True)
+
+ sidecar._load_locked("m", "whisper-server", path_revision = 1, device = "cpu")
+
+ assert sidecar._forced_cpu is True
+
+
+def test_stt_unload_can_skip_a_sidecar_that_is_mid_transcription():
+ """The Voice device switch is an early release, not a reclaim: draining a
+ live transcription for 30s and then killing it loses the recording."""
+ import inspect
+
+ from core.inference import stt_registry
+ from core.inference.orchestrator import InferenceOrchestrator
+ from routes import inference as ri
+
+ assert "wait" in inspect.signature(ri.stt_unload).parameters
+ assert inspect.signature(ri.stt_unload).parameters["wait"].default is True
+ # Both halves of _stt_lifecycle have to take it, or the route raises TypeError
+ # for whichever one is live.
+ assert "wait" in inspect.signature(InferenceOrchestrator.unload_stt_model).parameters
+ assert "wait" in inspect.signature(stt_registry.unload).parameters
diff --git a/studio/backend/tests/test_gpu_arbiter.py b/studio/backend/tests/test_gpu_arbiter.py
index de553b9de41..c2b06e44a65 100644
--- a/studio/backend/tests/test_gpu_arbiter.py
+++ b/studio/backend/tests/test_gpu_arbiter.py
@@ -343,6 +343,9 @@ def test_the_safetensors_load_yields_a_gpu_it_lost_while_loading():
load_impl = route_src[route_src.index("async def _load_model_impl") :]
unsloth_load = load_impl.index("success = await asyncio.to_thread(")
tail = load_impl[unsloth_load:]
- guard = tail.index("if current_owner() != CHAT:")
+ # Gated on chat_load_needs_gpu like the GGUF branch: a load that never took the
+ # arbiter has no ownership to lose, and checking against a None owner would 409
+ # every CPU-placed audio load.
+ guard = tail.index("if chat_load_needs_gpu and current_owner() != CHAT:")
assert "await asyncio.to_thread(backend.unload_model, config.identifier)" in tail[guard:]
assert tail.index("status_code = 409", guard) > guard
diff --git a/studio/backend/tests/test_openai_audio_transcriptions_route.py b/studio/backend/tests/test_openai_audio_transcriptions_route.py
index b6681b4afcb..78ee761ed3f 100644
--- a/studio/backend/tests/test_openai_audio_transcriptions_route.py
+++ b/studio/backend/tests/test_openai_audio_transcriptions_route.py
@@ -31,6 +31,7 @@ def _make_client(monkeypatch, transcribe = None):
fast,
engine = None,
request = None,
+ device = None,
):
calls.append(
{
diff --git a/studio/backend/tests/test_stt_registry.py b/studio/backend/tests/test_stt_registry.py
index 483a729cb2b..7d3b3e4a983 100644
--- a/studio/backend/tests/test_stt_registry.py
+++ b/studio/backend/tests/test_stt_registry.py
@@ -27,6 +27,7 @@ class _Sidecar:
self.unload_expected = []
self.loaded_with = None
self.load_cancel_event = None
+ self.loaded_device = None
def is_loading(self):
return self._loading
@@ -35,9 +36,11 @@ class _Sidecar:
self,
model,
request_cancel_event = None,
+ device = None,
):
self.loaded_with = model
self.load_cancel_event = request_cancel_event
+ self.loaded_device = device
def unload(
self,
@@ -85,7 +88,7 @@ def test_load_releases_the_other_engines_after_the_target_loads(monkeypatch):
f"unload:{name}"
)
target = sidecars["mtmd"]
- target.load = lambda model, request_cancel_event = None: order.append("load:mtmd")
+ target.load = lambda model, request_cancel_event = None, device = None: order.append("load:mtmd")
monkeypatch.setattr(stt_registry, "sidecar_for", lambda name: sidecars[name])
stt_registry.load("qwen3-asr-0.6b", "mtmd")
@@ -231,7 +234,11 @@ def test_a_failed_load_leaves_the_engine_the_user_was_using(monkeypatch):
"""The sidecars order preflight before release for this reason; so does the registry."""
sidecars = {name: _Sidecar(name) for name in stt_registry.STT_ENGINES}
- def refuse(model, request_cancel_event = None):
+ def refuse(
+ model,
+ request_cancel_event = None,
+ device = None,
+ ):
raise RuntimeError("STT model 'x' is not downloaded.")
sidecars["mtmd"].load = refuse
@@ -314,6 +321,7 @@ def test_a_downloaded_switch_releases_the_old_engine_before_allocating(monkeypat
self,
model,
request_cancel_event = None,
+ device = None,
):
order.append(("load", model))
@@ -340,6 +348,7 @@ def test_an_undownloaded_switch_keeps_the_resident_engine_until_the_load_succeed
self,
model,
request_cancel_event = None,
+ device = None,
):
order.append(("load", model))
@@ -442,7 +451,10 @@ def test_an_implicit_transcribe_load_releases_the_other_engines(monkeypatch):
monkeypatch.setattr(
ri,
"_stt_lifecycle",
- lambda: (lambda model, engine, cancel = None: loaded.append((model, engine)), lambda *a: []),
+ lambda: (
+ lambda model, engine, cancel = None, device = None: loaded.append((model, engine)),
+ lambda *a: [],
+ ),
)
result = asyncio.run(
diff --git a/studio/backend/tests/test_stt_sidecar.py b/studio/backend/tests/test_stt_sidecar.py
index 704037fc40e..79134626c1e 100644
--- a/studio/backend/tests/test_stt_sidecar.py
+++ b/studio/backend/tests/test_stt_sidecar.py
@@ -35,6 +35,18 @@ from core.inference.stt_sidecar import (
validate_remote_model,
)
+
[email protected](autouse = True)
+def _neutral_audio_device_env(monkeypatch):
+ """A server-wide default must not decide the outcome of these tests.
+
+ Placement here is asserted against no opinion, so a host that sets
+ UNSLOTH_AUDIO_DEVICE would fail these on correct behaviour, and that host is
+ exactly the one most likely to run them.
+ """
+ monkeypatch.delenv("UNSLOTH_AUDIO_DEVICE", raising = False)
+
+
_REAL_DECODE_AUDIO_BOUNDED = stt_sidecar_module._decode_audio_bounded
_REAL_ENSURE_STT_AVAILABLE = stt_sidecar_module.ensure_stt_available
_REAL_SNAPSHOT_IS_COMPLETE = stt_sidecar_module._snapshot_is_complete
@@ -479,7 +491,9 @@ def test_load_hands_the_cached_snapshot_to_the_worker_process(monkeypatch):
# never given back, which is the whole reason the engine is out of process.
_install_fake_torch(monkeypatch)
started = _install_fake_worker(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
sidecar.load("small")
@@ -493,7 +507,9 @@ def test_load_sends_the_dtype_by_name_so_torch_stays_out_of_the_command(monkeypa
fake_torch = _install_fake_torch(monkeypatch)
fake_torch.float16 = "torch.float16"
started = _install_fake_worker(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", fake_torch.float16))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cuda", fake_torch.float16)
+ )
WhisperSttSidecar(keep_alive_seconds = 0).load("small")
@@ -503,7 +519,9 @@ def test_load_sends_the_dtype_by_name_so_torch_stays_out_of_the_command(monkeypa
def test_a_worker_that_died_is_not_resident_and_is_replaced_on_the_next_load(monkeypatch):
_install_fake_torch(monkeypatch)
started = _install_fake_worker(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
worker = sidecar.load("small")
@@ -522,7 +540,9 @@ def test_unload_stops_the_worker_process(monkeypatch):
# empty_cache cannot return the context; ending the process is what does.
_install_fake_torch(monkeypatch)
_install_fake_worker(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
worker = sidecar.load("small")
@@ -537,7 +557,9 @@ def test_a_worker_rejected_by_a_late_cancel_is_stopped_rather_than_leaked(monkey
# start() came back with a live child. Nothing installed that child, and dropping
# the handle does not end the process holding the context training waits for.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
workers = []
@@ -583,7 +605,9 @@ def test_a_late_cancelled_worker_that_outlived_the_kill_stays_resident(monkeypat
# and kill and still holds its accelerator memory, so reporting nothing resident
# would let training be admitted against memory that is not free.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
workers = []
@@ -667,7 +691,9 @@ def test_a_worker_that_outlived_the_kill_stays_resident_rather_than_reported_unl
# still holds its memory, so reporting the model unloaded would let training be
# admitted against memory that is not free.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
workers = _install_unkillable_worker(monkeypatch)
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
@@ -683,7 +709,9 @@ def test_a_new_load_is_refused_while_the_previous_worker_still_holds_its_memory(
# Starting a second child over one that never exited doubles the memory the
# first was unloaded to release, so refuse and let the caller retry.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
workers = _install_unkillable_worker(monkeypatch)
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
@@ -702,7 +730,9 @@ def test_a_surviving_worker_is_not_handed_to_the_next_dictation(monkeypatch):
# terminate and a kill, so handing it to a transcription costs the caller the whole
# command timeout under the model lock. Refuse, and let the retry kill it again.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
workers = _install_unkillable_worker(monkeypatch)
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
@@ -723,7 +753,9 @@ def test_a_worker_wedged_by_a_cancelled_transcription_is_not_handed_to_the_next_
# close() answers False to nobody and the sidecar never learns the child outlived
# both signals. Handing it on spends the whole command timeout under the model lock.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
workers = []
class WedgedByCancelWorker:
@@ -822,7 +854,9 @@ def test_a_child_that_outlived_the_kill_inside_start_stays_accounted(monkeypatch
# installed that handle and it is the only one on the process: dropping it reports
# nothing resident while the context is taken, admitting training against it.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
workers = _install_worker_that_survives_its_own_start(
monkeypatch, SttLoadCancelledError("STT model loading was cancelled so training can start.")
)
@@ -842,7 +876,9 @@ def test_a_load_whose_child_outlived_the_kill_is_not_retried_onto_a_second_child
# The accelerator attempt failed leaving a live child, so the CPU retry would
# put a second child beside it and, installing that one, forget the first.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", "float16"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cuda", "float16")
+ )
workers = _install_worker_that_survives_its_own_start(
monkeypatch, RuntimeError("the dictation worker stopped responding")
)
@@ -861,7 +897,9 @@ def test_an_idle_unload_retries_a_worker_that_outlived_the_kill(monkeypatch):
# Keeping the survivor resident must not strand it: the idle timer is rearmed
# so the release is tried again once the child finally goes.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
monkeypatch.setattr(stt_sidecar_module.threading, "Timer", _FakeTimer)
workers = _install_unkillable_worker(monkeypatch)
@@ -917,7 +955,9 @@ def test_a_worker_whose_liveness_cannot_be_read_stays_resident(monkeypatch):
# An unanswerable probe is not evidence that the context was released, so reporting
# nothing resident would let training be admitted against memory still held.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
def unreadable(_worker):
raise OSError("the process handle cannot be inspected")
@@ -936,7 +976,9 @@ def test_a_close_that_raises_is_a_failed_release(monkeypatch):
# close() raising out of join/terminate/kill confirms nothing dead, so discarding
# the only handle would advertise memory the child is still holding as free.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
def raising(_worker):
raise OSError("the worker could not be joined")
@@ -956,7 +998,9 @@ def test_a_close_that_raises_over_a_child_already_gone_still_releases(monkeypatc
# The other direction: bookkeeping that raised over a confirmed dead child
# holds no memory, and keeping it would refuse every later load.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
def raising_after_death(worker):
worker.alive = False
@@ -980,7 +1024,9 @@ def test_a_worker_being_reaped_stays_visible_to_training_admission(monkeypatch):
# lock, so clearing them before the reap would report nothing resident for the
# whole close, and training would start into the child's accelerator memory.
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
release = threading.Event()
def slow_close(worker):
@@ -1018,7 +1064,9 @@ def test_a_host_that_cannot_spawn_keeps_dictation_in_process_on_cpu(monkeypatch)
import core.inference.stt_transformers_worker as worker_module
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", "float16"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cuda", "float16")
+ )
spawned = []
loaded = []
@@ -1084,7 +1132,9 @@ def test_load_reports_model_hub_cache_miss(monkeypatch):
raise SttModelNotDownloadedError("The dictation model is not downloaded.")
_install_fake_worker(monkeypatch, start = missing)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
with pytest.raises(SttModelNotDownloadedError, match = "large-v3"):
WhisperSttSidecar(keep_alive_seconds = 0).load("large-v3")
@@ -1109,7 +1159,7 @@ def test_unavailable_runtime_is_rejected_before_audio_decode(monkeypatch):
def test_missing_model_is_rejected_before_audio_decode(monkeypatch):
sidecar = WhisperSttSidecar(keep_alive_seconds = 0)
- def missing(_model_id):
+ def missing(_model_id, use_resident = True):
raise SttModelNotDownloadedError("not downloaded")
def should_not_decode(_audio):
@@ -1129,7 +1179,7 @@ def test_missing_model_switch_keeps_resident_model(monkeypatch):
sidecar._model_id = "small"
sidecar._device = "cpu"
- def missing(_model_id):
+ def missing(_model_id, use_resident = True):
raise SttModelNotDownloadedError("not downloaded")
monkeypatch.setattr(sidecar, "_ensure_model_downloaded", missing, raising = False)
@@ -1184,7 +1234,9 @@ def test_loaded_model_stays_warm_until_idle_timer_fires(monkeypatch):
monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None)
monkeypatch.setattr(stt_sidecar_module.threading, "Timer", make_timer)
monkeypatch.setattr(sidecar, "_build_model", lambda *_args: (object(), object()))
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
sidecar.load("small")
@@ -1210,7 +1262,9 @@ def test_reusing_loaded_model_refreshes_idle_timer(monkeypatch):
monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None)
monkeypatch.setattr(stt_sidecar_module.threading, "Timer", make_timer)
monkeypatch.setattr(sidecar, "_build_model", lambda *_args: (object(), object()))
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
sidecar.load("small")
first = timers[-1]
@@ -1316,7 +1370,9 @@ def test_accelerator_load_failure_retries_on_cpu(monkeypatch):
raise RuntimeError("accelerator allocation failed")
return object(), object()
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cuda", "float16"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cuda", "float16")
+ )
monkeypatch.setattr(sidecar, "_build_model", build)
sidecar.load("small")
@@ -1352,7 +1408,9 @@ def test_pending_load_can_be_cancelled_without_waiting_for_model_lock(monkeypatc
errors.append(exc)
monkeypatch.setattr(stt_sidecar_module, "ensure_stt_available", lambda: None)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("cpu", "float32"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("cpu", "float32")
+ )
monkeypatch.setattr(sidecar, "_build_model", build)
load_thread = threading.Thread(target = run_load)
@@ -1873,7 +1931,9 @@ def test_preflight_rejects_partial_snapshot(monkeypatch, tmp_path, model_id):
def test_cpu_retry_releases_failed_accelerator_load(monkeypatch):
_install_fake_torch(monkeypatch)
- monkeypatch.setattr(stt_sidecar_module, "_pick_device", lambda: ("mps", "float16"))
+ monkeypatch.setattr(
+ stt_sidecar_module, "_pick_device", lambda _preference = None: ("mps", "float16")
+ )
class Marker:
pass
diff --git a/studio/backend/tests/test_stt_transcription_cancellation.py b/studio/backend/tests/test_stt_transcription_cancellation.py
index 89657739673..fb792ed2b13 100644
--- a/studio/backend/tests/test_stt_transcription_cancellation.py
+++ b/studio/backend/tests/test_stt_transcription_cancellation.py
@@ -121,7 +121,12 @@ def test_disconnected_load_cancels_only_its_request_event(monkeypatch):
sidecar._load_owner_cancel_event = sibling_owner
cancelled_request = None
- def load(_model, _engine, request_cancel_event):
+ def load(
+ _model,
+ _engine,
+ request_cancel_event,
+ device = None,
+ ):
nonlocal cancelled_request
cancelled_request = request_cancel_event
assert request_cancel_event.wait(1), "disconnect must cancel this load request"
@@ -169,7 +174,12 @@ def test_disconnected_raw_transcription_cancels_its_sidecar(monkeypatch):
loaded = []
- def load(model, _engine, _request_cancel_event):
+ def load(
+ model,
+ _engine,
+ _request_cancel_event,
+ device = None,
+ ):
# Stubbed for the same reason the sibling test above stubs it: the real
# implicit load goes through the registry, which refuses with
# SttModelNotDownloadedError (409) unless a snapshot happens to be on disk.
diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.ts
index fe10bcb1a6f..53b5dfa3faf 100644
--- a/studio/frontend/scripts/check-bundle-budget.ts
+++ b/studio/frontend/scripts/check-bundle-budget.ts
@@ -38,7 +38,10 @@ const DIST = resolve(HERE, "..", "dist");
*/
export const BUDGET = {
// Measured 1,496.2 KB transfer / 5,207.2 KB raw at 17363f8a2.
- transferBytes: 1_600_000,
+ // Raised for the audio placement control: same build both sides, merge base
+ // 1,560.9 KB transfer against branch 1,562.6 KB, so it crossed the old 1,562.5 KB
+ // ceiling by a tenth of a kilobyte.
+ transferBytes: 1_620_000,
rawBytes: 5_500_000,
};
diff --git a/studio/frontend/src/features/audio/audio-page-policy.ts b/studio/frontend/src/features/audio/audio-page-policy.ts
index ab1eb341202..b1f489caf77 100644
--- a/studio/frontend/src/features/audio/audio-page-policy.ts
+++ b/studio/frontend/src/features/audio/audio-page-policy.ts
@@ -267,6 +267,35 @@ export function exactGgufLoadSelector(
return meta.ggufFilename ?? meta.ggufVariant ?? null;
}
+/** Whether a TTS pick loads through llama.cpp.
+ *
+ * A direct .gguf file and a GGUF repo id carry no variant filename, so the selector
+ * alone misses both. `meta.isGguf` wins where a caller has it; this covers the rest.
+ */
+export function isGgufTtsTarget({
+ repoId,
+ ggufFilename,
+ loadId,
+ isGguf,
+}: {
+ repoId: string;
+ ggufFilename?: string | null;
+ loadId?: string | null;
+ /** The catalog's own answer, when the caller has one. The tests below are
+ * name heuristics, blind to a GGUF repo whose ids do not spell it. */
+ isGguf?: boolean | null;
+}): boolean {
+ const endsWithGguf = (value: string | null | undefined): boolean =>
+ Boolean(value?.toLowerCase().endsWith(".gguf"));
+ return Boolean(
+ isGguf ||
+ ggufFilename ||
+ /(?:^|[-/])gguf(?:$|[-/])/i.test(repoId) ||
+ endsWithGguf(repoId) ||
+ endsWithGguf(loadId),
+ );
+}
+
export type MacTtsPickAction = "allow" | "use-gguf-sibling" | "reject";
/** MLX has no codec TTS decoder. Curated native PyTorch audio models bypass MLX;
diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsx
index c8c5b21c700..37634e0f283 100644
--- a/studio/frontend/src/features/audio/audio-page.tsx
+++ b/studio/frontend/src/features/audio/audio-page.tsx
@@ -86,6 +86,7 @@ import {
trackSttDownload,
} from "@/features/settings/lib/stt-download-mirror";
import { sttModelSize } from "@/features/settings/stores/stt-model-catalog";
+import { usePersistedChoice } from "@/hooks/use-persisted-choice";
import { usePersistedToggle } from "@/hooks/use-persisted-toggle";
import { useScrollFades } from "@/hooks/use-scroll-fades";
import { fetchSystemInfo } from "@/hooks/use-system";
@@ -120,6 +121,7 @@ import {
canTransitionAudioMode,
exactGgufLoadSelector,
expectedGgufDownloadBytes,
+ isGgufTtsTarget,
isTtsAudioType,
macTtsPickAction,
mergeGalleryPage,
@@ -379,6 +381,7 @@ export function AudioPage({
loadId?: string | null;
audioType?: string | null;
remoteCodeApproval?: RemoteCodeApproval;
+ isGguf?: boolean | null;
} | null>(null);
const ttsStatusRefreshGeneration = useRef(0);
const ttsLoadGeneration = useRef(0);
@@ -463,6 +466,7 @@ export function AudioPage({
loadId?: string | null;
audioType?: string | null;
remoteCodeApproval?: RemoteCodeApproval;
+ isGguf?: boolean | null;
generation: number;
} | null>(null);
const stagedTtsLoadDeferred = useRef(false);
@@ -526,6 +530,11 @@ export function AudioPage({
const [advancedOpen, setAdvancedOpen] = usePersistedToggle(
"unsloth_audio_advanced_open",
);
+ // Read at load time; the handler below ejects so a change takes effect.
+ const [audioDevice, setAudioDeviceState] = usePersistedChoice(
+ "unsloth_audio_device",
+ "auto",
+ );
const refreshStatus = useCallback(async () => {
const generation = ++ttsStatusRefreshGeneration.current;
@@ -894,6 +903,8 @@ export function AudioPage({
loadId?: string | null,
audioType?: string | null,
remoteCodeApproval?: RemoteCodeApproval,
+ // The catalog's answer: the ids alone miss a GGUF repo that does not spell it.
+ isGguf?: boolean | null,
) => {
// A routed pick that arrives while a previous load is still tearing down would
// otherwise be dropped here, and the route effect has already cleared ?model=, so
@@ -905,6 +916,7 @@ export function AudioPage({
loadId,
audioType,
remoteCodeApproval,
+ isGguf,
};
return;
}
@@ -944,6 +956,7 @@ export function AudioPage({
loadId,
audioType,
remoteCodeApproval,
+ isGguf,
};
return;
}
@@ -993,6 +1006,8 @@ export function AudioPage({
);
if (controller.signal.aborted || !isCurrent()) return;
}
+ const wantsCpu = audioDevice === "cpu";
+ const isGgufLoad = isGgufTtsTarget({ repoId, ggufFilename, loadId, isGguf });
const res = await loadModel(
{
model_path: loadId || repoId,
@@ -1005,6 +1020,18 @@ export function AudioPage({
gguf_variant: ggufFilename ?? null,
trust_remote_code: trustRemoteCode,
approved_remote_code_fingerprint: approvedRemoteCodeFingerprint,
+ audio_device: wantsCpu ? "cpu" : "auto",
+ // GGUF ignores audio_device: llama.cpp offloads unless told not to.
+ // An absent speculative_type resolves to "auto", which may attach a GPU
+ // drafter, and the backend then evicts image/video for a CPU load.
+ ...(wantsCpu && isGgufLoad
+ ? // biome-ignore lint/style/useNamingConvention: API schema
+ {
+ gpu_memory_mode: "manual" as const,
+ gpu_layers: 0,
+ speculative_type: "off" as const,
+ }
+ : {}),
},
{
signal: controller.signal,
@@ -1026,6 +1053,17 @@ export function AudioPage({
toast.success(`Model loaded (${res.audio_type ?? "audio"})`, {
id: toastId,
});
+ // Only the native runtime and GGUF can be held in RAM.
+ if (
+ wantsCpu &&
+ !isGgufLoad &&
+ !usesNativeAudioRuntime(repoId, res.audio_type)
+ ) {
+ toast.info(
+ "This model does not support CPU RAM yet, so it loaded on the GPU.",
+ { duration: 6000 },
+ );
+ }
} else {
toast.error(`${repoId} loaded but is not a supported TTS model.`, {
id: toastId,
@@ -1056,7 +1094,7 @@ export function AudioPage({
if (activeRef.current) replayQueuedTtsPick();
}
},
- [refreshStatus],
+ [refreshStatus, audioDevice],
);
// Stage uncached Hub GGUFs through the shared manager so Audio gets the same
@@ -1075,6 +1113,7 @@ export function AudioPage({
queued.loadId,
queued.audioType,
queued.remoteCodeApproval,
+ queued.isGguf,
);
}, []);
const invalidatePendingStagedTts = useCallback(() => {
@@ -1173,6 +1212,7 @@ export function AudioPage({
pending.loadId,
pending.audioType,
pending.remoteCodeApproval,
+ pending.isGguf,
);
},
});
@@ -1199,6 +1239,7 @@ export function AudioPage({
pending.loadId,
pending.audioType,
pending.remoteCodeApproval,
+ pending.isGguf,
);
}, [active, busy]);
@@ -1268,6 +1309,7 @@ export function AudioPage({
meta.loadId,
meta.audioType,
remoteCodeApproval,
+ meta.isGguf,
);
return;
}
@@ -1287,6 +1329,7 @@ export function AudioPage({
loadId: meta.loadId,
audioType: meta.audioType,
remoteCodeApproval,
+ isGguf: meta.isGguf,
generation,
};
stageTtsDownload(
@@ -1313,6 +1356,7 @@ export function AudioPage({
loadId: meta.loadId,
audioType: meta.audioType,
remoteCodeApproval,
+ isGguf: meta.isGguf,
generation,
};
stageTtsDownload([
@@ -1334,6 +1378,7 @@ export function AudioPage({
meta.loadId,
meta.audioType,
remoteCodeApproval,
+ meta.isGguf,
);
},
[stageTtsDownload],
@@ -1576,10 +1621,7 @@ export function AudioPage({
if (ttsPickGeneration.current !== selectionGeneration) return;
const exactGguf = exactGgufLoadSelector(meta);
const isGguf = Boolean(
- meta.isGguf ||
- exactGguf ||
- /(?:^|[-/])gguf(?:$|[-/])/i.test(id) ||
- id.toLowerCase().endsWith(".gguf"),
+ meta.isGguf || isGgufTtsTarget({ repoId: id, ggufFilename: exactGguf }),
);
const ggufSibling = isGguf ? null : ggufSiblingFor(id);
const nativeRuntime =
@@ -2629,6 +2671,47 @@ export function AudioPage({
/>
</Field>
) : null}
+ {/* Field inlined: its label needs a form control to point
+ at, and PillTabs is a tablist with its own name. */}
+ <div className="grid gap-1.5">
+ <span className="text-ui-13 font-medium text-foreground">
+ Load model into
+ </span>
+ <PillTabs
+ ariaLabel="Load model into"
+ value={audioDevice === "cpu" ? "cpu" : "auto"}
+ // The eject below applies the change and cannot interrupt a load.
+ disabled={busy !== null || isRecording}
+ onValueChange={(value) => {
+ const next = value === "cpu" ? "cpu" : "auto";
+ if (next === audioDevice) return;
+ // MiniMax needs CUDA, and the backend's refusal cannot save a
+ // model already ejected here.
+ if (next === "cpu" && status?.audio_type === "minimax_music3") {
+ toast.info(
+ "MiniMax Music 3 needs a GPU, so it cannot be held in CPU RAM.",
+ );
+ return;
+ }
+ setAudioDeviceState(next);
+ if (ttsLoaded) handleEject();
+ }}
+ fit={true}
+ className="h-[30px] self-start [&>button]:h-[30px] [&>button]:px-6"
+ tabs={[
+ { value: "auto", label: "GPU when available" },
+ { value: "cpu", label: "CPU RAM" },
+ ]}
+ />
+ {/* Phrased as what the next load will do, not as the resident
+ model's state: a model loaded by another tab or client can
+ be on the other device, and status does not report it. */}
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {audioDevice === "cpu"
+ ? "New loads go into system RAM instead of the GPU. Slower to generate, and no GPU memory is used."
+ : "New loads use the GPU when there is one, and the CPU otherwise."}
+ </p>
+ </div>
<AdvancedDisclosure
open={advancedOpen}
onOpenChange={setAdvancedOpen}
diff --git a/studio/frontend/src/features/chat/adapters/studio-model-dictation-adapter.ts b/studio/frontend/src/features/chat/adapters/studio-model-dictation-adapter.ts
index dfdc2c7b472..c612f5d196a 100644
--- a/studio/frontend/src/features/chat/adapters/studio-model-dictation-adapter.ts
+++ b/studio/frontend/src/features/chat/adapters/studio-model-dictation-adapter.ts
@@ -8,6 +8,7 @@ import { useSettingsDialogStore } from "@/features/settings/stores/settings-dial
import { requestSttDownload } from "@/features/settings/stores/stt-download-prompt-store";
import {
MTMD_STT_MODELS,
+ type SttDevice,
applyDictationDictionary,
isCuratedSttModel,
recordRecentDictation,
@@ -181,6 +182,7 @@ export async function transcribeAudioBlob(
const engine = options.engine ?? sttEngineFor(model);
const params = new URLSearchParams({ model, fast: "true", engine });
if (language) params.set("language", language);
+ params.set("device", settings.sttDevice);
const response = await authFetch(
`/api/inference/audio/transcribe/raw?${params.toString()}`,
{
@@ -309,15 +311,21 @@ export function loadSttModel(
model: string,
engine?: SttEngine,
signal?: AbortSignal,
+ device?: SttDevice,
): Promise<void> {
const resolvedEngine = engine ?? sttEngineFor(model);
+ const resolvedDevice = device ?? useVoiceSettingsStore.getState().sttDevice;
// Announced so the indicator shows the load immediately, as the toast does.
return queueSttLifecycle(() =>
withModelLoadNotice("stt", model, async () => {
const response = await authFetch("/api/inference/audio/stt/load", {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ model, engine: resolvedEngine }),
+ body: JSON.stringify({
+ model,
+ engine: resolvedEngine,
+ device: resolvedDevice,
+ }),
signal,
});
if (!response.ok) {
@@ -380,11 +388,15 @@ export async function cancelSttDownload(
export function unloadSttModel(
engine?: SttEngine,
model?: string,
+ options?: { wait?: boolean },
): Promise<void> {
return queueSttLifecycle(async () => {
const params = new URLSearchParams();
if (engine) params.set("engine", engine);
if (model) params.set("model", model);
+ // Opt-out only: the default drains an in-flight transcription, right when the
+ // caller needs the memory back now.
+ if (options?.wait === false) params.set("wait", "false");
const query = params.size ? `?${params}` : "";
const response = await authFetch(
`/api/inference/audio/stt/unload${query}`,
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index 7f6956af277..caa70a7e5d6 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -370,6 +370,8 @@ export async function validateModel(
tensor_parallel: payload.tensor_parallel ?? false,
disable_vision: payload.disable_vision ?? false,
gpu_ids: payload.gpu_ids,
+ // Takes no VRAM, so validate must not preflight it and refuse what /load takes.
+ audio_device: payload.audio_device ?? null,
// Manual placement is an explicit override: Auto layers use llama.cpp --fit, a pinned
// layer count is owned by the user. Tell validate so it applies the same policy as /load.
gpu_memory_mode: payload.gpu_memory_mode,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 65d464601b2..deb34e9bbc8 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -136,6 +136,9 @@ export interface LoadModelRequest {
tensor_split?: number[] | null;
/** Picked CUDA/ROCm physical IDs or Vulkan ordinals (omit/empty = automatic). */
gpu_ids?: number[];
+ /** Native audio (TTS / music) only: "cpu" holds the weights in system RAM
+ * rather than the GPU. Ignored for every non-audio model. */
+ audio_device?: "auto" | "cpu" | "gpu";
}
export interface ValidateModelResponse {
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx
index 472e661a4da..81c5e660eae 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx
@@ -21,6 +21,7 @@ export function PillTabs({
className,
compact = false,
fit = false,
+ disabled = false,
}: {
tabs: PillTab[];
value: string;
@@ -28,6 +29,8 @@ export function PillTabs({
ariaLabel: string;
className?: string;
compact?: boolean;
+ /** Block every tab, for a choice that cannot be applied right now. */
+ disabled?: boolean;
/** Size each tab to its label instead of equal widths. The active tab carries
* the pill background directly (the toggle never animates). Tabs only shrink
* when their combined intrinsic width exceeds the available space. */
@@ -70,6 +73,7 @@ export function PillTabs({
// Left/Right move between tabs (WAI-ARIA tablist pattern). ArrowDown
// is left to bubble so the picker's "enter the list" handler still runs.
tabIndex={value === tab.value ? 0 : -1}
+ disabled={disabled}
onKeyDown={(e) => {
if (e.key !== "ArrowRight" && e.key !== "ArrowLeft") return;
e.preventDefault();
@@ -85,6 +89,7 @@ export function PillTabs({
onClick={() => onValueChange(tab.value)}
className={cn(
"relative z-10 inline-flex items-center justify-center gap-1.5 rounded-full transition-colors",
+ disabled && "cursor-not-allowed opacity-50",
fit ? "min-w-0 shrink" : "min-w-0 flex-1",
compact
? "h-7 px-2.5 text-ui-11"
diff --git a/studio/frontend/src/features/settings/stores/voice-settings-store.ts b/studio/frontend/src/features/settings/stores/voice-settings-store.ts
index 3e1845d0d20..28ec3a7c6ce 100644
--- a/studio/frontend/src/features/settings/stores/voice-settings-store.ts
+++ b/studio/frontend/src/features/settings/stores/voice-settings-store.ts
@@ -80,6 +80,16 @@ export function isSttModelLanguageCompatible(
export type DictationEngine = "browser" | "model" | "custom";
+/** Mirrors the backend's `SttLoadRequest.device`. "gpu" is not offered in the
+ * UI: it differs from "auto" only where it cannot be honoured anyway. */
+export type SttDevice = "auto" | "cpu";
+
+export const DEFAULT_STT_DEVICE: SttDevice = "auto";
+
+function normalizeSttDevice(value: unknown): SttDevice {
+ return value === "cpu" ? "cpu" : DEFAULT_STT_DEVICE;
+}
+
export type TtsEngine = "system" | "studio" | "custom";
/**
@@ -107,6 +117,11 @@ export interface VoiceSettingsState {
sttModel: SttModel;
setSttModel: (value: SttModel) => void;
+ /** "cpu" holds the dictation model in system RAM instead of the GPU. Sent
+ * with every load and transcribe, so a change applies on the next load. */
+ sttDevice: SttDevice;
+ setSttDevice: (value: SttDevice) => void;
+
sttProviderId: string;
setSttProviderId: (value: string) => void;
sttProviderModel: string;
@@ -225,6 +240,9 @@ export const useVoiceSettingsStore = create<VoiceSettingsState>()(
};
}),
+ sttDevice: DEFAULT_STT_DEVICE,
+ setSttDevice: (value) => set({ sttDevice: normalizeSttDevice(value) }),
+
sttProviderId: "",
setSttProviderId: (sttProviderId) => set({ sttProviderId }),
sttProviderModel: "",
@@ -362,6 +380,7 @@ export const useVoiceSettingsStore = create<VoiceSettingsState>()(
micDeviceId: asString(saved?.micDeviceId, "default"),
dictationEngine,
sttModel,
+ sttDevice: normalizeSttDevice(saved?.sttDevice),
sttProviderId: asString(saved?.sttProviderId, ""),
sttProviderModel: asString(saved?.sttProviderModel, ""),
dictationLanguage,
diff --git a/studio/frontend/src/features/settings/tabs/voice-tab.tsx b/studio/frontend/src/features/settings/tabs/voice-tab.tsx
index 278ec78d985..ce93f44cba7 100644
--- a/studio/frontend/src/features/settings/tabs/voice-tab.tsx
+++ b/studio/frontend/src/features/settings/tabs/voice-tab.tsx
@@ -30,6 +30,7 @@ import {
loadSttModel,
releaseTtsAudioUrl,
startSttDownload,
+ sttEngineFor,
unloadSttModel,
useExternalProvidersStore,
validateSttModel,
@@ -406,6 +407,9 @@ export function VoiceTab() {
const setDictationEngine = useVoiceSettingsStore((s) => s.setDictationEngine);
const sttModel = useVoiceSettingsStore((s) => s.sttModel);
const setSttModel = useVoiceSettingsStore((s) => s.setSttModel);
+ // Named apart from the `sttDevice` state below, which the sidecar reports back.
+ const sttDevicePreference = useVoiceSettingsStore((s) => s.sttDevice);
+ const setSttDevicePreference = useVoiceSettingsStore((s) => s.setSttDevice);
const sttProviderId = useVoiceSettingsStore((s) => s.sttProviderId);
const setSttProviderId = useVoiceSettingsStore((s) => s.setSttProviderId);
const sttProviderModel = useVoiceSettingsStore((s) => s.sttProviderModel);
@@ -1130,6 +1134,51 @@ export function VoiceTab() {
)
) : null}
+ {isLocalEngine && modelSttSupported ? (
+ <SettingsRow
+ label={t("settings.voice.dictation.sttDeviceLabel")}
+ description={
+ sttDevicePreference === "cpu"
+ ? t("settings.voice.dictation.sttDeviceCpuDescription")
+ : t("settings.voice.dictation.sttDeviceAutoDescription")
+ }
+ >
+ <Select
+ value={sttDevicePreference}
+ onValueChange={(value) => {
+ const next = value === "cpu" ? "cpu" : "auto";
+ if (next === sttDevicePreference) return;
+ // Scoped, so moving our placement cannot evict a model another surface
+ // swapped in. wait:false so a decoding dictation is not killed for a
+ // setting the next load applies anyway.
+ void unloadSttModel(sttEngineFor(sttModel), sttModel, {
+ wait: false,
+ }).catch(() => {});
+ setSttPhase("on-demand");
+ setSttDevice(null);
+ setSttDevicePreference(next);
+ }}
+ >
+ <SelectTrigger
+ data-testid="stt-device-trigger"
+ aria-label={t("settings.voice.dictation.sttDeviceLabel")}
+ className="w-56"
+ size="sm"
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="auto" data-testid="stt-device-auto">
+ {t("settings.voice.dictation.sttDeviceAuto")}
+ </SelectItem>
+ <SelectItem value="cpu" data-testid="stt-device-cpu">
+ {t("settings.voice.dictation.sttDeviceCpu")}
+ </SelectItem>
+ </SelectContent>
+ </Select>
+ </SettingsRow>
+ ) : null}
+
<SettingsRow
label={t("settings.voice.dictation.microphoneLabel")}
description={
diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts
index 609c63fab76..8347defe5f9 100644
--- a/studio/frontend/src/i18n/locales/ar.ts
+++ b/studio/frontend/src/i18n/locales/ar.ts
@@ -491,6 +491,13 @@ export const ar = {
customModelDescription: "اسم النموذج المرسل إلى /v1/audio/transcriptions.",
sttModelLabel: "نموذج التعرّف على الكلام",
sttModelDescription: "اختر نموذج STT أو ابحث عنه لتشغيله محليًا.",
+ sttDeviceLabel: "التحميل إلى",
+ sttDeviceAuto: "وحدة معالجة الرسومات عند توفرها",
+ sttDeviceCpu: "ذاكرة المعالج",
+ sttDeviceAutoDescription:
+ "استخدم وحدة معالجة الرسومات إن وُجدت، وإلا فالمعالج.",
+ sttDeviceCpuDescription:
+ "احتفظ بالنموذج في ذاكرة النظام. النسخ أبطأ، لكن دون استخدام ذاكرة الرسومات.",
sttModelSearchPlaceholder: "ابحث عن نموذج",
sttModelSearching: "جارٍ البحث في Hugging Face…",
sttModelValidating: "جارٍ التحقق من التوافق مع Whisper…",
diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts
index 4d064dd3f4e..fbc32a2da04 100644
--- a/studio/frontend/src/i18n/locales/de.ts
+++ b/studio/frontend/src/i18n/locales/de.ts
@@ -497,6 +497,13 @@ export const de = {
sttModelLabel: "Spracherkennungsmodell",
sttModelDescription:
"Wählen oder suchen Sie ein STT-Modell für die lokale Ausführung.",
+ sttDeviceLabel: "Laden in",
+ sttDeviceAuto: "GPU, wenn verfügbar",
+ sttDeviceCpu: "CPU-RAM",
+ sttDeviceAutoDescription:
+ "Die GPU verwenden, sofern vorhanden, sonst die CPU.",
+ sttDeviceCpuDescription:
+ "Das Modell im Arbeitsspeicher halten. Die Transkription ist langsamer, belegt aber keinen GPU-Speicher.",
sttModelSearchPlaceholder: "Modell suchen",
sttModelSearching: "Hugging Face wird durchsucht…",
sttModelValidating: "Whisper-Kompatibilität wird geprüft…",
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index 956f6990fa5..915435cce37 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -493,6 +493,13 @@ export const en = {
customModelDescription: "Model name sent to /v1/audio/transcriptions.",
sttModelLabel: "Speech recognition model",
sttModelDescription: "Choose or search a STT model to run locally.",
+ sttDeviceLabel: "Load into",
+ sttDeviceAuto: "GPU when available",
+ sttDeviceCpu: "CPU RAM",
+ sttDeviceAutoDescription:
+ "Use the GPU when there is one, and the CPU otherwise.",
+ sttDeviceCpuDescription:
+ "Keep the model in system RAM. Transcription is slower, but no GPU memory is used.",
sttModelSearchPlaceholder: "Search any model on HF",
sttModelSearching: "Searching Hugging Face…",
sttModelValidating: "Checking Whisper compatibility…",
diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts
index 6120f03d59a..5b9f607eb33 100644
--- a/studio/frontend/src/i18n/locales/es.ts
+++ b/studio/frontend/src/i18n/locales/es.ts
@@ -496,6 +496,13 @@ export const es = {
sttModelLabel: "Modelo de reconocimiento de voz",
sttModelDescription:
"Elige o busca un modelo STT para ejecutarlo en local.",
+ sttDeviceLabel: "Cargar en",
+ sttDeviceAuto: "GPU cuando esté disponible",
+ sttDeviceCpu: "RAM de la CPU",
+ sttDeviceAutoDescription:
+ "Usar la GPU cuando haya una y, si no, la CPU.",
+ sttDeviceCpuDescription:
+ "Mantener el modelo en la RAM del sistema. La transcripción es más lenta, pero no usa memoria de la GPU.",
sttModelSearchPlaceholder: "Buscar modelo",
sttModelSearching: "Buscando en Hugging Face…",
sttModelValidating: "Comprobando la compatibilidad con Whisper…",
diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts
index 3222489f5cc..ce5ee71a352 100644
--- a/studio/frontend/src/i18n/locales/fr.ts
+++ b/studio/frontend/src/i18n/locales/fr.ts
@@ -497,6 +497,13 @@ export const fr = {
sttModelLabel: "Modèle de reconnaissance vocale",
sttModelDescription:
"Choisissez ou recherchez un modèle STT à exécuter en local.",
+ sttDeviceLabel: "Charger dans",
+ sttDeviceAuto: "GPU si disponible",
+ sttDeviceCpu: "RAM du CPU",
+ sttDeviceAutoDescription:
+ "Utiliser le GPU quand il y en a un, sinon le CPU.",
+ sttDeviceCpuDescription:
+ "Garder le modèle dans la RAM système. La transcription est plus lente, mais n'utilise pas de mémoire GPU.",
sttModelSearchPlaceholder: "Rechercher un modèle",
sttModelSearching: "Recherche sur Hugging Face…",
sttModelValidating: "Vérification de la compatibilité Whisper…",
diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts
index 3f2734b2ed0..22646e2e262 100644
--- a/studio/frontend/src/i18n/locales/hi.ts
+++ b/studio/frontend/src/i18n/locales/hi.ts
@@ -495,6 +495,13 @@ export const hi = {
"/v1/audio/transcriptions को भेजा जाने वाला मॉडल नाम।",
sttModelLabel: "स्पीच रिकग्निशन मॉडल",
sttModelDescription: "लोकली चलाने के लिए STT मॉडल चुनें या खोजें।",
+ sttDeviceLabel: "इसमें लोड करें",
+ sttDeviceAuto: "उपलब्ध होने पर GPU",
+ sttDeviceCpu: "CPU RAM",
+ sttDeviceAutoDescription:
+ "GPU उपलब्ध हो तो उसका उपयोग करें, अन्यथा CPU का।",
+ sttDeviceCpuDescription:
+ "मॉडल को सिस्टम RAM में रखें। ट्रांसक्रिप्शन धीमा होगा, पर GPU मेमोरी नहीं लगेगी।",
sttModelSearchPlaceholder: "मॉडल खोजें",
sttModelSearching: "Hugging Face पर खोजा जा रहा है…",
sttModelValidating: "Whisper संगतता जाँची जा रही है…",
diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.ts
index ef8a5eb2e36..bb3aa7d2c2d 100644
--- a/studio/frontend/src/i18n/locales/it.ts
+++ b/studio/frontend/src/i18n/locales/it.ts
@@ -462,6 +462,13 @@ export const it = {
sttModelLabel: "Modello di riconoscimento vocale",
sttModelDescription:
"Scegli o cerca un modello STT da eseguire in locale.",
+ sttDeviceLabel: "Carica in",
+ sttDeviceAuto: "GPU quando disponibile",
+ sttDeviceCpu: "RAM della CPU",
+ sttDeviceAutoDescription:
+ "Usa la GPU quando c'è, altrimenti la CPU.",
+ sttDeviceCpuDescription:
+ "Tieni il modello nella RAM di sistema. La trascrizione è più lenta, ma non usa memoria GPU.",
sttModelSearchPlaceholder: "Cerca un modello",
sttModelSearching: "Ricerca su Hugging Face…",
sttModelValidating: "Verifica della compatibilità con Whisper…",
diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts
index 4ef4cb463e9..f5b12fc7eb8 100644
--- a/studio/frontend/src/i18n/locales/ja.ts
+++ b/studio/frontend/src/i18n/locales/ja.ts
@@ -495,6 +495,13 @@ export const ja = {
sttModelLabel: "音声認識モデル",
sttModelDescription:
"ローカルで実行する STT モデルを選択または検索します。",
+ sttDeviceLabel: "読み込み先",
+ sttDeviceAuto: "GPU(利用できる場合)",
+ sttDeviceCpu: "CPU メモリ",
+ sttDeviceAutoDescription:
+ "GPU があれば GPU を、なければ CPU を使用します。",
+ sttDeviceCpuDescription:
+ "モデルをシステムメモリに保持します。文字起こしは遅くなりますが、GPU メモリを使いません。",
sttModelSearchPlaceholder: "モデルを検索",
sttModelSearching: "Hugging Face を検索中…",
sttModelValidating: "Whisper との互換性を確認中…",
diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts
index 24bef3b8ec9..1a58caa866e 100644
--- a/studio/frontend/src/i18n/locales/ko.ts
+++ b/studio/frontend/src/i18n/locales/ko.ts
@@ -492,6 +492,13 @@ export const ko = {
"/v1/audio/transcriptions에 전송할 모델 이름입니다.",
sttModelLabel: "음성 인식 모델",
sttModelDescription: "로컬에서 실행할 STT 모델을 선택하거나 검색하세요.",
+ sttDeviceLabel: "로드 위치",
+ sttDeviceAuto: "GPU(사용 가능할 때)",
+ sttDeviceCpu: "CPU RAM",
+ sttDeviceAutoDescription:
+ "GPU가 있으면 GPU를, 없으면 CPU를 사용합니다.",
+ sttDeviceCpuDescription:
+ "모델을 시스템 RAM에 유지합니다. 변환은 느리지만 GPU 메모리를 쓰지 않습니다.",
sttModelSearchPlaceholder: "모델 검색",
sttModelSearching: "Hugging Face에서 검색 중…",
sttModelValidating: "Whisper 호환성 확인 중…",
diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts
index 9c9bb570c3a..4e8cb7aefc3 100644
--- a/studio/frontend/src/i18n/locales/pt-br.ts
+++ b/studio/frontend/src/i18n/locales/pt-br.ts
@@ -495,6 +495,13 @@ export const ptBR = {
sttModelLabel: "Modelo de reconhecimento de fala",
sttModelDescription:
"Escolha ou busque um modelo STT para executar localmente.",
+ sttDeviceLabel: "Carregar em",
+ sttDeviceAuto: "GPU quando disponível",
+ sttDeviceCpu: "RAM da CPU",
+ sttDeviceAutoDescription:
+ "Usar a GPU quando houver uma e, caso contrário, a CPU.",
+ sttDeviceCpuDescription:
+ "Manter o modelo na RAM do sistema. A transcrição fica mais lenta, mas não usa memória da GPU.",
sttModelSearchPlaceholder: "Buscar modelo",
sttModelSearching: "Buscando no Hugging Face…",
sttModelValidating: "Verificando a compatibilidade com o Whisper…",
diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts
index ff30d75409d..65703730ca1 100644
--- a/studio/frontend/src/i18n/locales/ru.ts
+++ b/studio/frontend/src/i18n/locales/ru.ts
@@ -495,6 +495,13 @@ export const ru = {
sttModelLabel: "Модель распознавания речи",
sttModelDescription:
"Выберите или найдите модель STT для локального запуска.",
+ sttDeviceLabel: "Загружать в",
+ sttDeviceAuto: "GPU, если доступен",
+ sttDeviceCpu: "ОЗУ процессора",
+ sttDeviceAutoDescription:
+ "Использовать GPU, когда он есть, иначе CPU.",
+ sttDeviceCpuDescription:
+ "Держать модель в системной памяти. Расшифровка медленнее, но память GPU не занимается.",
sttModelSearchPlaceholder: "Поиск модели",
sttModelSearching: "Поиск на Hugging Face…",
sttModelValidating: "Проверка совместимости с Whisper…",
diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts
index 16f07e270a2..83a452eff4c 100644
--- a/studio/frontend/src/i18n/locales/zh-CN.ts
+++ b/studio/frontend/src/i18n/locales/zh-CN.ts
@@ -490,6 +490,12 @@ export const zhCN = {
"发送到 /v1/audio/transcriptions 的模型名称。",
sttModelLabel: "语音识别模型",
sttModelDescription: "选择或搜索要在本地运行的 STT 模型。",
+ sttDeviceLabel: "加载到",
+ sttDeviceAuto: "有 GPU 时使用 GPU",
+ sttDeviceCpu: "CPU 内存",
+ sttDeviceAutoDescription: "有 GPU 时使用 GPU,否则使用 CPU。",
+ sttDeviceCpuDescription:
+ "将模型保留在系统内存中。转写速度较慢,但不占用显存。",
sttModelSearchPlaceholder: "搜索模型",
sttModelSearching: "正在搜索 Hugging Face…",
sttModelValidating: "正在检查 Whisper 兼容性…",
diff --git a/studio/frontend/tests/audio-model-eject.test.ts b/studio/frontend/tests/audio-model-eject.test.ts
index 115ef887c9e..ef6b1f542c2 100644
--- a/studio/frontend/tests/audio-model-eject.test.ts
+++ b/studio/frontend/tests/audio-model-eject.test.ts
@@ -67,7 +67,7 @@ test("a Speak load asks the same question and forces from the answer", () => {
// The slot is claimed before the await, so a routed pick arriving mid-dialog queues.
assert.match(
source,
- /if \(ttsLoadInFlight\.current \|\| busyRef\.current === "generating"\) \{\s*pendingRoutedTtsPick\.current = \{\s*repoId,\s*ggufFilename,\s*loadId,\s*audioType,\s*remoteCodeApproval,\s*\};\s*return;\s*\}[\s\S]{0,400}?ttsLoadInFlight\.current = true;/,
+ /if \(ttsLoadInFlight\.current \|\| busyRef\.current === "generating"\) \{\s*pendingRoutedTtsPick\.current = \{\s*repoId,\s*ggufFilename,\s*loadId,\s*audioType,\s*remoteCodeApproval,\s*isGguf,\s*\};\s*return;\s*\}[\s\S]{0,400}?ttsLoadInFlight\.current = true;/,
);
// Declining releases the slot and drops the queued pick, which would else re-ask.
assert.match(
@@ -124,7 +124,7 @@ test("a load confirmed after Audio is hidden is deferred, not sent", () => {
// nothing to abort. Sending anyway let a hidden page replace the visible page's model.
assert.match(
source,
- /if \(!activeRef\.current\) \{\s*releaseLifecycle\(\);\s*ttsLoadInFlight\.current = false;\s*pendingRoutedTtsPick\.current = \{\s*repoId,\s*ggufFilename,\s*loadId,\s*audioType,\s*remoteCodeApproval,\s*\};\s*return;\s*\}/,
+ /if \(!activeRef\.current\) \{\s*releaseLifecycle\(\);\s*ttsLoadInFlight\.current = false;\s*pendingRoutedTtsPick\.current = \{\s*repoId,\s*ggufFilename,\s*loadId,\s*audioType,\s*remoteCodeApproval,\s*isGguf,\s*\};\s*return;\s*\}/,
);
// The activation effect replays exactly that queue, so the pick is not lost.
assert.match(
diff --git a/studio/frontend/tests/audio-page-policy.test.ts b/studio/frontend/tests/audio-page-policy.test.ts
index 6d6f6883aad..49dcf2de7f3 100644
--- a/studio/frontend/tests/audio-page-policy.test.ts
+++ b/studio/frontend/tests/audio-page-policy.test.ts
@@ -14,6 +14,7 @@ import {
canTransitionAudioMode,
exactGgufLoadSelector,
expectedGgufDownloadBytes,
+ isGgufTtsTarget,
isTtsAudioType,
macTtsPickAction,
mergeGalleryPage,
@@ -735,3 +736,65 @@ test("a capped restore refresh invalidates a page fetched from the older cursor"
/const cursor = galleryCache\.nextCursor;\s*try\s*{\s*const page = await listAudioGallery\(\s*0,\s*PAGE_SIZE,\s*cursor,?\s*\);\s*if \(\s*refreshGeneration !== galleryRefreshGeneration\.current \|\|\s*cursor !== galleryCache\.nextCursor\s*\)\s*return;/,
);
});
+
+test("a direct .gguf pick is a GGUF target even without a variant filename", () => {
+ // Local direct rows supply neither ggufFilename nor ggufVariant, so a check on the
+ // selector alone left them on GPU offload.
+ assert.equal(
+ isGgufTtsTarget({ repoId: "/models/orpheus-3b-Q4_K_M.gguf" }),
+ true,
+ );
+ assert.equal(
+ isGgufTtsTarget({ repoId: "Orpheus TTS", loadId: "/m/x.GGUF" }),
+ true,
+ );
+ assert.equal(isGgufTtsTarget({ repoId: "unsloth/orpheus-3b-0.1-ft-GGUF" }), true);
+ assert.equal(
+ isGgufTtsTarget({ repoId: "unsloth/orpheus", ggufFilename: "x-Q4.gguf" }),
+ true,
+ );
+});
+
+test("the catalog's own answer outranks the name heuristics", () => {
+ // Invisible to every test below; only the catalog knows. Losing it dropped offload.
+ assert.equal(
+ isGgufTtsTarget({ repoId: "acme/voicebox", isGguf: true }),
+ true,
+ );
+ assert.equal(
+ isGgufTtsTarget({ repoId: "acme/voicebox-GGUF", isGguf: false }),
+ true,
+ );
+ assert.equal(
+ isGgufTtsTarget({ repoId: "acme/voicebox", isGguf: null }),
+ false,
+ );
+});
+
+test("a safetensors pick is not a GGUF target", () => {
+ assert.equal(isGgufTtsTarget({ repoId: "unsloth/orpheus-3b-0.1-ft" }), false);
+ assert.equal(isGgufTtsTarget({ repoId: "bosonai/higgs-tts-2-3b-base" }), false);
+ // "gguf" only as a bare path segment, so a name merely containing it does not match.
+ assert.equal(isGgufTtsTarget({ repoId: "acme/ggufology" }), false);
+});
+
+test("a CPU GGUF audio load declares speculation off", () => {
+ // An absent speculative_type resolves to "auto", which can attach a GPU drafter;
+ // zero_vram_chat_load then takes the arbiter and cancels an image or video job.
+ assert.match(
+ audioPageSource,
+ /gpu_memory_mode: "manual" as const,\s*gpu_layers: 0,\s*speculative_type: "off" as const,/,
+ );
+});
+
+test("selecting CPU never ejects a resident MiniMax, which cannot load on CPU", () => {
+ // The backend's refusal cannot help once ejected: recovery needs the refused load.
+ const handler = audioPageSource.slice(
+ audioPageSource.indexOf('const next = value === "cpu" ? "cpu" : "auto";'),
+ );
+ const guard = handler.indexOf('status?.audio_type === "minimax_music3"');
+ const eject = handler.indexOf("handleEject()");
+ assert.ok(guard > -1, "no MiniMax guard on the placement control");
+ assert.ok(guard < eject, "the guard must return before the eject");
+ assert.match(handler.slice(guard, eject), /return;/);
+});
diff --git a/studio/frontend/tests/voice-stt-device-control.test.ts b/studio/frontend/tests/voice-stt-device-control.test.ts
new file mode 100644
index 00000000000..9903ed14baf
--- /dev/null
+++ b/studio/frontend/tests/voice-stt-device-control.test.ts
@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+const voiceTab = readFileSync(
+ new URL("../src/features/settings/tabs/voice-tab.tsx", import.meta.url),
+ "utf8",
+);
+const adapter = readFileSync(
+ new URL(
+ "../src/features/chat/adapters/studio-model-dictation-adapter.ts",
+ import.meta.url,
+ ),
+ "utf8",
+);
+
+test("changing the dictation device releases only this tab's model", () => {
+ // Unscoped, the unload can tear down a model this tab never owned.
+ assert.match(
+ voiceTab,
+ /void unloadSttModel\(sttEngineFor\(sttModel\), sttModel, \{\s*wait: false,\s*\}\)/,
+ );
+});
+
+test("switching device never kills a transcription being decoded", () => {
+ // The default drains for 30s and releases anyway, throwing the recording away.
+ assert.match(voiceTab, /wait: false/);
+ assert.match(adapter, /if \(options\?\.wait === false\) params\.set\("wait", "false"\)/);
+});
+
+test("the unload API still takes the engine and model that scoping needs", () => {
+ assert.match(
+ adapter,
+ /export function unloadSttModel\(\s*engine\?: SttEngine,\s*model\?: string,\s*options\?: \{ wait\?: boolean \},\s*\)/,
+ );
+ assert.match(adapter, /if \(model\) params\.set\("model", model\)/);
+});
+
+test("the device preference travels with every load and transcribe", () => {
+ // A load that omits it reads as "no opinion", so the setting never applies.
+ assert.match(adapter, /device: resolvedDevice/);
+ assert.match(adapter, /params\.set\("device", settings\.sttDevice\)/);
+});