Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the chat model loaded when an API request loads a speech model (#14369)

Daniel Han00cf2808db65
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index c014fe470ba..0ec3774ce65 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -69,7 +69,7 @@ import contextvars
import threading
import weakref
from concurrent.futures import ThreadPoolExecutor
-from contextlib import ExitStack, asynccontextmanager, contextmanager, nullcontext
+from contextlib import AsyncExitStack, ExitStack, asynccontextmanager, contextmanager, nullcontext
from dataclasses import dataclass, fields as dataclass_fields, replace
@@ -11560,6 +11560,22 @@ async def _auto_switch_npu_model(
_note_switch_waiter(key, -1)
+def _switch_loads_beside(target_speaks: Callable[[], bool]) -> bool:
+ """Under Keep multiple models loaded, a speech model loads beside a text one (or the reverse); the same kind replaces."""
+ from utils.multi_model_settings import get_multi_model_enabled
+
+ if not get_multi_model_enabled():
+ return False
+ if _llama_cpp_backend.is_active:
+ return target_speaks() != bool(getattr(_llama_cpp_backend, "_is_audio", False))
+ orchestrator = _peek_inference_backend()
+ active = getattr(orchestrator, "active_model_name", None)
+ if not active or getattr(orchestrator, "_managed_engine", None) is not None:
+ return False
+ info = orchestrator.models.get(active, {})
+ return target_speaks() != bool(info.get("is_audio") and info.get("audio_type") != "whisper")
+
+
async def _maybe_auto_switch_model(
requested_model: Optional[str],
fastapi_request: Request,
@@ -12204,7 +12220,21 @@ async def _maybe_auto_switch_model(
try:
# Hold the keep-warm gate across the swap so no new inference can
# start on the model while it is being torn down and replaced.
- async with model_load_gate(), inference_lifecycle_gate():
+ # Beside: load_model_gated takes the load gate itself; the primary keeps serving.
+ beside = await asyncio.to_thread(
+ _switch_loads_beside,
+ # Chat serves a TTS model too, so the target decides, not the endpoint.
+ lambda: require_speech
+ or (
+ resolved is not None
+ and _target_speech_audio_type(target_id, target_is_gguf, variant)
+ is not None
+ ),
+ )
+ async with AsyncExitStack() as swap_gates:
+ if not beside:
+ await swap_gates.enter_async_context(model_load_gate())
+ await swap_gates.enter_async_context(inference_lifecycle_gate())
# Re-read under the gate: the snapshot above predates the wait.
if ollama_target:
ollama_source_identity = await asyncio.to_thread(
@@ -12268,6 +12298,8 @@ async def _maybe_auto_switch_model(
)
if caller_hf_token:
load_kwargs["hf_token"] = caller_hf_token
+ if beside:
+ load_kwargs["alongside"] = True
saved_gpu_ids = load_kwargs.get("gpu_ids")
if saved_gpu_ids and not await _override_gpu_ids_still_resolve(
saved_gpu_ids, stored_gpu_index_kind(override)
@@ -12308,18 +12340,50 @@ async def _maybe_auto_switch_model(
load_internal_kw["speech_codec_path"] = speech_codec_path
if speech_type is not None and caller_hf_token is None:
load_internal_kw["anonymous_hf_access"] = True
- try:
- load_request = LoadRequest(**load_kwargs)
- load_request._gguf_companion_roots = gguf_companion_roots
- load_request._gguf_companion_roots_set = True
- load_request._override_alias_id = override_id
- await _load_model_impl(
+
+ async def _switch_load(load_request: LoadRequest) -> None:
+ nonlocal target_backend
+ if not beside:
+ await _load_model_impl(
+ load_request,
+ fastapi_request,
+ current_subject,
+ current_request_counted = True,
+ **load_internal_kw,
+ )
+ return
+ await load_model_gated(
load_request,
fastapi_request,
current_subject,
current_request_counted = True,
- **load_internal_kw,
+ impl_kwargs = load_internal_kw,
+ background = True,
)
+ slot = routed_slot.get()
+ if slot is None:
+ return
+ if not model_slots.reserve(slot):
+ # Unloaded since: never answer as the primary.
+ routed_slot.set(None)
+ raise HTTPException(
+ status_code = 503,
+ detail = openai_error_body(
+ "The requested model was unloaded before this request "
+ "could use it. Please retry shortly.",
+ status = 503,
+ code = "model_loading",
+ param = "model",
+ ),
+ )
+ target_backend = slot.llama if target_is_gguf else slot.orchestrator
+
+ try:
+ load_request = LoadRequest(**load_kwargs)
+ load_request._gguf_companion_roots = gguf_companion_roots
+ load_request._gguf_companion_roots_set = True
+ load_request._override_alias_id = override_id
+ await _switch_load(load_request)
except HTTPException as exc:
# The pre-flight check cannot mirror every loader gpu_ids rule,
# and a stale pin must never block a request, so retry without it.
@@ -12341,13 +12405,7 @@ async def _maybe_auto_switch_model(
load_request._gguf_companion_roots = gguf_companion_roots
load_request._gguf_companion_roots_set = True
load_request._override_alias_id = override_id
- await _load_model_impl(
- load_request,
- fastapi_request,
- current_subject,
- current_request_counted = True,
- **load_internal_kw,
- )
+ await _switch_load(load_request)
_switch_loaded_ok = True
# publish the completed load before a late cancellation is observed.
target_backend._openai_advertised_id = override_id
@@ -18629,6 +18687,12 @@ def _owner_session(fastapi_request) -> bool:
return not _request_has_api_key(fastapi_request)
+def _raise_if_load_cancelled(attempt: _ScopedLoadAttempt, impl_kwargs: Optional[dict]) -> None:
+ request_cancel = (impl_kwargs or {}).get("load_cancel_event")
+ if attempt.cancel_event.is_set() or (request_cancel is not None and request_cancel.is_set()):
+ raise HTTPException(status_code = 409, detail = "Model load cancelled")
+
+
async def _run_tracked_load_model_impl(
request: LoadRequest,
fastapi_request: Request,
@@ -18637,17 +18701,34 @@ async def _run_tracked_load_model_impl(
attempt: Optional[_ScopedLoadAttempt] = None,
current_request_counted: bool = False,
on_reload_confirmed = None,
+ impl_kwargs: Optional[dict] = None,
+ background: bool = False,
):
global _running_load_attempt
owned_attempt = attempt is None
attempt = attempt or _begin_load_attempt(request, current_subject)
+ impl_kwargs = dict(impl_kwargs or {})
+ request_cancel = impl_kwargs.pop("load_cancel_event", None)
+ follow = None
with _scoped_load_attempts_lock:
_running_load_attempt = attempt
try:
+ if request_cancel is not None:
+ # /unload, shutdown and account retirement cancel through the attempt's event.
+ async def _follow_request_cancel():
+ while not attempt.cancel_event.is_set():
+ if request_cancel.is_set():
+ _cancel_for_shutdown(attempt)
+ return
+ await asyncio.sleep(0.1)
+
+ if request_cancel.is_set():
+ _cancel_for_shutdown(attempt)
+ follow = asyncio.create_task(_follow_request_cancel())
if attempt.cancel_event.is_set():
raise HTTPException(status_code = 409, detail = "Model load cancelled")
- warnings_token = _load_warnings_reach_user.set(True)
+ warnings_token = _load_warnings_reach_user.set(not background)
try:
with collecting_hub_token_rejections() as token_rejections:
response = await _load_model_impl(
@@ -18657,6 +18738,7 @@ async def _run_tracked_load_model_impl(
current_request_counted = current_request_counted,
on_reload_confirmed = on_reload_confirmed,
load_cancel_event = attempt.cancel_event,
+ **impl_kwargs,
)
finally:
_load_warnings_reach_user.reset(warnings_token)
@@ -18664,6 +18746,8 @@ async def _run_tracked_load_model_impl(
_remember_hub_access_warning(response, token_rejections)
return response
finally:
+ if follow is not None:
+ follow.cancel()
if attempt.cancel_event.is_set() and not attempt.cancel_complete.is_set():
if not await asyncio.to_thread(
attempt.cancel_complete.wait, _SCOPED_LOAD_CANCEL_HANDSHAKE_TIMEOUT_S
@@ -18800,6 +18884,8 @@ async def load_model_gated(
*,
user_initiated: bool = False,
current_request_counted: bool = False,
+ impl_kwargs: Optional[dict] = None,
+ background: bool = False,
):
"""Everything ``POST /load`` does except the tunnel-safe padding.
@@ -18875,6 +18961,9 @@ async def load_model_gated(
force = request.force_cancel_active,
action = "Reloading this model",
)
+ elif background:
+ # Under the lifecycle gate the load waits for running chats instead.
+ reload_gate = None
else:
reload_gate = functools.partial(
_raise_or_cancel_active_generations,
@@ -18892,9 +18981,13 @@ async def load_model_gated(
attempt = attempt,
current_request_counted = current_request_counted,
on_reload_confirmed = reload_gate,
+ impl_kwargs = impl_kwargs,
+ background = background,
)
break
except (GpuMemoryShortError, HTTPException) as exc:
+ # Before any eviction below: a cancelled load unloads nothing to make room.
+ _raise_if_load_cancelled(attempt, impl_kwargs)
measured = isinstance(exc, GpuMemoryShortError)
if not measured:
if extra is None and _ran_out_of_memory(exc):
@@ -18959,11 +19052,12 @@ async def load_model_gated(
# Fits nowhere beside the others: replace the active model, as before,
# unless its chats refuse that, checked before anything else changes.
routed_slot.set(None)
- _raise_or_cancel_active_generations(
- force = request.force_cancel_active,
- action = "Loading a model",
- cancel = False,
- )
+ if not background:
+ _raise_or_cancel_active_generations(
+ force = request.force_cancel_active,
+ action = "Loading a model",
+ cancel = False,
+ )
await asyncio.to_thread(model_slots.drop, extra)
model_slots.loading, extra = None, None
replaced = _primary_model_label()
@@ -18974,9 +19068,18 @@ async def load_model_gated(
update = {"alongside": False, "force_alongside": False}
)
async with inference_lifecycle_gate() if new_slot else nullcontext():
- _raise_or_cancel_active_generations(
- force = request.force_cancel_active, action = "Loading a model"
- )
+ if background:
+ await _wait_for_model_switch_idle(
+ current_request_counted = current_request_counted
+ )
+ # The drain can outlast a sidecar install or a cancel: never unload for nothing.
+ _raise_if_sidecar_swap_in_progress()
+ _raise_if_load_cancelled(attempt, impl_kwargs)
+ else:
+ _raise_or_cancel_active_generations(
+ force = request.force_cancel_active,
+ action = "Loading a model",
+ )
# As a pick does before a switch: the active model goes first, so
# the new one is placed on a GPU it no longer shares.
await asyncio.to_thread(_unload_primary)
@@ -18990,6 +19093,8 @@ async def load_model_gated(
attempt = attempt,
current_request_counted = current_request_counted,
on_reload_confirmed = None,
+ impl_kwargs = impl_kwargs,
+ background = background,
)
evicted += [replaced] if replaced else []
break
diff --git a/studio/backend/tests/test_multi_model_serving.py b/studio/backend/tests/test_multi_model_serving.py
index 6483cb4e994..3fba518233f 100644
--- a/studio/backend/tests/test_multi_model_serving.py
+++ b/studio/backend/tests/test_multi_model_serving.py
@@ -438,6 +438,30 @@ def test_a_short_fit_evicts_the_least_recently_used_slot_and_names_it(backends,
assert model_slots.slots[-1].request.model_path == "org/C-GGUF"
+def test_a_load_cancelled_after_a_short_fit_evicts_nothing(backends, monkeypatch):
+ _, extra = backends
+ extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 5.0
+ older = _slot("org/D-GGUF", "Q4_K_M", last_used = 1.0)
+ model_slots.slots.append(older)
+ cancel = threading.Event()
+ _gated_load_fakes(monkeypatch, short_fits = [1])
+ short = inf._run_tracked_load_model_impl
+
+ async def load(request, *args, **kwargs):
+ cancel.set() # the API request is cancelled while the first attempt reports its shortfall
+ return await short(request, *args, **kwargs)
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(
+ inf.load_model_gated(
+ request, None, "s", background = True, impl_kwargs = {"load_cancel_event": cancel}
+ )
+ )
+ assert exc.value.status_code == 409 and older in model_slots.slots and older.llama.is_active
+
+
def test_eviction_takes_as_many_lru_slots_as_the_shortfall_needs(backends, monkeypatch):
_, extra = backends
extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 3.0
@@ -1456,6 +1480,69 @@ def test_the_replace_fallback_refuses_while_the_active_model_generates(backends,
assert model_slots.slots == []
+def test_an_automatic_replace_fallback_waits_for_the_active_chats(backends, monkeypatch):
+ primary, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])
+
+ def refuse(**_kw):
+ raise AssertionError("an automatic switch refused instead of waiting")
+
+ waited = []
+
+ async def wait(**kw):
+ waited.append(primary.model_identifier)
+
+ monkeypatch.setattr(inf, "_raise_or_cancel_active_generations", refuse)
+ monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert asyncio.run(inf.load_model_gated(request, None, "s", background = True)) == "loaded"
+ assert waited == ["org/A-GGUF"] and primary.model_identifier == "org/C-GGUF"
+
+
+def test_an_automatic_replace_fallback_rechecks_the_sidecar_after_the_drain(backends, monkeypatch):
+ primary, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])
+ installing = []
+
+ def sidecar():
+ if installing:
+ raise HTTPException(status_code = 409, detail = "installing")
+
+ async def wait(**kw):
+ installing.append(True) # a transformers install claims its window meanwhile
+
+ monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", sidecar)
+ monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inf.load_model_gated(request, None, "s", background = True))
+ assert exc.value.status_code == 409 and primary.model_identifier == "org/A-GGUF"
+
+
+def test_an_automatic_replace_fallback_cancelled_during_the_drain_keeps_the_active_model(
+ backends, monkeypatch
+):
+ primary, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])
+ cancel = threading.Event()
+
+ async def wait(**kw):
+ cancel.set() # the API request is cancelled while running chats drain
+
+ monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(
+ inf.load_model_gated(
+ request, None, "s", background = True, impl_kwargs = {"load_cancel_event": cancel}
+ )
+ )
+ assert exc.value.status_code == 409 and primary.model_identifier == "org/A-GGUF"
+
+
def test_training_stops_on_a_kept_model_that_would_not_unload(backends, monkeypatch):
_, extra = backends
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 1d5d0ebeb3c..21de2f14f70 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -13175,3 +13175,138 @@ def test_completions_on_mlx_refuses_an_audio_cpp_target_before_it_loads(monkeypa
with pytest.raises(HTTPException) as exc:
asyncio.run(inference_route.openai_completions(_json_body_request(body), "tester"))
assert exc.value.status_code == 400 and rec.calls == []
+
+
+def _beside_case(
+ monkeypatch,
+ mode,
+ primary_speaks,
+ audio_type = "snac",
+):
+ """A switch to a target of ``audio_type`` with a GGUF primary that is (or is not) a speech model."""
+ from core.inference import model_slots
+ from utils import multi_model_settings
+
+ backend, recorder = _speech_case(monkeypatch, audio_type)
+ primary = SimpleNamespace(is_active = True, _is_audio = primary_speaks)
+ monkeypatch.setattr(inference_route, "_llama_cpp_backend", primary)
+ monkeypatch.setattr(multi_model_settings, "get_multi_model_mode", lambda: mode)
+ monkeypatch.setattr(model_slots, "slots", [])
+ slot = model_slots.ExtraSlot(_FakeBackend(), SimpleNamespace(active_model_name = None), "owner")
+ gated = []
+
+ async def load_model_gated(request, fastapi_request, current_subject, **kwargs):
+ gated.append((request, kwargs))
+ model_slots.slots.append(slot)
+ inference_route.routed_slot.set(slot)
+
+ monkeypatch.setattr(inference_route, "load_model_gated", load_model_gated)
+ # The stub getters always answer with the primary, which a slot load leaves alone.
+ monkeypatch.setattr(inference_route, "_reject_unservable_model", _noop_reject)
+ return backend, recorder, slot, gated
+
+
[email protected]("mode", ["fit", "always"])
+def test_a_speech_switch_keeps_the_chat_model_loaded(monkeypatch, mode):
+ backend, recorder, slot, gated = _beside_case(monkeypatch, mode, primary_speaks = False)
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "org/B-GGUF", object(), "tester", require_speech = True
+ )
+ )
+ assert recorder.calls == [] and len(gated) == 1
+ request, kwargs = gated[0]
+ assert request.alongside and request.model_path == "/local/B.gguf"
+ assert kwargs["current_request_counted"] is True and kwargs["background"] is True
+ assert slot.llama._openai_advertised_id == "org/B-GGUF"
+ assert backend.model_identifier == "org/A-GGUF"
+
+
+ "mode,primary_speaks",
+ [("off", False), ("fit", True), ("always", True)],
+)
+def test_a_speech_switch_still_replaces_under_off_or_another_speech_model(
+ monkeypatch, mode, primary_speaks
+):
+ backend, recorder, _slot, gated = _beside_case(monkeypatch, mode, primary_speaks)
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "org/B-GGUF", object(), "tester", require_speech = True
+ )
+ )
+ assert gated == [] and len(recorder.calls) == 1
+ assert not recorder.calls[0].alongside
+
+
+def test_a_chat_switch_keeps_a_loaded_speech_model(monkeypatch):
+ _backend, recorder, _slot, gated = _beside_case(
+ monkeypatch, "fit", primary_speaks = True, audio_type = None
+ )
+ asyncio.run(inference_route._maybe_auto_switch_model("org/B-GGUF", object(), "tester"))
+ assert recorder.calls == [] and len(gated) == 1 and gated[0][0].alongside
+
+
[email protected]("primary_speaks", [False, True])
+def test_a_chat_request_naming_a_speech_model_is_classified_by_its_target(
+ monkeypatch, primary_speaks
+):
+ _backend, recorder, _slot, gated = _beside_case(monkeypatch, "fit", primary_speaks)
+ asyncio.run(inference_route._maybe_auto_switch_model("org/B-GGUF", object(), "tester"))
+ if primary_speaks:
+ assert gated == [] and len(recorder.calls) == 1
+ else:
+ assert recorder.calls == [] and len(gated) == 1 and gated[0][0].alongside
+
+
+def test_a_kept_slot_unloaded_before_it_is_reserved_refuses_the_request(monkeypatch):
+ from core.inference import model_slots
+
+ backend, recorder, slot, gated = _beside_case(monkeypatch, "fit", primary_speaks = False)
+ monkeypatch.setattr(model_slots, "reserve", lambda _slot: False)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "org/B-GGUF", object(), "tester", require_speech = True
+ )
+ )
+ assert exc.value.status_code == 503 and len(gated) == 1
+ assert getattr(backend, "_openai_advertised_id", None) != "org/B-GGUF"
+
+
+def test_a_background_tracked_load_keeps_the_request_cancel_and_no_user_warnings(monkeypatch):
+ seen = {}
+
+ async def impl(request, fastapi_request, current_subject, **kwargs):
+ seen.update(kwargs, warnings = inference_route._load_warnings_reach_user.get())
+ if "request_cancel" in seen:
+ seen["request_cancel"].set()
+ for _ in range(50):
+ if kwargs["load_cancel_event"].is_set():
+ break
+ await asyncio.sleep(0.02)
+
+ monkeypatch.setattr(inference_route, "_load_model_impl", impl)
+ request = LoadRequest(model_path = "org/B-GGUF")
+ for background in (True, False):
+ seen.clear()
+ attempt = inference_route._begin_load_attempt(request, "tester")
+ cancel = threading.Event()
+ impl_kwargs = None
+ if background:
+ seen["request_cancel"] = cancel
+ impl_kwargs = {"load_cancel_event": cancel}
+ asyncio.run(
+ inference_route._run_tracked_load_model_impl(
+ request,
+ object(),
+ "tester",
+ attempt = attempt,
+ impl_kwargs = impl_kwargs,
+ background = background,
+ )
+ )
+ inference_route._finish_load_attempt(attempt)
+ assert seen["load_cancel_event"] is attempt.cancel_event
+ assert seen["warnings"] is (not background)
+ assert attempt.cancel_event.is_set() is background