Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep the chat model loaded when an API request loads a speech model (#14369)
Daniel Han00cf2808db65
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex c014fe470ba..0ec3774ce65 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -69,7 +69,7 @@ import contextvarsimport threadingimport weakreffrom concurrent.futures import ThreadPoolExecutor-from contextlib import ExitStack, asynccontextmanager, contextmanager, nullcontext+from contextlib import AsyncExitStack, ExitStack, asynccontextmanager, contextmanager, nullcontextfrom dataclasses import dataclass, fields as dataclass_fields, replace@@ -11560,6 +11560,22 @@ async def _auto_switch_npu_model(_note_switch_waiter(key, -1)+def _switch_loads_beside(target_speaks: Callable[[], bool]) -> bool:+ """Under Keep multiple models loaded, a speech model loads beside a text one (or the reverse); the same kind replaces."""+ from utils.multi_model_settings import get_multi_model_enabled++ if not get_multi_model_enabled():+ return False+ if _llama_cpp_backend.is_active:+ return target_speaks() != bool(getattr(_llama_cpp_backend, "_is_audio", False))+ orchestrator = _peek_inference_backend()+ active = getattr(orchestrator, "active_model_name", None)+ if not active or getattr(orchestrator, "_managed_engine", None) is not None:+ return False+ info = orchestrator.models.get(active, {})+ return target_speaks() != bool(info.get("is_audio") and info.get("audio_type") != "whisper")++async def _maybe_auto_switch_model(requested_model: Optional[str],fastapi_request: Request,@@ -12204,7 +12220,21 @@ async def _maybe_auto_switch_model(try:# Hold the keep-warm gate across the swap so no new inference can# start on the model while it is being torn down and replaced.- async with model_load_gate(), inference_lifecycle_gate():+ # Beside: load_model_gated takes the load gate itself; the primary keeps serving.+ beside = await asyncio.to_thread(+ _switch_loads_beside,+ # Chat serves a TTS model too, so the target decides, not the endpoint.+ lambda: require_speech+ or (+ resolved is not None+ and _target_speech_audio_type(target_id, target_is_gguf, variant)+ is not None+ ),+ )+ async with AsyncExitStack() as swap_gates:+ if not beside:+ await swap_gates.enter_async_context(model_load_gate())+ await swap_gates.enter_async_context(inference_lifecycle_gate())# Re-read under the gate: the snapshot above predates the wait.if ollama_target:ollama_source_identity = await asyncio.to_thread(@@ -12268,6 +12298,8 @@ async def _maybe_auto_switch_model()if caller_hf_token:load_kwargs["hf_token"] = caller_hf_token+ if beside:+ load_kwargs["alongside"] = Truesaved_gpu_ids = load_kwargs.get("gpu_ids")if saved_gpu_ids and not await _override_gpu_ids_still_resolve(saved_gpu_ids, stored_gpu_index_kind(override)@@ -12308,18 +12340,50 @@ async def _maybe_auto_switch_model(load_internal_kw["speech_codec_path"] = speech_codec_pathif speech_type is not None and caller_hf_token is None:load_internal_kw["anonymous_hf_access"] = True- try:- load_request = LoadRequest(**load_kwargs)- load_request._gguf_companion_roots = gguf_companion_roots- load_request._gguf_companion_roots_set = True- load_request._override_alias_id = override_id- await _load_model_impl(++ async def _switch_load(load_request: LoadRequest) -> None:+ nonlocal target_backend+ if not beside:+ await _load_model_impl(+ load_request,+ fastapi_request,+ current_subject,+ current_request_counted = True,+ **load_internal_kw,+ )+ return+ await load_model_gated(load_request,fastapi_request,current_subject,current_request_counted = True,- **load_internal_kw,+ impl_kwargs = load_internal_kw,+ background = True,)+ slot = routed_slot.get()+ if slot is None:+ return+ if not model_slots.reserve(slot):+ # Unloaded since: never answer as the primary.+ routed_slot.set(None)+ raise HTTPException(+ status_code = 503,+ detail = openai_error_body(+ "The requested model was unloaded before this request "+ "could use it. Please retry shortly.",+ status = 503,+ code = "model_loading",+ param = "model",+ ),+ )+ target_backend = slot.llama if target_is_gguf else slot.orchestrator++ try:+ load_request = LoadRequest(**load_kwargs)+ load_request._gguf_companion_roots = gguf_companion_roots+ load_request._gguf_companion_roots_set = True+ load_request._override_alias_id = override_id+ await _switch_load(load_request)except HTTPException as exc:# The pre-flight check cannot mirror every loader gpu_ids rule,# and a stale pin must never block a request, so retry without it.@@ -12341,13 +12405,7 @@ async def _maybe_auto_switch_model(load_request._gguf_companion_roots = gguf_companion_rootsload_request._gguf_companion_roots_set = Trueload_request._override_alias_id = override_id- await _load_model_impl(- load_request,- fastapi_request,- current_subject,- current_request_counted = True,- **load_internal_kw,- )+ await _switch_load(load_request)_switch_loaded_ok = True# publish the completed load before a late cancellation is observed.target_backend._openai_advertised_id = override_id@@ -18629,6 +18687,12 @@ def _owner_session(fastapi_request) -> bool:return not _request_has_api_key(fastapi_request)+def _raise_if_load_cancelled(attempt: _ScopedLoadAttempt, impl_kwargs: Optional[dict]) -> None:+ request_cancel = (impl_kwargs or {}).get("load_cancel_event")+ if attempt.cancel_event.is_set() or (request_cancel is not None and request_cancel.is_set()):+ raise HTTPException(status_code = 409, detail = "Model load cancelled")++async def _run_tracked_load_model_impl(request: LoadRequest,fastapi_request: Request,@@ -18637,17 +18701,34 @@ async def _run_tracked_load_model_impl(attempt: Optional[_ScopedLoadAttempt] = None,current_request_counted: bool = False,on_reload_confirmed = None,+ impl_kwargs: Optional[dict] = None,+ background: bool = False,):global _running_load_attemptowned_attempt = attempt is Noneattempt = attempt or _begin_load_attempt(request, current_subject)+ impl_kwargs = dict(impl_kwargs or {})+ request_cancel = impl_kwargs.pop("load_cancel_event", None)+ follow = Nonewith _scoped_load_attempts_lock:_running_load_attempt = attempttry:+ if request_cancel is not None:+ # /unload, shutdown and account retirement cancel through the attempt's event.+ async def _follow_request_cancel():+ while not attempt.cancel_event.is_set():+ if request_cancel.is_set():+ _cancel_for_shutdown(attempt)+ return+ await asyncio.sleep(0.1)++ if request_cancel.is_set():+ _cancel_for_shutdown(attempt)+ follow = asyncio.create_task(_follow_request_cancel())if attempt.cancel_event.is_set():raise HTTPException(status_code = 409, detail = "Model load cancelled")- warnings_token = _load_warnings_reach_user.set(True)+ warnings_token = _load_warnings_reach_user.set(not background)try:with collecting_hub_token_rejections() as token_rejections:response = await _load_model_impl(@@ -18657,6 +18738,7 @@ async def _run_tracked_load_model_impl(current_request_counted = current_request_counted,on_reload_confirmed = on_reload_confirmed,load_cancel_event = attempt.cancel_event,+ **impl_kwargs,)finally:_load_warnings_reach_user.reset(warnings_token)@@ -18664,6 +18746,8 @@ async def _run_tracked_load_model_impl(_remember_hub_access_warning(response, token_rejections)return responsefinally:+ if follow is not None:+ follow.cancel()if attempt.cancel_event.is_set() and not attempt.cancel_complete.is_set():if not await asyncio.to_thread(attempt.cancel_complete.wait, _SCOPED_LOAD_CANCEL_HANDSHAKE_TIMEOUT_S@@ -18800,6 +18884,8 @@ async def load_model_gated(*,user_initiated: bool = False,current_request_counted: bool = False,+ impl_kwargs: Optional[dict] = None,+ background: bool = False,):"""Everything ``POST /load`` does except the tunnel-safe padding.@@ -18875,6 +18961,9 @@ async def load_model_gated(force = request.force_cancel_active,action = "Reloading this model",)+ elif background:+ # Under the lifecycle gate the load waits for running chats instead.+ reload_gate = Noneelse:reload_gate = functools.partial(_raise_or_cancel_active_generations,@@ -18892,9 +18981,13 @@ async def load_model_gated(attempt = attempt,current_request_counted = current_request_counted,on_reload_confirmed = reload_gate,+ impl_kwargs = impl_kwargs,+ background = background,)breakexcept (GpuMemoryShortError, HTTPException) as exc:+ # Before any eviction below: a cancelled load unloads nothing to make room.+ _raise_if_load_cancelled(attempt, impl_kwargs)measured = isinstance(exc, GpuMemoryShortError)if not measured:if extra is None and _ran_out_of_memory(exc):@@ -18959,11 +19052,12 @@ async def load_model_gated(# Fits nowhere beside the others: replace the active model, as before,# unless its chats refuse that, checked before anything else changes.routed_slot.set(None)- _raise_or_cancel_active_generations(- force = request.force_cancel_active,- action = "Loading a model",- cancel = False,- )+ if not background:+ _raise_or_cancel_active_generations(+ force = request.force_cancel_active,+ action = "Loading a model",+ cancel = False,+ )await asyncio.to_thread(model_slots.drop, extra)model_slots.loading, extra = None, Nonereplaced = _primary_model_label()@@ -18974,9 +19068,18 @@ async def load_model_gated(update = {"alongside": False, "force_alongside": False})async with inference_lifecycle_gate() if new_slot else nullcontext():- _raise_or_cancel_active_generations(- force = request.force_cancel_active, action = "Loading a model"- )+ if background:+ await _wait_for_model_switch_idle(+ current_request_counted = current_request_counted+ )+ # The drain can outlast a sidecar install or a cancel: never unload for nothing.+ _raise_if_sidecar_swap_in_progress()+ _raise_if_load_cancelled(attempt, impl_kwargs)+ else:+ _raise_or_cancel_active_generations(+ force = request.force_cancel_active,+ action = "Loading a model",+ )# As a pick does before a switch: the active model goes first, so# the new one is placed on a GPU it no longer shares.await asyncio.to_thread(_unload_primary)@@ -18990,6 +19093,8 @@ async def load_model_gated(attempt = attempt,current_request_counted = current_request_counted,on_reload_confirmed = None,+ impl_kwargs = impl_kwargs,+ background = background,)evicted += [replaced] if replaced else []breakdiff --git a/studio/backend/tests/test_multi_model_serving.py b/studio/backend/tests/test_multi_model_serving.pyindex 6483cb4e994..3fba518233f 100644--- a/studio/backend/tests/test_multi_model_serving.py+++ b/studio/backend/tests/test_multi_model_serving.py@@ -438,6 +438,30 @@ def test_a_short_fit_evicts_the_least_recently_used_slot_and_names_it(backends,assert model_slots.slots[-1].request.model_path == "org/C-GGUF"+def test_a_load_cancelled_after_a_short_fit_evicts_nothing(backends, monkeypatch):+ _, extra = backends+ extra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 5.0+ older = _slot("org/D-GGUF", "Q4_K_M", last_used = 1.0)+ model_slots.slots.append(older)+ cancel = threading.Event()+ _gated_load_fakes(monkeypatch, short_fits = [1])+ short = inf._run_tracked_load_model_impl++ async def load(request, *args, **kwargs):+ cancel.set() # the API request is cancelled while the first attempt reports its shortfall+ return await short(request, *args, **kwargs)++ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(+ inf.load_model_gated(+ request, None, "s", background = True, impl_kwargs = {"load_cancel_event": cancel}+ )+ )+ assert exc.value.status_code == 409 and older in model_slots.slots and older.llama.is_active++def test_eviction_takes_as_many_lru_slots_as_the_shortfall_needs(backends, monkeypatch):_, extra = backendsextra.request, extra.last_used = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0"), 3.0@@ -1456,6 +1480,69 @@ def test_the_replace_fallback_refuses_while_the_active_model_generates(backends,assert model_slots.slots == []+def test_an_automatic_replace_fallback_waits_for_the_active_chats(backends, monkeypatch):+ primary, extra = backends+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])++ def refuse(**_kw):+ raise AssertionError("an automatic switch refused instead of waiting")++ waited = []++ async def wait(**kw):+ waited.append(primary.model_identifier)++ monkeypatch.setattr(inf, "_raise_or_cancel_active_generations", refuse)+ monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)+ assert asyncio.run(inf.load_model_gated(request, None, "s", background = True)) == "loaded"+ assert waited == ["org/A-GGUF"] and primary.model_identifier == "org/C-GGUF"+++def test_an_automatic_replace_fallback_rechecks_the_sidecar_after_the_drain(backends, monkeypatch):+ primary, extra = backends+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])+ installing = []++ def sidecar():+ if installing:+ raise HTTPException(status_code = 409, detail = "installing")++ async def wait(**kw):+ installing.append(True) # a transformers install claims its window meanwhile++ monkeypatch.setattr(inf, "_raise_if_sidecar_swap_in_progress", sidecar)+ monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(inf.load_model_gated(request, None, "s", background = True))+ assert exc.value.status_code == 409 and primary.model_identifier == "org/A-GGUF"+++def test_an_automatic_replace_fallback_cancelled_during_the_drain_keeps_the_active_model(+ backends, monkeypatch+):+ primary, extra = backends+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])+ cancel = threading.Event()++ async def wait(**kw):+ cancel.set() # the API request is cancelled while running chats drain++ monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(+ inf.load_model_gated(+ request, None, "s", background = True, impl_kwargs = {"load_cancel_event": cancel}+ )+ )+ assert exc.value.status_code == 409 and primary.model_identifier == "org/A-GGUF"++def test_training_stops_on_a_kept_model_that_would_not_unload(backends, monkeypatch):_, extra = backendsdiff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 1d5d0ebeb3c..21de2f14f70 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -13175,3 +13175,138 @@ def test_completions_on_mlx_refuses_an_audio_cpp_target_before_it_loads(monkeypawith pytest.raises(HTTPException) as exc:asyncio.run(inference_route.openai_completions(_json_body_request(body), "tester"))assert exc.value.status_code == 400 and rec.calls == []+++def _beside_case(+ monkeypatch,+ mode,+ primary_speaks,+ audio_type = "snac",+):+ """A switch to a target of ``audio_type`` with a GGUF primary that is (or is not) a speech model."""+ from core.inference import model_slots+ from utils import multi_model_settings++ backend, recorder = _speech_case(monkeypatch, audio_type)+ primary = SimpleNamespace(is_active = True, _is_audio = primary_speaks)+ monkeypatch.setattr(inference_route, "_llama_cpp_backend", primary)+ monkeypatch.setattr(multi_model_settings, "get_multi_model_mode", lambda: mode)+ monkeypatch.setattr(model_slots, "slots", [])+ slot = model_slots.ExtraSlot(_FakeBackend(), SimpleNamespace(active_model_name = None), "owner")+ gated = []++ async def load_model_gated(request, fastapi_request, current_subject, **kwargs):+ gated.append((request, kwargs))+ model_slots.slots.append(slot)+ inference_route.routed_slot.set(slot)++ monkeypatch.setattr(inference_route, "load_model_gated", load_model_gated)+ # The stub getters always answer with the primary, which a slot load leaves alone.+ monkeypatch.setattr(inference_route, "_reject_unservable_model", _noop_reject)+ return backend, recorder, slot, gated++[email protected]("mode", ["fit", "always"])+def test_a_speech_switch_keeps_the_chat_model_loaded(monkeypatch, mode):+ backend, recorder, slot, gated = _beside_case(monkeypatch, mode, primary_speaks = False)+ asyncio.run(+ inference_route._maybe_auto_switch_model(+ "org/B-GGUF", object(), "tester", require_speech = True+ )+ )+ assert recorder.calls == [] and len(gated) == 1+ request, kwargs = gated[0]+ assert request.alongside and request.model_path == "/local/B.gguf"+ assert kwargs["current_request_counted"] is True and kwargs["background"] is True+ assert slot.llama._openai_advertised_id == "org/B-GGUF"+ assert backend.model_identifier == "org/A-GGUF"+++ "mode,primary_speaks",+ [("off", False), ("fit", True), ("always", True)],+)+def test_a_speech_switch_still_replaces_under_off_or_another_speech_model(+ monkeypatch, mode, primary_speaks+):+ backend, recorder, _slot, gated = _beside_case(monkeypatch, mode, primary_speaks)+ asyncio.run(+ inference_route._maybe_auto_switch_model(+ "org/B-GGUF", object(), "tester", require_speech = True+ )+ )+ assert gated == [] and len(recorder.calls) == 1+ assert not recorder.calls[0].alongside+++def test_a_chat_switch_keeps_a_loaded_speech_model(monkeypatch):+ _backend, recorder, _slot, gated = _beside_case(+ monkeypatch, "fit", primary_speaks = True, audio_type = None+ )+ asyncio.run(inference_route._maybe_auto_switch_model("org/B-GGUF", object(), "tester"))+ assert recorder.calls == [] and len(gated) == 1 and gated[0][0].alongside++[email protected]("primary_speaks", [False, True])+def test_a_chat_request_naming_a_speech_model_is_classified_by_its_target(+ monkeypatch, primary_speaks+):+ _backend, recorder, _slot, gated = _beside_case(monkeypatch, "fit", primary_speaks)+ asyncio.run(inference_route._maybe_auto_switch_model("org/B-GGUF", object(), "tester"))+ if primary_speaks:+ assert gated == [] and len(recorder.calls) == 1+ else:+ assert recorder.calls == [] and len(gated) == 1 and gated[0][0].alongside+++def test_a_kept_slot_unloaded_before_it_is_reserved_refuses_the_request(monkeypatch):+ from core.inference import model_slots++ backend, recorder, slot, gated = _beside_case(monkeypatch, "fit", primary_speaks = False)+ monkeypatch.setattr(model_slots, "reserve", lambda _slot: False)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(+ inference_route._maybe_auto_switch_model(+ "org/B-GGUF", object(), "tester", require_speech = True+ )+ )+ assert exc.value.status_code == 503 and len(gated) == 1+ assert getattr(backend, "_openai_advertised_id", None) != "org/B-GGUF"+++def test_a_background_tracked_load_keeps_the_request_cancel_and_no_user_warnings(monkeypatch):+ seen = {}++ async def impl(request, fastapi_request, current_subject, **kwargs):+ seen.update(kwargs, warnings = inference_route._load_warnings_reach_user.get())+ if "request_cancel" in seen:+ seen["request_cancel"].set()+ for _ in range(50):+ if kwargs["load_cancel_event"].is_set():+ break+ await asyncio.sleep(0.02)++ monkeypatch.setattr(inference_route, "_load_model_impl", impl)+ request = LoadRequest(model_path = "org/B-GGUF")+ for background in (True, False):+ seen.clear()+ attempt = inference_route._begin_load_attempt(request, "tester")+ cancel = threading.Event()+ impl_kwargs = None+ if background:+ seen["request_cancel"] = cancel+ impl_kwargs = {"load_cancel_event": cancel}+ asyncio.run(+ inference_route._run_tracked_load_model_impl(+ request,+ object(),+ "tester",+ attempt = attempt,+ impl_kwargs = impl_kwargs,+ background = background,+ )+ )+ inference_route._finish_load_attempt(attempt)+ assert seen["load_cancel_event"] is attempt.cancel_event+ assert seen["warnings"] is (not background)+ assert attempt.cancel_event.is_set() is background