Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: recheck that the user is still away before an automatic task swaps models (#15436)
Daniel Han3c4802b2f57f
diff --git a/studio/backend/core/scheduled_tasks/scheduler.py b/studio/backend/core/scheduled_tasks/scheduler.pyindex 0d8ef9b1560..40b7d3a0695 100644--- a/studio/backend/core/scheduled_tasks/scheduler.py+++ b/studio/backend/core/scheduled_tasks/scheduler.py@@ -379,6 +379,16 @@ async def _plan_swap(task: dict, model: str, unattended: bool, carried: Optional)+def _require_still_away(swap: _Swap) -> None:+ """Raise model_busy once someone came back since the swap was planned: an inference request ended,+ a model load landed, or a request is running beside this run."""+ from core.inference.llama_keepwarm import attended_stamp, other_inference_request_count+ if attended_stamp() != swap.attended or other_inference_request_count(+ current_request_counted = True+ ):+ raise RunFailed("model_busy")++async def _restore_after_swap(swap: _Swap, app: Any) -> None:"""Put back what the run's load replaced, unless someone used or loaded a model since. Best effort."""from core.inference.llama_keepwarm import (@@ -484,6 +494,7 @@ async def generate_answer(allow_model_switch_for_request,load_alongside_for_request,produce_openai_chat_completions,+ require_for_model_switch,)carried = holder.pending if holder is not None else None@@ -494,16 +505,8 @@ async def generate_answer(model_fields, provider, allow_switch = _model_fields(task.get("model"), bool(task.get("allowModelLoad") or run.get("allowModelLoad")))- swap = (- await _plan_swap(- task,- task["model"],- run.get("source") == "schedule" and not run.get("allowModelLoad"),- carried,- )- if allow_switch- else None- )+ unattended = run.get("source") == "schedule" and not run.get("allowModelLoad")+ swap = await _plan_swap(task, task["model"], unattended, carried) if allow_switch else Noneif holder is not None:holder.pending = swap or carriedloop = asyncio.get_running_loop()@@ -515,6 +518,18 @@ async def generate_answer(allow_model_switch_for_request(request.scope)# Keep multiple models loaded: the task model goes beside the user's when it fits, as from Chat's picker.load_alongside_for_request(request.scope)+ away_swap = swap if unattended else None+ refused: list[bool] = []+ if away_swap is not None:+ # The switch rechecks under its gates; it can only answer with an HTTP error, read back as model_busy.+ def still_away() -> None:+ try:+ _require_still_away(away_swap)+ except RunFailed:+ refused.append(True)+ raise HTTPException(status_code = 409, detail = "model_busy") from None++ require_for_model_switch(request.scope, still_away)with unattended_activity():activity = InferenceActivityReservation()activity.reserve()@@ -529,6 +544,9 @@ async def generate_answer(raise RunFailed("timeout") from Noneif cancel_event.is_set():raise RunFailed("cancelled")+ if away_swap is not None:+ # Admission can wait out a load the user started, so the idle check before it may be stale.+ _require_still_away(away_swap)# Built once admitted: a model kept alongside is chosen as it stands now, not before a load or unload.if not task.get("model") and (kept := _sole_kept_model()):model_fields = {"model": kept}@@ -596,6 +614,8 @@ async def generate_answer(try:response = producer.result()except HTTPException as exc:+ if refused:+ raise RunFailed("model_busy") from Noneif allow_switch and not _model_is_loaded(payload.model):raise RunFailed("model_load_failed", _error_text(exc.detail)) from None# A named kept model answered for itself, so its refusal is not "nothing loaded",@@ -609,6 +629,8 @@ async def generate_answer(raise RunFailed(None, _error_text(exc.detail)) from Noneexcept GpuBusyForAnotherAccountError as exc:raise RunFailed(None, _error_text(exc.as_http_exception().detail)) from None+ if refused:+ raise RunFailed("model_busy")if serving.get("loaded") is False:raise RunFailed("model_load_failed")finally:diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex d83797ce4a0..3ef38f75a5b 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -9493,6 +9493,23 @@ def _model_switch_allowed_for_request(fastapi_request) -> bool:return isinstance(scope, dict) and bool(scope.get(_ALLOW_MODEL_SWITCH_SCOPE_KEY))+_SWITCH_PRECONDITION_SCOPE_KEY = "unsloth.switch_precondition"+++def require_for_model_switch(scope, check) -> None:+ """Have this in-process request's switch call ``check()`` under the swap gate, just before it replaces the+ loaded model; ``check`` raises to refuse. A scope key, so no HTTP client can set it."""+ if isinstance(scope, dict):+ scope[_SWITCH_PRECONDITION_SCOPE_KEY] = check+++def _check_switch_precondition(fastapi_request) -> None:+ scope = getattr(fastapi_request, "scope", None)+ check = scope.get(_SWITCH_PRECONDITION_SCOPE_KEY) if isinstance(scope, dict) else None+ if callable(check):+ check()++_LOAD_ALONGSIDE_SCOPE_KEY = "unsloth.load_alongside"@@ -11319,6 +11336,8 @@ async def _maybe_auto_switch_model(_record_serving_alias()returnrequire_no_foreign_generations(path = switch_path)+ # Last chance to refuse: held swap gates keep inference and loads from starting before it.+ _check_switch_precondition(fastapi_request)# Order: variant-qualified keys before bare ids, load path before advertised id.from utils.openai_auto_switch_settings import (resolve_override_for_load,@@ -17272,6 +17291,9 @@ async def load_model_gated()async with nullcontext() if new_slot else inference_lifecycle_gate():_raise_if_sidecar_swap_in_progress()+ if not new_slot:+ # An auto-switch beside checked before this gate: this load replaces a loaded model.+ _check_switch_precondition(fastapi_request)# The 409 gate runs inside _load_model_impl, under the lifecycle gate, atomic with teardown.if new_slot:reload_gate = None@@ -17342,11 +17364,14 @@ async def load_model_gated(raise refusal_for(await asyncio.to_thread(chat_eviction_reports)) from exc+ # The new slot loads with no gate against chats: recheck before anything is unloaded.+ _check_switch_precondition(fastapi_request)# An idle image or video model goes before any chat model, as gpu_residency orders it.if await _free_for_chat(exc, evicted, media_only = True):continue# No chat may start on a victim between its pick and teardown.async with inference_lifecycle_gate() if new_slot else nullcontext():+ _check_switch_precondition(fastapi_request)victims = await asyncio.to_thread(model_slots.eviction_victims,extra,@@ -17397,6 +17422,8 @@ async def load_model_gated(# The drain can outlast a sidecar install or a cancel: never unload for nothing._raise_if_sidecar_swap_in_progress()_raise_if_load_cancelled(attempt, impl_kwargs)+ # A user back during the drain keeps their model.+ _check_switch_precondition(fastapi_request)else:_raise_or_cancel_active_generations(force = request.force_cancel_active,diff --git a/studio/backend/tests/test_multi_model_serving.py b/studio/backend/tests/test_multi_model_serving.pyindex 7be9d7eeb38..7bca84ee3a4 100644--- a/studio/backend/tests/test_multi_model_serving.py+++ b/studio/backend/tests/test_multi_model_serving.py@@ -1817,3 +1817,50 @@ def test_a_short_load_still_replaces_an_active_model_it_may_share_with(backends,assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"assert calls[-1] == (False, False)assert primary.model_identifier == "org/C-GGUF"+++def _away_request(check):+ from starlette.requests import Request++ request = Request({"type": "http", "method": "POST", "path": "/v1/chat/completions"})+ inf.require_for_model_switch(request.scope, check)+ return request+++def test_an_unattended_replace_fallback_rechecks_the_user_is_away_after_the_drain(+ backends, monkeypatch+):+ primary, extra = backends+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])+ back = []++ def still_away():+ if back:+ raise HTTPException(status_code = 409, detail = "model_busy")++ async def wait(**kw):+ back.append(True) # the user's chat on the active model ends during the drain++ monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(inf.load_model_gated(request, _away_request(still_away), "s", background = True))+ assert exc.value.status_code == 409 and exc.value.detail == "model_busy"+ assert primary.model_identifier == "org/A-GGUF" and primary.is_loaded+++def test_an_unattended_load_that_fits_beside_the_primary_is_not_rechecked(backends, monkeypatch):+ primary, _ = backends+ _gated_load_fakes(monkeypatch, short_fits = [])++ def refuse():+ raise AssertionError("an alongside load that touches no loaded model was rechecked")++ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)+ assert (+ asyncio.run(inf.load_model_gated(request, _away_request(refuse), "s", background = True))+ == "loaded"+ )+ assert primary.model_identifier == "org/A-GGUF"+ assert model_slots.slots[-1].request.model_path == "org/C-GGUF"diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex c6b1cd6d4a9..63f048fda6a 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -12438,3 +12438,66 @@ def test_a_background_tracked_load_keeps_the_request_cancel_and_no_user_warningsassert seen["load_cancel_event"] is attempt.cancel_eventassert seen["warnings"] is (not background)assert attempt.cancel_event.is_set() is background+++def test_an_unattended_switch_beside_rechecks_after_the_load_gate_before_replacing_the_primary(+ monkeypatch,+):+ # Keep multiple models loaded: the switch checks before load_model_gated takes the load gate,+ # so a user load that lands during that wait must still refuse a load onto the primary.+ from contextlib import asynccontextmanager+ from starlette.requests import Request+ from core.scheduled_tasks import scheduler++ _reset_keepwarm()+ monkeypatch.setattr(kw, "_inflight", 1)+ stamp = [1.0]+ checks = []+ loads = []+ backend, _ = _wired(monkeypatch, _FakeBackend("org/A-GGUF"), ("org/B-GGUF", None, "org/B-GGUF"))+ monkeypatch.setattr(kw, "attended_stamp", lambda: stamp[0])+ monkeypatch.setattr(inference_route, "_switch_loads_beside", lambda *_: True)+ monkeypatch.setattr(inference_route, "_raise_if_sidecar_swap_in_progress", lambda: None)++ @asynccontextmanager+ async def waited_load_gate():+ stamp[0] = 2.0 # the user's own load finishes while this one queues for the gate+ yield++ monkeypatch.setattr(kw, "model_load_gate", waited_load_gate)++ async def select_primary(_request):+ return None # as for a managed-engine or NPU primary: the load would replace it++ monkeypatch.setattr(inference_route, "_select_load_slot", select_primary)++ async def tracked_load(request, *_a, **_k):+ loads.append(request.model_path)+ backend.model_identifier = request.model_path+ return {}++ monkeypatch.setattr(inference_route, "_run_tracked_load_model_impl", tracked_load)+ request = Request(+ {+ "type": "http",+ "method": "POST",+ "path": "/v1/chat/completions",+ "headers": [],+ "query_string": b"",+ }+ )+ swap = types.SimpleNamespace(attended = 1.0)++ def require_away():+ checks.append(stamp[0])+ try:+ scheduler._require_still_away(swap)+ except scheduler.RunFailed:+ raise HTTPException(409, "model_busy") from None++ inference_route.require_for_model_switch(request.scope, require_away)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(inference_route._maybe_auto_switch_model("org/B-GGUF", request, "tester"))+ assert exc.value.status_code == 409+ assert checks == [1.0, 2.0]+ assert loads == [] and backend.model_identifier == "org/A-GGUF"diff --git a/studio/backend/tests/test_scheduled_tasks.py b/studio/backend/tests/test_scheduled_tasks.pyindex af3c1c7f691..49754101edc 100644--- a/studio/backend/tests/test_scheduled_tasks.py+++ b/studio/backend/tests/test_scheduled_tasks.py@@ -1422,6 +1422,76 @@ def test_a_scheduled_run_while_away_switches_back_to_the_earlier_model(monkeypatassert world.loaded == SwapWorld.EARLIER+def test_a_scheduled_run_stays_off_a_model_the_user_loaded_while_it_waited(monkeypatch):+ # The user's load holds the lifecycle gate, so admission waits it out; the load landing stamps attendance.+ world = SwapWorld(monkeypatch)+ stamp = [1.0]+ monkeypatch.setattr(llama_keepwarm, "attended_stamp", lambda: stamp[0])+ users_model = ("/models/llama.gguf", "Q4_K_M", "unsloth/Llama-3.2-1B-GGUF")+ start = llama_keepwarm.InferenceActivityReservation.start++ async def wait_out_the_users_load(self, cancel_event = None):+ world.loaded, world.idle = users_model, False+ stamp[0] = 2.0+ await start(self, cancel_event)++ monkeypatch.setattr(+ llama_keepwarm.InferenceActivityReservation, "start", wait_out_the_users_load+ )+ producer = world.producer()+ with pytest.raises(RunFailed) as failure:+ world.run(monkeypatch, {"id": "run-1", "source": "schedule"}, producer)+ assert failure.value.code == "model_busy"+ assert producer.payload is None+ assert world.loaded == users_model+ assert world.unloaded == [] and world.reloads == []+++def test_a_scheduled_switch_rechecks_under_its_gate_for_a_user_request(monkeypatch):+ # A chat that starts after admission is caught where the switch would replace the model.+ world = SwapWorld(monkeypatch)+ others = [0]+ monkeypatch.setattr(llama_keepwarm, "other_inference_request_count", lambda *a, **k: others[0])++ class SwitchesAfterAUserRequestStarts(FakeProducer):+ async def __call__(self, payload, request, current_subject, *, cancel_on_disconnect):+ self.payload = payload+ others[0] = 1+ inference_routes._check_switch_precondition(request)+ world.loaded = ("/models/qwen.gguf", "Q4_K_M", world.TASK_MODEL)+ return completion("Dawn light.")++ with pytest.raises(RunFailed) as failure:+ world.run(+ monkeypatch,+ {"id": "run-1", "source": "schedule"},+ SwitchesAfterAUserRequestStarts(),+ )+ assert failure.value.code == "model_busy"+ assert world.loaded == SwapWorld.EARLIER+++def test_a_run_the_user_started_is_not_held_to_the_away_check(monkeypatch):+ world = SwapWorld(monkeypatch, idle = False)+ seen = []++ class Switches(FakeProducer):+ async def __call__(self, payload, request, current_subject, *, cancel_on_disconnect):+ seen.append(request.scope.get(inference_routes._SWITCH_PRECONDITION_SCOPE_KEY))+ inference_routes._check_switch_precondition(request)+ world.loaded = ("/models/qwen.gguf", "Q4_K_M", world.TASK_MODEL)+ return await super().__call__(+ payload, request, current_subject, cancel_on_disconnect = cancel_on_disconnect+ )++ world.run(+ monkeypatch,+ {"id": "run-1", "source": "manual"},+ Switches(completion("Dawn light.")),+ )+ assert seen == [None]++def test_switching_back_unloads_only_a_task_model_loaded_beside_the_primary(monkeypatch):from core.inference.orchestrator import routed_slot