Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: recheck that the user is still away before an automatic task swaps models (#15436)

Daniel Han3c4802b2f57f
diff --git a/studio/backend/core/scheduled_tasks/scheduler.py b/studio/backend/core/scheduled_tasks/scheduler.py
index 0d8ef9b1560..40b7d3a0695 100644
--- a/studio/backend/core/scheduled_tasks/scheduler.py
+++ b/studio/backend/core/scheduled_tasks/scheduler.py
@@ -379,6 +379,16 @@ async def _plan_swap(task: dict, model: str, unattended: bool, carried: Optional
)
+def _require_still_away(swap: _Swap) -> None:
+ """Raise model_busy once someone came back since the swap was planned: an inference request ended,
+ a model load landed, or a request is running beside this run."""
+ from core.inference.llama_keepwarm import attended_stamp, other_inference_request_count
+ if attended_stamp() != swap.attended or other_inference_request_count(
+ current_request_counted = True
+ ):
+ raise RunFailed("model_busy")
+
+
async def _restore_after_swap(swap: _Swap, app: Any) -> None:
"""Put back what the run's load replaced, unless someone used or loaded a model since. Best effort."""
from core.inference.llama_keepwarm import (
@@ -484,6 +494,7 @@ async def generate_answer(
allow_model_switch_for_request,
load_alongside_for_request,
produce_openai_chat_completions,
+ require_for_model_switch,
)
carried = holder.pending if holder is not None else None
@@ -494,16 +505,8 @@ async def generate_answer(
model_fields, provider, allow_switch = _model_fields(
task.get("model"), bool(task.get("allowModelLoad") or run.get("allowModelLoad"))
)
- swap = (
- await _plan_swap(
- task,
- task["model"],
- run.get("source") == "schedule" and not run.get("allowModelLoad"),
- carried,
- )
- if allow_switch
- else None
- )
+ unattended = run.get("source") == "schedule" and not run.get("allowModelLoad")
+ swap = await _plan_swap(task, task["model"], unattended, carried) if allow_switch else None
if holder is not None:
holder.pending = swap or carried
loop = asyncio.get_running_loop()
@@ -515,6 +518,18 @@ async def generate_answer(
allow_model_switch_for_request(request.scope)
# Keep multiple models loaded: the task model goes beside the user's when it fits, as from Chat's picker.
load_alongside_for_request(request.scope)
+ away_swap = swap if unattended else None
+ refused: list[bool] = []
+ if away_swap is not None:
+ # The switch rechecks under its gates; it can only answer with an HTTP error, read back as model_busy.
+ def still_away() -> None:
+ try:
+ _require_still_away(away_swap)
+ except RunFailed:
+ refused.append(True)
+ raise HTTPException(status_code = 409, detail = "model_busy") from None
+
+ require_for_model_switch(request.scope, still_away)
with unattended_activity():
activity = InferenceActivityReservation()
activity.reserve()
@@ -529,6 +544,9 @@ async def generate_answer(
raise RunFailed("timeout") from None
if cancel_event.is_set():
raise RunFailed("cancelled")
+ if away_swap is not None:
+ # Admission can wait out a load the user started, so the idle check before it may be stale.
+ _require_still_away(away_swap)
# Built once admitted: a model kept alongside is chosen as it stands now, not before a load or unload.
if not task.get("model") and (kept := _sole_kept_model()):
model_fields = {"model": kept}
@@ -596,6 +614,8 @@ async def generate_answer(
try:
response = producer.result()
except HTTPException as exc:
+ if refused:
+ raise RunFailed("model_busy") from None
if allow_switch and not _model_is_loaded(payload.model):
raise RunFailed("model_load_failed", _error_text(exc.detail)) from None
# A named kept model answered for itself, so its refusal is not "nothing loaded",
@@ -609,6 +629,8 @@ async def generate_answer(
raise RunFailed(None, _error_text(exc.detail)) from None
except GpuBusyForAnotherAccountError as exc:
raise RunFailed(None, _error_text(exc.as_http_exception().detail)) from None
+ if refused:
+ raise RunFailed("model_busy")
if serving.get("loaded") is False:
raise RunFailed("model_load_failed")
finally:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index d83797ce4a0..3ef38f75a5b 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -9493,6 +9493,23 @@ def _model_switch_allowed_for_request(fastapi_request) -> bool:
return isinstance(scope, dict) and bool(scope.get(_ALLOW_MODEL_SWITCH_SCOPE_KEY))
+_SWITCH_PRECONDITION_SCOPE_KEY = "unsloth.switch_precondition"
+
+
+def require_for_model_switch(scope, check) -> None:
+ """Have this in-process request's switch call ``check()`` under the swap gate, just before it replaces the
+ loaded model; ``check`` raises to refuse. A scope key, so no HTTP client can set it."""
+ if isinstance(scope, dict):
+ scope[_SWITCH_PRECONDITION_SCOPE_KEY] = check
+
+
+def _check_switch_precondition(fastapi_request) -> None:
+ scope = getattr(fastapi_request, "scope", None)
+ check = scope.get(_SWITCH_PRECONDITION_SCOPE_KEY) if isinstance(scope, dict) else None
+ if callable(check):
+ check()
+
+
_LOAD_ALONGSIDE_SCOPE_KEY = "unsloth.load_alongside"
@@ -11319,6 +11336,8 @@ async def _maybe_auto_switch_model(
_record_serving_alias()
return
require_no_foreign_generations(path = switch_path)
+ # Last chance to refuse: held swap gates keep inference and loads from starting before it.
+ _check_switch_precondition(fastapi_request)
# Order: variant-qualified keys before bare ids, load path before advertised id.
from utils.openai_auto_switch_settings import (
resolve_override_for_load,
@@ -17272,6 +17291,9 @@ async def load_model_gated(
)
async with nullcontext() if new_slot else inference_lifecycle_gate():
_raise_if_sidecar_swap_in_progress()
+ if not new_slot:
+ # An auto-switch beside checked before this gate: this load replaces a loaded model.
+ _check_switch_precondition(fastapi_request)
# The 409 gate runs inside _load_model_impl, under the lifecycle gate, atomic with teardown.
if new_slot:
reload_gate = None
@@ -17342,11 +17364,14 @@ async def load_model_gated(
raise refusal_for(
await asyncio.to_thread(chat_eviction_reports)
) from exc
+ # The new slot loads with no gate against chats: recheck before anything is unloaded.
+ _check_switch_precondition(fastapi_request)
# An idle image or video model goes before any chat model, as gpu_residency orders it.
if await _free_for_chat(exc, evicted, media_only = True):
continue
# No chat may start on a victim between its pick and teardown.
async with inference_lifecycle_gate() if new_slot else nullcontext():
+ _check_switch_precondition(fastapi_request)
victims = await asyncio.to_thread(
model_slots.eviction_victims,
extra,
@@ -17397,6 +17422,8 @@ async def load_model_gated(
# The drain can outlast a sidecar install or a cancel: never unload for nothing.
_raise_if_sidecar_swap_in_progress()
_raise_if_load_cancelled(attempt, impl_kwargs)
+ # A user back during the drain keeps their model.
+ _check_switch_precondition(fastapi_request)
else:
_raise_or_cancel_active_generations(
force = request.force_cancel_active,
diff --git a/studio/backend/tests/test_multi_model_serving.py b/studio/backend/tests/test_multi_model_serving.py
index 7be9d7eeb38..7bca84ee3a4 100644
--- a/studio/backend/tests/test_multi_model_serving.py
+++ b/studio/backend/tests/test_multi_model_serving.py
@@ -1817,3 +1817,50 @@ def test_a_short_load_still_replaces_an_active_model_it_may_share_with(backends,
assert asyncio.run(inf.load_model_gated(request, None, "s")) == "loaded"
assert calls[-1] == (False, False)
assert primary.model_identifier == "org/C-GGUF"
+
+
+def _away_request(check):
+ from starlette.requests import Request
+
+ request = Request({"type": "http", "method": "POST", "path": "/v1/chat/completions"})
+ inf.require_for_model_switch(request.scope, check)
+ return request
+
+
+def test_an_unattended_replace_fallback_rechecks_the_user_is_away_after_the_drain(
+ backends, monkeypatch
+):
+ primary, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ _gated_load_fakes(monkeypatch, short_fits = [1, 1])
+ back = []
+
+ def still_away():
+ if back:
+ raise HTTPException(status_code = 409, detail = "model_busy")
+
+ async def wait(**kw):
+ back.append(True) # the user's chat on the active model ends during the drain
+
+ monkeypatch.setattr(inf, "_wait_for_model_switch_idle", wait)
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inf.load_model_gated(request, _away_request(still_away), "s", background = True))
+ assert exc.value.status_code == 409 and exc.value.detail == "model_busy"
+ assert primary.model_identifier == "org/A-GGUF" and primary.is_loaded
+
+
+def test_an_unattended_load_that_fits_beside_the_primary_is_not_rechecked(backends, monkeypatch):
+ primary, _ = backends
+ _gated_load_fakes(monkeypatch, short_fits = [])
+
+ def refuse():
+ raise AssertionError("an alongside load that touches no loaded model was rechecked")
+
+ request = LoadRequest(model_path = "org/C-GGUF", alongside = True)
+ assert (
+ asyncio.run(inf.load_model_gated(request, _away_request(refuse), "s", background = True))
+ == "loaded"
+ )
+ assert primary.model_identifier == "org/A-GGUF"
+ assert model_slots.slots[-1].request.model_path == "org/C-GGUF"
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index c6b1cd6d4a9..63f048fda6a 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -12438,3 +12438,66 @@ def test_a_background_tracked_load_keeps_the_request_cancel_and_no_user_warnings
assert seen["load_cancel_event"] is attempt.cancel_event
assert seen["warnings"] is (not background)
assert attempt.cancel_event.is_set() is background
+
+
+def test_an_unattended_switch_beside_rechecks_after_the_load_gate_before_replacing_the_primary(
+ monkeypatch,
+):
+ # Keep multiple models loaded: the switch checks before load_model_gated takes the load gate,
+ # so a user load that lands during that wait must still refuse a load onto the primary.
+ from contextlib import asynccontextmanager
+ from starlette.requests import Request
+ from core.scheduled_tasks import scheduler
+
+ _reset_keepwarm()
+ monkeypatch.setattr(kw, "_inflight", 1)
+ stamp = [1.0]
+ checks = []
+ loads = []
+ backend, _ = _wired(monkeypatch, _FakeBackend("org/A-GGUF"), ("org/B-GGUF", None, "org/B-GGUF"))
+ monkeypatch.setattr(kw, "attended_stamp", lambda: stamp[0])
+ monkeypatch.setattr(inference_route, "_switch_loads_beside", lambda *_: True)
+ monkeypatch.setattr(inference_route, "_raise_if_sidecar_swap_in_progress", lambda: None)
+
+ @asynccontextmanager
+ async def waited_load_gate():
+ stamp[0] = 2.0 # the user's own load finishes while this one queues for the gate
+ yield
+
+ monkeypatch.setattr(kw, "model_load_gate", waited_load_gate)
+
+ async def select_primary(_request):
+ return None # as for a managed-engine or NPU primary: the load would replace it
+
+ monkeypatch.setattr(inference_route, "_select_load_slot", select_primary)
+
+ async def tracked_load(request, *_a, **_k):
+ loads.append(request.model_path)
+ backend.model_identifier = request.model_path
+ return {}
+
+ monkeypatch.setattr(inference_route, "_run_tracked_load_model_impl", tracked_load)
+ request = Request(
+ {
+ "type": "http",
+ "method": "POST",
+ "path": "/v1/chat/completions",
+ "headers": [],
+ "query_string": b"",
+ }
+ )
+ swap = types.SimpleNamespace(attended = 1.0)
+
+ def require_away():
+ checks.append(stamp[0])
+ try:
+ scheduler._require_still_away(swap)
+ except scheduler.RunFailed:
+ raise HTTPException(409, "model_busy") from None
+
+ inference_route.require_for_model_switch(request.scope, require_away)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(inference_route._maybe_auto_switch_model("org/B-GGUF", request, "tester"))
+ assert exc.value.status_code == 409
+ assert checks == [1.0, 2.0]
+ assert loads == [] and backend.model_identifier == "org/A-GGUF"
diff --git a/studio/backend/tests/test_scheduled_tasks.py b/studio/backend/tests/test_scheduled_tasks.py
index af3c1c7f691..49754101edc 100644
--- a/studio/backend/tests/test_scheduled_tasks.py
+++ b/studio/backend/tests/test_scheduled_tasks.py
@@ -1422,6 +1422,76 @@ def test_a_scheduled_run_while_away_switches_back_to_the_earlier_model(monkeypat
assert world.loaded == SwapWorld.EARLIER
+def test_a_scheduled_run_stays_off_a_model_the_user_loaded_while_it_waited(monkeypatch):
+ # The user's load holds the lifecycle gate, so admission waits it out; the load landing stamps attendance.
+ world = SwapWorld(monkeypatch)
+ stamp = [1.0]
+ monkeypatch.setattr(llama_keepwarm, "attended_stamp", lambda: stamp[0])
+ users_model = ("/models/llama.gguf", "Q4_K_M", "unsloth/Llama-3.2-1B-GGUF")
+ start = llama_keepwarm.InferenceActivityReservation.start
+
+ async def wait_out_the_users_load(self, cancel_event = None):
+ world.loaded, world.idle = users_model, False
+ stamp[0] = 2.0
+ await start(self, cancel_event)
+
+ monkeypatch.setattr(
+ llama_keepwarm.InferenceActivityReservation, "start", wait_out_the_users_load
+ )
+ producer = world.producer()
+ with pytest.raises(RunFailed) as failure:
+ world.run(monkeypatch, {"id": "run-1", "source": "schedule"}, producer)
+ assert failure.value.code == "model_busy"
+ assert producer.payload is None
+ assert world.loaded == users_model
+ assert world.unloaded == [] and world.reloads == []
+
+
+def test_a_scheduled_switch_rechecks_under_its_gate_for_a_user_request(monkeypatch):
+ # A chat that starts after admission is caught where the switch would replace the model.
+ world = SwapWorld(monkeypatch)
+ others = [0]
+ monkeypatch.setattr(llama_keepwarm, "other_inference_request_count", lambda *a, **k: others[0])
+
+ class SwitchesAfterAUserRequestStarts(FakeProducer):
+ async def __call__(self, payload, request, current_subject, *, cancel_on_disconnect):
+ self.payload = payload
+ others[0] = 1
+ inference_routes._check_switch_precondition(request)
+ world.loaded = ("/models/qwen.gguf", "Q4_K_M", world.TASK_MODEL)
+ return completion("Dawn light.")
+
+ with pytest.raises(RunFailed) as failure:
+ world.run(
+ monkeypatch,
+ {"id": "run-1", "source": "schedule"},
+ SwitchesAfterAUserRequestStarts(),
+ )
+ assert failure.value.code == "model_busy"
+ assert world.loaded == SwapWorld.EARLIER
+
+
+def test_a_run_the_user_started_is_not_held_to_the_away_check(monkeypatch):
+ world = SwapWorld(monkeypatch, idle = False)
+ seen = []
+
+ class Switches(FakeProducer):
+ async def __call__(self, payload, request, current_subject, *, cancel_on_disconnect):
+ seen.append(request.scope.get(inference_routes._SWITCH_PRECONDITION_SCOPE_KEY))
+ inference_routes._check_switch_precondition(request)
+ world.loaded = ("/models/qwen.gguf", "Q4_K_M", world.TASK_MODEL)
+ return await super().__call__(
+ payload, request, current_subject, cancel_on_disconnect = cancel_on_disconnect
+ )
+
+ world.run(
+ monkeypatch,
+ {"id": "run-1", "source": "manual"},
+ Switches(completion("Dawn light.")),
+ )
+ assert seen == [None]
+
+
def test_switching_back_unloads_only_a_task_model_loaded_beside_the_primary(monkeypatch):
from core.inference.orchestrator import routed_slot