Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: unload a local model without stopping external provider chats (#13917)

Nilay6f4a3e3caefb
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 70f4339e378..afa37aa6737 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -5733,12 +5733,7 @@ def _prune_pending(now: float) -> None:
class _TrackedCancel:
- """Register cancel_event in _CANCEL_REGISTRY for the block's duration.
-
- Also records the run in state.active_generations so /load and /unload can
- see which chats a reload would interrupt. Both registries share this event,
- so either one cancels down the same per-request path.
- """
+ """track one event in both cancellation and active-generation registries."""
def __init__(
self,
@@ -5749,15 +5744,20 @@ class _TrackedCancel:
model = None,
kind = "chat",
track_run = True,
+ external = False,
):
self.event = event
self.keys = tuple(_account_cancel_key(k) for k in keys if k)
- # kind reaches the swap prompt: embeddings and raw completions have no conversation, so
- # naming them chats would offer to stop something the user never started from a thread.
- # track_run=False only adds cancel keys to a run another tracker already records.
+ # kind avoids calling embeddings and raw completions chats in the swap prompt.
+ # track_run=False adds cancel keys to a run recorded by another tracker.
self._active = (
active_generations.ActiveGeneration(
- event, thread_id = thread_id, run_id = run_id, model = model, kind = kind
+ event,
+ thread_id = thread_id,
+ run_id = run_id,
+ model = model,
+ kind = kind,
+ external = external,
)
if track_run
else None
@@ -5765,19 +5765,24 @@ class _TrackedCancel:
self._slot = None
@classmethod
- def for_payload(cls, event: threading.Event, payload, *keys):
- """Track the run against the conversation its request names."""
+ def for_payload(
+ cls,
+ event: threading.Event,
+ payload,
+ *keys,
+ external = False,
+ ):
return cls(
event,
*keys,
thread_id = getattr(payload, "thread_id", None),
run_id = getattr(payload, "generation_run_id", None),
model = getattr(payload, "model", None),
+ external = external,
)
def __enter__(self):
- # Register + consume-pending in one critical section to close the
- # TOCTOU race against a concurrent cancel POST.
+ # keep registration and pending-cancel consumption atomic with concurrent cancellation.
should_cancel = False
with _CANCEL_LOCK:
for k in self.keys:
@@ -9349,7 +9354,7 @@ async def _wait_for_model_switch_idle(
elsewhere = model_slots.all_generations()
active_others -= min(active_others, len(elsewhere))
if cancel_pending:
- cancellable = active_generations.count(account_id, elsewhere)
+ cancellable = active_generations.count(account_id, elsewhere, include_external = False)
active_others -= min(active_others, cancellable)
if active_others <= queued_switches:
return
@@ -17612,41 +17617,26 @@ def _raise_or_cancel_active_generations(
action: str,
cancel: bool = True,
) -> int:
- """Gate a model swap on the chats currently generating.
-
- Every open conversation decodes on the single llama-server this route is
- about to replace, so refuse with 409 and name them. force_cancel_active
- instead stops them through the same events an explicit Stop uses. Returns
- how many were cancelled. The frontend guard is bypassable from a second tab
- or curl; this one is not.
-
- ``cancel = False`` runs the refusal half only. /load calls it that way once
- up front, so a non-forced swap still fails fast, and again with cancel just
- before teardown: cancelling is destructive and unrecoverable, so it must not
- run ahead of preflight checks that can still reject the load (see
- _load_model_impl).
- """
+ """refuse model swaps that would stop managed chats, or cancel them only after preflight."""
from core.inference.gpu_arbiter import require_no_foreign_generations
scope = account_access.account_scope()
if scope is not None:
- # Before the count and the cancel: a foreign generation refuses the swap, so cancelling
- # first would end the caller's chats for nothing. Keyed on account_scope(), whose count
- # drops while a deactivated account's generation still holds the GPU.
+ # account_scope() catches deactivated GPU use; check it before any destructive cancellation.
require_no_foreign_generations(scope)
elsewhere = model_slots.all_generations()
- if not active_generations.count(scope, elsewhere):
+ if not active_generations.count(scope, elsewhere, include_external = False):
return 0
if not force:
raise _active_generations_conflict(
action,
- active_generations.count(scope, elsewhere),
- active_generations.active_thread_ids(scope, elsewhere),
+ active_generations.count(scope, elsewhere, include_external = False),
+ active_generations.active_thread_ids(scope, elsewhere, include_external = False),
)
if not cancel:
- # Refusal-only pass: the caller cancels later, once nothing can still reject the load.
+ # the caller cancels only after preflight can no longer reject the load.
return 0
- cancelled = active_generations.cancel_all(scope, elsewhere)
+ cancelled = active_generations.cancel_all(scope, elsewhere, include_external = False)
if cancelled:
logger.info(
"model_swap_cancelled_active_generations",
@@ -17659,23 +17649,7 @@ _POST_CANCEL_DRAIN_TIMEOUT_S = 5.0
async def _cancel_and_drain_for_sidecar_swap(timeout_s: Optional[float] = None) -> None:
- """Clear the way for a confirmed sidecar swap, then stop the chats it interrupts.
-
- The installer gates on the middleware's in-flight count, not on
- active_generations, so it also sees requests the cancel cannot stop. Drain
- those FIRST, discounting the registered chats (they are what the cancel is
- for, so waiting on them would wait out the point of the force). Only then
- cancel, and let the survivors unwind. Cancelling first meant an unrelated
- counted request -- a /v1/messages/count_tokens, say -- was still there for
- the caller's recheck, which then refused an install that had already stopped
- every chat for nothing.
-
- Bounded on both halves: the requests being waited on may never observe a
- cancel, and this holds the lifecycle gate and the sidecar reservation inside
- ``asyncio.shield``, so an unbounded wait would wedge the process. Expiring in
- the first half returns without cancelling, so the caller's recheck refuses
- with the chats untouched.
- """
+ """drain unrelated requests before cancellation so a rejected install cannot stop chats for nothing."""
from core.inference.llama_keepwarm import other_inference_request_count
budget = _POST_CANCEL_DRAIN_TIMEOUT_S if timeout_s is None else timeout_s
@@ -17686,16 +17660,14 @@ async def _cancel_and_drain_for_sidecar_swap(timeout_s: Optional[float] = None)
current_request_counted = False, include_pending = False
)
if discount_registered:
- counted -= min(counted, active_generations.count())
+ counted -= min(counted, active_generations.count(None, include_external = False))
if counted <= 0:
return True
if time.monotonic() >= deadline:
return False
await asyncio.sleep(0.02)
- # Weighted, not halved, so the total wait under the gate is unchanged. The first drain only
- # asks whether unrelated inference is in flight; cutting the second short refused installs
- # whose chats had already been stopped for nothing.
+ # reserve most of the unchanged budget for chats that have been cancelled and are unwinding.
if not await _drain(time.monotonic() + budget / 5, discount_registered = True):
return
_raise_or_cancel_active_generations(force = True, action = "Installing a new transformers version")
@@ -17892,16 +17864,9 @@ async def get_active_generations(
current_subject: str = Depends(get_current_subject),
model: Optional[str] = None,
):
- """Conversations currently generating, plus how many can decode at once.
-
- Lets a model swap name the chats it would interrupt, including runs this tab
- cannot see (another tab, or a reload behind a proxy). parallel_slots is the
- slot count actually in use, which the VRAM fit may have cut below the
- requested --parallel; chats beyond it queue rather than fail.
- """
+ """return interruptible local generations and the effective decode capacity."""
scope = account_access.account_scope()
- # ``model``: only the chats an unload of that model stops, the same split its scoped 409 uses.
- exclude, only, llama = (), None, None
+ exclude, only, llama = set(), None, None
if model and model_slots.slots:
slot = await asyncio.to_thread(
model_slots.serving_slot, model, model_slots.visible(), _loaded_satisfies
@@ -17909,12 +17874,11 @@ async def get_active_generations(
if slot is not None:
only, llama = set(slot.generations), slot.llama
elif await asyncio.to_thread(model_slots.in_slot, None, lambda: _loaded_satisfies(model)):
- exclude = model_slots.all_generations()
+ exclude |= model_slots.all_generations()
else:
- only = set() # no loaded model by that name: its unload stops nothing
- entries = active_generations.snapshot(scope, exclude, only)
- # A tracker's model can be a native local path (the legacy stream records active_model_name
- # verbatim); redact here, the one place that serialises it.
+ only = set() # an absent model cannot interrupt any generation
+ entries = active_generations.snapshot(scope, exclude, only, include_external = False)
+ # legacy streams record native model paths, so redact them at the serialization boundary.
for _entry in entries:
if isinstance(_entry.get("model"), str):
_entry["model"] = redact_native_paths(_entry["model"])
@@ -17926,19 +17890,15 @@ async def get_active_generations(
return {
"active": entries,
"count": len(entries),
- "thread_ids": active_generations.active_thread_ids(scope, exclude, only),
+ "thread_ids": active_generations.active_thread_ids(
+ scope, exclude, only, include_external = False
+ ),
"parallel_slots": max(1, int(slots)),
}
def _mlx_runtime_settings_match(backend, request) -> bool:
- """Whether a loaded model already runs the request's MLX runtime settings.
-
- Only the MLX backend records these, so a backend that never stored them
- (GGUF, CUDA safetensors) always matches and reuse is unaffected. Both sides
- are normalized, otherwise an out-of-domain value would differ from the
- stored None on every request and reload forever.
- """
+ """match normalized MLX settings without forcing reloads for backends that do not record them."""
entry = backend.models.get(backend.active_model_name, {}) or {}
if "mlx_kv_quant_requested" not in entry:
return True
@@ -30116,14 +30076,13 @@ async def _proxy_to_external_provider(
finally:
cancel_event.set()
if disconnect_task is not None:
- # Joined so its outcome is retrieved. No bounded stop needed: cancel_event is
- # set above, so the loop exits even if is_disconnected() eats the cancel (#7617).
+ # cancel_event ends the loop if is_disconnected() swallows cancellation (#7617).
disconnect_task.cancel()
await asyncio.gather(disconnect_task, return_exceptions = True)
try:
await gen.aclose()
except RuntimeError:
- pass # suppress httpcore asyncgen cleanup error (Python 3.13 + httpcore 1.0.x)
+ pass # httpcore cleanup may raise on Python 3.13 with httpcore 1.0.x
await client.close()
def _tracked_stream():
@@ -30132,12 +30091,14 @@ async def _proxy_to_external_provider(
async def _wrapped():
inner = _stream()
- with _TrackedCancel.for_payload(cancel_event, payload, *cancel_keys):
+ with _TrackedCancel.for_payload(
+ cancel_event, payload, *cancel_keys, external = managed is None
+ ):
try:
async for chunk in inner:
yield chunk
finally:
- # Close it now, not at garbage collection: that releases the upstream request.
+ # explicit close releases the upstream request before garbage collection.
await inner.aclose()
return _wrapped()
diff --git a/studio/backend/state/active_generations.py b/studio/backend/state/active_generations.py
index 495fa866532..a0549a0b7bf 100644
--- a/studio/backend/state/active_generations.py
+++ b/studio/backend/state/active_generations.py
@@ -34,10 +34,7 @@ _FENCED: set[str] = set()
class ActiveGeneration:
- """Registers one in-flight generation for the duration of the block.
-
- Each __enter__ mints its own handle, so overlapping uses never clobber.
- """
+ """uses a unique handle so overlapping registrations cannot clobber each other."""
__slots__ = (
"thread_id",
@@ -46,6 +43,7 @@ class ActiveGeneration:
"model",
"kind",
"account_id",
+ "external",
"_handle",
"_borrowed",
)
@@ -59,8 +57,10 @@ class ActiveGeneration:
model: Optional[str] = None,
kind: str = "chat",
account_id: Optional[str] = None,
+ external: bool = False,
):
self.account_id = account_id or current_account_id()
+ self.external = external
self.thread_id = thread_id or None
self.run_id = run_id or None
self.cancel_event = cancel_event
@@ -71,8 +71,7 @@ class ActiveGeneration:
def __enter__(self) -> "ActiveGeneration":
with _LOCK:
- # A durable supervisor registers before model loading starts and the route later enters its normal
- # tracker with that same event and run, so borrow the outer registration.
+ # borrow the supervisor's pre-load registration when the route reuses its event and run.
if self.run_id:
for entry in _ACTIVE.values():
if (
@@ -97,6 +96,7 @@ class ActiveGeneration:
"model": self.model,
"kind": self.kind,
"account_id": self.account_id,
+ "external": self.external,
"started_at": time.time(),
"event": self.cancel_event,
}
@@ -119,6 +119,8 @@ def _matching(
account_id: Optional[str],
exclude: Collection[threading.Event],
only: Optional[Collection[threading.Event]] = None,
+ *,
+ include_external: bool = True,
) -> list[dict[str, Any]]:
return [
e
@@ -126,6 +128,7 @@ def _matching(
if (account_id is None or e["account_id"] == account_id)
and e["event"] not in exclude
and (only is None or e["event"] in only)
+ and (include_external or not e["external"])
]
@@ -133,11 +136,12 @@ def snapshot(
account_id: Optional[str] = None,
exclude: Collection[threading.Event] = (),
only: Optional[Collection[threading.Event]] = None,
+ *,
+ include_external: bool = True,
) -> list[dict[str, Any]]:
- """In-flight generations, newest last; ``account_id`` None (all) is shutdown/arbiter only.
- ``exclude`` leaves out the runs holding those cancel events."""
+ """in-flight generations, newest last; ``account_id=None`` is for shutdown or the arbiter."""
with _LOCK:
- entries = _matching(account_id, exclude, only)
+ entries = _matching(account_id, exclude, only, include_external = include_external)
entries.sort(key = lambda e: e["started_at"])
return [
{
@@ -157,38 +161,49 @@ def active_thread_ids(
account_id: Optional[str] = None,
exclude: Collection[threading.Event] = (),
only: Optional[Collection[threading.Event]] = None,
+ *,
+ include_external: bool = True,
) -> list[str]:
- """Distinct conversation ids with a generation in flight, in start order.
-
- A first turn that races persistence has no thread id yet: count() sees it,
- this cannot name it.
- """
+ """distinct conversation ids in start order; an unpersisted first turn cannot be named."""
seen: list[str] = []
- for e in snapshot(account_id, exclude, only):
+ for e in snapshot(account_id, exclude, only, include_external = include_external):
tid = e["thread_id"]
if tid and tid not in seen:
seen.append(tid)
return seen
-def count(account_id: Optional[str] = None, exclude: Collection[threading.Event] = ()) -> int:
+def count(
+ account_id: Optional[str] = None,
+ exclude: Collection[threading.Event] = (),
+ *,
+ include_external: bool = True,
+) -> int:
with _LOCK:
- if account_id is None and not exclude:
+ if account_id is None and not exclude and include_external:
return len(_ACTIVE)
- return len(_matching(account_id, exclude))
+ return len(_matching(account_id, exclude, include_external = include_external))
def foreign_count(account_id: str) -> int:
- """Generations in flight for OTHER accounts, which a load or unload must not interrupt."""
+ """generations for other accounts that load and unload must not interrupt."""
with _LOCK:
- return sum(1 for e in _ACTIVE.values() if e["account_id"] != account_id)
+ return sum(
+ 1 for e in _ACTIVE.values() if e["account_id"] != account_id and not e["external"]
+ )
-def cancel_all(account_id: Optional[str] = None, exclude: Collection[threading.Event] = ()) -> int:
- """Signal in-flight generations to stop, returning the count. Request-driven callers must pass
- ``account_id``; None means everyone and is for shutdown only."""
+def cancel_all(
+ account_id: Optional[str] = None,
+ exclude: Collection[threading.Event] = (),
+ *,
+ include_external: bool = True,
+) -> int:
+ """request callers must pass ``account_id``; None cancels every account during shutdown."""
with _LOCK:
- events = [e["event"] for e in _matching(account_id, exclude)]
+ events = [
+ e["event"] for e in _matching(account_id, exclude, include_external = include_external)
+ ]
for ev in events:
try:
ev.set()
@@ -198,7 +213,7 @@ def cancel_all(account_id: Optional[str] = None, exclude: Collection[threading.E
def cancel_thread(thread_id: str, account_id: Optional[str] = None) -> int:
- """Signal ``thread_id``'s generations; thread ids are client-chosen, so scope by account."""
+ """thread ids are client-chosen, so cancellation is scoped by account."""
if not thread_id:
return 0
scope = account_id or current_account_id()
diff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.py
index 4b4ba5feffd..10a23edcbf9 100644
--- a/studio/backend/tests/test_active_generations.py
+++ b/studio/backend/tests/test_active_generations.py
@@ -2963,3 +2963,169 @@ def test_responses_stream_stamps_tool_call_deltas(monkeypatch):
[row] = [r for r in api_monitor.snapshot() if r["id"] == monitor_id]
assert row["ttft_ms"] is not None
assert row["stop_reason"] == "tool_calls"
+
+
+def _run_external_tool_chat(during):
+ import asyncio
+ import json
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+ from types import SimpleNamespace
+
+ import httpx
+ from starlette.requests import Request
+
+ import routes.inference as inf_mod
+ from core.inference import external_provider as ep_mod
+ from models.inference import ChatCompletionRequest
+
+ release = threading.Event()
+
+ class _SlowProvider(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def log_message(self, *args, **kwargs):
+ return
+
+ def do_POST(self):
+ self.rfile.read(int(self.headers.get("Content-Length") or 0))
+ self.send_response(200)
+ self.send_header("Content-Type", "text/event-stream")
+ self.send_header("Connection", "close")
+ self.end_headers()
+ for word in ("one ", "two "):
+ chunk = {"choices": [{"index": 0, "delta": {"content": word}}]}
+ self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
+ self.wfile.flush()
+ release.wait(10)
+ done = {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
+ self.wfile.write(f"data: {json.dumps(done)}\n\ndata: [DONE]\n\n".encode())
+ self.wfile.flush()
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), _SlowProvider)
+ threading.Thread(target = server.serve_forever, daemon = True).start()
+
+ async def receive():
+ await asyncio.Event().wait()
+
+ payload = ChatCompletionRequest(
+ provider_type = "custom",
+ provider_base_url = f"http://127.0.0.1:{server.server_address[1]}/v1",
+ model = "remote-model",
+ messages = [{"role": "user", "content": "hi"}],
+ stream = True,
+ thread_id = "external-chat",
+ cancel_id = "external-run",
+ enable_tools = True,
+ enabled_tools = ["python"],
+ )
+ request = Request(
+ {
+ "type": "http",
+ "http_version": "1.1",
+ "method": "POST",
+ "path": "/api/inference/chat/completions",
+ "raw_path": b"/api/inference/chat/completions",
+ "root_path": "",
+ "scheme": "http",
+ "query_string": b"",
+ "headers": [(b"content-type", b"application/json"), (b"x-unsloth-events", b"1")],
+ "client": ("127.0.0.1", 12345),
+ "server": ("127.0.0.1", 8000),
+ "app": SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1)),
+ },
+ receive,
+ )
+
+ async def run():
+ previous = ep_mod._http_client
+ ep_mod._http_client = httpx.AsyncClient()
+ try:
+ response = await inf_mod._proxy_to_external_provider(payload, request)
+ iterator = response.body_iterator
+ chunks = []
+ while active_generations.count() == 0:
+ chunks.append(await asyncio.wait_for(iterator.__anext__(), 10))
+ seen = await during(inf_mod, request)
+ release.set()
+ async for chunk in iterator:
+ chunks.append(chunk)
+ return seen, "".join(c.decode() if isinstance(c, bytes) else c for c in chunks)
+ finally:
+ release.set()
+ await ep_mod._http_client.aclose()
+ ep_mod._http_client = previous
+ server.shutdown()
+ server.server_close()
+
+ return asyncio.run(run())
+
+
+def test_an_external_tool_chat_is_not_stopped_by_a_local_model_swap():
+ _route_gate()
+
+ async def during(inf_mod, request):
+ [entry] = active_generations.snapshot()
+ listed = await inf_mod.get_active_generations(request, "tester")
+ inf_mod._raise_or_cancel_active_generations(force = False, action = "Unloading the model")
+ cancelled = inf_mod._raise_or_cancel_active_generations(
+ force = True, action = "Unloading the model"
+ )
+ return entry, listed, cancelled
+
+ (entry, listed, cancelled), body = _run_external_tool_chat(during)
+
+ assert entry["thread_id"] == "external-chat"
+ assert listed["count"] == 0 and listed["thread_ids"] == []
+ assert cancelled == 0
+ assert "two" in body and "[DONE]" in body
+ assert active_generations.count() == 0
+
+
+def test_external_registration_racing_the_swap_gate_stays_excluded(monkeypatch):
+ gate = _route_gate()
+ event = threading.Event()
+ tracker = active_generations.ActiveGeneration(
+ event,
+ thread_id = "external-chat",
+ external = True,
+ )
+ original_count = active_generations.count
+ entered = False
+
+ def count_after_registration(*args, **kwargs):
+ nonlocal entered
+ if not entered:
+ tracker.__enter__()
+ entered = True
+ return original_count(*args, **kwargs)
+
+ monkeypatch.setattr(active_generations, "count", count_after_registration)
+ try:
+ assert gate(force = False, action = "Unloading the model") == 0
+ assert not event.is_set()
+ finally:
+ if entered:
+ tracker.__exit__(None, None, None)
+
+
+def test_an_external_tool_chat_still_stops_on_its_own_stop():
+ _route_gate()
+
+ async def during(inf_mod, request):
+ return inf_mod._cancel_by_cancel_id_or_stash("external-run")
+
+ stopped, _ = _run_external_tool_chat(during)
+
+ assert stopped == 1
+ assert active_generations.count() == 0
+
+
+def test_deleting_an_external_tool_chat_still_stops_it():
+ _route_gate()
+
+ async def during(inf_mod, request):
+ return active_generations.cancel_thread("external-chat")
+
+ stopped, _ = _run_external_tool_chat(during)
+
+ assert stopped == 1