Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: unload a local model without stopping external provider chats (#13917)
Nilay6f4a3e3caefb
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 70f4339e378..afa37aa6737 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -5733,12 +5733,7 @@ def _prune_pending(now: float) -> None:class _TrackedCancel:- """Register cancel_event in _CANCEL_REGISTRY for the block's duration.-- Also records the run in state.active_generations so /load and /unload can- see which chats a reload would interrupt. Both registries share this event,- so either one cancels down the same per-request path.- """+ """track one event in both cancellation and active-generation registries."""def __init__(self,@@ -5749,15 +5744,20 @@ class _TrackedCancel:model = None,kind = "chat",track_run = True,+ external = False,):self.event = eventself.keys = tuple(_account_cancel_key(k) for k in keys if k)- # kind reaches the swap prompt: embeddings and raw completions have no conversation, so- # naming them chats would offer to stop something the user never started from a thread.- # track_run=False only adds cancel keys to a run another tracker already records.+ # kind avoids calling embeddings and raw completions chats in the swap prompt.+ # track_run=False adds cancel keys to a run recorded by another tracker.self._active = (active_generations.ActiveGeneration(- event, thread_id = thread_id, run_id = run_id, model = model, kind = kind+ event,+ thread_id = thread_id,+ run_id = run_id,+ model = model,+ kind = kind,+ external = external,)if track_runelse None@@ -5765,19 +5765,24 @@ class _TrackedCancel:self._slot = None@classmethod- def for_payload(cls, event: threading.Event, payload, *keys):- """Track the run against the conversation its request names."""+ def for_payload(+ cls,+ event: threading.Event,+ payload,+ *keys,+ external = False,+ ):return cls(event,*keys,thread_id = getattr(payload, "thread_id", None),run_id = getattr(payload, "generation_run_id", None),model = getattr(payload, "model", None),+ external = external,)def __enter__(self):- # Register + consume-pending in one critical section to close the- # TOCTOU race against a concurrent cancel POST.+ # keep registration and pending-cancel consumption atomic with concurrent cancellation.should_cancel = Falsewith _CANCEL_LOCK:for k in self.keys:@@ -9349,7 +9354,7 @@ async def _wait_for_model_switch_idle(elsewhere = model_slots.all_generations()active_others -= min(active_others, len(elsewhere))if cancel_pending:- cancellable = active_generations.count(account_id, elsewhere)+ cancellable = active_generations.count(account_id, elsewhere, include_external = False)active_others -= min(active_others, cancellable)if active_others <= queued_switches:return@@ -17612,41 +17617,26 @@ def _raise_or_cancel_active_generations(action: str,cancel: bool = True,) -> int:- """Gate a model swap on the chats currently generating.-- Every open conversation decodes on the single llama-server this route is- about to replace, so refuse with 409 and name them. force_cancel_active- instead stops them through the same events an explicit Stop uses. Returns- how many were cancelled. The frontend guard is bypassable from a second tab- or curl; this one is not.-- ``cancel = False`` runs the refusal half only. /load calls it that way once- up front, so a non-forced swap still fails fast, and again with cancel just- before teardown: cancelling is destructive and unrecoverable, so it must not- run ahead of preflight checks that can still reject the load (see- _load_model_impl).- """+ """refuse model swaps that would stop managed chats, or cancel them only after preflight."""from core.inference.gpu_arbiter import require_no_foreign_generationsscope = account_access.account_scope()if scope is not None:- # Before the count and the cancel: a foreign generation refuses the swap, so cancelling- # first would end the caller's chats for nothing. Keyed on account_scope(), whose count- # drops while a deactivated account's generation still holds the GPU.+ # account_scope() catches deactivated GPU use; check it before any destructive cancellation.require_no_foreign_generations(scope)elsewhere = model_slots.all_generations()- if not active_generations.count(scope, elsewhere):+ if not active_generations.count(scope, elsewhere, include_external = False):return 0if not force:raise _active_generations_conflict(action,- active_generations.count(scope, elsewhere),- active_generations.active_thread_ids(scope, elsewhere),+ active_generations.count(scope, elsewhere, include_external = False),+ active_generations.active_thread_ids(scope, elsewhere, include_external = False),)if not cancel:- # Refusal-only pass: the caller cancels later, once nothing can still reject the load.+ # the caller cancels only after preflight can no longer reject the load.return 0- cancelled = active_generations.cancel_all(scope, elsewhere)+ cancelled = active_generations.cancel_all(scope, elsewhere, include_external = False)if cancelled:logger.info("model_swap_cancelled_active_generations",@@ -17659,23 +17649,7 @@ _POST_CANCEL_DRAIN_TIMEOUT_S = 5.0async def _cancel_and_drain_for_sidecar_swap(timeout_s: Optional[float] = None) -> None:- """Clear the way for a confirmed sidecar swap, then stop the chats it interrupts.-- The installer gates on the middleware's in-flight count, not on- active_generations, so it also sees requests the cancel cannot stop. Drain- those FIRST, discounting the registered chats (they are what the cancel is- for, so waiting on them would wait out the point of the force). Only then- cancel, and let the survivors unwind. Cancelling first meant an unrelated- counted request -- a /v1/messages/count_tokens, say -- was still there for- the caller's recheck, which then refused an install that had already stopped- every chat for nothing.-- Bounded on both halves: the requests being waited on may never observe a- cancel, and this holds the lifecycle gate and the sidecar reservation inside- ``asyncio.shield``, so an unbounded wait would wedge the process. Expiring in- the first half returns without cancelling, so the caller's recheck refuses- with the chats untouched.- """+ """drain unrelated requests before cancellation so a rejected install cannot stop chats for nothing."""from core.inference.llama_keepwarm import other_inference_request_countbudget = _POST_CANCEL_DRAIN_TIMEOUT_S if timeout_s is None else timeout_s@@ -17686,16 +17660,14 @@ async def _cancel_and_drain_for_sidecar_swap(timeout_s: Optional[float] = None)current_request_counted = False, include_pending = False)if discount_registered:- counted -= min(counted, active_generations.count())+ counted -= min(counted, active_generations.count(None, include_external = False))if counted <= 0:return Trueif time.monotonic() >= deadline:return Falseawait asyncio.sleep(0.02)- # Weighted, not halved, so the total wait under the gate is unchanged. The first drain only- # asks whether unrelated inference is in flight; cutting the second short refused installs- # whose chats had already been stopped for nothing.+ # reserve most of the unchanged budget for chats that have been cancelled and are unwinding.if not await _drain(time.monotonic() + budget / 5, discount_registered = True):return_raise_or_cancel_active_generations(force = True, action = "Installing a new transformers version")@@ -17892,16 +17864,9 @@ async def get_active_generations(current_subject: str = Depends(get_current_subject),model: Optional[str] = None,):- """Conversations currently generating, plus how many can decode at once.-- Lets a model swap name the chats it would interrupt, including runs this tab- cannot see (another tab, or a reload behind a proxy). parallel_slots is the- slot count actually in use, which the VRAM fit may have cut below the- requested --parallel; chats beyond it queue rather than fail.- """+ """return interruptible local generations and the effective decode capacity."""scope = account_access.account_scope()- # ``model``: only the chats an unload of that model stops, the same split its scoped 409 uses.- exclude, only, llama = (), None, None+ exclude, only, llama = set(), None, Noneif model and model_slots.slots:slot = await asyncio.to_thread(model_slots.serving_slot, model, model_slots.visible(), _loaded_satisfies@@ -17909,12 +17874,11 @@ async def get_active_generations(if slot is not None:only, llama = set(slot.generations), slot.llamaelif await asyncio.to_thread(model_slots.in_slot, None, lambda: _loaded_satisfies(model)):- exclude = model_slots.all_generations()+ exclude |= model_slots.all_generations()else:- only = set() # no loaded model by that name: its unload stops nothing- entries = active_generations.snapshot(scope, exclude, only)- # A tracker's model can be a native local path (the legacy stream records active_model_name- # verbatim); redact here, the one place that serialises it.+ only = set() # an absent model cannot interrupt any generation+ entries = active_generations.snapshot(scope, exclude, only, include_external = False)+ # legacy streams record native model paths, so redact them at the serialization boundary.for _entry in entries:if isinstance(_entry.get("model"), str):_entry["model"] = redact_native_paths(_entry["model"])@@ -17926,19 +17890,15 @@ async def get_active_generations(return {"active": entries,"count": len(entries),- "thread_ids": active_generations.active_thread_ids(scope, exclude, only),+ "thread_ids": active_generations.active_thread_ids(+ scope, exclude, only, include_external = False+ ),"parallel_slots": max(1, int(slots)),}def _mlx_runtime_settings_match(backend, request) -> bool:- """Whether a loaded model already runs the request's MLX runtime settings.-- Only the MLX backend records these, so a backend that never stored them- (GGUF, CUDA safetensors) always matches and reuse is unaffected. Both sides- are normalized, otherwise an out-of-domain value would differ from the- stored None on every request and reload forever.- """+ """match normalized MLX settings without forcing reloads for backends that do not record them."""entry = backend.models.get(backend.active_model_name, {}) or {}if "mlx_kv_quant_requested" not in entry:return True@@ -30116,14 +30076,13 @@ async def _proxy_to_external_provider(finally:cancel_event.set()if disconnect_task is not None:- # Joined so its outcome is retrieved. No bounded stop needed: cancel_event is- # set above, so the loop exits even if is_disconnected() eats the cancel (#7617).+ # cancel_event ends the loop if is_disconnected() swallows cancellation (#7617).disconnect_task.cancel()await asyncio.gather(disconnect_task, return_exceptions = True)try:await gen.aclose()except RuntimeError:- pass # suppress httpcore asyncgen cleanup error (Python 3.13 + httpcore 1.0.x)+ pass # httpcore cleanup may raise on Python 3.13 with httpcore 1.0.xawait client.close()def _tracked_stream():@@ -30132,12 +30091,14 @@ async def _proxy_to_external_provider(async def _wrapped():inner = _stream()- with _TrackedCancel.for_payload(cancel_event, payload, *cancel_keys):+ with _TrackedCancel.for_payload(+ cancel_event, payload, *cancel_keys, external = managed is None+ ):try:async for chunk in inner:yield chunkfinally:- # Close it now, not at garbage collection: that releases the upstream request.+ # explicit close releases the upstream request before garbage collection.await inner.aclose()return _wrapped()diff --git a/studio/backend/state/active_generations.py b/studio/backend/state/active_generations.pyindex 495fa866532..a0549a0b7bf 100644--- a/studio/backend/state/active_generations.py+++ b/studio/backend/state/active_generations.py@@ -34,10 +34,7 @@ _FENCED: set[str] = set()class ActiveGeneration:- """Registers one in-flight generation for the duration of the block.-- Each __enter__ mints its own handle, so overlapping uses never clobber.- """+ """uses a unique handle so overlapping registrations cannot clobber each other."""__slots__ = ("thread_id",@@ -46,6 +43,7 @@ class ActiveGeneration:"model","kind","account_id",+ "external","_handle","_borrowed",)@@ -59,8 +57,10 @@ class ActiveGeneration:model: Optional[str] = None,kind: str = "chat",account_id: Optional[str] = None,+ external: bool = False,):self.account_id = account_id or current_account_id()+ self.external = externalself.thread_id = thread_id or Noneself.run_id = run_id or Noneself.cancel_event = cancel_event@@ -71,8 +71,7 @@ class ActiveGeneration:def __enter__(self) -> "ActiveGeneration":with _LOCK:- # A durable supervisor registers before model loading starts and the route later enters its normal- # tracker with that same event and run, so borrow the outer registration.+ # borrow the supervisor's pre-load registration when the route reuses its event and run.if self.run_id:for entry in _ACTIVE.values():if (@@ -97,6 +96,7 @@ class ActiveGeneration:"model": self.model,"kind": self.kind,"account_id": self.account_id,+ "external": self.external,"started_at": time.time(),"event": self.cancel_event,}@@ -119,6 +119,8 @@ def _matching(account_id: Optional[str],exclude: Collection[threading.Event],only: Optional[Collection[threading.Event]] = None,+ *,+ include_external: bool = True,) -> list[dict[str, Any]]:return [e@@ -126,6 +128,7 @@ def _matching(if (account_id is None or e["account_id"] == account_id)and e["event"] not in excludeand (only is None or e["event"] in only)+ and (include_external or not e["external"])]@@ -133,11 +136,12 @@ def snapshot(account_id: Optional[str] = None,exclude: Collection[threading.Event] = (),only: Optional[Collection[threading.Event]] = None,+ *,+ include_external: bool = True,) -> list[dict[str, Any]]:- """In-flight generations, newest last; ``account_id`` None (all) is shutdown/arbiter only.- ``exclude`` leaves out the runs holding those cancel events."""+ """in-flight generations, newest last; ``account_id=None`` is for shutdown or the arbiter."""with _LOCK:- entries = _matching(account_id, exclude, only)+ entries = _matching(account_id, exclude, only, include_external = include_external)entries.sort(key = lambda e: e["started_at"])return [{@@ -157,38 +161,49 @@ def active_thread_ids(account_id: Optional[str] = None,exclude: Collection[threading.Event] = (),only: Optional[Collection[threading.Event]] = None,+ *,+ include_external: bool = True,) -> list[str]:- """Distinct conversation ids with a generation in flight, in start order.-- A first turn that races persistence has no thread id yet: count() sees it,- this cannot name it.- """+ """distinct conversation ids in start order; an unpersisted first turn cannot be named."""seen: list[str] = []- for e in snapshot(account_id, exclude, only):+ for e in snapshot(account_id, exclude, only, include_external = include_external):tid = e["thread_id"]if tid and tid not in seen:seen.append(tid)return seen-def count(account_id: Optional[str] = None, exclude: Collection[threading.Event] = ()) -> int:+def count(+ account_id: Optional[str] = None,+ exclude: Collection[threading.Event] = (),+ *,+ include_external: bool = True,+) -> int:with _LOCK:- if account_id is None and not exclude:+ if account_id is None and not exclude and include_external:return len(_ACTIVE)- return len(_matching(account_id, exclude))+ return len(_matching(account_id, exclude, include_external = include_external))def foreign_count(account_id: str) -> int:- """Generations in flight for OTHER accounts, which a load or unload must not interrupt."""+ """generations for other accounts that load and unload must not interrupt."""with _LOCK:- return sum(1 for e in _ACTIVE.values() if e["account_id"] != account_id)+ return sum(+ 1 for e in _ACTIVE.values() if e["account_id"] != account_id and not e["external"]+ )-def cancel_all(account_id: Optional[str] = None, exclude: Collection[threading.Event] = ()) -> int:- """Signal in-flight generations to stop, returning the count. Request-driven callers must pass- ``account_id``; None means everyone and is for shutdown only."""+def cancel_all(+ account_id: Optional[str] = None,+ exclude: Collection[threading.Event] = (),+ *,+ include_external: bool = True,+) -> int:+ """request callers must pass ``account_id``; None cancels every account during shutdown."""with _LOCK:- events = [e["event"] for e in _matching(account_id, exclude)]+ events = [+ e["event"] for e in _matching(account_id, exclude, include_external = include_external)+ ]for ev in events:try:ev.set()@@ -198,7 +213,7 @@ def cancel_all(account_id: Optional[str] = None, exclude: Collection[threading.Edef cancel_thread(thread_id: str, account_id: Optional[str] = None) -> int:- """Signal ``thread_id``'s generations; thread ids are client-chosen, so scope by account."""+ """thread ids are client-chosen, so cancellation is scoped by account."""if not thread_id:return 0scope = account_id or current_account_id()diff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.pyindex 4b4ba5feffd..10a23edcbf9 100644--- a/studio/backend/tests/test_active_generations.py+++ b/studio/backend/tests/test_active_generations.py@@ -2963,3 +2963,169 @@ def test_responses_stream_stamps_tool_call_deltas(monkeypatch):[row] = [r for r in api_monitor.snapshot() if r["id"] == monitor_id]assert row["ttft_ms"] is not Noneassert row["stop_reason"] == "tool_calls"+++def _run_external_tool_chat(during):+ import asyncio+ import json+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+ from types import SimpleNamespace++ import httpx+ from starlette.requests import Request++ import routes.inference as inf_mod+ from core.inference import external_provider as ep_mod+ from models.inference import ChatCompletionRequest++ release = threading.Event()++ class _SlowProvider(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def log_message(self, *args, **kwargs):+ return++ def do_POST(self):+ self.rfile.read(int(self.headers.get("Content-Length") or 0))+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Connection", "close")+ self.end_headers()+ for word in ("one ", "two "):+ chunk = {"choices": [{"index": 0, "delta": {"content": word}}]}+ self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())+ self.wfile.flush()+ release.wait(10)+ done = {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}+ self.wfile.write(f"data: {json.dumps(done)}\n\ndata: [DONE]\n\n".encode())+ self.wfile.flush()++ server = ThreadingHTTPServer(("127.0.0.1", 0), _SlowProvider)+ threading.Thread(target = server.serve_forever, daemon = True).start()++ async def receive():+ await asyncio.Event().wait()++ payload = ChatCompletionRequest(+ provider_type = "custom",+ provider_base_url = f"http://127.0.0.1:{server.server_address[1]}/v1",+ model = "remote-model",+ messages = [{"role": "user", "content": "hi"}],+ stream = True,+ thread_id = "external-chat",+ cancel_id = "external-run",+ enable_tools = True,+ enabled_tools = ["python"],+ )+ request = Request(+ {+ "type": "http",+ "http_version": "1.1",+ "method": "POST",+ "path": "/api/inference/chat/completions",+ "raw_path": b"/api/inference/chat/completions",+ "root_path": "",+ "scheme": "http",+ "query_string": b"",+ "headers": [(b"content-type", b"application/json"), (b"x-unsloth-events", b"1")],+ "client": ("127.0.0.1", 12345),+ "server": ("127.0.0.1", 8000),+ "app": SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1)),+ },+ receive,+ )++ async def run():+ previous = ep_mod._http_client+ ep_mod._http_client = httpx.AsyncClient()+ try:+ response = await inf_mod._proxy_to_external_provider(payload, request)+ iterator = response.body_iterator+ chunks = []+ while active_generations.count() == 0:+ chunks.append(await asyncio.wait_for(iterator.__anext__(), 10))+ seen = await during(inf_mod, request)+ release.set()+ async for chunk in iterator:+ chunks.append(chunk)+ return seen, "".join(c.decode() if isinstance(c, bytes) else c for c in chunks)+ finally:+ release.set()+ await ep_mod._http_client.aclose()+ ep_mod._http_client = previous+ server.shutdown()+ server.server_close()++ return asyncio.run(run())+++def test_an_external_tool_chat_is_not_stopped_by_a_local_model_swap():+ _route_gate()++ async def during(inf_mod, request):+ [entry] = active_generations.snapshot()+ listed = await inf_mod.get_active_generations(request, "tester")+ inf_mod._raise_or_cancel_active_generations(force = False, action = "Unloading the model")+ cancelled = inf_mod._raise_or_cancel_active_generations(+ force = True, action = "Unloading the model"+ )+ return entry, listed, cancelled++ (entry, listed, cancelled), body = _run_external_tool_chat(during)++ assert entry["thread_id"] == "external-chat"+ assert listed["count"] == 0 and listed["thread_ids"] == []+ assert cancelled == 0+ assert "two" in body and "[DONE]" in body+ assert active_generations.count() == 0+++def test_external_registration_racing_the_swap_gate_stays_excluded(monkeypatch):+ gate = _route_gate()+ event = threading.Event()+ tracker = active_generations.ActiveGeneration(+ event,+ thread_id = "external-chat",+ external = True,+ )+ original_count = active_generations.count+ entered = False++ def count_after_registration(*args, **kwargs):+ nonlocal entered+ if not entered:+ tracker.__enter__()+ entered = True+ return original_count(*args, **kwargs)++ monkeypatch.setattr(active_generations, "count", count_after_registration)+ try:+ assert gate(force = False, action = "Unloading the model") == 0+ assert not event.is_set()+ finally:+ if entered:+ tracker.__exit__(None, None, None)+++def test_an_external_tool_chat_still_stops_on_its_own_stop():+ _route_gate()++ async def during(inf_mod, request):+ return inf_mod._cancel_by_cancel_id_or_stash("external-run")++ stopped, _ = _run_external_tool_chat(during)++ assert stopped == 1+ assert active_generations.count() == 0+++def test_deleting_an_external_tool_chat_still_stops_it():+ _route_gate()++ async def during(inf_mod, request):+ return active_generations.cancel_thread("external-chat")++ stopped, _ = _run_external_tool_chat(during)++ assert stopped == 1