Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop showing a crashed chat model as loaded (#14575)

Nilay8f402f50b44b
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index e42822157d3..a7cd3e9f660 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -2495,6 +2495,30 @@ class InferenceOrchestrator:
return getattr(self, "_managed_engine", None) is None
return False
+ def reap_dead_worker(self) -> bool:
+ proc = self._proc
+ if proc is None or proc.is_alive():
+ return False
+ with self._subprocess_shutdown_lock:
+ if (
+ self._proc is not proc
+ or self._managed_engine is not None
+ or not self.active_model_name
+ or self.loading_models
+ or self._replies_in_flight()
+ ):
+ return False
+ logger.warning(
+ "Inference worker for '%s' exited (exitcode=%s); clearing the loaded model",
+ self.active_model_name,
+ proc.exitcode,
+ )
+ self._retire_stderr_capture()
+ self._shutdown_subprocess_locked(2)
+ self.active_model_name = None
+ self.models.clear()
+ return True
+
def _load_managed_engine(
self,
engine,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index afb768f4de7..57a34e24485 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -9350,8 +9350,26 @@ def get_llama_cpp_backend() -> LlamaCppBackend:
return slot.llama if slot is not None else _llama_cpp_backend
+def _worker_exited(slot: Optional[_ExtraSlot]) -> bool:
+ proc = getattr(model_slots.in_slot(slot, _peek_inference_backend), "_proc", None)
+ return proc is not None and not proc.is_alive()
+
+
+def _reap_dead_workers(slots: list) -> None:
+ for slot in slots:
+ model_slots.in_slot(slot, lambda: reap_dead_worker(_peek_inference_backend()))
+
+
+async def _reap_exited_workers() -> None:
+ # A killed worker stays named until reaped; is_alive() is a non-blocking waitpid.
+ exited = [slot for slot in (None, *model_slots.slots) if _worker_exited(slot)]
+ if exited:
+ await asyncio.to_thread(_reap_dead_workers, exited)
+
+
async def _route_to_extra_slot(requested: Optional[str]) -> Optional[_ExtraSlot]:
"""Route this request to the kept model serving *requested*, if any. The primary wins a tie."""
+ await _reap_exited_workers()
return await model_slots.route(requested, _loaded_satisfies)
@@ -12834,6 +12852,16 @@ def reap_dead_managed_engine(backend) -> None:
account_access.clear_resident("chat")
+def reap_dead_worker(backend) -> None:
+ """Cleared, never respawned: every routed request runs this, so an OOM respawn would loop."""
+ from core.inference.gpu_arbiter import CHAT, holds
+
+ if getattr(backend, "_proc", None) is None:
+ return
+ if backend.reap_dead_worker() and (release_chat_after_kept_models() or not holds(CHAT)):
+ account_access.clear_resident("chat")
+
+
def _preview_same_checkpoint(loaded: str, requested: str) -> bool:
"""True when the resident slot already serves the preview's checkpoint. Exact string
match first: it is the fast path and the only comparison that makes sense for a non-path
@@ -12907,6 +12935,8 @@ async def load_model_for_preview(
detail = "A transformers installation is in progress. Retry when it completes.",
headers = {"Retry-After": "10"},
)
+ # Else a dead worker's name refuses every preview as Studio-owned.
+ await _reap_exited_workers()
loaded = _loaded_slot_ident()
# Borrow the resident model when it already serves this checkpoint, matching
# an equivalent spelling (relative vs the resolved absolute path) via realpath
@@ -19406,6 +19436,8 @@ async def _select_load_slot(request: LoadRequest) -> Optional[_ExtraSlot]:
"""The extra slot already serving the model, or a new one for ``alongside``. None is the primary."""
from core.inference.npu_backend import is_npu_model_path
+ # Before the early return: the NPU and managed-engine guards read the primary's name.
+ await _reap_exited_workers()
# The NPU backend is one per process and replaces the primary, so it never takes a slot. vLLM
# and SGLang reserve a fixed share of the GPU up front, so neither shares it with a kept model.
if is_npu_model_path(request.model_path) or request.engine != "auto":
@@ -20086,6 +20118,7 @@ async def _load_model_impl(
return reused
if not (request.gguf_variant or is_direct_gguf_request):
_inherit_resident_load_in_4bit(backend, request, model_identifier)
+ await asyncio.to_thread(reap_dead_worker, backend)
if (
_same_loaded_identifier(backend.active_model_name, model_identifier)
and (backend.models.get(backend.active_model_name) or {}).get("engine", "auto")
diff --git a/studio/backend/tests/test_killed_inference_worker.py b/studio/backend/tests/test_killed_inference_worker.py
new file mode 100644
index 00000000000..5ea142e01fa
--- /dev/null
+++ b/studio/backend/tests/test_killed_inference_worker.py
@@ -0,0 +1,239 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import asyncio
+import multiprocessing as mp
+import sys
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parents[1]
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+import routes.inference as inference_route # noqa: E402
+from core.inference.orchestrator import InferenceOrchestrator # noqa: E402
+
+MODEL = "unsloth/Qwen3-0.6B"
+
+
+def _orchestrator(killed = True):
+ backend = InferenceOrchestrator()
+ ctx = mp.get_context("spawn")
+ backend._proc = ctx.Process(target = time.sleep, args = (60,), daemon = True)
+ backend._proc.start()
+ backend.active_model_name = MODEL
+ backend.models = {MODEL: {"engine": "auto"}}
+ if killed:
+ backend._proc.kill()
+ backend._proc.join(timeout = 10)
+ return backend
+
+
+def released(monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ inference_route, "release_chat_gpu_claim", lambda: calls.append("release") or True
+ )
+ monkeypatch.setattr(inference_route.account_access, "clear_resident", calls.append)
+ return calls
+
+
+def _status(monkeypatch, backend):
+ monkeypatch.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)
+ monkeypatch.setattr(inference_route, "_probe_llama_cpp_status", lambda _b: (False, {}))
+ monkeypatch.setattr(inference_route, "_running_load_attempt", None)
+ monkeypatch.setattr(inference_route, "_pending_load_attempts", {})
+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)
+ return asyncio.run(inference_route.get_status(current_subject = "test"))
+
+
+def test_status_stops_reporting_a_model_whose_worker_was_killed(monkeypatch, released):
+ backend = _orchestrator()
+
+ response = _status(monkeypatch, backend)
+
+ assert response.active_model is None
+ assert response.loaded == []
+ assert backend.active_model_name is None and backend._proc is None
+ assert released == ["release", "chat"]
+
+
+def test_an_api_request_clears_a_model_whose_worker_was_killed(monkeypatch, released):
+ from starlette.requests import Request
+
+ backend = _orchestrator()
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)
+ request = Request(
+ {"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": []}
+ )
+
+ asyncio.run(inference_route._maybe_auto_switch_model(None, request, "user"))
+
+ assert backend.active_model_name is None and backend._proc is None
+
+
[email protected]("owner", ["user", "another-account"])
+def test_a_request_is_not_routed_to_a_kept_model_whose_worker_was_killed(
+ monkeypatch, released, owner
+):
+ from core.inference import model_slots
+
+ # A managed installation hides another account's slot from routing, not from the reap.
+ monkeypatch.setattr(model_slots.account_access, "managed_account", lambda: True)
+ monkeypatch.setattr(model_slots, "current_account_id", lambda: "user")
+ kept = _orchestrator()
+ llama = SimpleNamespace(is_active = False, is_loaded = False, model_identifier = None)
+ slot = model_slots.ExtraSlot(llama = llama, orchestrator = kept, account = owner)
+ monkeypatch.setattr(model_slots, "slots", [slot])
+
+ routed = asyncio.run(inference_route._route_to_extra_slot(MODEL))
+
+ assert routed is None
+ assert kept.active_model_name is None and kept._proc is None
+
+
+def test_reaping_the_last_gpu_kept_model_beside_a_cpu_primary_releases_chat(monkeypatch):
+ from core.inference import gpu_arbiter, model_slots
+
+ for name, value in (
+ ("_owner", gpu_arbiter.CHAT),
+ ("_owner_account", None),
+ ("_prior_account", None),
+ ("_residents", {gpu_arbiter.CHAT: None}),
+ ("_accounts", {}),
+ ("_media_needs", {}),
+ ):
+ monkeypatch.setattr(gpu_arbiter, name, value)
+ cleared = []
+ monkeypatch.setattr(inference_route.account_access, "clear_resident", cleared.append)
+ cpu_primary = SimpleNamespace(is_active = True, is_loaded = True, holds_no_vram = True)
+ monkeypatch.setattr(inference_route, "_llama_cpp_backend", cpu_primary)
+ kept = _orchestrator()
+ idle_llama = SimpleNamespace(is_active = False, is_loaded = False)
+ slot = model_slots.ExtraSlot(llama = idle_llama, orchestrator = kept, account = "user")
+ monkeypatch.setattr(model_slots, "slots", [slot])
+
+ model_slots.in_slot(slot, lambda: inference_route.reap_dead_worker(kept))
+
+ assert not gpu_arbiter.holds(gpu_arbiter.CHAT)
+ assert cleared == ["chat"]
+
+
[email protected]("state", ["alive", "loading", "replying"])
+def test_status_keeps_a_model_whose_worker_may_still_answer(monkeypatch, released, state):
+ backend = _orchestrator(killed = state != "alive")
+ if state == "loading":
+ backend.loading_models = {"org/incoming-model"}
+ if state == "replying":
+ backend._direct_mailboxes["request"] = object()
+ try:
+ response = _status(monkeypatch, backend)
+ finally:
+ backend._proc.kill()
+ backend._proc.join(timeout = 10)
+
+ assert response.active_model == MODEL
+ assert released == []
+
+
+def test_loading_the_same_model_again_restarts_a_killed_worker(monkeypatch, released):
+ from core.inference import gpu_arbiter
+ from hub.services.models import account_access
+ from models.inference import LoadRequest
+
+ # The load takes the real CHAT claim before its stubbed loader raises.
+ for name, empty in (
+ ("_owner", None),
+ ("_owner_account", None),
+ ("_prior_account", None),
+ ("_residents", {}),
+ ("_accounts", {}),
+ ("_media_needs", {}),
+ ):
+ monkeypatch.setattr(gpu_arbiter, name, empty)
+ backend = _orchestrator()
+ calls = []
+
+ def _record(step):
+ def _stop(*args, **kwargs):
+ calls.append(step)
+ raise RuntimeError(step)
+
+ return _stop
+
+ backend.set_parallel_slots = _record("reuse")
+ backend.load_model = _record("load")
+ llama = SimpleNamespace(is_loaded = False, is_active = False, model_identifier = None)
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama)
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)
+ monkeypatch.setattr(
+ inference_route,
+ "_resolve_model_identifier_for_request",
+ lambda *a, **k: (MODEL, MODEL, False),
+ )
+ monkeypatch.setattr(account_access, "require_model_access", lambda *a, **k: None)
+ fastapi_request = SimpleNamespace(
+ app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1))
+ )
+ with pytest.raises(Exception):
+ asyncio.run(
+ inference_route._load_model_impl(
+ LoadRequest(model_path = MODEL), fastapi_request, current_subject = "user"
+ )
+ )
+
+ assert calls[:1] == ["load"]
+
+
[email protected]("engine", ["vllm", "npu"])
+def test_a_managed_or_npu_load_reaps_a_killed_primary_first(monkeypatch, released, engine):
+ from core.inference import npu_backend
+ from models.inference import LoadRequest
+
+ backend = _orchestrator()
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)
+ monkeypatch.setattr(npu_backend, "is_npu_model_path", lambda _path: engine == "npu")
+ request = LoadRequest(model_path = "org/other", engine = "auto" if engine == "npu" else engine)
+
+ assert asyncio.run(inference_route._select_load_slot(request)) is None
+ assert backend.active_model_name is None and backend._proc is None
+
+
+def test_a_preview_of_another_checkpoint_is_not_refused_by_a_killed_worker(monkeypatch, released):
+ from fastapi import HTTPException
+
+ from core.inference import llama_keepwarm
+ from models.inference import LoadRequest
+
+ backend = _orchestrator()
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)
+ monkeypatch.setattr(
+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
+ )
+ monkeypatch.setattr(inference_route, "_resident_npu_model", lambda: None)
+ monkeypatch.setattr(inference_route.account_access, "managed_account", lambda: False)
+ monkeypatch.setattr(inference_route.account_access, "require_idle_other_accounts", lambda: None)
+ # Stop at the busy check right after the ownership one: reaching it means the dead model did not refuse.
+ monkeypatch.setattr(llama_keepwarm, "other_admitted_inference_count", lambda: 1)
+ request = SimpleNamespace(scope = {"type": "http"})
+
+ with pytest.raises(HTTPException) as refused:
+ asyncio.run(
+ inference_route.load_model_for_preview(
+ LoadRequest(model_path = "org/other"), request, current_subject = "user"
+ )
+ )
+
+ assert "busy" in refused.value.detail
+ assert backend.active_model_name is None and backend._proc is None
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index 2f3a81d9a65..1ab15af8497 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -1154,6 +1154,8 @@ def test_scoped_cancel_holds_load_ownership_until_backend_cancel_finishes(monkey
return _Backend()
monkeypatch.setattr(inference_route, "get_inference_backend", _get_backend)
+ # Routing peeks for a dead worker; only the cancel may reach the blocking getter.
+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: None)
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: _Llama())
monkeypatch.setattr(inference_route, "_raise_if_sidecar_swap_in_progress", lambda: None)
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 55bfaa6721b..4a235e9e3e8 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -247,7 +247,10 @@ import {
beginServerModelWait,
statusPollSignal,
} from "../lib/server-model-wait";
-import { syncModelCapabilities } from "../hooks/use-chat-model-runtime";
+import {
+ syncInferenceStatusToStore,
+ syncModelCapabilities,
+} from "../hooks/use-chat-model-runtime";
import {
clampReasoningEffortToLevels,
externalMaxOutputTokensNeedsConnectionCap,
@@ -9189,6 +9192,10 @@ export function createOpenAIStreamAdapter(
description: msg || "Unknown error",
});
}
+ // Only a status read shows a killed worker's model is gone; quiet, the failure has its toast.
+ if (!isExternalRequest) {
+ void syncInferenceStatusToStore({ includeLoras: false, quiet: true });
+ }
}
// An explicit Stop is an abort too, but it must persist its reason instead of
// leaving the last streamed yield's label standing: the replacement yield
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 4615d655b93..7743b1b50f1 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -688,12 +688,13 @@ function unloadKeptModel(keptId: string): Promise<boolean> {
);
}
-async function syncInferenceStatusToStore(options?: {
+export async function syncInferenceStatusToStore(options?: {
signal?: AbortSignal;
includeLoras?: boolean;
preserveIdleUnloaded?: boolean;
/** A TTS load owns the chat slot even though Chat did not start the load. */
externalChatSlotLoad?: boolean;
+ quiet?: boolean;
}): Promise<void> {
const signal = options?.signal;
const downWhenIssued = isBackendDownForDesktopUpdate();
@@ -834,7 +835,7 @@ async function syncInferenceStatusToStore(options?: {
if (wasResident) {
toast.info(`${wasResident} is no longer loaded`, {
description:
- "The server released it, which loading an image, video or audio model does. Pick it again to keep chatting.",
+ "The server released it, as loading an image, video or audio model does, or its process stopped. Pick it again to keep chatting.",
});
}
// Drop the pick too, as a server-side unload does: the name alone reads as "this is my model"
@@ -846,6 +847,7 @@ async function syncInferenceStatusToStore(options?: {
// A superseded refresh reports nothing, or a stale failure would raise a toast about a read
// whose answer would have been discarded. The LoRA inventory settles from its own request.
if (signal?.aborted || superseded()) return;
+ if (options?.quiet) return;
// The update screen already reports the backend it stopped.
if (isSilencedDesktopUpdateFailure(error, downWhenIssued)) return;
const message =
diff --git a/studio/frontend/tests/chat-failed-reply-rereads-status.test.ts b/studio/frontend/tests/chat-failed-reply-rereads-status.test.ts
new file mode 100644
index 00000000000..a86605413c8
--- /dev/null
+++ b/studio/frontend/tests/chat-failed-reply-rereads-status.test.ts
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const ADAPTER = readSrc("features/chat/api/chat-adapter.ts");
+const RUNTIME = readSrc("features/chat/hooks/use-chat-model-runtime.ts");
+
+const FAILURE = ADAPTER.slice(
+ ADAPTER.indexOf('toast.error("Generation failed"'),
+ ADAPTER.indexOf("// An explicit Stop is an abort too"),
+);
+const SYNC_FAILURE = RUNTIME.slice(
+ RUNTIME.indexOf("export async function syncInferenceStatusToStore("),
+ RUNTIME.indexOf('toast.error("Failed to refresh models"'),
+);
+
+test("a failed local reply quietly re-reads what the server still holds", () => {
+ assert.match(
+ FAILURE,
+ /if \(!isExternalRequest\) \{\s*void syncInferenceStatusToStore\(\{[^}]*quiet: true/,
+ );
+ // An idle-unload armed refresh would read the crash reap's empty status as idle-unloaded and keep the dead pick.
+ assert.doesNotMatch(FAILURE, /preserveIdleUnloaded/);
+ const catchBlock = SYNC_FAILURE.slice(SYNC_FAILURE.lastIndexOf("} catch (error) {"));
+ assert.match(catchBlock, /if \(options\?\.quiet\) return;[\s\S]*setModelsError\(message\)/);
+});