Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop showing a crashed chat model as loaded (#14575)
Nilay8f402f50b44b
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex e42822157d3..a7cd3e9f660 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -2495,6 +2495,30 @@ class InferenceOrchestrator:return getattr(self, "_managed_engine", None) is Nonereturn False+ def reap_dead_worker(self) -> bool:+ proc = self._proc+ if proc is None or proc.is_alive():+ return False+ with self._subprocess_shutdown_lock:+ if (+ self._proc is not proc+ or self._managed_engine is not None+ or not self.active_model_name+ or self.loading_models+ or self._replies_in_flight()+ ):+ return False+ logger.warning(+ "Inference worker for '%s' exited (exitcode=%s); clearing the loaded model",+ self.active_model_name,+ proc.exitcode,+ )+ self._retire_stderr_capture()+ self._shutdown_subprocess_locked(2)+ self.active_model_name = None+ self.models.clear()+ return True+def _load_managed_engine(self,engine,diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex afb768f4de7..57a34e24485 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -9350,8 +9350,26 @@ def get_llama_cpp_backend() -> LlamaCppBackend:return slot.llama if slot is not None else _llama_cpp_backend+def _worker_exited(slot: Optional[_ExtraSlot]) -> bool:+ proc = getattr(model_slots.in_slot(slot, _peek_inference_backend), "_proc", None)+ return proc is not None and not proc.is_alive()+++def _reap_dead_workers(slots: list) -> None:+ for slot in slots:+ model_slots.in_slot(slot, lambda: reap_dead_worker(_peek_inference_backend()))+++async def _reap_exited_workers() -> None:+ # A killed worker stays named until reaped; is_alive() is a non-blocking waitpid.+ exited = [slot for slot in (None, *model_slots.slots) if _worker_exited(slot)]+ if exited:+ await asyncio.to_thread(_reap_dead_workers, exited)++async def _route_to_extra_slot(requested: Optional[str]) -> Optional[_ExtraSlot]:"""Route this request to the kept model serving *requested*, if any. The primary wins a tie."""+ await _reap_exited_workers()return await model_slots.route(requested, _loaded_satisfies)@@ -12834,6 +12852,16 @@ def reap_dead_managed_engine(backend) -> None:account_access.clear_resident("chat")+def reap_dead_worker(backend) -> None:+ """Cleared, never respawned: every routed request runs this, so an OOM respawn would loop."""+ from core.inference.gpu_arbiter import CHAT, holds++ if getattr(backend, "_proc", None) is None:+ return+ if backend.reap_dead_worker() and (release_chat_after_kept_models() or not holds(CHAT)):+ account_access.clear_resident("chat")++def _preview_same_checkpoint(loaded: str, requested: str) -> bool:"""True when the resident slot already serves the preview's checkpoint. Exact stringmatch first: it is the fast path and the only comparison that makes sense for a non-path@@ -12907,6 +12935,8 @@ async def load_model_for_preview(detail = "A transformers installation is in progress. Retry when it completes.",headers = {"Retry-After": "10"},)+ # Else a dead worker's name refuses every preview as Studio-owned.+ await _reap_exited_workers()loaded = _loaded_slot_ident()# Borrow the resident model when it already serves this checkpoint, matching# an equivalent spelling (relative vs the resolved absolute path) via realpath@@ -19406,6 +19436,8 @@ async def _select_load_slot(request: LoadRequest) -> Optional[_ExtraSlot]:"""The extra slot already serving the model, or a new one for ``alongside``. None is the primary."""from core.inference.npu_backend import is_npu_model_path+ # Before the early return: the NPU and managed-engine guards read the primary's name.+ await _reap_exited_workers()# The NPU backend is one per process and replaces the primary, so it never takes a slot. vLLM# and SGLang reserve a fixed share of the GPU up front, so neither shares it with a kept model.if is_npu_model_path(request.model_path) or request.engine != "auto":@@ -20086,6 +20118,7 @@ async def _load_model_impl(return reusedif not (request.gguf_variant or is_direct_gguf_request):_inherit_resident_load_in_4bit(backend, request, model_identifier)+ await asyncio.to_thread(reap_dead_worker, backend)if (_same_loaded_identifier(backend.active_model_name, model_identifier)and (backend.models.get(backend.active_model_name) or {}).get("engine", "auto")diff --git a/studio/backend/tests/test_killed_inference_worker.py b/studio/backend/tests/test_killed_inference_worker.pynew file mode 100644index 00000000000..5ea142e01fa--- /dev/null+++ b/studio/backend/tests/test_killed_inference_worker.py@@ -0,0 +1,239 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import asyncio+import multiprocessing as mp+import sys+import time+from pathlib import Path+from types import SimpleNamespace++import pytest++_BACKEND = Path(__file__).resolve().parents[1]+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++import routes.inference as inference_route # noqa: E402+from core.inference.orchestrator import InferenceOrchestrator # noqa: E402++MODEL = "unsloth/Qwen3-0.6B"+++def _orchestrator(killed = True):+ backend = InferenceOrchestrator()+ ctx = mp.get_context("spawn")+ backend._proc = ctx.Process(target = time.sleep, args = (60,), daemon = True)+ backend._proc.start()+ backend.active_model_name = MODEL+ backend.models = {MODEL: {"engine": "auto"}}+ if killed:+ backend._proc.kill()+ backend._proc.join(timeout = 10)+ return backend+++def released(monkeypatch):+ calls = []+ monkeypatch.setattr(+ inference_route, "release_chat_gpu_claim", lambda: calls.append("release") or True+ )+ monkeypatch.setattr(inference_route.account_access, "clear_resident", calls.append)+ return calls+++def _status(monkeypatch, backend):+ monkeypatch.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)+ monkeypatch.setattr(inference_route, "_probe_llama_cpp_status", lambda _b: (False, {}))+ monkeypatch.setattr(inference_route, "_running_load_attempt", None)+ monkeypatch.setattr(inference_route, "_pending_load_attempts", {})+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)+ return asyncio.run(inference_route.get_status(current_subject = "test"))+++def test_status_stops_reporting_a_model_whose_worker_was_killed(monkeypatch, released):+ backend = _orchestrator()++ response = _status(monkeypatch, backend)++ assert response.active_model is None+ assert response.loaded == []+ assert backend.active_model_name is None and backend._proc is None+ assert released == ["release", "chat"]+++def test_an_api_request_clears_a_model_whose_worker_was_killed(monkeypatch, released):+ from starlette.requests import Request++ backend = _orchestrator()+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)+ request = Request(+ {"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": []}+ )++ asyncio.run(inference_route._maybe_auto_switch_model(None, request, "user"))++ assert backend.active_model_name is None and backend._proc is None++[email protected]("owner", ["user", "another-account"])+def test_a_request_is_not_routed_to_a_kept_model_whose_worker_was_killed(+ monkeypatch, released, owner+):+ from core.inference import model_slots++ # A managed installation hides another account's slot from routing, not from the reap.+ monkeypatch.setattr(model_slots.account_access, "managed_account", lambda: True)+ monkeypatch.setattr(model_slots, "current_account_id", lambda: "user")+ kept = _orchestrator()+ llama = SimpleNamespace(is_active = False, is_loaded = False, model_identifier = None)+ slot = model_slots.ExtraSlot(llama = llama, orchestrator = kept, account = owner)+ monkeypatch.setattr(model_slots, "slots", [slot])++ routed = asyncio.run(inference_route._route_to_extra_slot(MODEL))++ assert routed is None+ assert kept.active_model_name is None and kept._proc is None+++def test_reaping_the_last_gpu_kept_model_beside_a_cpu_primary_releases_chat(monkeypatch):+ from core.inference import gpu_arbiter, model_slots++ for name, value in (+ ("_owner", gpu_arbiter.CHAT),+ ("_owner_account", None),+ ("_prior_account", None),+ ("_residents", {gpu_arbiter.CHAT: None}),+ ("_accounts", {}),+ ("_media_needs", {}),+ ):+ monkeypatch.setattr(gpu_arbiter, name, value)+ cleared = []+ monkeypatch.setattr(inference_route.account_access, "clear_resident", cleared.append)+ cpu_primary = SimpleNamespace(is_active = True, is_loaded = True, holds_no_vram = True)+ monkeypatch.setattr(inference_route, "_llama_cpp_backend", cpu_primary)+ kept = _orchestrator()+ idle_llama = SimpleNamespace(is_active = False, is_loaded = False)+ slot = model_slots.ExtraSlot(llama = idle_llama, orchestrator = kept, account = "user")+ monkeypatch.setattr(model_slots, "slots", [slot])++ model_slots.in_slot(slot, lambda: inference_route.reap_dead_worker(kept))++ assert not gpu_arbiter.holds(gpu_arbiter.CHAT)+ assert cleared == ["chat"]++[email protected]("state", ["alive", "loading", "replying"])+def test_status_keeps_a_model_whose_worker_may_still_answer(monkeypatch, released, state):+ backend = _orchestrator(killed = state != "alive")+ if state == "loading":+ backend.loading_models = {"org/incoming-model"}+ if state == "replying":+ backend._direct_mailboxes["request"] = object()+ try:+ response = _status(monkeypatch, backend)+ finally:+ backend._proc.kill()+ backend._proc.join(timeout = 10)++ assert response.active_model == MODEL+ assert released == []+++def test_loading_the_same_model_again_restarts_a_killed_worker(monkeypatch, released):+ from core.inference import gpu_arbiter+ from hub.services.models import account_access+ from models.inference import LoadRequest++ # The load takes the real CHAT claim before its stubbed loader raises.+ for name, empty in (+ ("_owner", None),+ ("_owner_account", None),+ ("_prior_account", None),+ ("_residents", {}),+ ("_accounts", {}),+ ("_media_needs", {}),+ ):+ monkeypatch.setattr(gpu_arbiter, name, empty)+ backend = _orchestrator()+ calls = []++ def _record(step):+ def _stop(*args, **kwargs):+ calls.append(step)+ raise RuntimeError(step)++ return _stop++ backend.set_parallel_slots = _record("reuse")+ backend.load_model = _record("load")+ llama = SimpleNamespace(is_loaded = False, is_active = False, model_identifier = None)+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama)+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)+ monkeypatch.setattr(+ inference_route,+ "_resolve_model_identifier_for_request",+ lambda *a, **k: (MODEL, MODEL, False),+ )+ monkeypatch.setattr(account_access, "require_model_access", lambda *a, **k: None)+ fastapi_request = SimpleNamespace(+ app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1))+ )+ with pytest.raises(Exception):+ asyncio.run(+ inference_route._load_model_impl(+ LoadRequest(model_path = MODEL), fastapi_request, current_subject = "user"+ )+ )++ assert calls[:1] == ["load"]++[email protected]("engine", ["vllm", "npu"])+def test_a_managed_or_npu_load_reaps_a_killed_primary_first(monkeypatch, released, engine):+ from core.inference import npu_backend+ from models.inference import LoadRequest++ backend = _orchestrator()+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)+ monkeypatch.setattr(npu_backend, "is_npu_model_path", lambda _path: engine == "npu")+ request = LoadRequest(model_path = "org/other", engine = "auto" if engine == "npu" else engine)++ assert asyncio.run(inference_route._select_load_slot(request)) is None+ assert backend.active_model_name is None and backend._proc is None+++def test_a_preview_of_another_checkpoint_is_not_refused_by_a_killed_worker(monkeypatch, released):+ from fastapi import HTTPException++ from core.inference import llama_keepwarm+ from models.inference import LoadRequest++ backend = _orchestrator()+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: backend)+ monkeypatch.setattr(+ inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)+ )+ monkeypatch.setattr(inference_route, "_resident_npu_model", lambda: None)+ monkeypatch.setattr(inference_route.account_access, "managed_account", lambda: False)+ monkeypatch.setattr(inference_route.account_access, "require_idle_other_accounts", lambda: None)+ # Stop at the busy check right after the ownership one: reaching it means the dead model did not refuse.+ monkeypatch.setattr(llama_keepwarm, "other_admitted_inference_count", lambda: 1)+ request = SimpleNamespace(scope = {"type": "http"})++ with pytest.raises(HTTPException) as refused:+ asyncio.run(+ inference_route.load_model_for_preview(+ LoadRequest(model_path = "org/other"), request, current_subject = "user"+ )+ )++ assert "busy" in refused.value.detail+ assert backend.active_model_name is None and backend._proc is Nonediff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.pyindex 2f3a81d9a65..1ab15af8497 100644--- a/studio/backend/tests/test_orchestrator_unload_cancel.py+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py@@ -1154,6 +1154,8 @@ def test_scoped_cancel_holds_load_ownership_until_backend_cancel_finishes(monkeyreturn _Backend()monkeypatch.setattr(inference_route, "get_inference_backend", _get_backend)+ # Routing peeks for a dead worker; only the cancel may reach the blocking getter.+ monkeypatch.setattr(inference_route, "_peek_inference_backend", lambda: None)monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: _Llama())monkeypatch.setattr(inference_route, "_raise_if_sidecar_swap_in_progress", lambda: None)diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 55bfaa6721b..4a235e9e3e8 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -247,7 +247,10 @@ import {beginServerModelWait,statusPollSignal,} from "../lib/server-model-wait";-import { syncModelCapabilities } from "../hooks/use-chat-model-runtime";+import {+ syncInferenceStatusToStore,+ syncModelCapabilities,+} from "../hooks/use-chat-model-runtime";import {clampReasoningEffortToLevels,externalMaxOutputTokensNeedsConnectionCap,@@ -9189,6 +9192,10 @@ export function createOpenAIStreamAdapter(description: msg || "Unknown error",});}+ // Only a status read shows a killed worker's model is gone; quiet, the failure has its toast.+ if (!isExternalRequest) {+ void syncInferenceStatusToStore({ includeLoras: false, quiet: true });+ }}// An explicit Stop is an abort too, but it must persist its reason instead of// leaving the last streamed yield's label standing: the replacement yielddiff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex 4615d655b93..7743b1b50f1 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -688,12 +688,13 @@ function unloadKeptModel(keptId: string): Promise<boolean> {);}-async function syncInferenceStatusToStore(options?: {+export async function syncInferenceStatusToStore(options?: {signal?: AbortSignal;includeLoras?: boolean;preserveIdleUnloaded?: boolean;/** A TTS load owns the chat slot even though Chat did not start the load. */externalChatSlotLoad?: boolean;+ quiet?: boolean;}): Promise<void> {const signal = options?.signal;const downWhenIssued = isBackendDownForDesktopUpdate();@@ -834,7 +835,7 @@ async function syncInferenceStatusToStore(options?: {if (wasResident) {toast.info(`${wasResident} is no longer loaded`, {description:- "The server released it, which loading an image, video or audio model does. Pick it again to keep chatting.",+ "The server released it, as loading an image, video or audio model does, or its process stopped. Pick it again to keep chatting.",});}// Drop the pick too, as a server-side unload does: the name alone reads as "this is my model"@@ -846,6 +847,7 @@ async function syncInferenceStatusToStore(options?: {// A superseded refresh reports nothing, or a stale failure would raise a toast about a read// whose answer would have been discarded. The LoRA inventory settles from its own request.if (signal?.aborted || superseded()) return;+ if (options?.quiet) return;// The update screen already reports the backend it stopped.if (isSilencedDesktopUpdateFailure(error, downWhenIssued)) return;const message =diff --git a/studio/frontend/tests/chat-failed-reply-rereads-status.test.ts b/studio/frontend/tests/chat-failed-reply-rereads-status.test.tsnew file mode 100644index 00000000000..a86605413c8--- /dev/null+++ b/studio/frontend/tests/chat-failed-reply-rereads-status.test.ts@@ -0,0 +1,30 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { readSrc } from "./helpers/kit.ts";++const ADAPTER = readSrc("features/chat/api/chat-adapter.ts");+const RUNTIME = readSrc("features/chat/hooks/use-chat-model-runtime.ts");++const FAILURE = ADAPTER.slice(+ ADAPTER.indexOf('toast.error("Generation failed"'),+ ADAPTER.indexOf("// An explicit Stop is an abort too"),+);+const SYNC_FAILURE = RUNTIME.slice(+ RUNTIME.indexOf("export async function syncInferenceStatusToStore("),+ RUNTIME.indexOf('toast.error("Failed to refresh models"'),+);++test("a failed local reply quietly re-reads what the server still holds", () => {+ assert.match(+ FAILURE,+ /if \(!isExternalRequest\) \{\s*void syncInferenceStatusToStore\(\{[^}]*quiet: true/,+ );+ // An idle-unload armed refresh would read the crash reap's empty status as idle-unloaded and keep the dead pick.+ assert.doesNotMatch(FAILURE, /preserveIdleUnloaded/);+ const catchBlock = SYNC_FAILURE.slice(SYNC_FAILURE.lastIndexOf("} catch (error) {"));+ assert.match(catchBlock, /if \(options\?\.quiet\) return;[\s\S]*setModelsError\(message\)/);+});