Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: reload without MTP when llama-server dies mid-generation on any MTP launch (#13316)
Daniel Hanb5efe363b421
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 4046d013f3e..4d4b835844a 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -452,6 +452,7 @@ from utils.subprocess_compat import ()from utils.process_lifetime import child_popen_kwargs as _child_popen_kwargsfrom utils.process_lifetime import is_signalable_pid as _is_signalable_pid+from utils.process_lifetime import spawn_on_lifetime_thread as _spawn_on_lifetime_threadfrom core.inference.tool_call_parser import (BUDGET_EXHAUSTED_NUDGE,MAX_ACT_REPROMPTS as _MAX_REPROMPTS,@@ -8253,12 +8254,12 @@ class LlamaCppBackend:self._last_load_intent: Optional[GgufLoadIntent] = Noneself._mtp_runtime_fallback_lock = threading.Lock()self._mtp_runtime_fallback_in_progress = False- # Background watchdog so an MTP+tensor crash recovers even when no request+ # Background watchdog so an MTP crash recovers even when no request# observes it (direct proxy endpoints, or nothing in flight).self._mtp_watchdog_thread: Optional[threading.Thread] = Noneself._mtp_watchdog_stop = threading.Event()- # True when the launch actually runs MTP+tensor (Unsloth- or user/env-driven);- # gates the probe, watchdog, and recovery so pass-through MTP is covered.+ # True when the launch actually runs MTP (Unsloth- or user/env-driven);+ # gates the watchdog and recovery so pass-through MTP is covered.self._mtp_runtime_fallback_active = Falseself._stdout_lines: list[str] = []self._stdout_thread: Optional[threading.Thread] = None@@ -24175,16 +24176,19 @@ class LlamaCppBackend:self._close_attempt_log()self._health_wait_cancelled = Truereturn False- _spawned = subprocess.Popen(- cmd,- stdout = subprocess.PIPE,- stderr = subprocess.STDOUT,- text = True,- encoding = "utf-8",- errors = "replace",- env = env,- **_windows_hidden_subprocess_kwargs(),- **_child_popen_kwargs(),+ # PDEATHSIG follows the forking thread; the MTP-crash reload's thread exits.+ _spawned = _spawn_on_lifetime_thread(+ lambda: subprocess.Popen(+ cmd,+ stdout = subprocess.PIPE,+ stderr = subprocess.STDOUT,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ env = env,+ **_windows_hidden_subprocess_kwargs(),+ **_child_popen_kwargs(),+ ))self._process = _spawned_invalidate_gpu_memory("llama-server spawned")@@ -31140,17 +31144,19 @@ class LlamaCppBackend:self._close_attempt_log()self._health_wait_cancelled = Truereturn False- _spawned = subprocess.Popen(- run_cmd,- stdout = subprocess.PIPE,- stderr = subprocess.STDOUT,- text = True,- encoding = "utf-8",- errors = "replace",- env = env,- cwd = _spawn_cwd,- **_windows_hidden_subprocess_kwargs(),- **_child_popen_kwargs(),+ _spawned = _spawn_on_lifetime_thread(+ lambda: subprocess.Popen(+ run_cmd,+ stdout = subprocess.PIPE,+ stderr = subprocess.STDOUT,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ env = env,+ cwd = _spawn_cwd,+ **_windows_hidden_subprocess_kwargs(),+ **_child_popen_kwargs(),+ ))self._process = _spawned_invalidate_gpu_memory("llama-server spawned")@@ -32430,11 +32436,9 @@ class LlamaCppBackend:_spec_requested_dflash = any("draft-dflash" in str(t).lower() for t in spec_flags) or _extra_args_requests_dflash(extra_args, env = _launch_spec_env)- # Is the launched server actually running MTP+tensor? Gates the- # probe/watchdog/recovery; cleared if the MTP-drop fallback wins.- _mtp_active_for_launched_server = bool(- self._tensor_parallel and _spec_requested_mtp- )+ # Any MTP launch, not only tensor: a respawn would replay a drafter that+ # aborts every decode (#13253). Cleared if the MTP-drop fallback wins.+ _mtp_active_for_launched_server = bool(_spec_requested_mtp)# MTP can pass /health then crash the flash-attn kernel on the# first decode under tensor; probe one generation so the fallback# catches that too. Tensor-only, so ordinary MTP stays probe-free.@@ -33019,7 +33023,7 @@ class LlamaCppBackend:self._requested_spec_draft_cache_type = intent.spec_draft_cache_typeself._requested_ctx_checkpoints = intent.ctx_checkpointsself._requested_cache_ram = intent.cache_ram- # Commit the known-good snapshot + whether MTP+tensor is live, then+ # Commit the known-good snapshot + whether MTP is live, then# watch this load for a mid-generation crash. The verified-cache hint is# dropped: a respawn replays this snapshot arbitrarily later, so recovery# re-resolves the file instead of trusting a path this request verified.@@ -35906,12 +35910,12 @@ class LlamaCppBackend:def _maybe_recover_from_mtp_crash(self, exc: Optional[BaseException] = None) -> bool:"""Schedule one background reload without MTP after a mid-generation death.- MTP+tensor can crash the flash-attn kernel on a later request, after- load_model returned, past the load-time fallback and decode probe. Not a- persistent ban: a fresh load re-tries MTP. Returns True if scheduled.+ An MTP launch can pass /health and the load-time fallback, then abort on+ a later decode (MTP+tensor flash-attn, Gemma 4 on Apple Silicon #13253).+ Not a persistent ban: a fresh load re-tries MTP. Returns True if scheduled."""- # Cheap async-safe gate: only our live MTP+tensor launch, not cancelled,- # with a snapshot to replay.+ # Cheap async-safe gate: only our live MTP launch, not cancelled, with a+ # snapshot to replay.if self._cancel_event.is_set():return Falseif not self._mtp_runtime_fallback_active:@@ -35922,6 +35926,9 @@ class LlamaCppBackend:proc = self._processif snapshot is None or proc is None:return False+ # Still serving: a transient error, so leave the caller its normal retry.+ if proc.poll() is None and self._server_socket_is_open():+ return False# Single-flight: the first failure claims the reload.with self._mtp_runtime_fallback_lock:if self._mtp_runtime_fallback_in_progress:@@ -35939,8 +35946,8 @@ class LlamaCppBackend:logger.debug("Generation error but llama-server is alive; keeping MTP.")returnlogger.warning(- "llama-server exited mid-generation with MTP under tensor "- "parallelism (%s); reloading without speculative decoding.",+ "llama-server exited mid-generation with MTP speculative "+ "decoding (%s); reloading without speculative decoding.",type(exc).__name__ if exc is not None else "server exited",)# Re-check under the load lock (RLock allows the nested@@ -36001,7 +36008,12 @@ class LlamaCppBackend:)# Restore the requested mode + reason load_model("off") cleared,# so /status shows the user's mode + note (like the startup fallback).- self._requested_spec_mode = _canonicalize_spec_mode(requested_mode)+ # None when the extras own --spec-type, as the duplicate-load match expects.+ self._requested_spec_mode = (+ None+ if _extra_args_set_spec_type(_ea)+ else _canonicalize_spec_mode(requested_mode)+ )self._spec_fallback_reason = "runtime_error"if undo_reload:logger.info(@@ -36009,7 +36021,7 @@ class LlamaCppBackend:)self.unload_model()return- logger.info("Reloaded without MTP after the tensor-parallel crash.")+ logger.info("Reloaded without MTP after the mid-generation crash.")except Exception as e:logger.error(f"Reload without MTP failed: {e}")finally:@@ -36027,11 +36039,11 @@ class LlamaCppBackend:return Truedef _start_mtp_crash_watchdog(self) -> None:- """Background poll that recovers on an MTP+tensor crash even when no- request observes it (direct proxy endpoints, or nothing in flight).+ """Background poll that recovers on an MTP crash even when no request+ observes it (direct proxy endpoints, or nothing in flight).- Armed only for a live MTP+tensor launch; the no-MTP reload disarms it, so- it can't loop.+ Armed only for a live MTP launch; the no-MTP reload disarms it, so it+ can't loop."""if not self._mtp_runtime_fallback_active:return@@ -37457,7 +37469,7 @@ class LlamaCppBackend:except _LlamaStreamCancelled:returnexcept httpx.ConnectError as e:- # Server already down. If this was an MTP+tensor crash, recover by+ # Server already down. If this was an MTP crash, recover by# reloading without MTP (scheduled in the background) and fail this# request. Otherwise the server was likely SIGKILL'd by GPU pressure# from a concurrent training run: respawn the same config and retry thediff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex c4affa450e9..7df356c3adb 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -32136,7 +32136,7 @@ async def produce_openai_chat_completions(except Exception as e:logger.error(f"Error during GGUF tool streaming: {e}", exc_info = True)api_monitor.fail(monitor_id, _friendly_error(e))- # Recover if an MTP+tensor crash killed the server mid-stream.+ # Recover if an MTP crash killed the server mid-stream.get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e)error_chunk = _openai_stream_error_chunk(e)yield _openai_stream_error_sse(error_chunk)@@ -32516,7 +32516,7 @@ async def produce_openai_chat_completions(except Exception as e:logger.error(f"Error during GGUF tool completion: {e}", exc_info = True)api_monitor.fail(monitor_id, _friendly_error(e))- # Recover if an MTP+tensor crash killed the server.+ # Recover if an MTP crash killed the server.get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e)# An over-context prompt makes llama-server return 400; map any# upstream 4xx to a 400 client error rather than leaking a 500.@@ -33204,7 +33204,7 @@ async def produce_openai_chat_completions(except Exception as e:logger.error(f"Error during GGUF completion: {e}", exc_info = True)api_monitor.fail(monitor_id, _friendly_error(e))- # Recover if an MTP+tensor crash killed the server.+ # Recover if an MTP crash killed the server.get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e)# An over-context prompt makes llama-server return 400; map any# upstream 4xx to a 400 client error rather than leaking a 500.@@ -43349,7 +43349,7 @@ async def _passthrough_retry_url(llama_backend, exc):A crashed server relaunches on a NEW ephemeral port, so a passthrough stillholding the old base_url keeps failing until the next load. Mirrors the- respawn-and-retry in generate_chat_completion. None when an MTP+tensor crash+ respawn-and-retry in generate_chat_completion. None when an MTP crashalready scheduled its own recovery, or when nothing needed respawning.Shared by both passthrough surfaces: /v1/messages and /v1/chat/completions postdiff --git a/studio/backend/tests/test_mtp_runtime_fallback_any_launch.py b/studio/backend/tests/test_mtp_runtime_fallback_any_launch.pynew file mode 100644index 00000000000..7bf9830126e--- /dev/null+++ b/studio/backend/tests/test_mtp_runtime_fallback_any_launch.py@@ -0,0 +1,121 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MTP crash fallback on every MTP launch, and its reload surviving its thread (#13253)."""++from __future__ import annotations++import inspect+import os+import re+import sys+import threading+import time+import types as _types+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++_loggers_stub = _types.ModuleType("loggers")+_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)+sys.modules.setdefault("loggers", _loggers_stub)++from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend # noqa: E402+from utils import process_lifetime # noqa: E402+++def test_runtime_fallback_armed_without_tensor_parallel():+ src = inspect.getsource(LlamaCppBackend.load_model)+ m = re.search(r"_mtp_active_for_launched_server = bool\(([^)]*)\)", src)+ assert m, "load_model must record whether the launched server runs MTP"+ assert m.group(1).strip() == "_spec_requested_mtp"+ assert "self._mtp_runtime_fallback_active = _mtp_active_for_launched_server" in src++[email protected](not sys.platform.startswith("linux"), reason = "PDEATHSIG is Linux-only")+def test_llama_server_started_from_a_short_thread_outlives_it(tmp_path, monkeypatch):+ if not process_lifetime._pdeathsig_available():+ pytest.skip(reason = "PR_SET_PDEATHSIG blocked here (seccomp), so the hazard cannot occur")+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ b = LlamaCppBackend()+ started = {}++ def _reload_thread():+ with b._lock:+ started["ok"] = b._start_llama_process(+ [sys.executable, "-c", "import time; time.sleep(60)"],+ dict(os.environ),+ child_gpu_physical_ids = None,+ )++ t = threading.Thread(target = _reload_thread, name = "mtp-crash-reload")+ t.start()+ t.join()+ try:+ assert started["ok"] is True+ time.sleep(1.0)+ assert b._process.poll() is None, "child died with the thread that spawned it"+ finally:+ b._kill_process()+++class _AliveProcess:+ returncode = None++ def poll(self):+ return None+++class _DeadProcess:+ returncode = -6++ def poll(self):+ return -6++ def terminate(self):+ pass++ def kill(self):+ pass++ def wait(self, timeout = None):+ return -6+++def _armed_backend(proc, extra_args = None):+ b = LlamaCppBackend()+ b._mtp_runtime_fallback_active = True+ b._process = proc+ b._last_load_intent = GgufLoadIntent(+ model_identifier = "unsloth/gemma-4-E2B-it-GGUF",+ speculative_type = "auto",+ extra_args = extra_args,+ )+ return b+++def test_transient_error_on_a_live_server_keeps_the_normal_retry(monkeypatch):+ b = _armed_backend(_AliveProcess())+ monkeypatch.setattr(b, "_server_socket_is_open", lambda *a, **k: True)+ monkeypatch.setattr(b, "load_model", lambda intent: pytest.fail("no reload for a live server"))+ assert b._maybe_recover_from_mtp_crash(RuntimeError("reset")) is False+ assert b._mtp_runtime_fallback_in_progress is False+++def test_recovery_with_extras_owned_mtp_matches_an_identical_apply(monkeypatch):+ extras = ("--spec-type", "draft-mtp")+ b = _armed_backend(_DeadProcess(), extra_args = extras)+ loads = []+ monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)+ assert b._maybe_recover_from_mtp_crash(RuntimeError("peer closed")) is True+ deadline = time.monotonic() + 5+ while (not loads or b._mtp_runtime_fallback_in_progress) and time.monotonic() < deadline:+ time.sleep(0.02)+ assert loads and loads[0].speculative_type == "off"+ assert "draft-mtp" not in (loads[0].extra_args or ())+ assert b._spec_fallback_reason == "runtime_error"+ assert b._requested_spec_mode is None