Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reload without MTP when llama-server dies mid-generation on any MTP launch (#13316)

Daniel Hanb5efe363b421
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 4046d013f3e..4d4b835844a 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -452,6 +452,7 @@ from utils.subprocess_compat import (
)
from utils.process_lifetime import child_popen_kwargs as _child_popen_kwargs
from utils.process_lifetime import is_signalable_pid as _is_signalable_pid
+from utils.process_lifetime import spawn_on_lifetime_thread as _spawn_on_lifetime_thread
from core.inference.tool_call_parser import (
BUDGET_EXHAUSTED_NUDGE,
MAX_ACT_REPROMPTS as _MAX_REPROMPTS,
@@ -8253,12 +8254,12 @@ class LlamaCppBackend:
self._last_load_intent: Optional[GgufLoadIntent] = None
self._mtp_runtime_fallback_lock = threading.Lock()
self._mtp_runtime_fallback_in_progress = False
- # Background watchdog so an MTP+tensor crash recovers even when no request
+ # Background watchdog so an MTP crash recovers even when no request
# observes it (direct proxy endpoints, or nothing in flight).
self._mtp_watchdog_thread: Optional[threading.Thread] = None
self._mtp_watchdog_stop = threading.Event()
- # True when the launch actually runs MTP+tensor (Unsloth- or user/env-driven);
- # gates the probe, watchdog, and recovery so pass-through MTP is covered.
+ # True when the launch actually runs MTP (Unsloth- or user/env-driven);
+ # gates the watchdog and recovery so pass-through MTP is covered.
self._mtp_runtime_fallback_active = False
self._stdout_lines: list[str] = []
self._stdout_thread: Optional[threading.Thread] = None
@@ -24175,16 +24176,19 @@ class LlamaCppBackend:
self._close_attempt_log()
self._health_wait_cancelled = True
return False
- _spawned = subprocess.Popen(
- cmd,
- stdout = subprocess.PIPE,
- stderr = subprocess.STDOUT,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- env = env,
- **_windows_hidden_subprocess_kwargs(),
- **_child_popen_kwargs(),
+ # PDEATHSIG follows the forking thread; the MTP-crash reload's thread exits.
+ _spawned = _spawn_on_lifetime_thread(
+ lambda: subprocess.Popen(
+ cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ env = env,
+ **_windows_hidden_subprocess_kwargs(),
+ **_child_popen_kwargs(),
+ )
)
self._process = _spawned
_invalidate_gpu_memory("llama-server spawned")
@@ -31140,17 +31144,19 @@ class LlamaCppBackend:
self._close_attempt_log()
self._health_wait_cancelled = True
return False
- _spawned = subprocess.Popen(
- run_cmd,
- stdout = subprocess.PIPE,
- stderr = subprocess.STDOUT,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- env = env,
- cwd = _spawn_cwd,
- **_windows_hidden_subprocess_kwargs(),
- **_child_popen_kwargs(),
+ _spawned = _spawn_on_lifetime_thread(
+ lambda: subprocess.Popen(
+ run_cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ env = env,
+ cwd = _spawn_cwd,
+ **_windows_hidden_subprocess_kwargs(),
+ **_child_popen_kwargs(),
+ )
)
self._process = _spawned
_invalidate_gpu_memory("llama-server spawned")
@@ -32430,11 +32436,9 @@ class LlamaCppBackend:
_spec_requested_dflash = any(
"draft-dflash" in str(t).lower() for t in spec_flags
) or _extra_args_requests_dflash(extra_args, env = _launch_spec_env)
- # Is the launched server actually running MTP+tensor? Gates the
- # probe/watchdog/recovery; cleared if the MTP-drop fallback wins.
- _mtp_active_for_launched_server = bool(
- self._tensor_parallel and _spec_requested_mtp
- )
+ # Any MTP launch, not only tensor: a respawn would replay a drafter that
+ # aborts every decode (#13253). Cleared if the MTP-drop fallback wins.
+ _mtp_active_for_launched_server = bool(_spec_requested_mtp)
# MTP can pass /health then crash the flash-attn kernel on the
# first decode under tensor; probe one generation so the fallback
# catches that too. Tensor-only, so ordinary MTP stays probe-free.
@@ -33019,7 +33023,7 @@ class LlamaCppBackend:
self._requested_spec_draft_cache_type = intent.spec_draft_cache_type
self._requested_ctx_checkpoints = intent.ctx_checkpoints
self._requested_cache_ram = intent.cache_ram
- # Commit the known-good snapshot + whether MTP+tensor is live, then
+ # Commit the known-good snapshot + whether MTP is live, then
# watch this load for a mid-generation crash. The verified-cache hint is
# dropped: a respawn replays this snapshot arbitrarily later, so recovery
# re-resolves the file instead of trusting a path this request verified.
@@ -35906,12 +35910,12 @@ class LlamaCppBackend:
def _maybe_recover_from_mtp_crash(self, exc: Optional[BaseException] = None) -> bool:
"""Schedule one background reload without MTP after a mid-generation death.
- MTP+tensor can crash the flash-attn kernel on a later request, after
- load_model returned, past the load-time fallback and decode probe. Not a
- persistent ban: a fresh load re-tries MTP. Returns True if scheduled.
+ An MTP launch can pass /health and the load-time fallback, then abort on
+ a later decode (MTP+tensor flash-attn, Gemma 4 on Apple Silicon #13253).
+ Not a persistent ban: a fresh load re-tries MTP. Returns True if scheduled.
"""
- # Cheap async-safe gate: only our live MTP+tensor launch, not cancelled,
- # with a snapshot to replay.
+ # Cheap async-safe gate: only our live MTP launch, not cancelled, with a
+ # snapshot to replay.
if self._cancel_event.is_set():
return False
if not self._mtp_runtime_fallback_active:
@@ -35922,6 +35926,9 @@ class LlamaCppBackend:
proc = self._process
if snapshot is None or proc is None:
return False
+ # Still serving: a transient error, so leave the caller its normal retry.
+ if proc.poll() is None and self._server_socket_is_open():
+ return False
# Single-flight: the first failure claims the reload.
with self._mtp_runtime_fallback_lock:
if self._mtp_runtime_fallback_in_progress:
@@ -35939,8 +35946,8 @@ class LlamaCppBackend:
logger.debug("Generation error but llama-server is alive; keeping MTP.")
return
logger.warning(
- "llama-server exited mid-generation with MTP under tensor "
- "parallelism (%s); reloading without speculative decoding.",
+ "llama-server exited mid-generation with MTP speculative "
+ "decoding (%s); reloading without speculative decoding.",
type(exc).__name__ if exc is not None else "server exited",
)
# Re-check under the load lock (RLock allows the nested
@@ -36001,7 +36008,12 @@ class LlamaCppBackend:
)
# Restore the requested mode + reason load_model("off") cleared,
# so /status shows the user's mode + note (like the startup fallback).
- self._requested_spec_mode = _canonicalize_spec_mode(requested_mode)
+ # None when the extras own --spec-type, as the duplicate-load match expects.
+ self._requested_spec_mode = (
+ None
+ if _extra_args_set_spec_type(_ea)
+ else _canonicalize_spec_mode(requested_mode)
+ )
self._spec_fallback_reason = "runtime_error"
if undo_reload:
logger.info(
@@ -36009,7 +36021,7 @@ class LlamaCppBackend:
)
self.unload_model()
return
- logger.info("Reloaded without MTP after the tensor-parallel crash.")
+ logger.info("Reloaded without MTP after the mid-generation crash.")
except Exception as e:
logger.error(f"Reload without MTP failed: {e}")
finally:
@@ -36027,11 +36039,11 @@ class LlamaCppBackend:
return True
def _start_mtp_crash_watchdog(self) -> None:
- """Background poll that recovers on an MTP+tensor crash even when no
- request observes it (direct proxy endpoints, or nothing in flight).
+ """Background poll that recovers on an MTP crash even when no request
+ observes it (direct proxy endpoints, or nothing in flight).
- Armed only for a live MTP+tensor launch; the no-MTP reload disarms it, so
- it can't loop.
+ Armed only for a live MTP launch; the no-MTP reload disarms it, so it
+ can't loop.
"""
if not self._mtp_runtime_fallback_active:
return
@@ -37457,7 +37469,7 @@ class LlamaCppBackend:
except _LlamaStreamCancelled:
return
except httpx.ConnectError as e:
- # Server already down. If this was an MTP+tensor crash, recover by
+ # Server already down. If this was an MTP crash, recover by
# reloading without MTP (scheduled in the background) and fail this
# request. Otherwise the server was likely SIGKILL'd by GPU pressure
# from a concurrent training run: respawn the same config and retry the
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index c4affa450e9..7df356c3adb 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -32136,7 +32136,7 @@ async def produce_openai_chat_completions(
except Exception as e:
logger.error(f"Error during GGUF tool streaming: {e}", exc_info = True)
api_monitor.fail(monitor_id, _friendly_error(e))
- # Recover if an MTP+tensor crash killed the server mid-stream.
+ # Recover if an MTP crash killed the server mid-stream.
get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e)
error_chunk = _openai_stream_error_chunk(e)
yield _openai_stream_error_sse(error_chunk)
@@ -32516,7 +32516,7 @@ async def produce_openai_chat_completions(
except Exception as e:
logger.error(f"Error during GGUF tool completion: {e}", exc_info = True)
api_monitor.fail(monitor_id, _friendly_error(e))
- # Recover if an MTP+tensor crash killed the server.
+ # Recover if an MTP crash killed the server.
get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e)
# An over-context prompt makes llama-server return 400; map any
# upstream 4xx to a 400 client error rather than leaking a 500.
@@ -33204,7 +33204,7 @@ async def produce_openai_chat_completions(
except Exception as e:
logger.error(f"Error during GGUF completion: {e}", exc_info = True)
api_monitor.fail(monitor_id, _friendly_error(e))
- # Recover if an MTP+tensor crash killed the server.
+ # Recover if an MTP crash killed the server.
get_llama_cpp_backend()._maybe_recover_from_mtp_crash(e)
# An over-context prompt makes llama-server return 400; map any
# upstream 4xx to a 400 client error rather than leaking a 500.
@@ -43349,7 +43349,7 @@ async def _passthrough_retry_url(llama_backend, exc):
A crashed server relaunches on a NEW ephemeral port, so a passthrough still
holding the old base_url keeps failing until the next load. Mirrors the
- respawn-and-retry in generate_chat_completion. None when an MTP+tensor crash
+ respawn-and-retry in generate_chat_completion. None when an MTP crash
already scheduled its own recovery, or when nothing needed respawning.
Shared by both passthrough surfaces: /v1/messages and /v1/chat/completions post
diff --git a/studio/backend/tests/test_mtp_runtime_fallback_any_launch.py b/studio/backend/tests/test_mtp_runtime_fallback_any_launch.py
new file mode 100644
index 00000000000..7bf9830126e
--- /dev/null
+++ b/studio/backend/tests/test_mtp_runtime_fallback_any_launch.py
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MTP crash fallback on every MTP launch, and its reload surviving its thread (#13253)."""
+
+from __future__ import annotations
+
+import inspect
+import os
+import re
+import sys
+import threading
+import time
+import types as _types
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+_loggers_stub = _types.ModuleType("loggers")
+_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
+sys.modules.setdefault("loggers", _loggers_stub)
+
+from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend # noqa: E402
+from utils import process_lifetime # noqa: E402
+
+
+def test_runtime_fallback_armed_without_tensor_parallel():
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ m = re.search(r"_mtp_active_for_launched_server = bool\(([^)]*)\)", src)
+ assert m, "load_model must record whether the launched server runs MTP"
+ assert m.group(1).strip() == "_spec_requested_mtp"
+ assert "self._mtp_runtime_fallback_active = _mtp_active_for_launched_server" in src
+
+
[email protected](not sys.platform.startswith("linux"), reason = "PDEATHSIG is Linux-only")
+def test_llama_server_started_from_a_short_thread_outlives_it(tmp_path, monkeypatch):
+ if not process_lifetime._pdeathsig_available():
+ pytest.skip(reason = "PR_SET_PDEATHSIG blocked here (seccomp), so the hazard cannot occur")
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ b = LlamaCppBackend()
+ started = {}
+
+ def _reload_thread():
+ with b._lock:
+ started["ok"] = b._start_llama_process(
+ [sys.executable, "-c", "import time; time.sleep(60)"],
+ dict(os.environ),
+ child_gpu_physical_ids = None,
+ )
+
+ t = threading.Thread(target = _reload_thread, name = "mtp-crash-reload")
+ t.start()
+ t.join()
+ try:
+ assert started["ok"] is True
+ time.sleep(1.0)
+ assert b._process.poll() is None, "child died with the thread that spawned it"
+ finally:
+ b._kill_process()
+
+
+class _AliveProcess:
+ returncode = None
+
+ def poll(self):
+ return None
+
+
+class _DeadProcess:
+ returncode = -6
+
+ def poll(self):
+ return -6
+
+ def terminate(self):
+ pass
+
+ def kill(self):
+ pass
+
+ def wait(self, timeout = None):
+ return -6
+
+
+def _armed_backend(proc, extra_args = None):
+ b = LlamaCppBackend()
+ b._mtp_runtime_fallback_active = True
+ b._process = proc
+ b._last_load_intent = GgufLoadIntent(
+ model_identifier = "unsloth/gemma-4-E2B-it-GGUF",
+ speculative_type = "auto",
+ extra_args = extra_args,
+ )
+ return b
+
+
+def test_transient_error_on_a_live_server_keeps_the_normal_retry(monkeypatch):
+ b = _armed_backend(_AliveProcess())
+ monkeypatch.setattr(b, "_server_socket_is_open", lambda *a, **k: True)
+ monkeypatch.setattr(b, "load_model", lambda intent: pytest.fail("no reload for a live server"))
+ assert b._maybe_recover_from_mtp_crash(RuntimeError("reset")) is False
+ assert b._mtp_runtime_fallback_in_progress is False
+
+
+def test_recovery_with_extras_owned_mtp_matches_an_identical_apply(monkeypatch):
+ extras = ("--spec-type", "draft-mtp")
+ b = _armed_backend(_DeadProcess(), extra_args = extras)
+ loads = []
+ monkeypatch.setattr(b, "load_model", lambda intent: loads.append(intent) or True)
+ assert b._maybe_recover_from_mtp_crash(RuntimeError("peer closed")) is True
+ deadline = time.monotonic() + 5
+ while (not loads or b._mtp_runtime_fallback_in_progress) and time.monotonic() < deadline:
+ time.sleep(0.02)
+ assert loads and loads[0].speculative_type == "off"
+ assert "draft-mtp" not in (loads[0].extra_args or ())
+ assert b._spec_fallback_reason == "runtime_error"
+ assert b._requested_spec_mode is None