Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: restart llama-server after a lost GPU device instead of locking the chat (#13044)

Daniel Hanfc56663ef21c
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 9c07077f6d1..64ddc9005ab 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -607,6 +607,17 @@ class _LlamaStreamCancelled(Exception):
__slots__ = ()
+# vulkan-hpp formats DeviceLostError as "vk::Queue::submit: ErrorDeviceLost".
+_GPU_DEVICE_LOST_MARKERS = ("ErrorDeviceLost", "VK_ERROR_DEVICE_LOST")
+_DEVICE_LOST_LOG_GRACE_S = 2.0
+
+
+def _is_gpu_device_lost(exc: BaseException) -> bool:
+ """True when llama-server reported a lost GPU device (500 body or SSE error, #11453)."""
+ text = str(exc)
+ return any(marker in text for marker in _GPU_DEVICE_LOST_MARKERS)
+
+
class _CombinedCancelEvent:
__slots__ = ("_events",)
@@ -17892,8 +17903,10 @@ class LlamaCppBackend:
startup_len: Optional[int] = None
log_bytes = 0
levelled = to_info = False
+ proc = self._process
+ device_lost_seen = False
try:
- for line in self._process.stdout:
+ for line in proc.stdout:
line = line.rstrip()
if line:
lines = self._stdout_lines
@@ -17918,6 +17931,14 @@ class LlamaCppBackend:
if level is not None:
levelled = True
to_info = level.group(1) in "WE" or (ready and level.group(1) == "I")
+ # E lines only: unprefixed lines can be a request dump with user text.
+ if (
+ not device_lost_seen
+ and level.group(1) == "E"
+ and any(marker in line for marker in _GPU_DEVICE_LOST_MARKERS)
+ ):
+ device_lost_seen = True
+ self._retire_device_lost_server_soon(proc)
elif not levelled:
to_info = ready or _llama_line_is_warning_or_error(line, line_lower)
try:
@@ -35727,6 +35748,52 @@ class LlamaCppBackend:
return False
return started
+ def _retire_device_lost_server(self, served_by) -> None:
+ """Kill the llama-server that hit a lost GPU device, if it is still current (#11453).
+
+ It survives VK_ERROR_DEVICE_LOST but fails every later request; a dead child is
+ what _respawn_if_dead already recovers.
+ """
+ with self._respawn_lock:
+ if (
+ served_by is None
+ or served_by is not self._process
+ or served_by.poll() is not None
+ or self._cancel_event.is_set()
+ ):
+ return
+ # Else the MTP watchdog reads this kill as an MTP crash.
+ self._stop_mtp_crash_watchdog()
+ try:
+ served_by.kill()
+ served_by.wait(timeout = 10)
+ except (OSError, subprocess.TimeoutExpired):
+ pass
+ try:
+ logger.warning(
+ f"llama-server for '{self._model_identifier}' lost its GPU device "
+ "(the graphics driver reset it); restarted it. On Intel Arc with "
+ "Vulkan, GGML_VK_DISABLE_COOPMAT=1 may avoid the reset."
+ )
+ except Exception:
+ # The logger raises once stdout is closed; recovery must not.
+ pass
+
+ def _retire_device_lost_server_soon(self, served_by) -> None:
+ """Retire after a device loss in the server log, for the passthrough routes.
+
+ The grace lets a chat request that hit the same error retire and retry first.
+ """
+
+ def _retire():
+ time.sleep(_DEVICE_LOST_LOG_GRACE_S)
+ self._retire_device_lost_server(served_by)
+
+ try:
+ threading.Thread(target = _retire, daemon = True, name = "llama-device-lost").start()
+ except RuntimeError:
+ pass
+
@contextlib.contextmanager
def _open_chat_stream_with_respawn_retry(
self,
@@ -35752,12 +35819,26 @@ class LlamaCppBackend:
"""
for attempt in range(2):
response_opened = False
+ served_by = getattr(self, "_process", None)
try:
url = f"{self.base_url}/v1/chat/completions"
with self._open_stream(url, payload, cancel_event) as opened:
response_opened = True
yield opened
return
+ except RuntimeError as exc:
+ if not _is_gpu_device_lost(exc):
+ raise
+ # Retry only before the 200; once opened the consumer may have emitted output.
+ self._retire_device_lost_server(served_by)
+ if response_opened or attempt > 0 or not self._respawn_if_dead():
+ raise
+ logger.warning(
+ "llama-server lost its GPU device; restarted it and retrying the generation"
+ )
+ if on_respawn is not None:
+ on_respawn()
+ continue
except (httpx.NetworkError, httpx.RemoteProtocolError) as exc:
if response_opened:
raise
@@ -36027,6 +36108,7 @@ class LlamaCppBackend:
_apply_seeded_llama_request(payload, seed)
payload["stream_options"] = {"include_usage": True}
+ served_by = getattr(self, "_process", None)
url = f"{self.base_url}/v1/chat/completions"
cumulative = ""
in_thinking = False
@@ -36035,6 +36117,55 @@ class LlamaCppBackend:
_metadata_timings = None
_metadata_finish_reason = None
+ def _replay_on_replacement_server():
+ context_overflow_ = retry_context_overflow
+ max_tokens_ = retry_max_tokens
+ if (
+ retry_preflight_context_length is not None
+ and retry_preflight_context_length != self._effective_context_length
+ ):
+ # Refit the compacted prompt against the replacement server's window;
+ # any event now reports only additional evictions.
+ context_overflow_ = context_overflow
+ if max_tokens is None:
+ max_tokens_ = None
+ yield from self.generate_chat_completion(
+ retry_messages,
+ image_b64 = retry_image_b64,
+ temperature = temperature,
+ top_p = top_p,
+ top_k = top_k,
+ min_p = min_p,
+ max_tokens = max_tokens_,
+ repetition_penalty = repetition_penalty,
+ presence_penalty = presence_penalty,
+ frequency_penalty = frequency_penalty,
+ logit_bias = logit_bias,
+ stop = stop,
+ cancel_event = cancel_event,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
+ continue_final_message = continue_final_message,
+ seed = seed,
+ promote_reasoning_only = promote_reasoning_only,
+ perf_callback = perf_callback,
+ reasoning_provenance = reasoning_provenance,
+ context_overflow = context_overflow_,
+ context_policy = context_policy,
+ compaction_headroom_ratio = compaction_headroom_ratio,
+ # The retry refits for the replacement window and can evict more than
+ # the first attempt did. Without the thread those extra turns are
+ # archived nowhere and no reserve or boundary applies, on the one path
+ # that deliberately compacts again.
+ thread_id = thread_id,
+ # The retry refits, so it must be told the same about this request's
+ # tools as the first attempt was.
+ tools_withheld = tools_withheld,
+ thinking_budget_tokens = thinking_budget_tokens,
+ _allow_respawn_retry = False,
+ )
+
try:
with self._open_stream(url, payload, cancel_event) as (
response,
@@ -36189,56 +36320,21 @@ class LlamaCppBackend:
logger.warning(
"llama-server was unreachable; respawned it and retrying the generation"
)
- if (
- retry_preflight_context_length is not None
- and retry_preflight_context_length != self._effective_context_length
- ):
- # Refit the compacted prompt against the replacement server's window;
- # any event now reports only additional evictions.
- retry_context_overflow = context_overflow
- if max_tokens is None:
- retry_max_tokens = None
- yield from self.generate_chat_completion(
- retry_messages,
- image_b64 = retry_image_b64,
- temperature = temperature,
- top_p = top_p,
- top_k = top_k,
- min_p = min_p,
- max_tokens = retry_max_tokens,
- repetition_penalty = repetition_penalty,
- presence_penalty = presence_penalty,
- frequency_penalty = frequency_penalty,
- logit_bias = logit_bias,
- stop = stop,
- cancel_event = cancel_event,
- enable_thinking = enable_thinking,
- reasoning_effort = reasoning_effort,
- preserve_thinking = preserve_thinking,
- continue_final_message = continue_final_message,
- seed = seed,
- promote_reasoning_only = promote_reasoning_only,
- perf_callback = perf_callback,
- reasoning_provenance = reasoning_provenance,
- context_overflow = retry_context_overflow,
- context_policy = context_policy,
- compaction_headroom_ratio = compaction_headroom_ratio,
- # The retry refits for the replacement window and can evict more than
- # the first attempt did. Without the thread those extra turns are
- # archived nowhere and no reserve or boundary applies, on the one path
- # that deliberately compacts again.
- thread_id = thread_id,
- # The retry refits, so it must be told the same about this request's
- # tools as the first attempt was.
- tools_withheld = tools_withheld,
- thinking_budget_tokens = thinking_budget_tokens,
- _allow_respawn_retry = False,
- )
+ yield from _replay_on_replacement_server()
return
raise RuntimeError("Lost connection to llama-server")
except Exception as e:
if cancel_event is not None and cancel_event.is_set():
return
+ if _is_gpu_device_lost(e):
+ self._retire_device_lost_server(served_by)
+ if _allow_respawn_retry and not cumulative and self._respawn_if_dead():
+ logger.warning(
+ "llama-server lost its GPU device; restarted it and retrying the generation"
+ )
+ yield from _replay_on_replacement_server()
+ return
+ raise
# Died mid-generation: recover MTP, re-raise unchanged for this request.
self._maybe_recover_from_mtp_crash(e)
raise
diff --git a/studio/backend/tests/test_llama_cpp_device_lost_respawn.py b/studio/backend/tests/test_llama_cpp_device_lost_respawn.py
new file mode 100644
index 00000000000..abd83539b31
--- /dev/null
+++ b/studio/backend/tests/test_llama_cpp_device_lost_respawn.py
@@ -0,0 +1,325 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A llama-server whose GPU device was lost is restarted instead of failing forever (#11453).
+
+Vulkan reports VK_ERROR_DEVICE_LOST as an exception llama-server catches, so the process
+stays up and fails every later request, either as a 500 before the stream opens or as an
+in-band SSE error chunk after the 200. The dead-process respawn never fired, and the chat
+stayed stuck until a manual eject and reload.
+"""
+
+from __future__ import annotations
+
+import http.server
+import json
+import subprocess
+import sys
+import threading
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.llama_cpp import LlamaCppBackend
+from core.inference.stream_errors import stream_error_from_chunk
+
+# The reporter's exact llama-server error.
+DEVICE_LOST_ERROR = {
+ "error": {
+ "code": 500,
+ "message": "got exception: vk::Queue::submit: ErrorDeviceLost",
+ "type": "server_error",
+ }
+}
+DEVICE_LOST_BODY = json.dumps(DEVICE_LOST_ERROR)
+OTHER_500_BODY = json.dumps(
+ {"error": {"code": 500, "message": "got exception: bad request", "type": "server_error"}}
+)
+_PROGRESS = (
+ 'data: {"choices":[{"index":0,"delta":{}}],'
+ '"prompt_progress":{"total":8,"cache":0,"processed":0,"time_ms":0}}\n\n'
+)
+# What llama-server sends when the loss happens after the 200 (Studio asks for
+# return_progress, so the headers and a progress event go out before the decode).
+SSE_DEVICE_LOST = _PROGRESS + f"data: {DEVICE_LOST_BODY}\n\n"
+SSE_KV_STARVATION = (
+ _PROGRESS + 'data: {"error":{"code":500,"message":"Context size has been exceeded.",'
+ '"type":"server_error"}}\n\n'
+)
+SSE_PARTIAL_THEN_DEVICE_LOST = (
+ 'data: {"choices":[{"index":0,"delta":{"content":"Partial"}}]}\n\n'
+ f"data: {DEVICE_LOST_BODY}\n\n"
+)
+SSE_OK = 'data: {"choices":[{"index":0,"delta":{"content":"Recovered."}}]}\n\n' "data: [DONE]\n\n"
+
+
+def _serve(status: int, body: str, content_type: str):
+ hits = []
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_POST(self):
+ self.rfile.read(int(self.headers.get("Content-Length", 0)))
+ hits.append(self.path)
+ data = body.encode()
+ self.send_response(status)
+ self.send_header("Content-Type", content_type)
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def log_message(self, *args):
+ pass
+
+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ threading.Thread(target = server.serve_forever, daemon = True).start()
+ return server, hits
+
+
+def _child():
+ return subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"])
+
+
+def _close_port_when_exited(child, server):
+ # A real llama-server's port closes with it, so the next request gets ConnectError.
+ closed = threading.Event()
+
+ def watch():
+ child.wait()
+ server.shutdown()
+ server.server_close()
+ closed.set()
+
+ threading.Thread(target = watch, daemon = True).start()
+ return closed
+
+
+def harness():
+ servers, children = [], []
+
+ def make(
+ broken_body: str,
+ *,
+ status: int = 500,
+ reload_ok: bool = True,
+ ):
+ content_type = "application/json" if status != 200 else "text/event-stream"
+ broken, broken_hits = _serve(status, broken_body, content_type)
+ healthy, healthy_hits = _serve(200, SSE_OK, "text/event-stream")
+ servers.extend([broken, healthy])
+ backend = LlamaCppBackend()
+ first = _child()
+ children.append(first)
+ backend.port_closed = _close_port_when_exited(first, broken)
+ backend._process = first
+ backend._port = broken.server_address[1]
+ backend._healthy = True
+ backend._model_identifier = "unsloth/test-GGUF"
+ backend._last_load_intent = object()
+ loads = []
+
+ def fake_load_model(intent):
+ loads.append(intent)
+ if not reload_ok:
+ return False
+ replacement = _child()
+ children.append(replacement)
+ backend._process = replacement
+ backend._port = healthy.server_address[1]
+ backend._healthy = True
+ return True
+
+ backend.load_model = fake_load_model
+ return backend, first, loads, broken_hits, healthy_hits
+
+ yield make
+ for child in children:
+ if child.poll() is None:
+ child.kill()
+ child.wait()
+ for server in servers:
+ if server.socket.fileno() != -1:
+ server.shutdown()
+ server.server_close()
+
+
+def _chat(backend):
+ return list(
+ backend.generate_chat_completion(
+ [{"role": "user", "content": "Write space invaders in HTML"}],
+ max_tokens = 16,
+ )
+ )
+
+
+def _text(events):
+ return "".join(str(e) for e in events)
+
+
+ "status, body", [(500, DEVICE_LOST_BODY), (200, SSE_DEVICE_LOST)], ids = ["http500", "sse"]
+)
+def test_device_lost_restarts_the_server_and_retries_the_chat(harness, status, body):
+ backend, first, loads, broken_hits, healthy_hits = harness(body, status = status)
+
+ events = _chat(backend)
+
+ assert len(loads) == 1
+ assert first.poll() is not None
+ assert backend._process is not first and backend._process.poll() is None
+ assert len(broken_hits) == 1 and len(healthy_hits) == 1
+ assert "Recovered." in _text(events)
+
+
+def test_device_lost_after_output_fails_once_then_the_next_turn_recovers(harness):
+ backend, first, loads, broken_hits, healthy_hits = harness(
+ SSE_PARTIAL_THEN_DEVICE_LOST, status = 200
+ )
+
+ with pytest.raises(RuntimeError) as info:
+ _chat(backend)
+ # Output was already streamed, so this turn is not replayed; the server is retired.
+ assert "ErrorDeviceLost" in str(info.value)
+ assert first.poll() is not None and loads == []
+ assert backend.port_closed.wait(5)
+
+ events = _chat(backend)
+
+ assert len(loads) == 1 and "Recovered." in _text(events)
+ assert len(broken_hits) == 1 and len(healthy_hits) == 1
+
+
+def test_device_lost_restarts_the_server_on_the_tool_stream_path(harness):
+ backend, first, loads, broken_hits, healthy_hits = harness(DEVICE_LOST_BODY)
+ respawned = []
+ payload = {"messages": [{"role": "user", "content": "hi"}], "stream": True}
+
+ with backend._open_chat_stream_with_respawn_retry(
+ payload, None, on_respawn = lambda: respawned.append(True)
+ ) as (response, _deadline):
+ assert response.status_code == 200
+
+ assert len(loads) == 1 and respawned == [True]
+ assert first.poll() is not None
+ assert len(broken_hits) == 1 and len(healthy_hits) == 1
+
+
+def test_device_lost_mid_stream_on_the_tool_path_retires_the_server(harness):
+ backend, first, loads, _broken_hits, _healthy_hits = harness(SSE_DEVICE_LOST, status = 200)
+ payload = {"messages": [{"role": "user", "content": "hi"}], "stream": True}
+
+ with pytest.raises(RuntimeError, match = "ErrorDeviceLost"):
+ with backend._open_chat_stream_with_respawn_retry(payload, None):
+ # What the tool loops do with an in-band error chunk.
+ raise stream_error_from_chunk(DEVICE_LOST_ERROR)
+
+ assert first.poll() is not None and loads == []
+ assert backend.port_closed.wait(5)
+ with backend._open_chat_stream_with_respawn_retry(payload, None) as (response, _deadline):
+ assert response.status_code == 200
+ assert len(loads) == 1
+
+
+ "status, body", [(500, OTHER_500_BODY), (200, SSE_KV_STARVATION)], ids = ["http500", "sse"]
+)
+def test_other_server_errors_do_not_restart_the_server(harness, status, body):
+ backend, first, loads, broken_hits, _healthy_hits = harness(body, status = status)
+
+ with pytest.raises(RuntimeError) as info:
+ _chat(backend)
+
+ if status == 500:
+ assert str(info.value) == f"llama-server returned 500: {OTHER_500_BODY}"
+ assert loads == []
+ assert first.poll() is None and backend._process is first
+ assert len(broken_hits) == 1
+
+
+def test_a_failed_restart_reraises_the_original_error(harness):
+ backend, _first, loads, broken_hits, healthy_hits = harness(DEVICE_LOST_BODY, reload_ok = False)
+
+ with pytest.raises(RuntimeError) as info:
+ _chat(backend)
+
+ assert str(info.value) == f"llama-server returned 500: {DEVICE_LOST_BODY}"
+ assert len(loads) == 1
+ assert len(broken_hits) == 1 and healthy_hits == []
+
+
+def test_a_late_caller_does_not_kill_the_replacement(harness):
+ backend, first, loads, _broken_hits, _healthy_hits = harness(DEVICE_LOST_BODY)
+ _chat(backend)
+ replacement = backend._process
+ assert replacement is not first and len(loads) == 1
+
+ # A second request that also failed on the lost server reports it after the reload.
+ backend._retire_device_lost_server(first)
+
+ assert backend._process is replacement and replacement.poll() is None
+ assert len(loads) == 1
+
+
+def _logging_child(line: str):
+ return subprocess.Popen(
+ [
+ sys.executable,
+ "-c",
+ f"import sys, time; print({line!r}, flush = True); time.sleep(120)",
+ ],
+ stdout = subprocess.PIPE,
+ text = True,
+ )
+
+
+ "line, retired",
+ [
+ (
+ "0.01.000.000 E slot iterate: id 0 | got exception: vk::Queue::submit: ErrorDeviceLost",
+ True,
+ ),
+ (
+ "0.01.000.000 I slot launch_slot_: id 0 | prompt: what does ErrorDeviceLost mean?",
+ False,
+ ),
+ ("what does vk::Queue::submit: ErrorDeviceLost mean?", False),
+ ],
+ ids = ["server_error", "info_line", "unprefixed_request_dump"],
+)
+def test_a_device_loss_in_the_server_log_retires_the_server(monkeypatch, line, retired):
+ # The passthrough endpoints forward llama-server's error untouched, so the
+ # server's own error log is what retires the server for them.
+ import core.inference.llama_cpp as llama_cpp
+
+ monkeypatch.setattr(llama_cpp, "_DEVICE_LOST_LOG_GRACE_S", 0.0)
+ child = _logging_child(line)
+ try:
+ backend = LlamaCppBackend()
+ backend._process = child
+ backend._model_identifier = "unsloth/test-GGUF"
+ drain = threading.Thread(target = backend._drain_stdout, daemon = True)
+ drain.start()
+ try:
+ child.wait(timeout = 5)
+ except subprocess.TimeoutExpired:
+ pass
+ assert (child.poll() is not None) is retired
+ finally:
+ if child.poll() is None:
+ child.kill()
+ child.wait()
+
+
+def test_retiring_disarms_the_mtp_crash_watchdog(harness):
+ backend, first, _loads, _broken_hits, _healthy_hits = harness(DEVICE_LOST_BODY)
+ stop = threading.Event()
+ backend._mtp_watchdog_stop = stop
+
+ backend._retire_device_lost_server(first)
+
+ assert stop.is_set() and first.poll() is not None