Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: restart llama-server after a lost GPU device instead of locking the chat (#13044)
Daniel Hanfc56663ef21c
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 9c07077f6d1..64ddc9005ab 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -607,6 +607,17 @@ class _LlamaStreamCancelled(Exception):__slots__ = ()+# vulkan-hpp formats DeviceLostError as "vk::Queue::submit: ErrorDeviceLost".+_GPU_DEVICE_LOST_MARKERS = ("ErrorDeviceLost", "VK_ERROR_DEVICE_LOST")+_DEVICE_LOST_LOG_GRACE_S = 2.0+++def _is_gpu_device_lost(exc: BaseException) -> bool:+ """True when llama-server reported a lost GPU device (500 body or SSE error, #11453)."""+ text = str(exc)+ return any(marker in text for marker in _GPU_DEVICE_LOST_MARKERS)++class _CombinedCancelEvent:__slots__ = ("_events",)@@ -17892,8 +17903,10 @@ class LlamaCppBackend:startup_len: Optional[int] = Nonelog_bytes = 0levelled = to_info = False+ proc = self._process+ device_lost_seen = Falsetry:- for line in self._process.stdout:+ for line in proc.stdout:line = line.rstrip()if line:lines = self._stdout_lines@@ -17918,6 +17931,14 @@ class LlamaCppBackend:if level is not None:levelled = Trueto_info = level.group(1) in "WE" or (ready and level.group(1) == "I")+ # E lines only: unprefixed lines can be a request dump with user text.+ if (+ not device_lost_seen+ and level.group(1) == "E"+ and any(marker in line for marker in _GPU_DEVICE_LOST_MARKERS)+ ):+ device_lost_seen = True+ self._retire_device_lost_server_soon(proc)elif not levelled:to_info = ready or _llama_line_is_warning_or_error(line, line_lower)try:@@ -35727,6 +35748,52 @@ class LlamaCppBackend:return Falsereturn started+ def _retire_device_lost_server(self, served_by) -> None:+ """Kill the llama-server that hit a lost GPU device, if it is still current (#11453).++ It survives VK_ERROR_DEVICE_LOST but fails every later request; a dead child is+ what _respawn_if_dead already recovers.+ """+ with self._respawn_lock:+ if (+ served_by is None+ or served_by is not self._process+ or served_by.poll() is not None+ or self._cancel_event.is_set()+ ):+ return+ # Else the MTP watchdog reads this kill as an MTP crash.+ self._stop_mtp_crash_watchdog()+ try:+ served_by.kill()+ served_by.wait(timeout = 10)+ except (OSError, subprocess.TimeoutExpired):+ pass+ try:+ logger.warning(+ f"llama-server for '{self._model_identifier}' lost its GPU device "+ "(the graphics driver reset it); restarted it. On Intel Arc with "+ "Vulkan, GGML_VK_DISABLE_COOPMAT=1 may avoid the reset."+ )+ except Exception:+ # The logger raises once stdout is closed; recovery must not.+ pass++ def _retire_device_lost_server_soon(self, served_by) -> None:+ """Retire after a device loss in the server log, for the passthrough routes.++ The grace lets a chat request that hit the same error retire and retry first.+ """++ def _retire():+ time.sleep(_DEVICE_LOST_LOG_GRACE_S)+ self._retire_device_lost_server(served_by)++ try:+ threading.Thread(target = _retire, daemon = True, name = "llama-device-lost").start()+ except RuntimeError:+ pass+@contextlib.contextmanagerdef _open_chat_stream_with_respawn_retry(self,@@ -35752,12 +35819,26 @@ class LlamaCppBackend:"""for attempt in range(2):response_opened = False+ served_by = getattr(self, "_process", None)try:url = f"{self.base_url}/v1/chat/completions"with self._open_stream(url, payload, cancel_event) as opened:response_opened = Trueyield openedreturn+ except RuntimeError as exc:+ if not _is_gpu_device_lost(exc):+ raise+ # Retry only before the 200; once opened the consumer may have emitted output.+ self._retire_device_lost_server(served_by)+ if response_opened or attempt > 0 or not self._respawn_if_dead():+ raise+ logger.warning(+ "llama-server lost its GPU device; restarted it and retrying the generation"+ )+ if on_respawn is not None:+ on_respawn()+ continueexcept (httpx.NetworkError, httpx.RemoteProtocolError) as exc:if response_opened:raise@@ -36027,6 +36108,7 @@ class LlamaCppBackend:_apply_seeded_llama_request(payload, seed)payload["stream_options"] = {"include_usage": True}+ served_by = getattr(self, "_process", None)url = f"{self.base_url}/v1/chat/completions"cumulative = ""in_thinking = False@@ -36035,6 +36117,55 @@ class LlamaCppBackend:_metadata_timings = None_metadata_finish_reason = None+ def _replay_on_replacement_server():+ context_overflow_ = retry_context_overflow+ max_tokens_ = retry_max_tokens+ if (+ retry_preflight_context_length is not None+ and retry_preflight_context_length != self._effective_context_length+ ):+ # Refit the compacted prompt against the replacement server's window;+ # any event now reports only additional evictions.+ context_overflow_ = context_overflow+ if max_tokens is None:+ max_tokens_ = None+ yield from self.generate_chat_completion(+ retry_messages,+ image_b64 = retry_image_b64,+ temperature = temperature,+ top_p = top_p,+ top_k = top_k,+ min_p = min_p,+ max_tokens = max_tokens_,+ repetition_penalty = repetition_penalty,+ presence_penalty = presence_penalty,+ frequency_penalty = frequency_penalty,+ logit_bias = logit_bias,+ stop = stop,+ cancel_event = cancel_event,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,+ continue_final_message = continue_final_message,+ seed = seed,+ promote_reasoning_only = promote_reasoning_only,+ perf_callback = perf_callback,+ reasoning_provenance = reasoning_provenance,+ context_overflow = context_overflow_,+ context_policy = context_policy,+ compaction_headroom_ratio = compaction_headroom_ratio,+ # The retry refits for the replacement window and can evict more than+ # the first attempt did. Without the thread those extra turns are+ # archived nowhere and no reserve or boundary applies, on the one path+ # that deliberately compacts again.+ thread_id = thread_id,+ # The retry refits, so it must be told the same about this request's+ # tools as the first attempt was.+ tools_withheld = tools_withheld,+ thinking_budget_tokens = thinking_budget_tokens,+ _allow_respawn_retry = False,+ )+try:with self._open_stream(url, payload, cancel_event) as (response,@@ -36189,56 +36320,21 @@ class LlamaCppBackend:logger.warning("llama-server was unreachable; respawned it and retrying the generation")- if (- retry_preflight_context_length is not None- and retry_preflight_context_length != self._effective_context_length- ):- # Refit the compacted prompt against the replacement server's window;- # any event now reports only additional evictions.- retry_context_overflow = context_overflow- if max_tokens is None:- retry_max_tokens = None- yield from self.generate_chat_completion(- retry_messages,- image_b64 = retry_image_b64,- temperature = temperature,- top_p = top_p,- top_k = top_k,- min_p = min_p,- max_tokens = retry_max_tokens,- repetition_penalty = repetition_penalty,- presence_penalty = presence_penalty,- frequency_penalty = frequency_penalty,- logit_bias = logit_bias,- stop = stop,- cancel_event = cancel_event,- enable_thinking = enable_thinking,- reasoning_effort = reasoning_effort,- preserve_thinking = preserve_thinking,- continue_final_message = continue_final_message,- seed = seed,- promote_reasoning_only = promote_reasoning_only,- perf_callback = perf_callback,- reasoning_provenance = reasoning_provenance,- context_overflow = retry_context_overflow,- context_policy = context_policy,- compaction_headroom_ratio = compaction_headroom_ratio,- # The retry refits for the replacement window and can evict more than- # the first attempt did. Without the thread those extra turns are- # archived nowhere and no reserve or boundary applies, on the one path- # that deliberately compacts again.- thread_id = thread_id,- # The retry refits, so it must be told the same about this request's- # tools as the first attempt was.- tools_withheld = tools_withheld,- thinking_budget_tokens = thinking_budget_tokens,- _allow_respawn_retry = False,- )+ yield from _replay_on_replacement_server()returnraise RuntimeError("Lost connection to llama-server")except Exception as e:if cancel_event is not None and cancel_event.is_set():return+ if _is_gpu_device_lost(e):+ self._retire_device_lost_server(served_by)+ if _allow_respawn_retry and not cumulative and self._respawn_if_dead():+ logger.warning(+ "llama-server lost its GPU device; restarted it and retrying the generation"+ )+ yield from _replay_on_replacement_server()+ return+ raise# Died mid-generation: recover MTP, re-raise unchanged for this request.self._maybe_recover_from_mtp_crash(e)raisediff --git a/studio/backend/tests/test_llama_cpp_device_lost_respawn.py b/studio/backend/tests/test_llama_cpp_device_lost_respawn.pynew file mode 100644index 00000000000..abd83539b31--- /dev/null+++ b/studio/backend/tests/test_llama_cpp_device_lost_respawn.py@@ -0,0 +1,325 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A llama-server whose GPU device was lost is restarted instead of failing forever (#11453).++Vulkan reports VK_ERROR_DEVICE_LOST as an exception llama-server catches, so the process+stays up and fails every later request, either as a 500 before the stream opens or as an+in-band SSE error chunk after the 200. The dead-process respawn never fired, and the chat+stayed stuck until a manual eject and reload.+"""++from __future__ import annotations++import http.server+import json+import subprocess+import sys+import threading+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from core.inference.llama_cpp import LlamaCppBackend+from core.inference.stream_errors import stream_error_from_chunk++# The reporter's exact llama-server error.+DEVICE_LOST_ERROR = {+ "error": {+ "code": 500,+ "message": "got exception: vk::Queue::submit: ErrorDeviceLost",+ "type": "server_error",+ }+}+DEVICE_LOST_BODY = json.dumps(DEVICE_LOST_ERROR)+OTHER_500_BODY = json.dumps(+ {"error": {"code": 500, "message": "got exception: bad request", "type": "server_error"}}+)+_PROGRESS = (+ 'data: {"choices":[{"index":0,"delta":{}}],'+ '"prompt_progress":{"total":8,"cache":0,"processed":0,"time_ms":0}}\n\n'+)+# What llama-server sends when the loss happens after the 200 (Studio asks for+# return_progress, so the headers and a progress event go out before the decode).+SSE_DEVICE_LOST = _PROGRESS + f"data: {DEVICE_LOST_BODY}\n\n"+SSE_KV_STARVATION = (+ _PROGRESS + 'data: {"error":{"code":500,"message":"Context size has been exceeded.",'+ '"type":"server_error"}}\n\n'+)+SSE_PARTIAL_THEN_DEVICE_LOST = (+ 'data: {"choices":[{"index":0,"delta":{"content":"Partial"}}]}\n\n'+ f"data: {DEVICE_LOST_BODY}\n\n"+)+SSE_OK = 'data: {"choices":[{"index":0,"delta":{"content":"Recovered."}}]}\n\n' "data: [DONE]\n\n"+++def _serve(status: int, body: str, content_type: str):+ hits = []++ class Handler(http.server.BaseHTTPRequestHandler):+ def do_POST(self):+ self.rfile.read(int(self.headers.get("Content-Length", 0)))+ hits.append(self.path)+ data = body.encode()+ self.send_response(status)+ self.send_header("Content-Type", content_type)+ self.send_header("Content-Length", str(len(data)))+ self.end_headers()+ self.wfile.write(data)++ def log_message(self, *args):+ pass++ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ threading.Thread(target = server.serve_forever, daemon = True).start()+ return server, hits+++def _child():+ return subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"])+++def _close_port_when_exited(child, server):+ # A real llama-server's port closes with it, so the next request gets ConnectError.+ closed = threading.Event()++ def watch():+ child.wait()+ server.shutdown()+ server.server_close()+ closed.set()++ threading.Thread(target = watch, daemon = True).start()+ return closed+++def harness():+ servers, children = [], []++ def make(+ broken_body: str,+ *,+ status: int = 500,+ reload_ok: bool = True,+ ):+ content_type = "application/json" if status != 200 else "text/event-stream"+ broken, broken_hits = _serve(status, broken_body, content_type)+ healthy, healthy_hits = _serve(200, SSE_OK, "text/event-stream")+ servers.extend([broken, healthy])+ backend = LlamaCppBackend()+ first = _child()+ children.append(first)+ backend.port_closed = _close_port_when_exited(first, broken)+ backend._process = first+ backend._port = broken.server_address[1]+ backend._healthy = True+ backend._model_identifier = "unsloth/test-GGUF"+ backend._last_load_intent = object()+ loads = []++ def fake_load_model(intent):+ loads.append(intent)+ if not reload_ok:+ return False+ replacement = _child()+ children.append(replacement)+ backend._process = replacement+ backend._port = healthy.server_address[1]+ backend._healthy = True+ return True++ backend.load_model = fake_load_model+ return backend, first, loads, broken_hits, healthy_hits++ yield make+ for child in children:+ if child.poll() is None:+ child.kill()+ child.wait()+ for server in servers:+ if server.socket.fileno() != -1:+ server.shutdown()+ server.server_close()+++def _chat(backend):+ return list(+ backend.generate_chat_completion(+ [{"role": "user", "content": "Write space invaders in HTML"}],+ max_tokens = 16,+ )+ )+++def _text(events):+ return "".join(str(e) for e in events)+++ "status, body", [(500, DEVICE_LOST_BODY), (200, SSE_DEVICE_LOST)], ids = ["http500", "sse"]+)+def test_device_lost_restarts_the_server_and_retries_the_chat(harness, status, body):+ backend, first, loads, broken_hits, healthy_hits = harness(body, status = status)++ events = _chat(backend)++ assert len(loads) == 1+ assert first.poll() is not None+ assert backend._process is not first and backend._process.poll() is None+ assert len(broken_hits) == 1 and len(healthy_hits) == 1+ assert "Recovered." in _text(events)+++def test_device_lost_after_output_fails_once_then_the_next_turn_recovers(harness):+ backend, first, loads, broken_hits, healthy_hits = harness(+ SSE_PARTIAL_THEN_DEVICE_LOST, status = 200+ )++ with pytest.raises(RuntimeError) as info:+ _chat(backend)+ # Output was already streamed, so this turn is not replayed; the server is retired.+ assert "ErrorDeviceLost" in str(info.value)+ assert first.poll() is not None and loads == []+ assert backend.port_closed.wait(5)++ events = _chat(backend)++ assert len(loads) == 1 and "Recovered." in _text(events)+ assert len(broken_hits) == 1 and len(healthy_hits) == 1+++def test_device_lost_restarts_the_server_on_the_tool_stream_path(harness):+ backend, first, loads, broken_hits, healthy_hits = harness(DEVICE_LOST_BODY)+ respawned = []+ payload = {"messages": [{"role": "user", "content": "hi"}], "stream": True}++ with backend._open_chat_stream_with_respawn_retry(+ payload, None, on_respawn = lambda: respawned.append(True)+ ) as (response, _deadline):+ assert response.status_code == 200++ assert len(loads) == 1 and respawned == [True]+ assert first.poll() is not None+ assert len(broken_hits) == 1 and len(healthy_hits) == 1+++def test_device_lost_mid_stream_on_the_tool_path_retires_the_server(harness):+ backend, first, loads, _broken_hits, _healthy_hits = harness(SSE_DEVICE_LOST, status = 200)+ payload = {"messages": [{"role": "user", "content": "hi"}], "stream": True}++ with pytest.raises(RuntimeError, match = "ErrorDeviceLost"):+ with backend._open_chat_stream_with_respawn_retry(payload, None):+ # What the tool loops do with an in-band error chunk.+ raise stream_error_from_chunk(DEVICE_LOST_ERROR)++ assert first.poll() is not None and loads == []+ assert backend.port_closed.wait(5)+ with backend._open_chat_stream_with_respawn_retry(payload, None) as (response, _deadline):+ assert response.status_code == 200+ assert len(loads) == 1+++ "status, body", [(500, OTHER_500_BODY), (200, SSE_KV_STARVATION)], ids = ["http500", "sse"]+)+def test_other_server_errors_do_not_restart_the_server(harness, status, body):+ backend, first, loads, broken_hits, _healthy_hits = harness(body, status = status)++ with pytest.raises(RuntimeError) as info:+ _chat(backend)++ if status == 500:+ assert str(info.value) == f"llama-server returned 500: {OTHER_500_BODY}"+ assert loads == []+ assert first.poll() is None and backend._process is first+ assert len(broken_hits) == 1+++def test_a_failed_restart_reraises_the_original_error(harness):+ backend, _first, loads, broken_hits, healthy_hits = harness(DEVICE_LOST_BODY, reload_ok = False)++ with pytest.raises(RuntimeError) as info:+ _chat(backend)++ assert str(info.value) == f"llama-server returned 500: {DEVICE_LOST_BODY}"+ assert len(loads) == 1+ assert len(broken_hits) == 1 and healthy_hits == []+++def test_a_late_caller_does_not_kill_the_replacement(harness):+ backend, first, loads, _broken_hits, _healthy_hits = harness(DEVICE_LOST_BODY)+ _chat(backend)+ replacement = backend._process+ assert replacement is not first and len(loads) == 1++ # A second request that also failed on the lost server reports it after the reload.+ backend._retire_device_lost_server(first)++ assert backend._process is replacement and replacement.poll() is None+ assert len(loads) == 1+++def _logging_child(line: str):+ return subprocess.Popen(+ [+ sys.executable,+ "-c",+ f"import sys, time; print({line!r}, flush = True); time.sleep(120)",+ ],+ stdout = subprocess.PIPE,+ text = True,+ )+++ "line, retired",+ [+ (+ "0.01.000.000 E slot iterate: id 0 | got exception: vk::Queue::submit: ErrorDeviceLost",+ True,+ ),+ (+ "0.01.000.000 I slot launch_slot_: id 0 | prompt: what does ErrorDeviceLost mean?",+ False,+ ),+ ("what does vk::Queue::submit: ErrorDeviceLost mean?", False),+ ],+ ids = ["server_error", "info_line", "unprefixed_request_dump"],+)+def test_a_device_loss_in_the_server_log_retires_the_server(monkeypatch, line, retired):+ # The passthrough endpoints forward llama-server's error untouched, so the+ # server's own error log is what retires the server for them.+ import core.inference.llama_cpp as llama_cpp++ monkeypatch.setattr(llama_cpp, "_DEVICE_LOST_LOG_GRACE_S", 0.0)+ child = _logging_child(line)+ try:+ backend = LlamaCppBackend()+ backend._process = child+ backend._model_identifier = "unsloth/test-GGUF"+ drain = threading.Thread(target = backend._drain_stdout, daemon = True)+ drain.start()+ try:+ child.wait(timeout = 5)+ except subprocess.TimeoutExpired:+ pass+ assert (child.poll() is not None) is retired+ finally:+ if child.poll() is None:+ child.kill()+ child.wait()+++def test_retiring_disarms_the_mtp_crash_watchdog(harness):+ backend, first, _loads, _broken_hits, _healthy_hits = harness(DEVICE_LOST_BODY)+ stop = threading.Event()+ backend._mtp_watchdog_stop = stop++ backend._retire_device_lost_server(first)++ assert stop.is_set() and first.poll() is not None