Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: respawn a llama-server that ran out of GPU memory mid-session with more VRAM headroom (#16388)

Daniel Han741815577cd7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index c8dd066da95..db6ff584af1 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2313,6 +2313,10 @@ def _inject_recall(conversation: list[dict], recall: dict, style: str) -> Option
# Transport errors can beat the child to reapable; a request cannot wait the 5s MTP reload spends.
_RESPAWN_REAP_GRACE_S = 1.0
+# Extra --fit-target per mid-session GPU OOM of one load: an identical respawn hits the same
+# allocation once the conversation is replayed (#16177).
+_GPU_OOM_HEADROOM_STEP_MIB = 1024.0
+_GPU_OOM_HEADROOM_MAX_STRIKES = 3
def _finalize_reasoning_only_cumulative(
@@ -8361,6 +8365,9 @@ class LlamaCppBackend:
# (load key, count) of Metal OOM retirements, and the key that stopped respawning.
self._metal_oom_strikes = None
self._metal_oom_terminal_key = None
+ # (load key, count) of mid-session GPU OOM deaths, and the child that logged one.
+ self._gpu_oom_strikes = None
+ self._gpu_oom_process = None
self._stdout_lines: list[str] = []
self._stdout_thread: Optional[threading.Thread] = None
# Wakes the health probe on subprocess readiness/exit, not left to the 500 ms fallback poll.
@@ -18192,6 +18199,9 @@ class LlamaCppBackend:
if level.group(1) == "E":
if any(m in line for m in _DRAFT_CONTEXT_FAILURE_MARKERS):
metal_poison["draft"] = True
+ # Startup OOM has its own retries; this one is a served child's.
+ if startup_len is not None and self._is_gpu_memory_start_failure(line):
+ self._gpu_oom_process = proc
if not device_lost_seen and any(
marker in line for marker in _GPU_DEVICE_LOST_MARKERS
):
@@ -19359,6 +19369,7 @@ class LlamaCppBackend:
self._mtp_runtime_fallback_active = False
self._metal_oom_strikes = None
self._metal_oom_terminal_key = None
+ self._gpu_oom_strikes = None
# Record the applied split and reset the other knobs so status and reload dedup agree.
self._gpu_memory_mode = gpu_memory_mode if manual_ngl is not None else "auto"
self._gpu_layers = manual_ngl if manual_ngl is not None else -1
@@ -24508,6 +24519,23 @@ class LlamaCppBackend:
for _idx, _free, _total in _gpu_mem
}
+ # Each card keeps its own margin plus the headroom a mid-session OOM earned.
+ _oom_headroom_mib = self._gpu_oom_headroom_mib(intent)
+ if _oom_headroom_mib:
+ _min_delta = _VRAM_FLOOR_RESERVE_MIB - (
+ _VRAM_FLOOR_RESERVE_MIB
+ if gpu_memory_mode == "manual" and gpu_layers < 0
+ else _LLAMA_FIT_TARGET_DEFAULT_MIB
+ )
+ _fit_target_delta_mib = (
+ max(_min_delta, _fit_target_delta_mib) + _oom_headroom_mib
+ )
+ if _fit_target_delta_by_idx:
+ _fit_target_delta_by_idx = {
+ _idx: max(_min_delta, _delta) + _oom_headroom_mib
+ for _idx, _delta in _fit_target_delta_by_idx.items()
+ }
+
_sysmem_fallback = self._sysmem_fallback_risk(binary)
def _gpu_usable(g, frac = _vram_frac):
@@ -32616,6 +32644,7 @@ class LlamaCppBackend:
# the strike count.
self._metal_oom_strikes = None
self._metal_oom_terminal_key = None
+ self._gpu_oom_strikes = None
self._mtp_runtime_fallback_active = _mtp_active_for_launched_server
self._start_mtp_crash_watchdog()
@@ -33535,6 +33564,7 @@ class LlamaCppBackend:
self._mtp_runtime_fallback_active = False
self._metal_oom_strikes = None
self._metal_oom_terminal_key = None
+ self._gpu_oom_strikes = None
self._hf_variant = None
self._is_vision = False
self._disable_vision = False
@@ -36339,6 +36369,8 @@ class LlamaCppBackend:
f"llama-server for '{self._model_identifier}' exited "
f"(code {proc.returncode}); respawning to recover the session"
)
+ if self._died_of_gpu_oom(proc):
+ self._note_gpu_oom_strike(intent)
try:
with _automatic_recovery_load():
started = bool(self.load_model(intent))
@@ -36503,6 +36535,36 @@ class LlamaCppBackend:
except TypeError:
return intent
+ def _died_of_gpu_oom(self, proc) -> bool:
+ """Whether the dead served child logged a GPU out-of-memory error before exiting."""
+ drain = getattr(self, "_stdout_thread", None)
+ if drain is not None and drain is not threading.current_thread():
+ # Its last lines can still be in the pipe when the exit status lands.
+ drain.join(timeout = 2)
+ return proc is not None and getattr(self, "_gpu_oom_process", None) is proc
+
+ def _note_gpu_oom_strike(self, intent) -> None:
+ key = self._metal_oom_key(intent)
+ prior = getattr(self, "_gpu_oom_strikes", None)
+ strikes = prior[1] + 1 if prior is not None and prior[0] == key else 1
+ strikes = min(strikes, _GPU_OOM_HEADROOM_MAX_STRIKES)
+ self._gpu_oom_strikes = (key, strikes)
+ try:
+ logger.warning(
+ f"llama-server for '{self._model_identifier}' ran out of GPU memory; "
+ f"respawning it with {int(strikes * _GPU_OOM_HEADROOM_STEP_MIB)} MiB more "
+ "free VRAM kept (--fit-target)."
+ )
+ except Exception:
+ pass
+
+ def _gpu_oom_headroom_mib(self, intent) -> float:
+ """Extra --fit-target for a reload of the load that ran out of GPU memory; 0 otherwise."""
+ prior = getattr(self, "_gpu_oom_strikes", None)
+ if prior is None or intent is None or prior[0] != self._metal_oom_key(intent):
+ return 0.0
+ return prior[1] * _GPU_OOM_HEADROOM_STEP_MIB
+
def _metal_oom_is_terminal(self, intent) -> bool:
"""Whether ``intent`` already ran out of Metal memory twice (no more replays)."""
terminal = getattr(self, "_metal_oom_terminal_key", None)
diff --git a/studio/backend/tests/test_llama_cpp_gpu_oom_respawn.py b/studio/backend/tests/test_llama_cpp_gpu_oom_respawn.py
new file mode 100644
index 00000000000..050e8abb55c
--- /dev/null
+++ b/studio/backend/tests/test_llama_cpp_gpu_oom_respawn.py
@@ -0,0 +1,176 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A served llama-server that dies of a GPU OOM is respawned with more --fit-target (#16177)."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import threading
+from unittest.mock import patch
+
+import pytest
+
+import core.inference.llama_cpp as llama_cpp
+from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend
+
+from . import _sidecar_fit_helpers as h
+
+# The reporter's crash (A40, Windows, --moe-cache-mib auto, --fit-target 563, ~49K tokens in).
+READY = "3.49.882.278 I srv server_run_i: model loaded"
+CUDA_OOM = [
+ "118.04.263.283 E CUDA error: out of memory",
+ "118.04.263.313 E D:\\a\\llama.cpp\\llama.cpp\\src\\ggml\\src\\ggml-cuda\\ggml-cuda.cu:113: CUDA error",
+ " current device: 0, in function ggml_cuda_graph_launch_chunk",
+]
+OTHER_CRASH = ["5.00.000.000 E GGML_ASSERT(n_tokens > 0) failed"]
+
+
[email protected](autouse = True)
+def _studio_home(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))
+
+
+def _dead_child(lines):
+ code = "".join(f"print({line!r}, flush = True)\n" for line in lines)
+ proc = subprocess.Popen(
+ [sys.executable, "-c", code or "pass"], stdout = subprocess.PIPE, text = True
+ )
+ proc.wait(timeout = 30)
+ return proc
+
+
+def _crashed_backend(lines, intent):
+ backend = LlamaCppBackend()
+ backend._process = _dead_child(lines)
+ backend._stdout_thread = threading.Thread(target = backend._drain_stdout, daemon = True)
+ backend._stdout_thread.start()
+ backend._healthy = True
+ backend._model_identifier = "unsloth/Qwen3.8-Flash-Next-GGUF"
+ backend._last_load_intent = intent
+ backend._server_socket_is_open = lambda: False
+ loads = []
+
+ def fake_load_model(new_intent):
+ loads.append(new_intent)
+ backend._process = _dead_child([READY])
+ backend._stdout_thread = None
+ return True
+
+ backend.load_model = fake_load_model
+ return backend, loads
+
+
+INTENT = GgufLoadIntent(gguf_path = "/m.gguf", model_identifier = "m", n_ctx = 191360)
+
+
+def test_served_cuda_oom_records_a_strike_for_the_respawn():
+ backend, loads = _crashed_backend([READY, *CUDA_OOM], INTENT)
+ assert backend._respawn_if_dead()
+ assert loads == [INTENT]
+ assert backend._gpu_oom_strikes == (backend._metal_oom_key(INTENT), 1)
+
+
+ "lines",
+ [[READY, *OTHER_CRASH], CUDA_OOM, [READY, "5.00.000.000 I prompt: CUDA error: out of memory"]],
+ ids = ["other-crash", "startup-oom", "info-line-echo"],
+)
+def test_no_strike_without_a_served_gpu_oom(lines):
+ backend, loads = _crashed_backend(lines, INTENT)
+ assert backend._respawn_if_dead()
+ assert len(loads) == 1
+ assert backend._gpu_oom_strikes is None
+
+
+def test_strikes_grow_per_crash_of_the_same_load_and_cap():
+ backend = LlamaCppBackend()
+ backend._model_identifier = "m"
+ for _ in range(5):
+ backend._note_gpu_oom_strike(INTENT)
+ assert backend._gpu_oom_strikes[1] == llama_cpp._GPU_OOM_HEADROOM_MAX_STRIKES
+ other = GgufLoadIntent(gguf_path = "/m.gguf", model_identifier = "m", n_ctx = 65536)
+ backend._note_gpu_oom_strike(other)
+ assert backend._gpu_oom_strikes == (backend._metal_oom_key(other), 1)
+ # A speculative-mode change keeps the strike: it is the same placement.
+ off = GgufLoadIntent(
+ gguf_path = "/m.gguf", model_identifier = "m", n_ctx = 65536, speculative_type = "off"
+ )
+ assert backend._gpu_oom_headroom_mib(off) == llama_cpp._GPU_OOM_HEADROOM_STEP_MIB
+ assert backend._gpu_oom_headroom_mib(INTENT) == 0.0
+
+
+def _spawn_cmd(backend, intent):
+ captured = {}
+
+ def fake_popen(cmd, **kwargs):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return h.placement._REAL_POPEN(cmd, **kwargs)
+ captured["cmd"] = list(cmd)
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 2**31 - 1,
+ "stdout": (),
+ "poll": lambda self: None,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: 0,
+ "kill": lambda self: None,
+ },
+ )()
+
+ with patch.object(subprocess, "Popen", side_effect = fake_popen):
+ assert backend.load_model(intent)
+ return captured["cmd"]
+
+
+ "vram_fraction, before, after",
+ [(0.97, None, "2048"), (0.98, "563", "1587")],
+ ids = ["default-budget", "reporter-98pct"],
+)
+def test_respawn_after_gpu_oom_keeps_more_free_vram(
+ tmp_path, monkeypatch, vram_fraction, before, after
+):
+ monkeypatch.setattr(llama_cpp, "_active_vram_fraction", lambda: vram_fraction)
+ backend, gguf = h.cuda_backend(tmp_path, memory = ((0, 46_000, 46_000),))
+ intent = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 8192)
+ assert h.last(_spawn_cmd(backend, intent), "--fit-target") == before
+
+ backend, gguf = h.cuda_backend(tmp_path, memory = ((0, 46_000, 46_000),))
+ backend._model_identifier = "test"
+ backend._note_gpu_oom_strike(intent)
+ with llama_cpp._automatic_recovery_load():
+ assert h.last(_spawn_cmd(backend, intent), "--fit-target") == after
+ # The recovery load keeps the strike; an unload starts over.
+ assert backend._gpu_oom_strikes is not None
+ backend.unload_model()
+ assert backend._gpu_oom_strikes is None
+
+
+def test_mixed_cards_keep_their_own_margins_plus_the_headroom(tmp_path, monkeypatch):
+ B = LlamaCppBackend
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ for k in ("CUDA_VISIBLE_DEVICES", "LLAMA_ARG_DEVICE", "LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_RPC"):
+ monkeypatch.delenv(k, raising = False)
+ monkeypatch.setattr(llama_cpp, "_metal_capable_host", lambda: False)
+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", True)
+ monkeypatch.setattr(B, "_smi_uuid_mask_order", staticmethod(lambda: None))
+ monkeypatch.setattr(B, "_visibility_mask_is_unmappable", staticmethod(lambda: False))
+ monkeypatch.setattr(B, "_SMI_MIG_INDICES", frozenset())
+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: False))
+ monkeypatch.setattr(llama_cpp, "_active_vram_fraction", lambda: 0.8)
+ memory = ((0, 80_000, 80_000), (1, 8_000, 8_000))
+ backend, gguf = h.cards(tmp_path, memory = memory)
+ intent = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 8192)
+ before = h.margins(_spawn_cmd(backend, intent))
+
+ backend, gguf = h.cards(tmp_path, memory = memory)
+ backend._model_identifier = "test"
+ backend._note_gpu_oom_strike(intent)
+ with llama_cpp._automatic_recovery_load():
+ after = h.margins(_spawn_cmd(backend, intent))
+ assert len(before) == 2
+ assert after == [m + 1024 for m in before]