Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: respawn a llama-server that ran out of GPU memory mid-session with more VRAM headroom (#16388)
Daniel Han741815577cd7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex c8dd066da95..db6ff584af1 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -2313,6 +2313,10 @@ def _inject_recall(conversation: list[dict], recall: dict, style: str) -> Option# Transport errors can beat the child to reapable; a request cannot wait the 5s MTP reload spends._RESPAWN_REAP_GRACE_S = 1.0+# Extra --fit-target per mid-session GPU OOM of one load: an identical respawn hits the same+# allocation once the conversation is replayed (#16177).+_GPU_OOM_HEADROOM_STEP_MIB = 1024.0+_GPU_OOM_HEADROOM_MAX_STRIKES = 3def _finalize_reasoning_only_cumulative(@@ -8361,6 +8365,9 @@ class LlamaCppBackend:# (load key, count) of Metal OOM retirements, and the key that stopped respawning.self._metal_oom_strikes = Noneself._metal_oom_terminal_key = None+ # (load key, count) of mid-session GPU OOM deaths, and the child that logged one.+ self._gpu_oom_strikes = None+ self._gpu_oom_process = Noneself._stdout_lines: list[str] = []self._stdout_thread: Optional[threading.Thread] = None# Wakes the health probe on subprocess readiness/exit, not left to the 500 ms fallback poll.@@ -18192,6 +18199,9 @@ class LlamaCppBackend:if level.group(1) == "E":if any(m in line for m in _DRAFT_CONTEXT_FAILURE_MARKERS):metal_poison["draft"] = True+ # Startup OOM has its own retries; this one is a served child's.+ if startup_len is not None and self._is_gpu_memory_start_failure(line):+ self._gpu_oom_process = procif not device_lost_seen and any(marker in line for marker in _GPU_DEVICE_LOST_MARKERS):@@ -19359,6 +19369,7 @@ class LlamaCppBackend:self._mtp_runtime_fallback_active = Falseself._metal_oom_strikes = Noneself._metal_oom_terminal_key = None+ self._gpu_oom_strikes = None# Record the applied split and reset the other knobs so status and reload dedup agree.self._gpu_memory_mode = gpu_memory_mode if manual_ngl is not None else "auto"self._gpu_layers = manual_ngl if manual_ngl is not None else -1@@ -24508,6 +24519,23 @@ class LlamaCppBackend:for _idx, _free, _total in _gpu_mem}+ # Each card keeps its own margin plus the headroom a mid-session OOM earned.+ _oom_headroom_mib = self._gpu_oom_headroom_mib(intent)+ if _oom_headroom_mib:+ _min_delta = _VRAM_FLOOR_RESERVE_MIB - (+ _VRAM_FLOOR_RESERVE_MIB+ if gpu_memory_mode == "manual" and gpu_layers < 0+ else _LLAMA_FIT_TARGET_DEFAULT_MIB+ )+ _fit_target_delta_mib = (+ max(_min_delta, _fit_target_delta_mib) + _oom_headroom_mib+ )+ if _fit_target_delta_by_idx:+ _fit_target_delta_by_idx = {+ _idx: max(_min_delta, _delta) + _oom_headroom_mib+ for _idx, _delta in _fit_target_delta_by_idx.items()+ }+_sysmem_fallback = self._sysmem_fallback_risk(binary)def _gpu_usable(g, frac = _vram_frac):@@ -32616,6 +32644,7 @@ class LlamaCppBackend:# the strike count.self._metal_oom_strikes = Noneself._metal_oom_terminal_key = None+ self._gpu_oom_strikes = Noneself._mtp_runtime_fallback_active = _mtp_active_for_launched_serverself._start_mtp_crash_watchdog()@@ -33535,6 +33564,7 @@ class LlamaCppBackend:self._mtp_runtime_fallback_active = Falseself._metal_oom_strikes = Noneself._metal_oom_terminal_key = None+ self._gpu_oom_strikes = Noneself._hf_variant = Noneself._is_vision = Falseself._disable_vision = False@@ -36339,6 +36369,8 @@ class LlamaCppBackend:f"llama-server for '{self._model_identifier}' exited "f"(code {proc.returncode}); respawning to recover the session")+ if self._died_of_gpu_oom(proc):+ self._note_gpu_oom_strike(intent)try:with _automatic_recovery_load():started = bool(self.load_model(intent))@@ -36503,6 +36535,36 @@ class LlamaCppBackend:except TypeError:return intent+ def _died_of_gpu_oom(self, proc) -> bool:+ """Whether the dead served child logged a GPU out-of-memory error before exiting."""+ drain = getattr(self, "_stdout_thread", None)+ if drain is not None and drain is not threading.current_thread():+ # Its last lines can still be in the pipe when the exit status lands.+ drain.join(timeout = 2)+ return proc is not None and getattr(self, "_gpu_oom_process", None) is proc++ def _note_gpu_oom_strike(self, intent) -> None:+ key = self._metal_oom_key(intent)+ prior = getattr(self, "_gpu_oom_strikes", None)+ strikes = prior[1] + 1 if prior is not None and prior[0] == key else 1+ strikes = min(strikes, _GPU_OOM_HEADROOM_MAX_STRIKES)+ self._gpu_oom_strikes = (key, strikes)+ try:+ logger.warning(+ f"llama-server for '{self._model_identifier}' ran out of GPU memory; "+ f"respawning it with {int(strikes * _GPU_OOM_HEADROOM_STEP_MIB)} MiB more "+ "free VRAM kept (--fit-target)."+ )+ except Exception:+ pass++ def _gpu_oom_headroom_mib(self, intent) -> float:+ """Extra --fit-target for a reload of the load that ran out of GPU memory; 0 otherwise."""+ prior = getattr(self, "_gpu_oom_strikes", None)+ if prior is None or intent is None or prior[0] != self._metal_oom_key(intent):+ return 0.0+ return prior[1] * _GPU_OOM_HEADROOM_STEP_MIB+def _metal_oom_is_terminal(self, intent) -> bool:"""Whether ``intent`` already ran out of Metal memory twice (no more replays)."""terminal = getattr(self, "_metal_oom_terminal_key", None)diff --git a/studio/backend/tests/test_llama_cpp_gpu_oom_respawn.py b/studio/backend/tests/test_llama_cpp_gpu_oom_respawn.pynew file mode 100644index 00000000000..050e8abb55c--- /dev/null+++ b/studio/backend/tests/test_llama_cpp_gpu_oom_respawn.py@@ -0,0 +1,176 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A served llama-server that dies of a GPU OOM is respawned with more --fit-target (#16177)."""++from __future__ import annotations++import subprocess+import sys+import threading+from unittest.mock import patch++import pytest++import core.inference.llama_cpp as llama_cpp+from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend++from . import _sidecar_fit_helpers as h++# The reporter's crash (A40, Windows, --moe-cache-mib auto, --fit-target 563, ~49K tokens in).+READY = "3.49.882.278 I srv server_run_i: model loaded"+CUDA_OOM = [+ "118.04.263.283 E CUDA error: out of memory",+ "118.04.263.313 E D:\\a\\llama.cpp\\llama.cpp\\src\\ggml\\src\\ggml-cuda\\ggml-cuda.cu:113: CUDA error",+ " current device: 0, in function ggml_cuda_graph_launch_chunk",+]+OTHER_CRASH = ["5.00.000.000 E GGML_ASSERT(n_tokens > 0) failed"]++[email protected](autouse = True)+def _studio_home(tmp_path, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio_home"))+++def _dead_child(lines):+ code = "".join(f"print({line!r}, flush = True)\n" for line in lines)+ proc = subprocess.Popen(+ [sys.executable, "-c", code or "pass"], stdout = subprocess.PIPE, text = True+ )+ proc.wait(timeout = 30)+ return proc+++def _crashed_backend(lines, intent):+ backend = LlamaCppBackend()+ backend._process = _dead_child(lines)+ backend._stdout_thread = threading.Thread(target = backend._drain_stdout, daemon = True)+ backend._stdout_thread.start()+ backend._healthy = True+ backend._model_identifier = "unsloth/Qwen3.8-Flash-Next-GGUF"+ backend._last_load_intent = intent+ backend._server_socket_is_open = lambda: False+ loads = []++ def fake_load_model(new_intent):+ loads.append(new_intent)+ backend._process = _dead_child([READY])+ backend._stdout_thread = None+ return True++ backend.load_model = fake_load_model+ return backend, loads+++INTENT = GgufLoadIntent(gguf_path = "/m.gguf", model_identifier = "m", n_ctx = 191360)+++def test_served_cuda_oom_records_a_strike_for_the_respawn():+ backend, loads = _crashed_backend([READY, *CUDA_OOM], INTENT)+ assert backend._respawn_if_dead()+ assert loads == [INTENT]+ assert backend._gpu_oom_strikes == (backend._metal_oom_key(INTENT), 1)+++ "lines",+ [[READY, *OTHER_CRASH], CUDA_OOM, [READY, "5.00.000.000 I prompt: CUDA error: out of memory"]],+ ids = ["other-crash", "startup-oom", "info-line-echo"],+)+def test_no_strike_without_a_served_gpu_oom(lines):+ backend, loads = _crashed_backend(lines, INTENT)+ assert backend._respawn_if_dead()+ assert len(loads) == 1+ assert backend._gpu_oom_strikes is None+++def test_strikes_grow_per_crash_of_the_same_load_and_cap():+ backend = LlamaCppBackend()+ backend._model_identifier = "m"+ for _ in range(5):+ backend._note_gpu_oom_strike(INTENT)+ assert backend._gpu_oom_strikes[1] == llama_cpp._GPU_OOM_HEADROOM_MAX_STRIKES+ other = GgufLoadIntent(gguf_path = "/m.gguf", model_identifier = "m", n_ctx = 65536)+ backend._note_gpu_oom_strike(other)+ assert backend._gpu_oom_strikes == (backend._metal_oom_key(other), 1)+ # A speculative-mode change keeps the strike: it is the same placement.+ off = GgufLoadIntent(+ gguf_path = "/m.gguf", model_identifier = "m", n_ctx = 65536, speculative_type = "off"+ )+ assert backend._gpu_oom_headroom_mib(off) == llama_cpp._GPU_OOM_HEADROOM_STEP_MIB+ assert backend._gpu_oom_headroom_mib(INTENT) == 0.0+++def _spawn_cmd(backend, intent):+ captured = {}++ def fake_popen(cmd, **kwargs):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return h.placement._REAL_POPEN(cmd, **kwargs)+ captured["cmd"] = list(cmd)+ return type(+ "Process",+ (),+ {+ "pid": 2**31 - 1,+ "stdout": (),+ "poll": lambda self: None,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: 0,+ "kill": lambda self: None,+ },+ )()++ with patch.object(subprocess, "Popen", side_effect = fake_popen):+ assert backend.load_model(intent)+ return captured["cmd"]+++ "vram_fraction, before, after",+ [(0.97, None, "2048"), (0.98, "563", "1587")],+ ids = ["default-budget", "reporter-98pct"],+)+def test_respawn_after_gpu_oom_keeps_more_free_vram(+ tmp_path, monkeypatch, vram_fraction, before, after+):+ monkeypatch.setattr(llama_cpp, "_active_vram_fraction", lambda: vram_fraction)+ backend, gguf = h.cuda_backend(tmp_path, memory = ((0, 46_000, 46_000),))+ intent = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 8192)+ assert h.last(_spawn_cmd(backend, intent), "--fit-target") == before++ backend, gguf = h.cuda_backend(tmp_path, memory = ((0, 46_000, 46_000),))+ backend._model_identifier = "test"+ backend._note_gpu_oom_strike(intent)+ with llama_cpp._automatic_recovery_load():+ assert h.last(_spawn_cmd(backend, intent), "--fit-target") == after+ # The recovery load keeps the strike; an unload starts over.+ assert backend._gpu_oom_strikes is not None+ backend.unload_model()+ assert backend._gpu_oom_strikes is None+++def test_mixed_cards_keep_their_own_margins_plus_the_headroom(tmp_path, monkeypatch):+ B = LlamaCppBackend+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ for k in ("CUDA_VISIBLE_DEVICES", "LLAMA_ARG_DEVICE", "LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_RPC"):+ monkeypatch.delenv(k, raising = False)+ monkeypatch.setattr(llama_cpp, "_metal_capable_host", lambda: False)+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", True)+ monkeypatch.setattr(B, "_smi_uuid_mask_order", staticmethod(lambda: None))+ monkeypatch.setattr(B, "_visibility_mask_is_unmappable", staticmethod(lambda: False))+ monkeypatch.setattr(B, "_SMI_MIG_INDICES", frozenset())+ monkeypatch.setattr(B, "_host_torch_is_rocm", staticmethod(lambda: False))+ monkeypatch.setattr(llama_cpp, "_active_vram_fraction", lambda: 0.8)+ memory = ((0, 80_000, 80_000), (1, 8_000, 8_000))+ backend, gguf = h.cards(tmp_path, memory = memory)+ intent = GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 8192)+ before = h.margins(_spawn_cmd(backend, intent))++ backend, gguf = h.cards(tmp_path, memory = memory)+ backend._model_identifier = "test"+ backend._note_gpu_oom_strike(intent)+ with llama_cpp._automatic_recovery_load():+ after = h.margins(_spawn_cmd(backend, intent))+ assert len(before) == 2+ assert after == [m + 1024 for m in before]