Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fit a GGUF reload only after the stopped llama-server's VRAM shows as free (#17345)

oobaboogad4c684edfbe0
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 91c4f16394d..9ecfe3d65ef 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -8660,6 +8660,8 @@ class LlamaCppBackend:
self._mmproj_accepts_image: bool = True
# Set in _kill_process (monotonic); load_model reads it to decide whether to wait for VRAM reclaim.
self._last_kill_monotonic: float = 0.0
+ # {GPU: (free MiB, child's weights MiB)} read just before the last kill.
+ self._last_kill_vram_baseline: Optional[dict[int, tuple[int, float]]] = None
self._planned_vram_mib: dict[int, int] = {}
self._pending_plan_mib: dict[int, int] = {}
@@ -16431,14 +16433,65 @@ class LlamaCppBackend:
# Skip the wait when the last kill is older than this; the driver has already reclaimed it.
_VRAM_SETTLE_WINDOW_S: float = 15.0
+ # Weights under this on a GPU cannot be told from noise in its free-VRAM figure.
+ _VRAM_RELEASE_MIN_HELD_MIB: float = 512.0
+ # Windows (WDDM) reports a dead child's VRAM as free after it exits. A bound, not a measurement.
+ _VRAM_RELEASE_WAIT_S: float = 10.0
+
+ @staticmethod
+ def _free_vram_counts_only_this_process() -> bool:
+ """Windows ROCm free VRAM excludes other processes; checks torch too, as IS_ROCM is set late."""
+ if sys.platform != "win32":
+ return False
+ from utils.hardware import rocm_windows_free_is_untrusted
+
+ if rocm_windows_free_is_untrusted():
+ return True
+ torch = sys.modules.get("torch")
+ return torch is not None and LlamaCppBackend._torch_is_rocm(torch)
+
+ def _vram_baseline_before_kill(self) -> Optional[dict[int, tuple[int, float]]]:
+ """{GPU: (free MiB, child's weights MiB)} while the child lives; None if any GPU's reading
+ does not count those weights as used (cannot show their release) or on Windows ROCm."""
+ try:
+ if self._free_vram_counts_only_this_process():
+ return None
+ held = {
+ idx: float(gb) * 1024.0
+ for idx, gb in (self.reclaimable_gpu_memory_gb() or {}).items()
+ if float(gb) * 1024.0 >= self._VRAM_RELEASE_MIN_HELD_MIB
+ }
+ if not held:
+ return None
+ from utils.hardware import gpu_query
+
+ with gpu_query.fresh_reads():
+ readings = {
+ i: (free, total) for i, free, total in LlamaCppBackend._get_gpu_memory()
+ }
+ # Again: the read itself may be what imported torch.
+ if self._free_vram_counts_only_this_process():
+ return None
+ baseline: dict[int, tuple[int, float]] = {}
+ for idx, held_mib in held.items():
+ free, total = readings.get(idx, (0, 0))
+ if total <= 0 or total - free < held_mib / 2.0:
+ return None
+ baseline[idx] = (int(free), held_mib)
+ return baseline
+ except Exception:
+ return None
+
@staticmethod
def _wait_for_vram_settle(
max_wait: float = 2.0,
interval: float = 0.25,
tolerance_mib: int = 256,
since_kill: float = 0.0,
+ baseline: Optional[dict[int, tuple[int, float]]] = None,
) -> None:
- """Polls until VRAM settles after a kill; skips on cold start, stale kill, CPU-only, probe errors, GPU-set change; max_wait caps a wedged nvidia-smi."""
+ """Polls until VRAM settles after a kill; skips on cold start, stale kill, CPU-only, probe errors, GPU-set change; max_wait caps a wedged nvidia-smi.
+ With ``baseline``, samples count only once the child's VRAM shows freed (up to ``_VRAM_RELEASE_WAIT_S``)."""
now = time.monotonic()
if since_kill <= 0.0:
return
@@ -16458,18 +16511,40 @@ class LlamaCppBackend:
except Exception:
return None
+ def _released(sample) -> bool:
+ # Per GPU; half the weights, since another tenant may take some of the freed memory.
+ free = dict(sample)
+ return all(
+ free.get(idx, before) - before >= held / 2.0
+ for idx, (before, held) in baseline.items()
+ )
+
prev = _probe_or_none()
if prev is None or not prev:
return
+ # A GPU the probe no longer lists cannot be tracked.
+ awaiting_release = (
+ bool(baseline) and set(baseline) <= {idx for idx, _free in prev} and not _released(prev)
+ )
+ if awaiting_release:
+ deadline = now + max(max_wait, LlamaCppBackend._VRAM_RELEASE_WAIT_S)
while time.monotonic() < deadline:
remaining = deadline - time.monotonic()
if remaining <= 0:
- return
+ break
# Clip the nap so a near-zero ``max_wait`` is respected.
time.sleep(min(interval, remaining))
+ if awaiting_release and time.monotonic() >= deadline:
+ break
curr = _probe_or_none()
if curr is None or not curr or len(curr) != len(prev):
return
+ if awaiting_release:
+ if _released(curr):
+ awaiting_release = False
+ deadline = time.monotonic() + max_wait
+ prev = curr
+ continue
prev_map = dict(prev)
stable = True
for idx, free in curr:
@@ -16485,6 +16560,13 @@ class LlamaCppBackend:
if stable:
return
prev = curr
+ if awaiting_release:
+ logger.warning(
+ "Free VRAM did not rise within %.1f s of stopping the previous llama-server "
+ "(it held about %.0f MiB); the GPU fit that follows uses the reported figure.",
+ max(max_wait, LlamaCppBackend._VRAM_RELEASE_WAIT_S),
+ sum(held for _before, held in baseline.values()),
+ )
# Free-VRAM fraction at which Unsloth pins the GPU instead of deferring to --fit on.
# Guards fragmentation and per-device context; 0.90 sent 91-94% fits to CPU offload.
@@ -24304,7 +24386,10 @@ class LlamaCppBackend:
self._vram_fraction_launched = None
# Outside the lock, as for llama-server: the Auto layer fit reads free VRAM,
# which a just-killed runner still holds until the driver reclaims it.
- self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)
+ self._wait_for_vram_settle(
+ since_kill = self._last_kill_monotonic,
+ baseline = getattr(self, "_last_kill_vram_baseline", None),
+ )
with self._lock:
if _load_cancelled():
logger.info("Load cancelled before diffusion server start")
@@ -24340,7 +24425,10 @@ class LlamaCppBackend:
self._resumes_thoughts = self.resumes_thoughts(binary)
# Outside self._lock so /unload, /cancel, /status aren't blocked.
- self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)
+ self._wait_for_vram_settle(
+ since_kill = self._last_kill_monotonic,
+ baseline = getattr(self, "_last_kill_vram_baseline", None),
+ )
# ── Phase 3: start llama-server (under lock) ──────────────
with self._lock:
@@ -34568,6 +34656,9 @@ class LlamaCppBackend:
"""Read both before terminate: getpgid fails once the leader is reaped and children reparent on exit; the visual server must be named while the link exists"""
self._stop_mtp_crash_watchdog()
self._reset_effective_parallel_slots()
+ if self._process is not None:
+ # While the child still holds its VRAM and its GPU ids; a shutdown has no load to plan for.
+ self._last_kill_vram_baseline = None if teardown else self._vram_baseline_before_kill()
# _process may be a test stand-in lacking terminate(); skip signalling, fall through to finally.
self._diffusion_requested_ngl = None
self._child_gpu_physical_ids = None
diff --git a/studio/backend/tests/test_diffusion_auto_fit_ngl.py b/studio/backend/tests/test_diffusion_auto_fit_ngl.py
index 6f425bafd51..96c430c384b 100644
--- a/studio/backend/tests/test_diffusion_auto_fit_ngl.py
+++ b/studio/backend/tests/test_diffusion_auto_fit_ngl.py
@@ -131,7 +131,7 @@ def test_diffusion_load_settles_vram_outside_the_lock():
src = inspect.getsource(LlamaCppBackend.load_model)
branch = src[src.index("if self._is_diffusion:") :]
- settle = branch.index("self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)")
+ settle = branch.index("self._wait_for_vram_settle(")
assert settle < branch.index("with self._lock:") < branch.index("_start_diffusion_server(")
diff --git a/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py b/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py
index a4ce16bbbfc..d7ab81a055a 100644
--- a/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py
+++ b/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py
@@ -871,3 +871,318 @@ def test_kill_orphaned_servers_procfs_still_kills_the_same_process(tmp_path):
assert killed == [mypid + 1]
assert n == 1
+
+
+_HELD = 16000.0 # MiB of weights the killed child had on a GPU
+_BEFORE = [(0, 4000)] # free while it was alive
+_AFTER = [(0, 20000)] # free once the driver reports the release
+_BASELINE = {0: (4000, _HELD)}
+
+
+class _Warnings:
+ """Stands in for the module logger; structlog loggers are not caplog's to capture."""
+
+ def __init__(self):
+ self.messages: list[str] = []
+
+ def warning(self, msg, *args, **_kw):
+ self.messages.append(msg % args if args else msg)
+
+ def __getattr__(self, _name):
+ return lambda *a, **k: None
+
+
+def _settle(
+ samples,
+ baseline,
+ *,
+ interval = 0.01,
+):
+ ctx, state = _patch_probe(samples)
+ with ctx, _Sleeps() as sleeps:
+ LlamaCppBackend._wait_for_vram_settle(
+ **_kw(max_wait = 2.0, interval = interval, baseline = baseline)
+ )
+ return state["calls"], len(sleeps.durations)
+
+
+def test_without_a_baseline_a_stale_stable_reading_still_ends_the_wait():
+ assert _settle([_BEFORE, _BEFORE, _AFTER], None) == (2, 1)
+
+
+def test_a_stale_stable_reading_waits_for_the_release():
+ assert _settle([_BEFORE, _BEFORE, _BEFORE, _AFTER, _AFTER], _BASELINE) == (5, 4)
+
+
+def test_a_release_already_reported_costs_the_same_single_poll():
+ assert _settle([_AFTER, _AFTER], _BASELINE) == (2, 1)
+
+
+def test_a_partial_release_is_not_the_release():
+ partial = [(0, 9000)]
+ assert _settle([_BEFORE, partial, partial, _AFTER, _AFTER], _BASELINE)[0] == 5
+
+
+def test_a_gpu_that_reports_late_is_not_covered_by_one_that_already_did():
+ baseline = {0: (2000, _HELD), 1: (2000, _HELD)}
+ before = [(0, 2000), (1, 2000)]
+ first_only = [(0, 18000), (1, 2000)]
+ after = [(0, 18000), (1, 18000)]
+ assert _settle([before, first_only, first_only, after, after], baseline)[0] == 5
+
+
+def test_a_gpu_the_child_did_not_occupy_is_not_waited_on():
+ before = [(0, 4000), (1, 9000)]
+ after = [(0, 20000), (1, 9000)]
+ assert _settle([before, before, after, after], _BASELINE)[0] == 4
+
+
+def test_a_baseline_gpu_the_probe_no_longer_lists_falls_back_to_the_plain_settle():
+ other = [(1, 9000)]
+ assert _settle([other, other, other], _BASELINE) == (2, 1)
+
+
+def test_a_release_that_never_shows_stops_at_the_bound_and_says_so():
+ log = _Warnings()
+ ctx, state = _patch_probe([_BEFORE])
+ with (
+ _Clock() as clock,
+ ctx,
+ _Sleeps(clock) as sleeps,
+ patch.object(llama_cpp_module, "logger", log),
+ ):
+ LlamaCppBackend._wait_for_vram_settle(
+ **_kw(max_wait = 2.0, interval = 0.25, baseline = _BASELINE)
+ )
+ assert sleeps.total == pytest.approx(LlamaCppBackend._VRAM_RELEASE_WAIT_S)
+ assert state["calls"] >= 2
+ assert len(log.messages) == 1 and "did not rise" in log.messages[0]
+
+
+def test_the_settle_budget_restarts_when_the_release_shows():
+ """A release near the end of the bound still gets its stable read."""
+ draining = [(0, 18000)]
+ log = _Warnings()
+ stale = int(LlamaCppBackend._VRAM_RELEASE_WAIT_S / 0.25) - 1
+ samples = [_BEFORE] * stale + [draining, _AFTER, _AFTER]
+ ctx, state = _patch_probe(samples)
+ with _Clock() as clock, ctx, _Sleeps(clock), patch.object(llama_cpp_module, "logger", log):
+ LlamaCppBackend._wait_for_vram_settle(
+ **_kw(max_wait = 2.0, interval = 0.25, baseline = _BASELINE)
+ )
+ assert state["calls"] == len(samples)
+ assert log.messages == []
+
+
+class _LiveThenDead:
+ def __init__(self, order):
+ self._order = order
+ self._dead = False
+
+ def poll(self):
+ return 0 if self._dead else None
+
+ def terminate(self):
+ self._order.append("terminate")
+ self._dead = True
+
+ def wait(self, timeout = None):
+ return 0
+
+ def kill(self):
+ self._dead = True
+
+
+def _killable_backend(order, buffers_mib):
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._process = _LiveThenDead(order)
+ backend._healthy = True
+ backend._stats_logger = None
+ backend._stdout_thread = None
+ backend._llama_log_fh = None
+ backend._last_kill_monotonic = 0.0
+ backend._last_kill_vram_baseline = None
+ backend._child_gpu_physical_ids = tuple(range(len(buffers_mib)))
+ backend._stdout_lines = [
+ f"load_tensors: CUDA{i} model buffer size = {mib:.2f} MiB"
+ for i, mib in enumerate(buffers_mib)
+ ]
+ return backend
+
+
+def _ordered_memory(order, rows):
+ def _probe(*_a, **_k):
+ order.append("probe")
+ return rows
+
+ return patch.object(LlamaCppBackend, "_get_gpu_memory", staticmethod(_probe))
+
+
+def test_kill_reads_free_vram_while_a_gpu_child_is_still_alive():
+ order: list[str] = []
+ backend = _killable_backend(order, [_HELD])
+ with _ordered_memory(order, [(0, 4000, 24000)]):
+ backend._kill_process()
+ assert order == ["probe", "terminate"]
+ assert backend._last_kill_vram_baseline == {0: (4000, _HELD)}
+
+
+def test_kill_keeps_a_baseline_per_gpu_the_child_occupies():
+ order: list[str] = []
+ backend = _killable_backend(order, [_HELD, 12000.0])
+ with _ordered_memory(order, [(0, 4000, 24000), (1, 10000, 24000), (2, 23000, 24000)]):
+ backend._kill_process()
+ assert backend._last_kill_vram_baseline == {0: (4000, _HELD), 1: (10000, 12000.0)}
+
+
+ "rows",
+ [
+ [(0, 24000, 24000)],
+ [(0, 20000, 24000)],
+ [(0, 4000, 0)],
+ [(1, 4000, 24000)],
+ ],
+ ids = ["ceiling", "own-allocations-only", "unknown-total", "gpu-not-listed"],
+)
+def test_a_reading_that_does_not_count_the_childs_weights_is_no_baseline(rows):
+ order: list[str] = []
+ backend = _killable_backend(order, [_HELD])
+ with _ordered_memory(order, rows):
+ backend._kill_process()
+ assert order == ["probe", "terminate"]
+ assert backend._last_kill_vram_baseline is None
+
+
[email protected]("untrusted", [False, True], ids = ["live-figure", "windows-rocm"])
+def test_windows_rocm_takes_no_baseline_even_when_studios_own_vram_passes_the_check(untrusted):
+ """Studio's own 6 GiB on a 24 GiB card passes the used check for 8 GiB of child weights."""
+ order: list[str] = []
+ backend = _killable_backend(order, [8192.0])
+ with (
+ _ordered_memory(order, [(0, 18432, 24576)]),
+ patch.object(
+ LlamaCppBackend,
+ "_free_vram_counts_only_this_process",
+ staticmethod(lambda: untrusted),
+ ),
+ ):
+ backend._kill_process()
+ if untrusted:
+ assert order == ["terminate"]
+ assert backend._last_kill_vram_baseline is None
+ else:
+ assert backend._last_kill_vram_baseline == {0: (18432, 8192.0)}
+
+
+def test_a_rocm_torch_imported_by_the_read_itself_still_drops_the_baseline():
+ """Before hardware detection the first check passes; the read then brings torch in."""
+ order: list[str] = []
+ backend = _killable_backend(order, [8192.0])
+ answers = iter([False, True])
+ with (
+ _ordered_memory(order, [(0, 18432, 24576)]),
+ patch.object(
+ LlamaCppBackend,
+ "_free_vram_counts_only_this_process",
+ staticmethod(lambda: next(answers)),
+ ),
+ ):
+ backend._kill_process()
+ assert order == ["probe", "terminate"]
+ assert backend._last_kill_vram_baseline is None
+
+
+ "platform, is_rocm, torch_hip, expected",
+ [
+ ("win32", True, None, True),
+ ("win32", False, "6.4", True),
+ ("win32", False, None, False),
+ ("linux", True, "6.4", False),
+ ],
+ ids = ["detected-rocm", "rocm-torch-before-detection", "windows-cuda", "linux-rocm"],
+)
+def test_only_windows_rocm_free_vram_leaves_the_child_out(
+ monkeypatch, platform, is_rocm, torch_hip, expected
+):
+ import utils.hardware as hardware
+
+ fake_torch = _types.SimpleNamespace(
+ version = _types.SimpleNamespace(hip = torch_hip), __version__ = "2.14.1"
+ )
+ monkeypatch.setattr(sys, "platform", platform)
+ monkeypatch.setattr(hardware, "rocm_windows_free_is_untrusted", lambda: is_rocm)
+ monkeypatch.setitem(sys.modules, "torch", fake_torch)
+ assert LlamaCppBackend._free_vram_counts_only_this_process() is expected
+
+
+def test_one_untracked_gpu_drops_the_whole_baseline():
+ order: list[str] = []
+ backend = _killable_backend(order, [_HELD, _HELD])
+ with _ordered_memory(order, [(0, 4000, 24000), (1, 24000, 24000)]):
+ backend._kill_process()
+ assert backend._last_kill_vram_baseline is None
+
+
[email protected]("buffers", [[], [100.0]], ids = ["no-gpu-weights", "tiny"])
+def test_kill_does_not_probe_for_a_child_with_no_gpu_weights_to_see(buffers):
+ order: list[str] = []
+ backend = _killable_backend(order, buffers)
+ with _ordered_memory(order, [(0, 4000, 24000)]):
+ backend._kill_process()
+ assert order == ["terminate"]
+ assert backend._last_kill_vram_baseline is None
+
+
+def test_kill_takes_no_baseline_on_shutdown_or_for_an_exited_child():
+ order: list[str] = []
+ backend = _killable_backend(order, [_HELD])
+ backend._teardown_lock = __import__("threading").RLock()
+ backend._spawn_lock = __import__("threading").RLock()
+ backend._last_kill_vram_baseline = dict(_BASELINE)
+ with (
+ _ordered_memory(order, [(0, 4000, 24000)]),
+ patch("utils.process_lifetime.mark_process_shutting_down", lambda: None),
+ ):
+ backend._kill_process(teardown = True)
+ assert "probe" not in order
+ assert backend._last_kill_vram_baseline is None
+
+ order.clear()
+ backend = _killable_backend(order, [_HELD])
+ backend._process._dead = True # crashed before the kill: its free figure is no baseline
+ with _ordered_memory(order, [(0, 4000, 24000)]):
+ backend._kill_process()
+ assert "probe" not in order
+ assert backend._last_kill_vram_baseline is None
+
+
+def test_a_kill_with_no_process_keeps_the_baseline_of_the_kill_before_it():
+ backend = _killable_backend([], [_HELD])
+ backend._process = None
+ backend._last_kill_vram_baseline = dict(_BASELINE)
+ backend._kill_process()
+ assert backend._last_kill_vram_baseline == _BASELINE
+
+
+def test_a_failed_baseline_read_leaves_the_kill_and_the_wait_as_before():
+ order: list[str] = []
+ backend = _killable_backend(order, [_HELD])
+
+ def _boom(*_a, **_k):
+ raise RuntimeError("nvidia-smi wedged")
+
+ with patch.object(LlamaCppBackend, "_get_gpu_memory", staticmethod(_boom)):
+ backend._kill_process()
+ assert order == ["terminate"]
+ assert backend._last_kill_vram_baseline is None
+ assert backend._process is None
+
+
+def test_both_load_paths_hand_the_baseline_to_the_wait():
+ import inspect
+
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ assert src.count("_wait_for_vram_settle(") == 2
+ assert src.count('baseline = getattr(self, "_last_kill_vram_baseline", None)') == 2