Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fit a GGUF reload only after the stopped llama-server's VRAM shows as free (#17345)
oobaboogad4c684edfbe0
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 91c4f16394d..9ecfe3d65ef 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -8660,6 +8660,8 @@ class LlamaCppBackend:self._mmproj_accepts_image: bool = True# Set in _kill_process (monotonic); load_model reads it to decide whether to wait for VRAM reclaim.self._last_kill_monotonic: float = 0.0+ # {GPU: (free MiB, child's weights MiB)} read just before the last kill.+ self._last_kill_vram_baseline: Optional[dict[int, tuple[int, float]]] = Noneself._planned_vram_mib: dict[int, int] = {}self._pending_plan_mib: dict[int, int] = {}@@ -16431,14 +16433,65 @@ class LlamaCppBackend:# Skip the wait when the last kill is older than this; the driver has already reclaimed it._VRAM_SETTLE_WINDOW_S: float = 15.0+ # Weights under this on a GPU cannot be told from noise in its free-VRAM figure.+ _VRAM_RELEASE_MIN_HELD_MIB: float = 512.0+ # Windows (WDDM) reports a dead child's VRAM as free after it exits. A bound, not a measurement.+ _VRAM_RELEASE_WAIT_S: float = 10.0++ @staticmethod+ def _free_vram_counts_only_this_process() -> bool:+ """Windows ROCm free VRAM excludes other processes; checks torch too, as IS_ROCM is set late."""+ if sys.platform != "win32":+ return False+ from utils.hardware import rocm_windows_free_is_untrusted++ if rocm_windows_free_is_untrusted():+ return True+ torch = sys.modules.get("torch")+ return torch is not None and LlamaCppBackend._torch_is_rocm(torch)++ def _vram_baseline_before_kill(self) -> Optional[dict[int, tuple[int, float]]]:+ """{GPU: (free MiB, child's weights MiB)} while the child lives; None if any GPU's reading+ does not count those weights as used (cannot show their release) or on Windows ROCm."""+ try:+ if self._free_vram_counts_only_this_process():+ return None+ held = {+ idx: float(gb) * 1024.0+ for idx, gb in (self.reclaimable_gpu_memory_gb() or {}).items()+ if float(gb) * 1024.0 >= self._VRAM_RELEASE_MIN_HELD_MIB+ }+ if not held:+ return None+ from utils.hardware import gpu_query++ with gpu_query.fresh_reads():+ readings = {+ i: (free, total) for i, free, total in LlamaCppBackend._get_gpu_memory()+ }+ # Again: the read itself may be what imported torch.+ if self._free_vram_counts_only_this_process():+ return None+ baseline: dict[int, tuple[int, float]] = {}+ for idx, held_mib in held.items():+ free, total = readings.get(idx, (0, 0))+ if total <= 0 or total - free < held_mib / 2.0:+ return None+ baseline[idx] = (int(free), held_mib)+ return baseline+ except Exception:+ return None+@staticmethoddef _wait_for_vram_settle(max_wait: float = 2.0,interval: float = 0.25,tolerance_mib: int = 256,since_kill: float = 0.0,+ baseline: Optional[dict[int, tuple[int, float]]] = None,) -> None:- """Polls until VRAM settles after a kill; skips on cold start, stale kill, CPU-only, probe errors, GPU-set change; max_wait caps a wedged nvidia-smi."""+ """Polls until VRAM settles after a kill; skips on cold start, stale kill, CPU-only, probe errors, GPU-set change; max_wait caps a wedged nvidia-smi.+ With ``baseline``, samples count only once the child's VRAM shows freed (up to ``_VRAM_RELEASE_WAIT_S``)."""now = time.monotonic()if since_kill <= 0.0:return@@ -16458,18 +16511,40 @@ class LlamaCppBackend:except Exception:return None+ def _released(sample) -> bool:+ # Per GPU; half the weights, since another tenant may take some of the freed memory.+ free = dict(sample)+ return all(+ free.get(idx, before) - before >= held / 2.0+ for idx, (before, held) in baseline.items()+ )+prev = _probe_or_none()if prev is None or not prev:return+ # A GPU the probe no longer lists cannot be tracked.+ awaiting_release = (+ bool(baseline) and set(baseline) <= {idx for idx, _free in prev} and not _released(prev)+ )+ if awaiting_release:+ deadline = now + max(max_wait, LlamaCppBackend._VRAM_RELEASE_WAIT_S)while time.monotonic() < deadline:remaining = deadline - time.monotonic()if remaining <= 0:- return+ break# Clip the nap so a near-zero ``max_wait`` is respected.time.sleep(min(interval, remaining))+ if awaiting_release and time.monotonic() >= deadline:+ breakcurr = _probe_or_none()if curr is None or not curr or len(curr) != len(prev):return+ if awaiting_release:+ if _released(curr):+ awaiting_release = False+ deadline = time.monotonic() + max_wait+ prev = curr+ continueprev_map = dict(prev)stable = Truefor idx, free in curr:@@ -16485,6 +16560,13 @@ class LlamaCppBackend:if stable:returnprev = curr+ if awaiting_release:+ logger.warning(+ "Free VRAM did not rise within %.1f s of stopping the previous llama-server "+ "(it held about %.0f MiB); the GPU fit that follows uses the reported figure.",+ max(max_wait, LlamaCppBackend._VRAM_RELEASE_WAIT_S),+ sum(held for _before, held in baseline.values()),+ )# Free-VRAM fraction at which Unsloth pins the GPU instead of deferring to --fit on.# Guards fragmentation and per-device context; 0.90 sent 91-94% fits to CPU offload.@@ -24304,7 +24386,10 @@ class LlamaCppBackend:self._vram_fraction_launched = None# Outside the lock, as for llama-server: the Auto layer fit reads free VRAM,# which a just-killed runner still holds until the driver reclaims it.- self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)+ self._wait_for_vram_settle(+ since_kill = self._last_kill_monotonic,+ baseline = getattr(self, "_last_kill_vram_baseline", None),+ )with self._lock:if _load_cancelled():logger.info("Load cancelled before diffusion server start")@@ -24340,7 +24425,10 @@ class LlamaCppBackend:self._resumes_thoughts = self.resumes_thoughts(binary)# Outside self._lock so /unload, /cancel, /status aren't blocked.- self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)+ self._wait_for_vram_settle(+ since_kill = self._last_kill_monotonic,+ baseline = getattr(self, "_last_kill_vram_baseline", None),+ )# ── Phase 3: start llama-server (under lock) ──────────────with self._lock:@@ -34568,6 +34656,9 @@ class LlamaCppBackend:"""Read both before terminate: getpgid fails once the leader is reaped and children reparent on exit; the visual server must be named while the link exists"""self._stop_mtp_crash_watchdog()self._reset_effective_parallel_slots()+ if self._process is not None:+ # While the child still holds its VRAM and its GPU ids; a shutdown has no load to plan for.+ self._last_kill_vram_baseline = None if teardown else self._vram_baseline_before_kill()# _process may be a test stand-in lacking terminate(); skip signalling, fall through to finally.self._diffusion_requested_ngl = Noneself._child_gpu_physical_ids = Nonediff --git a/studio/backend/tests/test_diffusion_auto_fit_ngl.py b/studio/backend/tests/test_diffusion_auto_fit_ngl.pyindex 6f425bafd51..96c430c384b 100644--- a/studio/backend/tests/test_diffusion_auto_fit_ngl.py+++ b/studio/backend/tests/test_diffusion_auto_fit_ngl.py@@ -131,7 +131,7 @@ def test_diffusion_load_settles_vram_outside_the_lock():src = inspect.getsource(LlamaCppBackend.load_model)branch = src[src.index("if self._is_diffusion:") :]- settle = branch.index("self._wait_for_vram_settle(since_kill = self._last_kill_monotonic)")+ settle = branch.index("self._wait_for_vram_settle(")assert settle < branch.index("with self._lock:") < branch.index("_start_diffusion_server(")diff --git a/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py b/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.pyindex a4ce16bbbfc..d7ab81a055a 100644--- a/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py+++ b/studio/backend/tests/test_llama_cpp_wait_for_vram_settle.py@@ -871,3 +871,318 @@ def test_kill_orphaned_servers_procfs_still_kills_the_same_process(tmp_path):assert killed == [mypid + 1]assert n == 1+++_HELD = 16000.0 # MiB of weights the killed child had on a GPU+_BEFORE = [(0, 4000)] # free while it was alive+_AFTER = [(0, 20000)] # free once the driver reports the release+_BASELINE = {0: (4000, _HELD)}+++class _Warnings:+ """Stands in for the module logger; structlog loggers are not caplog's to capture."""++ def __init__(self):+ self.messages: list[str] = []++ def warning(self, msg, *args, **_kw):+ self.messages.append(msg % args if args else msg)++ def __getattr__(self, _name):+ return lambda *a, **k: None+++def _settle(+ samples,+ baseline,+ *,+ interval = 0.01,+):+ ctx, state = _patch_probe(samples)+ with ctx, _Sleeps() as sleeps:+ LlamaCppBackend._wait_for_vram_settle(+ **_kw(max_wait = 2.0, interval = interval, baseline = baseline)+ )+ return state["calls"], len(sleeps.durations)+++def test_without_a_baseline_a_stale_stable_reading_still_ends_the_wait():+ assert _settle([_BEFORE, _BEFORE, _AFTER], None) == (2, 1)+++def test_a_stale_stable_reading_waits_for_the_release():+ assert _settle([_BEFORE, _BEFORE, _BEFORE, _AFTER, _AFTER], _BASELINE) == (5, 4)+++def test_a_release_already_reported_costs_the_same_single_poll():+ assert _settle([_AFTER, _AFTER], _BASELINE) == (2, 1)+++def test_a_partial_release_is_not_the_release():+ partial = [(0, 9000)]+ assert _settle([_BEFORE, partial, partial, _AFTER, _AFTER], _BASELINE)[0] == 5+++def test_a_gpu_that_reports_late_is_not_covered_by_one_that_already_did():+ baseline = {0: (2000, _HELD), 1: (2000, _HELD)}+ before = [(0, 2000), (1, 2000)]+ first_only = [(0, 18000), (1, 2000)]+ after = [(0, 18000), (1, 18000)]+ assert _settle([before, first_only, first_only, after, after], baseline)[0] == 5+++def test_a_gpu_the_child_did_not_occupy_is_not_waited_on():+ before = [(0, 4000), (1, 9000)]+ after = [(0, 20000), (1, 9000)]+ assert _settle([before, before, after, after], _BASELINE)[0] == 4+++def test_a_baseline_gpu_the_probe_no_longer_lists_falls_back_to_the_plain_settle():+ other = [(1, 9000)]+ assert _settle([other, other, other], _BASELINE) == (2, 1)+++def test_a_release_that_never_shows_stops_at_the_bound_and_says_so():+ log = _Warnings()+ ctx, state = _patch_probe([_BEFORE])+ with (+ _Clock() as clock,+ ctx,+ _Sleeps(clock) as sleeps,+ patch.object(llama_cpp_module, "logger", log),+ ):+ LlamaCppBackend._wait_for_vram_settle(+ **_kw(max_wait = 2.0, interval = 0.25, baseline = _BASELINE)+ )+ assert sleeps.total == pytest.approx(LlamaCppBackend._VRAM_RELEASE_WAIT_S)+ assert state["calls"] >= 2+ assert len(log.messages) == 1 and "did not rise" in log.messages[0]+++def test_the_settle_budget_restarts_when_the_release_shows():+ """A release near the end of the bound still gets its stable read."""+ draining = [(0, 18000)]+ log = _Warnings()+ stale = int(LlamaCppBackend._VRAM_RELEASE_WAIT_S / 0.25) - 1+ samples = [_BEFORE] * stale + [draining, _AFTER, _AFTER]+ ctx, state = _patch_probe(samples)+ with _Clock() as clock, ctx, _Sleeps(clock), patch.object(llama_cpp_module, "logger", log):+ LlamaCppBackend._wait_for_vram_settle(+ **_kw(max_wait = 2.0, interval = 0.25, baseline = _BASELINE)+ )+ assert state["calls"] == len(samples)+ assert log.messages == []+++class _LiveThenDead:+ def __init__(self, order):+ self._order = order+ self._dead = False++ def poll(self):+ return 0 if self._dead else None++ def terminate(self):+ self._order.append("terminate")+ self._dead = True++ def wait(self, timeout = None):+ return 0++ def kill(self):+ self._dead = True+++def _killable_backend(order, buffers_mib):+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._process = _LiveThenDead(order)+ backend._healthy = True+ backend._stats_logger = None+ backend._stdout_thread = None+ backend._llama_log_fh = None+ backend._last_kill_monotonic = 0.0+ backend._last_kill_vram_baseline = None+ backend._child_gpu_physical_ids = tuple(range(len(buffers_mib)))+ backend._stdout_lines = [+ f"load_tensors: CUDA{i} model buffer size = {mib:.2f} MiB"+ for i, mib in enumerate(buffers_mib)+ ]+ return backend+++def _ordered_memory(order, rows):+ def _probe(*_a, **_k):+ order.append("probe")+ return rows++ return patch.object(LlamaCppBackend, "_get_gpu_memory", staticmethod(_probe))+++def test_kill_reads_free_vram_while_a_gpu_child_is_still_alive():+ order: list[str] = []+ backend = _killable_backend(order, [_HELD])+ with _ordered_memory(order, [(0, 4000, 24000)]):+ backend._kill_process()+ assert order == ["probe", "terminate"]+ assert backend._last_kill_vram_baseline == {0: (4000, _HELD)}+++def test_kill_keeps_a_baseline_per_gpu_the_child_occupies():+ order: list[str] = []+ backend = _killable_backend(order, [_HELD, 12000.0])+ with _ordered_memory(order, [(0, 4000, 24000), (1, 10000, 24000), (2, 23000, 24000)]):+ backend._kill_process()+ assert backend._last_kill_vram_baseline == {0: (4000, _HELD), 1: (10000, 12000.0)}+++ "rows",+ [+ [(0, 24000, 24000)],+ [(0, 20000, 24000)],+ [(0, 4000, 0)],+ [(1, 4000, 24000)],+ ],+ ids = ["ceiling", "own-allocations-only", "unknown-total", "gpu-not-listed"],+)+def test_a_reading_that_does_not_count_the_childs_weights_is_no_baseline(rows):+ order: list[str] = []+ backend = _killable_backend(order, [_HELD])+ with _ordered_memory(order, rows):+ backend._kill_process()+ assert order == ["probe", "terminate"]+ assert backend._last_kill_vram_baseline is None++[email protected]("untrusted", [False, True], ids = ["live-figure", "windows-rocm"])+def test_windows_rocm_takes_no_baseline_even_when_studios_own_vram_passes_the_check(untrusted):+ """Studio's own 6 GiB on a 24 GiB card passes the used check for 8 GiB of child weights."""+ order: list[str] = []+ backend = _killable_backend(order, [8192.0])+ with (+ _ordered_memory(order, [(0, 18432, 24576)]),+ patch.object(+ LlamaCppBackend,+ "_free_vram_counts_only_this_process",+ staticmethod(lambda: untrusted),+ ),+ ):+ backend._kill_process()+ if untrusted:+ assert order == ["terminate"]+ assert backend._last_kill_vram_baseline is None+ else:+ assert backend._last_kill_vram_baseline == {0: (18432, 8192.0)}+++def test_a_rocm_torch_imported_by_the_read_itself_still_drops_the_baseline():+ """Before hardware detection the first check passes; the read then brings torch in."""+ order: list[str] = []+ backend = _killable_backend(order, [8192.0])+ answers = iter([False, True])+ with (+ _ordered_memory(order, [(0, 18432, 24576)]),+ patch.object(+ LlamaCppBackend,+ "_free_vram_counts_only_this_process",+ staticmethod(lambda: next(answers)),+ ),+ ):+ backend._kill_process()+ assert order == ["probe", "terminate"]+ assert backend._last_kill_vram_baseline is None+++ "platform, is_rocm, torch_hip, expected",+ [+ ("win32", True, None, True),+ ("win32", False, "6.4", True),+ ("win32", False, None, False),+ ("linux", True, "6.4", False),+ ],+ ids = ["detected-rocm", "rocm-torch-before-detection", "windows-cuda", "linux-rocm"],+)+def test_only_windows_rocm_free_vram_leaves_the_child_out(+ monkeypatch, platform, is_rocm, torch_hip, expected+):+ import utils.hardware as hardware++ fake_torch = _types.SimpleNamespace(+ version = _types.SimpleNamespace(hip = torch_hip), __version__ = "2.14.1"+ )+ monkeypatch.setattr(sys, "platform", platform)+ monkeypatch.setattr(hardware, "rocm_windows_free_is_untrusted", lambda: is_rocm)+ monkeypatch.setitem(sys.modules, "torch", fake_torch)+ assert LlamaCppBackend._free_vram_counts_only_this_process() is expected+++def test_one_untracked_gpu_drops_the_whole_baseline():+ order: list[str] = []+ backend = _killable_backend(order, [_HELD, _HELD])+ with _ordered_memory(order, [(0, 4000, 24000), (1, 24000, 24000)]):+ backend._kill_process()+ assert backend._last_kill_vram_baseline is None++[email protected]("buffers", [[], [100.0]], ids = ["no-gpu-weights", "tiny"])+def test_kill_does_not_probe_for_a_child_with_no_gpu_weights_to_see(buffers):+ order: list[str] = []+ backend = _killable_backend(order, buffers)+ with _ordered_memory(order, [(0, 4000, 24000)]):+ backend._kill_process()+ assert order == ["terminate"]+ assert backend._last_kill_vram_baseline is None+++def test_kill_takes_no_baseline_on_shutdown_or_for_an_exited_child():+ order: list[str] = []+ backend = _killable_backend(order, [_HELD])+ backend._teardown_lock = __import__("threading").RLock()+ backend._spawn_lock = __import__("threading").RLock()+ backend._last_kill_vram_baseline = dict(_BASELINE)+ with (+ _ordered_memory(order, [(0, 4000, 24000)]),+ patch("utils.process_lifetime.mark_process_shutting_down", lambda: None),+ ):+ backend._kill_process(teardown = True)+ assert "probe" not in order+ assert backend._last_kill_vram_baseline is None++ order.clear()+ backend = _killable_backend(order, [_HELD])+ backend._process._dead = True # crashed before the kill: its free figure is no baseline+ with _ordered_memory(order, [(0, 4000, 24000)]):+ backend._kill_process()+ assert "probe" not in order+ assert backend._last_kill_vram_baseline is None+++def test_a_kill_with_no_process_keeps_the_baseline_of_the_kill_before_it():+ backend = _killable_backend([], [_HELD])+ backend._process = None+ backend._last_kill_vram_baseline = dict(_BASELINE)+ backend._kill_process()+ assert backend._last_kill_vram_baseline == _BASELINE+++def test_a_failed_baseline_read_leaves_the_kill_and_the_wait_as_before():+ order: list[str] = []+ backend = _killable_backend(order, [_HELD])++ def _boom(*_a, **_k):+ raise RuntimeError("nvidia-smi wedged")++ with patch.object(LlamaCppBackend, "_get_gpu_memory", staticmethod(_boom)):+ backend._kill_process()+ assert order == ["terminate"]+ assert backend._last_kill_vram_baseline is None+ assert backend._process is None+++def test_both_load_paths_hand_the_baseline_to_the_wait():+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ assert src.count("_wait_for_vram_settle(") == 2+ assert src.count('baseline = getattr(self, "_last_kill_vram_baseline", None)') == 2