Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep waiting on a llama-server load that is still reading its model (#11055)

oobabooga98f15dae33f3
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 16fc1b305d6..b2e40b8beeb 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -16798,7 +16798,7 @@ class LlamaCppBackend:
self._effective_context_length = maxtok or self._context_length
self._max_context_length = self._context_length or maxtok or None
- healthy = self._wait_for_health(timeout = 600.0, cancelled = cancelled)
+ healthy = self._wait_for_health(timeout = self._HEALTH_STALL_TIMEOUT_S, cancelled = cancelled)
if healthy:
if not self._publish_healthy():
# A teardown between the probe and this commit is already killing
@@ -19075,13 +19075,12 @@ class LlamaCppBackend:
"expected; otherwise check the llama-server log for the cause."
)
- # A live server that never answered 200 on /health is not a bad GGUF:
- # the load is too large for VRAM/context, or a local proxy/VPN grabbed
- # the loopback probe (#5740).
+ # A startup stall is not a bad GGUF; memory pressure or a proxy intercepting
+ # the loopback probe can also keep /health from succeeding (#5740).
if "health check timed out" in lowered:
return (
- "llama-server started but never became healthy on its local "
- "/health endpoint. Try a smaller context length or a more "
+ "llama-server started but stopped making progress before it became "
+ "healthy on its local /health endpoint. Try a smaller context length or a more "
"quantized GGUF, and if you use a VPN or HTTP proxy make sure "
"localhost bypasses it (NO_PROXY=127.0.0.1,localhost)."
)
@@ -27056,7 +27055,7 @@ class LlamaCppBackend:
# mark_process_shutting_down does not take _spawn_lock, so the
# check above is not atomic against it for a helper-owned
# backend. Without this recheck a child spawned in that gap sits
- # outside the completed sweep for the whole 600s health wait.
+ # outside the completed sweep for the whole health wait.
if self._spawn_is_stale():
logger.info(
"shutdown began during the spawn; killing the new llama-server"
@@ -27075,7 +27074,9 @@ class LlamaCppBackend:
target = self._drain_stdout, daemon = True, name = "llama-stdout"
)
self._stdout_thread.start()
- if self._wait_for_health(timeout = 600.0, cancelled = _load_cancelled):
+ if self._wait_for_health(
+ timeout = self._HEALTH_STALL_TIMEOUT_S, cancelled = _load_cancelled
+ ):
if _did_rocm_retry:
# Proved on this host: every later child skips the
# prepend instead of crashing into it first.
@@ -28481,7 +28482,9 @@ class LlamaCppBackend:
# the old child the teardown is clearing.
_cleanup_cancelled_load("App shut down during the text-only retry")
return False
- if self._wait_for_health(timeout = 600.0, cancelled = _load_cancelled):
+ if self._wait_for_health(
+ timeout = self._HEALTH_STALL_TIMEOUT_S, cancelled = _load_cancelled
+ ):
healthy = True
# The child that serves this session never read the
# CPU-pinned projector, so the host-memory advisory the
@@ -31429,14 +31432,85 @@ class LlamaCppBackend:
stop.set()
self._mtp_watchdog_thread = None
+ # Extend the health deadline while startup work continues.
+ _HEALTH_STALL_TIMEOUT_S = 600.0
+ _STARTUP_PROGRESS_SAMPLE_S = 5.0
+ # Ignore activity from the health probes themselves.
+ _STARTUP_PROGRESS_MIN_BYTES = 1 << 20
+ _STARTUP_PROGRESS_MIN_PAGE_FAULTS = 64
+ _STARTUP_PROGRESS_MIN_CPU_FRACTION = 0.05
+
+ @staticmethod
+ def _page_faults(member, memory) -> int:
+ # mmap loading can progress through page faults while RSS stays flat.
+ for field in ("pfaults", "num_page_faults"): # macOS, Windows
+ if hasattr(memory, field):
+ return getattr(memory, field)
+ try:
+ with open(f"/proc/{member.pid}/stat", "rb") as f:
+ fields = f.read().rsplit(b")", 1)[1].split()
+ return int(fields[7]) + int(fields[9]) # minflt + majflt
+ except (OSError, IndexError, ValueError):
+ return 0
+
+ @classmethod
+ def _startup_work_sample(cls, process) -> Optional[tuple[float, int, int, int]]:
+ """Return cumulative (CPU, RSS, bytes read, page faults) for the process tree."""
+ pid = getattr(process, "pid", None)
+ if not isinstance(pid, int):
+ return None
+ try:
+ import psutil
+ root = psutil.Process(pid)
+ members = [root, *root.children(recursive = True)]
+ except Exception:
+ return None
+ cpu = 0.0
+ rss = read = faults = 0
+ for member in members:
+ try:
+ with member.oneshot():
+ times = member.cpu_times()
+ memory = member.memory_info()
+ try:
+ io = member.io_counters()
+ except (AttributeError, psutil.AccessDenied):
+ io = None # macOS has no per-process I/O counters
+ member_faults = cls._page_faults(member, memory)
+ except Exception:
+ if member is root:
+ return None
+ continue
+ cpu += times.user + times.system
+ rss += memory.rss
+ faults += member_faults
+ if io is not None:
+ read += getattr(io, "read_chars", io.read_bytes) # Windows has no read_chars
+ return (cpu, rss, read, faults)
+
+ @classmethod
+ def _made_startup_progress(
+ cls, before: tuple[float, int, int, int], after: tuple[float, int, int, int], elapsed: float
+ ) -> bool:
+ # RSS is a high-water mark so evicted pages are not counted twice.
+ return (
+ after[1] - before[1] >= cls._STARTUP_PROGRESS_MIN_BYTES
+ or after[2] - before[2] >= cls._STARTUP_PROGRESS_MIN_BYTES
+ or after[3] - before[3] >= cls._STARTUP_PROGRESS_MIN_PAGE_FAULTS
+ or after[0] - before[0] >= cls._STARTUP_PROGRESS_MIN_CPU_FRACTION * elapsed
+ )
+
def _wait_for_health(
self,
timeout: float = 120.0,
interval: float = 0.5,
cancelled: Optional[Callable[[], bool]] = None,
) -> bool:
- """Poll llama-server's /health until 200; also detect early exit/crash."""
+ """Poll /health until success, exit, cancellation, or a startup stall."""
deadline = time.monotonic() + timeout
+ work_sample = None
+ work_sampled_at = None
+ work_checked_at = None
url = f"{self.base_url}/health"
health_probe_event = getattr(self, "_health_probe_event", None)
if health_probe_event is None: # Backward-compatible with lightweight test doubles.
@@ -31455,7 +31529,7 @@ class LlamaCppBackend:
# spawn and this line. _torn_down_process is matched by identity instead.
process = None # the child this wait last looked at, read again after the loop
- while time.monotonic() < deadline:
+ while (now := time.monotonic()) < deadline:
# Durable, unlike the per-process marker below: once teardown begins,
# every later iteration sees it.
if getattr(self, "_shutting_down", False):
@@ -31504,6 +31578,18 @@ class LlamaCppBackend:
)
return False
+ if work_checked_at is None or now - work_checked_at >= self._STARTUP_PROGRESS_SAMPLE_S:
+ work_checked_at = now
+ sample = self._startup_work_sample(process)
+ if sample is not None:
+ if work_sample is not None and self._made_startup_progress(
+ work_sample, sample, now - work_sampled_at
+ ):
+ deadline = now + timeout
+ peak = sample[1] if work_sample is None else max(work_sample[1], sample[1])
+ work_sample = (sample[0], peak, sample[2], sample[3])
+ work_sampled_at = now
+
try:
# trust_env=False: skip ambient HTTP(S)_PROXY, which if it 503s
# for 127.0.0.1 loops the probe until timeout and hangs load.
@@ -31552,8 +31638,9 @@ class LlamaCppBackend:
# Leave a marker so _classify_llama_start_failure tells a live but
# never-healthy load (too large, or a proxy hijacking the loopback
# probe) apart from a bad GGUF (#5740).
- self._stdout_lines.append(f"llama-server health check timed out after {timeout}s")
- logger.error(f"llama-server health check timed out after {timeout}s")
+ marker = f"llama-server health check timed out: no startup progress for {timeout:g}s"
+ self._stdout_lines.append(marker)
+ logger.error(marker)
return False
@staticmethod
diff --git a/studio/backend/tests/test_llama_cpp_start_failure_classification.py b/studio/backend/tests/test_llama_cpp_start_failure_classification.py
index 030cc568216..4c54324d416 100644
--- a/studio/backend/tests/test_llama_cpp_start_failure_classification.py
+++ b/studio/backend/tests/test_llama_cpp_start_failure_classification.py
@@ -206,7 +206,9 @@ class TestOllamaAndFallback:
# A live server that never returns 200 on /health must name the probe and
# proxy/context causes, not blame a bad GGUF (#5740).
msg = _classify(
- "llama-server health check timed out after 600.0s", "/models/x.gguf", "local/x"
+ "llama-server health check timed out: no startup progress for 600s",
+ "/models/x.gguf",
+ "local/x",
)
assert "/health" in msg
assert "NO_PROXY" in msg
@@ -962,7 +964,7 @@ class TestMacOSLoaderEdgeCases:
out = (
"dyld[1]: Library not loaded: @rpath/libllama.dylib\n"
" Reason: tried: '/x/libllama.dylib' (no such file)\n"
- "llama-server health check timed out after 600.0s"
+ "llama-server health check timed out: no startup progress for 600s"
)
msg = _classify(out, "/models/x.gguf", "local/x", 1)
assert "health check timed out" not in msg
diff --git a/studio/backend/tests/test_llama_cpp_wait_for_health.py b/studio/backend/tests/test_llama_cpp_wait_for_health.py
index 723b1f6d7b8..e3ae0562586 100644
--- a/studio/backend/tests/test_llama_cpp_wait_for_health.py
+++ b/studio/backend/tests/test_llama_cpp_wait_for_health.py
@@ -11,6 +11,7 @@ subprocess.poll() branch so a crashed llama-server surfaces a structured
from __future__ import annotations
+import os
import subprocess
import sys
import threading
@@ -1585,3 +1586,140 @@ def test_the_lock_order_is_teardown_then_spawn():
):
inversions.append(inner.lineno)
assert not inversions, f"_teardown_lock taken inside _spawn_lock at {inversions}"
+
+
+class TestHealthWaitMeasuresStalls:
+ # Grow RSS, then idle.
+ _WORKER = (
+ "import sys, time\n"
+ "held = []\n"
+ "end = time.monotonic() + float(sys.argv[1])\n"
+ "while time.monotonic() < end:\n"
+ " held.append(b'\\x01' * (4 << 20))\n"
+ " time.sleep(0.05)\n"
+ "time.sleep(60)\n"
+ )
+
+ # Page mmap views without retaining RSS or calling read().
+ _MMAP_WORKER = (
+ "import mmap, os, sys, time\n"
+ "fd = os.open(sys.argv[1], os.O_RDONLY | getattr(os, 'O_BINARY', 0))\n"
+ "view = 16 << 20\n"
+ "end = time.monotonic() + float(sys.argv[2])\n"
+ "offset = 0\n"
+ "while time.monotonic() < end:\n"
+ " if hasattr(os, 'posix_fadvise'):\n"
+ " os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)\n"
+ " with mmap.mmap(fd, view, access = mmap.ACCESS_READ, offset = offset) as m:\n"
+ " sum(m[p] for p in range(0, view, mmap.PAGESIZE))\n"
+ " offset = view - offset\n"
+ " time.sleep(0.05)\n"
+ "time.sleep(60)\n"
+ )
+
+ def _wait_on_child(
+ self,
+ monkeypatch,
+ argv,
+ *,
+ healthy_after = None,
+ timeout = 0.6,
+ ):
+ monkeypatch.setattr(LlamaCppBackend, "_STARTUP_PROGRESS_SAMPLE_S", 0.1)
+ b = _make_backend()
+ b._process = subprocess.Popen([sys.executable, "-c", *argv])
+ started = time.monotonic()
+
+ def probe(*a, **kw):
+ if healthy_after is not None and time.monotonic() - started >= healthy_after:
+ return mock.Mock(status_code = 200)
+ return mock.Mock(status_code = 503)
+
+ monkeypatch.setattr(httpx, "get", probe)
+ try:
+ ok = b._wait_for_health(timeout = timeout, interval = 0.02)
+ finally:
+ import psutil
+
+ for descendant in psutil.Process(b._process.pid).children(recursive = True):
+ descendant.kill()
+ b._process.kill()
+ b._process.wait()
+ return b, ok, time.monotonic() - started
+
+ def test_a_load_that_keeps_working_outlives_the_timeout(self, monkeypatch):
+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "3.0"], healthy_after = 2.5)
+ assert ok is True
+ assert elapsed >= 2.5
+ assert not any("health check timed out" in ln for ln in b._stdout_lines)
+
+ def test_a_child_doing_nothing_still_times_out(self, monkeypatch):
+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "0.0"])
+ assert ok is False
+ assert elapsed < 3.0
+ assert any("no startup progress for 0.6s" in ln for ln in b._stdout_lines)
+
+ def test_a_load_that_stalls_times_out_one_timeout_after_its_last_work(self, monkeypatch):
+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "1.5"])
+ assert ok is False
+ assert 1.5 + 0.6 - 0.2 <= elapsed < 1.5 + 3.0
+ assert any("health check timed out" in ln for ln in b._stdout_lines)
+
+ def test_work_done_by_a_descendant_counts(self, monkeypatch):
+ shim = (
+ "import subprocess, sys, time\n"
+ "subprocess.Popen([sys.executable, '-c', sys.argv[1], '3.0'])\n"
+ "time.sleep(60)\n"
+ )
+ b, ok, elapsed = self._wait_on_child(monkeypatch, [shim, self._WORKER], healthy_after = 2.5)
+ assert ok is True
+ assert elapsed >= 2.5
+
+ def test_mmap_page_ins_with_flat_resident_memory_count(self, monkeypatch, tmp_path):
+ # Make page faults the only enabled progress signal.
+ monkeypatch.setattr(LlamaCppBackend, "_STARTUP_PROGRESS_MIN_CPU_FRACTION", 100.0)
+ monkeypatch.setattr(LlamaCppBackend, "_STARTUP_PROGRESS_MIN_BYTES", 1 << 40)
+ model = tmp_path / "model.gguf"
+ # Small writes avoid collapsing the file into 2 MiB cache folios.
+ with open(model, "wb") as f:
+ for _ in range(512):
+ f.write(os.urandom(64 << 10))
+ b, ok, elapsed = self._wait_on_child(
+ monkeypatch, [self._MMAP_WORKER, str(model), "3.0"], healthy_after = 2.5
+ )
+ assert ok is True
+ assert elapsed >= 2.5
+
+ def test_unreadable_counters_keep_the_fixed_deadline(self, monkeypatch):
+ monkeypatch.setattr(LlamaCppBackend, "_startup_work_sample", staticmethod(lambda p: None))
+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "3.0"])
+ assert ok is False
+ assert elapsed < 1.5
+
+ def test_cpu_after_an_unreadable_sample_is_measured_from_the_last_readable_one(
+ self, monkeypatch
+ ):
+ # 20 ms over 0.9s remains below the threshold.
+ samples = iter([(1.0, 0, 0, 0), *[None] * 8, (1.02, 0, 0, 0)])
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_startup_work_sample",
+ staticmethod(lambda p: next(samples, None)),
+ )
+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "0.0"], timeout = 1.5)
+ assert ok is False
+ assert elapsed < 1.95
+
+ def test_resident_memory_regained_after_eviction_is_not_progress(self):
+ peak = (10.0, 500 << 20, 0, 0)
+ assert not LlamaCppBackend._made_startup_progress(peak, (10.0, 499 << 20, 0, 0), 5.0)
+ assert LlamaCppBackend._made_startup_progress(peak, (10.0, 502 << 20, 0, 0), 5.0)
+
+ def test_answering_health_probes_is_not_progress(self):
+ # Idle health probes stay below both thresholds.
+ before = (1.0, 100 << 20, 50 << 20, 1000)
+ assert not LlamaCppBackend._made_startup_progress(
+ before, (1.02, 100 << 20, 50 << 20, 1033), 5.0
+ )
+ assert LlamaCppBackend._made_startup_progress(before, (1.5, 100 << 20, 50 << 20, 1000), 5.0)
+ assert LlamaCppBackend._made_startup_progress(before, (1.0, 100 << 20, 50 << 20, 1064), 5.0)