Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep waiting on a llama-server load that is still reading its model (#11055)
oobabooga98f15dae33f3
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 16fc1b305d6..b2e40b8beeb 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -16798,7 +16798,7 @@ class LlamaCppBackend:self._effective_context_length = maxtok or self._context_lengthself._max_context_length = self._context_length or maxtok or None- healthy = self._wait_for_health(timeout = 600.0, cancelled = cancelled)+ healthy = self._wait_for_health(timeout = self._HEALTH_STALL_TIMEOUT_S, cancelled = cancelled)if healthy:if not self._publish_healthy():# A teardown between the probe and this commit is already killing@@ -19075,13 +19075,12 @@ class LlamaCppBackend:"expected; otherwise check the llama-server log for the cause.")- # A live server that never answered 200 on /health is not a bad GGUF:- # the load is too large for VRAM/context, or a local proxy/VPN grabbed- # the loopback probe (#5740).+ # A startup stall is not a bad GGUF; memory pressure or a proxy intercepting+ # the loopback probe can also keep /health from succeeding (#5740).if "health check timed out" in lowered:return (- "llama-server started but never became healthy on its local "- "/health endpoint. Try a smaller context length or a more "+ "llama-server started but stopped making progress before it became "+ "healthy on its local /health endpoint. Try a smaller context length or a more ""quantized GGUF, and if you use a VPN or HTTP proxy make sure ""localhost bypasses it (NO_PROXY=127.0.0.1,localhost).")@@ -27056,7 +27055,7 @@ class LlamaCppBackend:# mark_process_shutting_down does not take _spawn_lock, so the# check above is not atomic against it for a helper-owned# backend. Without this recheck a child spawned in that gap sits- # outside the completed sweep for the whole 600s health wait.+ # outside the completed sweep for the whole health wait.if self._spawn_is_stale():logger.info("shutdown began during the spawn; killing the new llama-server"@@ -27075,7 +27074,9 @@ class LlamaCppBackend:target = self._drain_stdout, daemon = True, name = "llama-stdout")self._stdout_thread.start()- if self._wait_for_health(timeout = 600.0, cancelled = _load_cancelled):+ if self._wait_for_health(+ timeout = self._HEALTH_STALL_TIMEOUT_S, cancelled = _load_cancelled+ ):if _did_rocm_retry:# Proved on this host: every later child skips the# prepend instead of crashing into it first.@@ -28481,7 +28482,9 @@ class LlamaCppBackend:# the old child the teardown is clearing._cleanup_cancelled_load("App shut down during the text-only retry")return False- if self._wait_for_health(timeout = 600.0, cancelled = _load_cancelled):+ if self._wait_for_health(+ timeout = self._HEALTH_STALL_TIMEOUT_S, cancelled = _load_cancelled+ ):healthy = True# The child that serves this session never read the# CPU-pinned projector, so the host-memory advisory the@@ -31429,14 +31432,85 @@ class LlamaCppBackend:stop.set()self._mtp_watchdog_thread = None+ # Extend the health deadline while startup work continues.+ _HEALTH_STALL_TIMEOUT_S = 600.0+ _STARTUP_PROGRESS_SAMPLE_S = 5.0+ # Ignore activity from the health probes themselves.+ _STARTUP_PROGRESS_MIN_BYTES = 1 << 20+ _STARTUP_PROGRESS_MIN_PAGE_FAULTS = 64+ _STARTUP_PROGRESS_MIN_CPU_FRACTION = 0.05++ @staticmethod+ def _page_faults(member, memory) -> int:+ # mmap loading can progress through page faults while RSS stays flat.+ for field in ("pfaults", "num_page_faults"): # macOS, Windows+ if hasattr(memory, field):+ return getattr(memory, field)+ try:+ with open(f"/proc/{member.pid}/stat", "rb") as f:+ fields = f.read().rsplit(b")", 1)[1].split()+ return int(fields[7]) + int(fields[9]) # minflt + majflt+ except (OSError, IndexError, ValueError):+ return 0++ @classmethod+ def _startup_work_sample(cls, process) -> Optional[tuple[float, int, int, int]]:+ """Return cumulative (CPU, RSS, bytes read, page faults) for the process tree."""+ pid = getattr(process, "pid", None)+ if not isinstance(pid, int):+ return None+ try:+ import psutil+ root = psutil.Process(pid)+ members = [root, *root.children(recursive = True)]+ except Exception:+ return None+ cpu = 0.0+ rss = read = faults = 0+ for member in members:+ try:+ with member.oneshot():+ times = member.cpu_times()+ memory = member.memory_info()+ try:+ io = member.io_counters()+ except (AttributeError, psutil.AccessDenied):+ io = None # macOS has no per-process I/O counters+ member_faults = cls._page_faults(member, memory)+ except Exception:+ if member is root:+ return None+ continue+ cpu += times.user + times.system+ rss += memory.rss+ faults += member_faults+ if io is not None:+ read += getattr(io, "read_chars", io.read_bytes) # Windows has no read_chars+ return (cpu, rss, read, faults)++ @classmethod+ def _made_startup_progress(+ cls, before: tuple[float, int, int, int], after: tuple[float, int, int, int], elapsed: float+ ) -> bool:+ # RSS is a high-water mark so evicted pages are not counted twice.+ return (+ after[1] - before[1] >= cls._STARTUP_PROGRESS_MIN_BYTES+ or after[2] - before[2] >= cls._STARTUP_PROGRESS_MIN_BYTES+ or after[3] - before[3] >= cls._STARTUP_PROGRESS_MIN_PAGE_FAULTS+ or after[0] - before[0] >= cls._STARTUP_PROGRESS_MIN_CPU_FRACTION * elapsed+ )+def _wait_for_health(self,timeout: float = 120.0,interval: float = 0.5,cancelled: Optional[Callable[[], bool]] = None,) -> bool:- """Poll llama-server's /health until 200; also detect early exit/crash."""+ """Poll /health until success, exit, cancellation, or a startup stall."""deadline = time.monotonic() + timeout+ work_sample = None+ work_sampled_at = None+ work_checked_at = Noneurl = f"{self.base_url}/health"health_probe_event = getattr(self, "_health_probe_event", None)if health_probe_event is None: # Backward-compatible with lightweight test doubles.@@ -31455,7 +31529,7 @@ class LlamaCppBackend:# spawn and this line. _torn_down_process is matched by identity instead.process = None # the child this wait last looked at, read again after the loop- while time.monotonic() < deadline:+ while (now := time.monotonic()) < deadline:# Durable, unlike the per-process marker below: once teardown begins,# every later iteration sees it.if getattr(self, "_shutting_down", False):@@ -31504,6 +31578,18 @@ class LlamaCppBackend:)return False+ if work_checked_at is None or now - work_checked_at >= self._STARTUP_PROGRESS_SAMPLE_S:+ work_checked_at = now+ sample = self._startup_work_sample(process)+ if sample is not None:+ if work_sample is not None and self._made_startup_progress(+ work_sample, sample, now - work_sampled_at+ ):+ deadline = now + timeout+ peak = sample[1] if work_sample is None else max(work_sample[1], sample[1])+ work_sample = (sample[0], peak, sample[2], sample[3])+ work_sampled_at = now+try:# trust_env=False: skip ambient HTTP(S)_PROXY, which if it 503s# for 127.0.0.1 loops the probe until timeout and hangs load.@@ -31552,8 +31638,9 @@ class LlamaCppBackend:# Leave a marker so _classify_llama_start_failure tells a live but# never-healthy load (too large, or a proxy hijacking the loopback# probe) apart from a bad GGUF (#5740).- self._stdout_lines.append(f"llama-server health check timed out after {timeout}s")- logger.error(f"llama-server health check timed out after {timeout}s")+ marker = f"llama-server health check timed out: no startup progress for {timeout:g}s"+ self._stdout_lines.append(marker)+ logger.error(marker)return False@staticmethoddiff --git a/studio/backend/tests/test_llama_cpp_start_failure_classification.py b/studio/backend/tests/test_llama_cpp_start_failure_classification.pyindex 030cc568216..4c54324d416 100644--- a/studio/backend/tests/test_llama_cpp_start_failure_classification.py+++ b/studio/backend/tests/test_llama_cpp_start_failure_classification.py@@ -206,7 +206,9 @@ class TestOllamaAndFallback:# A live server that never returns 200 on /health must name the probe and# proxy/context causes, not blame a bad GGUF (#5740).msg = _classify(- "llama-server health check timed out after 600.0s", "/models/x.gguf", "local/x"+ "llama-server health check timed out: no startup progress for 600s",+ "/models/x.gguf",+ "local/x",)assert "/health" in msgassert "NO_PROXY" in msg@@ -962,7 +964,7 @@ class TestMacOSLoaderEdgeCases:out = ("dyld[1]: Library not loaded: @rpath/libllama.dylib\n"" Reason: tried: '/x/libllama.dylib' (no such file)\n"- "llama-server health check timed out after 600.0s"+ "llama-server health check timed out: no startup progress for 600s")msg = _classify(out, "/models/x.gguf", "local/x", 1)assert "health check timed out" not in msgdiff --git a/studio/backend/tests/test_llama_cpp_wait_for_health.py b/studio/backend/tests/test_llama_cpp_wait_for_health.pyindex 723b1f6d7b8..e3ae0562586 100644--- a/studio/backend/tests/test_llama_cpp_wait_for_health.py+++ b/studio/backend/tests/test_llama_cpp_wait_for_health.py@@ -11,6 +11,7 @@ subprocess.poll() branch so a crashed llama-server surfaces a structuredfrom __future__ import annotations+import osimport subprocessimport sysimport threading@@ -1585,3 +1586,140 @@ def test_the_lock_order_is_teardown_then_spawn():):inversions.append(inner.lineno)assert not inversions, f"_teardown_lock taken inside _spawn_lock at {inversions}"+++class TestHealthWaitMeasuresStalls:+ # Grow RSS, then idle.+ _WORKER = (+ "import sys, time\n"+ "held = []\n"+ "end = time.monotonic() + float(sys.argv[1])\n"+ "while time.monotonic() < end:\n"+ " held.append(b'\\x01' * (4 << 20))\n"+ " time.sleep(0.05)\n"+ "time.sleep(60)\n"+ )++ # Page mmap views without retaining RSS or calling read().+ _MMAP_WORKER = (+ "import mmap, os, sys, time\n"+ "fd = os.open(sys.argv[1], os.O_RDONLY | getattr(os, 'O_BINARY', 0))\n"+ "view = 16 << 20\n"+ "end = time.monotonic() + float(sys.argv[2])\n"+ "offset = 0\n"+ "while time.monotonic() < end:\n"+ " if hasattr(os, 'posix_fadvise'):\n"+ " os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)\n"+ " with mmap.mmap(fd, view, access = mmap.ACCESS_READ, offset = offset) as m:\n"+ " sum(m[p] for p in range(0, view, mmap.PAGESIZE))\n"+ " offset = view - offset\n"+ " time.sleep(0.05)\n"+ "time.sleep(60)\n"+ )++ def _wait_on_child(+ self,+ monkeypatch,+ argv,+ *,+ healthy_after = None,+ timeout = 0.6,+ ):+ monkeypatch.setattr(LlamaCppBackend, "_STARTUP_PROGRESS_SAMPLE_S", 0.1)+ b = _make_backend()+ b._process = subprocess.Popen([sys.executable, "-c", *argv])+ started = time.monotonic()++ def probe(*a, **kw):+ if healthy_after is not None and time.monotonic() - started >= healthy_after:+ return mock.Mock(status_code = 200)+ return mock.Mock(status_code = 503)++ monkeypatch.setattr(httpx, "get", probe)+ try:+ ok = b._wait_for_health(timeout = timeout, interval = 0.02)+ finally:+ import psutil++ for descendant in psutil.Process(b._process.pid).children(recursive = True):+ descendant.kill()+ b._process.kill()+ b._process.wait()+ return b, ok, time.monotonic() - started++ def test_a_load_that_keeps_working_outlives_the_timeout(self, monkeypatch):+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "3.0"], healthy_after = 2.5)+ assert ok is True+ assert elapsed >= 2.5+ assert not any("health check timed out" in ln for ln in b._stdout_lines)++ def test_a_child_doing_nothing_still_times_out(self, monkeypatch):+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "0.0"])+ assert ok is False+ assert elapsed < 3.0+ assert any("no startup progress for 0.6s" in ln for ln in b._stdout_lines)++ def test_a_load_that_stalls_times_out_one_timeout_after_its_last_work(self, monkeypatch):+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "1.5"])+ assert ok is False+ assert 1.5 + 0.6 - 0.2 <= elapsed < 1.5 + 3.0+ assert any("health check timed out" in ln for ln in b._stdout_lines)++ def test_work_done_by_a_descendant_counts(self, monkeypatch):+ shim = (+ "import subprocess, sys, time\n"+ "subprocess.Popen([sys.executable, '-c', sys.argv[1], '3.0'])\n"+ "time.sleep(60)\n"+ )+ b, ok, elapsed = self._wait_on_child(monkeypatch, [shim, self._WORKER], healthy_after = 2.5)+ assert ok is True+ assert elapsed >= 2.5++ def test_mmap_page_ins_with_flat_resident_memory_count(self, monkeypatch, tmp_path):+ # Make page faults the only enabled progress signal.+ monkeypatch.setattr(LlamaCppBackend, "_STARTUP_PROGRESS_MIN_CPU_FRACTION", 100.0)+ monkeypatch.setattr(LlamaCppBackend, "_STARTUP_PROGRESS_MIN_BYTES", 1 << 40)+ model = tmp_path / "model.gguf"+ # Small writes avoid collapsing the file into 2 MiB cache folios.+ with open(model, "wb") as f:+ for _ in range(512):+ f.write(os.urandom(64 << 10))+ b, ok, elapsed = self._wait_on_child(+ monkeypatch, [self._MMAP_WORKER, str(model), "3.0"], healthy_after = 2.5+ )+ assert ok is True+ assert elapsed >= 2.5++ def test_unreadable_counters_keep_the_fixed_deadline(self, monkeypatch):+ monkeypatch.setattr(LlamaCppBackend, "_startup_work_sample", staticmethod(lambda p: None))+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "3.0"])+ assert ok is False+ assert elapsed < 1.5++ def test_cpu_after_an_unreadable_sample_is_measured_from_the_last_readable_one(+ self, monkeypatch+ ):+ # 20 ms over 0.9s remains below the threshold.+ samples = iter([(1.0, 0, 0, 0), *[None] * 8, (1.02, 0, 0, 0)])+ monkeypatch.setattr(+ LlamaCppBackend,+ "_startup_work_sample",+ staticmethod(lambda p: next(samples, None)),+ )+ b, ok, elapsed = self._wait_on_child(monkeypatch, [self._WORKER, "0.0"], timeout = 1.5)+ assert ok is False+ assert elapsed < 1.95++ def test_resident_memory_regained_after_eviction_is_not_progress(self):+ peak = (10.0, 500 << 20, 0, 0)+ assert not LlamaCppBackend._made_startup_progress(peak, (10.0, 499 << 20, 0, 0), 5.0)+ assert LlamaCppBackend._made_startup_progress(peak, (10.0, 502 << 20, 0, 0), 5.0)++ def test_answering_health_probes_is_not_progress(self):+ # Idle health probes stay below both thresholds.+ before = (1.0, 100 << 20, 50 << 20, 1000)+ assert not LlamaCppBackend._made_startup_progress(+ before, (1.02, 100 << 20, 50 << 20, 1033), 5.0+ )+ assert LlamaCppBackend._made_startup_progress(before, (1.5, 100 << 20, 50 << 20, 1000), 5.0)+ assert LlamaCppBackend._made_startup_progress(before, (1.0, 100 << 20, 50 << 20, 1064), 5.0)