Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: do not crash a model load when shutdown tears the server down under it (#10369)

Daniel Han03dabb263a83
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 6be0583401d..ecc6a6d3e67 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -23752,8 +23752,14 @@ class LlamaCppBackend:
return True
if getattr(self, "_health_wait_cancelled", False):
return False
+ # Read once, like the wait itself: a cleared reference is a
+ # teardown, not a startup crash, and re-reading the
+ # attribute per term would race the shutdown thread again.
+ _crashed_proc = self._process
_startup_crashed = (
- self._process.poll() is not None and self._process.returncode != 0
+ _crashed_proc is not None
+ and _crashed_proc.poll() is not None
+ and _crashed_proc.returncode != 0
)
# A split-axis abort (#6415) is fit-independent: skip the
# --fit off retry and let the caller latch it. So is a
@@ -23792,7 +23798,7 @@ class LlamaCppBackend:
"because a bundled ROCm library could not resolve a "
"symbol against the system ROCm; retrying once "
"with the bundled runtime only. Crash log: %s",
- self._process.returncode,
+ _crashed_proc.returncode,
self._llama_log_path,
)
env["LD_LIBRARY_PATH"] = _retry_ld
@@ -23822,7 +23828,7 @@ class LlamaCppBackend:
"with a tensor-spill plan; the plan was optimistic, "
"retrying once with --fit on so llama.cpp can place "
"the model itself. Crash log: %s",
- self._process.returncode,
+ _crashed_proc.returncode,
self._llama_log_path,
)
run_cmd = _reverted
@@ -23847,7 +23853,7 @@ class LlamaCppBackend:
"with forced --fit off; the fit estimate was optimistic, "
"retrying once with --fit on so it can offload. "
"Crash log: %s",
- self._process.returncode,
+ _crashed_proc.returncode,
self._llama_log_path,
)
# Flip Unsloth's own --fit off (added first, before any
@@ -23909,7 +23915,7 @@ class LlamaCppBackend:
"with the default memory-fit step enabled; Unsloth "
"already verified the model fits, retrying once "
"with --fit off. Crash log: %s",
- self._process.returncode,
+ _crashed_proc.returncode,
self._llama_log_path,
)
run_cmd = [*run_cmd, "--fit", "off"]
@@ -27860,8 +27866,16 @@ class LlamaCppBackend:
# Cleared before probing so output during the request stays latched for
# the fallback wait below.
health_probe_event.clear()
+ # Read once: shutdown clears the reference from another thread (#10353).
+ process = self._process
+ if process is None:
+ logger.info("llama-server was torn down while waiting for it to become healthy")
+ # Terminal like a cancel: the caller must not read the cleared
+ # reference, nor respawn a server shutdown just killed.
+ self._health_wait_cancelled = True
+ return False
# Process crashed?
- if self._process.poll() is not None:
+ if process.poll() is not None:
# Let the drain thread collect final output.
if self._stdout_thread is not None:
self._stdout_thread.join(timeout = 2)
@@ -27875,7 +27889,7 @@ class LlamaCppBackend:
else ""
)
logger.error(
- f"llama-server exited with code {self._process.returncode}. "
+ f"llama-server exited with code {process.returncode}. "
f"Output (tail): {output[-2000:]}{_log_hint}"
)
return False
diff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.py
index 98a64b728b6..062323d597a 100644
--- a/studio/backend/tests/test_chat_generation_supervisor.py
+++ b/studio/backend/tests/test_chat_generation_supervisor.py
@@ -197,6 +197,58 @@ async def test_background_producer_persists_chunks_and_completes(durable_run, mo
] == chunks
+async def test_a_prefill_reporting_only_progress_renews_the_lease(durable_run, monkeypatch):
+ """A 250K prefill outruns the 1200s lease before its first token, and the
+ write is what renews it, so dropping content-less progress chunks would reap a
+ healthy prefill. Sampled mid-run: afterwards every chunk has landed."""
+ released = asyncio.Event()
+ sampled: dict = {}
+
+ def _progress(processed):
+ # What llama-server sends under return_progress: a content-less delta.
+ return {
+ "choices": [{"delta": {"role": "assistant", "content": None}, "finish_reason": None}],
+ "prompt_progress": {
+ "total": 250000,
+ "cache": 0,
+ "processed": processed,
+ "time_ms": processed,
+ },
+ }
+
+ async def body():
+ for processed in (1024, 8192, 65536):
+ yield f"data: {json.dumps(_progress(processed))}\n\n"
+ # Polled, not slept: the idle flush is on a 0.1s timer
+ # (_EVENT_BATCH_SECONDS) and a sleep sized against it flakes under load.
+ _deadline = time.monotonic() + 10.0
+ while time.monotonic() < _deadline:
+ sampled["events"] = [
+ e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"
+ ]
+ if len(sampled["events"]) >= 3:
+ break
+ await asyncio.sleep(0.01)
+ sampled["progress"] = runs_db.get_progress("run-1")
+ released.set()
+ yield f"data: {json.dumps({'choices': [{'delta': {'content': 'Hi'}, 'finish_reason': 'stop'}]})}\n\n"
+ yield "data: [DONE]\n\n"
+
+ async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
+ return SimpleNamespace(status_code = 200, body_iterator = body())
+
+ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
+ await supervisor._produce("run-1")
+ assert released.is_set()
+ assert len(sampled["events"]) == 3, sampled["events"]
+ assert all("prompt_progress" in e for e in sampled["events"])
+ # The lease counts writes, so it renewed three times before the first token.
+ assert sampled["progress"][1] == 3, sampled["progress"]
+ assert runs_db.get_run("run-1", "alice")["status"] == "completed"
+
+
async def _subscriber_sequences(after = 0):
response = await run_routes.chat_generation_events(
"run-1",
diff --git a/studio/backend/tests/test_llama_cpp_wait_for_health.py b/studio/backend/tests/test_llama_cpp_wait_for_health.py
index 6d49c51b25e..4ae2433fe4b 100644
--- a/studio/backend/tests/test_llama_cpp_wait_for_health.py
+++ b/studio/backend/tests/test_llama_cpp_wait_for_health.py
@@ -76,6 +76,107 @@ class TestWaitForHealthResilience:
assert b._wait_for_health(timeout = 0.02, interval = 0.01) is False
assert any("health check timed out" in ln for ln in b._stdout_lines)
+ def test_a_teardown_during_the_wait_is_a_failed_load_not_a_crash(self, monkeypatch):
+ """Shutdown clears the reference mid-wait; reading it once and treating
+ its absence as "gone" is what stops the AttributeError the user saw as
+ "Error loading model: 'NoneType' object has no attribute 'poll'" (#10353)."""
+ b = _make_backend()
+ b._process.poll.return_value = None
+ probes = []
+
+ def probe(*a, **kw):
+ probes.append(1)
+ if len(probes) == 2:
+ b._process = None # what _kill_process does on shutdown
+ return mock.Mock(status_code = 503)
+
+ monkeypatch.setattr(httpx, "get", probe)
+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False
+ assert len(probes) == 2
+ assert not any("health check timed out" in ln for ln in b._stdout_lines)
+ assert b._health_wait_cancelled is True
+
+ def test_a_teardown_before_the_first_probe_is_still_not_a_crash(self, monkeypatch):
+ b = _make_backend()
+ b._process = None
+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 200))
+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is True
+
+ def test_a_teardown_is_terminal_so_the_caller_reads_no_exit_code(self, monkeypatch):
+ """False alone is not enough: _spawn_and_wait stops only on this flag, else
+ it reads the cleared reference and raises one frame up, and respawns."""
+ b = _make_backend()
+ b._process = None
+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))
+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is True
+ # Exactly what the caller does at the `if not healthy` branch.
+ assert (
+ b._process is not None and b._process.poll() is not None and b._process.returncode != 0
+ ) is False
+
+ def test_a_crash_leaves_the_wait_unmarked_so_the_retries_still_run(self, monkeypatch):
+ """The other side: a real crash must NOT look terminal, or the --fit off
+ and CPU fallbacks stop recovering loads that used to recover."""
+ b = _make_backend()
+ b._process.poll.return_value = 1
+ b._process.returncode = 1
+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))
+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is False
+
+ def test_a_teardown_marker_does_not_leak_into_the_next_load(self, monkeypatch):
+ b = _make_backend()
+ b._process = None
+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 200))
+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False
+ assert b._health_wait_cancelled is True
+ b._process = mock.Mock()
+ b._process.poll.return_value = None
+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is True
+ assert b._health_wait_cancelled is False
+
+ def test_a_teardown_from_a_real_thread_is_not_a_crash(self, monkeypatch):
+ """Two real threads, not a probe callback: the once-per-iteration read is
+ what makes it safe, so drive it that way."""
+ b = _make_backend()
+ b._process.poll.return_value = None
+ in_probe = threading.Event()
+ cleared = threading.Event()
+
+ def probe(*a, **kw):
+ in_probe.set()
+ cleared.wait(2.0)
+ return mock.Mock(status_code = 503)
+
+ monkeypatch.setattr(httpx, "get", probe)
+
+ def shutdown():
+ in_probe.wait(2.0)
+ b._process = None # _kill_process, from run.py's shutdown path
+ cleared.set()
+
+ t = threading.Thread(target = shutdown)
+ t.start()
+ try:
+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False
+ finally:
+ t.join(5.0)
+ assert b._health_wait_cancelled is True
+
+ def test_a_crash_still_reports_the_exit_code(self, monkeypatch):
+ """The teardown guard must not swallow the crash branch: an exited
+ process still has to name its exit code and its output."""
+ b = _make_backend()
+ b._process.poll.return_value = 1
+ b._process.returncode = 1
+ b._stdout_lines = ["ROCm error: out of memory"]
+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))
+ with mock.patch("core.inference.llama_cpp.logger") as log:
+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False
+ assert any("exited with code 1" in str(c) for c in log.error.call_args_list)
+
def test_cancel_stops_the_wait_without_a_timeout_marker(self, monkeypatch):
b = _make_backend()
b._process.poll.return_value = None
@@ -424,6 +525,63 @@ class TestCancelledWaitEndsTheLoad:
is False
)
+ def test_a_teardown_mid_load_ends_the_load_instead_of_raising(self, tmp_path, monkeypatch):
+ """The whole race through the real caller (#10353): the waiter tests above
+ never reach _spawn_and_wait, which is where the traceback resurfaced."""
+ import subprocess
+
+ from core.inference.llama_cpp import GgufLoadIntent
+
+ gguf = tmp_path / "model.gguf"
+ gguf.write_bytes(b"GGUF" + b"\0" * 4096)
+
+ b = LlamaCppBackend()
+ b._find_llama_server_binary = lambda *a, **kw: "/usr/bin/true"
+ b._non_chat_gguf_refusal_for_path = lambda *a, **kw: None
+ b._non_chat_gguf_refusal = lambda *a, **kw: None
+ # The drain thread would iterate a Mock's stdout forever.
+ b._drain_stdout = lambda *a, **kw: None
+
+ def _kill():
+ b._process = None # the one line of _kill_process this race turns on
+
+ b._kill_process = _kill
+
+ spawns = []
+ _real_popen = subprocess.Popen
+
+ def _popen(*a, **kw):
+ argv = [str(x) for x in (a[0] if a else kw.get("args") or [])]
+ # `--help` and nvidia-smi are not launches; run() needs a real Popen.
+ if str(gguf) not in argv:
+ return _real_popen(*a, **kw)
+ spawns.append(argv)
+ proc = mock.Mock()
+ proc.poll.return_value = None # alive: shutdown kills it, it does not crash
+ proc.pid = 424242
+ return proc
+
+ monkeypatch.setattr(subprocess, "Popen", _popen)
+
+ def probe(*a, **kw):
+ _kill() # run.py's shutdown, arriving while the load waits
+ return mock.Mock(status_code = 503)
+
+ monkeypatch.setattr(httpx, "get", probe)
+
+ assert (
+ b.load_model(
+ GgufLoadIntent(
+ model_identifier = "owner/model",
+ gguf_path = str(gguf),
+ n_ctx = 4096,
+ ),
+ )
+ is False
+ )
+ # A second server would outlive the app whose shutdown killed the first.
+ assert len(spawns) == 1, f"respawned after shutdown (spawns={len(spawns)})"
+
def test_a_cancelled_diffusion_start_reaps_the_runner():
"""An automatic switch cancels without unloading, so nothing else reaps the shim