Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: do not crash a model load when shutdown tears the server down under it (#10369)
Daniel Han03dabb263a83
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 6be0583401d..ecc6a6d3e67 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -23752,8 +23752,14 @@ class LlamaCppBackend:return Trueif getattr(self, "_health_wait_cancelled", False):return False+ # Read once, like the wait itself: a cleared reference is a+ # teardown, not a startup crash, and re-reading the+ # attribute per term would race the shutdown thread again.+ _crashed_proc = self._process_startup_crashed = (- self._process.poll() is not None and self._process.returncode != 0+ _crashed_proc is not None+ and _crashed_proc.poll() is not None+ and _crashed_proc.returncode != 0)# A split-axis abort (#6415) is fit-independent: skip the# --fit off retry and let the caller latch it. So is a@@ -23792,7 +23798,7 @@ class LlamaCppBackend:"because a bundled ROCm library could not resolve a ""symbol against the system ROCm; retrying once ""with the bundled runtime only. Crash log: %s",- self._process.returncode,+ _crashed_proc.returncode,self._llama_log_path,)env["LD_LIBRARY_PATH"] = _retry_ld@@ -23822,7 +23828,7 @@ class LlamaCppBackend:"with a tensor-spill plan; the plan was optimistic, ""retrying once with --fit on so llama.cpp can place ""the model itself. Crash log: %s",- self._process.returncode,+ _crashed_proc.returncode,self._llama_log_path,)run_cmd = _reverted@@ -23847,7 +23853,7 @@ class LlamaCppBackend:"with forced --fit off; the fit estimate was optimistic, ""retrying once with --fit on so it can offload. ""Crash log: %s",- self._process.returncode,+ _crashed_proc.returncode,self._llama_log_path,)# Flip Unsloth's own --fit off (added first, before any@@ -23909,7 +23915,7 @@ class LlamaCppBackend:"with the default memory-fit step enabled; Unsloth ""already verified the model fits, retrying once ""with --fit off. Crash log: %s",- self._process.returncode,+ _crashed_proc.returncode,self._llama_log_path,)run_cmd = [*run_cmd, "--fit", "off"]@@ -27860,8 +27866,16 @@ class LlamaCppBackend:# Cleared before probing so output during the request stays latched for# the fallback wait below.health_probe_event.clear()+ # Read once: shutdown clears the reference from another thread (#10353).+ process = self._process+ if process is None:+ logger.info("llama-server was torn down while waiting for it to become healthy")+ # Terminal like a cancel: the caller must not read the cleared+ # reference, nor respawn a server shutdown just killed.+ self._health_wait_cancelled = True+ return False# Process crashed?- if self._process.poll() is not None:+ if process.poll() is not None:# Let the drain thread collect final output.if self._stdout_thread is not None:self._stdout_thread.join(timeout = 2)@@ -27875,7 +27889,7 @@ class LlamaCppBackend:else "")logger.error(- f"llama-server exited with code {self._process.returncode}. "+ f"llama-server exited with code {process.returncode}. "f"Output (tail): {output[-2000:]}{_log_hint}")return Falsediff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.pyindex 98a64b728b6..062323d597a 100644--- a/studio/backend/tests/test_chat_generation_supervisor.py+++ b/studio/backend/tests/test_chat_generation_supervisor.py@@ -197,6 +197,58 @@ async def test_background_producer_persists_chunks_and_completes(durable_run, mo] == chunks+async def test_a_prefill_reporting_only_progress_renews_the_lease(durable_run, monkeypatch):+ """A 250K prefill outruns the 1200s lease before its first token, and the+ write is what renews it, so dropping content-less progress chunks would reap a+ healthy prefill. Sampled mid-run: afterwards every chunk has landed."""+ released = asyncio.Event()+ sampled: dict = {}++ def _progress(processed):+ # What llama-server sends under return_progress: a content-less delta.+ return {+ "choices": [{"delta": {"role": "assistant", "content": None}, "finish_reason": None}],+ "prompt_progress": {+ "total": 250000,+ "cache": 0,+ "processed": processed,+ "time_ms": processed,+ },+ }++ async def body():+ for processed in (1024, 8192, 65536):+ yield f"data: {json.dumps(_progress(processed))}\n\n"+ # Polled, not slept: the idle flush is on a 0.1s timer+ # (_EVENT_BATCH_SECONDS) and a sleep sized against it flakes under load.+ _deadline = time.monotonic() + 10.0+ while time.monotonic() < _deadline:+ sampled["events"] = [+ e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"+ ]+ if len(sampled["events"]) >= 3:+ break+ await asyncio.sleep(0.01)+ sampled["progress"] = runs_db.get_progress("run-1")+ released.set()+ yield f"data: {json.dumps({'choices': [{'delta': {'content': 'Hi'}, 'finish_reason': 'stop'}]})}\n\n"+ yield "data: [DONE]\n\n"++ async def fake(_payload, _request, _subject, *, cancel_on_disconnect):+ return SimpleNamespace(status_code = 200, body_iterator = body())++ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))+ await supervisor._produce("run-1")+ assert released.is_set()+ assert len(sampled["events"]) == 3, sampled["events"]+ assert all("prompt_progress" in e for e in sampled["events"])+ # The lease counts writes, so it renewed three times before the first token.+ assert sampled["progress"][1] == 3, sampled["progress"]+ assert runs_db.get_run("run-1", "alice")["status"] == "completed"++async def _subscriber_sequences(after = 0):response = await run_routes.chat_generation_events("run-1",diff --git a/studio/backend/tests/test_llama_cpp_wait_for_health.py b/studio/backend/tests/test_llama_cpp_wait_for_health.pyindex 6d49c51b25e..4ae2433fe4b 100644--- a/studio/backend/tests/test_llama_cpp_wait_for_health.py+++ b/studio/backend/tests/test_llama_cpp_wait_for_health.py@@ -76,6 +76,107 @@ class TestWaitForHealthResilience:assert b._wait_for_health(timeout = 0.02, interval = 0.01) is Falseassert any("health check timed out" in ln for ln in b._stdout_lines)+ def test_a_teardown_during_the_wait_is_a_failed_load_not_a_crash(self, monkeypatch):+ """Shutdown clears the reference mid-wait; reading it once and treating+ its absence as "gone" is what stops the AttributeError the user saw as+ "Error loading model: 'NoneType' object has no attribute 'poll'" (#10353)."""+ b = _make_backend()+ b._process.poll.return_value = None+ probes = []++ def probe(*a, **kw):+ probes.append(1)+ if len(probes) == 2:+ b._process = None # what _kill_process does on shutdown+ return mock.Mock(status_code = 503)++ monkeypatch.setattr(httpx, "get", probe)+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False+ assert len(probes) == 2+ assert not any("health check timed out" in ln for ln in b._stdout_lines)+ assert b._health_wait_cancelled is True++ def test_a_teardown_before_the_first_probe_is_still_not_a_crash(self, monkeypatch):+ b = _make_backend()+ b._process = None+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 200))+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False+ assert b._health_wait_cancelled is True++ def test_a_teardown_is_terminal_so_the_caller_reads_no_exit_code(self, monkeypatch):+ """False alone is not enough: _spawn_and_wait stops only on this flag, else+ it reads the cleared reference and raises one frame up, and respawns."""+ b = _make_backend()+ b._process = None+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False+ assert b._health_wait_cancelled is True+ # Exactly what the caller does at the `if not healthy` branch.+ assert (+ b._process is not None and b._process.poll() is not None and b._process.returncode != 0+ ) is False++ def test_a_crash_leaves_the_wait_unmarked_so_the_retries_still_run(self, monkeypatch):+ """The other side: a real crash must NOT look terminal, or the --fit off+ and CPU fallbacks stop recovering loads that used to recover."""+ b = _make_backend()+ b._process.poll.return_value = 1+ b._process.returncode = 1+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False+ assert b._health_wait_cancelled is False++ def test_a_teardown_marker_does_not_leak_into_the_next_load(self, monkeypatch):+ b = _make_backend()+ b._process = None+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 200))+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False+ assert b._health_wait_cancelled is True+ b._process = mock.Mock()+ b._process.poll.return_value = None+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is True+ assert b._health_wait_cancelled is False++ def test_a_teardown_from_a_real_thread_is_not_a_crash(self, monkeypatch):+ """Two real threads, not a probe callback: the once-per-iteration read is+ what makes it safe, so drive it that way."""+ b = _make_backend()+ b._process.poll.return_value = None+ in_probe = threading.Event()+ cleared = threading.Event()++ def probe(*a, **kw):+ in_probe.set()+ cleared.wait(2.0)+ return mock.Mock(status_code = 503)++ monkeypatch.setattr(httpx, "get", probe)++ def shutdown():+ in_probe.wait(2.0)+ b._process = None # _kill_process, from run.py's shutdown path+ cleared.set()++ t = threading.Thread(target = shutdown)+ t.start()+ try:+ assert b._wait_for_health(timeout = 30.0, interval = 0.01) is False+ finally:+ t.join(5.0)+ assert b._health_wait_cancelled is True++ def test_a_crash_still_reports_the_exit_code(self, monkeypatch):+ """The teardown guard must not swallow the crash branch: an exited+ process still has to name its exit code and its output."""+ b = _make_backend()+ b._process.poll.return_value = 1+ b._process.returncode = 1+ b._stdout_lines = ["ROCm error: out of memory"]+ monkeypatch.setattr(httpx, "get", lambda *a, **kw: mock.Mock(status_code = 503))+ with mock.patch("core.inference.llama_cpp.logger") as log:+ assert b._wait_for_health(timeout = 1.0, interval = 0.01) is False+ assert any("exited with code 1" in str(c) for c in log.error.call_args_list)+def test_cancel_stops_the_wait_without_a_timeout_marker(self, monkeypatch):b = _make_backend()b._process.poll.return_value = None@@ -424,6 +525,63 @@ class TestCancelledWaitEndsTheLoad:is False)+ def test_a_teardown_mid_load_ends_the_load_instead_of_raising(self, tmp_path, monkeypatch):+ """The whole race through the real caller (#10353): the waiter tests above+ never reach _spawn_and_wait, which is where the traceback resurfaced."""+ import subprocess++ from core.inference.llama_cpp import GgufLoadIntent++ gguf = tmp_path / "model.gguf"+ gguf.write_bytes(b"GGUF" + b"\0" * 4096)++ b = LlamaCppBackend()+ b._find_llama_server_binary = lambda *a, **kw: "/usr/bin/true"+ b._non_chat_gguf_refusal_for_path = lambda *a, **kw: None+ b._non_chat_gguf_refusal = lambda *a, **kw: None+ # The drain thread would iterate a Mock's stdout forever.+ b._drain_stdout = lambda *a, **kw: None++ def _kill():+ b._process = None # the one line of _kill_process this race turns on++ b._kill_process = _kill++ spawns = []+ _real_popen = subprocess.Popen++ def _popen(*a, **kw):+ argv = [str(x) for x in (a[0] if a else kw.get("args") or [])]+ # `--help` and nvidia-smi are not launches; run() needs a real Popen.+ if str(gguf) not in argv:+ return _real_popen(*a, **kw)+ spawns.append(argv)+ proc = mock.Mock()+ proc.poll.return_value = None # alive: shutdown kills it, it does not crash+ proc.pid = 424242+ return proc++ monkeypatch.setattr(subprocess, "Popen", _popen)++ def probe(*a, **kw):+ _kill() # run.py's shutdown, arriving while the load waits+ return mock.Mock(status_code = 503)++ monkeypatch.setattr(httpx, "get", probe)++ assert (+ b.load_model(+ GgufLoadIntent(+ model_identifier = "owner/model",+ gguf_path = str(gguf),+ n_ctx = 4096,+ ),+ )+ is False+ )+ # A second server would outlive the app whose shutdown killed the first.+ assert len(spawns) == 1, f"respawned after shutdown (spawns={len(spawns)})"+def test_a_cancelled_diffusion_start_reaps_the_runner():"""An automatic switch cancels without unloading, so nothing else reaps the shim