Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop unsloth start from killing a model download that is still running (#10453)

Nilay4962fca61614
diff --git a/tests/studio/test_cli_start_download_liveness.py b/tests/studio/test_cli_start_download_liveness.py
new file mode 100644
index 00000000000..c2b6766792f
--- /dev/null
+++ b/tests/studio/test_cli_start_download_liveness.py
@@ -0,0 +1,404 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""`unsloth start` must not kill an auto-started server whose model is still downloading."""
+
+from __future__ import annotations
+
+import pytest
+import typer
+
+import unsloth_cli.commands.start as start_cli
+
+
+BASE = "http://127.0.0.1:8888"
+MODEL = "unsloth/Qwen3-Coder-480B-A35B-Instruct-GGUF"
+KEY_LINE = f"{start_cli._START_API_KEY_PREFIX}sk-unsloth-test\n"
+EXPECTED_BYTES = 500 * 1024**3
+STEP_S = 120.0
+# A loop that keeps resetting its deadline never returns, so stop it and say so rather
+# than hanging the suite. At STEP_S this is ~7 days of fake wall clock.
+MAX_ITERATIONS = 5000
+
+
+def health_poll_line(n):
+ """What `LoggingMiddleware` logs for the `/api/health` GET this loop just made."""
+ return (
+ f"2026-09-08 03:{n // 60:02d}:{n % 60:02d} [info ] request_completed"
+ " method=GET path=/api/health process_time_ms=0.4 status_code=200\n"
+ )
+
+
+def failed_health_poll_record(n):
+ """A health probe that raises: the JSON record, then the traceback `with_readable_traceback`
+ echoes on the lines after it, each carrying `loggers.config._TRACEBACK_ECHO_PREFIX`."""
+ return (
+ f'{{"event": "request_failed", "path": "/api/health", "status_code": 500, "n": {n}}}\n'
+ "| Traceback (most recent call last):\n"
+ '| File "/opt/unsloth/studio/backend/main.py", line 1796, in health\n'
+ "| return await _hardware_snapshot()\n"
+ "| RuntimeError: hardware probe timed out\n"
+ )
+
+
+def load_watchdog_heartbeat(n):
+ """`start_watchdog`'s on_heartbeat -> `{"type": "status"}` -> the orchestrator's
+ `Subprocess status` INFO line. Timer driven, so it keeps coming while a load is wedged."""
+ return (
+ f"2026-09-08 03:{n // 60:02d}:{n % 60:02d} [info ] Subprocess status: still loading\n"
+ )
+
+
+class FakeClock:
+ """monotonic() only moves when the readiness loop sleeps, so runs are deterministic."""
+
+ def __init__(self, step):
+ self.now = 1000.0
+ self.start = 1000.0
+ self.step = step
+
+ def monotonic(self):
+ return self.now
+
+ def sleep(self, seconds):
+ self.now += self.step
+
+ @property
+ def elapsed(self):
+ return self.now - self.start
+
+
+class FakePopen:
+ def poll(self):
+ return None
+
+
+class Harness:
+ def __init__(
+ self,
+ monkeypatch,
+ *,
+ downloaded_bytes = 0,
+ chunk_bytes = 0,
+ chatter = None,
+ fail_every = 0,
+ fail_first = 0,
+ unmeasured_every = 0,
+ unmeasured_grows = False,
+ vanish_every = 0,
+ rebound = False,
+ ready_at = None,
+ tail = KEY_LINE,
+ ):
+ self.clock = FakeClock(STEP_S)
+ self.log_path = None
+ self.chatter = chatter
+ self.fail_every = fail_every
+ self.fail_first = fail_first
+ self.failures = 0
+ self.unmeasured_every = unmeasured_every
+ self.unmeasured_grows = unmeasured_grows
+ self.unmeasured = 0
+ self.vanish_every = vanish_every
+ self.vanished = 0
+ self.rebound = rebound
+ self.downloaded_bytes = downloaded_bytes
+ self.chunk_bytes = chunk_bytes
+ self.ready_at = ready_at
+ self.tail = tail
+ self.server = FakePopen()
+ self.iterations = 0
+ self.polls = 0
+ self.shutdowns = []
+ monkeypatch.setattr(start_cli, "time", self.clock)
+ monkeypatch.setattr(start_cli, "_http_json", self.http_json)
+ monkeypatch.setattr(start_cli, "_log_tail", self.log_tail)
+ monkeypatch.setattr(start_cli, "_studio_healthy", self.studio_healthy)
+ monkeypatch.setattr(start_cli, "_shutdown_server", self.shutdowns.append)
+ monkeypatch.setattr(start_cli, "_auto_served_server", None)
+ monkeypatch.setattr(start_cli.atexit, "register", lambda *a, **k: None)
+ monkeypatch.setattr(start_cli.subprocess, "Popen", lambda *a, **k: self.server)
+
+ def http_json(
+ self,
+ method,
+ url,
+ token,
+ payload = None,
+ timeout = 30,
+ error = None,
+ ):
+ if "gguf-variants" in url:
+ return {
+ "default_variant": "Q4_K_M",
+ "variants": [{"quant": "Q4_K_M", "download_size_bytes": EXPECTED_BYTES}],
+ }
+ if "download-progress" in url:
+ self.polls += 1
+ if self.polls <= self.fail_first or (
+ self.fail_every and self.polls % self.fail_every == 0
+ ):
+ self.failures += 1
+ raise TimeoutError("the server took too long to answer")
+ if self.rebound:
+ # Two roots, one intermittently unreadable and nothing downloading: the
+ # partial scan sees less than the complete one, over and over.
+ if self.polls % 2 == 0:
+ return {
+ "downloaded_bytes": 12 * 1024**3,
+ "expected_bytes": EXPECTED_BYTES,
+ "progress": 0,
+ "cache_measured": True,
+ }
+ self.unmeasured += 1
+ return {
+ "downloaded_bytes": 5 * 1024**3,
+ "expected_bytes": EXPECTED_BYTES,
+ "progress": 0,
+ "cache_measured": False,
+ }
+ if self.vanish_every and self.polls % self.vanish_every == 0:
+ # A cache mount that disappears cleanly is a MEASURED absence -- no scan
+ # error -- so the count drops to zero and returns on the next remount.
+ self.vanished += 1
+ return {
+ "downloaded_bytes": 0,
+ "expected_bytes": EXPECTED_BYTES,
+ "progress": 0,
+ "cache_measured": True,
+ }
+ if self.unmeasured_every and self.polls % self.unmeasured_every == 0:
+ # A scan the server could not finish: 200, cache_measured false. Zero bytes
+ # is the unreadable-root case; a growing count is the readable-root one,
+ # which `snapshot_progress.py` documents as a real lower bound.
+ self.unmeasured += 1
+ if self.unmeasured_grows:
+ self.downloaded_bytes += self.chunk_bytes
+ return {
+ "downloaded_bytes": self.downloaded_bytes if self.unmeasured_grows else 0,
+ "expected_bytes": EXPECTED_BYTES,
+ "progress": 0,
+ "cache_measured": False,
+ }
+ self.downloaded_bytes += self.chunk_bytes
+ return {
+ "downloaded_bytes": self.downloaded_bytes,
+ "expected_bytes": EXPECTED_BYTES,
+ "progress": self.downloaded_bytes / EXPECTED_BYTES,
+ "cache_measured": True,
+ }
+ raise AssertionError(f"unexpected request: {method} {url}")
+
+ def log_tail(
+ self,
+ path,
+ lines = 20,
+ ):
+ # The real file the child writes to, so `_ServerLogProgress` reads it for real.
+ self.log_path = path
+ return self.tail
+
+ def studio_healthy(
+ self,
+ base,
+ timeout = 3.0,
+ ):
+ self.iterations += 1
+ assert self.iterations <= MAX_ITERATIONS, (
+ f"readiness loop still running after {self.iterations} passes "
+ f"({self.clock.elapsed:.0f}s of fake clock); its deadline never expires"
+ )
+ if self.chatter and self.log_path is not None:
+ with open(self.log_path, "ab") as handle:
+ handle.write(self.chatter(self.iterations).encode())
+ if self.ready_at is not None and self.iterations >= self.ready_at:
+ self.tail = f"{KEY_LINE}Model loaded: {MODEL}\n"
+ return True
+ return False
+
+ def start(self):
+ return start_cli._start_studio_server(BASE, MODEL, start_cli.LoadOptions())
+
+
+def test_a_live_download_survives_past_the_idle_cap(monkeypatch):
+ harness = Harness(
+ monkeypatch,
+ chunk_bytes = 1024**3,
+ ready_at = 40,
+ )
+
+ server = harness.start()
+
+ assert server is harness.server
+ assert harness.shutdowns == []
+ assert harness.polls >= 40
+ # The transfer outlives the cap in wall clock: the case that used to be killed.
+ assert harness.clock.elapsed > start_cli._SERVER_START_TIMEOUT_S
+
+
+def test_a_stalled_download_still_times_out(monkeypatch, capsys):
+ harness = Harness(
+ monkeypatch,
+ downloaded_bytes = 12 * 1024**3,
+ chunk_bytes = 0,
+ )
+
+ with pytest.raises(typer.Exit):
+ harness.start()
+
+ assert harness.shutdowns == [harness.server]
+ message = capsys.readouterr().err
+ assert f"made no progress for {start_cli._SERVER_START_TIMEOUT_S}s" in message
+ assert harness.clock.elapsed < 2 * start_cli._SERVER_START_TIMEOUT_S
+
+
+def test_a_server_that_never_downloads_still_times_out(monkeypatch, capsys):
+ harness = Harness(monkeypatch, tail = "starting\n")
+
+ with pytest.raises(typer.Exit):
+ harness.start()
+
+ assert harness.polls == 0
+ assert harness.shutdowns == [harness.server]
+ assert f"made no progress for {start_cli._SERVER_START_TIMEOUT_S}s" in capsys.readouterr().err
+ assert harness.clock.elapsed < 2 * start_cli._SERVER_START_TIMEOUT_S
+
+
+ "chatter",
+ [
+ pytest.param(health_poll_line, id = "the loop's own health poll"),
+ pytest.param(failed_health_poll_record, id = "traceback echo lines"),
+ pytest.param(load_watchdog_heartbeat, id = "load watchdog heartbeat"),
+ ],
+)
+def test_a_wedged_server_that_keeps_writing_still_times_out(monkeypatch, capsys, chatter):
+ # Every one of these keeps the log growing on a timer while nothing loads, so log
+ # growth cannot stand in for progress: only fresh download bytes may move the deadline.
+ harness = Harness(
+ monkeypatch,
+ downloaded_bytes = EXPECTED_BYTES,
+ chunk_bytes = 0,
+ chatter = chatter,
+ )
+
+ with pytest.raises(typer.Exit):
+ harness.start()
+
+ assert harness.shutdowns == [harness.server]
+ assert f"made no progress for {start_cli._SERVER_START_TIMEOUT_S}s" in capsys.readouterr().err
+ assert harness.clock.elapsed < 2 * start_cli._SERVER_START_TIMEOUT_S
+
+
+def test_a_transient_progress_error_does_not_blind_the_loop(monkeypatch):
+ # One slow reading used to disable the reader for good, which would now leave a live
+ # download with no signal at all and kill it at the cap.
+ harness = Harness(
+ monkeypatch,
+ chunk_bytes = 1024**3,
+ fail_every = 3,
+ ready_at = 40,
+ )
+
+ server = harness.start()
+
+ assert server is harness.server
+ assert harness.shutdowns == []
+ assert harness.failures >= 10
+ assert harness.clock.elapsed > start_cli._SERVER_START_TIMEOUT_S
+
+
+def test_an_unmeasured_reading_is_not_progress(monkeypatch, capsys):
+ # `snapshot_progress_response` answers 200 with zero bytes when it cannot finish the
+ # cache scan. Believing it would drop the count to zero, so the next real reading of
+ # the same cached bytes would read as fresh growth and renew the deadline forever.
+ harness = Harness(
+ monkeypatch,
+ downloaded_bytes = 12 * 1024**3,
+ chunk_bytes = 0,
+ unmeasured_every = 2,
+ )
+
+ with pytest.raises(typer.Exit):
+ harness.start()
+
+ assert harness.unmeasured > 0
+ assert harness.shutdowns == [harness.server]
+ assert f"made no progress for {start_cli._SERVER_START_TIMEOUT_S}s" in capsys.readouterr().err
+ assert harness.clock.elapsed < 2 * start_cli._SERVER_START_TIMEOUT_S
+
+
+def test_polling_keeps_probing_after_a_long_burst_of_errors(monkeypatch):
+ # Bytes are the only thing separating a live transfer from a wedged server, so a run
+ # of failures may back the reader off but must never retire it: the download it would
+ # abandon is the one this whole path exists to keep alive. Six in a row is past the
+ # point where the reader used to disable itself for the rest of the startup.
+ harness = Harness(
+ monkeypatch,
+ chunk_bytes = 1024**3,
+ fail_first = 6,
+ ready_at = 40,
+ )
+
+ server = harness.start()
+
+ assert server is harness.server
+ assert harness.shutdowns == []
+ assert harness.failures >= 6
+ assert harness.downloaded_bytes > 0 # it recovered and saw the transfer again
+ assert harness.clock.elapsed > start_cli._SERVER_START_TIMEOUT_S
+
+
+def test_a_growing_unmeasured_reading_still_counts(monkeypatch):
+ # One cache root unreadable while the download lands in another: the backend answers
+ # `cache_measured: false` with the readable root's real, growing byte count. That is a
+ # lower bound, not an unknown, so rejecting it would kill a live transfer at the cap.
+ harness = Harness(
+ monkeypatch,
+ chunk_bytes = 1024**3,
+ unmeasured_every = 1,
+ unmeasured_grows = True,
+ ready_at = 40,
+ )
+
+ server = harness.start()
+
+ assert server is harness.server
+ assert harness.shutdowns == []
+ assert harness.unmeasured >= 40 # every reading came back unmeasured
+ assert harness.clock.elapsed > start_cli._SERVER_START_TIMEOUT_S
+
+
+def test_a_partial_scan_rebound_is_not_progress(monkeypatch, capsys):
+ # Nothing is downloading; one cache root simply comes and goes. Letting the partial
+ # scan lower the baseline would turn the next complete scan of the very same bytes
+ # into growth, and a wedged server would be renewed for as long as the mount flaps.
+ harness = Harness(monkeypatch, rebound = True)
+
+ with pytest.raises(typer.Exit):
+ harness.start()
+
+ assert harness.unmeasured > 0
+ assert harness.shutdowns == [harness.server]
+ assert f"made no progress for {start_cli._SERVER_START_TIMEOUT_S}s" in capsys.readouterr().err
+ assert harness.clock.elapsed < 2 * start_cli._SERVER_START_TIMEOUT_S
+
+
+def test_a_vanished_cache_mount_is_not_progress(monkeypatch, capsys):
+ # Nothing is downloading; a cache mount just comes and goes. Its absence is reported
+ # as a MEASURED zero, so a baseline that followed readings down would read every
+ # remount as fresh growth and keep a wedged server waiting for as long as it flaps.
+ harness = Harness(
+ monkeypatch,
+ downloaded_bytes = 12 * 1024**3,
+ chunk_bytes = 0,
+ vanish_every = 2,
+ )
+
+ with pytest.raises(typer.Exit):
+ harness.start()
+
+ assert harness.vanished > 0
+ assert harness.shutdowns == [harness.server]
+ assert f"made no progress for {start_cli._SERVER_START_TIMEOUT_S}s" in capsys.readouterr().err
+ assert harness.clock.elapsed < 2 * start_cli._SERVER_START_TIMEOUT_S
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index 04cfa0bcf55..73eca70c19e 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -873,9 +873,12 @@ def _http_json(
# failure paths and the atexit backstop can tear it down without threading a handle
# through all six agent commands. Only one agent runs per process, so one slot is enough.
_auto_served_server: Optional[subprocess.Popen] = None
-# Model download + load can be slow; give the auto-started server room before giving up.
+# Model download + load can be slow, so this caps time since the download last
+# advanced, not total elapsed time (see `_start_studio_server`).
_SERVER_START_TIMEOUT_S = 900
_DOWNLOAD_POLL_INTERVAL_S = 1.0
+# Ceiling on the doubling back-off the progress reader uses after a polling error.
+_DOWNLOAD_POLL_MAX_BACKOFF_S = 60.0
_START_API_KEY_PREFIX = "UNSLOTH_START_API_KEY: "
_START_API_KEY_MARKER_ENV = "_UNSLOTH_START_API_KEY_MARKER"
@@ -1004,6 +1007,9 @@ class _ModelDownloadProgress:
self._model = model
self._variant = variant or ""
self._expected_bytes = 0
+ self._downloaded_bytes = 0
+ self._failures = 0
+ self._retry_at = 0.0
self._display = _DownloadProgressDisplay()
self._configured = False
self._disabled = not _is_hub_model_id(model)
@@ -1054,6 +1060,8 @@ class _ModelDownloadProgress:
self._configure()
if self._disabled:
return
+ if time.monotonic() < self._retry_at:
+ return
try:
if self._variant or "gguf" in self._model.lower():
params = urlencode(
@@ -1077,10 +1085,34 @@ class _ModelDownloadProgress:
self._progress_prefix = "/api/models"
self.poll()
return
+ # The liveness baseline only ever rises. A reading falls for reasons that are
+ # not "bytes left the disk": an incomplete scan reporting a lower bound, a
+ # cache mount vanishing cleanly (`hf_cache_state._safe_is_dir` calls that a
+ # measured absence, not an error), an XET run purging its partial. Following a
+ # reading down would make the recovery back to the same figure look like fresh
+ # growth and renew the deadline for a server that is downloading nothing, and a
+ # flapping mount could do that forever. The cost is that a transfer which truly
+ # restarts is not counted again until it passes its own high mark; that failure
+ # is bounded and says so, where a false renewal is an unbounded wait.
+ self._downloaded_bytes = max(
+ self._downloaded_bytes, max(0, int(reading.get("downloaded_bytes") or 0))
+ )
+ self._failures = 0
+ self._retry_at = 0.0
self._display.update(reading)
except Exception:
- # Progress is best-effort; never fail the load over a polling error.
- self._disabled = True
+ # Progress is best-effort and never fails the load, but `_start_studio_server`
+ # reads `downloaded_bytes` to tell a live transfer from a wedged one. Backing
+ # off keeps a broken endpoint cheap; giving up for good would kill the very
+ # download this exists to protect, so it never stops probing.
+ self._failures += 1
+ self._retry_at = time.monotonic() + min(
+ 2.0**self._failures, _DOWNLOAD_POLL_MAX_BACKOFF_S
+ )
+
+ @property
+ def downloaded_bytes(self) -> int:
+ return self._downloaded_bytes
def close(self) -> None:
self._display.close()
@@ -1323,6 +1355,7 @@ def _start_studio_server(
deadline = time.monotonic() + _SERVER_START_TIMEOUT_S
progress: Optional[_ModelDownloadProgress] = None
+ downloaded_bytes = 0
early_key_seen = False
try:
while time.monotonic() < deadline:
@@ -1348,6 +1381,15 @@ def _start_studio_server(
)
if progress is not None:
progress.poll()
+ # Fresh bytes are the one unambiguous sign the child is moving, so the cap
+ # measures time since the transfer last advanced rather than total elapsed.
+ # Server log growth deliberately does NOT count: the loop's own health poll,
+ # echoed tracebacks and the loader's watchdog heartbeat all keep the log
+ # growing while nothing loads, which would make this deadline unreachable.
+ bytes_now = progress.downloaded_bytes if progress is not None else 0
+ if bytes_now > downloaded_bytes:
+ deadline = time.monotonic() + _SERVER_START_TIMEOUT_S
+ downloaded_bytes = bytes_now
# New children emit an early key marker, so wait for the final model banner;
# older children only print the key after load, so fall back to that.
ready_signal = "Model loaded:" in tail if early_key_seen else "sk-unsloth-" in tail
@@ -1363,7 +1405,8 @@ def _start_studio_server(
progress.close()
_shutdown_auto_served()
_fail(
- f"The Unsloth server didn't become ready within {_SERVER_START_TIMEOUT_S}s. See {log_path}."
+ "The Unsloth server didn't become ready and made no progress for "
+ f"{_SERVER_START_TIMEOUT_S}s. See {log_path}."
)
diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py
index 7fb7063fe24..db47c3417de 100644
--- a/unsloth_cli/tests/test_start.py
+++ b/unsloth_cli/tests/test_start.py
@@ -3944,6 +3944,8 @@ def test_start_studio_server_polls_progress_from_early_key(monkeypatch):
created = []
class FakeProgress:
+ downloaded_bytes = 0
+
def __init__(self, base, key, model, variant):
created.append((base, key, model, variant, "created"))