Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Installer downloads: retry a body cut short or trickling instead of hashing it (#15763)

Daniel Han93426bf43540
diff --git a/studio/install_node_prebuilt.py b/studio/install_node_prebuilt.py
index ac07031eef9..cf845a4e730 100644
--- a/studio/install_node_prebuilt.py
+++ b/studio/install_node_prebuilt.py
@@ -38,7 +38,7 @@ _STUDIO_DIR = str(Path(__file__).resolve().parent)
if _STUDIO_DIR not in sys.path:
sys.path.insert(0, _STUDIO_DIR)
-from prebuilt_core import DownloadProgress, make_staging_dir # noqa: E402
+from prebuilt_core import DownloadProgress, copy_response_body, make_staging_dir # noqa: E402
try:
from filelock import FileLock, Timeout as FileLockTimeout
@@ -301,15 +301,9 @@ def download_file(url: str, destination: Path) -> None:
if _LOG_TO_STDOUT:
progress.stream = sys.stdout
progress.is_tty = progress.is_tty and sys.stdout.isatty()
- downloaded_bytes = 0
- while True:
- chunk = response.read(1024 * 1024)
- if not chunk:
- break
- handle.write(chunk)
- downloaded_bytes += len(chunk)
- progress.update(downloaded_bytes)
- progress.finish(downloaded_bytes)
+ copy_response_body(
+ response, handle.write, total_bytes = total_bytes, progress = progress
+ )
handle.flush()
os.fsync(handle.fileno())
if not tmp_path.exists() or tmp_path.stat().st_size == 0:
diff --git a/studio/prebuilt_core.py b/studio/prebuilt_core.py
index bc3d711dbc7..d83da798b20 100644
--- a/studio/prebuilt_core.py
+++ b/studio/prebuilt_core.py
@@ -720,6 +720,80 @@ class DownloadProgress:
self.stream.flush()
+class TruncatedDownload(http.client.IncompleteRead):
+ """The server closed the body before Content-Length bytes arrived. http.client's read(amt)
+ returns b"" here instead of raising, so without this a short file is hashed as if whole."""
+
+ def __init__(self, received: int, expected: int) -> None:
+ super().__init__(b"", expected - received)
+ self.received = received
+ self.expected = expected
+
+ def __str__(self) -> str:
+ return f"connection closed after {self.received} of {self.expected} bytes"
+
+ __repr__ = __str__
+
+
+class StalledDownload(TimeoutError):
+ """The body is still arriving but too slowly to finish; the socket timeout never fires on a
+ trickle, so one stalled connection could otherwise hold an install for an hour."""
+
+
+# Bytes that must arrive per window, else the attempt is abandoned and retried on a new connection.
+DOWNLOAD_STALL_WINDOW_SECONDS = 60.0
+DOWNLOAD_STALL_MIN_BYTES = 256 * 1024
+
+
+def copy_response_body(
+ response: Any,
+ write: Callable[[bytes], Any],
+ *,
+ total_bytes: int | None,
+ progress: "DownloadProgress | None" = None,
+ chunk_size: int = 64 * 1024,
+) -> int:
+ """Stream ``response`` into ``write`` and return the byte count. Raises StalledDownload when a
+ window passes with too few bytes, and TruncatedDownload when the body ends short of
+ ``total_bytes``; both are retryable, so the caller's loop reconnects.
+
+ A real HTTPResponse is read with read1, which returns what one receive brings: read(n) waits
+ for all n bytes, so a trickle would block it for hours and the window check would never run.
+ Anything else (file:// bodies, test doubles) keeps plain read."""
+ if isinstance(response, http.client.HTTPResponse):
+ read = response.read1
+ else:
+ read = response.read
+ downloaded = 0
+ window_start = time.monotonic()
+ window_bytes = 0
+ while True:
+ chunk = read(chunk_size)
+ if not chunk:
+ break
+ write(chunk)
+ downloaded += len(chunk)
+ window_bytes += len(chunk)
+ if progress is not None:
+ progress.update(downloaded)
+ now = time.monotonic()
+ if now - window_start >= DOWNLOAD_STALL_WINDOW_SECONDS:
+ if window_bytes < DOWNLOAD_STALL_MIN_BYTES and (
+ total_bytes is None or downloaded < total_bytes
+ ):
+ raise StalledDownload(
+ f"download stalled: {window_bytes} bytes in the last "
+ f"{now - window_start:.0f}s ({downloaded} received so far)"
+ )
+ window_start = now
+ window_bytes = 0
+ if progress is not None:
+ progress.finish(downloaded)
+ if total_bytes is not None and downloaded < total_bytes:
+ raise TruncatedDownload(downloaded, total_bytes)
+ return downloaded
+
+
def download_label_from_url(url: str) -> str:
name = Path(urllib.parse.urlparse(url).path).name
return name or url
@@ -748,15 +822,9 @@ def download_bytes(
total_bytes = int(content_length)
progress = DownloadProgress(progress_label, total_bytes) if progress_label else None
data = bytearray()
- while True:
- chunk = response.read(1024 * 1024)
- if not chunk:
- break
- data.extend(chunk)
- if progress is not None:
- progress.update(len(data))
- if progress is not None:
- progress.finish(len(data))
+ copy_response_body(
+ response, data.extend, total_bytes = total_bytes, progress = progress
+ )
return bytes(data)
except Exception as exc:
last_exc = exc
@@ -830,15 +898,9 @@ def download_file(ops: ModuleOps, url: str, destination: Path) -> None:
if content_length and content_length.isdigit():
total_bytes = int(content_length)
progress = DownloadProgress(f"Downloading {destination.name}", total_bytes)
- downloaded_bytes = 0
- while True:
- chunk = response.read(1024 * 1024)
- if not chunk:
- break
- handle.write(chunk)
- downloaded_bytes += len(chunk)
- progress.update(downloaded_bytes)
- progress.finish(downloaded_bytes)
+ copy_response_body(
+ response, handle.write, total_bytes = total_bytes, progress = progress
+ )
handle.flush()
os.fsync(handle.fileno())
if not tmp_path.exists() or tmp_path.stat().st_size == 0:
diff --git a/tests/studio/install/test_download_truncation_and_stall.py b/tests/studio/install/test_download_truncation_and_stall.py
new file mode 100644
index 00000000000..b5d783e2b06
--- /dev/null
+++ b/tests/studio/install/test_download_truncation_and_stall.py
@@ -0,0 +1,231 @@
+"""Installer downloads retry a body cut short or trickling, instead of hashing or waiting on it.
+
+http.client's read(amt) returns b"" when the server closes before Content-Length, so the copy loop
+used to accept a truncated archive and report it as a checksum mismatch (or, with no published
+digest, hand it to the extractor). A real local server is used because that EOF behaviour is what
+is under test; a BytesIO double cannot reproduce it.
+"""
+
+import importlib.util
+import http.server
+import sys
+import threading
+import time
+from pathlib import Path
+
+import pytest
+
+
+PACKAGE_ROOT = Path(__file__).resolve().parents[3]
+
+
+def _load(name: str, filename: str):
+ spec = importlib.util.spec_from_file_location(name, PACKAGE_ROOT / "studio" / filename)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+NODE = _load("studio_install_node_prebuilt_truncation", "install_node_prebuilt.py")
+LLAMA = _load("studio_install_llama_prebuilt_truncation", "install_llama_prebuilt.py")
+CORE = sys.modules["prebuilt_core"]
+
+PAYLOAD = bytes(range(256)) * 4096 # 1 MiB
+
+
+class _Server:
+ """Serves PAYLOAD with its full Content-Length, closing early for the first `short` requests."""
+
+ def __init__(self, short: int) -> None:
+ self.short = short
+ self.requests = 0
+ server = self
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self): # noqa: N802
+ server.requests += 1
+ self.send_response(200)
+ self.send_header("Content-Length", str(len(PAYLOAD)))
+ self.end_headers()
+ cut = server.requests <= server.short
+ self.wfile.write(PAYLOAD[: len(PAYLOAD) // 4] if cut else PAYLOAD)
+ self.wfile.flush()
+ self.close_connection = True
+
+ def log_message(self, *args):
+ pass
+
+ self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}/node.tar.gz"
+ self.thread = threading.Thread(target = self.httpd.serve_forever, daemon = True)
+
+ def __enter__(self):
+ self.thread.start()
+ return self
+
+ def __exit__(self, *exc):
+ self.httpd.shutdown()
+ self.httpd.server_close()
+
+
[email protected](autouse = True)
+def _no_backoff(monkeypatch):
+ monkeypatch.setattr(NODE, "sleep_backoff", lambda *a, **k: None)
+ monkeypatch.setattr(CORE, "sleep_backoff", lambda *a, **k: None)
+
+
+def test_a_truncated_body_is_retryable_not_accepted():
+ with _Server(short = 1) as srv:
+ import urllib.request
+ with urllib.request.urlopen(srv.url, timeout = 10) as response:
+ sink = bytearray()
+ with pytest.raises(CORE.TruncatedDownload) as info:
+ CORE.copy_response_body(
+ response, sink.extend, total_bytes = int(response.headers["Content-Length"])
+ )
+ assert info.value.received == len(PAYLOAD) // 4
+ assert info.value.expected == len(PAYLOAD)
+ assert NODE.is_retryable_url_error(info.value)
+ assert CORE.is_retryable_url_error(info.value)
+
+
[email protected]("which", ["node", "llama"])
+def test_download_file_reconnects_after_a_truncated_body(tmp_path, which):
+ download_file = NODE.download_file if which == "node" else LLAMA.download_file
+ with _Server(short = 2) as srv:
+ destination = tmp_path / "node.tar.gz"
+ download_file(srv.url, destination)
+ assert srv.requests == 3
+ assert destination.read_bytes() == PAYLOAD
+
+
+def test_download_bytes_reconnects_after_a_truncated_body():
+ with _Server(short = 1) as srv:
+ data = LLAMA.download_bytes(srv.url)
+ assert srv.requests == 2
+ assert data == PAYLOAD
+
+
+def test_a_body_that_never_finishes_short_is_still_refused(tmp_path):
+ """Every attempt truncated: the error surfaces, and no partial file is left behind."""
+ with _Server(short = 99) as srv:
+ destination = tmp_path / "node.tar.gz"
+ with pytest.raises(CORE.TruncatedDownload):
+ NODE.download_file(srv.url, destination)
+ assert srv.requests == NODE.HTTP_FETCH_ATTEMPTS
+ assert not destination.exists()
+ assert list(tmp_path.iterdir()) == []
+
+
+class _Trickle:
+ """A response that keeps sending tiny chunks, so no single read ever times out."""
+
+ def __init__(self, chunks: int, delay: float) -> None:
+ self.left = chunks
+ self.delay = delay
+
+ def read(self, amt):
+ if self.left == 0:
+ return b""
+ self.left -= 1
+ time.sleep(self.delay)
+ return b"x"
+
+
+def test_a_trickling_body_is_abandoned(monkeypatch):
+ monkeypatch.setattr(CORE, "DOWNLOAD_STALL_WINDOW_SECONDS", 0.05)
+ with pytest.raises(CORE.StalledDownload) as info:
+ CORE.copy_response_body(_Trickle(1000, 0.01), lambda _b: None, total_bytes = 10_000)
+ assert CORE.is_retryable_url_error(info.value)
+ assert NODE.is_retryable_url_error(info.value)
+
+
+def test_a_slow_but_moving_body_is_not_abandoned(monkeypatch):
+ monkeypatch.setattr(CORE, "DOWNLOAD_STALL_WINDOW_SECONDS", 0.05)
+ monkeypatch.setattr(CORE, "DOWNLOAD_STALL_MIN_BYTES", 1)
+ sink = bytearray()
+ n = CORE.copy_response_body(_Trickle(20, 0.01), sink.extend, total_bytes = 20)
+ assert n == 20 and bytes(sink) == b"x" * 20
+
+
+def test_the_last_bytes_arriving_slowly_do_not_count_as_a_stall(monkeypatch):
+ """A window that closes on the final byte has nothing left to wait for."""
+ monkeypatch.setattr(CORE, "DOWNLOAD_STALL_WINDOW_SECONDS", 0.0)
+ n = CORE.copy_response_body(_Trickle(1, 0.0), lambda _b: None, total_bytes = 1)
+ assert n == 1
+
+
+class _TrickleServer:
+ """The first `slow` requests get the full Content-Length but one byte every 50 ms, for at most
+ ten seconds; later requests get the whole body at once."""
+
+ def __init__(self, slow: int) -> None:
+ self.slow = slow
+ self.requests = 0
+ server = self
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self): # noqa: N802
+ server.requests += 1
+ self.send_response(200)
+ self.send_header("Content-Length", str(len(PAYLOAD)))
+ self.end_headers()
+ try:
+ if server.requests <= server.slow:
+ for i in range(200):
+ self.wfile.write(PAYLOAD[i : i + 1])
+ self.wfile.flush()
+ time.sleep(0.05)
+ else:
+ self.wfile.write(PAYLOAD)
+ self.wfile.flush()
+ except OSError:
+ pass # the client gave up on the trickle
+ self.close_connection = True
+
+ def log_message(self, *args):
+ pass
+
+ self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.httpd.daemon_threads = True
+ self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}/node.tar.gz"
+ self.thread = threading.Thread(target = self.httpd.serve_forever, daemon = True)
+
+ def __enter__(self):
+ self.thread.start()
+ return self
+
+ def __exit__(self, *exc):
+ self.httpd.shutdown()
+ self.httpd.server_close()
+
+
+def test_a_real_trickling_connection_is_abandoned_within_the_window(monkeypatch):
+ """read(n) on an HTTPResponse waits for all n bytes, so the watchdog only works if the copy
+ returns per receive. With the old read the call blocks until the server gives up (~10 s)."""
+ import urllib.request
+
+ monkeypatch.setattr(CORE, "DOWNLOAD_STALL_WINDOW_SECONDS", 0.3)
+ with _TrickleServer(slow = 1) as srv:
+ start = time.monotonic()
+ with urllib.request.urlopen(srv.url, timeout = 10) as response:
+ with pytest.raises(CORE.StalledDownload):
+ CORE.copy_response_body(
+ response, lambda _b: None, total_bytes = int(response.headers["Content-Length"])
+ )
+ assert time.monotonic() - start < 3.0
+
+
[email protected]("which", ["node", "llama"])
+def test_download_file_reconnects_after_a_real_trickle(monkeypatch, tmp_path, which):
+ monkeypatch.setattr(CORE, "DOWNLOAD_STALL_WINDOW_SECONDS", 0.3)
+ download_file = NODE.download_file if which == "node" else LLAMA.download_file
+ with _TrickleServer(slow = 1) as srv:
+ destination = tmp_path / "node.tar.gz"
+ start = time.monotonic()
+ download_file(srv.url, destination)
+ assert srv.requests == 2
+ assert time.monotonic() - start < 5.0
+ assert destination.read_bytes() == PAYLOAD