The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(pm): avoid delaying available mirrors
akfoster8607c509f072
diff --git a/pm/downloader.py b/pm/downloader.pyindex c1b00d5d917..d3fbe3dbbbe 100644--- a/pm/downloader.py+++ b/pm/downloader.py@@ -393,7 +393,8 @@ class Download:configured_connections = self.connectionstry:size = retry_network(fetch, wait=self._wait_retry,- progress=(lambda: self._durable_done) if self.resume and resumable else None)+ progress=(lambda: self._durable_done) if self.resume and resumable else None,+ fallback_available=bool(source.fallbacks))except (OSError, http.client.HTTPException) as exc:if isinstance(exc, urllib.error.URLError) or is_transient(exc):raise DownloadTransportError(source.url, exc) from excdiff --git a/pm/network.py b/pm/network.pyindex 070e1b6d042..d12cc1b1927 100644--- a/pm/network.py+++ b/pm/network.py@@ -90,7 +90,7 @@ def _delay(exc: Exception, attempt: int) -> float:def retry_network(operation: Callable[[], _T], *, wait: Callable[[float], None] | None = None,- progress: Callable[[], int] | None = None,+ progress: Callable[[], int] | None = None, fallback_available: bool = False,) -> _T:"""Retry the complete request and body read, never an install or publish.@@ -98,6 +98,7 @@ def retry_network(A download supplies an interruptible wait so pause does not wait for backoff.Downloads supply ``progress``: durable forward progress renews theirbudget. Metadata and non-resumable requests still get four attempts.+ A resumable request yields to an available fallback before a long wait."""failures_since_progress = 0backoff = 0@@ -122,7 +123,7 @@ def retry_network(delay = _delay(exc, backoff)if progress is None:delay = min(delay, _MAX_DELAY)- elif delay > _MAX_RECOVERY_DELAY:+ elif delay > (_MAX_DELAY if fallback_available else _MAX_RECOVERY_DELAY):raisebackoff += 1logging.getLogger(__name__).warning(diff --git a/tests/pm/test_download_continuity.py b/tests/pm/test_download_continuity.pyindex c588bd81e02..af843d994e7 100644--- a/tests/pm/test_download_continuity.py+++ b/tests/pm/test_download_continuity.py@@ -10,18 +10,18 @@ import subprocessimport sysfrom http.server import BaseHTTPRequestHandlerfrom threading import Event-from types import SimpleNamespaceimport pytestfrom pm import downloader, filesystem-from pm.downloader import Download, Source+from pm.downloader import Download, DownloadTransportError, Sourcefrom tests.pm._fixtures import threaded_serverfrom tests.pm._range_server import RangeHandler, dl_server as dl_server, url # noqa: PLC0414 -- pytest fixture re-exportfrom tests.waiting import wait_until-def test_useful_progress_survives_more_than_four_interrupted_bodies(tmp_path, monkeypatch):[email protected]("stalls_after_progress", [False, True])+def test_download_retries_renew_only_while_bytes_are_saved(tmp_path, monkeypatch, stalls_after_progress):block = 1 << 20payload = bytes(range(256)) * (10 * block // 256)requests = []@@ -38,6 +38,8 @@ def test_useful_progress_survives_more_than_four_interrupted_bodies(tmp_path, moif not probe:requests.append((start, end))stop = min(end + 1, start + block) if not probe and len(requests) <= 6 else end + 1+ if not probe and stalls_after_progress and len(requests) > 1:+ stop = startself.wfile.write(payload[start:stop])self.wfile.flush()self.close_connection = True@@ -51,10 +53,16 @@ def test_useful_progress_survives_more_than_four_interrupted_bodies(tmp_path, mohashlib.sha256(payload).hexdigest())download = Download([source], connections=1, partials_dir=tmp_path / "partials")monkeypatch.setattr(download, "_wait_retry", lambda _: None)- download.run()- assert destination.read_bytes() == payload- assert len(requests) == 7- assert [start for start, _ in requests] == [index * block for index in range(7)]+ if stalls_after_progress:+ with pytest.raises(DownloadTransportError):+ download.run()+ assert not destination.exists()+ assert download._durable_done == block+ assert [start for start, _ in requests] == [0, block, block, block, block]+ else:+ download.run()+ assert destination.read_bytes() == payload+ assert [start for start, _ in requests] == [index * block for index in range(7)]def test_forced_termination_recovers_checkpoint_before_request_finishes(tmp_path):@@ -121,60 +129,6 @@ def test_forced_termination_recovers_checkpoint_before_request_finishes(tmp_pathassert requests[1][0] >= checkpoint-def test_checkpoints_are_time_based_not_byte_based(tmp_path, monkeypatch, dl_server):- payload = b"w" * (32 << 20)- RangeHandler.payloads = {"/fast": payload}- download = downloader.Download(- [downloader.Source(url(dl_server, "/fast"), tmp_path / "model", hashlib.sha256(payload).hexdigest())],- partials_dir=tmp_path / "partials", connections=1,- )- checkpoints = []- original = download._write_sidecar-- def save(*args):- checkpoints.append(None)- return original(*args)-- monkeypatch.setattr(download, "_write_sidecar", save)- monkeypatch.setattr(downloader, "time", SimpleNamespace(monotonic=lambda: 0.0))- download.run()- assert (tmp_path / "model").read_bytes() == payload- assert len(checkpoints) == 1 # The attempt still saves its final checkpoint.--[email protected]("failed_stage", ["probe", "single-stream"])-def test_non_resumable_requests_keep_the_existing_retry_after_cap(tmp_path, monkeypatch, failed_stage):- payload = b"model bytes"- failed = False-- class Handler(BaseHTTPRequestHandler):- def do_GET(self):- nonlocal failed- probe = self.headers.get("Range") == "bytes=0-0"- if not failed and probe == (failed_stage == "probe"):- failed = True- self.send_response(503)- self.send_header("Retry-After", "60")- self.end_headers()- return- self.send_response(200) # This source does not support ranges.- self.send_header("Content-Length", str(len(payload)))- self.end_headers()- self.wfile.write(payload)-- def log_message(self, *args):- pass-- waits = []- with threaded_server(Handler) as server:- source = downloader.Source(f"http://127.0.0.1:{server.server_port}/model", tmp_path / "model")- download = downloader.Download([source], partials_dir=tmp_path / "partials")- monkeypatch.setattr(download, "_wait_retry", waits.append)- download.run()- assert waits == [30]- assert source.dest.read_bytes() == payload--def test_failed_periodic_checkpoint_retries_without_claiming_saved_bytes(tmp_path, monkeypatch, caplog):payload = b"checkpointed model bytes"part = tmp_path / "model.part"@@ -209,22 +163,32 @@ def test_failed_periodic_checkpoint_retries_without_claiming_saved_bytes(tmp_patassert any(record.name == "pm.downloader" and record.levelname == "WARNING" for record in caplog.records)-def test_final_checkpoint_write_failure_does_not_publish_the_model(tmp_path, monkeypatch, dl_server):- payload = b"model bytes"- RangeHandler.payloads = {"/strict": payload}- source = Source(url(dl_server, "/strict"), tmp_path / "model", hashlib.sha256(payload).hexdigest())- download = Download([source], partials_dir=tmp_path / "partials", connections=1)- side = download.partials_dir / f"{download._key(source.url)}.ranges"- replace = filesystem.os.replace-- def publish(temporary, destination):- if Path(destination) == side:- raise PermissionError(errno.EACCES, "checkpoint temporarily held")- return replace(temporary, destination)[email protected]("retry_after", [5, 120, 290, 600])+def test_long_primary_wait_yields_to_a_mirror_that_can_wait_longer(tmp_path, monkeypatch, dl_server, retry_after):+ payload = b"pinned mirror bytes"+ RangeHandler.payloads = {"/primary": payload, "/mirror": payload}+ original = RangeHandler.do_GET+ mirror_requests = []++ def respond(handler):+ if handler.headers.get("Range") != "bytes=0-0":+ if handler.path == "/mirror":+ mirror_requests.append(None)+ if handler.path == "/primary" or len(mirror_requests) == 1:+ handler.send_response(503)+ handler.send_header("Retry-After", str(retry_after if handler.path == "/primary" else 60))+ handler.send_header("Content-Length", "0")+ handler.end_headers()+ return+ original(handler)- monkeypatch.setattr(filesystem.os, "replace", publish)- monkeypatch.setattr(filesystem, "_HELD_RETRY_DELAYS", ())- monkeypatch.setattr(downloader, "time", SimpleNamespace(monotonic=lambda: 0.0))- with pytest.raises(PermissionError):- download.run()- assert not source.dest.exists() and download._durable_done == 0+ monkeypatch.setattr(RangeHandler, "do_GET", respond)+ destination = tmp_path / "model"+ source = Source(url(dl_server, "/primary"), destination, hashlib.sha256(payload).hexdigest(),+ fallbacks=(url(dl_server, "/mirror"),))+ download = Download([source], partials_dir=tmp_path / "partials", connections=1)+ waits = []+ monkeypatch.setattr(download, "_wait_retry", waits.append)+ download.run()+ assert destination.read_bytes() == payload+ assert waits == ([5, 5, 5] if retry_after == 5 else []) + [60]diff --git a/tests/pm/test_network.py b/tests/pm/test_network.pyindex 5f094b7e501..a0d27759e68 100644--- a/tests/pm/test_network.py+++ b/tests/pm/test_network.py@@ -134,20 +134,3 @@ def test_resumable_transfer_honors_retry_after_or_stops(retry_after, expected):else:assert network.retry_network(request, wait=waits.append, progress=lambda: 0) == "complete"assert waits == [expected]---def test_progress_renews_only_the_consecutive_failure_budget():- attempts = []- waits = []- durable = [0]-- def request():- attempts.append(None)- if len(attempts) == 1:- durable[0] += 1- raise ConnectionResetError("disconnected")-- with pytest.raises(ConnectionResetError):- network.retry_network(request, wait=waits.append, progress=lambda: durable[0])- assert len(attempts) == 1 + network._ATTEMPTS- assert waits == [1, 2, 4, 8]diff --git a/website/docs/reference/package-management.md b/website/docs/reference/package-management.mdindex 0db3bc6ccf5..61b44aefc39 100644--- a/website/docs/reference/package-management.md+++ b/website/docs/reference/package-management.md@@ -573,7 +573,8 @@ version lookups. Probes and non-resumable transfers get at most four attempts.Their backoff waits are 1, 2 and 4 seconds. A `Retry-After` header can extendthese waits up to 30 seconds. Resumable transfers renew their retry budgetwhen they save new progress; four failures without new progress stop retries.-They honor `Retry-After` up to five minutes and stop for longer requested waits.+With a pinned mirror available, waits above 30 seconds switch to that mirror.+The last source honors `Retry-After` up to five minutes and stops for longer waits.Each retry logs its cause, delay and consecutive failures without progress.Retryable HTTP statuses are 408, 429, 500, 502, 503 and 504. Connection resets,