The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(pm): tolerate failed periodic checkpoints
akfoster938be3a6d4b9
diff --git a/pm/downloader.py b/pm/downloader.pyindex 81cdff8d82d..c1b00d5d917 100644--- a/pm/downloader.py+++ b/pm/downloader.py@@ -471,14 +471,14 @@ class Download:@staticmethoddef _write_sidecar(side: Path, part: Path, covered: _Ranges, remote: _Remote, sha256: str) -> None:- from pm.filesystem import durable_write_bytes+ from pm.filesystem import durable_write_bytes, retry_held# Workers flush before announcing coverage. Snapshot those ranges,# fsync the data, then atomically publish only that durable snapshot.with part.open("r+b") as stream:os.fsync(stream.fileno())record = {"total": remote.total, "etag": remote.etag, "sha256": sha256, "ranges": covered}- durable_write_bytes(side, json.dumps(record).encode("utf-8"))+ retry_held(lambda: durable_write_bytes(side, json.dumps(record).encode("utf-8")))def _fetch_ranged(self, source: Source, remote: _Remote, tick) -> int:key = self._key(source.url)@@ -511,7 +511,7 @@ class Download:self.connections = 1continueraise next((error for error in errors if not is_transient(error)), errors[0])- written = sum(end - start for start, end in covered)+ written = self._durable_doneif written != remote.total:raise http.client.IncompleteRead(b"", remote.total - written)return written@@ -595,16 +595,20 @@ class Download:finished = pending == 0updated.clear()now = time.monotonic()- written = sum(end - start for start, end in snapshot)if now >= next_checkpoint:if snapshot != checkpointed:try:self._write_sidecar(part.with_suffix(".ranges"), part, snapshot, remote, source.sha256)+ except OSError as exc:+ logging.getLogger(__name__).warning(+ "could not save download progress for %s; continuing: %s", source.dest.name, exc,+ )except BaseException:stop.set()raise- self._durable_done = written- checkpointed = snapshot+ else:+ self._durable_done = sum(end - start for start, end in snapshot)+ checkpointed = snapshotnext_checkpoint = time.monotonic() + _CHECKPOINT_SECONDS# Only the coordinator observes progress. Slow UI/worker IPC# coalesces intermediate snapshots, never holds up range writes.diff --git a/pm/network.py b/pm/network.pyindex cb45a5f76e2..070e1b6d042 100644--- a/pm/network.py+++ b/pm/network.py@@ -69,7 +69,7 @@ def edge_block(exc: BaseException) -> str | None:"retry from another network")-def _delay(exc: Exception, attempt: int, *, maximum: float | None = _MAX_DELAY) -> float:+def _delay(exc: Exception, attempt: int) -> float:delay = float(2 ** attempt)if isinstance(exc, urllib.error.HTTPError):value = exc.headers.get("Retry-After", "") if exc.headers else ""@@ -85,7 +85,7 @@ def _delay(exc: Exception, attempt: int, *, maximum: float | None = _MAX_DELAY)except (TypeError, ValueError, OverflowError):retry_after = 0.0delay = max(delay, retry_after)- return delay if maximum is None else min(delay, maximum)+ return delaydef retry_network(@@ -120,12 +120,10 @@ def retry_network(if failures_since_progress >= _ATTEMPTS:raisedelay = _delay(exc, backoff)- if progress is not None:- # Do not retry before Retry-After or wait hours for an embargo.- requested = _delay(exc, 0, maximum=None)- if requested > _MAX_RECOVERY_DELAY:- raise- delay = max(delay, requested)+ if progress is None:+ delay = min(delay, _MAX_DELAY)+ elif delay > _MAX_RECOVERY_DELAY:+ raisebackoff += 1logging.getLogger(__name__).warning("network request failed (%s); retrying in %gs (%d/%d failures without progress)",diff --git a/tests/hermes_cli/test_local_models_routes.py b/tests/hermes_cli/test_local_models_routes.pyindex e3f28d73c20..9b42c00b2ee 100644--- a/tests/hermes_cli/test_local_models_routes.py+++ b/tests/hermes_cli/test_local_models_routes.py@@ -333,6 +333,7 @@ def test_download_short_of_server_length_errors_and_cleans_up(client, monkeypatcrequests.append(req.headers.get('Range'))return super().open(req, timeout)monkeypatch.setattr("pm.downloader._OPENER", Truncated(b"not the real body", content_length=32))+ monkeypatch.setattr("pm.downloader.Download._wait_retry", lambda self, seconds: None)destination = models_dir() / 'truncated.gguf'monkeypatch.setattr(lm, '_download_plan', lambda *_: [('https://fixture.invalid/model', destination, 32)])monkeypatch.setattr('pm.paths.partials_root', lambda: tmp_path / 'partials')diff --git a/tests/pm/test_download_checkpoint_policy.py b/tests/pm/test_download_checkpoint_policy.pydeleted file mode 100644index 6d4f7d0273f..00000000000--- a/tests/pm/test_download_checkpoint_policy.py+++ /dev/null@@ -1,66 +0,0 @@-"""Checkpoint rate and retry delays do not grow with connection speed."""-from __future__ import annotations--import hashlib-from http.server import BaseHTTPRequestHandler-from types import SimpleNamespace--import pytest--from pm import downloader-from tests.pm._fixtures import threaded_server-from tests.pm._range_server import RangeHandler, dl_server as dl_server, url # noqa: PLC0414 -- pytest fixture re-export---def test_fast_transfer_does_not_sync_each_byte_threshold(tmp_path, monkeypatch, dl_server):- payload = b"w" * (32 << 20)- RangeHandler.payloads = {"/fast": payload}- download = downloader.Download(- [downloader.Source(url(dl_server, "/fast"), tmp_path / "model", hashlib.sha256(payload).hexdigest())],- partials_dir=tmp_path / "partials", connections=1,- )- checkpoints = []- original = download._write_sidecar-- def save(*args):- checkpoints.append(None)- return original(*args)-- monkeypatch.setattr(download, "_write_sidecar", save)- monkeypatch.setattr(downloader, "time", SimpleNamespace(monotonic=lambda: 0.0))- download.run()- assert (tmp_path / "model").read_bytes() == payload- assert len(checkpoints) == 1 # The attempt still saves its final checkpoint.--[email protected]("failed_stage", ["probe", "single-stream"])-def test_non_resumable_requests_keep_the_existing_retry_after_cap(tmp_path, monkeypatch, failed_stage):- payload = b"model bytes"- failed = False-- class Handler(BaseHTTPRequestHandler):- def do_GET(self):- nonlocal failed- probe = self.headers.get("Range") == "bytes=0-0"- if not failed and probe == (failed_stage == "probe"):- failed = True- self.send_response(503)- self.send_header("Retry-After", "60")- self.end_headers()- return- self.send_response(200) # This source does not support ranges.- self.send_header("Content-Length", str(len(payload)))- self.end_headers()- self.wfile.write(payload)-- def log_message(self, *args):- pass-- waits = []- with threaded_server(Handler) as server:- source = downloader.Source(f"http://127.0.0.1:{server.server_port}/model", tmp_path / "model")- download = downloader.Download([source], partials_dir=tmp_path / "partials")- monkeypatch.setattr(download, "_wait_retry", waits.append)- download.run()- assert waits == [30]- assert source.dest.read_bytes() == payloaddiff --git a/tests/pm/test_download_continuity.py b/tests/pm/test_download_continuity.pyindex 4eaf3929861..c588bd81e02 100644--- a/tests/pm/test_download_continuity.py+++ b/tests/pm/test_download_continuity.py@@ -1,6 +1,7 @@-"""Interrupted bodies retain durable progress across attempts and processes."""+"""Durable checkpoints preserve downloads across interruptions and process termination."""from __future__ import annotations+import errnoimport hashlibimport jsonfrom pathlib import Path@@ -9,9 +10,15 @@ import subprocessimport sysfrom http.server import BaseHTTPRequestHandlerfrom threading import Event+from types import SimpleNamespace+import pytest++from pm import downloader, filesystemfrom pm.downloader import Download, Sourcefrom tests.pm._fixtures import threaded_server+from tests.pm._range_server import RangeHandler, dl_server as dl_server, url # noqa: PLC0414 -- pytest fixture re-export+from tests.waiting import wait_untildef test_useful_progress_survives_more_than_four_interrupted_bodies(tmp_path, monkeypatch):@@ -88,7 +95,8 @@ def test_forced_termination_recovers_checkpoint_before_request_finishes(tmp_pathurl = f"http://127.0.0.1:{server.server_port}/model"side = partials / (hashlib.sha256(url.encode()).hexdigest() + ".ranges")script = (- "from pathlib import Path\nimport sys\nfrom pm.downloader import Download, Source\n"+ "from pathlib import Path\nimport sys\nfrom pm import downloader\nfrom pm.downloader import Download, Source\n"+ "downloader._CHECKPOINT_SECONDS = 0.2\n""Download([Source(sys.argv[1],Path(sys.argv[2]),sys.argv[4])],""partials_dir=Path(sys.argv[3]),connections=1).run()\n")@@ -98,8 +106,6 @@ def test_forced_termination_recovers_checkpoint_before_request_finishes(tmp_pathstdout=subprocess.PIPE, stderr=subprocess.PIPE)try:assert waiting.wait(10), "child never received model data"- from tests.waiting import wait_until-assert wait_until(side.is_file, timeout=10), "no checkpoint during an unfinished request"saved = json.loads(side.read_text())retained = sum(end - start for start, end in saved["ranges"])@@ -113,3 +119,112 @@ def test_forced_termination_recovers_checkpoint_before_request_finishes(tmp_pathpartials_dir=partials, connections=1).run()assert destination.read_bytes() == payloadassert requests[1][0] >= checkpoint+++def test_checkpoints_are_time_based_not_byte_based(tmp_path, monkeypatch, dl_server):+ payload = b"w" * (32 << 20)+ RangeHandler.payloads = {"/fast": payload}+ download = downloader.Download(+ [downloader.Source(url(dl_server, "/fast"), tmp_path / "model", hashlib.sha256(payload).hexdigest())],+ partials_dir=tmp_path / "partials", connections=1,+ )+ checkpoints = []+ original = download._write_sidecar++ def save(*args):+ checkpoints.append(None)+ return original(*args)++ monkeypatch.setattr(download, "_write_sidecar", save)+ monkeypatch.setattr(downloader, "time", SimpleNamespace(monotonic=lambda: 0.0))+ download.run()+ assert (tmp_path / "model").read_bytes() == payload+ assert len(checkpoints) == 1 # The attempt still saves its final checkpoint.++[email protected]("failed_stage", ["probe", "single-stream"])+def test_non_resumable_requests_keep_the_existing_retry_after_cap(tmp_path, monkeypatch, failed_stage):+ payload = b"model bytes"+ failed = False++ class Handler(BaseHTTPRequestHandler):+ def do_GET(self):+ nonlocal failed+ probe = self.headers.get("Range") == "bytes=0-0"+ if not failed and probe == (failed_stage == "probe"):+ failed = True+ self.send_response(503)+ self.send_header("Retry-After", "60")+ self.end_headers()+ return+ self.send_response(200) # This source does not support ranges.+ self.send_header("Content-Length", str(len(payload)))+ self.end_headers()+ self.wfile.write(payload)++ def log_message(self, *args):+ pass++ waits = []+ with threaded_server(Handler) as server:+ source = downloader.Source(f"http://127.0.0.1:{server.server_port}/model", tmp_path / "model")+ download = downloader.Download([source], partials_dir=tmp_path / "partials")+ monkeypatch.setattr(download, "_wait_retry", waits.append)+ download.run()+ assert waits == [30]+ assert source.dest.read_bytes() == payload+++def test_failed_periodic_checkpoint_retries_without_claiming_saved_bytes(tmp_path, monkeypatch, caplog):+ payload = b"checkpointed model bytes"+ part = tmp_path / "model.part"+ side = part.with_suffix(".ranges")+ part.write_bytes(payload)+ source = Source("https://fixture.invalid/model", tmp_path / "model")+ download = Download([source], partials_dir=tmp_path, connections=1)+ remote = downloader._Remote(total=len(payload), ranged=True, etag='"stable"')+ attempts = []+ replace = filesystem.os.replace++ def publish(temporary, destination):+ if Path(destination) == side:+ attempts.append(download._durable_done)+ if len(attempts) == 1:+ raise PermissionError(errno.EACCES, "checkpoint temporarily held")+ return replace(temporary, destination)++ def read_range(source, remote, part, start, end, stop, record):+ record(start, end)+ assert wait_until(lambda: side.is_file() or stop.is_set(), timeout=5)++ monkeypatch.setattr(filesystem.os, "replace", publish)+ monkeypatch.setattr(filesystem, "_HELD_RETRY_DELAYS", ())+ monkeypatch.setattr(downloader, "_CHECKPOINT_SECONDS", 0.01)+ monkeypatch.setattr(download, "_read_range", read_range)+ covered, errors = download._ranged_attempt(source, remote, part, [], 1, lambda _: None)+ assert not errors and covered == [(0, len(payload))]+ assert attempts == [0, 0]+ assert json.loads(side.read_text())["ranges"] == [[0, len(payload)]]+ assert download._durable_done == len(payload)+ assert any(record.name == "pm.downloader" and record.levelname == "WARNING" for record in caplog.records)+++def test_final_checkpoint_write_failure_does_not_publish_the_model(tmp_path, monkeypatch, dl_server):+ payload = b"model bytes"+ RangeHandler.payloads = {"/strict": payload}+ source = Source(url(dl_server, "/strict"), tmp_path / "model", hashlib.sha256(payload).hexdigest())+ download = Download([source], partials_dir=tmp_path / "partials", connections=1)+ side = download.partials_dir / f"{download._key(source.url)}.ranges"+ replace = filesystem.os.replace++ def publish(temporary, destination):+ if Path(destination) == side:+ raise PermissionError(errno.EACCES, "checkpoint temporarily held")+ return replace(temporary, destination)++ monkeypatch.setattr(filesystem.os, "replace", publish)+ monkeypatch.setattr(filesystem, "_HELD_RETRY_DELAYS", ())+ monkeypatch.setattr(downloader, "time", SimpleNamespace(monotonic=lambda: 0.0))+ with pytest.raises(PermissionError):+ download.run()+ assert not source.dest.exists() and download._durable_done == 0diff --git a/website/docs/reference/package-management.md b/website/docs/reference/package-management.mdindex 870c3cf2833..0db3bc6ccf5 100644--- a/website/docs/reference/package-management.md+++ b/website/docs/reference/package-management.md@@ -578,7 +578,8 @@ Each retry logs its cause, delay and consecutive failures without progress.Retryable HTTP statuses are 408, 429, 500, 502, 503 and 504. Connection resets,timeouts, temporary DNS failures and interrupted response bodies also retry.-Ranged downloads save durable progress every five seconds and after each attempt.+Ranged downloads checkpoint every five seconds and after each attempt. A failed+periodic checkpoint logs a warning and retries later; the final checkpoint is required.Servers without range support require a fresh stream. Pause interrupts backoffand preserves the partial download.