Skip to content

The agent that grows with you

fix(update): a backed-off failed dependency sync boots instead of re-execing forever

Teknium3eeeb172dd8d
diff --git a/hermes_cli/venv_sync.py b/hermes_cli/venv_sync.py
index efbf252bdca..e4fa41a82aa 100644
--- a/hermes_cli/venv_sync.py
+++ b/hermes_cli/venv_sync.py
@@ -370,6 +370,49 @@ def _tree_matches_completed_stamp(root: Path) -> bool:
return info.commit == commit and not info.dirty
+def _may_retry_completion(root: Path, command: list[str], *, current: bool) -> bool:
+ """Whether this launch runs the owed repair now; a suppressed one says why when it matters."""
+ import sys
+
+ if not current and command[:1] == ["update"]:
+ # `hermes update` is the remedy every backed-off launch names: it always tries the repair.
+ return True
+ may_retry, attempts, backoff = completion_retry_state(root)
+ if not may_retry and attempts >= COMPLETION_RETRY_MAX_ATTEMPTS:
+ # The tail has failed often enough that every relaunch re-running it
+ # does more harm than good (#122206: "every launch burns ~4 minutes").
+ # Leave the marker for an explicit `hermes update`; say so once.
+ print(
+ f"hermes: a source update could not be finished automatically "
+ f"({attempts} attempts); run `hermes update` from a shell to finish it",
+ file=sys.stderr, flush=True,
+ )
+ elif not may_retry and not current:
+ # Inside the backoff window a finished-dependencies tail waits silently (the record's age
+ # tracks the wait); stale dependencies are worth one line in front of the user.
+ print(
+ f"hermes: running with the previous dependencies after {attempts} failed syncs; "
+ f"next automatic retry in {max(1, backoff // 60)} min, or run `hermes update` to finish it",
+ file=sys.stderr, flush=True,
+ )
+ return may_retry
+
+
+def _relaunch_target(python: Path, *, same: bool) -> Path | None:
+ """*python*, unless this process is already a relaunch into that same interpreter.
+
+ A relaunched child that asks to relaunch into itself again would find the same install state
+ in the next child too: exec'ing would loop forever (#122513), so it boots as it is.
+ """
+ import sys
+
+ if same and getattr(sys, "_hermes_relaunched", False):
+ print("hermes: relaunching did not leave this install current; running with the "
+ "dependencies it has — run `hermes update` to finish it", file=sys.stderr, flush=True)
+ return None
+ return python
+
+
def prepare_launch(project_root: Path, argv: list[str]) -> Path | None:
"""Finish a self-managed source update before importing app dependencies.
@@ -426,20 +469,12 @@ def prepare_launch(project_root: Path, argv: list[str]) -> Path | None:
return _prepare_borrowed_launch(root, owner, current=current)
pending = completion_pending_path(root)
owed_to_cli = current and pending.is_file() and _supervised_child()
- _may_retry, _attempts, _backoff = completion_retry_state(root)
- if not _may_retry and _attempts >= COMPLETION_RETRY_MAX_ATTEMPTS:
- # The tail has failed often enough that every relaunch re-running it
- # does more harm than good (#122206: "every launch burns ~4 minutes").
- # Leave the marker for an explicit `hermes update`; say so once.
- print(
- f"hermes: a source update could not be finished automatically "
- f"({_attempts} attempts); run `hermes update` from a shell to finish it",
- file=sys.stderr, flush=True,
- )
- elif not _may_retry:
- # Backoff window not yet elapsed: skip this launch's retry without
- # noise (the record's age tracks the wait), leaving the marker armed.
- pass
+ if not _may_retry_completion(root, command, current=current):
+ if not current:
+ # No repair ran, so this install is exactly as stale as it was: a relaunch would land
+ # back here with the same answer and exec again, forever. Boot on the previous
+ # dependency generation, as a launch whose sync failed does.
+ return None
elif not owed_to_cli and (not current or pending.is_file()):
# The marker alone first (a process the live update spawned runs under its claim and
# inherits no checkout lock); install_root still names the checkout whose held lock
@@ -480,7 +515,7 @@ def prepare_launch(project_root: Path, argv: list[str]) -> Path | None:
same = os.path.normcase(os.path.abspath(python)) == os.path.normcase(os.path.abspath(sys.executable))
if not current or not same:
publish_launchers(root)
- return python
+ return _relaunch_target(python, same=same)
if owed_to_cli:
# Left owed, not dropped: say so (once, in the process that boots) where an
# operator of the unit will read it.
@@ -656,7 +691,9 @@ def relaunch_command(
if option in ("-W", "-X") and index < len(original):
options.append(original[index])
index += 1
- prefix = f"import sys, runpy; sys.path.insert(0, {str(root)!r}); sys.argv = {argv!r}; "
+ # _hermes_relaunched marks the child this starts (not processes it spawns) for _relaunch_target.
+ prefix = (f"import sys, runpy; sys.path.insert(0, {str(root)!r}); sys.argv = {argv!r}; "
+ "sys._hermes_relaunched = True; ")
if argv[0] == "-c":
body = f"exec({original[index + 1]!r})"
elif module and module != "__main__":
diff --git a/tests/hermes_cli/test_launch_relaunch_loop.py b/tests/hermes_cli/test_launch_relaunch_loop.py
new file mode 100644
index 00000000000..8a0ebb4184d
--- /dev/null
+++ b/tests/hermes_cli/test_launch_relaunch_loop.py
@@ -0,0 +1,132 @@
+"""A launch whose dependencies stay stale boots; it never re-execs itself forever.
+
+Each test runs real launches as separate processes through the real decision chain the
+bootstrap runs (``prepare_launch`` -> ``relaunch_command`` -> ``os.execv``). Only PM's sync and
+currency answers and the store interpreter are stubbed, to model an install whose repair keeps
+failing (or whose sync never makes it current). Every exec'd generation is counted.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parents[2]
+GENERATION_CAP = 8
+
+_LAUNCH = r'''
+import os, sys
+from pathlib import Path
+
+repo, root, scratch, sync_mode, launch = sys.argv[1:6]
+import json
+command_args = json.loads(os.environ["LAUNCH_COMMAND"]) # an argv naming `update` alarms the test guard
+sys.path.insert(0, repo)
+import pm, pm.client
+from hermes_cli import _launchers, venv_sync
+
+generations = Path(scratch, f"generations-{launch}")
+with generations.open("a") as fh:
+ fh.write(f"{os.getpid()}\n")
+if len(generations.read_text().split()) > int(os.environ["GENERATION_CAP"]):
+ raise SystemExit(99)
+
+def sync(*_args, **_kwargs):
+ with Path(scratch, "syncs").open("a") as fh:
+ fh.write(f"{launch}\n")
+ if sync_mode == "fail":
+ raise RuntimeError("sync failed")
+
+pm.venv_is_current = lambda **_kw: False
+pm.sync_venv = sync
+pm.client.ensure_tools_for_sync = lambda: None
+_launchers.resolve_store_python = lambda *_a, **_k: Path(sys.executable)
+venv_sync.publish_launchers = lambda *_a, **_k: None
+
+try:
+ python = venv_sync.prepare_launch(Path(root), command_args)
+ if python is not None:
+ os.execv(str(python), venv_sync.relaunch_command(python, Path(repo), sys.argv, sys.orig_argv, None))
+except RuntimeError as exc:
+ print(f"degraded: {exc}", file=sys.stderr)
+print("BOOTED")
+'''
+
+
+def _launch(tmp_path: Path, root: Path, sync_mode: str, launch: int, argv: list[str]):
+ script = tmp_path / "launch.py"
+ script.write_text(_LAUNCH, encoding="utf-8")
+ env = {"PATH": "/usr/bin:/bin", "HOME": str(tmp_path), "HERMES_HOME": str(tmp_path / "home"),
+ "LAUNCH_COMMAND": json.dumps(argv), "GENERATION_CAP": str(GENERATION_CAP)}
+ result = subprocess.run(
+ [sys.executable, str(script), str(REPO), str(root), str(tmp_path), sync_mode, str(launch)],
+ env=env, capture_output=True, text=True, timeout=120, check=False,
+ )
+ generations = len((tmp_path / f"generations-{launch}").read_text().split())
+ return result, generations
+
+
+def _syncs(tmp_path: Path) -> int:
+ path = tmp_path / "syncs"
+ return len(path.read_text().split()) if path.exists() else 0
+
+
[email protected]("argv", [[], ["update"]], ids=["launch", "update"])
+def test_backed_off_failed_sync_boots_on_the_previous_dependencies(tmp_path, monkeypatch, argv):
+ """#139952 records a failed launch sync so relaunches back off. A launch inside that backoff
+ (or past the cap) has run no repair, so relaunching it would re-enter the same decision and
+ exec again forever; it must boot on the previous dependencies, and `hermes update` (the
+ remedy every backed-off launch names) still tries the repair."""
+ from hermes_cli import venv_sync
+
+ root = tmp_path / "checkout"
+ root.mkdir()
+ (root / ".git").mkdir()
+ (root / "pyproject.toml").write_text("[project]\nname='example'\n", encoding="utf-8")
+ (root / "install-stamp.json").write_text(json.dumps({"updateMechanism": "self"}), encoding="utf-8")
+
+ for launch in range(1, 5):
+ result, generations = _launch(tmp_path, root, "fail", launch, argv)
+ assert result.returncode == 0, f"launch {launch} looped: {generations} generations\n{result.stderr}"
+ assert result.stdout.strip() == "BOOTED"
+ assert generations == 1
+ expected = 4 if argv else venv_sync.COMPLETION_RETRY_BACKOFF_ATTEMPTS
+ assert _syncs(tmp_path) == expected, "a backed-off launch re-ran the failing repair"
+
+ if not argv:
+ # Past the cap, at any age: still one generation, and one pointer at `hermes update`.
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))
+ record = venv_sync._completion_attempts_path(root)
+ assert record.is_file(), "the failed syncs were not recorded where completion_retry_state reads"
+ record.write_text(f"{venv_sync.COMPLETION_RETRY_MAX_ATTEMPTS}\n", encoding="utf-8")
+ result, generations = _launch(tmp_path, root, "fail", 5, argv)
+ assert (result.returncode, generations) == (0, 1), result.stderr
+ assert result.stderr.count("could not be finished automatically") == 1
+
+
+def test_relaunched_child_never_relaunches_into_itself(tmp_path):
+ """A sync that never makes the install current (the PM answer disagrees with what the sync
+ did) must not turn every launch into an exec chain: the relaunched child boots."""
+ from hermes_cli.source_stamp import write_source_stamp
+
+ root = tmp_path / "checkout"
+ root.mkdir()
+ (root / "pyproject.toml").write_text("[project]\nname='example'\n", encoding="utf-8")
+ (root / ".gitignore").write_text("install-stamp.json\n", encoding="utf-8")
+ (root / "install-stamp.json").write_text(json.dumps({"updateMechanism": "self"}), encoding="utf-8")
+ git_env = {"PATH": "/usr/bin:/bin", "HOME": str(tmp_path)}
+ for args in (["init"], ["config", "user.email", "[email protected]"], ["config", "user.name", "test"],
+ ["add", "-A"], ["commit", "-m", "init"]):
+ subprocess.run(["git", *args], cwd=root, env=git_env, check=True, capture_output=True, timeout=30)
+ assert write_source_stamp(root)["dirty"] is False
+
+ result, generations = _launch(tmp_path, root, "ok", 1, [])
+ assert result.returncode == 0, f"looped: {generations} generations\n{result.stderr}"
+ assert result.stdout.strip() == "BOOTED"
+ assert generations == 2, "the first relaunch into the store interpreter is still taken"
+ assert "relaunching did not leave this install current" in result.stderr
diff --git a/tests/hermes_cli/test_update_launch_completion.py b/tests/hermes_cli/test_update_launch_completion.py
index fd5f1cd36ff..9bd381acb78 100644
--- a/tests/hermes_cli/test_update_launch_completion.py
+++ b/tests/hermes_cli/test_update_launch_completion.py
@@ -370,7 +370,7 @@ def test_failed_dependency_sync_backs_off_only_for_the_same_inputs(tmp_path, mon
venv_sync.prepare_launch(root, [])
assert venv_sync.completion_pending_path(root).is_file()
- assert venv_sync.prepare_launch(root, []) == Path(sys.executable)
+ assert venv_sync.prepare_launch(root, []) is None, "a backed-off stale launch must not relaunch"
assert len(syncs) == venv_sync.COMPLETION_RETRY_BACKOFF_ATTEMPTS, "the same failing repair re-ran"
(root / "uv.lock").write_text("# corrected source update\n", encoding="utf-8")