The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(update): a backed-off failed dependency sync boots instead of re-execing forever
Teknium3eeeb172dd8d
diff --git a/hermes_cli/venv_sync.py b/hermes_cli/venv_sync.pyindex efbf252bdca..e4fa41a82aa 100644--- a/hermes_cli/venv_sync.py+++ b/hermes_cli/venv_sync.py@@ -370,6 +370,49 @@ def _tree_matches_completed_stamp(root: Path) -> bool:return info.commit == commit and not info.dirty+def _may_retry_completion(root: Path, command: list[str], *, current: bool) -> bool:+ """Whether this launch runs the owed repair now; a suppressed one says why when it matters."""+ import sys++ if not current and command[:1] == ["update"]:+ # `hermes update` is the remedy every backed-off launch names: it always tries the repair.+ return True+ may_retry, attempts, backoff = completion_retry_state(root)+ if not may_retry and attempts >= COMPLETION_RETRY_MAX_ATTEMPTS:+ # The tail has failed often enough that every relaunch re-running it+ # does more harm than good (#122206: "every launch burns ~4 minutes").+ # Leave the marker for an explicit `hermes update`; say so once.+ print(+ f"hermes: a source update could not be finished automatically "+ f"({attempts} attempts); run `hermes update` from a shell to finish it",+ file=sys.stderr, flush=True,+ )+ elif not may_retry and not current:+ # Inside the backoff window a finished-dependencies tail waits silently (the record's age+ # tracks the wait); stale dependencies are worth one line in front of the user.+ print(+ f"hermes: running with the previous dependencies after {attempts} failed syncs; "+ f"next automatic retry in {max(1, backoff // 60)} min, or run `hermes update` to finish it",+ file=sys.stderr, flush=True,+ )+ return may_retry+++def _relaunch_target(python: Path, *, same: bool) -> Path | None:+ """*python*, unless this process is already a relaunch into that same interpreter.++ A relaunched child that asks to relaunch into itself again would find the same install state+ in the next child too: exec'ing would loop forever (#122513), so it boots as it is.+ """+ import sys++ if same and getattr(sys, "_hermes_relaunched", False):+ print("hermes: relaunching did not leave this install current; running with the "+ "dependencies it has — run `hermes update` to finish it", file=sys.stderr, flush=True)+ return None+ return python++def prepare_launch(project_root: Path, argv: list[str]) -> Path | None:"""Finish a self-managed source update before importing app dependencies.@@ -426,20 +469,12 @@ def prepare_launch(project_root: Path, argv: list[str]) -> Path | None:return _prepare_borrowed_launch(root, owner, current=current)pending = completion_pending_path(root)owed_to_cli = current and pending.is_file() and _supervised_child()- _may_retry, _attempts, _backoff = completion_retry_state(root)- if not _may_retry and _attempts >= COMPLETION_RETRY_MAX_ATTEMPTS:- # The tail has failed often enough that every relaunch re-running it- # does more harm than good (#122206: "every launch burns ~4 minutes").- # Leave the marker for an explicit `hermes update`; say so once.- print(- f"hermes: a source update could not be finished automatically "- f"({_attempts} attempts); run `hermes update` from a shell to finish it",- file=sys.stderr, flush=True,- )- elif not _may_retry:- # Backoff window not yet elapsed: skip this launch's retry without- # noise (the record's age tracks the wait), leaving the marker armed.- pass+ if not _may_retry_completion(root, command, current=current):+ if not current:+ # No repair ran, so this install is exactly as stale as it was: a relaunch would land+ # back here with the same answer and exec again, forever. Boot on the previous+ # dependency generation, as a launch whose sync failed does.+ return Noneelif not owed_to_cli and (not current or pending.is_file()):# The marker alone first (a process the live update spawned runs under its claim and# inherits no checkout lock); install_root still names the checkout whose held lock@@ -480,7 +515,7 @@ def prepare_launch(project_root: Path, argv: list[str]) -> Path | None:same = os.path.normcase(os.path.abspath(python)) == os.path.normcase(os.path.abspath(sys.executable))if not current or not same:publish_launchers(root)- return python+ return _relaunch_target(python, same=same)if owed_to_cli:# Left owed, not dropped: say so (once, in the process that boots) where an# operator of the unit will read it.@@ -656,7 +691,9 @@ def relaunch_command(if option in ("-W", "-X") and index < len(original):options.append(original[index])index += 1- prefix = f"import sys, runpy; sys.path.insert(0, {str(root)!r}); sys.argv = {argv!r}; "+ # _hermes_relaunched marks the child this starts (not processes it spawns) for _relaunch_target.+ prefix = (f"import sys, runpy; sys.path.insert(0, {str(root)!r}); sys.argv = {argv!r}; "+ "sys._hermes_relaunched = True; ")if argv[0] == "-c":body = f"exec({original[index + 1]!r})"elif module and module != "__main__":diff --git a/tests/hermes_cli/test_launch_relaunch_loop.py b/tests/hermes_cli/test_launch_relaunch_loop.pynew file mode 100644index 00000000000..8a0ebb4184d--- /dev/null+++ b/tests/hermes_cli/test_launch_relaunch_loop.py@@ -0,0 +1,132 @@+"""A launch whose dependencies stay stale boots; it never re-execs itself forever.++Each test runs real launches as separate processes through the real decision chain the+bootstrap runs (``prepare_launch`` -> ``relaunch_command`` -> ``os.execv``). Only PM's sync and+currency answers and the store interpreter are stubbed, to model an install whose repair keeps+failing (or whose sync never makes it current). Every exec'd generation is counted.+"""+from __future__ import annotations++import json+import subprocess+import sys+from pathlib import Path++import pytest++REPO = Path(__file__).resolve().parents[2]+GENERATION_CAP = 8++_LAUNCH = r'''+import os, sys+from pathlib import Path++repo, root, scratch, sync_mode, launch = sys.argv[1:6]+import json+command_args = json.loads(os.environ["LAUNCH_COMMAND"]) # an argv naming `update` alarms the test guard+sys.path.insert(0, repo)+import pm, pm.client+from hermes_cli import _launchers, venv_sync++generations = Path(scratch, f"generations-{launch}")+with generations.open("a") as fh:+ fh.write(f"{os.getpid()}\n")+if len(generations.read_text().split()) > int(os.environ["GENERATION_CAP"]):+ raise SystemExit(99)++def sync(*_args, **_kwargs):+ with Path(scratch, "syncs").open("a") as fh:+ fh.write(f"{launch}\n")+ if sync_mode == "fail":+ raise RuntimeError("sync failed")++pm.venv_is_current = lambda **_kw: False+pm.sync_venv = sync+pm.client.ensure_tools_for_sync = lambda: None+_launchers.resolve_store_python = lambda *_a, **_k: Path(sys.executable)+venv_sync.publish_launchers = lambda *_a, **_k: None++try:+ python = venv_sync.prepare_launch(Path(root), command_args)+ if python is not None:+ os.execv(str(python), venv_sync.relaunch_command(python, Path(repo), sys.argv, sys.orig_argv, None))+except RuntimeError as exc:+ print(f"degraded: {exc}", file=sys.stderr)+print("BOOTED")+'''+++def _launch(tmp_path: Path, root: Path, sync_mode: str, launch: int, argv: list[str]):+ script = tmp_path / "launch.py"+ script.write_text(_LAUNCH, encoding="utf-8")+ env = {"PATH": "/usr/bin:/bin", "HOME": str(tmp_path), "HERMES_HOME": str(tmp_path / "home"),+ "LAUNCH_COMMAND": json.dumps(argv), "GENERATION_CAP": str(GENERATION_CAP)}+ result = subprocess.run(+ [sys.executable, str(script), str(REPO), str(root), str(tmp_path), sync_mode, str(launch)],+ env=env, capture_output=True, text=True, timeout=120, check=False,+ )+ generations = len((tmp_path / f"generations-{launch}").read_text().split())+ return result, generations+++def _syncs(tmp_path: Path) -> int:+ path = tmp_path / "syncs"+ return len(path.read_text().split()) if path.exists() else 0++[email protected]("posix")[email protected]("argv", [[], ["update"]], ids=["launch", "update"])+def test_backed_off_failed_sync_boots_on_the_previous_dependencies(tmp_path, monkeypatch, argv):+ """#139952 records a failed launch sync so relaunches back off. A launch inside that backoff+ (or past the cap) has run no repair, so relaunching it would re-enter the same decision and+ exec again forever; it must boot on the previous dependencies, and `hermes update` (the+ remedy every backed-off launch names) still tries the repair."""+ from hermes_cli import venv_sync++ root = tmp_path / "checkout"+ root.mkdir()+ (root / ".git").mkdir()+ (root / "pyproject.toml").write_text("[project]\nname='example'\n", encoding="utf-8")+ (root / "install-stamp.json").write_text(json.dumps({"updateMechanism": "self"}), encoding="utf-8")++ for launch in range(1, 5):+ result, generations = _launch(tmp_path, root, "fail", launch, argv)+ assert result.returncode == 0, f"launch {launch} looped: {generations} generations\n{result.stderr}"+ assert result.stdout.strip() == "BOOTED"+ assert generations == 1+ expected = 4 if argv else venv_sync.COMPLETION_RETRY_BACKOFF_ATTEMPTS+ assert _syncs(tmp_path) == expected, "a backed-off launch re-ran the failing repair"++ if not argv:+ # Past the cap, at any age: still one generation, and one pointer at `hermes update`.+ monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))+ record = venv_sync._completion_attempts_path(root)+ assert record.is_file(), "the failed syncs were not recorded where completion_retry_state reads"+ record.write_text(f"{venv_sync.COMPLETION_RETRY_MAX_ATTEMPTS}\n", encoding="utf-8")+ result, generations = _launch(tmp_path, root, "fail", 5, argv)+ assert (result.returncode, generations) == (0, 1), result.stderr+ assert result.stderr.count("could not be finished automatically") == 1++[email protected]("posix")+def test_relaunched_child_never_relaunches_into_itself(tmp_path):+ """A sync that never makes the install current (the PM answer disagrees with what the sync+ did) must not turn every launch into an exec chain: the relaunched child boots."""+ from hermes_cli.source_stamp import write_source_stamp++ root = tmp_path / "checkout"+ root.mkdir()+ (root / "pyproject.toml").write_text("[project]\nname='example'\n", encoding="utf-8")+ (root / ".gitignore").write_text("install-stamp.json\n", encoding="utf-8")+ (root / "install-stamp.json").write_text(json.dumps({"updateMechanism": "self"}), encoding="utf-8")+ git_env = {"PATH": "/usr/bin:/bin", "HOME": str(tmp_path)}+ for args in (["init"], ["config", "user.email", "[email protected]"], ["config", "user.name", "test"],+ ["add", "-A"], ["commit", "-m", "init"]):+ subprocess.run(["git", *args], cwd=root, env=git_env, check=True, capture_output=True, timeout=30)+ assert write_source_stamp(root)["dirty"] is False++ result, generations = _launch(tmp_path, root, "ok", 1, [])+ assert result.returncode == 0, f"looped: {generations} generations\n{result.stderr}"+ assert result.stdout.strip() == "BOOTED"+ assert generations == 2, "the first relaunch into the store interpreter is still taken"+ assert "relaunching did not leave this install current" in result.stderrdiff --git a/tests/hermes_cli/test_update_launch_completion.py b/tests/hermes_cli/test_update_launch_completion.pyindex fd5f1cd36ff..9bd381acb78 100644--- a/tests/hermes_cli/test_update_launch_completion.py+++ b/tests/hermes_cli/test_update_launch_completion.py@@ -370,7 +370,7 @@ def test_failed_dependency_sync_backs_off_only_for_the_same_inputs(tmp_path, monvenv_sync.prepare_launch(root, [])assert venv_sync.completion_pending_path(root).is_file()- assert venv_sync.prepare_launch(root, []) == Path(sys.executable)+ assert venv_sync.prepare_launch(root, []) is None, "a backed-off stale launch must not relaunch"assert len(syncs) == venv_sync.COMPLETION_RETRY_BACKOFF_ATTEMPTS, "the same failing repair re-ran"(root / "uv.lock").write_text("# corrected source update\n", encoding="utf-8")