The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(process): a supervisor killed young gets one restart, not a restart loop
kshitijk4poorf22cb45be0b0
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.pyindex b034f52a73c..0e2e753e0a0 100644--- a/hermes_cli/process_identity.py+++ b/hermes_cli/process_identity.py@@ -858,12 +858,19 @@ def _spawn_supervisor():return proc-# A supervisor that EXITS sooner than this after starting cannot stay up (a broken install: sh exiting-# 2 or 127): the waiter leaves it to the next registration change instead of restarting it in a loop.-# One killed by a signal is restarted whatever its age.+# A supervisor that dies sooner than this after starting may not be able to stay up (a broken install:+# sh exiting 2 or 127; a sandbox or OOM killer that keeps killing it): the waiter leaves it to the next+# registration change instead of restarting it in a loop. One killed by a signal gets one young restart+# (a new supervisor starts after every idle period, so a `pkill -f hermes` often meets a young one); if+# its replacement is killed young as well, that one waits for the next registration change too._SUPERVISOR_RESTART_MIN_UPTIME_S = 5.0+# Set (under _supervisor_lock) while the waiter starts the young restart, so that replacement's own+# waiter knows it already had one.+_starting_young_restart = False++def _watch_supervisor(proc) -> None:"""One waiter thread per supervisor, blocked in the untimed ``wait()`` (waitpid, no polling). Itreaps the supervisor the moment it exits, so no zombie child is left for psutil walkers of this@@ -871,9 +878,10 @@ def _watch_supervisor(proc) -> None:(``pkill -f hermes``, the OOM killer) is restarted and replayed at once, so coverage does not waitfor the next registration change; a released one (EOF with an empty set) is only reaped."""started = time.monotonic()+ young_restart = _starting_young_restartdef wait_then_restart() -> None:- global _supervisor+ global _supervisor, _starting_young_restartproc.wait()with _supervisor_lock:if _supervisor is not None and _supervisor is not proc:@@ -884,13 +892,19 @@ def _watch_supervisor(proc) -> None:_prune_dead_supervised_pgids()if not _supervised_pgids:return- restart = proc.returncode < 0 or time.monotonic() - started >= _SUPERVISOR_RESTART_MIN_UPTIME_S+ old_enough = time.monotonic() - started >= _SUPERVISOR_RESTART_MIN_UPTIME_S+ young_signal = not old_enough and proc.returncode < 0 and not young_restart+ restart = old_enough or young_signallogger.warning("owner-death supervisor exited (status %s) with %d process group(s) registered; %s",proc.returncode, len(_supervised_pgids),"restarting it" if restart else "it could not stay up, retrying at the next registration")if restart:- _start_supervisor_locked()+ _starting_young_restart = young_signal+ try:+ _start_supervisor_locked()+ finally:+ _starting_young_restart = Falsetry:# health: allow HX012 -- the thread waitpid()s one child and touches only process-global supervisor statediff --git a/tests/hermes_cli/test_process_identity_supervisor.py b/tests/hermes_cli/test_process_identity_supervisor.pyindex 13610f6336f..cfabfd36250 100644--- a/tests/hermes_cli/test_process_identity_supervisor.py+++ b/tests/hermes_cli/test_process_identity_supervisor.py@@ -951,3 +951,34 @@ def test_a_process_that_cannot_start_a_thread_still_gets_its_child_and_releases_assert process_identity._supervisor is None, "with no timer thread the supervisor is released at once"if supervisor is not None:supervisor.wait(timeout=10)+++def test_a_supervisor_killed_every_time_it_starts_is_not_restarted_in_a_loop(monkeypatch, tmp_path):+ """A sandbox or OOM killer that kills every new /bin/sh at once must not turn the waiter's+ restart into a fork loop: one young restart, then the next registration change retries."""+ die = tmp_path / "die.sh"+ die.write_text("kill -9 $$\n", encoding="utf-8")+ monkeypatch.setattr(process_identity, "_SUPERVISOR_SCRIPT", die)+ spawned = []+ real_spawn = process_identity._spawn_supervisor++ def counting():+ proc = real_spawn()+ spawned.append(proc)+ return proc++ monkeypatch.setattr(process_identity, "_spawn_supervisor", counting)+ child = subprocess.Popen(_VICTIM, start_new_session=True)+ try:+ process_identity.own_process_groups([child.pid])+ wait_until(lambda: len(spawned) >= 2 and process_identity._supervisor is None+ and all(p.returncode is not None for p in spawned), timeout=15,+ what="the killed supervisor and its one young restart to settle")+ time.sleep(1.0) # health: allow HX013 -- no further restart over this window IS the behaviour+ assert len(spawned) == 2, f"{len(spawned)} supervisors spawned for one registration"+ finally:+ _kill(child.pid)+ child.wait(timeout=10)+ for proc in spawned:+ if proc is not None:+ proc.wait(timeout=10)