Skip to content

The agent that grows with you

fix(process): a supervisor killed young gets one restart, not a restart loop

kshitijk4poorf22cb45be0b0
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py
index b034f52a73c..0e2e753e0a0 100644
--- a/hermes_cli/process_identity.py
+++ b/hermes_cli/process_identity.py
@@ -858,12 +858,19 @@ def _spawn_supervisor():
return proc
-# A supervisor that EXITS sooner than this after starting cannot stay up (a broken install: sh exiting
-# 2 or 127): the waiter leaves it to the next registration change instead of restarting it in a loop.
-# One killed by a signal is restarted whatever its age.
+# A supervisor that dies sooner than this after starting may not be able to stay up (a broken install:
+# sh exiting 2 or 127; a sandbox or OOM killer that keeps killing it): the waiter leaves it to the next
+# registration change instead of restarting it in a loop. One killed by a signal gets one young restart
+# (a new supervisor starts after every idle period, so a `pkill -f hermes` often meets a young one); if
+# its replacement is killed young as well, that one waits for the next registration change too.
_SUPERVISOR_RESTART_MIN_UPTIME_S = 5.0
+# Set (under _supervisor_lock) while the waiter starts the young restart, so that replacement's own
+# waiter knows it already had one.
+_starting_young_restart = False
+
+
def _watch_supervisor(proc) -> None:
"""One waiter thread per supervisor, blocked in the untimed ``wait()`` (waitpid, no polling). It
reaps the supervisor the moment it exits, so no zombie child is left for psutil walkers of this
@@ -871,9 +878,10 @@ def _watch_supervisor(proc) -> None:
(``pkill -f hermes``, the OOM killer) is restarted and replayed at once, so coverage does not wait
for the next registration change; a released one (EOF with an empty set) is only reaped."""
started = time.monotonic()
+ young_restart = _starting_young_restart
def wait_then_restart() -> None:
- global _supervisor
+ global _supervisor, _starting_young_restart
proc.wait()
with _supervisor_lock:
if _supervisor is not None and _supervisor is not proc:
@@ -884,13 +892,19 @@ def _watch_supervisor(proc) -> None:
_prune_dead_supervised_pgids()
if not _supervised_pgids:
return
- restart = proc.returncode < 0 or time.monotonic() - started >= _SUPERVISOR_RESTART_MIN_UPTIME_S
+ old_enough = time.monotonic() - started >= _SUPERVISOR_RESTART_MIN_UPTIME_S
+ young_signal = not old_enough and proc.returncode < 0 and not young_restart
+ restart = old_enough or young_signal
logger.warning(
"owner-death supervisor exited (status %s) with %d process group(s) registered; %s",
proc.returncode, len(_supervised_pgids),
"restarting it" if restart else "it could not stay up, retrying at the next registration")
if restart:
- _start_supervisor_locked()
+ _starting_young_restart = young_signal
+ try:
+ _start_supervisor_locked()
+ finally:
+ _starting_young_restart = False
try:
# health: allow HX012 -- the thread waitpid()s one child and touches only process-global supervisor state
diff --git a/tests/hermes_cli/test_process_identity_supervisor.py b/tests/hermes_cli/test_process_identity_supervisor.py
index 13610f6336f..cfabfd36250 100644
--- a/tests/hermes_cli/test_process_identity_supervisor.py
+++ b/tests/hermes_cli/test_process_identity_supervisor.py
@@ -951,3 +951,34 @@ def test_a_process_that_cannot_start_a_thread_still_gets_its_child_and_releases_
assert process_identity._supervisor is None, "with no timer thread the supervisor is released at once"
if supervisor is not None:
supervisor.wait(timeout=10)
+
+
+def test_a_supervisor_killed_every_time_it_starts_is_not_restarted_in_a_loop(monkeypatch, tmp_path):
+ """A sandbox or OOM killer that kills every new /bin/sh at once must not turn the waiter's
+ restart into a fork loop: one young restart, then the next registration change retries."""
+ die = tmp_path / "die.sh"
+ die.write_text("kill -9 $$\n", encoding="utf-8")
+ monkeypatch.setattr(process_identity, "_SUPERVISOR_SCRIPT", die)
+ spawned = []
+ real_spawn = process_identity._spawn_supervisor
+
+ def counting():
+ proc = real_spawn()
+ spawned.append(proc)
+ return proc
+
+ monkeypatch.setattr(process_identity, "_spawn_supervisor", counting)
+ child = subprocess.Popen(_VICTIM, start_new_session=True)
+ try:
+ process_identity.own_process_groups([child.pid])
+ wait_until(lambda: len(spawned) >= 2 and process_identity._supervisor is None
+ and all(p.returncode is not None for p in spawned), timeout=15,
+ what="the killed supervisor and its one young restart to settle")
+ time.sleep(1.0) # health: allow HX013 -- no further restart over this window IS the behaviour
+ assert len(spawned) == 2, f"{len(spawned)} supervisors spawned for one registration"
+ finally:
+ _kill(child.pid)
+ child.wait(timeout=10)
+ for proc in spawned:
+ if proc is not None:
+ proc.wait(timeout=10)