The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(scratch): keep owning a registered gateway across a wall-clock step
Teknium9b443ff6e413
diff --git a/gateway/status.py b/gateway/status.pyindex 455da6f5099..ab160695016 100644--- a/gateway/status.py+++ b/gateway/status.py@@ -489,8 +489,8 @@ def _get_process_start_time(pid: int) -> Optional[int]:"""stat_path = Path(f"/proc/{pid}/stat")try:- # Field 22 in /proc/<pid>/stat is process start time (clock ticks).- return int(stat_path.read_text(encoding="utf-8").split()[21]) # windows-footgun: ok (/proc is BOM-free)+ # Field 22 (ticks) after comm's last ")": comm may hold spaces or non-UTF-8 bytes.+ return int(stat_path.read_bytes().rsplit(b")", 1)[1].split()[19])except (FileNotFoundError, IndexError, PermissionError, ValueError, OSError):passdiff --git a/hermes_cli/data_cleanup_holders.py b/hermes_cli/data_cleanup_holders.pyindex 48dbe2590bb..7c020061330 100644--- a/hermes_cli/data_cleanup_holders.py+++ b/hermes_cli/data_cleanup_holders.py@@ -71,7 +71,7 @@ def _drain_manual_gateway(home: Path) -> None:def _refuse_backend_writers(home: Path) -> None:from hermes_constants import get_default_hermes_root- from hermes_cli.process_identity import LEDGER_FILENAME, _pid_alive_matches, _read_ledger+ from hermes_cli.process_identity import LEDGER_FILENAME, _read_ledger, entry_still_runningroot = get_default_hermes_root(home=home).resolve()rows = _read_ledger(root / LEDGER_FILENAME)@@ -85,7 +85,7 @@ def _refuse_backend_writers(home: Path) -> None:pid = row.get("pid")if type(pid) is not int or pid <= 0:raise RuntimeError("backend ownership contains an invalid PID")- if _pid_alive_matches(pid, row.get("create_time")) is not False:+ if entry_still_running(row, keep_unprovable=True):raise RuntimeError(f"backend PID {pid} still owns {home}; close its desktop/dashboard before retrying")diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.pyindex d055ce0d466..51eb8c2f858 100644--- a/hermes_cli/process_identity.py+++ b/hermes_cli/process_identity.py@@ -101,6 +101,21 @@ def worker_birth(proc) -> float:return process_birth(proc)+def boot_start(proc) -> Optional[float]:+ """Seconds from boot to the start of the ``psutil.Process`` *proc* on Linux; ``None`` elsewhere.++ The ledger's epoch ``create_time`` is :func:`process_birth`, which on Linux is the kernel's+ start ticks plus the ``/proc/stat`` btime psutil re-reads on every call, so an NTP or VM-resume+ clock step moves it by whole seconds and a live process stops matching its entry. Ledger+ entries carry this alongside it (``boot_start`` / ``spawner_boot``) and compare it exactly.+ Raises ``psutil.Error`` like ``create_time()``.+ """+ import psutil+ if getattr(psutil, "LINUX", False):+ return float(proc._proc.create_time(monotonic=True))+ return None++def _process_create_time(pid: Optional[int] = None) -> Optional[float]:""":func:`process_birth` of ``pid`` (default: this process); ``None`` when psutil can't say."""try:@@ -111,6 +126,30 @@ def _process_create_time(pid: Optional[int] = None) -> Optional[float]:return None+def _verified_boot_start(pid: Optional[int], create_time: Optional[float]) -> Optional[float]:+ """:func:`boot_start` of ``pid`` while it still matches ``create_time``; ``None`` when unproven."""+ if not isinstance(pid, int) or pid <= 0 or create_time is None:+ return None+ try:+ import psutil++ proc = psutil.Process(pid)+ return boot_start(proc) if _same_incarnation(proc, create_time) else None+ except Exception:+ return None+++def _step_can_explain_mismatch(pid: int, boot: Optional[float]) -> bool:+ """A live ``pid`` failed an epoch-only match on Linux: a clock step explains that as well as a+ reused pid, so the mismatch proves nothing. ``False`` when ``boot`` was recorded (exact)."""+ try:+ import psutil++ return boot is None and getattr(psutil, "LINUX", False) and psutil.pid_exists(pid)+ except Exception:+ return False++# Layer 1 — spawn tags@@ -172,6 +211,10 @@ class LedgerEntry:# `serve --isolated`: opted out of the host singleton (Desktop's SSH backend for another# machine). Attach-first readers must never adopt it; argv is truncated, so this is canonical.isolated: bool = False+ # Linux boot-relative births (:func:`boot_start`) of the process and its spawner: exact+ # identity across clock steps. ``None`` elsewhere and in older entries (epoch match only).+ boot_start: Optional[float] = None+ spawner_boot: Optional[float] = Nonedef _ledger_path() -> Path:@@ -220,12 +263,20 @@ def _read_ledger_or_quarantine(path: Path) -> Optional[list[dict]]:return entries-def _same_incarnation(proc, create_time: Optional[float]) -> bool:- """Does the live ``proc`` match a recorded ``create_time`` (2 s tolerance; ``None`` matches)?"""+def _same_incarnation(proc, create_time: Optional[float], boot: Optional[float] = None, *, exact: bool = False) -> bool:+ """Does the live ``proc`` match a recorded birth? A recorded ``boot`` (:func:`boot_start`)+ compares exactly where the live one is readable; otherwise ``create_time`` (2 s tolerance, or+ equality when ``exact``; ``None`` matches unless ``exact``)."""+ if boot is not None and (live_boot := boot_start(proc)) is not None:+ return live_boot == float(boot)+ if exact:+ return create_time is not None and process_birth(proc) == create_timereturn create_time is None or abs(process_birth(proc) - float(create_time)) < 2.0-def _pid_alive_matches(pid: int, create_time: Optional[float], *, strict: bool = False) -> Optional[bool]:+def _pid_alive_matches(+ pid: int, create_time: Optional[float], *, strict: bool = False, boot: Optional[float] = None,+) -> Optional[bool]:"""True/False when provable; ``None`` when psutil can't say."""try:import psutil@@ -234,12 +285,12 @@ def _pid_alive_matches(pid: int, create_time: Optional[float], *, strict: bool =try:proc = psutil.Process(int(pid))if strict:- return (create_time is not None and process_birth(proc) == create_time+ return (_same_incarnation(proc, create_time, boot, exact=True)and proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE)# A zombie keeps its create_time until its parent reaps it, but it is already dead: the# dashboard stop check (``gateway.status._pid_exists``) books it stopped, so the ledger must# not report it as a live pre-update survivor. Windows has no zombies (and status() is slow there).- return _same_incarnation(proc, create_time) and (+ return _same_incarnation(proc, create_time, boot) and (os.name == "nt" or proc.status() != getattr(psutil, "STATUS_ZOMBIE", "zombie"))except psutil.NoSuchProcess:return False@@ -260,6 +311,7 @@ def register_self(purpose: str, *, project_root: Optional[Path] = None, detail:tag = parse_spawn_tag(os.environ.get(SPAWN_ENV_VAR))spawner_pid, spawner_create = (tag.spawner_pid, tag.spawner_create) if tag else _desktop_spawner_identity()entry = _new_entry(os.getpid(), _process_create_time(), purpose, project_root, spawner_pid, spawner_create)+ entry.spawner_boot = _verified_boot_start(spawner_pid, spawner_create)entry.hermes_home = hermes_home_key()if detail:try:@@ -352,7 +404,8 @@ def _new_entry(spawner_pid: Optional[int], spawner_create: Optional[float],) -> LedgerEntry:return LedgerEntry(- pid, create_time, purpose, install_id(project_root), spawner_pid, spawner_create, time.time(), argv=""+ pid, create_time, purpose, install_id(project_root), spawner_pid, spawner_create, time.time(), argv="",+ boot_start=_verified_boot_start(pid, create_time),)@@ -372,7 +425,7 @@ def _append_entry(entry: LedgerEntry) -> bool:e for e in entriesif isinstance(e.get("pid"), int)and e["pid"] != entry.pid- and _pid_alive_matches(e["pid"], e.get("create_time")) is not False+ and entry_still_running(e, keep_unprovable=True)]pruned.append(asdict(entry))try:@@ -403,6 +456,7 @@ def register_child(pid: int, purpose: str, *, project_root: Optional[Path] = Nonif child_create is None:return Falseentry = _new_entry(pid, child_create, purpose, project_root, os.getpid(), _process_create_time())+ entry.spawner_boot = _verified_boot_start(os.getpid(), entry.spawner_create)try:import psutil@@ -434,27 +488,46 @@ def ledger_entries(e for e in entriesif (all_installs or e.get("install") == want_install)and isinstance(e.get("pid"), int)- and (_pid_alive_matches(e["pid"], e.get("create_time"), strict=True) is True- if verified_only else _pid_alive_matches(e["pid"], e.get("create_time")) is not False)+ and (_pid_alive_matches(e["pid"], e.get("create_time"), strict=True, boot=e.get("boot_start")) is True+ if verified_only else _pid_alive_matches(e["pid"], e.get("create_time"), boot=e.get("boot_start")) is not False)]+def entry_still_running(entry: dict, *, keep_unprovable: bool = False) -> bool:+ """Is the ledger *entry*'s process still the recorded incarnation (``None`` from psutil+ counts as running)? ``keep_unprovable``: an older entry with only an epoch birth whose live+ pid disagrees on Linux also counts, because a clock step explains that as well as pid reuse.+ Readers that protect a process (ledger pruning, the scratch prune's owner check) pass it;+ killers never do."""+ pid, boot = entry.get("pid"), entry.get("boot_start")+ if not isinstance(pid, int) or pid <= 0:+ return False+ if _pid_alive_matches(pid, entry.get("create_time"), boot=boot) is not False:+ return True+ return keep_unprovable and _step_can_explain_mismatch(pid, boot)++def spawner_is_dead(entry: dict) -> Optional[bool]:"""Is the recorded spawner of this entry provably gone? ``None`` when unrecorded/unprovable."""spawner_pid = entry.get("spawner_pid")if not isinstance(spawner_pid, int) or spawner_pid <= 0:return None- alive = _pid_alive_matches(spawner_pid, entry.get("spawner_create"))+ boot = entry.get("spawner_boot")+ alive = _pid_alive_matches(spawner_pid, entry.get("spawner_create"), boot=boot)+ # An entry that survives a clock step (``boot_start``) with an epoch-only spawner: a step+ # would read its live spawner as dead and let a reaper kill the child.+ if alive is False and entry.get("boot_start") is not None and _step_can_explain_mismatch(spawner_pid, boot):+ return Nonereturn None if alive is None else not alive-def _terminate_then_kill(pid: int, create_time: Optional[float]) -> bool:+def _terminate_then_kill(pid: int, create_time: Optional[float], boot: Optional[float] = None) -> bool:"""SIGTERM, 2 s grace, then SIGKILL. False when the incarnation moved on or is already gone."""try:import psutilproc = psutil.Process(pid)- if not _same_incarnation(proc, create_time):+ if not _same_incarnation(proc, create_time, boot):return False # PID reused since registration — never signal a strangerproc.terminate()try:@@ -529,7 +602,7 @@ def reap_orphaned_backend_owner(try:if kill_fn is not None:kill_fn(pid)- elif not _terminate_then_kill(pid, entry.get("create_time")):+ elif not _terminate_then_kill(pid, entry.get("create_time"), entry.get("boot_start")):return Noneexcept Exception:logger.debug("backend owner reap failed for %s", entry, exc_info=True)@@ -564,7 +637,7 @@ def reap_orphaned_mcp_helpers(*, project_root: Optional[Path] = None, kill_fn=Noimport psutilproc = psutil.Process(pid)- if not _same_incarnation(proc, entry.get("create_time")):+ if not _same_incarnation(proc, entry.get("create_time"), entry.get("boot_start")):continue # PID reused since registration# Windows: descendants (npx.cmd → node.exe) have no pgid to group-kill and# reparent with ParentId=null when the direct child exits first (#61059), sodiff --git a/hermes_constants_scratch.py b/hermes_constants_scratch.pyindex ea9c11cd88e..b8571dc6aeb 100644--- a/hermes_constants_scratch.py+++ b/hermes_constants_scratch.py@@ -245,7 +245,7 @@ def _registered_live_pids(home: Path) -> set[int]:matches is a reused pid, not ownership."""import psutil- from hermes_cli.process_identity import spawner_is_dead+ from hermes_cli.process_identity import entry_still_running, spawner_is_deadfrom hermes_constants import get_default_hermes_rootroot = get_default_hermes_root(home=home)@@ -267,10 +267,12 @@ def _registered_live_pids(home: Path) -> set[int]:if entry.get("purpose") == "mcp-helper" and spawner_is_dead(entry) is True:continuetry:- if stamp is None:+ if stamp_key == "create_time":+ # Boot-relative when recorded; an older epoch-only entry that a clock step+ # could have shifted stays owned rather than getting its tree killed.+ alive = entry_still_running(entry, keep_unprovable=True)+ elif stamp is None:alive = psutil.pid_exists(pid)- elif stamp_key == "create_time":- alive = abs(psutil.Process(pid).create_time() - float(stamp)) < 2.0else:from gateway.status import get_process_start_timediff --git a/tests/test_scratch_registered_clock_step.py b/tests/test_scratch_registered_clock_step.pynew file mode 100644index 00000000000..b38374a5c51--- /dev/null+++ b/tests/test_scratch_registered_clock_step.py@@ -0,0 +1,52 @@+"""The scratch prune's owner check survives a wall-clock step (NTP, VM resume)."""++import os+import subprocess+import sys+import time+from pathlib import Path++import psutil+import pytest++from hermes_constants import get_scratch_dir, prune_scratch_dir+from tests.waiting import wait_until++[email protected]("linux") # psutil's create_time adds the live /proc/stat btime only here[email protected]("step", [5.0, -5.0])+def test_prune_keeps_a_registered_gateways_shell_after_a_clock_step(tmp_path, monkeypatch, step):+ """A gateway registered in the spawn ledger runs a shell inside an idle scratch entry. The+ wall clock then steps by ``step`` seconds, which moves psutil's epoch ``create_time`` of every+ process on Linux. The gateway is still the same process: the prune must keep owning it, never+ kill its shell or delete the entry the shell works in."""+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))+ scratch = get_scratch_dir(tmp_path, prune=False)+ lane = scratch / "gateway-lane"+ lane.mkdir()+ shell_pidfile = tmp_path / "shell.pid"+ repo = str(Path(__file__).resolve().parents[1])+ gateway = subprocess.Popen(+ [sys.executable, "-c",+ f"import sys, subprocess, time; sys.path.insert(0, {repo!r}); "+ "from hermes_cli.process_identity import register_self; assert register_self('gateway'); "+ f"shell = subprocess.Popen(['sleep', '60'], cwd={str(lane)!r}); "+ f"open({str(shell_pidfile)!r}, 'w').write(str(shell.pid)); time.sleep(60)"],+ stdin=subprocess.DEVNULL, env={**os.environ, "HERMES_HOME": str(tmp_path)})+ try:+ shell = int(wait_until(lambda: shell_pidfile.exists() and shell_pidfile.read_text().strip(),+ timeout=20, what="registered gateway started its shell"))+ real_btime = psutil.boot_time()+ monkeypatch.setattr(psutil._pslinux, "boot_time", lambda: real_btime + step)+ ancient = time.time() - 30 * 3600+ os.utime(lane, (ancient, ancient))++ prune_scratch_dir(scratch)++ assert psutil.Process(shell).status() != psutil.STATUS_ZOMBIE, "the gateway's shell was killed"+ assert lane.exists(), "the entry the gateway's shell works in was deleted"+ finally:+ for child in psutil.Process(gateway.pid).children(recursive=True):+ child.kill()+ gateway.kill()+ gateway.wait(timeout=10)