Skip to content

The agent that grows with you

fix(scratch): keep owning a registered gateway across a wall-clock step

Teknium9b443ff6e413
diff --git a/gateway/status.py b/gateway/status.py
index 455da6f5099..ab160695016 100644
--- a/gateway/status.py
+++ b/gateway/status.py
@@ -489,8 +489,8 @@ def _get_process_start_time(pid: int) -> Optional[int]:
"""
stat_path = Path(f"/proc/{pid}/stat")
try:
- # Field 22 in /proc/<pid>/stat is process start time (clock ticks).
- return int(stat_path.read_text(encoding="utf-8").split()[21]) # windows-footgun: ok (/proc is BOM-free)
+ # Field 22 (ticks) after comm's last ")": comm may hold spaces or non-UTF-8 bytes.
+ return int(stat_path.read_bytes().rsplit(b")", 1)[1].split()[19])
except (FileNotFoundError, IndexError, PermissionError, ValueError, OSError):
pass
diff --git a/hermes_cli/data_cleanup_holders.py b/hermes_cli/data_cleanup_holders.py
index 48dbe2590bb..7c020061330 100644
--- a/hermes_cli/data_cleanup_holders.py
+++ b/hermes_cli/data_cleanup_holders.py
@@ -71,7 +71,7 @@ def _drain_manual_gateway(home: Path) -> None:
def _refuse_backend_writers(home: Path) -> None:
from hermes_constants import get_default_hermes_root
- from hermes_cli.process_identity import LEDGER_FILENAME, _pid_alive_matches, _read_ledger
+ from hermes_cli.process_identity import LEDGER_FILENAME, _read_ledger, entry_still_running
root = get_default_hermes_root(home=home).resolve()
rows = _read_ledger(root / LEDGER_FILENAME)
@@ -85,7 +85,7 @@ def _refuse_backend_writers(home: Path) -> None:
pid = row.get("pid")
if type(pid) is not int or pid <= 0:
raise RuntimeError("backend ownership contains an invalid PID")
- if _pid_alive_matches(pid, row.get("create_time")) is not False:
+ if entry_still_running(row, keep_unprovable=True):
raise RuntimeError(f"backend PID {pid} still owns {home}; close its desktop/dashboard before retrying")
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py
index d055ce0d466..51eb8c2f858 100644
--- a/hermes_cli/process_identity.py
+++ b/hermes_cli/process_identity.py
@@ -101,6 +101,21 @@ def worker_birth(proc) -> float:
return process_birth(proc)
+def boot_start(proc) -> Optional[float]:
+ """Seconds from boot to the start of the ``psutil.Process`` *proc* on Linux; ``None`` elsewhere.
+
+ The ledger's epoch ``create_time`` is :func:`process_birth`, which on Linux is the kernel's
+ start ticks plus the ``/proc/stat`` btime psutil re-reads on every call, so an NTP or VM-resume
+ clock step moves it by whole seconds and a live process stops matching its entry. Ledger
+ entries carry this alongside it (``boot_start`` / ``spawner_boot``) and compare it exactly.
+ Raises ``psutil.Error`` like ``create_time()``.
+ """
+ import psutil
+ if getattr(psutil, "LINUX", False):
+ return float(proc._proc.create_time(monotonic=True))
+ return None
+
+
def _process_create_time(pid: Optional[int] = None) -> Optional[float]:
""":func:`process_birth` of ``pid`` (default: this process); ``None`` when psutil can't say."""
try:
@@ -111,6 +126,30 @@ def _process_create_time(pid: Optional[int] = None) -> Optional[float]:
return None
+def _verified_boot_start(pid: Optional[int], create_time: Optional[float]) -> Optional[float]:
+ """:func:`boot_start` of ``pid`` while it still matches ``create_time``; ``None`` when unproven."""
+ if not isinstance(pid, int) or pid <= 0 or create_time is None:
+ return None
+ try:
+ import psutil
+
+ proc = psutil.Process(pid)
+ return boot_start(proc) if _same_incarnation(proc, create_time) else None
+ except Exception:
+ return None
+
+
+def _step_can_explain_mismatch(pid: int, boot: Optional[float]) -> bool:
+ """A live ``pid`` failed an epoch-only match on Linux: a clock step explains that as well as a
+ reused pid, so the mismatch proves nothing. ``False`` when ``boot`` was recorded (exact)."""
+ try:
+ import psutil
+
+ return boot is None and getattr(psutil, "LINUX", False) and psutil.pid_exists(pid)
+ except Exception:
+ return False
+
+
# Layer 1 — spawn tags
@@ -172,6 +211,10 @@ class LedgerEntry:
# `serve --isolated`: opted out of the host singleton (Desktop's SSH backend for another
# machine). Attach-first readers must never adopt it; argv is truncated, so this is canonical.
isolated: bool = False
+ # Linux boot-relative births (:func:`boot_start`) of the process and its spawner: exact
+ # identity across clock steps. ``None`` elsewhere and in older entries (epoch match only).
+ boot_start: Optional[float] = None
+ spawner_boot: Optional[float] = None
def _ledger_path() -> Path:
@@ -220,12 +263,20 @@ def _read_ledger_or_quarantine(path: Path) -> Optional[list[dict]]:
return entries
-def _same_incarnation(proc, create_time: Optional[float]) -> bool:
- """Does the live ``proc`` match a recorded ``create_time`` (2 s tolerance; ``None`` matches)?"""
+def _same_incarnation(proc, create_time: Optional[float], boot: Optional[float] = None, *, exact: bool = False) -> bool:
+ """Does the live ``proc`` match a recorded birth? A recorded ``boot`` (:func:`boot_start`)
+ compares exactly where the live one is readable; otherwise ``create_time`` (2 s tolerance, or
+ equality when ``exact``; ``None`` matches unless ``exact``)."""
+ if boot is not None and (live_boot := boot_start(proc)) is not None:
+ return live_boot == float(boot)
+ if exact:
+ return create_time is not None and process_birth(proc) == create_time
return create_time is None or abs(process_birth(proc) - float(create_time)) < 2.0
-def _pid_alive_matches(pid: int, create_time: Optional[float], *, strict: bool = False) -> Optional[bool]:
+def _pid_alive_matches(
+ pid: int, create_time: Optional[float], *, strict: bool = False, boot: Optional[float] = None,
+) -> Optional[bool]:
"""True/False when provable; ``None`` when psutil can't say."""
try:
import psutil
@@ -234,12 +285,12 @@ def _pid_alive_matches(pid: int, create_time: Optional[float], *, strict: bool =
try:
proc = psutil.Process(int(pid))
if strict:
- return (create_time is not None and process_birth(proc) == create_time
+ return (_same_incarnation(proc, create_time, boot, exact=True)
and proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE)
# A zombie keeps its create_time until its parent reaps it, but it is already dead: the
# dashboard stop check (``gateway.status._pid_exists``) books it stopped, so the ledger must
# not report it as a live pre-update survivor. Windows has no zombies (and status() is slow there).
- return _same_incarnation(proc, create_time) and (
+ return _same_incarnation(proc, create_time, boot) and (
os.name == "nt" or proc.status() != getattr(psutil, "STATUS_ZOMBIE", "zombie"))
except psutil.NoSuchProcess:
return False
@@ -260,6 +311,7 @@ def register_self(purpose: str, *, project_root: Optional[Path] = None, detail:
tag = parse_spawn_tag(os.environ.get(SPAWN_ENV_VAR))
spawner_pid, spawner_create = (tag.spawner_pid, tag.spawner_create) if tag else _desktop_spawner_identity()
entry = _new_entry(os.getpid(), _process_create_time(), purpose, project_root, spawner_pid, spawner_create)
+ entry.spawner_boot = _verified_boot_start(spawner_pid, spawner_create)
entry.hermes_home = hermes_home_key()
if detail:
try:
@@ -352,7 +404,8 @@ def _new_entry(
spawner_pid: Optional[int], spawner_create: Optional[float],
) -> LedgerEntry:
return LedgerEntry(
- pid, create_time, purpose, install_id(project_root), spawner_pid, spawner_create, time.time(), argv=""
+ pid, create_time, purpose, install_id(project_root), spawner_pid, spawner_create, time.time(), argv="",
+ boot_start=_verified_boot_start(pid, create_time),
)
@@ -372,7 +425,7 @@ def _append_entry(entry: LedgerEntry) -> bool:
e for e in entries
if isinstance(e.get("pid"), int)
and e["pid"] != entry.pid
- and _pid_alive_matches(e["pid"], e.get("create_time")) is not False
+ and entry_still_running(e, keep_unprovable=True)
]
pruned.append(asdict(entry))
try:
@@ -403,6 +456,7 @@ def register_child(pid: int, purpose: str, *, project_root: Optional[Path] = Non
if child_create is None:
return False
entry = _new_entry(pid, child_create, purpose, project_root, os.getpid(), _process_create_time())
+ entry.spawner_boot = _verified_boot_start(os.getpid(), entry.spawner_create)
try:
import psutil
@@ -434,27 +488,46 @@ def ledger_entries(
e for e in entries
if (all_installs or e.get("install") == want_install)
and isinstance(e.get("pid"), int)
- and (_pid_alive_matches(e["pid"], e.get("create_time"), strict=True) is True
- if verified_only else _pid_alive_matches(e["pid"], e.get("create_time")) is not False)
+ and (_pid_alive_matches(e["pid"], e.get("create_time"), strict=True, boot=e.get("boot_start")) is True
+ if verified_only else _pid_alive_matches(e["pid"], e.get("create_time"), boot=e.get("boot_start")) is not False)
]
+def entry_still_running(entry: dict, *, keep_unprovable: bool = False) -> bool:
+ """Is the ledger *entry*'s process still the recorded incarnation (``None`` from psutil
+ counts as running)? ``keep_unprovable``: an older entry with only an epoch birth whose live
+ pid disagrees on Linux also counts, because a clock step explains that as well as pid reuse.
+ Readers that protect a process (ledger pruning, the scratch prune's owner check) pass it;
+ killers never do."""
+ pid, boot = entry.get("pid"), entry.get("boot_start")
+ if not isinstance(pid, int) or pid <= 0:
+ return False
+ if _pid_alive_matches(pid, entry.get("create_time"), boot=boot) is not False:
+ return True
+ return keep_unprovable and _step_can_explain_mismatch(pid, boot)
+
+
def spawner_is_dead(entry: dict) -> Optional[bool]:
"""Is the recorded spawner of this entry provably gone? ``None`` when unrecorded/unprovable."""
spawner_pid = entry.get("spawner_pid")
if not isinstance(spawner_pid, int) or spawner_pid <= 0:
return None
- alive = _pid_alive_matches(spawner_pid, entry.get("spawner_create"))
+ boot = entry.get("spawner_boot")
+ alive = _pid_alive_matches(spawner_pid, entry.get("spawner_create"), boot=boot)
+ # An entry that survives a clock step (``boot_start``) with an epoch-only spawner: a step
+ # would read its live spawner as dead and let a reaper kill the child.
+ if alive is False and entry.get("boot_start") is not None and _step_can_explain_mismatch(spawner_pid, boot):
+ return None
return None if alive is None else not alive
-def _terminate_then_kill(pid: int, create_time: Optional[float]) -> bool:
+def _terminate_then_kill(pid: int, create_time: Optional[float], boot: Optional[float] = None) -> bool:
"""SIGTERM, 2 s grace, then SIGKILL. False when the incarnation moved on or is already gone."""
try:
import psutil
proc = psutil.Process(pid)
- if not _same_incarnation(proc, create_time):
+ if not _same_incarnation(proc, create_time, boot):
return False # PID reused since registration — never signal a stranger
proc.terminate()
try:
@@ -529,7 +602,7 @@ def reap_orphaned_backend_owner(
try:
if kill_fn is not None:
kill_fn(pid)
- elif not _terminate_then_kill(pid, entry.get("create_time")):
+ elif not _terminate_then_kill(pid, entry.get("create_time"), entry.get("boot_start")):
return None
except Exception:
logger.debug("backend owner reap failed for %s", entry, exc_info=True)
@@ -564,7 +637,7 @@ def reap_orphaned_mcp_helpers(*, project_root: Optional[Path] = None, kill_fn=No
import psutil
proc = psutil.Process(pid)
- if not _same_incarnation(proc, entry.get("create_time")):
+ if not _same_incarnation(proc, entry.get("create_time"), entry.get("boot_start")):
continue # PID reused since registration
# Windows: descendants (npx.cmd → node.exe) have no pgid to group-kill and
# reparent with ParentId=null when the direct child exits first (#61059), so
diff --git a/hermes_constants_scratch.py b/hermes_constants_scratch.py
index ea9c11cd88e..b8571dc6aeb 100644
--- a/hermes_constants_scratch.py
+++ b/hermes_constants_scratch.py
@@ -245,7 +245,7 @@ def _registered_live_pids(home: Path) -> set[int]:
matches is a reused pid, not ownership."""
import psutil
- from hermes_cli.process_identity import spawner_is_dead
+ from hermes_cli.process_identity import entry_still_running, spawner_is_dead
from hermes_constants import get_default_hermes_root
root = get_default_hermes_root(home=home)
@@ -267,10 +267,12 @@ def _registered_live_pids(home: Path) -> set[int]:
if entry.get("purpose") == "mcp-helper" and spawner_is_dead(entry) is True:
continue
try:
- if stamp is None:
+ if stamp_key == "create_time":
+ # Boot-relative when recorded; an older epoch-only entry that a clock step
+ # could have shifted stays owned rather than getting its tree killed.
+ alive = entry_still_running(entry, keep_unprovable=True)
+ elif stamp is None:
alive = psutil.pid_exists(pid)
- elif stamp_key == "create_time":
- alive = abs(psutil.Process(pid).create_time() - float(stamp)) < 2.0
else:
from gateway.status import get_process_start_time
diff --git a/tests/test_scratch_registered_clock_step.py b/tests/test_scratch_registered_clock_step.py
new file mode 100644
index 00000000000..b38374a5c51
--- /dev/null
+++ b/tests/test_scratch_registered_clock_step.py
@@ -0,0 +1,52 @@
+"""The scratch prune's owner check survives a wall-clock step (NTP, VM resume)."""
+
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import psutil
+import pytest
+
+from hermes_constants import get_scratch_dir, prune_scratch_dir
+from tests.waiting import wait_until
+
+
[email protected]("linux") # psutil's create_time adds the live /proc/stat btime only here
[email protected]("step", [5.0, -5.0])
+def test_prune_keeps_a_registered_gateways_shell_after_a_clock_step(tmp_path, monkeypatch, step):
+ """A gateway registered in the spawn ledger runs a shell inside an idle scratch entry. The
+ wall clock then steps by ``step`` seconds, which moves psutil's epoch ``create_time`` of every
+ process on Linux. The gateway is still the same process: the prune must keep owning it, never
+ kill its shell or delete the entry the shell works in."""
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+ scratch = get_scratch_dir(tmp_path, prune=False)
+ lane = scratch / "gateway-lane"
+ lane.mkdir()
+ shell_pidfile = tmp_path / "shell.pid"
+ repo = str(Path(__file__).resolve().parents[1])
+ gateway = subprocess.Popen(
+ [sys.executable, "-c",
+ f"import sys, subprocess, time; sys.path.insert(0, {repo!r}); "
+ "from hermes_cli.process_identity import register_self; assert register_self('gateway'); "
+ f"shell = subprocess.Popen(['sleep', '60'], cwd={str(lane)!r}); "
+ f"open({str(shell_pidfile)!r}, 'w').write(str(shell.pid)); time.sleep(60)"],
+ stdin=subprocess.DEVNULL, env={**os.environ, "HERMES_HOME": str(tmp_path)})
+ try:
+ shell = int(wait_until(lambda: shell_pidfile.exists() and shell_pidfile.read_text().strip(),
+ timeout=20, what="registered gateway started its shell"))
+ real_btime = psutil.boot_time()
+ monkeypatch.setattr(psutil._pslinux, "boot_time", lambda: real_btime + step)
+ ancient = time.time() - 30 * 3600
+ os.utime(lane, (ancient, ancient))
+
+ prune_scratch_dir(scratch)
+
+ assert psutil.Process(shell).status() != psutil.STATUS_ZOMBIE, "the gateway's shell was killed"
+ assert lane.exists(), "the entry the gateway's shell works in was deleted"
+ finally:
+ for child in psutil.Process(gateway.pid).children(recursive=True):
+ child.kill()
+ gateway.kill()
+ gateway.wait(timeout=10)