Skip to content

The agent that grows with you

fix(kanban): read process start fingerprints from the kernel birth on macOS

Teknium377be426fde8
diff --git a/gateway/status.py b/gateway/status.py
index 093b882d271..5fa2a99238e 100644
--- a/gateway/status.py
+++ b/gateway/status.py
@@ -497,9 +497,16 @@ def _get_process_start_time(pid: int) -> Optional[int]:
# No /proc (macOS / Windows): psutil is a hard dependency and exposes a
# cross-platform creation time. Quantize to centiseconds so repeated reads
# of the same process compare equal without float-precision fragility.
+ # ``process_birth`` is the kernel start on macOS: psutil's ``create_time()``
+ # there shifts by a boot-time reference frozen per calling process, so after a
+ # clock step a long-lived gateway and a fresh worker fingerprinted the same
+ # live PID differently (Kanban ``_watch_submitter`` then exited; #140580).
+ # With no step (|drift| < 1 s) psutil returns the kernel start unchanged, so
+ # stored fingerprints keep matching.
try:
import psutil # type: ignore
- return round(psutil.Process(pid).create_time() * 100)
+ from hermes_cli.process_identity import process_birth
+ return round(process_birth(psutil.Process(pid)) * 100)
except Exception:
return None
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py
index 7611621c0bd..009e556cfb2 100644
--- a/hermes_cli/process_identity.py
+++ b/hermes_cli/process_identity.py
@@ -85,11 +85,11 @@ def process_birth(proc) -> float:
def _process_create_time(pid: Optional[int] = None) -> Optional[float]:
- """``psutil`` create time for ``pid`` (default: this process); ``None`` when psutil can't say."""
+ """:func:`process_birth` of ``pid`` (default: this process); ``None`` when psutil can't say."""
try:
import psutil
- return float(psutil.Process(os.getpid() if pid is None else pid).create_time())
+ return process_birth(psutil.Process(os.getpid() if pid is None else pid))
except Exception:
return None
@@ -205,7 +205,7 @@ def _read_ledger_or_quarantine(path: Path) -> Optional[list[dict]]:
def _same_incarnation(proc, create_time: Optional[float]) -> bool:
"""Does the live ``proc`` match a recorded ``create_time`` (2 s tolerance; ``None`` matches)?"""
- return create_time is None or abs(float(proc.create_time()) - float(create_time)) < 2.0
+ return create_time is None or abs(process_birth(proc) - float(create_time)) < 2.0
def _pid_alive_matches(pid: int, create_time: Optional[float], *, strict: bool = False) -> Optional[bool]:
@@ -217,7 +217,7 @@ def _pid_alive_matches(pid: int, create_time: Optional[float], *, strict: bool =
try:
proc = psutil.Process(int(pid))
if strict:
- return (create_time is not None and proc.create_time() == create_time
+ return (create_time is not None and process_birth(proc) == create_time
and proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE)
# A zombie keeps its create_time until its parent reaps it, but it is already dead: the
# dashboard stop check (``gateway.status._pid_exists``) books it stopped, so the ledger must
diff --git a/tests/gateway/test_worker_admission_reservation.py b/tests/gateway/test_worker_admission_reservation.py
index 5bdb0fdfa02..5fe01d96a17 100644
--- a/tests/gateway/test_worker_admission_reservation.py
+++ b/tests/gateway/test_worker_admission_reservation.py
@@ -141,27 +141,32 @@ async def test_input_queued_behind_a_live_worker_is_released_then_runs_when_it_f
db.close()
+def stale_psutil_boot_reference(monkeypatch):
+ """Step psutil's macOS boot-time reference the way a clock step leaves it in a long-lived
+ process: ``_psosx.INIT_BOOT_TIME`` is frozen at import, and ``create_time()`` adds
+ ``abs(INIT_BOOT_TIME - boot_time())``. The real macOS psutil code runs; only the reference,
+ which is what a step makes stale, moves (stepping the host clock needs root)."""
+ def step(drift):
+ monkeypatch.setattr(psutil._psplatform, 'INIT_BOOT_TIME', psutil.boot_time() + drift)
+ return step
+
+
@pytest.mark.parametrize('drift', [5.0, -5.0])
-def test_reservation_survives_a_clock_step_under_emulated_macos_psutil(tmp_path, monkeypatch, drift):
- """#140580, on every platform: emulate psutil's macOS ``create_time()``. The kernel records an
- epoch start; psutil adds ``abs(INIT_BOOT_TIME - boot_time())`` (``_psosx.adjust_proc_create_time``)
- with a reference frozen at import. A long-lived owner's reference is stale after a clock step
- either way, a fresh worker's is not, so the exact match refused every managed worker.
- Monkeypatched: ``psutil.MACOS`` and the platform ``create_time``; the reservation is real."""
+def test_reservation_survives_a_clock_step_on_macos(tmp_path, stale_psutil_boot_reference, drift):
+ """#140580: after a clock step a long-lived owner's psutil reads a fresh worker's birth
+ shifted, while the worker reports its own unshifted one, so the exact match refused every
+ managed worker. ``process_birth`` (the kernel start) is what both processes see."""
from gateway.session_worker_reservation import reserve_admission_worker
+ from hermes_cli.process_identity import process_birth
db = SessionDB(tmp_path / 'state.db')
process = subprocess.Popen([sys.executable, '-c', 'import sys; sys.stdin.read()'], stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
- frame = hello(process) # what the freshly started worker reports (its reference is current)
- kernel = psutil._psplatform.Process.create_time
-
- def owner_view(self, monotonic=False, **kwargs): # the owner's stale reference
- start = kernel(self, **kwargs) # an epoch start, as macOS kinfo_proc records it
- return start if monotonic else start + abs(drift)
-
- monkeypatch.setattr(psutil, 'MACOS', True)
- monkeypatch.setattr(psutil._psplatform.Process, 'create_time', owner_view)
+ frame = hello(process) | {'birth': process_birth(psutil.Process(process.pid))} # the worker's own read
+ stale_psutil_boot_reference(drift)
+ assert psutil.Process(process.pid).create_time() != frame['birth'] # the step is real
db.create_session('owned', 'cli')
epoch = begin_runtime_epoch(db, instance_id='owner')
authority = SimpleNamespace(db=db, epoch=epoch, profile_id=str(tmp_path), _require_admission_open=lambda: None)
@@ -176,3 +181,41 @@ def test_reservation_survives_a_clock_step_under_emulated_macos_psutil(tmp_path,
process.stdin.close()
process.wait(timeout=5)
db.close()
+
+
[email protected]('drift', [5.0, -5.0])
+def test_process_fingerprints_survive_a_clock_step_on_macos(stale_psutil_boot_reference, drift):
+ """KB-6 beyond worker birth: the Kanban submitter fingerprint (``_process_fingerprint`` via
+ ``get_process_start_time``; ``_watch_submitter`` exits on a mismatch), the delegation
+ ``owner_started_at`` and the spawn ledger's ``_pid_alive_matches`` read the same live process
+ the same way before and after a clock step."""
+ from gateway.status import get_process_start_time
+ from hermes_cli.kanban_db_dispatch import _pid_recycled, _process_fingerprint
+ from hermes_cli.process_identity import _pid_alive_matches, _process_create_time
+ process = subprocess.Popen([sys.executable, '-c', 'import sys; sys.stdin.read()'], stdin=subprocess.PIPE,
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+ try:
+ start, fingerprint, create = (get_process_start_time(process.pid), _process_fingerprint(process.pid),
+ _process_create_time(process.pid))
+ stale_psutil_boot_reference(drift)
+ assert abs(psutil.Process(process.pid).create_time() - create) >= 4 # the step is real
+ assert get_process_start_time(process.pid) == start
+ assert _process_fingerprint(process.pid) == fingerprint and not _pid_recycled(process.pid, fingerprint)
+ assert _pid_alive_matches(process.pid, create, strict=True) is True
+ assert _pid_alive_matches(process.pid, create) is True
+ finally:
+ process.stdin.close()
+ process.wait(timeout=5)
+
+
+def test_process_birth_is_create_time_where_psutil_applies_no_reference():
+ """Linux and Windows read ``create_time()`` unchanged, so every stored fingerprint and ledger
+ row written before KB-6 still matches there; on macOS with no step psutil returns the kernel
+ start unchanged too (``adjust_proc_create_time`` ignores |drift| < 1 s)."""
+ from gateway.status import get_process_start_time
+ from hermes_cli.process_identity import _pid_alive_matches, process_birth
+ proc = psutil.Process()
+ assert process_birth(proc) == proc.create_time()
+ assert _pid_alive_matches(os.getpid(), proc.create_time(), strict=True) is True
+ assert get_process_start_time(os.getpid()) is not None