Skip to content

The agent that grows with you

fix(gateway): compare a managed worker's birth by its kernel start on macOS

Tekniumffbeb735f12e
diff --git a/agent/managed_worker.py b/agent/managed_worker.py
index f9843d74c36..5939edb5762 100644
--- a/agent/managed_worker.py
+++ b/agent/managed_worker.py
@@ -442,8 +442,9 @@ def hello():
the ancestor chain it observes. Launchers (uv's venv python.exe) put the owner's Popen
handle one or two hops above; the owner, never this process, decides whether they match."""
import psutil
+ from hermes_cli.process_identity import process_birth
me = psutil.Process()
- return {'pid': me.pid, 'birth': me.create_time(), 'ancestors': [p.pid for p in me.parents()[:3]]}
+ return {'pid': me.pid, 'birth': process_birth(me), 'ancestors': [p.pid for p in me.parents()[:3]]}
def main():
diff --git a/gateway/session_kanban.py b/gateway/session_kanban.py
index d23f09477d4..9e13ce38854 100644
--- a/gateway/session_kanban.py
+++ b/gateway/session_kanban.py
@@ -104,6 +104,7 @@ async def run_task(connection, params):
from hermes_cli import kanban_db as kb
import os
import psutil
+ from hermes_cli.process_identity import process_birth
context = json.loads(policy.kanban_json)
def admit_claim():
@@ -124,7 +125,7 @@ async def run_task(connection, params):
raise RuntimeStoreError('stale_kanban_claim')
kb._append_event(conn, task.id, 'owner_admitted',
{'db': owner_db, 'session_id': ref.session_id,
- 'request_id': request_id, 'pid': os.getpid(), 'birth': psutil.Process().create_time()},
+ 'request_id': request_id, 'pid': os.getpid(), 'birth': process_birth(psutil.Process())},
run_id=task.current_run_id)
from gateway.session_runtime_workers import tracked_write
await tracked_write(authority, admit_claim)
diff --git a/gateway/session_managed_worker.py b/gateway/session_managed_worker.py
index 4ff71708965..fa7c88dc903 100644
--- a/gateway/session_managed_worker.py
+++ b/gateway/session_managed_worker.py
@@ -291,7 +291,8 @@ class ManagedWorker:
return
try:
proc = psutil.Process(self.worker[0])
- if proc.create_time() == self.worker[1]:
+ from hermes_cli.process_identity import process_birth
+ if process_birth(proc) == self.worker[1]:
(proc.kill if kill else proc.terminate)()
except psutil.Error:
pass
diff --git a/gateway/session_worker.py b/gateway/session_worker.py
index 6c3f162600f..914280114bc 100644
--- a/gateway/session_worker.py
+++ b/gateway/session_worker.py
@@ -30,7 +30,8 @@ def _claim(connection, ref, params):
raise RuntimeStoreError('invalid_params')
try:
process = psutil.Process(pid)
- if (process.create_time() != birth or not process.is_running()
+ from hermes_cli.process_identity import process_birth
+ if (process_birth(process) != birth or not process.is_running()
or process.status() == psutil.STATUS_ZOMBIE
or process.username() != psutil.Process().username()):
raise RuntimeStoreError('permission_denied')
diff --git a/gateway/session_worker_delegation.py b/gateway/session_worker_delegation.py
index 90838b2aa73..11d773d825d 100644
--- a/gateway/session_worker_delegation.py
+++ b/gateway/session_worker_delegation.py
@@ -69,7 +69,8 @@ def _dispatch(db, conn, session_id, payload, *, execution_id, worker_pid, worker
import psutil
from gateway.status import get_process_start_time
try:
- if psutil.Process(worker_pid).create_time() != worker_birth:
+ from hermes_cli.process_identity import process_birth
+ if process_birth(psutil.Process(worker_pid)) != worker_birth:
raise RuntimeStoreError('permission_denied')
owner_started_at = get_process_start_time(worker_pid)
except psutil.Error as exc:
diff --git a/gateway/session_worker_reservation.py b/gateway/session_worker_reservation.py
index 2e36168dfe0..bcde646e8e9 100644
--- a/gateway/session_worker_reservation.py
+++ b/gateway/session_worker_reservation.py
@@ -37,7 +37,8 @@ def verify_worker_identity(process, hello):
if handle.ppid() != os.getpid() or handle.status() == psutil.STATUS_ZOMBIE:
raise RuntimeStoreError('permission_denied')
worker = psutil.Process(pid)
- birth = worker.create_time()
+ from hermes_cli.process_identity import process_birth
+ birth = process_birth(worker) # clock-step stable on macOS (#140580)
if birth != hello['birth'] or not worker.is_running() or worker.status() == psutil.STATUS_ZOMBIE:
raise RuntimeStoreError('permission_denied')
# psutil's parent() already refuses a recycled ppid (parent born after child).
diff --git a/hermes_cli/kanban_owner_recovery.py b/hermes_cli/kanban_owner_recovery.py
index ce379829037..36703081880 100644
--- a/hermes_cli/kanban_owner_recovery.py
+++ b/hermes_cli/kanban_owner_recovery.py
@@ -24,7 +24,8 @@ def owner_reclaim_paused(conn, task_id):
return row[1] == 'interrupted'
if row is None or row[0] in {'queued', 'unknown'}:
return True
- return psutil.Process(binding['pid']).create_time() != binding['birth']
+ from hermes_cli.process_identity import process_birth
+ return process_birth(psutil.Process(binding['pid'])) != binding['birth']
except (OSError, ValueError, KeyError, sqlite3.Error):
return True
except psutil.Error:
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py
index ddae5d5ea16..7611621c0bd 100644
--- a/hermes_cli/process_identity.py
+++ b/hermes_cli/process_identity.py
@@ -61,6 +61,29 @@ def install_id(project_root: Optional[Path] = None) -> str:
return hashlib.sha256(canonical.encode("utf-8", "replace")).hexdigest()[:12]
+def process_birth(proc) -> float:
+ """A managed worker's identity birth for the ``psutil.Process`` *proc*, the same in every process.
+
+ On macOS psutil shifts ``create_time()`` by ``abs(INIT_BOOT_TIME - boot_time())``
+ (``psutil._psosx.adjust_proc_create_time``), with a boot-time reference frozen when psutil
+ was imported in the CALLING process. After a clock step (NTP, sleep/wake) a long-lived
+ gateway owner and a freshly spawned worker read different births for the same process, and
+ the exact identity match refused every managed worker (#140580). The kernel's own start
+ (``monotonic=True``: the unadjusted ``kinfo_proc`` epoch start) is what every process sees.
+ Other platforms keep ``create_time()``: Linux and Windows apply no per-process reference.
+ Raises ``psutil.Error`` like ``create_time()``.
+ """
+ import psutil
+ if getattr(psutil, "MACOS", False):
+ platform_proc = getattr(proc, "_proc", None)
+ if platform_proc is not None:
+ try:
+ return float(platform_proc.create_time(monotonic=True))
+ except TypeError: # a psutil without the monotonic read: fall back to the adjusted one
+ pass
+ return float(proc.create_time())
+
+
def _process_create_time(pid: Optional[int] = None) -> Optional[float]:
"""``psutil`` create time for ``pid`` (default: this process); ``None`` when psutil can't say."""
try:
diff --git a/tests/gateway/test_worker_admission_reservation.py b/tests/gateway/test_worker_admission_reservation.py
index 49b64b77365..5bdb0fdfa02 100644
--- a/tests/gateway/test_worker_admission_reservation.py
+++ b/tests/gateway/test_worker_admission_reservation.py
@@ -139,3 +139,40 @@ async def test_input_queued_behind_a_live_worker_is_released_then_runs_when_it_f
process.stdin.close()
process.wait(timeout=5)
db.close()
+
+
[email protected]('drift', [5.0, -5.0])
+def test_reservation_survives_a_clock_step_under_emulated_macos_psutil(tmp_path, monkeypatch, drift):
+ """#140580, on every platform: emulate psutil's macOS ``create_time()``. The kernel records an
+ epoch start; psutil adds ``abs(INIT_BOOT_TIME - boot_time())`` (``_psosx.adjust_proc_create_time``)
+ with a reference frozen at import. A long-lived owner's reference is stale after a clock step
+ either way, a fresh worker's is not, so the exact match refused every managed worker.
+ Monkeypatched: ``psutil.MACOS`` and the platform ``create_time``; the reservation is real."""
+ from gateway.session_worker_reservation import reserve_admission_worker
+ db = SessionDB(tmp_path / 'state.db')
+ process = subprocess.Popen([sys.executable, '-c', 'import sys; sys.stdin.read()'], stdin=subprocess.PIPE,
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+ try:
+ frame = hello(process) # what the freshly started worker reports (its reference is current)
+ kernel = psutil._psplatform.Process.create_time
+
+ def owner_view(self, monotonic=False, **kwargs): # the owner's stale reference
+ start = kernel(self, **kwargs) # an epoch start, as macOS kinfo_proc records it
+ return start if monotonic else start + abs(drift)
+
+ monkeypatch.setattr(psutil, 'MACOS', True)
+ monkeypatch.setattr(psutil._psplatform.Process, 'create_time', owner_view)
+ db.create_session('owned', 'cli')
+ epoch = begin_runtime_epoch(db, instance_id='owner')
+ authority = SimpleNamespace(db=db, epoch=epoch, profile_id=str(tmp_path), _require_admission_open=lambda: None)
+ admitted = admit_session_input(db, epoch=epoch, principal_id='human', session_id='owned', request_id='input', payload={})
+ claim_session_input(db, epoch=epoch, session_id='owned')
+ kwargs = dict(admission_id=admitted['admission_id'], process=process, principal_id='human')
+ with pytest.raises(Exception, match='permission_denied'): # a forged birth still fails
+ reserve_admission_worker(authority, hello=frame | {'birth': frame['birth'] + 1}, **kwargs)
+ scope = reserve_admission_worker(authority, hello=frame, **kwargs)
+ assert scope['pid'] == process.pid and scope['birth'] == frame['birth']
+ finally:
+ process.stdin.close()
+ process.wait(timeout=5)
+ db.close()