Skip to content

The agent that grows with you

fix(gateway): compare a managed worker's birth by its kernel start on Linux

unsupportedpastelsdfe5b10677ae
diff --git a/agent/managed_worker.py b/agent/managed_worker.py
index a20db791096..b49a97acd0c 100644
--- a/agent/managed_worker.py
+++ b/agent/managed_worker.py
@@ -446,9 +446,9 @@ def hello():
the ancestor chain it observes. Launchers (uv's venv python.exe) put the owner's Popen
handle one or two hops above; the owner, never this process, decides whether they match."""
import psutil
- from hermes_cli.process_identity import process_birth
+ from hermes_cli.process_identity import worker_birth
me = psutil.Process()
- return {'pid': me.pid, 'birth': process_birth(me), 'ancestors': [p.pid for p in me.parents()[:3]]}
+ return {'pid': me.pid, 'birth': worker_birth(me), 'ancestors': [p.pid for p in me.parents()[:3]]}
def main():
diff --git a/gateway/session_kanban.py b/gateway/session_kanban.py
index 6b89dfa3737..d59ed2eecf7 100644
--- a/gateway/session_kanban.py
+++ b/gateway/session_kanban.py
@@ -105,7 +105,7 @@ async def run_task(connection, params):
from hermes_cli import kanban_db as kb
import os
import psutil
- from hermes_cli.process_identity import process_birth
+ from hermes_cli.process_identity import worker_birth
context = json.loads(policy.kanban_json)
def admit_claim():
@@ -126,7 +126,7 @@ async def run_task(connection, params):
raise RuntimeStoreError('stale_kanban_claim')
kb._append_event(conn, task.id, 'owner_admitted',
{'db': owner_db, 'session_id': ref.session_id,
- 'request_id': request_id, 'pid': os.getpid(), 'birth': process_birth(psutil.Process())},
+ 'request_id': request_id, 'pid': os.getpid(), 'birth': worker_birth(psutil.Process())},
run_id=task.current_run_id)
from gateway.session_runtime_workers import tracked_write
await tracked_write(authority, admit_claim)
diff --git a/gateway/session_managed_worker.py b/gateway/session_managed_worker.py
index 86baeb5c763..cf672513369 100644
--- a/gateway/session_managed_worker.py
+++ b/gateway/session_managed_worker.py
@@ -293,8 +293,8 @@ class ManagedWorker:
return
try:
proc = psutil.Process(self.worker[0])
- from hermes_cli.process_identity import process_birth
- if process_birth(proc) == self.worker[1]:
+ from hermes_cli.process_identity import worker_birth
+ if worker_birth(proc) == self.worker[1]:
(proc.kill if kill else proc.terminate)()
except psutil.Error:
pass
diff --git a/gateway/session_worker.py b/gateway/session_worker.py
index aaa3a6bac27..aa527defc48 100644
--- a/gateway/session_worker.py
+++ b/gateway/session_worker.py
@@ -30,8 +30,8 @@ def _claim(connection, ref, params):
raise RuntimeStoreError('invalid_params')
try:
process = psutil.Process(pid)
- from hermes_cli.process_identity import process_birth
- if (process_birth(process) != birth or not process.is_running()
+ from hermes_cli.process_identity import worker_birth
+ if (worker_birth(process) != birth or not process.is_running()
or process.status() == psutil.STATUS_ZOMBIE
or process.username() != psutil.Process().username()):
raise RuntimeStoreError('permission_denied')
diff --git a/gateway/session_worker_delegation.py b/gateway/session_worker_delegation.py
index 11d773d825d..5bb3640249c 100644
--- a/gateway/session_worker_delegation.py
+++ b/gateway/session_worker_delegation.py
@@ -69,8 +69,8 @@ def _dispatch(db, conn, session_id, payload, *, execution_id, worker_pid, worker
import psutil
from gateway.status import get_process_start_time
try:
- from hermes_cli.process_identity import process_birth
- if process_birth(psutil.Process(worker_pid)) != worker_birth:
+ from hermes_cli import process_identity
+ if process_identity.worker_birth(psutil.Process(worker_pid)) != worker_birth:
raise RuntimeStoreError('permission_denied')
owner_started_at = get_process_start_time(worker_pid)
except psutil.Error as exc:
diff --git a/gateway/session_worker_reservation.py b/gateway/session_worker_reservation.py
index bcde646e8e9..1be3bc9e87a 100644
--- a/gateway/session_worker_reservation.py
+++ b/gateway/session_worker_reservation.py
@@ -37,8 +37,8 @@ def verify_worker_identity(process, hello):
if handle.ppid() != os.getpid() or handle.status() == psutil.STATUS_ZOMBIE:
raise RuntimeStoreError('permission_denied')
worker = psutil.Process(pid)
- from hermes_cli.process_identity import process_birth
- birth = process_birth(worker) # clock-step stable on macOS (#140580)
+ from hermes_cli.process_identity import worker_birth
+ birth = worker_birth(worker) # kernel start: stable across clock steps (#140580) and btime re-anchors
if birth != hello['birth'] or not worker.is_running() or worker.status() == psutil.STATUS_ZOMBIE:
raise RuntimeStoreError('permission_denied')
# psutil's parent() already refuses a recycled ppid (parent born after child).
diff --git a/hermes_cli/kanban_owner_recovery.py b/hermes_cli/kanban_owner_recovery.py
index 90b51bd61c4..ece6c5cdadc 100644
--- a/hermes_cli/kanban_owner_recovery.py
+++ b/hermes_cli/kanban_owner_recovery.py
@@ -25,8 +25,8 @@ def owner_reclaim_paused(conn, task_id):
return row[1] == 'interrupted'
if row is None or row[0] in {'queued', 'unknown'}:
return True
- from hermes_cli.process_identity import process_birth
- return process_birth(psutil.Process(binding['pid'])) != binding['birth']
+ from hermes_cli.process_identity import worker_birth
+ return worker_birth(psutil.Process(binding['pid'])) != binding['birth']
except (OSError, ValueError, KeyError, sqlite3.Error):
return True
except psutil.Error:
@@ -81,8 +81,8 @@ def _owner_process_gone(receipt) -> bool:
if not kb._pid_alive(pid):
return True # absent, or an exited zombie still waiting to be reaped
try:
- from hermes_cli.process_identity import process_birth
- current = process_birth(psutil.Process(pid))
+ from hermes_cli.process_identity import worker_birth
+ current = worker_birth(psutil.Process(pid))
except psutil.NoSuchProcess:
return True
except psutil.Error:
diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py
index 1471c81138f..d055ce0d466 100644
--- a/hermes_cli/process_identity.py
+++ b/hermes_cli/process_identity.py
@@ -86,6 +86,21 @@ def process_birth(proc) -> float:
return float(proc.create_time())
+def worker_birth(proc) -> float:
+ """The birth a managed worker's identity fences compare: its hello, its scope and the Kanban
+ owner receipt. :func:`process_birth`, except on Linux, where psutil's ``create_time()`` adds the
+ ``/proc/stat`` btime it re-reads on every call. WSL2 re-anchors btime after host sleep, so the
+ owner read a live worker's birth shifted from the one it reported and refused it (KB-12). The
+ kernel's start ticks since boot (``monotonic=True``) never move. The spawn ledger keeps
+ :func:`process_birth`: its persisted epoch births must keep matching across an update.
+ Raises ``psutil.Error`` like ``create_time()``.
+ """
+ import psutil
+ if getattr(psutil, "LINUX", False):
+ return float(proc._proc.create_time(monotonic=True))
+ return process_birth(proc)
+
+
def _process_create_time(pid: Optional[int] = None) -> Optional[float]:
""":func:`process_birth` of ``pid`` (default: this process); ``None`` when psutil can't say."""
try:
diff --git a/tests/gateway/fixtures/agent_persistence_worker.py b/tests/gateway/fixtures/agent_persistence_worker.py
index e60467fdaef..371ca82143e 100644
--- a/tests/gateway/fixtures/agent_persistence_worker.py
+++ b/tests/gateway/fixtures/agent_persistence_worker.py
@@ -6,6 +6,7 @@ import sys
import psutil
+from hermes_cli.process_identity import worker_birth
from agent.runtime_session_store import RuntimeSessionStore, WorkerRPC
@@ -36,7 +37,7 @@ def main():
rpc = WorkerRPC(home)
scope = dict(profile_id=str(home), session_id=command['session_id'],
execution_id='agent-worker', generation=0, pid=os.getpid(),
- birth=psutil.Process().create_time(), secret='private-agent-worker')
+ birth=worker_birth(psutil.Process()), secret='private-agent-worker')
registered = rpc('worker.register', **scope, kind='compute')
scope['epoch'] = registered['owner_epoch']
store = RuntimeSessionStore(rpc, scope, home / 'worker-outboxes' / 'agent-worker')
diff --git a/tests/gateway/fixtures/compression_persistence_worker.py b/tests/gateway/fixtures/compression_persistence_worker.py
index 0fc798061d9..af81db18713 100644
--- a/tests/gateway/fixtures/compression_persistence_worker.py
+++ b/tests/gateway/fixtures/compression_persistence_worker.py
@@ -6,6 +6,7 @@ import sys
import psutil
+from hermes_cli.process_identity import worker_birth
from agent.runtime_session_store import RuntimeSessionStore, WorkerRPC
from tests.gateway.fixtures.agent_persistence_worker import writable_fds
@@ -18,7 +19,7 @@ def main():
if event == 'sqlite3.connect' and 'state.db' in str(args[0]) else None)
rpc = WorkerRPC(home)
scope = dict(profile_id=str(home), session_id=command['session_id'], execution_id='compression-worker',
- generation=0, pid=os.getpid(), birth=psutil.Process().create_time(), secret='private-compression-worker')
+ generation=0, pid=os.getpid(), birth=worker_birth(psutil.Process()), secret='private-compression-worker')
registered = rpc('worker.register', **scope, kind='compute')
scope['epoch'] = registered['owner_epoch']
store = RuntimeSessionStore(rpc, scope, home / 'worker-outboxes' / 'compression-worker')
diff --git a/tests/gateway/fixtures/ledger_tool_worker.py b/tests/gateway/fixtures/ledger_tool_worker.py
index 63e8987cb89..f44107e6414 100644
--- a/tests/gateway/fixtures/ledger_tool_worker.py
+++ b/tests/gateway/fixtures/ledger_tool_worker.py
@@ -6,6 +6,7 @@ import sys
import psutil
+from hermes_cli.process_identity import worker_birth
from agent.runtime_session_store import RuntimeSessionStore, WorkerRPC
from tests.gateway.fixtures.agent_persistence_worker import writable_fds
@@ -20,7 +21,7 @@ def main():
sys.addaudithook(audit)
rpc = WorkerRPC(home)
scope = dict(profile_id=str(home), session_id=command['session_id'], execution_id='tool-worker', generation=0,
- pid=os.getpid(), birth=psutil.Process().create_time(), secret='private-tool-worker')
+ pid=os.getpid(), birth=worker_birth(psutil.Process()), secret='private-tool-worker')
registered = rpc('worker.register', **scope, kind='compute')
scope['epoch'] = registered['owner_epoch']
store = RuntimeSessionStore(rpc, scope, home / 'worker-outboxes' / 'tool-worker')
diff --git a/tests/gateway/fixtures/lifecycle_persistence_worker.py b/tests/gateway/fixtures/lifecycle_persistence_worker.py
index 72feaa81199..881de159940 100644
--- a/tests/gateway/fixtures/lifecycle_persistence_worker.py
+++ b/tests/gateway/fixtures/lifecycle_persistence_worker.py
@@ -7,6 +7,7 @@ import time
import psutil
+from hermes_cli.process_identity import worker_birth
from agent.runtime_session_store import RuntimeSessionStore, WorkerRPC
from tests.gateway.fixtures.agent_persistence_worker import writable_fds
@@ -19,7 +20,7 @@ def main():
if event == 'sqlite3.connect' and 'state.db' in str(args[0]) else None)
rpc = WorkerRPC(home)
scope = dict(profile_id=str(home), session_id=sid, execution_id='lifecycle-worker', generation=0,
- pid=os.getpid(), birth=psutil.Process().create_time(), secret='private-lifecycle-worker')
+ pid=os.getpid(), birth=worker_birth(psutil.Process()), secret='private-lifecycle-worker')
scope['epoch'] = rpc('worker.register', **scope, kind='compute')['owner_epoch']
store = RuntimeSessionStore(rpc, scope, home / 'worker-outboxes' / 'lifecycle-worker')
from run_agent import AIAgent
diff --git a/tests/gateway/fixtures/persistence_worker.py b/tests/gateway/fixtures/persistence_worker.py
index 3b2abc51e23..60231d59f75 100644
--- a/tests/gateway/fixtures/persistence_worker.py
+++ b/tests/gateway/fixtures/persistence_worker.py
@@ -6,6 +6,8 @@ import sys
import psutil
+from hermes_cli.process_identity import worker_birth
+
def writable_fds(home):
result = []
@@ -45,7 +47,7 @@ for line in sys.stdin:
transport = module.WorkerRPC(home)
scope = {'profile_id': str(home), 'session_id': command['session_id'],
'execution_id': 'owned-worker', 'generation': 0,
- 'pid': os.getpid(), 'birth': psutil.Process().create_time(), 'secret': 'private-worker-secret'}
+ 'pid': os.getpid(), 'birth': worker_birth(psutil.Process()), 'secret': 'private-worker-secret'}
registration = transport('worker.register', **scope, kind='compute')
assert reason(lambda: transport('session.create', request_id='forbidden-worker-create', source='cli')) == 'permission_denied'
scope['epoch'] = registration['owner_epoch']
diff --git a/tests/gateway/fixtures/rotation_persistence_worker.py b/tests/gateway/fixtures/rotation_persistence_worker.py
index 98c0c1794d7..fb098e04150 100644
--- a/tests/gateway/fixtures/rotation_persistence_worker.py
+++ b/tests/gateway/fixtures/rotation_persistence_worker.py
@@ -6,6 +6,7 @@ import sys
import psutil
+from hermes_cli.process_identity import worker_birth
from agent.runtime_session_store import RuntimeSessionStore, WorkerRPC
from tests.gateway.fixtures.agent_persistence_worker import writable_fds
@@ -26,7 +27,7 @@ def main():
if event == 'sqlite3.connect' and 'state.db' in str(args[0]) else None)
rpc = WorkerRPC(home)
scope = dict(profile_id=str(home), session_id=command['session_id'], execution_id='compression-worker',
- generation=0, pid=os.getpid(), birth=psutil.Process().create_time(), secret='private-compression-worker')
+ generation=0, pid=os.getpid(), birth=worker_birth(psutil.Process()), secret='private-compression-worker')
registered = rpc('worker.register', **scope, kind='compute')
scope['epoch'] = registered['owner_epoch']
store = RuntimeSessionStore(rpc, scope, home / 'worker-outboxes' / 'compression-worker')
diff --git a/tests/gateway/test_lost_worker_receipt_compaction.py b/tests/gateway/test_lost_worker_receipt_compaction.py
index 4d135bf6d79..9231a9ab122 100644
--- a/tests/gateway/test_lost_worker_receipt_compaction.py
+++ b/tests/gateway/test_lost_worker_receipt_compaction.py
@@ -8,6 +8,7 @@ from types import SimpleNamespace
import psutil
+from hermes_cli.process_identity import worker_birth
from hermes_state import SessionDB
from hermes_state_runtime import (
admit_session_input, begin_runtime_epoch, claim_session_input, mutate_worker_execution,
@@ -19,7 +20,7 @@ READ = dict(target='s', include_ancestors=False, include_inactive=False, repair_
def hello(process):
- return {'type': 'hello', 'pid': process.pid, 'birth': psutil.Process(process.pid).create_time(), 'ancestors': [os.getpid()]}
+ return {'type': 'hello', 'pid': process.pid, 'birth': worker_birth(psutil.Process(process.pid)), 'ancestors': [os.getpid()]}
def _receipt_sizes(db):
diff --git a/tests/gateway/test_managed_worker_fencing.py b/tests/gateway/test_managed_worker_fencing.py
index 7d22111049b..dd410c3562b 100644
--- a/tests/gateway/test_managed_worker_fencing.py
+++ b/tests/gateway/test_managed_worker_fencing.py
@@ -8,13 +8,14 @@ from types import SimpleNamespace
import psutil
import pytest
+from hermes_cli.process_identity import worker_birth
from hermes_state import SessionDB
from hermes_state_runtime import admit_session_input, begin_runtime_epoch, claim_session_input
def hello(process):
"""A direct child's self-introduction (no launcher: chain length 0)."""
- return {'type': 'hello', 'pid': process.pid, 'birth': psutil.Process(process.pid).create_time(), 'ancestors': [os.getpid()]}
+ return {'type': 'hello', 'pid': process.pid, 'birth': worker_birth(psutil.Process(process.pid)), 'ancestors': [os.getpid()]}
def test_worker_loss_fences_same_generation_parent_result(tmp_path):
diff --git a/tests/gateway/test_managed_worker_finished_exit.py b/tests/gateway/test_managed_worker_finished_exit.py
index 0a3c5824a61..79834ca3da7 100644
--- a/tests/gateway/test_managed_worker_finished_exit.py
+++ b/tests/gateway/test_managed_worker_finished_exit.py
@@ -12,11 +12,12 @@ import pytest
from gateway.session_contract import Principal, SessionRef
CHILD = """import json, os, sys, time, psutil
+from hermes_cli.process_identity import worker_birth
me = psutil.Process()
out = os.fdopen(os.dup(sys.stdout.fileno()), 'wb', buffering=0)
def send(frame):
out.write((json.dumps(frame) + '\\n').encode())
-send({'type': 'hello', 'pid': me.pid, 'birth': me.create_time(), 'ancestors': [p.pid for p in me.parents()[:3]]})
+send({'type': 'hello', 'pid': me.pid, 'birth': worker_birth(me), 'ancestors': [p.pid for p in me.parents()[:3]]})
for line in sys.stdin.buffer:
frame = json.loads(line)
if 'version' in frame:
diff --git a/tests/gateway/test_managed_worker_hard_stop_frame_fence.py b/tests/gateway/test_managed_worker_hard_stop_frame_fence.py
index 425ba3ff00c..8fc20259049 100644
--- a/tests/gateway/test_managed_worker_hard_stop_frame_fence.py
+++ b/tests/gateway/test_managed_worker_hard_stop_frame_fence.py
@@ -68,10 +68,11 @@ def test_worker_stop_hard_cancels_the_agent_before_and_after_construction(tmp_pa
CHILD = """import json, os, sys, psutil
+from hermes_cli.process_identity import worker_birth
me = psutil.Process()
out = os.fdopen(os.dup(sys.stdout.fileno()), 'wb', buffering=0)
send = lambda f: out.write((json.dumps(f) + '\\n').encode())
-send({'type': 'hello', 'pid': me.pid, 'birth': me.create_time(), 'ancestors': [p.pid for p in me.parents()[:3]]})
+send({'type': 'hello', 'pid': me.pid, 'birth': worker_birth(me), 'ancestors': [p.pid for p in me.parents()[:3]]})
for line in sys.stdin.buffer:
for i in range(20):
send({'type': 'delta', 'text': str(i)})
diff --git a/tests/gateway/test_managed_worker_reservation_off_loop.py b/tests/gateway/test_managed_worker_reservation_off_loop.py
index 569e78d02d6..d6ae50ce4c5 100644
--- a/tests/gateway/test_managed_worker_reservation_off_loop.py
+++ b/tests/gateway/test_managed_worker_reservation_off_loop.py
@@ -16,11 +16,12 @@ from gateway.session_contract import Principal, SessionRef
from tests.waiting import wait_until
CHILD = """import json, os, sys, time, psutil
+from hermes_cli.process_identity import worker_birth
me = psutil.Process()
out = os.fdopen(os.dup(sys.stdout.fileno()), 'wb', buffering=0)
def send(frame):
out.write((json.dumps(frame) + '\\n').encode())
-send({'type': 'hello', 'pid': me.pid, 'birth': me.create_time(), 'ancestors': [p.pid for p in me.parents()[:3]]})
+send({'type': 'hello', 'pid': me.pid, 'birth': worker_birth(me), 'ancestors': [p.pid for p in me.parents()[:3]]})
for line in sys.stdin.buffer:
frame = json.loads(line)
if 'version' in frame:
diff --git a/tests/gateway/test_managed_worker_stop_before_hello.py b/tests/gateway/test_managed_worker_stop_before_hello.py
index 30e80738b3a..0ab723665a2 100644
--- a/tests/gateway/test_managed_worker_stop_before_hello.py
+++ b/tests/gateway/test_managed_worker_stop_before_hello.py
@@ -49,11 +49,12 @@ def test_stop_before_hello_terminates_child_and_settles_interrupted(monkeypatch)
CHILD = """import json, os, sys, psutil
+from hermes_cli.process_identity import worker_birth
me = psutil.Process()
out = os.fdopen(os.dup(sys.stdout.fileno()), 'wb', buffering=0)
def send(frame):
out.write((json.dumps(frame) + '\\n').encode())
-send({'type': 'hello', 'pid': me.pid, 'birth': me.create_time(), 'ancestors': [p.pid for p in me.parents()[:3]]})
+send({'type': 'hello', 'pid': me.pid, 'birth': worker_birth(me), 'ancestors': [p.pid for p in me.parents()[:3]]})
for line in sys.stdin.buffer:
frame = json.loads(line)
with open(sys.argv[1], 'a') as log:
diff --git a/tests/gateway/test_managed_worker_trampoline.py b/tests/gateway/test_managed_worker_trampoline.py
index ea14d690512..e45181cedc2 100644
--- a/tests/gateway/test_managed_worker_trampoline.py
+++ b/tests/gateway/test_managed_worker_trampoline.py
@@ -18,6 +18,7 @@ from types import SimpleNamespace
import psutil
import pytest
+from hermes_cli.process_identity import worker_birth
from hermes_state import SessionDB
from hermes_state_runtime import admit_session_input, begin_runtime_epoch, claim_session_input
from tests.gateway.fixtures.local_recovery_probe import child_env, daemon, rpc, websocket
@@ -31,7 +32,7 @@ LAUNCHER = ('import subprocess, sys\n'
def _hello(pid):
proc = psutil.Process(pid)
- return {'type': 'hello', 'pid': pid, 'birth': proc.create_time(),
+ return {'type': 'hello', 'pid': pid, 'birth': worker_birth(proc),
'ancestors': [p.pid for p in proc.parents()[:4]]}
@@ -63,7 +64,7 @@ def test_owner_binds_reservation_to_verified_descendant_of_the_handle_only(tmp_p
reserve_admission_worker(authority, hello=_hello(os.getpid()), **kwargs)
assert db._read_one('SELECT COUNT(*) FROM worker_executions')[0] == 0
scope = reserve_admission_worker(authority, hello=_hello(real), **kwargs)
- assert (scope['pid'], scope['birth']) == (real, psutil.Process(real).create_time())
+ assert (scope['pid'], scope['birth']) == (real, worker_birth(psutil.Process(real)))
assert scope['pid'] != launcher.pid
finally:
# The real interpreter first, while it is still the launcher's child: killing the launcher
diff --git a/tests/gateway/test_managed_worker_unknown_status.py b/tests/gateway/test_managed_worker_unknown_status.py
index ed563acb058..b59abbbef13 100644
--- a/tests/gateway/test_managed_worker_unknown_status.py
+++ b/tests/gateway/test_managed_worker_unknown_status.py
@@ -10,9 +10,10 @@ import pytest
from gateway.session_contract import Principal, SessionRef
CHILD = """import json, os, sys, psutil
+from hermes_cli.process_identity import worker_birth
me = psutil.Process()
out = os.fdopen(os.dup(sys.stdout.fileno()), 'wb', buffering=0)
-out.write((json.dumps({'type': 'hello', 'pid': me.pid, 'birth': me.create_time(),
+out.write((json.dumps({'type': 'hello', 'pid': me.pid, 'birth': worker_birth(me),
'ancestors': [p.pid for p in me.parents()[:3]]}) + '\\n').encode())
for line in sys.stdin.buffer:
out.write(b'{"type": "bogus"}\\n')
diff --git a/tests/gateway/test_restart_interrupt_notice.py b/tests/gateway/test_restart_interrupt_notice.py
index 20845c5ccca..81ac6f42d2c 100644
--- a/tests/gateway/test_restart_interrupt_notice.py
+++ b/tests/gateway/test_restart_interrupt_notice.py
@@ -148,11 +148,12 @@ async def test_user_stop_is_not_relabelled_when_shutdown_starts_during_settlemen
STOPPABLE_CHILD = """import json, os, sys, psutil
+from hermes_cli.process_identity import worker_birth
me = psutil.Process()
out = os.fdopen(os.dup(sys.stdout.fileno()), 'wb', buffering=0)
def send(frame):
out.write((json.dumps(frame) + '\\n').encode())
-send({'type': 'hello', 'pid': me.pid, 'birth': me.create_time(), 'ancestors': [p.pid for p in me.parents()[:3]]})
+send({'type': 'hello', 'pid': me.pid, 'birth': worker_birth(me), 'ancestors': [p.pid for p in me.parents()[:3]]})
for line in sys.stdin.buffer:
frame = json.loads(line)
if frame == {'type': 'stop'}:
diff --git a/tests/gateway/test_unknown_resolution_physical_worker.py b/tests/gateway/test_unknown_resolution_physical_worker.py
index 6d7f5eac9ea..fc6bfa7252e 100644
--- a/tests/gateway/test_unknown_resolution_physical_worker.py
+++ b/tests/gateway/test_unknown_resolution_physical_worker.py
@@ -9,12 +9,13 @@ from types import SimpleNamespace
import psutil
import pytest
+from hermes_cli.process_identity import worker_birth
from hermes_state import SessionDB
import hermes_state_runtime as rt
def hello(process):
- return {'type': 'hello', 'pid': process.pid, 'birth': psutil.Process(process.pid).create_time(),
+ return {'type': 'hello', 'pid': process.pid, 'birth': worker_birth(psutil.Process(process.pid)),
'ancestors': [os.getpid()]}
diff --git a/tests/gateway/test_worker_admission_reservation.py b/tests/gateway/test_worker_admission_reservation.py
index 5fe01d96a17..154093ec8d4 100644
--- a/tests/gateway/test_worker_admission_reservation.py
+++ b/tests/gateway/test_worker_admission_reservation.py
@@ -1,7 +1,9 @@
"""Owner-issued admission reservations never weaken idle public registration."""
+import json
import os
import subprocess
import sys
+from pathlib import Path
from types import SimpleNamespace
import psutil
@@ -14,7 +16,8 @@ from hermes_state_runtime import (admit_session_input, begin_runtime_epoch, clai
def hello(process):
"""A direct child's self-introduction (no launcher: chain length 0)."""
- return {'type': 'hello', 'pid': process.pid, 'birth': psutil.Process(process.pid).create_time(), 'ancestors': [os.getpid()]}
+ from hermes_cli.process_identity import worker_birth
+ return {'type': 'hello', 'pid': process.pid, 'birth': worker_birth(psutil.Process(process.pid)), 'ancestors': [os.getpid()]}
def test_started_admission_binds_only_exact_owned_process_and_principal(tmp_path):
@@ -119,7 +122,7 @@ async def test_input_queued_behind_a_live_worker_is_released_then_runs_when_it_f
connection = SimpleNamespace(authority=authority, actor=actor)
ref = SessionRef('p', 'owned')
scope = {'profile_id': 'p', 'session_id': 'owned', 'execution_id': 'compute', 'generation': 0,
- 'pid': process.pid, 'birth': psutil.Process(process.pid).create_time(), 'secret': 'w' * 32}
+ 'pid': process.pid, 'birth': hello(process)['birth'], 'secret': 'w' * 32}
try:
registered = await worker_request(connection, ref, scope | {'kind': 'compute'}, operation='register')
# Native ingress waits on the admission it committed; a blocked FIFO must answer it.
@@ -209,6 +212,39 @@ def test_process_fingerprints_survive_a_clock_step_on_macos(stale_psutil_boot_re
process.wait(timeout=5)
+
+def test_reservation_survives_a_boot_time_reanchor_on_linux(tmp_path, monkeypatch):
+ """KB-12: Linux psutil's ``create_time()`` adds the ``/proc/stat`` btime it re-reads on every
+ call, and WSL2 re-anchors btime after host sleep, so the owner read a live worker's birth
+ shifted from the one the worker reported and refused every managed worker."""
+ from gateway.session_worker_reservation import reserve_admission_worker
+ code = 'import json, sys; from agent.managed_worker import hello; print(json.dumps(hello()), flush=True); sys.stdin.read()'
+ process = subprocess.Popen([sys.executable, '-c', code], cwd=Path(__file__).resolve().parents[2],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+ db = SessionDB(tmp_path / 'state.db')
+ try:
+ frame = {'type': 'hello', **json.loads(process.stdout.readline())} # the real worker's hello
+ reported = psutil.Process(process.pid).create_time()
+ boot = psutil.boot_time()
+ monkeypatch.setattr(psutil._pslinux, 'boot_time', lambda: boot + 3600)
+ assert psutil.Process(process.pid).create_time() != reported # the re-anchor is real
+ db.create_session('owned', 'cli')
+ epoch = begin_runtime_epoch(db, instance_id='owner')
+ authority = SimpleNamespace(db=db, epoch=epoch, profile_id=str(tmp_path), _require_admission_open=lambda: None)
+ admitted = admit_session_input(db, epoch=epoch, principal_id='human', session_id='owned', request_id='input', payload={})
+ claim_session_input(db, epoch=epoch, session_id='owned')
+ kwargs = dict(admission_id=admitted['admission_id'], process=process, principal_id='human')
+ with pytest.raises(Exception, match='permission_denied'): # a forged birth still fails
+ reserve_admission_worker(authority, hello=frame | {'birth': frame['birth'] + 1}, **kwargs)
+ scope = reserve_admission_worker(authority, hello=frame, **kwargs)
+ assert scope['pid'] == process.pid and scope['birth'] == frame['birth']
+ finally:
+ process.stdin.close()
+ process.wait(timeout=10)
+ process.stdout.close()
+ db.close()
+
def test_process_birth_is_create_time_where_psutil_applies_no_reference():
"""Linux and Windows read ``create_time()`` unchanged, so every stored fingerprint and ledger
row written before KB-6 still matches there; on macOS with no step psutil returns the kernel
diff --git a/tests/gateway/test_worker_delegation_bridge.py b/tests/gateway/test_worker_delegation_bridge.py
index fc6bfe981ff..384e3f037e6 100644
--- a/tests/gateway/test_worker_delegation_bridge.py
+++ b/tests/gateway/test_worker_delegation_bridge.py
@@ -13,6 +13,7 @@ from types import SimpleNamespace
import psutil
import pytest
+from hermes_cli.process_identity import worker_birth
from hermes_state import SessionDB
from hermes_state_runtime import RuntimeStoreError, begin_runtime_epoch, register_worker_execution
from gateway.session_contract import SessionRef
@@ -38,7 +39,7 @@ def test_bridge_publishes_committed_completion_and_rejects_another_principal(tmp
connection = SimpleNamespace(authority=authority, actor=actor)
ref = SessionRef('profile', 'parent')
scope = dict(profile_id='profile', session_id='parent', execution_id='worker', generation=0,
- pid=worker.pid, birth=psutil.Process(worker.pid).create_time(), secret='private-worker-secret')
+ pid=worker.pid, birth=worker_birth(psutil.Process(worker.pid)), secret='private-worker-secret')
claim = _claim(connection, ref, scope)
register_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,
kind='compute', adoption_secret=claim)
diff --git a/tests/gateway/test_worker_delegation_receipts.py b/tests/gateway/test_worker_delegation_receipts.py
index 5f791c1429e..493e9cfd8e8 100644
--- a/tests/gateway/test_worker_delegation_receipts.py
+++ b/tests/gateway/test_worker_delegation_receipts.py
@@ -7,6 +7,7 @@ import time
import psutil
import pytest
+from hermes_cli.process_identity import worker_birth
from agent.runtime_session_store import RuntimeSessionStore
from hermes_state import SessionDB
from hermes_state_runtime import (RuntimeStoreError, begin_runtime_epoch, register_worker_execution,
@@ -25,7 +26,7 @@ def test_lost_ack_reopens_owner_and_replays_same_ledger_receipt(tmp_path):
register_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent',
generation=0, kind='compute', adoption_secret='private')
scope = dict(epoch=epoch, execution_id='worker', session_id='parent', generation=0,
- pid=os.getpid(), birth=psutil.Process().create_time())
+ pid=os.getpid(), birth=worker_birth(psutil.Process()))
lost = [True]
def rpc(method, **params):
assert method == 'worker.persist'
@@ -75,7 +76,7 @@ def test_receipt_fences_wrong_session_generation_and_owner_epoch(tmp_path):
generation=0, kind='compute', adoption_secret='private')
scope = dict(epoch=epoch, execution_id='worker', session_id='parent', generation=0,
sequence=1, operation='delegation.read', payload={'delegation_id': 'none'},
- worker_pid=os.getpid(), worker_birth=psutil.Process().create_time())
+ worker_pid=os.getpid(), worker_birth=worker_birth(psutil.Process()))
for override, error in [({'session_id': 'foreign'}, 'permission_denied'),
({'generation': 1}, 'stale_generation'),
({'epoch': epoch + 1}, 'stale_epoch'),
diff --git a/tests/gateway/test_worker_register_restriction.py b/tests/gateway/test_worker_register_restriction.py
index 2e3a62ec0d6..5469c3497d8 100644
--- a/tests/gateway/test_worker_register_restriction.py
+++ b/tests/gateway/test_worker_register_restriction.py
@@ -9,6 +9,8 @@ from types import SimpleNamespace
import psutil
import pytest
+from hermes_cli.process_identity import worker_birth
+
@pytest.mark.asyncio
async def test_worker_register_requires_the_owners_launched_worker(tmp_path, monkeypatch):
@@ -33,7 +35,7 @@ async def test_worker_register_requires_the_owners_launched_worker(tmp_path, mon
def scope(pid):
return {'profile_id': 'p', 'session_id': 'owned', 'execution_id': f'compute-{pid}', 'generation': 0,
- 'pid': pid, 'birth': psutil.Process(pid).create_time(), 'secret': 'w' * 32, 'kind': 'compute'}
+ 'pid': pid, 'birth': worker_birth(psutil.Process(pid)), 'secret': 'w' * 32, 'kind': 'compute'}
interactive = SimpleNamespace(authority=authority, actor=Principal(
'human', 'p', frozenset({'session:create', 'session:read', 'session:submit', 'session:control'}), 't'))
worker = SimpleNamespace(authority=authority, actor=Principal('human', 'p', frozenset({'worker:adopt'}), 't'))
diff --git a/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py
index ea9c67cdb92..7364fc62366 100644
--- a/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py
+++ b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py
@@ -34,7 +34,7 @@ from hermes_cli import kanban_db_connect as kbc
from hermes_cli import kanban_db_dispatch as kbd
from hermes_cli import kanban_diagnostics as kd
from hermes_cli import kanban_owner_recovery
-from hermes_cli.process_identity import process_birth
+from hermes_cli.process_identity import worker_birth
from hermes_cli.kanban_owner_recovery import (
bound_interpreter_gone,
owner_attempt_provably_over,
@@ -268,7 +268,7 @@ def test_a_timed_out_managed_run_is_booked_timed_out_not_left_for_the_crash_swee
try:
run = board.dead_attempt(
"t_slow", worker_pid=worker.pid, worker_fingerprint=kbd._process_fingerprint(worker.pid),
- owner_pid=os.getpid(), owner_birth=process_birth(psutil.Process()), admission=("started", None))
+ owner_pid=os.getpid(), owner_birth=worker_birth(psutil.Process()), admission=("started", None))
board.event("t_slow", run, "worker_bound", {"pid": interpreter.pid,
"started_at": kbd._process_fingerprint(interpreter.pid)}, NOW)
board.conn.execute("UPDATE tasks SET max_runtime_seconds = 1 WHERE id = 't_slow'")
@@ -296,7 +296,7 @@ def test_an_exited_unreaped_owner_no_longer_holds_the_slot(board):
"""A zombie keeps its birth until reaped, but it has exited: it cannot be executing anything."""
child = subprocess.Popen([sys.executable, "-c", "input()"], stdin=subprocess.PIPE)
try:
- birth = process_birth(psutil.Process(child.pid))
+ birth = worker_birth(psutil.Process(child.pid))
child.stdin.close()
deadline = time.time() + 10
while psutil.Process(child.pid).status() != psutil.STATUS_ZOMBIE:
diff --git a/tests/hermes_state/test_runtime_worker_children.py b/tests/hermes_state/test_runtime_worker_children.py
index c44bf421350..5dfcbeca031 100644
--- a/tests/hermes_state/test_runtime_worker_children.py
+++ b/tests/hermes_state/test_runtime_worker_children.py
@@ -6,6 +6,7 @@ from functools import partial
import psutil
import pytest
+from hermes_cli.process_identity import worker_birth
from gateway.session_worker_child import reserve_child_execution
from hermes_state import SessionDB
import hermes_state_runtime as rt
@@ -13,7 +14,7 @@ import hermes_state_runtime as rt
def _reserve(db, epoch, sequence, *, limits=(2, 1), refused=None, execution_id='worker'):
handler = partial(reserve_child_execution, epoch=epoch, execution_id=execution_id,
- identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},
+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': worker_birth(psutil.Process())},
principal='local', limits=limits, refused=refused)
return rt.apply_worker_receipt(db, epoch=epoch, execution_id=execution_id, session_id='parent', generation=0,
sequence=sequence, operation='child.reserve',
@@ -93,7 +94,7 @@ def test_depth_cap_profile_and_an_early_child_finish_settle_the_child_row(ledger
assert db.get_session(child['session_id'])['profile_name'] == 'coder'
# The child is already at max_spawn_depth=1: the owner refuses its own child.request.
nested = partial(reserve_child_execution, epoch=epoch, execution_id=child['execution_id'],
- identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},
+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': worker_birth(psutil.Process())},
principal='local', limits=(2, 1), refused=None)
assert rt.apply_worker_receipt(db, epoch=epoch, execution_id=child['execution_id'], session_id=child['session_id'],
generation=0, sequence=1, operation='child.reserve',
@@ -137,7 +138,7 @@ async def test_child_reserve_verifies_the_parent_claim_and_refuses_while_drainin
connection = SimpleNamespace(authority=authority, actor=SimpleNamespace(
subject='human', profile_id='p', capabilities={'session:submit', 'worker:adopt'}))
scope = {'profile_id': 'p', 'session_id': 'parent', 'execution_id': 'compute', 'generation': 0,
- 'pid': process.pid, 'birth': psutil.Process(process.pid).create_time(), 'secret': 'w' * 32}
+ 'pid': process.pid, 'birth': worker_birth(psutil.Process(process.pid)), 'secret': 'w' * 32}
try:
epoch = (await worker_request(connection, None, scope | {'kind': 'compute'}, operation='register'))['owner_epoch']
@@ -166,7 +167,7 @@ def test_a_child_store_closed_before_any_turn_finishes_its_execution_and_ends_it
from agent.runtime_session_store import RuntimeSessionStore
db, epoch = ledger
reserve = partial(reserve_child_execution, epoch=epoch, execution_id='worker',
- identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},
+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': worker_birth(psutil.Process())},
principal='local', limits=(2, 1), refused=None)
def rpc(method, **params):
diff --git a/tests/tools/test_async_delegation_worker.py b/tests/tools/test_async_delegation_worker.py
index 54348a5542d..a5b7e39897a 100644
--- a/tests/tools/test_async_delegation_worker.py
+++ b/tests/tools/test_async_delegation_worker.py
@@ -6,6 +6,7 @@ import time
import pytest
import psutil
+from hermes_cli.process_identity import worker_birth
from hermes_state import SessionDB
from hermes_state_runtime import RuntimeStoreError
from tools import async_delegation as ledger
@@ -16,7 +17,7 @@ def test_worker_dispatch_is_bound_to_execution_and_cannot_replace_foreign_rows(t
db = SessionDB(tmp_path / 'state.db')
db.create_session('parent', source='cli')
db.create_session('other', source='cli')
- handlers = worker_delegation_handlers('worker-a', os.getpid(), psutil.Process().create_time())
+ handlers = worker_delegation_handlers('worker-a', os.getpid(), worker_birth(psutil.Process()))
payload = {'delegation_id': 'deleg_owned', 'task': {'goal': 'inert'}, 'dispatched_at': time.time(),
'parent_session_id': 'parent', 'session_key': '', 'origin_ui_session_id': '', 'origin_session_id': ''}
def apply(name, payload, sid='parent', table=handlers):
@@ -27,7 +28,7 @@ def test_worker_dispatch_is_bound_to_execution_and_cannot_replace_foreign_rows(t
with closing(sqlite3.connect(tmp_path / 'state.db')) as conn:
ledger._initialize_schema(conn)
apply('delegation.dispatch', payload)
- foreign = worker_delegation_handlers('worker-b', os.getpid(), psutil.Process().create_time())
+ foreign = worker_delegation_handlers('worker-b', os.getpid(), worker_birth(psutil.Process()))
for sid, table in [('other', handlers), ('parent', foreign)]:
with pytest.raises(RuntimeStoreError, match='permission_denied'):
apply('delegation.read', {'delegation_id': 'deleg_owned'}, sid, table)
@@ -47,7 +48,7 @@ def test_worker_completion_claims_refund_and_preserve_terminal_result(tmp_path):
from gateway.session_worker_delegation import worker_delegation_handlers
db = SessionDB(tmp_path / 'state.db')
db.create_session('parent', source='cli')
- handlers = worker_delegation_handlers('worker-a', os.getpid(), psutil.Process().create_time())
+ handlers = worker_delegation_handlers('worker-a', os.getpid(), worker_birth(psutil.Process()))
def apply(name, **payload):
return db._execute_write(lambda conn: handlers['delegation.' + name](db, conn, 'parent', payload))
try: