Skip to content

The agent that grows with you

fix(managed-worker): delegated children build, close and settle off Relay's loop

Teknium1f91fb386a60
diff --git a/agent/runtime_session_child.py b/agent/runtime_session_child.py
index 31866cc5d4d..68d9dec723b 100644
--- a/agent/runtime_session_child.py
+++ b/agent/runtime_session_child.py
@@ -6,6 +6,7 @@ child persists through the ordinary ``worker.persist`` path under its own claim.
(capacity, depth, draining) is a ValueError so ``delegate_task`` reports it as a tool error.
"""
import secrets
+from functools import partial
from agent.runtime_session_store import RuntimeSessionStore, call_off_event_loop
@@ -50,4 +51,7 @@ def reserve_child_store(parent):
child = result['child']
scope = dict(parent.scope, **child, secret=secret)
outbox = parent.path.parent / ('child-' + child['execution_id'].rsplit(':', 1)[1])
- return ChildSessionStore(parent.rpc, scope, outbox)
+ # Every child RPC hops off a running loop, not only this reservation: the child is built, run
+ # and closed inside delegate_task's callback, and its constructor already reads the store
+ # (ContextCompressor.bind_session_state), which WorkerRPC refuses on Relay's loop.
+ return ChildSessionStore(partial(call_off_event_loop, parent.rpc), scope, outbox)
diff --git a/gateway/session_worker_child.py b/gateway/session_worker_child.py
index 24cc9941630..9918bd49dff 100644
--- a/gateway/session_worker_child.py
+++ b/gateway/session_worker_child.py
@@ -11,7 +11,7 @@ import secrets
from functools import partial
from hermes_state_runtime import RuntimeStoreError, apply_worker_receipt
-from hermes_state_runtime_workers import CHILD_SEPARATOR
+from hermes_state_runtime_workers import CHILD_SEPARATOR, descendant_range
_DEFAULTS = {'max_concurrent_children': 10, 'max_spawn_depth': 1}
@@ -49,9 +49,9 @@ def reserve_child_execution(db, conn, session_id, payload, *, epoch, execution_i
raise RuntimeStoreError('invalid_params')
max_children, max_depth = limits
prefix = execution_id + CHILD_SEPARATOR
- live = conn.execute("SELECT COUNT(*) FROM worker_executions WHERE substr(execution_id,1,?)=? "
+ live = conn.execute("SELECT COUNT(*) FROM worker_executions WHERE execution_id>=? AND execution_id<? "
"AND instr(substr(execution_id,?),'/')=0 AND status!='terminal'",
- (len(prefix), prefix, len(prefix) + 1)).fetchone()[0]
+ (*descendant_range(execution_id), len(prefix) + 1)).fetchone()[0]
if refused is None and prefix.count(CHILD_SEPARATOR) > max_depth:
refused = 'delegation_depth_exceeded'
if refused is None and live >= max_children:
diff --git a/hermes_state_runtime_workers.py b/hermes_state_runtime_workers.py
index 118d2158d30..7519ad68575 100644
--- a/hermes_state_runtime_workers.py
+++ b/hermes_state_runtime_workers.py
@@ -54,10 +54,15 @@ def retire_child_executions(conn, execution_id):
"""A child never outlives the execution that reserved it: every live descendant goes terminal
with it, its read receipts shrink and its open session row is ended. A late child write is
then refused ``stale_generation`` like any write after retirement."""
- prefix = execution_id + CHILD_SEPARATOR
_retire_children(conn, conn.execute("SELECT execution_id,session_id FROM worker_executions "
- "WHERE substr(execution_id,1,?)=? AND status!='terminal'",
- (len(prefix), prefix)).fetchall())
+ "WHERE execution_id>=? AND execution_id<? AND status!='terminal'",
+ descendant_range(execution_id)).fetchall())
+
+
+def descendant_range(execution_id):
+ """``(low, high)`` bounds every descendant id of ``execution_id`` on the primary key index
+ (';' sorts right after the separator's ':'), so a no-child finish costs one index probe."""
+ return execution_id + CHILD_SEPARATOR, execution_id + CHILD_SEPARATOR[:-1] + ';'
def retire_stale_children(conn, epoch):
@@ -68,12 +73,16 @@ def retire_stale_children(conn, epoch):
def _retire_children(conn, rows):
- import time
- for child, session_id in rows:
+ for child, _session_id in rows:
conn.execute("UPDATE worker_executions SET status='terminal' WHERE execution_id=?", (child,))
_compact(conn, child)
- conn.execute("UPDATE sessions SET ended_at=?,end_reason='agent_close' WHERE id=? AND ended_at IS NULL",
- (time.time(), session_id))
+ _end_child_row(conn, child)
+
+
+def _end_child_row(conn, execution_id):
+ import time
+ conn.execute("UPDATE sessions SET ended_at=?,end_reason='agent_close' WHERE ended_at IS NULL "
+ "AND id=(SELECT session_id FROM worker_executions WHERE execution_id=?)", (time.time(), execution_id))
def compact_terminal_receipts(conn, execution_id):
@@ -86,6 +95,10 @@ def compact_terminal_receipts(conn, execution_id):
conflicts. Mutation acknowledgements (counts, row annotations, assignments) stay replayable."""
retire_child_executions(conn, execution_id)
_compact(conn, execution_id)
+ if CHILD_SEPARATOR in execution_id:
+ # A child that finished itself (an early close: failed constructor, refused sibling) never
+ # reaches _retire_children, which only selects live children: its row ends here.
+ _end_child_row(conn, execution_id)
def _compact(conn, execution_id):
diff --git a/hermes_state_worker_lifecycle.py b/hermes_state_worker_lifecycle.py
index 86b12b2c24e..420408b8a66 100644
--- a/hermes_state_worker_lifecycle.py
+++ b/hermes_state_worker_lifecycle.py
@@ -31,6 +31,11 @@ def worker_create(db, conn, session_id, payload):
if config is not None and (not isinstance(config, dict) or set(config) - _CONSTRUCTOR_CONFIG):
raise RuntimeStoreError('invalid_params')
insert_session_row_in_transaction(db, conn, session_id, **payload)
+ if config:
+ # An owner-reserved child row already holds its lineage marker, so the keep-existing upsert
+ # would drop these facts; merge them beside it (keys are the constructor set only).
+ conn.execute("UPDATE sessions SET model_config=json_patch(model_config,?) WHERE id=? "
+ "AND json_type(model_config,'$._delegate_from') IS NOT NULL", (json.dumps(config), session_id))
return {'value': session_id}
diff --git a/tests/gateway/test_managed_worker_delegation.py b/tests/gateway/test_managed_worker_delegation.py
index 3897d6c9753..4940d35f11f 100644
--- a/tests/gateway/test_managed_worker_delegation.py
+++ b/tests/gateway/test_managed_worker_delegation.py
@@ -148,3 +148,37 @@ def test_owner_restart_mid_child_retires_the_child_and_never_replays_it(tmp_path
"WHERE w.kind='child'") == [('agent_close',)]
# The lost turn is not replayed: the parent never got a child result to finish on.
assert not [s for s in peer.seen if s[0] == ['DELEGATE_BLOCK'] and s[1] == 'PARENT_DONE'], peer.seen
+
+
+def test_relay_driven_worker_builds_and_runs_its_child_off_the_relay_loop(tmp_path):
+ """NeMo Relay drives tool callbacks on its own event loop: the child's construction reads the
+ store (bind_session_state) and its close finishes the execution, and the worker RPC refuses a
+ synchronous call on a running loop, so every child store call must hop off it."""
+ relay = pytest.importorskip('nemo_relay')
+ if getattr(relay, '_native', None) is None:
+ pytest.skip('NeMo Relay native binding unavailable')
+ home, peer, url, env = _setup(tmp_path)
+ plugins = home / 'plugins.toml'
+ plugins.write_text('version = 1\n[[components]]\nkind = "observability"\nenabled = true\n'
+ '[components.config]\nversion = 4\n[components.config.atof]\nenabled = true\n'
+ '[[components.config.atof.sinks]]\ntype = "file"\noutput_directory = '
+ + json.dumps(str(home / 'relay-events')) + '\nfilename = "events.jsonl"\nmode = "overwrite"\n')
+ env['HERMES_NEMO_RELAY_PLUGINS_TOML'] = str(plugins)
+
+ async def exercise(desc):
+ sid = await _submit(home, desc, url, 'DELEGATE')
+ await async_wait_until(lambda: _rows(home, "SELECT status FROM session_admissions WHERE request_id='run'") == [
+ ('terminal',)], timeout=120, interval=.1, what='run terminal')
+ return sid
+ try:
+ with daemon(ROOT, home, env, barrier=False) as (_owner, desc):
+ sid = asyncio.run(exercise(desc))
+ finally:
+ peer.shutdown()
+ peer.server_close()
+ assert (home / 'relay-events' / 'events.jsonl').exists(), 'Relay never drove this turn'
+ tool = [c for r, c in _rows(home, 'SELECT role, content FROM messages WHERE session_id=?', sid) if r == 'tool']
+ assert tool and 'CHILD_RESULT_42' in tool[-1], tool
+ assert _rows(home, "SELECT w.status, s.end_reason FROM worker_executions w JOIN sessions s ON s.id=w.session_id "
+ "WHERE w.kind='child'") == [('terminal', 'agent_close')]
diff --git a/tests/hermes_state/test_runtime_worker_children.py b/tests/hermes_state/test_runtime_worker_children.py
index a06d350110c..c44bf421350 100644
--- a/tests/hermes_state/test_runtime_worker_children.py
+++ b/tests/hermes_state/test_runtime_worker_children.py
@@ -29,7 +29,7 @@ def _child_append(db, epoch, child, sequence, content, session_id=None):
@pytest.fixture
def ledger(tmp_path):
db = SessionDB(db_path=tmp_path / 'state.db')
- db.create_session('parent', source='kanban')
+ db.create_session('parent', source='kanban', profile_name='coder')
db.create_session('other', source='cli')
epoch = rt.begin_runtime_epoch(db, instance_id='boot')
rt.register_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,
@@ -85,3 +85,103 @@ def test_owner_restart_retires_in_flight_children_instead_of_recovering_them(led
assert db.get_session(child_sid)['end_reason'] == 'agent_close'
with pytest.raises(rt.RuntimeStoreError, match='stale_generation|stale_epoch'):
_child_append(db, replacement, {'execution_id': child_eid, 'session_id': child_sid}, 1, 'late')
+
+
+def test_depth_cap_profile_and_an_early_child_finish_settle_the_child_row(ledger):
+ db, epoch = ledger
+ child = _reserve(db, epoch, 1)['child']
+ assert db.get_session(child['session_id'])['profile_name'] == 'coder'
+ # The child is already at max_spawn_depth=1: the owner refuses its own child.request.
+ nested = partial(reserve_child_execution, epoch=epoch, execution_id=child['execution_id'],
+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},
+ principal='local', limits=(2, 1), refused=None)
+ assert rt.apply_worker_receipt(db, epoch=epoch, execution_id=child['execution_id'], session_id=child['session_id'],
+ generation=0, sequence=1, operation='child.reserve',
+ payload={'secret': 'nested-secret-00000001'},
+ handlers={'child.reserve': nested}) == {'refused': 'delegation_depth_exceeded'}
+ # The child's constructor facts land beside the owner's lineage marker.
+ rt.mutate_worker_execution(db, epoch=epoch, execution_id=child['execution_id'], session_id=child['session_id'],
+ generation=0, sequence=2, operation='session.create',
+ payload={'source': 'subagent', 'model_config': {'max_iterations': 7}})
+ import json
+ assert json.loads(db.get_session(child['session_id'])['model_config']) == {
+ '_delegate_from': 'parent', 'max_iterations': 7}
+ # An early close finishes the child itself (failed constructor, refused sibling): its row ends
+ # then, while the parent is still live, since parent retirement only selects live children.
+ rt.mutate_worker_execution(db, epoch=epoch, execution_id=child['execution_id'], session_id=child['session_id'],
+ generation=0, sequence=3, operation='execution.finish', payload={})
+ assert db.get_session(child['session_id'])['end_reason'] == 'agent_close'
+ assert db.get_session('parent')['ended_at'] is None
+
+
+async def test_child_reserve_verifies_the_parent_claim_and_refuses_while_draining(tmp_path):
+ """The request path, not just the handler: a forged parent secret never reaches the ledger,
+ and a draining owner answers with a refusal receipt the parent's journal can move past."""
+ import subprocess
+ import sys
+ from types import SimpleNamespace
+ from gateway.session_worker import worker_request
+ db = SessionDB(db_path=tmp_path / 'state.db')
+ db.create_session('parent', source='kanban')
+ process = subprocess.Popen([sys.executable, '-c', 'import sys; sys.stdin.read()'], stdin=subprocess.PIPE) # noqa: ASYNC220 -- returns at once; the test only needs a live pid
+ draining = []
+
+ def admission_open():
+ if draining:
+ raise rt.RuntimeStoreError('runtime_draining')
+ authority = SimpleNamespace(db=db, epoch=rt.begin_runtime_epoch(db, instance_id='owner'), profile_id='p',
+ _require_admission_open=admission_open, wake_after_worker=lambda session_id: None,
+ authorize=lambda actor, ref, capability: None,
+ sessions={}, logical_owner=lambda sid: sid)
+ connection = SimpleNamespace(authority=authority, actor=SimpleNamespace(
+ subject='human', profile_id='p', capabilities={'session:submit', 'worker:adopt'}))
+ scope = {'profile_id': 'p', 'session_id': 'parent', 'execution_id': 'compute', 'generation': 0,
+ 'pid': process.pid, 'birth': psutil.Process(process.pid).create_time(), 'secret': 'w' * 32}
+ try:
+ epoch = (await worker_request(connection, None, scope | {'kind': 'compute'}, operation='register'))['owner_epoch']
+
+ def reserve(sequence, **override):
+ return worker_request(connection, None, scope | override | {
+ 'epoch': epoch, 'sequence': sequence, 'operation': 'child.reserve',
+ 'payload': {'secret': f'child-secret-{sequence:08d}'}}, operation='persist')
+ with pytest.raises(rt.RuntimeStoreError, match='permission_denied'):
+ await reserve(1, secret='f' * 32)
+ draining.append(True)
+ assert await reserve(1) == {'refused': 'runtime_draining'}
+ draining.clear()
+ assert 'child' in await reserve(2)
+ with db._read_ctx() as conn:
+ assert conn.execute("SELECT COUNT(*) FROM worker_executions WHERE kind='child'").fetchone()[0] == 1
+ finally:
+ process.stdin.close()
+ process.wait(timeout=5)
+ db.close()
+
+
+def test_a_child_store_closed_before_any_turn_finishes_its_execution_and_ends_its_row(ledger, tmp_path):
+ """The constructor-failure / refused-sibling path: the child's store closes having written
+ nothing, and that close alone settles the reservation (the parent is still live)."""
+ from agent.runtime_session_child import reserve_child_store
+ from agent.runtime_session_store import RuntimeSessionStore
+ db, epoch = ledger
+ reserve = partial(reserve_child_execution, epoch=epoch, execution_id='worker',
+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},
+ principal='local', limits=(2, 1), refused=None)
+
+ def rpc(method, **params):
+ params = {k: params[k] for k in ('epoch', 'execution_id', 'session_id', 'generation', 'sequence', 'operation', 'payload')}
+ if params['operation'] == 'child.reserve':
+ return rt.apply_worker_receipt(db, **params, handlers={'child.reserve': reserve})
+ return rt.mutate_worker_execution(db, **params)
+ parent = RuntimeSessionStore(rpc, dict(epoch=epoch, execution_id='worker', session_id='parent', generation=0),
+ tmp_path / 'outbox')
+ try:
+ child = reserve_child_store(parent)
+ child.close()
+ with db._read_ctx() as conn:
+ assert conn.execute("SELECT status FROM worker_executions WHERE session_id=?",
+ (child.reserved_session_id,)).fetchone()[0] == 'terminal'
+ assert db.get_session(child.reserved_session_id)['end_reason'] == 'agent_close'
+ finally:
+ parent.close()
diff --git a/tests/tools/test_delegate_batch_validation.py b/tests/tools/test_delegate_batch_validation.py
index 32e03e2c05f..d3d57e61bcf 100644
--- a/tests/tools/test_delegate_batch_validation.py
+++ b/tests/tools/test_delegate_batch_validation.py
@@ -179,3 +179,18 @@ class TestValidBatchStillRuns(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class TestOwnerRefusalMidBatch(unittest.TestCase):
+ def test_siblings_built_before_a_refusal_release_their_reservations(self):
+ """A managed worker's owner can refuse the Nth child (capacity, depth, draining) after
+ earlier siblings already hold reserved executions: those are closed, not leaked."""
+ built = MagicMock()
+ parent = _make_mock_parent()
+ refusal = ValueError("the gateway owner refused another subagent")
+ with patch("tools.delegate_tool._build_child_preserving_parent_tools", side_effect=[built, refusal]), \
+ patch("tools.delegate_tool._run_single_child") as mock_run:
+ result = json.loads(delegate_task(tasks=[{"goal": GOOD_A}, {"goal": GOOD_B}], parent_agent=parent))
+ self.assertIn("refused another subagent", result["error"])
+ mock_run.assert_not_called()
+ built.close.assert_called_once_with()