The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(managed-worker): delegated children build, close and settle off Relay's loop
Teknium1f91fb386a60
diff --git a/agent/runtime_session_child.py b/agent/runtime_session_child.pyindex 31866cc5d4d..68d9dec723b 100644--- a/agent/runtime_session_child.py+++ b/agent/runtime_session_child.py@@ -6,6 +6,7 @@ child persists through the ordinary ``worker.persist`` path under its own claim.(capacity, depth, draining) is a ValueError so ``delegate_task`` reports it as a tool error."""import secrets+from functools import partialfrom agent.runtime_session_store import RuntimeSessionStore, call_off_event_loop@@ -50,4 +51,7 @@ def reserve_child_store(parent):child = result['child']scope = dict(parent.scope, **child, secret=secret)outbox = parent.path.parent / ('child-' + child['execution_id'].rsplit(':', 1)[1])- return ChildSessionStore(parent.rpc, scope, outbox)+ # Every child RPC hops off a running loop, not only this reservation: the child is built, run+ # and closed inside delegate_task's callback, and its constructor already reads the store+ # (ContextCompressor.bind_session_state), which WorkerRPC refuses on Relay's loop.+ return ChildSessionStore(partial(call_off_event_loop, parent.rpc), scope, outbox)diff --git a/gateway/session_worker_child.py b/gateway/session_worker_child.pyindex 24cc9941630..9918bd49dff 100644--- a/gateway/session_worker_child.py+++ b/gateway/session_worker_child.py@@ -11,7 +11,7 @@ import secretsfrom functools import partialfrom hermes_state_runtime import RuntimeStoreError, apply_worker_receipt-from hermes_state_runtime_workers import CHILD_SEPARATOR+from hermes_state_runtime_workers import CHILD_SEPARATOR, descendant_range_DEFAULTS = {'max_concurrent_children': 10, 'max_spawn_depth': 1}@@ -49,9 +49,9 @@ def reserve_child_execution(db, conn, session_id, payload, *, epoch, execution_iraise RuntimeStoreError('invalid_params')max_children, max_depth = limitsprefix = execution_id + CHILD_SEPARATOR- live = conn.execute("SELECT COUNT(*) FROM worker_executions WHERE substr(execution_id,1,?)=? "+ live = conn.execute("SELECT COUNT(*) FROM worker_executions WHERE execution_id>=? AND execution_id<? ""AND instr(substr(execution_id,?),'/')=0 AND status!='terminal'",- (len(prefix), prefix, len(prefix) + 1)).fetchone()[0]+ (*descendant_range(execution_id), len(prefix) + 1)).fetchone()[0]if refused is None and prefix.count(CHILD_SEPARATOR) > max_depth:refused = 'delegation_depth_exceeded'if refused is None and live >= max_children:diff --git a/hermes_state_runtime_workers.py b/hermes_state_runtime_workers.pyindex 118d2158d30..7519ad68575 100644--- a/hermes_state_runtime_workers.py+++ b/hermes_state_runtime_workers.py@@ -54,10 +54,15 @@ def retire_child_executions(conn, execution_id):"""A child never outlives the execution that reserved it: every live descendant goes terminalwith it, its read receipts shrink and its open session row is ended. A late child write isthen refused ``stale_generation`` like any write after retirement."""- prefix = execution_id + CHILD_SEPARATOR_retire_children(conn, conn.execute("SELECT execution_id,session_id FROM worker_executions "- "WHERE substr(execution_id,1,?)=? AND status!='terminal'",- (len(prefix), prefix)).fetchall())+ "WHERE execution_id>=? AND execution_id<? AND status!='terminal'",+ descendant_range(execution_id)).fetchall())+++def descendant_range(execution_id):+ """``(low, high)`` bounds every descendant id of ``execution_id`` on the primary key index+ (';' sorts right after the separator's ':'), so a no-child finish costs one index probe."""+ return execution_id + CHILD_SEPARATOR, execution_id + CHILD_SEPARATOR[:-1] + ';'def retire_stale_children(conn, epoch):@@ -68,12 +73,16 @@ def retire_stale_children(conn, epoch):def _retire_children(conn, rows):- import time- for child, session_id in rows:+ for child, _session_id in rows:conn.execute("UPDATE worker_executions SET status='terminal' WHERE execution_id=?", (child,))_compact(conn, child)- conn.execute("UPDATE sessions SET ended_at=?,end_reason='agent_close' WHERE id=? AND ended_at IS NULL",- (time.time(), session_id))+ _end_child_row(conn, child)+++def _end_child_row(conn, execution_id):+ import time+ conn.execute("UPDATE sessions SET ended_at=?,end_reason='agent_close' WHERE ended_at IS NULL "+ "AND id=(SELECT session_id FROM worker_executions WHERE execution_id=?)", (time.time(), execution_id))def compact_terminal_receipts(conn, execution_id):@@ -86,6 +95,10 @@ def compact_terminal_receipts(conn, execution_id):conflicts. Mutation acknowledgements (counts, row annotations, assignments) stay replayable."""retire_child_executions(conn, execution_id)_compact(conn, execution_id)+ if CHILD_SEPARATOR in execution_id:+ # A child that finished itself (an early close: failed constructor, refused sibling) never+ # reaches _retire_children, which only selects live children: its row ends here.+ _end_child_row(conn, execution_id)def _compact(conn, execution_id):diff --git a/hermes_state_worker_lifecycle.py b/hermes_state_worker_lifecycle.pyindex 86b12b2c24e..420408b8a66 100644--- a/hermes_state_worker_lifecycle.py+++ b/hermes_state_worker_lifecycle.py@@ -31,6 +31,11 @@ def worker_create(db, conn, session_id, payload):if config is not None and (not isinstance(config, dict) or set(config) - _CONSTRUCTOR_CONFIG):raise RuntimeStoreError('invalid_params')insert_session_row_in_transaction(db, conn, session_id, **payload)+ if config:+ # An owner-reserved child row already holds its lineage marker, so the keep-existing upsert+ # would drop these facts; merge them beside it (keys are the constructor set only).+ conn.execute("UPDATE sessions SET model_config=json_patch(model_config,?) WHERE id=? "+ "AND json_type(model_config,'$._delegate_from') IS NOT NULL", (json.dumps(config), session_id))return {'value': session_id}diff --git a/tests/gateway/test_managed_worker_delegation.py b/tests/gateway/test_managed_worker_delegation.pyindex 3897d6c9753..4940d35f11f 100644--- a/tests/gateway/test_managed_worker_delegation.py+++ b/tests/gateway/test_managed_worker_delegation.py@@ -148,3 +148,37 @@ def test_owner_restart_mid_child_retires_the_child_and_never_replays_it(tmp_path"WHERE w.kind='child'") == [('agent_close',)]# The lost turn is not replayed: the parent never got a child result to finish on.assert not [s for s in peer.seen if s[0] == ['DELEGATE_BLOCK'] and s[1] == 'PARENT_DONE'], peer.seen++[email protected]("linux")+def test_relay_driven_worker_builds_and_runs_its_child_off_the_relay_loop(tmp_path):+ """NeMo Relay drives tool callbacks on its own event loop: the child's construction reads the+ store (bind_session_state) and its close finishes the execution, and the worker RPC refuses a+ synchronous call on a running loop, so every child store call must hop off it."""+ relay = pytest.importorskip('nemo_relay')+ if getattr(relay, '_native', None) is None:+ pytest.skip('NeMo Relay native binding unavailable')+ home, peer, url, env = _setup(tmp_path)+ plugins = home / 'plugins.toml'+ plugins.write_text('version = 1\n[[components]]\nkind = "observability"\nenabled = true\n'+ '[components.config]\nversion = 4\n[components.config.atof]\nenabled = true\n'+ '[[components.config.atof.sinks]]\ntype = "file"\noutput_directory = '+ + json.dumps(str(home / 'relay-events')) + '\nfilename = "events.jsonl"\nmode = "overwrite"\n')+ env['HERMES_NEMO_RELAY_PLUGINS_TOML'] = str(plugins)++ async def exercise(desc):+ sid = await _submit(home, desc, url, 'DELEGATE')+ await async_wait_until(lambda: _rows(home, "SELECT status FROM session_admissions WHERE request_id='run'") == [+ ('terminal',)], timeout=120, interval=.1, what='run terminal')+ return sid+ try:+ with daemon(ROOT, home, env, barrier=False) as (_owner, desc):+ sid = asyncio.run(exercise(desc))+ finally:+ peer.shutdown()+ peer.server_close()+ assert (home / 'relay-events' / 'events.jsonl').exists(), 'Relay never drove this turn'+ tool = [c for r, c in _rows(home, 'SELECT role, content FROM messages WHERE session_id=?', sid) if r == 'tool']+ assert tool and 'CHILD_RESULT_42' in tool[-1], tool+ assert _rows(home, "SELECT w.status, s.end_reason FROM worker_executions w JOIN sessions s ON s.id=w.session_id "+ "WHERE w.kind='child'") == [('terminal', 'agent_close')]diff --git a/tests/hermes_state/test_runtime_worker_children.py b/tests/hermes_state/test_runtime_worker_children.pyindex a06d350110c..c44bf421350 100644--- a/tests/hermes_state/test_runtime_worker_children.py+++ b/tests/hermes_state/test_runtime_worker_children.py@@ -29,7 +29,7 @@ def _child_append(db, epoch, child, sequence, content, session_id=None):@pytest.fixturedef ledger(tmp_path):db = SessionDB(db_path=tmp_path / 'state.db')- db.create_session('parent', source='kanban')+ db.create_session('parent', source='kanban', profile_name='coder')db.create_session('other', source='cli')epoch = rt.begin_runtime_epoch(db, instance_id='boot')rt.register_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,@@ -85,3 +85,103 @@ def test_owner_restart_retires_in_flight_children_instead_of_recovering_them(ledassert db.get_session(child_sid)['end_reason'] == 'agent_close'with pytest.raises(rt.RuntimeStoreError, match='stale_generation|stale_epoch'):_child_append(db, replacement, {'execution_id': child_eid, 'session_id': child_sid}, 1, 'late')+++def test_depth_cap_profile_and_an_early_child_finish_settle_the_child_row(ledger):+ db, epoch = ledger+ child = _reserve(db, epoch, 1)['child']+ assert db.get_session(child['session_id'])['profile_name'] == 'coder'+ # The child is already at max_spawn_depth=1: the owner refuses its own child.request.+ nested = partial(reserve_child_execution, epoch=epoch, execution_id=child['execution_id'],+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},+ principal='local', limits=(2, 1), refused=None)+ assert rt.apply_worker_receipt(db, epoch=epoch, execution_id=child['execution_id'], session_id=child['session_id'],+ generation=0, sequence=1, operation='child.reserve',+ payload={'secret': 'nested-secret-00000001'},+ handlers={'child.reserve': nested}) == {'refused': 'delegation_depth_exceeded'}+ # The child's constructor facts land beside the owner's lineage marker.+ rt.mutate_worker_execution(db, epoch=epoch, execution_id=child['execution_id'], session_id=child['session_id'],+ generation=0, sequence=2, operation='session.create',+ payload={'source': 'subagent', 'model_config': {'max_iterations': 7}})+ import json+ assert json.loads(db.get_session(child['session_id'])['model_config']) == {+ '_delegate_from': 'parent', 'max_iterations': 7}+ # An early close finishes the child itself (failed constructor, refused sibling): its row ends+ # then, while the parent is still live, since parent retirement only selects live children.+ rt.mutate_worker_execution(db, epoch=epoch, execution_id=child['execution_id'], session_id=child['session_id'],+ generation=0, sequence=3, operation='execution.finish', payload={})+ assert db.get_session(child['session_id'])['end_reason'] == 'agent_close'+ assert db.get_session('parent')['ended_at'] is None+++async def test_child_reserve_verifies_the_parent_claim_and_refuses_while_draining(tmp_path):+ """The request path, not just the handler: a forged parent secret never reaches the ledger,+ and a draining owner answers with a refusal receipt the parent's journal can move past."""+ import subprocess+ import sys+ from types import SimpleNamespace+ from gateway.session_worker import worker_request+ db = SessionDB(db_path=tmp_path / 'state.db')+ db.create_session('parent', source='kanban')+ process = subprocess.Popen([sys.executable, '-c', 'import sys; sys.stdin.read()'], stdin=subprocess.PIPE) # noqa: ASYNC220 -- returns at once; the test only needs a live pid+ draining = []++ def admission_open():+ if draining:+ raise rt.RuntimeStoreError('runtime_draining')+ authority = SimpleNamespace(db=db, epoch=rt.begin_runtime_epoch(db, instance_id='owner'), profile_id='p',+ _require_admission_open=admission_open, wake_after_worker=lambda session_id: None,+ authorize=lambda actor, ref, capability: None,+ sessions={}, logical_owner=lambda sid: sid)+ connection = SimpleNamespace(authority=authority, actor=SimpleNamespace(+ subject='human', profile_id='p', capabilities={'session:submit', 'worker:adopt'}))+ scope = {'profile_id': 'p', 'session_id': 'parent', 'execution_id': 'compute', 'generation': 0,+ 'pid': process.pid, 'birth': psutil.Process(process.pid).create_time(), 'secret': 'w' * 32}+ try:+ epoch = (await worker_request(connection, None, scope | {'kind': 'compute'}, operation='register'))['owner_epoch']++ def reserve(sequence, **override):+ return worker_request(connection, None, scope | override | {+ 'epoch': epoch, 'sequence': sequence, 'operation': 'child.reserve',+ 'payload': {'secret': f'child-secret-{sequence:08d}'}}, operation='persist')+ with pytest.raises(rt.RuntimeStoreError, match='permission_denied'):+ await reserve(1, secret='f' * 32)+ draining.append(True)+ assert await reserve(1) == {'refused': 'runtime_draining'}+ draining.clear()+ assert 'child' in await reserve(2)+ with db._read_ctx() as conn:+ assert conn.execute("SELECT COUNT(*) FROM worker_executions WHERE kind='child'").fetchone()[0] == 1+ finally:+ process.stdin.close()+ process.wait(timeout=5)+ db.close()+++def test_a_child_store_closed_before_any_turn_finishes_its_execution_and_ends_its_row(ledger, tmp_path):+ """The constructor-failure / refused-sibling path: the child's store closes having written+ nothing, and that close alone settles the reservation (the parent is still live)."""+ from agent.runtime_session_child import reserve_child_store+ from agent.runtime_session_store import RuntimeSessionStore+ db, epoch = ledger+ reserve = partial(reserve_child_execution, epoch=epoch, execution_id='worker',+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},+ principal='local', limits=(2, 1), refused=None)++ def rpc(method, **params):+ params = {k: params[k] for k in ('epoch', 'execution_id', 'session_id', 'generation', 'sequence', 'operation', 'payload')}+ if params['operation'] == 'child.reserve':+ return rt.apply_worker_receipt(db, **params, handlers={'child.reserve': reserve})+ return rt.mutate_worker_execution(db, **params)+ parent = RuntimeSessionStore(rpc, dict(epoch=epoch, execution_id='worker', session_id='parent', generation=0),+ tmp_path / 'outbox')+ try:+ child = reserve_child_store(parent)+ child.close()+ with db._read_ctx() as conn:+ assert conn.execute("SELECT status FROM worker_executions WHERE session_id=?",+ (child.reserved_session_id,)).fetchone()[0] == 'terminal'+ assert db.get_session(child.reserved_session_id)['end_reason'] == 'agent_close'+ finally:+ parent.close()diff --git a/tests/tools/test_delegate_batch_validation.py b/tests/tools/test_delegate_batch_validation.pyindex 32e03e2c05f..d3d57e61bcf 100644--- a/tests/tools/test_delegate_batch_validation.py+++ b/tests/tools/test_delegate_batch_validation.py@@ -179,3 +179,18 @@ class TestValidBatchStillRuns(unittest.TestCase):if __name__ == "__main__":unittest.main()+++class TestOwnerRefusalMidBatch(unittest.TestCase):+ def test_siblings_built_before_a_refusal_release_their_reservations(self):+ """A managed worker's owner can refuse the Nth child (capacity, depth, draining) after+ earlier siblings already hold reserved executions: those are closed, not leaked."""+ built = MagicMock()+ parent = _make_mock_parent()+ refusal = ValueError("the gateway owner refused another subagent")+ with patch("tools.delegate_tool._build_child_preserving_parent_tools", side_effect=[built, refusal]), \+ patch("tools.delegate_tool._run_single_child") as mock_run:+ result = json.loads(delegate_task(tasks=[{"goal": GOOD_A}, {"goal": GOOD_B}], parent_agent=parent))+ self.assertIn("refused another subagent", result["error"])+ mock_run.assert_not_called()+ built.close.assert_called_once_with()