The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(managed-worker): delegate_task works in Kanban workers through owner-reserved children
Tekniumbf2b3713ab28
diff --git a/agent/managed_worker.py b/agent/managed_worker.pyindex 5939edb5762..a41728cce01 100644--- a/agent/managed_worker.py+++ b/agent/managed_worker.py@@ -24,9 +24,6 @@ import threadinglogger = logging.getLogger(__name__)MAX_FRAME = 4 * 1024 * 1024-# Toolsets a managed worker turn never offers (subtracted last, so the tool_search bridge and a-# restored tools[] pin drop them too): delegate_task's child has no owner registration here.-WORKER_DISABLED_TOOLSETS = ('delegation',)BOOTSTRAP_FIELDS = {'version', 'home', 'scope', 'policy', 'api_key', 'text', 'route', 'user_id', 'chat_id','turn_author', 'safe_mode', 'ignore_user_config'}# The bounded run_conversation fields a turn receipt keeps (outcome, exit reason, tokens, cost,@@ -360,10 +357,9 @@ def execute(frame, channel):with policy_scope(policy):agent = _construct_agent(frame, AIAgent, policy, model=policy.model, provider=policy.provider, base_url=policy.base_url,api_key=frame['api_key'], session_db=store, session_id=scope['session_id'],+ # delegate_task stays offered: each child is reserved by the owner under this+ # execution (agent/runtime_session_child.py) and retires with it.enabled_toolsets=list(policy.toolsets), max_iterations=policy.max_turns,- # A delegated child needs its own owner-registered execution, which this single-turn- # worker cannot reserve; never offer a tool whose only outcome here is a refusal.- disabled_toolsets=list(WORKER_DISABLED_TOOLSETS),reasoning_config=policy.reasoning_config,# ``tool`` / ``oneshot`` are storage labels for a CLI-surface agent (session_policy.SURFACES).platform=policy.platform if policy.source in ('tool', 'oneshot') else policy.source,diff --git a/agent/runtime_session_child.py b/agent/runtime_session_child.pynew file mode 100644index 00000000000..31866cc5d4d--- /dev/null+++ b/agent/runtime_session_child.py@@ -0,0 +1,53 @@+"""A delegated child's store inside a worker: owner-reserved, never self-registered (#141793).++The parent's ``RuntimeSessionStore`` asks the owner for a child on its own receipt stream; the+owner mints the child's session row and execution (gateway/session_worker_child.py) and the+child persists through the ordinary ``worker.persist`` path under its own claim. A refusal+(capacity, depth, draining) is a ValueError so ``delegate_task`` reports it as a tool error.+"""+import secrets++from agent.runtime_session_store import RuntimeSessionStore, call_off_event_loop++_REFUSALS = {+ 'delegation_capacity': 'the gateway owner refused another subagent: this execution already runs '+ 'delegation.max_concurrent_children of them',+ 'delegation_depth_exceeded': 'the gateway owner refused a subagent this deep (delegation.max_spawn_depth)',+}+++class ChildSessionStore(RuntimeSessionStore):+ binds_delegation_ledger = False++ @property+ def reserved_session_id(self):+ return self.scope['session_id']++ def create_session(self, session_id, source, **fields):+ # The owner created this row at reservation with its source, lineage and profile; only+ # the constructor facts are the child's to add.+ from hermes_state_worker_lifecycle import _CONSTRUCTOR_CONFIG+ config = {k: v for k, v in (fields.get('model_config') or {}).items() if k in _CONSTRUCTOR_CONFIG}+ return super().create_session(session_id, 'subagent', model=fields.get('model'),+ model_config=config or None, system_prompt=fields.get('system_prompt'))++ def close(self):+ # The child's agent closes once, after its last write: that is this execution's end.+ try:+ if not self._outbox_owner.closed and not self.failure:+ self.finish()+ finally:+ super().close()+++def reserve_child_store(parent):+ """A ``ChildSessionStore`` the owner reserved under ``parent``'s verified execution."""+ secret = secrets.token_urlsafe(32)+ # Reached from delegate_task's tool callback, which can run on Relay's loop (#139029).+ result = call_off_event_loop(parent._apply, 'child.reserve', {'secret': secret})+ if 'refused' in result:+ raise ValueError(_REFUSALS.get(result['refused'], 'the gateway owner refused a subagent: ' + result['refused']))+ child = result['child']+ scope = dict(parent.scope, **child, secret=secret)+ outbox = parent.path.parent / ('child-' + child['execution_id'].rsplit(':', 1)[1])+ return ChildSessionStore(parent.rpc, scope, outbox)diff --git a/agent/runtime_session_store.py b/agent/runtime_session_store.pyindex a187aa51ce0..3393da5fce7 100644--- a/agent/runtime_session_store.py+++ b/agent/runtime_session_store.py@@ -220,6 +220,9 @@ class RuntimeSessionStore(RuntimeSessionCompressionMixin, RuntimeSessionLifecycladoption. Local queue acceptance is never returned as canonical commit.Pending journals live outside age-pruned cache trees."""+ # The process's async-delegation ledger belongs to the adopted execution, never a reserved child.+ binds_delegation_ledger = True+def __init__(self, rpc, scope, outbox_dir, adopted=None, *, max_bytes=4 * 1024 * 1024):self.rpc = rpcself.scope = dict(scope)@@ -258,7 +261,7 @@ class RuntimeSessionStore(RuntimeSessionCompressionMixin, RuntimeSessionLifecycl# Replay must land the journal exactly on the adopted assignment (a pending# publication moves it to the child); anything else is not this successor's outbox.raise WorkerPersistenceError('outbox_scope_mismatch')- if is_worker_process():+ if is_worker_process() and self.binds_delegation_ledger:from tools.async_delegation_worker import bind_worker_delegation_storebind_worker_delegation_store(self)except Exception:diff --git a/gateway/session_worker.py b/gateway/session_worker.pyindex 914280114bc..aaa3a6bac27 100644--- a/gateway/session_worker.py+++ b/gateway/session_worker.py@@ -1,7 +1,7 @@"""Execution-scoped worker operations on the ordinary authenticated authority.-Only explicit local compute registrations are enabled. Cron/child claims need-producer-specific validation before their consumers can use this transport.+Only explicit local compute registrations are enabled here. A delegated child is reserved by+the owner on its parent's receipt stream (gateway/session_worker_child.py), never registered."""import asyncioimport hmac@@ -68,6 +68,9 @@ async def worker_request(connection, ref, params, *, operation):if operation == "persist" and isinstance(params.get("operation"), str) and params["operation"].startswith("delegation."):from gateway.session_worker_delegation import worker_delegation_requestreturn await worker_delegation_request(connection, ref, params)+ if operation == "persist" and params.get("operation") == "child.reserve":+ from gateway.session_worker_child import worker_child_request+ return await worker_child_request(connection, ref, params)extra = {'register': {'kind'}, 'adopt': set(), 'persist': {'epoch', 'sequence', 'operation', 'payload'}}[operation]if set(params) != _SCOPE | extra:raise RuntimeStoreError('invalid_params')diff --git a/gateway/session_worker_child.py b/gateway/session_worker_child.pynew file mode 100644index 00000000000..24cc9941630--- /dev/null+++ b/gateway/session_worker_child.py@@ -0,0 +1,97 @@+"""Owner-reserved delegated children of a verified worker execution (#141793).++A worker never mints a child. It asks on its OWN fenced receipt stream (epoch, assignment,+generation, sequence), and the owner creates the child's session row and execution, binds it to+the parent's verified process identity and the worker's fresh secret, and enforces the frozen+policy's delegation caps. The child then persists through the ordinary ``worker.persist`` path+under its own claim. Expected refusals (caps, draining) are receipts, not errors: an error would+leave the parent's journal pending and refuse every later write of its turn.+"""+import secrets+from functools import partial++from hermes_state_runtime import RuntimeStoreError, apply_worker_receipt+from hermes_state_runtime_workers import CHILD_SEPARATOR++_DEFAULTS = {'max_concurrent_children': 10, 'max_spawn_depth': 1}+++def _limit(section, key):+ try:+ return max(1, int(section.get(key, _DEFAULTS[key])))+ except (TypeError, ValueError):+ return _DEFAULTS[key]+++def delegation_limits(authority, execution_id, session_id):+ """``(max_concurrent_children, max_spawn_depth)`` from the root conversation's frozen policy+ (the profile's config for a session with no local policy), never from the asking worker."""+ from gateway.session_policy import policy_for_source, profile_config+ root = execution_id.split(CHILD_SEPARATOR, 1)[0]+ if root != execution_id:+ with authority.db._read_ctx() as conn:+ row = conn.execute('SELECT session_id FROM worker_executions WHERE execution_id=?', (root,)).fetchone()+ session_id = row[0] if row else session_id+ live = authority.sessions.get(authority.logical_owner(session_id))+ policy = policy_for_source(authority.runner, live.source) if live is not None else None+ config = policy.config(authority) if policy is not None else profile_config(authority)+ section = config.get('delegation') or {}+ return _limit(section, 'max_concurrent_children'), _limit(section, 'max_spawn_depth')+++def reserve_child_execution(db, conn, session_id, payload, *, epoch, execution_id, identity, principal, limits, refused):+ from hermes_state_ids import new_session_id+ from hermes_state_keys import admission_fingerprint+ from hermes_state_runtime import _secret_digest+ from hermes_state_worker_lifecycle import insert_session_row_in_transaction+ secret = payload.get('secret') if isinstance(payload, dict) else None+ if set(payload) != {'secret'} or not isinstance(secret, str) or not 16 <= len(secret) <= 256:+ raise RuntimeStoreError('invalid_params')+ max_children, max_depth = limits+ prefix = execution_id + CHILD_SEPARATOR+ live = conn.execute("SELECT COUNT(*) FROM worker_executions WHERE substr(execution_id,1,?)=? "+ "AND instr(substr(execution_id,?),'/')=0 AND status!='terminal'",+ (len(prefix), prefix, len(prefix) + 1)).fetchone()[0]+ if refused is None and prefix.count(CHILD_SEPARATOR) > max_depth:+ refused = 'delegation_depth_exceeded'+ if refused is None and live >= max_children:+ refused = 'delegation_capacity'+ if refused is not None:+ return {'refused': refused}+ parent = conn.execute('SELECT profile_name FROM sessions WHERE id=?', (session_id,)).fetchone()+ child_sid, child_eid = new_session_id(), prefix + secrets.token_hex(8)+ insert_session_row_in_transaction(db, conn, child_sid, 'subagent', model_config={'_delegate_from': session_id},+ parent_session_id=session_id, profile_name=parent['profile_name'])+ scope = dict(identity, session_id=child_sid, execution_id=child_eid, generation=0, secret=secret)+ claim = admission_fingerprint(canonical_target=child_sid, payload=scope | {'principal': principal})+ conn.execute("INSERT INTO worker_executions(execution_id,session_id,kind,owner_epoch,generation,status,adoption_digest) "+ "VALUES(?,?,'child',?,0,'running',?)", (child_eid, child_sid, epoch, _secret_digest(claim)))+ return {'child': {'session_id': child_sid, 'execution_id': child_eid, 'generation': 0}}+++async def worker_child_request(connection, ref, params):+ """worker.persist ``child.reserve``: the parent's full claim and assignment are verified first."""+ import asyncio+ from gateway.session_worker import _SCOPE, _claim, _verify+ if set(params) != _SCOPE | {'epoch', 'sequence', 'operation', 'payload'}:+ raise RuntimeStoreError('invalid_params')+ claim = _claim(connection, ref, params)+ if 'worker:adopt' not in connection.actor.capabilities:+ raise RuntimeStoreError('permission_denied')+ _verify(connection, ref, params, claim)+ authority = connection.authority++ def reserve():+ refused, limits = None, (0, 0)+ try:+ authority._require_admission_open()+ limits = delegation_limits(authority, params['execution_id'], params['session_id'])+ except RuntimeStoreError as exc:+ refused = exc.reason+ handler = partial(reserve_child_execution, epoch=params['epoch'], execution_id=params['execution_id'],+ identity={key: params[key] for key in ('profile_id', 'pid', 'birth')},+ principal=connection.actor.subject, limits=limits, refused=refused)+ return apply_worker_receipt(authority.db, **{key: params[key] for key in (+ 'epoch', 'execution_id', 'session_id', 'generation', 'sequence', 'operation', 'payload')},+ handlers={'child.reserve': handler})+ return await asyncio.to_thread(reserve)diff --git a/hermes_state_runtime.py b/hermes_state_runtime.pyindex 838e6fc2971..60aaf594952 100644--- a/hermes_state_runtime.py+++ b/hermes_state_runtime.py@@ -448,6 +448,8 @@ def recover_session_inputs(db, *, epoch: int) -> int:"""Never replay started work. Live worker adoption is a separate explicit operation."""def write(conn):_epoch(conn, epoch)+ from hermes_state_runtime_workers import retire_stale_children+ retire_stale_children(conn, epoch)conn.execute("UPDATE worker_executions SET status='unknown' WHERE status IN ('registered','running') AND owner_epoch!=?", (epoch,))# A steer handed to a lost claim may or may not have reached its agent: never replayed.conn.execute("UPDATE session_admissions SET outcome='interrupted' WHERE status='terminal' AND outcome=? "diff --git a/hermes_state_runtime_workers.py b/hermes_state_runtime_workers.pyindex a7d9a90e881..118d2158d30 100644--- a/hermes_state_runtime_workers.py+++ b/hermes_state_runtime_workers.py@@ -45,15 +45,50 @@ def discard_orphan_workers_on_reset(conn, session_ids):RETIRED_RECEIPT = {'retired': True}+# A delegated child's execution id is its parent's plus this and a random suffix, so every+# descendant of an execution is one prefix probe away (gateway/session_worker_child.py).+CHILD_SEPARATOR = '/child:'+++def retire_child_executions(conn, execution_id):+ """A child never outlives the execution that reserved it: every live descendant goes terminal+ with it, its read receipts shrink and its open session row is ended. A late child write is+ then refused ``stale_generation`` like any write after retirement."""+ prefix = execution_id + CHILD_SEPARATOR+ _retire_children(conn, conn.execute("SELECT execution_id,session_id FROM worker_executions "+ "WHERE substr(execution_id,1,?)=? AND status!='terminal'",+ (len(prefix), prefix)).fetchall())+++def retire_stale_children(conn, epoch):+ """Owner restart: a child dies with its worker, which dies with its owner, so it is never+ adoptable; only its root execution can be recovered as ``unknown``."""+ _retire_children(conn, conn.execute("SELECT execution_id,session_id FROM worker_executions WHERE kind='child' "+ "AND status!='terminal' AND owner_epoch!=?", (epoch,)).fetchall())+++def _retire_children(conn, rows):+ import time+ for child, session_id in rows:+ conn.execute("UPDATE worker_executions SET status='terminal' WHERE execution_id=?", (child,))+ _compact(conn, child)+ conn.execute("UPDATE sessions SET ended_at=?,end_reason='agent_close' WHERE id=? AND ended_at IS NULL",+ (time.time(), session_id))def compact_terminal_receipts(conn, execution_id):- """Once an execution is terminal its read receipts are dead weight: a ``compression.history``+ """Every terminal transition calls this, so it also retires the execution's delegated children.+ Once an execution is terminal its read receipts are dead weight: a ``compression.history``read stores the whole transcript and a context read a full session row, so keeping them leavesone full-history copy per managed turn and ``state.db`` grows quadratically. They shrink to amarker in the closing transaction (an exact late replay is refused ``stale_generation``, asafter retirement); ``payload_digest`` stays, so a late duplicate with other content stillconflicts. Mutation acknowledgements (counts, row annotations, assignments) stay replayable."""+ retire_child_executions(conn, execution_id)+ _compact(conn, execution_id)+++def _compact(conn, execution_id):conn.execute("UPDATE worker_receipts SET result_json=? WHERE execution_id=? AND json_valid(result_json) ""AND (json_type(result_json, '$.messages') IS NOT NULL ""OR json_type(result_json, '$.session') IS NOT NULL)",diff --git a/tests/gateway/fixtures/agent_persistence_worker.py b/tests/gateway/fixtures/agent_persistence_worker.pyindex 8f0f1ab34aa..e60467fdaef 100644--- a/tests/gateway/fixtures/agent_persistence_worker.py+++ b/tests/gateway/fixtures/agent_persistence_worker.py@@ -60,7 +60,7 @@ def main():opened = fn()if opened is not None:release_or_close(opened)- return name, 'unexpected_success'+ return name, 'opened:' + type(opened).__name__except Exception as exc:return name, str(exc)with ThreadPoolExecutor(max_workers=1) as pool:diff --git a/tests/gateway/test_managed_worker_delegation.py b/tests/gateway/test_managed_worker_delegation.pynew file mode 100644index 00000000000..3897d6c9753--- /dev/null+++ b/tests/gateway/test_managed_worker_delegation.py@@ -0,0 +1,150 @@+"""delegate_task inside a managed worker (#141793): real daemon, real worker, loopback model.++Every child is reserved by the owner under the worker's execution, persists through its own+claim, returns its result to the parent's turn, and dies with the owner on a restart."""+import asyncio+from contextlib import closing+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+import json+import os+from pathlib import Path+import sqlite3+import threading++import pytest++from tests.gateway.fixtures.local_recovery_probe import daemon, rpc, websocket+from tests.waiting import async_wait_until++ROOT = Path(__file__).resolve().parents[2]+++class Model(BaseHTTPRequestHandler):+ """The parent delegates once; the child answers (or blocks on BLOCK); the parent then finishes."""+ def log_message(self, *args):+ pass++ def do_POST(self):+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))+ messages = body.get('messages') or [{'role': 'system', 'content': ''}]+ users = [m.get('content') for m in messages if m['role'] == 'user']+ text = json.dumps(messages)+ name = None+ if users and users[0] in ('DELEGATE', 'DELEGATE_BLOCK') and messages[-1]['role'] == 'user':+ goal = 'CHILD_GOAL_BLOCK' if users[0] == 'DELEGATE_BLOCK' else 'CHILD_GOAL'+ name, args = 'delegate_task', {'goal': goal}+ elif 'CHILD_GOAL' in text and not any(u in ('DELEGATE', 'DELEGATE_BLOCK') for u in users):+ if 'CHILD_GOAL_BLOCK' in text:+ self.server.blocked.set()+ self.server.release.wait(60)+ message = {'role': 'assistant', 'content': 'CHILD_RESULT_42'}+ else:+ message = {'role': 'assistant', 'content': 'PARENT_DONE'}+ if name:+ message = {'role': 'assistant', 'content': None, 'tool_calls': [{'id': 'd1', 'index': 0, 'type': 'function',+ 'function': {'name': name, 'arguments': json.dumps(args)}}]}+ self.server.seen.append((users[:1], message.get('content'), [t['function']['name'] for t in body.get('tools') or []]))+ choice = {'index': 0, 'delta': message, 'finish_reason': 'tool_calls' if name else 'stop'}+ frame = {'id': 'm', 'model': 'm', 'choices': [choice], 'usage': {'prompt_tokens': 3, 'completion_tokens': 2, 'total_tokens': 5}}+ if body.get('stream'):+ payload, kind = ('data: ' + json.dumps(frame) + '\n\ndata: [DONE]\n\n').encode(), 'text/event-stream'+ else:+ choice['message'] = choice.pop('delta')+ payload, kind = json.dumps(frame).encode(), 'application/json'+ try:+ self.send_response(200)+ self.send_header('Content-Type', kind)+ self.send_header('Content-Length', str(len(payload)))+ self.end_headers()+ self.wfile.write(payload)+ except (BrokenPipeError, ConnectionResetError):+ pass+++def _setup(tmp_path):+ home, user = tmp_path / 'state', tmp_path / 'user'+ home.mkdir(mode=0o700)+ user.mkdir()+ peer = ThreadingHTTPServer(('127.0.0.1', 0), Model)+ peer.seen, peer.blocked, peer.release = [], threading.Event(), threading.Event()+ threading.Thread(target=peer.serve_forever, daemon=True).start()+ url = f'http://127.0.0.1:{peer.server_port}/v1'+ (home / 'config.yaml').write_text(json.dumps({+ 'gateway': {'multiplex_profiles': False, 'managed_workers': True},+ 'model': {'provider': 'custom', 'default': 'm', 'base_url': url},+ 'auxiliary': {'title_generation': {'enabled': False}},+ 'platform_toolsets': {'cli': ['delegation', 'todo']}}))+ env = {k: os.environ[k] for k in ('PATH', 'LANG', 'TZ') if k in os.environ}+ env.update(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home), PYTHONPATH=str(ROOT),+ OPENAI_API_KEY='loopback-only', OPENAI_BASE_URL=url)+ return home, peer, url, env+++def _rows(home, sql, *args):+ with closing(sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True)) as db:+ return db.execute(sql, args).fetchall()+++async def _submit(home, desc, url, text):+ async with websocket(home, desc) as ws:+ created = await rpc(ws, 'session.create', request_id='parent', source='cli', cwd=str(home), model='m',+ provider='custom', base_url=url, api_key='loopback-only',+ toolsets=['delegation', 'todo'], ignore_rules=True)+ sid = created['result']['session_id']+ assert 'result' in await rpc(ws, 'prompt.submit', session_id=sid, input_id='run', text=text)+ return sid++[email protected]("linux")+def test_managed_worker_delegates_through_an_owner_reserved_child(tmp_path):+ home, peer, url, env = _setup(tmp_path)++ async def exercise(desc):+ sid = await _submit(home, desc, url, 'DELEGATE')+ await async_wait_until(lambda: _rows(home, "SELECT status FROM session_admissions WHERE request_id='run'") == [+ ('terminal',)], timeout=120, interval=.1, what='run terminal')+ async with websocket(home, desc) as ws:+ return sid, (await rpc(ws, 'session.resume', session_id=sid))['result']['messages']+ try:+ with daemon(ROOT, home, env, barrier=False) as (_owner, desc):+ sid, messages = asyncio.run(exercise(desc))+ finally:+ peer.release.set()+ peer.shutdown()+ peer.server_close()+ assert any('delegate_task' in tools for _, _, tools in peer.seen), peer.seen+ tool = [m['content'] for m in messages if m['role'] == 'tool'][-1]+ assert 'CHILD_RESULT_42' in tool, tool+ assert messages[-1]['content'] == 'PARENT_DONE', messages+ children = _rows(home, "SELECT w.status, s.source, s.parent_session_id, s.end_reason, s.input_tokens FROM worker_executions w "+ "JOIN sessions s ON s.id=w.session_id WHERE w.kind='child'")+ assert children == [('terminal', 'subagent', sid, 'agent_close', 3)], children+ child_sid = _rows(home, "SELECT session_id FROM worker_executions WHERE kind='child'")[0][0]+ assert ('assistant', 'CHILD_RESULT_42') in _rows(home, 'SELECT role, content FROM messages WHERE session_id=?', child_sid)++[email protected]("linux")+def test_owner_restart_mid_child_retires_the_child_and_never_replays_it(tmp_path):+ home, peer, url, env = _setup(tmp_path)++ async def first(desc):+ await _submit(home, desc, url, 'DELEGATE_BLOCK')+ assert await asyncio.to_thread(peer.blocked.wait, 90), peer.seen+ assert _rows(home, "SELECT status FROM worker_executions WHERE kind='child'") == [('running',)]+ try:+ with daemon(ROOT, home, env, barrier=False) as (owner, desc):+ asyncio.run(first(desc))+ owner.kill()+ owner.wait(timeout=10)+ peer.release.set()+ with daemon(ROOT, home, env, barrier=False):+ states = dict(_rows(home, 'SELECT kind, status FROM worker_executions'))+ finally:+ peer.release.set()+ peer.shutdown()+ peer.server_close()+ assert states == {'compute': 'unknown', 'child': 'terminal'}, states+ assert _rows(home, "SELECT s.end_reason FROM worker_executions w JOIN sessions s ON s.id=w.session_id "+ "WHERE w.kind='child'") == [('agent_close',)]+ # The lost turn is not replayed: the parent never got a child result to finish on.+ assert not [s for s in peer.seen if s[0] == ['DELEGATE_BLOCK'] and s[1] == 'PARENT_DONE'], peer.seendiff --git a/tests/gateway/test_managed_worker_no_delegation.py b/tests/gateway/test_managed_worker_no_delegation.pydeleted file mode 100644index ae5bd50f188..00000000000--- a/tests/gateway/test_managed_worker_no_delegation.py+++ /dev/null@@ -1,83 +0,0 @@-"""A managed worker turn never offers ``delegate_task``: its child would need an owner-registered-execution this single-turn worker cannot reserve, so the tool's only outcome would be a refusal-(real daemon, real managed worker, loopback model)."""-import asyncio-from contextlib import closing-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer-import json-import os-from pathlib import Path-import sqlite3-import threading--import pytest--from tests.gateway.fixtures.local_recovery_probe import daemon, rpc, websocket-from tests.waiting import async_wait_until---class Model(BaseHTTPRequestHandler):- def log_message(self, *args):- pass-- def do_POST(self):- body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))- if body.get('messages'):- self.server.offered.append(json.dumps(body.get('tools') or []))- message = {'role': 'assistant', 'content': 'WORKER_DONE'}- frame = {'id': 'tools', 'model': 'm', 'choices': [{'index': 0, 'delta': message, 'finish_reason': 'stop'}],- 'usage': {'prompt_tokens': 1, 'completion_tokens': 1, 'total_tokens': 2}}- if body.get('stream'):- payload, kind = ('data: ' + json.dumps(frame) + '\n\ndata: [DONE]\n\n').encode(), 'text/event-stream'- else:- frame['choices'][0]['message'] = frame['choices'][0].pop('delta')- payload, kind = json.dumps(frame).encode(), 'application/json'- self.send_response(200)- self.send_header('Content-Type', kind)- self.send_header('Content-Length', str(len(payload)))- self.end_headers()- self.wfile.write(payload)--[email protected]("linux")-def test_managed_worker_turn_does_not_offer_delegate_task(tmp_path):- root = Path(__file__).resolve().parents[2]- home, user = tmp_path / 'state', tmp_path / 'user'- home.mkdir(mode=0o700)- user.mkdir()- peer = ThreadingHTTPServer(('127.0.0.1', 0), Model)- peer.offered = []- threading.Thread(target=peer.serve_forever, daemon=True).start()- url = f'http://127.0.0.1:{peer.server_port}/v1'- (home / 'config.yaml').write_text(json.dumps({- 'gateway': {'multiplex_profiles': False, 'managed_workers': True},- 'model': {'provider': 'custom', 'default': 'm', 'base_url': url},- 'auxiliary': {'title_generation': {'enabled': False}},- 'platform_toolsets': {'cli': ['delegation', 'todo']}}))- env = {k: os.environ[k] for k in ('PATH', 'LANG', 'TZ') if k in os.environ}- env.update(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home), PYTHONPATH=str(root),- OPENAI_API_KEY='loopback-only', OPENAI_BASE_URL=url)-- def rows(sql):- with closing(sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True)) as db:- return db.execute(sql).fetchall()-- async def exercise(desc):- async with websocket(home, desc) as ws:- session = await rpc(ws, 'session.create', request_id='tools', source='cli', cwd=str(home), model='m',- provider='custom', base_url=url, api_key='loopback-only',- toolsets=['delegation', 'todo'], ignore_rules=True)- sid = session['result']['session_id']- submitted = await rpc(ws, 'prompt.submit', session_id=sid, input_id='run', text='hello')- assert 'result' in submitted, submitted- await async_wait_until(lambda: rows("SELECT status FROM session_admissions WHERE request_id='run'") == [- ('terminal',)], timeout=90, interval=.1, what='run terminal')- try:- with daemon(root, home, env, barrier=False) as (_owner, desc):- asyncio.run(exercise(desc))- finally:- peer.shutdown()- peer.server_close()- assert peer.offered, 'the managed worker never reached the model'- assert all('todo_list' in tools for tools in peer.offered), peer.offered # the turn kept its other tools- assert not any('delegate_task' in tools for tools in peer.offered), peer.offereddiff --git a/tests/gateway/test_worker_agent_execution.py b/tests/gateway/test_worker_agent_execution.pyindex ce971664f30..b0d40e6dbca 100644--- a/tests/gateway/test_worker_agent_execution.py+++ b/tests/gateway/test_worker_agent_execution.py@@ -78,7 +78,8 @@ def test_real_agent_worker_persists_context_usage_and_releases_lease(tmp_path, passert not proof['failed'], proofassert 'RECOVERY_ACK_WORKER_INFERENCE' in proof['result'], proofif probe_constructors:- assert proof['refusals']['child'] == 'worker_child_registration_required', proof['refusals']+ # A child is reserved by the owner under this execution, never opened on SQLite (#141793).+ assert proof['refusals']['child'] == 'opened:ChildSessionStore', proof['refusals']assert proof['refusals']['cron'] == 'worker_cron_registration_required', proof['refusals']assert proof['refusals']['ledger'] == 'worker_delegation_ledger_unavailable', proof['refusals']success, doc, final_response, error = proof['refusals']['script_result']diff --git a/tests/hermes_state/test_runtime_worker_children.py b/tests/hermes_state/test_runtime_worker_children.pynew file mode 100644index 00000000000..a06d350110c--- /dev/null+++ b/tests/hermes_state/test_runtime_worker_children.py@@ -0,0 +1,87 @@+"""Owner-reserved delegated children of a worker execution (#141793): the owner mints every child,+caps them, scopes their writes to their own row, and retires them with their parent."""+import os+from functools import partial++import psutil+import pytest++from gateway.session_worker_child import reserve_child_execution+from hermes_state import SessionDB+import hermes_state_runtime as rt+++def _reserve(db, epoch, sequence, *, limits=(2, 1), refused=None, execution_id='worker'):+ handler = partial(reserve_child_execution, epoch=epoch, execution_id=execution_id,+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},+ principal='local', limits=limits, refused=refused)+ return rt.apply_worker_receipt(db, epoch=epoch, execution_id=execution_id, session_id='parent', generation=0,+ sequence=sequence, operation='child.reserve',+ payload={'secret': f'child-secret-{sequence:08d}'}, handlers={'child.reserve': handler})+++def _child_append(db, epoch, child, sequence, content, session_id=None):+ return rt.mutate_worker_execution(db, epoch=epoch, execution_id=child['execution_id'],+ session_id=session_id or child['session_id'], generation=0, sequence=sequence,+ operation='transcript.append', payload={'messages': [{'role': 'assistant', 'content': content}]})+++def ledger(tmp_path):+ db = SessionDB(db_path=tmp_path / 'state.db')+ db.create_session('parent', source='kanban')+ db.create_session('other', source='cli')+ epoch = rt.begin_runtime_epoch(db, instance_id='boot')+ rt.register_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,+ kind='compute', adoption_secret='private-fixture')+ yield db, epoch+ db.close()+++def test_owner_mints_caps_and_scopes_children_and_retires_them_with_the_parent(ledger):+ db, epoch = ledger+ first = _reserve(db, epoch, 1)['child']+ second = _reserve(db, epoch, 2)['child']+ # Concurrency and draining caps are receipts, so the parent's journal never sticks.+ assert _reserve(db, epoch, 3) == {'refused': 'delegation_capacity'}+ assert _reserve(db, epoch, 4, refused='runtime_draining') == {'refused': 'runtime_draining'}+ row = db.get_session(first['session_id'])+ assert (row['source'], row['parent_session_id']) == ('subagent', 'parent')+ with db._read_ctx() as conn:+ kinds = conn.execute("SELECT kind,status FROM worker_executions WHERE execution_id!='worker'").fetchall()+ assert [tuple(r) for r in kinds] == [('child', 'running')] * 2+ # A child writes its own row only.+ _child_append(db, epoch, first, 1, 'CHILD_OUTPUT')+ with pytest.raises(rt.RuntimeStoreError, match='permission_denied'):+ _child_append(db, epoch, first, 2, 'escape', session_id='parent')+ assert [m['content'] for m in db.get_messages(first['session_id'])] == ['CHILD_OUTPUT']+ assert db.get_messages('parent') == []+ db.create_session(first['session_id'] + '-x', source='subagent') # unrelated row, untouched by retirement+ # The parent's finish retires every live child: it can no longer write and its row is ended.+ rt.mutate_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,+ sequence=5, operation='execution.finish', payload={})+ with pytest.raises(rt.RuntimeStoreError, match='stale_generation'):+ _child_append(db, epoch, second, 1, 'late')+ with db._read_ctx() as conn:+ assert {r[0] for r in conn.execute("SELECT status FROM worker_executions")} == {'terminal'}+ assert db.get_session(second['session_id'])['end_reason'] == 'agent_close'+ assert db.get_session(first['session_id'] + '-x')['ended_at'] is None+++def test_owner_restart_retires_in_flight_children_instead_of_recovering_them(ledger):+ db, epoch = ledger+ rt.adopt_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,+ adoption_secret='private-fixture')+ assert _reserve(db, epoch, 1, limits=(1, 1), execution_id='worker')['child']+ with db._read_ctx() as conn:+ child_eid, child_sid = conn.execute("SELECT execution_id,session_id FROM worker_executions WHERE kind='child'").fetchone()+ assert _reserve(db, epoch, 2, limits=(1, 1)) == {'refused': 'delegation_capacity'}+ replacement = rt.begin_runtime_epoch(db, instance_id='restart')+ rt.recover_session_inputs(db, epoch=replacement)+ with db._read_ctx() as conn:+ states = dict(conn.execute('SELECT execution_id,status FROM worker_executions').fetchall())+ # The root is the recoverable unknown; its child died with the worker and is never adoptable.+ assert states == {'worker': 'unknown', child_eid: 'terminal'}+ assert db.get_session(child_sid)['end_reason'] == 'agent_close'+ with pytest.raises(rt.RuntimeStoreError, match='stale_generation|stale_epoch'):+ _child_append(db, replacement, {'execution_id': child_eid, 'session_id': child_sid}, 1, 'late')diff --git a/tools/delegate_tool.py b/tools/delegate_tool.pyindex 486acbab106..56c58a154b4 100644--- a/tools/delegate_tool.py+++ b/tools/delegate_tool.py@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)# The delegate_tool_* siblings hold the pieces split out of this module; every name callers or patching tests reach as# ``tools.delegate_tool.<name>`` is re-imported here. Mutable flag globals live only in their owning module.from tools.delegate_tool_child_run import (- _ChildRun, _attach_child, _build_child_goal_message, _build_result_entry, _dump_subagent_timeout_diagnostic, _fabricated_entry,+ _ChildRun, _attach_child, _build_child_goal_message, _close_child, _detach_child, _build_result_entry, _dump_subagent_timeout_diagnostic, _fabricated_entry,_lease_child_credential, _merge_late_steer, _register_child, _start_heartbeat, _validate_child_output_schema,)from tools.delegate_tool_config import (@@ -137,9 +137,12 @@ def _open_child_session_db(parent_agent) -> Any:from agent.runtime_session_store import RuntimeSessionStore, WorkerPersistenceError, is_worker_processparent_session_db = getattr(parent_agent, "_session_db", None)if is_worker_process() or isinstance(parent_session_db, RuntimeSessionStore):- # Missing db_path on a scoped store is NOT permission to open the- # launch profile. Child identity must first be reserved by the owner.- raise WorkerPersistenceError('worker_child_registration_required')+ # Missing db_path on a scoped store is NOT permission to open the launch profile: the+ # owner reserves the child's row and execution under the parent's own (#141793).+ if not isinstance(parent_session_db, RuntimeSessionStore):+ raise WorkerPersistenceError('worker_child_registration_required')+ from agent.runtime_session_child import reserve_child_store+ return reserve_child_store(parent_session_db)if parent_session_db is None:return Nonewith _quiet("subagent: failed to open dedicated SessionDB; child persistence disabled", exc_info=True):@@ -287,6 +290,7 @@ def _build_child_agent(if child_progress_cb else None),session_db=child_session_db, parent_session_id=parent_sid, request_overrides=request_overrides,+ session_id=getattr(child_session_db, "reserved_session_id", None),tool_progress_callback=child_progress_cb,iteration_budget=None, # fresh budget per subagent)@@ -441,6 +445,10 @@ def _build_children(parent_agent=parent_agent, role=_normalize_role(t.get("role") or top_role), **overrides,)except ValueError as exc:+ # An owner refusal can land mid-batch: the siblings built so far hold reservations.+ for _, _, built in children:+ _detach_child(parent_agent, built)+ _close_child(built, "Failed to close child after a refused sibling")return [], str(exc)if _task_schema is not None:with _quiet("Could not attach output schema to child %d", i):diff --git a/website/docs/user-guide/features/kanban.md b/website/docs/user-guide/features/kanban.mdindex 0502a9fc852..d45fc03ec9d 100644--- a/website/docs/user-guide/features/kanban.md+++ b/website/docs/user-guide/features/kanban.md@@ -119,7 +119,7 @@ A worker that is blocked on `t_parent` and creates a support card for the missin**Use Kanban when** work crosses agent boundaries, needs to survive restarts, might need human input, might be picked up by a different role, or needs to be discoverable after the fact.-They coexist, but not inside one run: a kanban task executes in an isolated managed worker under the profile's gateway, and that worker is not offered `delegate_task` (a delegated child would need its own owner-registered execution, which the single-turn worker cannot reserve). Split work that needs fan-out into child cards instead (`kanban_create` with `parents`).+They also coexist inside one run. A kanban task executes in an isolated managed worker under the profile's gateway, and its `delegate_task` children are reserved by that gateway under the worker's execution: each child gets its own session (listed as a subagent of the card's session), counts against `delegation.max_concurrent_children` and `delegation.max_spawn_depth`, returns its result inside the card's turn, and ends with the worker. If the gateway goes down mid-child, the child is retired, never resumed. Work that must outlive the gateway, or needs its own reviewer, still belongs in child cards (`kanban_create` with `parents`).## Core concepts