Skip to content

The agent that grows with you

fix(managed-worker): delegate_task works in Kanban workers through owner-reserved children

Tekniumbf2b3713ab28
diff --git a/agent/managed_worker.py b/agent/managed_worker.py
index 5939edb5762..a41728cce01 100644
--- a/agent/managed_worker.py
+++ b/agent/managed_worker.py
@@ -24,9 +24,6 @@ import threading
logger = logging.getLogger(__name__)
MAX_FRAME = 4 * 1024 * 1024
-# Toolsets a managed worker turn never offers (subtracted last, so the tool_search bridge and a
-# restored tools[] pin drop them too): delegate_task's child has no owner registration here.
-WORKER_DISABLED_TOOLSETS = ('delegation',)
BOOTSTRAP_FIELDS = {'version', 'home', 'scope', 'policy', 'api_key', 'text', 'route', 'user_id', 'chat_id',
'turn_author', 'safe_mode', 'ignore_user_config'}
# The bounded run_conversation fields a turn receipt keeps (outcome, exit reason, tokens, cost,
@@ -360,10 +357,9 @@ def execute(frame, channel):
with policy_scope(policy):
agent = _construct_agent(frame, AIAgent, policy, model=policy.model, provider=policy.provider, base_url=policy.base_url,
api_key=frame['api_key'], session_db=store, session_id=scope['session_id'],
+ # delegate_task stays offered: each child is reserved by the owner under this
+ # execution (agent/runtime_session_child.py) and retires with it.
enabled_toolsets=list(policy.toolsets), max_iterations=policy.max_turns,
- # A delegated child needs its own owner-registered execution, which this single-turn
- # worker cannot reserve; never offer a tool whose only outcome here is a refusal.
- disabled_toolsets=list(WORKER_DISABLED_TOOLSETS),
reasoning_config=policy.reasoning_config,
# ``tool`` / ``oneshot`` are storage labels for a CLI-surface agent (session_policy.SURFACES).
platform=policy.platform if policy.source in ('tool', 'oneshot') else policy.source,
diff --git a/agent/runtime_session_child.py b/agent/runtime_session_child.py
new file mode 100644
index 00000000000..31866cc5d4d
--- /dev/null
+++ b/agent/runtime_session_child.py
@@ -0,0 +1,53 @@
+"""A delegated child's store inside a worker: owner-reserved, never self-registered (#141793).
+
+The parent's ``RuntimeSessionStore`` asks the owner for a child on its own receipt stream; the
+owner mints the child's session row and execution (gateway/session_worker_child.py) and the
+child persists through the ordinary ``worker.persist`` path under its own claim. A refusal
+(capacity, depth, draining) is a ValueError so ``delegate_task`` reports it as a tool error.
+"""
+import secrets
+
+from agent.runtime_session_store import RuntimeSessionStore, call_off_event_loop
+
+_REFUSALS = {
+ 'delegation_capacity': 'the gateway owner refused another subagent: this execution already runs '
+ 'delegation.max_concurrent_children of them',
+ 'delegation_depth_exceeded': 'the gateway owner refused a subagent this deep (delegation.max_spawn_depth)',
+}
+
+
+class ChildSessionStore(RuntimeSessionStore):
+ binds_delegation_ledger = False
+
+ @property
+ def reserved_session_id(self):
+ return self.scope['session_id']
+
+ def create_session(self, session_id, source, **fields):
+ # The owner created this row at reservation with its source, lineage and profile; only
+ # the constructor facts are the child's to add.
+ from hermes_state_worker_lifecycle import _CONSTRUCTOR_CONFIG
+ config = {k: v for k, v in (fields.get('model_config') or {}).items() if k in _CONSTRUCTOR_CONFIG}
+ return super().create_session(session_id, 'subagent', model=fields.get('model'),
+ model_config=config or None, system_prompt=fields.get('system_prompt'))
+
+ def close(self):
+ # The child's agent closes once, after its last write: that is this execution's end.
+ try:
+ if not self._outbox_owner.closed and not self.failure:
+ self.finish()
+ finally:
+ super().close()
+
+
+def reserve_child_store(parent):
+ """A ``ChildSessionStore`` the owner reserved under ``parent``'s verified execution."""
+ secret = secrets.token_urlsafe(32)
+ # Reached from delegate_task's tool callback, which can run on Relay's loop (#139029).
+ result = call_off_event_loop(parent._apply, 'child.reserve', {'secret': secret})
+ if 'refused' in result:
+ raise ValueError(_REFUSALS.get(result['refused'], 'the gateway owner refused a subagent: ' + result['refused']))
+ child = result['child']
+ scope = dict(parent.scope, **child, secret=secret)
+ outbox = parent.path.parent / ('child-' + child['execution_id'].rsplit(':', 1)[1])
+ return ChildSessionStore(parent.rpc, scope, outbox)
diff --git a/agent/runtime_session_store.py b/agent/runtime_session_store.py
index a187aa51ce0..3393da5fce7 100644
--- a/agent/runtime_session_store.py
+++ b/agent/runtime_session_store.py
@@ -220,6 +220,9 @@ class RuntimeSessionStore(RuntimeSessionCompressionMixin, RuntimeSessionLifecycl
adoption. Local queue acceptance is never returned as canonical commit.
Pending journals live outside age-pruned cache trees.
"""
+ # The process's async-delegation ledger belongs to the adopted execution, never a reserved child.
+ binds_delegation_ledger = True
+
def __init__(self, rpc, scope, outbox_dir, adopted=None, *, max_bytes=4 * 1024 * 1024):
self.rpc = rpc
self.scope = dict(scope)
@@ -258,7 +261,7 @@ class RuntimeSessionStore(RuntimeSessionCompressionMixin, RuntimeSessionLifecycl
# Replay must land the journal exactly on the adopted assignment (a pending
# publication moves it to the child); anything else is not this successor's outbox.
raise WorkerPersistenceError('outbox_scope_mismatch')
- if is_worker_process():
+ if is_worker_process() and self.binds_delegation_ledger:
from tools.async_delegation_worker import bind_worker_delegation_store
bind_worker_delegation_store(self)
except Exception:
diff --git a/gateway/session_worker.py b/gateway/session_worker.py
index 914280114bc..aaa3a6bac27 100644
--- a/gateway/session_worker.py
+++ b/gateway/session_worker.py
@@ -1,7 +1,7 @@
"""Execution-scoped worker operations on the ordinary authenticated authority.
-Only explicit local compute registrations are enabled. Cron/child claims need
-producer-specific validation before their consumers can use this transport.
+Only explicit local compute registrations are enabled here. A delegated child is reserved by
+the owner on its parent's receipt stream (gateway/session_worker_child.py), never registered.
"""
import asyncio
import hmac
@@ -68,6 +68,9 @@ async def worker_request(connection, ref, params, *, operation):
if operation == "persist" and isinstance(params.get("operation"), str) and params["operation"].startswith("delegation."):
from gateway.session_worker_delegation import worker_delegation_request
return await worker_delegation_request(connection, ref, params)
+ if operation == "persist" and params.get("operation") == "child.reserve":
+ from gateway.session_worker_child import worker_child_request
+ return await worker_child_request(connection, ref, params)
extra = {'register': {'kind'}, 'adopt': set(), 'persist': {'epoch', 'sequence', 'operation', 'payload'}}[operation]
if set(params) != _SCOPE | extra:
raise RuntimeStoreError('invalid_params')
diff --git a/gateway/session_worker_child.py b/gateway/session_worker_child.py
new file mode 100644
index 00000000000..24cc9941630
--- /dev/null
+++ b/gateway/session_worker_child.py
@@ -0,0 +1,97 @@
+"""Owner-reserved delegated children of a verified worker execution (#141793).
+
+A worker never mints a child. It asks on its OWN fenced receipt stream (epoch, assignment,
+generation, sequence), and the owner creates the child's session row and execution, binds it to
+the parent's verified process identity and the worker's fresh secret, and enforces the frozen
+policy's delegation caps. The child then persists through the ordinary ``worker.persist`` path
+under its own claim. Expected refusals (caps, draining) are receipts, not errors: an error would
+leave the parent's journal pending and refuse every later write of its turn.
+"""
+import secrets
+from functools import partial
+
+from hermes_state_runtime import RuntimeStoreError, apply_worker_receipt
+from hermes_state_runtime_workers import CHILD_SEPARATOR
+
+_DEFAULTS = {'max_concurrent_children': 10, 'max_spawn_depth': 1}
+
+
+def _limit(section, key):
+ try:
+ return max(1, int(section.get(key, _DEFAULTS[key])))
+ except (TypeError, ValueError):
+ return _DEFAULTS[key]
+
+
+def delegation_limits(authority, execution_id, session_id):
+ """``(max_concurrent_children, max_spawn_depth)`` from the root conversation's frozen policy
+ (the profile's config for a session with no local policy), never from the asking worker."""
+ from gateway.session_policy import policy_for_source, profile_config
+ root = execution_id.split(CHILD_SEPARATOR, 1)[0]
+ if root != execution_id:
+ with authority.db._read_ctx() as conn:
+ row = conn.execute('SELECT session_id FROM worker_executions WHERE execution_id=?', (root,)).fetchone()
+ session_id = row[0] if row else session_id
+ live = authority.sessions.get(authority.logical_owner(session_id))
+ policy = policy_for_source(authority.runner, live.source) if live is not None else None
+ config = policy.config(authority) if policy is not None else profile_config(authority)
+ section = config.get('delegation') or {}
+ return _limit(section, 'max_concurrent_children'), _limit(section, 'max_spawn_depth')
+
+
+def reserve_child_execution(db, conn, session_id, payload, *, epoch, execution_id, identity, principal, limits, refused):
+ from hermes_state_ids import new_session_id
+ from hermes_state_keys import admission_fingerprint
+ from hermes_state_runtime import _secret_digest
+ from hermes_state_worker_lifecycle import insert_session_row_in_transaction
+ secret = payload.get('secret') if isinstance(payload, dict) else None
+ if set(payload) != {'secret'} or not isinstance(secret, str) or not 16 <= len(secret) <= 256:
+ raise RuntimeStoreError('invalid_params')
+ max_children, max_depth = limits
+ prefix = execution_id + CHILD_SEPARATOR
+ live = conn.execute("SELECT COUNT(*) FROM worker_executions WHERE substr(execution_id,1,?)=? "
+ "AND instr(substr(execution_id,?),'/')=0 AND status!='terminal'",
+ (len(prefix), prefix, len(prefix) + 1)).fetchone()[0]
+ if refused is None and prefix.count(CHILD_SEPARATOR) > max_depth:
+ refused = 'delegation_depth_exceeded'
+ if refused is None and live >= max_children:
+ refused = 'delegation_capacity'
+ if refused is not None:
+ return {'refused': refused}
+ parent = conn.execute('SELECT profile_name FROM sessions WHERE id=?', (session_id,)).fetchone()
+ child_sid, child_eid = new_session_id(), prefix + secrets.token_hex(8)
+ insert_session_row_in_transaction(db, conn, child_sid, 'subagent', model_config={'_delegate_from': session_id},
+ parent_session_id=session_id, profile_name=parent['profile_name'])
+ scope = dict(identity, session_id=child_sid, execution_id=child_eid, generation=0, secret=secret)
+ claim = admission_fingerprint(canonical_target=child_sid, payload=scope | {'principal': principal})
+ conn.execute("INSERT INTO worker_executions(execution_id,session_id,kind,owner_epoch,generation,status,adoption_digest) "
+ "VALUES(?,?,'child',?,0,'running',?)", (child_eid, child_sid, epoch, _secret_digest(claim)))
+ return {'child': {'session_id': child_sid, 'execution_id': child_eid, 'generation': 0}}
+
+
+async def worker_child_request(connection, ref, params):
+ """worker.persist ``child.reserve``: the parent's full claim and assignment are verified first."""
+ import asyncio
+ from gateway.session_worker import _SCOPE, _claim, _verify
+ if set(params) != _SCOPE | {'epoch', 'sequence', 'operation', 'payload'}:
+ raise RuntimeStoreError('invalid_params')
+ claim = _claim(connection, ref, params)
+ if 'worker:adopt' not in connection.actor.capabilities:
+ raise RuntimeStoreError('permission_denied')
+ _verify(connection, ref, params, claim)
+ authority = connection.authority
+
+ def reserve():
+ refused, limits = None, (0, 0)
+ try:
+ authority._require_admission_open()
+ limits = delegation_limits(authority, params['execution_id'], params['session_id'])
+ except RuntimeStoreError as exc:
+ refused = exc.reason
+ handler = partial(reserve_child_execution, epoch=params['epoch'], execution_id=params['execution_id'],
+ identity={key: params[key] for key in ('profile_id', 'pid', 'birth')},
+ principal=connection.actor.subject, limits=limits, refused=refused)
+ return apply_worker_receipt(authority.db, **{key: params[key] for key in (
+ 'epoch', 'execution_id', 'session_id', 'generation', 'sequence', 'operation', 'payload')},
+ handlers={'child.reserve': handler})
+ return await asyncio.to_thread(reserve)
diff --git a/hermes_state_runtime.py b/hermes_state_runtime.py
index 838e6fc2971..60aaf594952 100644
--- a/hermes_state_runtime.py
+++ b/hermes_state_runtime.py
@@ -448,6 +448,8 @@ def recover_session_inputs(db, *, epoch: int) -> int:
"""Never replay started work. Live worker adoption is a separate explicit operation."""
def write(conn):
_epoch(conn, epoch)
+ from hermes_state_runtime_workers import retire_stale_children
+ retire_stale_children(conn, epoch)
conn.execute("UPDATE worker_executions SET status='unknown' WHERE status IN ('registered','running') AND owner_epoch!=?", (epoch,))
# A steer handed to a lost claim may or may not have reached its agent: never replayed.
conn.execute("UPDATE session_admissions SET outcome='interrupted' WHERE status='terminal' AND outcome=? "
diff --git a/hermes_state_runtime_workers.py b/hermes_state_runtime_workers.py
index a7d9a90e881..118d2158d30 100644
--- a/hermes_state_runtime_workers.py
+++ b/hermes_state_runtime_workers.py
@@ -45,15 +45,50 @@ def discard_orphan_workers_on_reset(conn, session_ids):
RETIRED_RECEIPT = {'retired': True}
+# A delegated child's execution id is its parent's plus this and a random suffix, so every
+# descendant of an execution is one prefix probe away (gateway/session_worker_child.py).
+CHILD_SEPARATOR = '/child:'
+
+
+def retire_child_executions(conn, execution_id):
+ """A child never outlives the execution that reserved it: every live descendant goes terminal
+ with it, its read receipts shrink and its open session row is ended. A late child write is
+ then refused ``stale_generation`` like any write after retirement."""
+ prefix = execution_id + CHILD_SEPARATOR
+ _retire_children(conn, conn.execute("SELECT execution_id,session_id FROM worker_executions "
+ "WHERE substr(execution_id,1,?)=? AND status!='terminal'",
+ (len(prefix), prefix)).fetchall())
+
+
+def retire_stale_children(conn, epoch):
+ """Owner restart: a child dies with its worker, which dies with its owner, so it is never
+ adoptable; only its root execution can be recovered as ``unknown``."""
+ _retire_children(conn, conn.execute("SELECT execution_id,session_id FROM worker_executions WHERE kind='child' "
+ "AND status!='terminal' AND owner_epoch!=?", (epoch,)).fetchall())
+
+
+def _retire_children(conn, rows):
+ import time
+ for child, session_id in rows:
+ conn.execute("UPDATE worker_executions SET status='terminal' WHERE execution_id=?", (child,))
+ _compact(conn, child)
+ conn.execute("UPDATE sessions SET ended_at=?,end_reason='agent_close' WHERE id=? AND ended_at IS NULL",
+ (time.time(), session_id))
def compact_terminal_receipts(conn, execution_id):
- """Once an execution is terminal its read receipts are dead weight: a ``compression.history``
+ """Every terminal transition calls this, so it also retires the execution's delegated children.
+ Once an execution is terminal its read receipts are dead weight: a ``compression.history``
read stores the whole transcript and a context read a full session row, so keeping them leaves
one full-history copy per managed turn and ``state.db`` grows quadratically. They shrink to a
marker in the closing transaction (an exact late replay is refused ``stale_generation``, as
after retirement); ``payload_digest`` stays, so a late duplicate with other content still
conflicts. Mutation acknowledgements (counts, row annotations, assignments) stay replayable."""
+ retire_child_executions(conn, execution_id)
+ _compact(conn, execution_id)
+
+
+def _compact(conn, execution_id):
conn.execute("UPDATE worker_receipts SET result_json=? WHERE execution_id=? AND json_valid(result_json) "
"AND (json_type(result_json, '$.messages') IS NOT NULL "
"OR json_type(result_json, '$.session') IS NOT NULL)",
diff --git a/tests/gateway/fixtures/agent_persistence_worker.py b/tests/gateway/fixtures/agent_persistence_worker.py
index 8f0f1ab34aa..e60467fdaef 100644
--- a/tests/gateway/fixtures/agent_persistence_worker.py
+++ b/tests/gateway/fixtures/agent_persistence_worker.py
@@ -60,7 +60,7 @@ def main():
opened = fn()
if opened is not None:
release_or_close(opened)
- return name, 'unexpected_success'
+ return name, 'opened:' + type(opened).__name__
except Exception as exc:
return name, str(exc)
with ThreadPoolExecutor(max_workers=1) as pool:
diff --git a/tests/gateway/test_managed_worker_delegation.py b/tests/gateway/test_managed_worker_delegation.py
new file mode 100644
index 00000000000..3897d6c9753
--- /dev/null
+++ b/tests/gateway/test_managed_worker_delegation.py
@@ -0,0 +1,150 @@
+"""delegate_task inside a managed worker (#141793): real daemon, real worker, loopback model.
+
+Every child is reserved by the owner under the worker's execution, persists through its own
+claim, returns its result to the parent's turn, and dies with the owner on a restart."""
+import asyncio
+from contextlib import closing
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import json
+import os
+from pathlib import Path
+import sqlite3
+import threading
+
+import pytest
+
+from tests.gateway.fixtures.local_recovery_probe import daemon, rpc, websocket
+from tests.waiting import async_wait_until
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class Model(BaseHTTPRequestHandler):
+ """The parent delegates once; the child answers (or blocks on BLOCK); the parent then finishes."""
+ def log_message(self, *args):
+ pass
+
+ def do_POST(self):
+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
+ messages = body.get('messages') or [{'role': 'system', 'content': ''}]
+ users = [m.get('content') for m in messages if m['role'] == 'user']
+ text = json.dumps(messages)
+ name = None
+ if users and users[0] in ('DELEGATE', 'DELEGATE_BLOCK') and messages[-1]['role'] == 'user':
+ goal = 'CHILD_GOAL_BLOCK' if users[0] == 'DELEGATE_BLOCK' else 'CHILD_GOAL'
+ name, args = 'delegate_task', {'goal': goal}
+ elif 'CHILD_GOAL' in text and not any(u in ('DELEGATE', 'DELEGATE_BLOCK') for u in users):
+ if 'CHILD_GOAL_BLOCK' in text:
+ self.server.blocked.set()
+ self.server.release.wait(60)
+ message = {'role': 'assistant', 'content': 'CHILD_RESULT_42'}
+ else:
+ message = {'role': 'assistant', 'content': 'PARENT_DONE'}
+ if name:
+ message = {'role': 'assistant', 'content': None, 'tool_calls': [{'id': 'd1', 'index': 0, 'type': 'function',
+ 'function': {'name': name, 'arguments': json.dumps(args)}}]}
+ self.server.seen.append((users[:1], message.get('content'), [t['function']['name'] for t in body.get('tools') or []]))
+ choice = {'index': 0, 'delta': message, 'finish_reason': 'tool_calls' if name else 'stop'}
+ frame = {'id': 'm', 'model': 'm', 'choices': [choice], 'usage': {'prompt_tokens': 3, 'completion_tokens': 2, 'total_tokens': 5}}
+ if body.get('stream'):
+ payload, kind = ('data: ' + json.dumps(frame) + '\n\ndata: [DONE]\n\n').encode(), 'text/event-stream'
+ else:
+ choice['message'] = choice.pop('delta')
+ payload, kind = json.dumps(frame).encode(), 'application/json'
+ try:
+ self.send_response(200)
+ self.send_header('Content-Type', kind)
+ self.send_header('Content-Length', str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+ except (BrokenPipeError, ConnectionResetError):
+ pass
+
+
+def _setup(tmp_path):
+ home, user = tmp_path / 'state', tmp_path / 'user'
+ home.mkdir(mode=0o700)
+ user.mkdir()
+ peer = ThreadingHTTPServer(('127.0.0.1', 0), Model)
+ peer.seen, peer.blocked, peer.release = [], threading.Event(), threading.Event()
+ threading.Thread(target=peer.serve_forever, daemon=True).start()
+ url = f'http://127.0.0.1:{peer.server_port}/v1'
+ (home / 'config.yaml').write_text(json.dumps({
+ 'gateway': {'multiplex_profiles': False, 'managed_workers': True},
+ 'model': {'provider': 'custom', 'default': 'm', 'base_url': url},
+ 'auxiliary': {'title_generation': {'enabled': False}},
+ 'platform_toolsets': {'cli': ['delegation', 'todo']}}))
+ env = {k: os.environ[k] for k in ('PATH', 'LANG', 'TZ') if k in os.environ}
+ env.update(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home), PYTHONPATH=str(ROOT),
+ OPENAI_API_KEY='loopback-only', OPENAI_BASE_URL=url)
+ return home, peer, url, env
+
+
+def _rows(home, sql, *args):
+ with closing(sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True)) as db:
+ return db.execute(sql, args).fetchall()
+
+
+async def _submit(home, desc, url, text):
+ async with websocket(home, desc) as ws:
+ created = await rpc(ws, 'session.create', request_id='parent', source='cli', cwd=str(home), model='m',
+ provider='custom', base_url=url, api_key='loopback-only',
+ toolsets=['delegation', 'todo'], ignore_rules=True)
+ sid = created['result']['session_id']
+ assert 'result' in await rpc(ws, 'prompt.submit', session_id=sid, input_id='run', text=text)
+ return sid
+
+
+def test_managed_worker_delegates_through_an_owner_reserved_child(tmp_path):
+ home, peer, url, env = _setup(tmp_path)
+
+ async def exercise(desc):
+ sid = await _submit(home, desc, url, 'DELEGATE')
+ await async_wait_until(lambda: _rows(home, "SELECT status FROM session_admissions WHERE request_id='run'") == [
+ ('terminal',)], timeout=120, interval=.1, what='run terminal')
+ async with websocket(home, desc) as ws:
+ return sid, (await rpc(ws, 'session.resume', session_id=sid))['result']['messages']
+ try:
+ with daemon(ROOT, home, env, barrier=False) as (_owner, desc):
+ sid, messages = asyncio.run(exercise(desc))
+ finally:
+ peer.release.set()
+ peer.shutdown()
+ peer.server_close()
+ assert any('delegate_task' in tools for _, _, tools in peer.seen), peer.seen
+ tool = [m['content'] for m in messages if m['role'] == 'tool'][-1]
+ assert 'CHILD_RESULT_42' in tool, tool
+ assert messages[-1]['content'] == 'PARENT_DONE', messages
+ children = _rows(home, "SELECT w.status, s.source, s.parent_session_id, s.end_reason, s.input_tokens FROM worker_executions w "
+ "JOIN sessions s ON s.id=w.session_id WHERE w.kind='child'")
+ assert children == [('terminal', 'subagent', sid, 'agent_close', 3)], children
+ child_sid = _rows(home, "SELECT session_id FROM worker_executions WHERE kind='child'")[0][0]
+ assert ('assistant', 'CHILD_RESULT_42') in _rows(home, 'SELECT role, content FROM messages WHERE session_id=?', child_sid)
+
+
+def test_owner_restart_mid_child_retires_the_child_and_never_replays_it(tmp_path):
+ home, peer, url, env = _setup(tmp_path)
+
+ async def first(desc):
+ await _submit(home, desc, url, 'DELEGATE_BLOCK')
+ assert await asyncio.to_thread(peer.blocked.wait, 90), peer.seen
+ assert _rows(home, "SELECT status FROM worker_executions WHERE kind='child'") == [('running',)]
+ try:
+ with daemon(ROOT, home, env, barrier=False) as (owner, desc):
+ asyncio.run(first(desc))
+ owner.kill()
+ owner.wait(timeout=10)
+ peer.release.set()
+ with daemon(ROOT, home, env, barrier=False):
+ states = dict(_rows(home, 'SELECT kind, status FROM worker_executions'))
+ finally:
+ peer.release.set()
+ peer.shutdown()
+ peer.server_close()
+ assert states == {'compute': 'unknown', 'child': 'terminal'}, states
+ assert _rows(home, "SELECT s.end_reason FROM worker_executions w JOIN sessions s ON s.id=w.session_id "
+ "WHERE w.kind='child'") == [('agent_close',)]
+ # The lost turn is not replayed: the parent never got a child result to finish on.
+ assert not [s for s in peer.seen if s[0] == ['DELEGATE_BLOCK'] and s[1] == 'PARENT_DONE'], peer.seen
diff --git a/tests/gateway/test_managed_worker_no_delegation.py b/tests/gateway/test_managed_worker_no_delegation.py
deleted file mode 100644
index ae5bd50f188..00000000000
--- a/tests/gateway/test_managed_worker_no_delegation.py
+++ /dev/null
@@ -1,83 +0,0 @@
-"""A managed worker turn never offers ``delegate_task``: its child would need an owner-registered
-execution this single-turn worker cannot reserve, so the tool's only outcome would be a refusal
-(real daemon, real managed worker, loopback model)."""
-import asyncio
-from contextlib import closing
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-import json
-import os
-from pathlib import Path
-import sqlite3
-import threading
-
-import pytest
-
-from tests.gateway.fixtures.local_recovery_probe import daemon, rpc, websocket
-from tests.waiting import async_wait_until
-
-
-class Model(BaseHTTPRequestHandler):
- def log_message(self, *args):
- pass
-
- def do_POST(self):
- body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
- if body.get('messages'):
- self.server.offered.append(json.dumps(body.get('tools') or []))
- message = {'role': 'assistant', 'content': 'WORKER_DONE'}
- frame = {'id': 'tools', 'model': 'm', 'choices': [{'index': 0, 'delta': message, 'finish_reason': 'stop'}],
- 'usage': {'prompt_tokens': 1, 'completion_tokens': 1, 'total_tokens': 2}}
- if body.get('stream'):
- payload, kind = ('data: ' + json.dumps(frame) + '\n\ndata: [DONE]\n\n').encode(), 'text/event-stream'
- else:
- frame['choices'][0]['message'] = frame['choices'][0].pop('delta')
- payload, kind = json.dumps(frame).encode(), 'application/json'
- self.send_response(200)
- self.send_header('Content-Type', kind)
- self.send_header('Content-Length', str(len(payload)))
- self.end_headers()
- self.wfile.write(payload)
-
-
-def test_managed_worker_turn_does_not_offer_delegate_task(tmp_path):
- root = Path(__file__).resolve().parents[2]
- home, user = tmp_path / 'state', tmp_path / 'user'
- home.mkdir(mode=0o700)
- user.mkdir()
- peer = ThreadingHTTPServer(('127.0.0.1', 0), Model)
- peer.offered = []
- threading.Thread(target=peer.serve_forever, daemon=True).start()
- url = f'http://127.0.0.1:{peer.server_port}/v1'
- (home / 'config.yaml').write_text(json.dumps({
- 'gateway': {'multiplex_profiles': False, 'managed_workers': True},
- 'model': {'provider': 'custom', 'default': 'm', 'base_url': url},
- 'auxiliary': {'title_generation': {'enabled': False}},
- 'platform_toolsets': {'cli': ['delegation', 'todo']}}))
- env = {k: os.environ[k] for k in ('PATH', 'LANG', 'TZ') if k in os.environ}
- env.update(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home), PYTHONPATH=str(root),
- OPENAI_API_KEY='loopback-only', OPENAI_BASE_URL=url)
-
- def rows(sql):
- with closing(sqlite3.connect(f'file:{home / "state.db"}?mode=ro', uri=True)) as db:
- return db.execute(sql).fetchall()
-
- async def exercise(desc):
- async with websocket(home, desc) as ws:
- session = await rpc(ws, 'session.create', request_id='tools', source='cli', cwd=str(home), model='m',
- provider='custom', base_url=url, api_key='loopback-only',
- toolsets=['delegation', 'todo'], ignore_rules=True)
- sid = session['result']['session_id']
- submitted = await rpc(ws, 'prompt.submit', session_id=sid, input_id='run', text='hello')
- assert 'result' in submitted, submitted
- await async_wait_until(lambda: rows("SELECT status FROM session_admissions WHERE request_id='run'") == [
- ('terminal',)], timeout=90, interval=.1, what='run terminal')
- try:
- with daemon(root, home, env, barrier=False) as (_owner, desc):
- asyncio.run(exercise(desc))
- finally:
- peer.shutdown()
- peer.server_close()
- assert peer.offered, 'the managed worker never reached the model'
- assert all('todo_list' in tools for tools in peer.offered), peer.offered # the turn kept its other tools
- assert not any('delegate_task' in tools for tools in peer.offered), peer.offered
diff --git a/tests/gateway/test_worker_agent_execution.py b/tests/gateway/test_worker_agent_execution.py
index ce971664f30..b0d40e6dbca 100644
--- a/tests/gateway/test_worker_agent_execution.py
+++ b/tests/gateway/test_worker_agent_execution.py
@@ -78,7 +78,8 @@ def test_real_agent_worker_persists_context_usage_and_releases_lease(tmp_path, p
assert not proof['failed'], proof
assert 'RECOVERY_ACK_WORKER_INFERENCE' in proof['result'], proof
if probe_constructors:
- assert proof['refusals']['child'] == 'worker_child_registration_required', proof['refusals']
+ # A child is reserved by the owner under this execution, never opened on SQLite (#141793).
+ assert proof['refusals']['child'] == 'opened:ChildSessionStore', proof['refusals']
assert proof['refusals']['cron'] == 'worker_cron_registration_required', proof['refusals']
assert proof['refusals']['ledger'] == 'worker_delegation_ledger_unavailable', proof['refusals']
success, doc, final_response, error = proof['refusals']['script_result']
diff --git a/tests/hermes_state/test_runtime_worker_children.py b/tests/hermes_state/test_runtime_worker_children.py
new file mode 100644
index 00000000000..a06d350110c
--- /dev/null
+++ b/tests/hermes_state/test_runtime_worker_children.py
@@ -0,0 +1,87 @@
+"""Owner-reserved delegated children of a worker execution (#141793): the owner mints every child,
+caps them, scopes their writes to their own row, and retires them with their parent."""
+import os
+from functools import partial
+
+import psutil
+import pytest
+
+from gateway.session_worker_child import reserve_child_execution
+from hermes_state import SessionDB
+import hermes_state_runtime as rt
+
+
+def _reserve(db, epoch, sequence, *, limits=(2, 1), refused=None, execution_id='worker'):
+ handler = partial(reserve_child_execution, epoch=epoch, execution_id=execution_id,
+ identity={'profile_id': 'home', 'pid': os.getpid(), 'birth': psutil.Process().create_time()},
+ principal='local', limits=limits, refused=refused)
+ return rt.apply_worker_receipt(db, epoch=epoch, execution_id=execution_id, session_id='parent', generation=0,
+ sequence=sequence, operation='child.reserve',
+ payload={'secret': f'child-secret-{sequence:08d}'}, handlers={'child.reserve': handler})
+
+
+def _child_append(db, epoch, child, sequence, content, session_id=None):
+ return rt.mutate_worker_execution(db, epoch=epoch, execution_id=child['execution_id'],
+ session_id=session_id or child['session_id'], generation=0, sequence=sequence,
+ operation='transcript.append', payload={'messages': [{'role': 'assistant', 'content': content}]})
+
+
+def ledger(tmp_path):
+ db = SessionDB(db_path=tmp_path / 'state.db')
+ db.create_session('parent', source='kanban')
+ db.create_session('other', source='cli')
+ epoch = rt.begin_runtime_epoch(db, instance_id='boot')
+ rt.register_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,
+ kind='compute', adoption_secret='private-fixture')
+ yield db, epoch
+ db.close()
+
+
+def test_owner_mints_caps_and_scopes_children_and_retires_them_with_the_parent(ledger):
+ db, epoch = ledger
+ first = _reserve(db, epoch, 1)['child']
+ second = _reserve(db, epoch, 2)['child']
+ # Concurrency and draining caps are receipts, so the parent's journal never sticks.
+ assert _reserve(db, epoch, 3) == {'refused': 'delegation_capacity'}
+ assert _reserve(db, epoch, 4, refused='runtime_draining') == {'refused': 'runtime_draining'}
+ row = db.get_session(first['session_id'])
+ assert (row['source'], row['parent_session_id']) == ('subagent', 'parent')
+ with db._read_ctx() as conn:
+ kinds = conn.execute("SELECT kind,status FROM worker_executions WHERE execution_id!='worker'").fetchall()
+ assert [tuple(r) for r in kinds] == [('child', 'running')] * 2
+ # A child writes its own row only.
+ _child_append(db, epoch, first, 1, 'CHILD_OUTPUT')
+ with pytest.raises(rt.RuntimeStoreError, match='permission_denied'):
+ _child_append(db, epoch, first, 2, 'escape', session_id='parent')
+ assert [m['content'] for m in db.get_messages(first['session_id'])] == ['CHILD_OUTPUT']
+ assert db.get_messages('parent') == []
+ db.create_session(first['session_id'] + '-x', source='subagent') # unrelated row, untouched by retirement
+ # The parent's finish retires every live child: it can no longer write and its row is ended.
+ rt.mutate_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,
+ sequence=5, operation='execution.finish', payload={})
+ with pytest.raises(rt.RuntimeStoreError, match='stale_generation'):
+ _child_append(db, epoch, second, 1, 'late')
+ with db._read_ctx() as conn:
+ assert {r[0] for r in conn.execute("SELECT status FROM worker_executions")} == {'terminal'}
+ assert db.get_session(second['session_id'])['end_reason'] == 'agent_close'
+ assert db.get_session(first['session_id'] + '-x')['ended_at'] is None
+
+
+def test_owner_restart_retires_in_flight_children_instead_of_recovering_them(ledger):
+ db, epoch = ledger
+ rt.adopt_worker_execution(db, epoch=epoch, execution_id='worker', session_id='parent', generation=0,
+ adoption_secret='private-fixture')
+ assert _reserve(db, epoch, 1, limits=(1, 1), execution_id='worker')['child']
+ with db._read_ctx() as conn:
+ child_eid, child_sid = conn.execute("SELECT execution_id,session_id FROM worker_executions WHERE kind='child'").fetchone()
+ assert _reserve(db, epoch, 2, limits=(1, 1)) == {'refused': 'delegation_capacity'}
+ replacement = rt.begin_runtime_epoch(db, instance_id='restart')
+ rt.recover_session_inputs(db, epoch=replacement)
+ with db._read_ctx() as conn:
+ states = dict(conn.execute('SELECT execution_id,status FROM worker_executions').fetchall())
+ # The root is the recoverable unknown; its child died with the worker and is never adoptable.
+ assert states == {'worker': 'unknown', child_eid: 'terminal'}
+ assert db.get_session(child_sid)['end_reason'] == 'agent_close'
+ with pytest.raises(rt.RuntimeStoreError, match='stale_generation|stale_epoch'):
+ _child_append(db, replacement, {'execution_id': child_eid, 'session_id': child_sid}, 1, 'late')
diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py
index 486acbab106..56c58a154b4 100644
--- a/tools/delegate_tool.py
+++ b/tools/delegate_tool.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
# The delegate_tool_* siblings hold the pieces split out of this module; every name callers or patching tests reach as
# ``tools.delegate_tool.<name>`` is re-imported here. Mutable flag globals live only in their owning module.
from tools.delegate_tool_child_run import (
- _ChildRun, _attach_child, _build_child_goal_message, _build_result_entry, _dump_subagent_timeout_diagnostic, _fabricated_entry,
+ _ChildRun, _attach_child, _build_child_goal_message, _close_child, _detach_child, _build_result_entry, _dump_subagent_timeout_diagnostic, _fabricated_entry,
_lease_child_credential, _merge_late_steer, _register_child, _start_heartbeat, _validate_child_output_schema,
)
from tools.delegate_tool_config import (
@@ -137,9 +137,12 @@ def _open_child_session_db(parent_agent) -> Any:
from agent.runtime_session_store import RuntimeSessionStore, WorkerPersistenceError, is_worker_process
parent_session_db = getattr(parent_agent, "_session_db", None)
if is_worker_process() or isinstance(parent_session_db, RuntimeSessionStore):
- # Missing db_path on a scoped store is NOT permission to open the
- # launch profile. Child identity must first be reserved by the owner.
- raise WorkerPersistenceError('worker_child_registration_required')
+ # Missing db_path on a scoped store is NOT permission to open the launch profile: the
+ # owner reserves the child's row and execution under the parent's own (#141793).
+ if not isinstance(parent_session_db, RuntimeSessionStore):
+ raise WorkerPersistenceError('worker_child_registration_required')
+ from agent.runtime_session_child import reserve_child_store
+ return reserve_child_store(parent_session_db)
if parent_session_db is None:
return None
with _quiet("subagent: failed to open dedicated SessionDB; child persistence disabled", exc_info=True):
@@ -287,6 +290,7 @@ def _build_child_agent(
if child_progress_cb else None
),
session_db=child_session_db, parent_session_id=parent_sid, request_overrides=request_overrides,
+ session_id=getattr(child_session_db, "reserved_session_id", None),
tool_progress_callback=child_progress_cb,
iteration_budget=None, # fresh budget per subagent
)
@@ -441,6 +445,10 @@ def _build_children(
parent_agent=parent_agent, role=_normalize_role(t.get("role") or top_role), **overrides,
)
except ValueError as exc:
+ # An owner refusal can land mid-batch: the siblings built so far hold reservations.
+ for _, _, built in children:
+ _detach_child(parent_agent, built)
+ _close_child(built, "Failed to close child after a refused sibling")
return [], str(exc)
if _task_schema is not None:
with _quiet("Could not attach output schema to child %d", i):
diff --git a/website/docs/user-guide/features/kanban.md b/website/docs/user-guide/features/kanban.md
index 0502a9fc852..d45fc03ec9d 100644
--- a/website/docs/user-guide/features/kanban.md
+++ b/website/docs/user-guide/features/kanban.md
@@ -119,7 +119,7 @@ A worker that is blocked on `t_parent` and creates a support card for the missin
**Use Kanban when** work crosses agent boundaries, needs to survive restarts, might need human input, might be picked up by a different role, or needs to be discoverable after the fact.
-They coexist, but not inside one run: a kanban task executes in an isolated managed worker under the profile's gateway, and that worker is not offered `delegate_task` (a delegated child would need its own owner-registered execution, which the single-turn worker cannot reserve). Split work that needs fan-out into child cards instead (`kanban_create` with `parents`).
+They also coexist inside one run. A kanban task executes in an isolated managed worker under the profile's gateway, and its `delegate_task` children are reserved by that gateway under the worker's execution: each child gets its own session (listed as a subagent of the card's session), counts against `delegation.max_concurrent_children` and `delegation.max_spawn_depth`, returns its result inside the card's turn, and ends with the worker. If the gateway goes down mid-child, the child is retired, never resumed. Work that must outlive the gateway, or needs its own reviewer, still belongs in child cards (`kanban_create` with `parents`).
## Core concepts