The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a lost managed worker's board note lands before its admission settles, boundedly
kshitijk4poor08527eef1ce3
diff --git a/gateway/session_kanban.py b/gateway/session_kanban.pyindex 93e58088498..31ab6a1b617 100644--- a/gateway/session_kanban.py+++ b/gateway/session_kanban.py@@ -313,23 +313,15 @@ def _record_worker_result(context, code, last_output):def record_worker_loss(policy, reason, *, pid=None, bound=False):- """Durably record that this attempt's managed execution was LOST — on the card's board.-- ``worker.adopt`` runs before ``bind_worker_context``, so a worker that dies in the adopt- handshake writes neither ``worker_bound`` nor ``worker_result``: the run learns nothing- from its own loss, and the dispatcher can only guess from a dead pid. 2026-10-22- (t_f5fad63f) the gateway logged ``Managed worker lost: managed_worker_failed`` 24 s in- and the card still said ``running`` 49 minutes later. A warning in a log file is not a- disposition, so the owner writes the disposition where the board can read it.-- Best-effort and run-scoped: a loss recorded against a run the card has already moved- past is not evidence about the current attempt, so it is dropped. Never raises — the- loss itself must reach the caller.+ """Append a run-scoped ``worker_lost`` event for this attempt to the card's board.++ Dropped when the card has moved past the run (it says nothing about the current attempt).+ Never raises: the caller's own loss handling must not depend on the board write."""- context = json.loads(getattr(policy, 'kanban_json', None) or 'null')- if not context:- returntry:+ context = json.loads(getattr(policy, 'kanban_json', None) or 'null')+ if not context:+ returnfrom hermes_cli.kanban_db_connect import connect_closingfrom hermes_cli import kanban_db as kbrun_id = int(context['run_id'])@@ -342,12 +334,9 @@ def record_worker_loss(policy, reason, *, pid=None, bound=False):{'reason': str(reason)[:200], 'claim_lock': context['claim_lock'],'run_id': run_id, 'pid': pid, 'bound': bool(bound)},run_id=run_id)- except Exception as exc:- # The loss must reach its caller, never be masked; the traceback is the only place a- # failing board write is diagnosable (the caller keeps the original loss either way).+ except Exception:import logging- logging.getLogger(__name__).exception('kanban worker_lost record failed: %s',- type(exc).__name__)+ logging.getLogger(__name__).exception('kanban worker_lost record failed')def _bound_worker_matches(conn, context):diff --git a/gateway/session_managed_worker.py b/gateway/session_managed_worker.pyindex 0c361d6e392..97a4dc9f75b 100644--- a/gateway/session_managed_worker.py+++ b/gateway/session_managed_worker.py@@ -24,6 +24,9 @@ HELLO_SECONDS = 60# After Stop the child interrupts its agent and emits its result; the owner terminates a# child that has not acknowledged within this window instead of waiting on the pipe forever.STOP_ACK_SECONDS = 30+# How long a lost attempt's board note may hold up settling its admission (the note is+# advisory; a busy board's write finishes in the background).+LOSS_RECORD_WAIT_S = 10# Own session (POSIX): a worker whose owner died kills its whole group (managed_worker# _die_with_owner); on Windows it exits itself.WORKER_POPEN = {'start_new_session': True, 'close_fds': True}@@ -545,21 +548,37 @@ def _stopped_before_spawn(authority, ref, row):return True-async def _record_lost_attempt(policy, reason, scope):- """Leave a BOARD disposition for a lost attempt, not only the log line above.+def _record_lost_attempt(authority, policy, exc, reason, scope):+ """Start the board disposition (``worker_lost``) for a lost Kanban card attempt; the task, or None.- ``worker.adopt`` precedes ``bind_worker_context``, so a worker that dies in the handshake- never writes ``worker_bound``/``worker_result`` and the card's run learns nothing- (2026-10-22: 49 minutes of "in progress" on a corpse). A deliberate Stop or a superseded- generation is not a loss, and a turn with no board has nowhere to write.+ A worker that dies in ``worker.adopt`` never writes ``worker_bound``/``worker_result``, so+ without it the card's run learns nothing from its own loss. A deliberate Stop or a+ superseded generation is not a loss. Tracked, so retirement joins the write."""- if not getattr(policy, 'kanban_json', None):- return- if reason in ('managed_worker_stopped', 'stale_generation', 'CancelledError'):- return+ if (not getattr(policy, 'kanban_json', None) or isinstance(exc, asyncio.CancelledError)+ or reason in ('managed_worker_stopped', 'stale_generation')):+ return Nonefrom gateway.session_kanban import record_worker_loss- await asyncio.to_thread(record_worker_loss, policy, reason,- pid=(scope or {}).get('pid'), bound=scope is not None)+ from gateway.session_runtime_workers import track_mutation+ return track_mutation(authority, asyncio.to_thread(partial(record_worker_loss, policy, reason,+ pid=(scope or {}).get('pid'), bound=scope is not None)))+++async def _settle_lost(authority, row, scope, recording):+ """Settle a lost worker's admission ``unknown`` once its board note (``recording``) landed.++ The note comes first: the submitter exits on the settled 'unknown' and the crash sweep then+ books the bound interpreter, so a later write is dropped and an exit-0 worker_result reads+ as a protocol violation. A busy board holds this up ``LOSS_RECORD_WAIT_S`` at most (the+ write carries on, tracked), and a cancellation during the wait still settles.+ """+ from gateway.session_worker_reservation import lose_admission_worker+ from gateway.session_runtime_workers import tracked_write+ try:+ if recording is not None:+ await asyncio.wait({recording}, timeout=LOSS_RECORD_WAIT_S)+ finally:+ await tracked_write(authority, partial(lose_admission_worker, authority, row, scope))async def _spawn_registered_worker(authority, ref, row):@@ -694,17 +713,15 @@ async def execute_managed(authority, ref, row, policy):import loggingreason = exc.reason if isinstance(exc, RuntimeStoreError) else type(exc).__name__logging.getLogger(__name__).warning('Managed worker lost: %s', reason)- await _record_lost_attempt(policy, reason, scope)if scope is None:if isinstance(exc, asyncio.CancelledError) or (isinstance(exc, RuntimeStoreError) and exc.reason == 'managed_worker_stopped'):# Stopped before the child ever received its bootstrap: nothing executed, so# this settles like an ordinary interrupted turn (the finally kills the child).return _interrupted_before_bootstrap(authority, row, worker.stop_cause)+ _record_lost_attempt(authority, policy, exc, reason, scope) # nothing bound: a late note is dropped, never misbookedraise- from gateway.session_worker_reservation import lose_admission_worker- from gateway.session_runtime_workers import tracked_write- await tracked_write(authority, partial(lose_admission_worker, authority, row, scope))+ await _settle_lost(authority, row, scope, _record_lost_attempt(authority, policy, exc, reason, scope))live = authority.sessions[ref.session_id]with live.event_stream.lock:live.controls.snapshot(ref.session_id, None)diff --git a/tests/gateway/test_managed_worker_unknown_status.py b/tests/gateway/test_managed_worker_unknown_status.pyindex b59abbbef13..3e979e0cdc0 100644--- a/tests/gateway/test_managed_worker_unknown_status.py+++ b/tests/gateway/test_managed_worker_unknown_status.py@@ -21,8 +21,8 @@ for line in sys.stdin.buffer:"""[email protected]("linux")-def test_unknown_worker_loss_publishes_error_status(tmp_path, monkeypatch):+def _authority_with_a_lost_worker(tmp_path, monkeypatch):+ """A real authority whose next managed turn's child says hello, then sends a bogus frame."""from gateway import session_managed_worker as managedfrom gateway.session_authority import LiveSession, SessionAuthorityfrom gateway.session_contract import Submission@@ -52,9 +52,19 @@ def test_unknown_worker_loss_publishes_error_status(tmp_path, monkeypatch):monkeypatch.setattr(managed.subprocess, 'Popen', Child)policy = build_policy({'cwd': str(tmp_path), 'model': 'm'}, {}, profile_terminal=False)- async def scenario():+ async def claim():await authority.submit(actor, Submission('turn', ref, {'text': 'go'}, 'queue'))- row = claim_session_input(db, epoch=authority.epoch, session_id='s')+ return claim_session_input(db, epoch=authority.epoch, session_id='s')+ return authority, ref, policy, claim, published, db++[email protected]("linux")+def test_unknown_worker_loss_publishes_error_status(tmp_path, monkeypatch):+ from gateway import session_managed_worker as managed+ authority, ref, policy, claim, published, db = _authority_with_a_lost_worker(tmp_path, monkeypatch)++ async def scenario():+ row = await claim()with pytest.raises(managed.ManagedExecutionUnknown):async with asyncio.timeout(30):await managed.execute_managed(authority, ref, row, policy)@@ -62,3 +72,73 @@ def test_unknown_worker_loss_publishes_error_status(tmp_path, monkeypatch):asyncio.run(scenario())complete = [p for kind, p in published if kind == 'message.complete' and p.get('outcome') == 'unknown']assert complete and complete[0]['status'] == 'error', published+++def _kanban_loss_recorder(monkeypatch, policy, started, release, write_s):+ """Record the order of the board's worker_lost write (held until ``release``) and the+ admission's settlement."""+ import dataclasses+ from gateway import session_kanban, session_worker_reservation+ order = []++ def record(*_, **__):+ started.set()+ release.wait(write_s)+ order.append('worker_lost')+ settle = session_worker_reservation.lose_admission_worker+ monkeypatch.setattr(session_kanban, 'record_worker_loss', record)+ monkeypatch.setattr(session_worker_reservation, 'lose_admission_worker',+ lambda *a, **kw: (order.append('settled'), settle(*a, **kw))[1])+ return dataclasses.replace(policy, kanban_json='{"task_id": "t"}'), order++[email protected]("linux")+def test_a_lost_card_worker_is_on_the_board_before_its_admission_settles(tmp_path, monkeypatch):+ """The settled 'unknown' lets the submitter exit and the crash sweep book the run, so the+ worker_lost note must already be there."""+ from gateway import session_managed_worker as managed+ import threading+ authority, ref, policy, claim, _published, db = _authority_with_a_lost_worker(tmp_path, monkeypatch)+ # ``release`` is never set: the write takes its full 1 s, slower than the settlement.+ started, release = threading.Event(), threading.Event()+ policy, order = _kanban_loss_recorder(monkeypatch, policy, started, release, write_s=1)++ async def scenario():+ row = await claim()+ authority.executing[ref.session_id] = row['generation'] # as the drain stamps it+ with pytest.raises(managed.ManagedExecutionUnknown):+ async with asyncio.timeout(30):+ await managed.execute_managed(authority, ref, row, policy)+ with db:+ asyncio.run(scenario())+ assert order == ['worker_lost', 'settled']++[email protected]("linux")+def test_a_cancellation_while_the_board_is_busy_still_settles_the_admission(tmp_path, monkeypatch):+ import threading+ from tests.waiting import async_wait_until+ from gateway import session_managed_worker as managed+ authority, ref, policy, claim, _published, db = _authority_with_a_lost_worker(tmp_path, monkeypatch)+ started, busy_board_frees = threading.Event(), threading.Event()+ policy, order = _kanban_loss_recorder(monkeypatch, policy, started, busy_board_frees, write_s=30)++ async def scenario():+ row = await claim()+ authority.executing[ref.session_id] = row['generation']+ drain = asyncio.create_task(managed.execute_managed(authority, ref, row, policy))+ await async_wait_until(started.is_set, timeout=30)+ drain.cancel() # while the loss handler waits on the board write+ with pytest.raises(asyncio.CancelledError):+ await drain+ busy_board_frees.set()+ await asyncio.gather(*authority._mutation_tasks)+ return row+ with db:+ row = asyncio.run(scenario())+ assert order == ['settled', 'worker_lost']+ import sqlite3+ with sqlite3.connect(tmp_path / 'state.db') as ledger:+ status = ledger.execute('SELECT status FROM session_admissions WHERE admission_id=?',+ (row['admission_id'],)).fetchone()[0]+ assert status == 'unknown' # settled, not left 'started' by the cancellationdiff --git a/tests/hermes_cli/test_kanban_managed_worker_loss.py b/tests/hermes_cli/test_kanban_managed_worker_loss.pyindex d93c37e7543..3284917293e 100644--- a/tests/hermes_cli/test_kanban_managed_worker_loss.py+++ b/tests/hermes_cli/test_kanban_managed_worker_loss.py@@ -223,10 +223,6 @@ def test_the_incident_shape_names_the_loss_on_the_reclaim(board):run = board.run(run_id)assert run["outcome"] == "reclaimed"assert "managed worker lost: managed_worker_failed" in run["error"]- event = board.conn.execute(- "SELECT payload FROM task_events WHERE task_id = 't_730' AND kind = 'reclaimed' "- "ORDER BY id DESC LIMIT 1").fetchone()- assert json.loads(event["payload"])["attempt_note"] == "managed worker lost: managed_worker_failed"def test_a_settled_receipt_with_a_live_gateway_owner_no_longer_holds_the_claim(board):@@ -266,16 +262,25 @@ def _owner_policy(board, tid, run_id):}))-def test_the_owner_records_the_loss_on_the_board_and_one_tick_disposes_of_it(board):- from gateway.session_kanban import record_worker_loss+async def test_the_owner_records_the_loss_on_the_board_and_one_tick_disposes_of_it(board):+ """Through the gateway's loss handler: a Stop, a superseded generation or a cancellation+ records nothing; a real loss leaves one worker_lost event that the next tick books."""+ import asyncio+ from types import SimpleNamespace+ from gateway.session_managed_worker import _record_lost_attemptrun_id = board.running("t_owner", claim_expires_in=900, heartbeat_age=30)policy = _owner_policy(board, "t_owner", run_id)- record_worker_loss(policy, "managed_worker_failed", pid=board.state("t_owner")["worker_pid"],- bound=False)- event = board.conn.execute(- "SELECT payload FROM task_events WHERE task_id='t_owner' AND kind='worker_lost' "- "ORDER BY id DESC LIMIT 1").fetchone()- assert json.loads(event["payload"])["reason"] == "managed_worker_failed"+ authority = SimpleNamespace()+ for exc, reason in [(RuntimeError(), "managed_worker_stopped"), (RuntimeError(), "stale_generation"),+ (asyncio.CancelledError(), "CancelledError"),+ (RuntimeError(), "managed_worker_failed")]:+ recording = _record_lost_attempt(authority, policy, exc, reason, None)+ if recording is not None:+ await recording+ events = [json.loads(r[0]) for r in board.conn.execute(+ "SELECT payload FROM task_events WHERE task_id='t_owner' AND kind='worker_lost'")]+ assert [e["reason"] for e in events] == ["managed_worker_failed"]board.tick(failure_limit=2)run = board.run(run_id)@@ -289,10 +294,13 @@ def test_a_loss_recorded_against_a_superseded_run_is_dropped(board):from gateway.session_kanban import record_worker_lossrun_id = board.running("t_stale", claim_expires_in=900, heartbeat_age=30)policy = _owner_policy(board, "t_stale", run_id)++ def count():+ return board.conn.execute(+ "SELECT count(*) FROM task_events WHERE task_id='t_stale' AND kind='worker_lost'").fetchone()[0]+ record_worker_loss(policy, "managed_worker_failed", pid=1234, bound=False)+ assert count() == 1board.conn.execute("UPDATE tasks SET status='ready', current_run_id=NULL WHERE id='t_stale'")board.conn.commit()record_worker_loss(policy, "managed_worker_failed", pid=1234, bound=False)- n = board.conn.execute(- "SELECT count(*) FROM task_events WHERE task_id='t_stale' AND kind='worker_lost'"- ).fetchone()[0]- assert n == 0+ assert count() == 1