Skip to content

The agent that grows with you

fix(gateway): a lost managed worker's board note lands before its admission settles, boundedly

kshitijk4poor08527eef1ce3
diff --git a/gateway/session_kanban.py b/gateway/session_kanban.py
index 93e58088498..31ab6a1b617 100644
--- a/gateway/session_kanban.py
+++ b/gateway/session_kanban.py
@@ -313,23 +313,15 @@ def _record_worker_result(context, code, last_output):
def record_worker_loss(policy, reason, *, pid=None, bound=False):
- """Durably record that this attempt's managed execution was LOST — on the card's board.
-
- ``worker.adopt`` runs before ``bind_worker_context``, so a worker that dies in the adopt
- handshake writes neither ``worker_bound`` nor ``worker_result``: the run learns nothing
- from its own loss, and the dispatcher can only guess from a dead pid. 2026-10-22
- (t_f5fad63f) the gateway logged ``Managed worker lost: managed_worker_failed`` 24 s in
- and the card still said ``running`` 49 minutes later. A warning in a log file is not a
- disposition, so the owner writes the disposition where the board can read it.
-
- Best-effort and run-scoped: a loss recorded against a run the card has already moved
- past is not evidence about the current attempt, so it is dropped. Never raises — the
- loss itself must reach the caller.
+ """Append a run-scoped ``worker_lost`` event for this attempt to the card's board.
+
+ Dropped when the card has moved past the run (it says nothing about the current attempt).
+ Never raises: the caller's own loss handling must not depend on the board write.
"""
- context = json.loads(getattr(policy, 'kanban_json', None) or 'null')
- if not context:
- return
try:
+ context = json.loads(getattr(policy, 'kanban_json', None) or 'null')
+ if not context:
+ return
from hermes_cli.kanban_db_connect import connect_closing
from hermes_cli import kanban_db as kb
run_id = int(context['run_id'])
@@ -342,12 +334,9 @@ def record_worker_loss(policy, reason, *, pid=None, bound=False):
{'reason': str(reason)[:200], 'claim_lock': context['claim_lock'],
'run_id': run_id, 'pid': pid, 'bound': bool(bound)},
run_id=run_id)
- except Exception as exc:
- # The loss must reach its caller, never be masked; the traceback is the only place a
- # failing board write is diagnosable (the caller keeps the original loss either way).
+ except Exception:
import logging
- logging.getLogger(__name__).exception('kanban worker_lost record failed: %s',
- type(exc).__name__)
+ logging.getLogger(__name__).exception('kanban worker_lost record failed')
def _bound_worker_matches(conn, context):
diff --git a/gateway/session_managed_worker.py b/gateway/session_managed_worker.py
index 0c361d6e392..97a4dc9f75b 100644
--- a/gateway/session_managed_worker.py
+++ b/gateway/session_managed_worker.py
@@ -24,6 +24,9 @@ HELLO_SECONDS = 60
# After Stop the child interrupts its agent and emits its result; the owner terminates a
# child that has not acknowledged within this window instead of waiting on the pipe forever.
STOP_ACK_SECONDS = 30
+# How long a lost attempt's board note may hold up settling its admission (the note is
+# advisory; a busy board's write finishes in the background).
+LOSS_RECORD_WAIT_S = 10
# Own session (POSIX): a worker whose owner died kills its whole group (managed_worker
# _die_with_owner); on Windows it exits itself.
WORKER_POPEN = {'start_new_session': True, 'close_fds': True}
@@ -545,21 +548,37 @@ def _stopped_before_spawn(authority, ref, row):
return True
-async def _record_lost_attempt(policy, reason, scope):
- """Leave a BOARD disposition for a lost attempt, not only the log line above.
+def _record_lost_attempt(authority, policy, exc, reason, scope):
+ """Start the board disposition (``worker_lost``) for a lost Kanban card attempt; the task, or None.
- ``worker.adopt`` precedes ``bind_worker_context``, so a worker that dies in the handshake
- never writes ``worker_bound``/``worker_result`` and the card's run learns nothing
- (2026-10-22: 49 minutes of "in progress" on a corpse). A deliberate Stop or a superseded
- generation is not a loss, and a turn with no board has nowhere to write.
+ A worker that dies in ``worker.adopt`` never writes ``worker_bound``/``worker_result``, so
+ without it the card's run learns nothing from its own loss. A deliberate Stop or a
+ superseded generation is not a loss. Tracked, so retirement joins the write.
"""
- if not getattr(policy, 'kanban_json', None):
- return
- if reason in ('managed_worker_stopped', 'stale_generation', 'CancelledError'):
- return
+ if (not getattr(policy, 'kanban_json', None) or isinstance(exc, asyncio.CancelledError)
+ or reason in ('managed_worker_stopped', 'stale_generation')):
+ return None
from gateway.session_kanban import record_worker_loss
- await asyncio.to_thread(record_worker_loss, policy, reason,
- pid=(scope or {}).get('pid'), bound=scope is not None)
+ from gateway.session_runtime_workers import track_mutation
+ return track_mutation(authority, asyncio.to_thread(partial(record_worker_loss, policy, reason,
+ pid=(scope or {}).get('pid'), bound=scope is not None)))
+
+
+async def _settle_lost(authority, row, scope, recording):
+ """Settle a lost worker's admission ``unknown`` once its board note (``recording``) landed.
+
+ The note comes first: the submitter exits on the settled 'unknown' and the crash sweep then
+ books the bound interpreter, so a later write is dropped and an exit-0 worker_result reads
+ as a protocol violation. A busy board holds this up ``LOSS_RECORD_WAIT_S`` at most (the
+ write carries on, tracked), and a cancellation during the wait still settles.
+ """
+ from gateway.session_worker_reservation import lose_admission_worker
+ from gateway.session_runtime_workers import tracked_write
+ try:
+ if recording is not None:
+ await asyncio.wait({recording}, timeout=LOSS_RECORD_WAIT_S)
+ finally:
+ await tracked_write(authority, partial(lose_admission_worker, authority, row, scope))
async def _spawn_registered_worker(authority, ref, row):
@@ -694,17 +713,15 @@ async def execute_managed(authority, ref, row, policy):
import logging
reason = exc.reason if isinstance(exc, RuntimeStoreError) else type(exc).__name__
logging.getLogger(__name__).warning('Managed worker lost: %s', reason)
- await _record_lost_attempt(policy, reason, scope)
if scope is None:
if isinstance(exc, asyncio.CancelledError) or (
isinstance(exc, RuntimeStoreError) and exc.reason == 'managed_worker_stopped'):
# Stopped before the child ever received its bootstrap: nothing executed, so
# this settles like an ordinary interrupted turn (the finally kills the child).
return _interrupted_before_bootstrap(authority, row, worker.stop_cause)
+ _record_lost_attempt(authority, policy, exc, reason, scope) # nothing bound: a late note is dropped, never misbooked
raise
- from gateway.session_worker_reservation import lose_admission_worker
- from gateway.session_runtime_workers import tracked_write
- await tracked_write(authority, partial(lose_admission_worker, authority, row, scope))
+ await _settle_lost(authority, row, scope, _record_lost_attempt(authority, policy, exc, reason, scope))
live = authority.sessions[ref.session_id]
with live.event_stream.lock:
live.controls.snapshot(ref.session_id, None)
diff --git a/tests/gateway/test_managed_worker_unknown_status.py b/tests/gateway/test_managed_worker_unknown_status.py
index b59abbbef13..3e979e0cdc0 100644
--- a/tests/gateway/test_managed_worker_unknown_status.py
+++ b/tests/gateway/test_managed_worker_unknown_status.py
@@ -21,8 +21,8 @@ for line in sys.stdin.buffer:
"""
-def test_unknown_worker_loss_publishes_error_status(tmp_path, monkeypatch):
+def _authority_with_a_lost_worker(tmp_path, monkeypatch):
+ """A real authority whose next managed turn's child says hello, then sends a bogus frame."""
from gateway import session_managed_worker as managed
from gateway.session_authority import LiveSession, SessionAuthority
from gateway.session_contract import Submission
@@ -52,9 +52,19 @@ def test_unknown_worker_loss_publishes_error_status(tmp_path, monkeypatch):
monkeypatch.setattr(managed.subprocess, 'Popen', Child)
policy = build_policy({'cwd': str(tmp_path), 'model': 'm'}, {}, profile_terminal=False)
- async def scenario():
+ async def claim():
await authority.submit(actor, Submission('turn', ref, {'text': 'go'}, 'queue'))
- row = claim_session_input(db, epoch=authority.epoch, session_id='s')
+ return claim_session_input(db, epoch=authority.epoch, session_id='s')
+ return authority, ref, policy, claim, published, db
+
+
+def test_unknown_worker_loss_publishes_error_status(tmp_path, monkeypatch):
+ from gateway import session_managed_worker as managed
+ authority, ref, policy, claim, published, db = _authority_with_a_lost_worker(tmp_path, monkeypatch)
+
+ async def scenario():
+ row = await claim()
with pytest.raises(managed.ManagedExecutionUnknown):
async with asyncio.timeout(30):
await managed.execute_managed(authority, ref, row, policy)
@@ -62,3 +72,73 @@ def test_unknown_worker_loss_publishes_error_status(tmp_path, monkeypatch):
asyncio.run(scenario())
complete = [p for kind, p in published if kind == 'message.complete' and p.get('outcome') == 'unknown']
assert complete and complete[0]['status'] == 'error', published
+
+
+def _kanban_loss_recorder(monkeypatch, policy, started, release, write_s):
+ """Record the order of the board's worker_lost write (held until ``release``) and the
+ admission's settlement."""
+ import dataclasses
+ from gateway import session_kanban, session_worker_reservation
+ order = []
+
+ def record(*_, **__):
+ started.set()
+ release.wait(write_s)
+ order.append('worker_lost')
+ settle = session_worker_reservation.lose_admission_worker
+ monkeypatch.setattr(session_kanban, 'record_worker_loss', record)
+ monkeypatch.setattr(session_worker_reservation, 'lose_admission_worker',
+ lambda *a, **kw: (order.append('settled'), settle(*a, **kw))[1])
+ return dataclasses.replace(policy, kanban_json='{"task_id": "t"}'), order
+
+
+def test_a_lost_card_worker_is_on_the_board_before_its_admission_settles(tmp_path, monkeypatch):
+ """The settled 'unknown' lets the submitter exit and the crash sweep book the run, so the
+ worker_lost note must already be there."""
+ from gateway import session_managed_worker as managed
+ import threading
+ authority, ref, policy, claim, _published, db = _authority_with_a_lost_worker(tmp_path, monkeypatch)
+ # ``release`` is never set: the write takes its full 1 s, slower than the settlement.
+ started, release = threading.Event(), threading.Event()
+ policy, order = _kanban_loss_recorder(monkeypatch, policy, started, release, write_s=1)
+
+ async def scenario():
+ row = await claim()
+ authority.executing[ref.session_id] = row['generation'] # as the drain stamps it
+ with pytest.raises(managed.ManagedExecutionUnknown):
+ async with asyncio.timeout(30):
+ await managed.execute_managed(authority, ref, row, policy)
+ with db:
+ asyncio.run(scenario())
+ assert order == ['worker_lost', 'settled']
+
+
+def test_a_cancellation_while_the_board_is_busy_still_settles_the_admission(tmp_path, monkeypatch):
+ import threading
+ from tests.waiting import async_wait_until
+ from gateway import session_managed_worker as managed
+ authority, ref, policy, claim, _published, db = _authority_with_a_lost_worker(tmp_path, monkeypatch)
+ started, busy_board_frees = threading.Event(), threading.Event()
+ policy, order = _kanban_loss_recorder(monkeypatch, policy, started, busy_board_frees, write_s=30)
+
+ async def scenario():
+ row = await claim()
+ authority.executing[ref.session_id] = row['generation']
+ drain = asyncio.create_task(managed.execute_managed(authority, ref, row, policy))
+ await async_wait_until(started.is_set, timeout=30)
+ drain.cancel() # while the loss handler waits on the board write
+ with pytest.raises(asyncio.CancelledError):
+ await drain
+ busy_board_frees.set()
+ await asyncio.gather(*authority._mutation_tasks)
+ return row
+ with db:
+ row = asyncio.run(scenario())
+ assert order == ['settled', 'worker_lost']
+ import sqlite3
+ with sqlite3.connect(tmp_path / 'state.db') as ledger:
+ status = ledger.execute('SELECT status FROM session_admissions WHERE admission_id=?',
+ (row['admission_id'],)).fetchone()[0]
+ assert status == 'unknown' # settled, not left 'started' by the cancellation
diff --git a/tests/hermes_cli/test_kanban_managed_worker_loss.py b/tests/hermes_cli/test_kanban_managed_worker_loss.py
index d93c37e7543..3284917293e 100644
--- a/tests/hermes_cli/test_kanban_managed_worker_loss.py
+++ b/tests/hermes_cli/test_kanban_managed_worker_loss.py
@@ -223,10 +223,6 @@ def test_the_incident_shape_names_the_loss_on_the_reclaim(board):
run = board.run(run_id)
assert run["outcome"] == "reclaimed"
assert "managed worker lost: managed_worker_failed" in run["error"]
- event = board.conn.execute(
- "SELECT payload FROM task_events WHERE task_id = 't_730' AND kind = 'reclaimed' "
- "ORDER BY id DESC LIMIT 1").fetchone()
- assert json.loads(event["payload"])["attempt_note"] == "managed worker lost: managed_worker_failed"
def test_a_settled_receipt_with_a_live_gateway_owner_no_longer_holds_the_claim(board):
@@ -266,16 +262,25 @@ def _owner_policy(board, tid, run_id):
}))
-def test_the_owner_records_the_loss_on_the_board_and_one_tick_disposes_of_it(board):
- from gateway.session_kanban import record_worker_loss
+async def test_the_owner_records_the_loss_on_the_board_and_one_tick_disposes_of_it(board):
+ """Through the gateway's loss handler: a Stop, a superseded generation or a cancellation
+ records nothing; a real loss leaves one worker_lost event that the next tick books."""
+ import asyncio
+ from types import SimpleNamespace
+ from gateway.session_managed_worker import _record_lost_attempt
run_id = board.running("t_owner", claim_expires_in=900, heartbeat_age=30)
policy = _owner_policy(board, "t_owner", run_id)
- record_worker_loss(policy, "managed_worker_failed", pid=board.state("t_owner")["worker_pid"],
- bound=False)
- event = board.conn.execute(
- "SELECT payload FROM task_events WHERE task_id='t_owner' AND kind='worker_lost' "
- "ORDER BY id DESC LIMIT 1").fetchone()
- assert json.loads(event["payload"])["reason"] == "managed_worker_failed"
+ authority = SimpleNamespace()
+ for exc, reason in [(RuntimeError(), "managed_worker_stopped"), (RuntimeError(), "stale_generation"),
+ (asyncio.CancelledError(), "CancelledError"),
+ (RuntimeError(), "managed_worker_failed")]:
+ recording = _record_lost_attempt(authority, policy, exc, reason, None)
+ if recording is not None:
+ await recording
+ events = [json.loads(r[0]) for r in board.conn.execute(
+ "SELECT payload FROM task_events WHERE task_id='t_owner' AND kind='worker_lost'")]
+ assert [e["reason"] for e in events] == ["managed_worker_failed"]
board.tick(failure_limit=2)
run = board.run(run_id)
@@ -289,10 +294,13 @@ def test_a_loss_recorded_against_a_superseded_run_is_dropped(board):
from gateway.session_kanban import record_worker_loss
run_id = board.running("t_stale", claim_expires_in=900, heartbeat_age=30)
policy = _owner_policy(board, "t_stale", run_id)
+
+ def count():
+ return board.conn.execute(
+ "SELECT count(*) FROM task_events WHERE task_id='t_stale' AND kind='worker_lost'").fetchone()[0]
+ record_worker_loss(policy, "managed_worker_failed", pid=1234, bound=False)
+ assert count() == 1
board.conn.execute("UPDATE tasks SET status='ready', current_run_id=NULL WHERE id='t_stale'")
board.conn.commit()
record_worker_loss(policy, "managed_worker_failed", pid=1234, bound=False)
- n = board.conn.execute(
- "SELECT count(*) FROM task_events WHERE task_id='t_stale' AND kind='worker_lost'"
- ).fetchone()[0]
- assert n == 0
+ assert count() == 1