Skip to content

The agent that grows with you

fix(kanban): bound kanban.run's admission wait by the board lock budget

Teknium3d5cf0cc58bd
diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.py
index cfed689b9e2..72521866b75 100644
--- a/hermes_cli/kanban_worker_client.py
+++ b/hermes_cli/kanban_worker_client.py
@@ -8,7 +8,6 @@ except ModuleNotFoundError as exc:
raise
import asyncio
-import math
import os
from pathlib import Path
import sys
@@ -20,6 +19,17 @@ import time
# until an owner is ready, so waiting can never run the claim twice.
OWNER_WAIT_S = 180.0
_TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})
+# Past the board lock wait: the owner's session setup and receipt before and after the write.
+ADMISSION_ANSWER_MARGIN_S = 60.0
+
+
+def admission_answer_budget():
+ """Seconds to wait for ``kanban.run``'s answer: the longest the owner's admission write can
+ wait on the board lock (``write_txn`` retries ``BEGIN IMMEDIATE`` ``_BUSY_MAX_RETRIES`` times,
+ each under the busy timeout) plus ``ADMISSION_ANSWER_MARGIN_S``."""
+ from hermes_cli import kanban_db_connect
+ lock_wait = (kanban_db_connect._BUSY_MAX_RETRIES + 1) * kanban_db_connect._resolve_busy_timeout_ms() / 1000
+ return lock_wait + ADMISSION_ANSWER_MARGIN_S
async def run(params, board_db):
@@ -40,13 +50,14 @@ async def run(params, board_db):
async def _submit_claim(client, params, board_db):
from hermes_cli.gateway_client import GatewayClientError
db = str(await asyncio.to_thread(Path(board_db).resolve))
- # No deadline on the admission answer: the owner checks the claim and writes its
- # ``owner_admitted`` marker in one board write transaction first, and the board (the
- # cross-profile dispatch bus) has a 120 s busy timeout. Giving up at the 30 s default left
- # the owner admitting an attempt whose submitter had exited, so the later-bound worker's
- # submitter-liveness watcher ended it and the admission was left ``unknown``. A hang-up still
- # fails the wait; the dispatcher's runtime cap and reclaim bound a stuck submitter.
- accepted = await client.rpc('kanban.run', _timeout=math.inf, **dict(params, db=db))
+ # The owner answers only after it checks the claim and writes its ``owner_admitted`` marker
+ # in one board write transaction. The board (the cross-profile dispatch bus) can make that
+ # wait out its busy timeout, so the 30 s default left the owner admitting an attempt whose
+ # submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the
+ # admission was left ``unknown``). The wait is still finite: once the marker commits,
+ # ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner
+ # that stalls after admitting is bounded only here. A hang-up fails the wait sooner.
+ accepted = await client.rpc('kanban.run', _timeout=admission_answer_budget(), **dict(params, db=db))
sid, receipt = accepted['session_id'], accepted['receipt']
print(f'Session: {sid}', file=sys.stderr)
while receipt['status'] not in {'terminal', 'unknown'}:
@@ -81,7 +92,8 @@ def _submit():
board_db = os.environ['HERMES_KANBAN_DB']
# Owner startup must never inherit this worker's task/tool/goal identity.
for key in list(os.environ):
- if key.startswith('HERMES_KANBAN_') and key != 'HERMES_KANBAN_HOME':
+ # The busy timeout is a host tuning knob, not task identity: the answer budget reads it.
+ if key.startswith('HERMES_KANBAN_') and key not in ('HERMES_KANBAN_HOME', 'HERMES_KANBAN_BUSY_TIMEOUT_MS'):
os.environ.pop(key)
for key in ('HERMES_SESSION_SOURCE', 'HERMES_TENANT', 'TERMINAL_CWD'):
os.environ.pop(key, None)
diff --git a/tests/hermes_cli/test_kanban_worker_admission_wait.py b/tests/hermes_cli/test_kanban_worker_admission_wait.py
index d259f101c8c..e46dc238b12 100644
--- a/tests/hermes_cli/test_kanban_worker_admission_wait.py
+++ b/tests/hermes_cli/test_kanban_worker_admission_wait.py
@@ -54,3 +54,35 @@ async def test_a_slow_admission_answer_is_awaited_not_failed(monkeypatch, tmp_pa
# One submission: the wait never resends the claim, and the budget never goes on the wire.
assert [r['method'] for r in owner.sent] == ['kanban.run']
assert all('_timeout' not in r['params'] for r in owner.sent)
+
+
+class _StalledOwner(_Owner):
+ """Admits (its ``owner_admitted`` marker is committed) and then never answers or hangs up."""
+
+ async def send(self, raw):
+ self.sent.append(json.loads(raw))
+
+
+async def test_an_owner_that_stalls_after_admitting_fails_the_wait_within_the_lock_budget(monkeypatch, tmp_path):
+ """Once ``owner_admitted`` commits, ``enforce_max_runtime`` and TTL reclaim both skip the run
+ (``owner_reclaim_paused``), so nothing outside the submitter bounds it: the wait for the
+ answer must end once the board's real lock wait (``write_txn``'s retried ``BEGIN IMMEDIATE``
+ under the busy timeout) plus a margin has passed. Time is scaled 1000x."""
+ from hermes_cli import kanban_db_connect
+ monkeypatch.setenv('HERMES_KANBAN_BUSY_TIMEOUT_MS', '1000')
+ budget = kanban_worker_client.admission_answer_budget()
+ assert budget == (kanban_db_connect._BUSY_MAX_RETRIES + 1) * 1.0 + kanban_worker_client.ADMISSION_ANSWER_MARGIN_S
+ real_wait_for, budgets = asyncio.wait_for, []
+
+ def scaled(fut, timeout):
+ budgets.append(timeout)
+ return real_wait_for(fut, None if timeout is None else timeout / 1000)
+ monkeypatch.setattr(asyncio, 'wait_for', scaled)
+ owner = _StalledOwner()
+ params = {'board': 'b', 'task_id': 't1', 'run_id': 1, 'claim_lock': 'lock'}
+ async with GatewayClient(owner) as client:
+ with pytest.raises(TimeoutError):
+ # The outer 5 s guard only stops an unbounded wait from hanging the test.
+ await real_wait_for(kanban_worker_client._submit_claim(client, params, str(tmp_path / 'kanban.db')), 5)
+ assert budgets == [budget] and [r['method'] for r in owner.sent] == ['kanban.run']