The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(kanban): bound kanban.run's admission wait by the board lock budget
Teknium3d5cf0cc58bd
diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.pyindex cfed689b9e2..72521866b75 100644--- a/hermes_cli/kanban_worker_client.py+++ b/hermes_cli/kanban_worker_client.py@@ -8,7 +8,6 @@ except ModuleNotFoundError as exc:raiseimport asyncio-import mathimport osfrom pathlib import Pathimport sys@@ -20,6 +19,17 @@ import time# until an owner is ready, so waiting can never run the claim twice.OWNER_WAIT_S = 180.0_TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})+# Past the board lock wait: the owner's session setup and receipt before and after the write.+ADMISSION_ANSWER_MARGIN_S = 60.0+++def admission_answer_budget():+ """Seconds to wait for ``kanban.run``'s answer: the longest the owner's admission write can+ wait on the board lock (``write_txn`` retries ``BEGIN IMMEDIATE`` ``_BUSY_MAX_RETRIES`` times,+ each under the busy timeout) plus ``ADMISSION_ANSWER_MARGIN_S``."""+ from hermes_cli import kanban_db_connect+ lock_wait = (kanban_db_connect._BUSY_MAX_RETRIES + 1) * kanban_db_connect._resolve_busy_timeout_ms() / 1000+ return lock_wait + ADMISSION_ANSWER_MARGIN_Sasync def run(params, board_db):@@ -40,13 +50,14 @@ async def run(params, board_db):async def _submit_claim(client, params, board_db):from hermes_cli.gateway_client import GatewayClientErrordb = str(await asyncio.to_thread(Path(board_db).resolve))- # No deadline on the admission answer: the owner checks the claim and writes its- # ``owner_admitted`` marker in one board write transaction first, and the board (the- # cross-profile dispatch bus) has a 120 s busy timeout. Giving up at the 30 s default left- # the owner admitting an attempt whose submitter had exited, so the later-bound worker's- # submitter-liveness watcher ended it and the admission was left ``unknown``. A hang-up still- # fails the wait; the dispatcher's runtime cap and reclaim bound a stuck submitter.- accepted = await client.rpc('kanban.run', _timeout=math.inf, **dict(params, db=db))+ # The owner answers only after it checks the claim and writes its ``owner_admitted`` marker+ # in one board write transaction. The board (the cross-profile dispatch bus) can make that+ # wait out its busy timeout, so the 30 s default left the owner admitting an attempt whose+ # submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the+ # admission was left ``unknown``). The wait is still finite: once the marker commits,+ # ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner+ # that stalls after admitting is bounded only here. A hang-up fails the wait sooner.+ accepted = await client.rpc('kanban.run', _timeout=admission_answer_budget(), **dict(params, db=db))sid, receipt = accepted['session_id'], accepted['receipt']print(f'Session: {sid}', file=sys.stderr)while receipt['status'] not in {'terminal', 'unknown'}:@@ -81,7 +92,8 @@ def _submit():board_db = os.environ['HERMES_KANBAN_DB']# Owner startup must never inherit this worker's task/tool/goal identity.for key in list(os.environ):- if key.startswith('HERMES_KANBAN_') and key != 'HERMES_KANBAN_HOME':+ # The busy timeout is a host tuning knob, not task identity: the answer budget reads it.+ if key.startswith('HERMES_KANBAN_') and key not in ('HERMES_KANBAN_HOME', 'HERMES_KANBAN_BUSY_TIMEOUT_MS'):os.environ.pop(key)for key in ('HERMES_SESSION_SOURCE', 'HERMES_TENANT', 'TERMINAL_CWD'):os.environ.pop(key, None)diff --git a/tests/hermes_cli/test_kanban_worker_admission_wait.py b/tests/hermes_cli/test_kanban_worker_admission_wait.pyindex d259f101c8c..e46dc238b12 100644--- a/tests/hermes_cli/test_kanban_worker_admission_wait.py+++ b/tests/hermes_cli/test_kanban_worker_admission_wait.py@@ -54,3 +54,35 @@ async def test_a_slow_admission_answer_is_awaited_not_failed(monkeypatch, tmp_pa# One submission: the wait never resends the claim, and the budget never goes on the wire.assert [r['method'] for r in owner.sent] == ['kanban.run']assert all('_timeout' not in r['params'] for r in owner.sent)+++class _StalledOwner(_Owner):+ """Admits (its ``owner_admitted`` marker is committed) and then never answers or hangs up."""++ async def send(self, raw):+ self.sent.append(json.loads(raw))+++async def test_an_owner_that_stalls_after_admitting_fails_the_wait_within_the_lock_budget(monkeypatch, tmp_path):+ """Once ``owner_admitted`` commits, ``enforce_max_runtime`` and TTL reclaim both skip the run+ (``owner_reclaim_paused``), so nothing outside the submitter bounds it: the wait for the+ answer must end once the board's real lock wait (``write_txn``'s retried ``BEGIN IMMEDIATE``+ under the busy timeout) plus a margin has passed. Time is scaled 1000x."""+ from hermes_cli import kanban_db_connect+ monkeypatch.setenv('HERMES_KANBAN_BUSY_TIMEOUT_MS', '1000')+ budget = kanban_worker_client.admission_answer_budget()+ assert budget == (kanban_db_connect._BUSY_MAX_RETRIES + 1) * 1.0 + kanban_worker_client.ADMISSION_ANSWER_MARGIN_S+ real_wait_for, budgets = asyncio.wait_for, []++ def scaled(fut, timeout):+ budgets.append(timeout)+ return real_wait_for(fut, None if timeout is None else timeout / 1000)+ monkeypatch.setattr(asyncio, 'wait_for', scaled)+ owner = _StalledOwner()+ params = {'board': 'b', 'task_id': 't1', 'run_id': 1, 'claim_lock': 'lock'}+ async with GatewayClient(owner) as client:+ with pytest.raises(TimeoutError):+ # The outer 5 s guard only stops an unbounded wait from hanging the test.+ await real_wait_for(kanban_worker_client._submit_claim(client, params, str(tmp_path / 'kanban.db')), 5)+ assert budgets == [budget] and [r['method'] for r in owner.sent] == ['kanban.run']