The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(kanban): a restarting owner's control socket or handshake failure during a redial also retries
kshitijk4poorf4e042e944df
diff --git a/hermes_cli/gateway_client.py b/hermes_cli/gateway_client.pyindex cbb02af51f4..a20e3fe661a 100644--- a/hermes_cli/gateway_client.py+++ b/hermes_cli/gateway_client.py@@ -98,9 +98,9 @@ class GatewayUnavailableError(GatewayClientError):"""No ready owner to dial: ``state`` is ``ensure_gateway_runtime``'s verdict (``draining`` while anupdate holds the install, ``starting`` past the deadline, ...). Nothing was submitted."""- def __init__(self, message, state):+ def __init__(self, message, state, reason_code=None):super().__init__(message)- self.state = state+ self.state, self.reason_code = state, reason_codeclass EventQueue(asyncio.Queue):@@ -299,10 +299,10 @@ async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 10detail = getattr(result, "detail", None)if result.reason_code == "runtime_exited" and detail:# A multi-line startup report (redacted traceback tail + where the full log is).- raise GatewayUnavailableError(f"Gateway could not start: {detail}", result.state)+ raise GatewayUnavailableError(f"Gateway could not start: {detail}", result.state, result.reason_code)detail = f" ({detail})" if detail else ""raise GatewayUnavailableError(f"Gateway {result.state}: {result.reason_code or 'not_ready'}{detail}",- result.state)+ result.state, result.reason_code)endpoint = result.endpointticket = await asyncio.to_thread(_session_ticket, home, endpoint)url, protocols = gateway_ws_target(endpoint, ticket)diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.pyindex cd2f4d68fbe..447f2808ada 100644--- a/hermes_cli/kanban_worker_client.py+++ b/hermes_cli/kanban_worker_client.py@@ -8,20 +8,30 @@ except ModuleNotFoundError as exc:raiseimport asyncio+from dataclasses import dataclassimport osfrom pathlib import Pathimport sysimport time--# A worker the dispatcher spawns while `hermes update` drains the owner (or while the new owner is-# still starting) waits for it instead of exiting: the condition is the host's, not the card's, and-# a failed exit here spends the card's failure budget on an update window. Nothing is submitted-# until an owner is ready, so waiting can never run the claim twice.+from typing import Optional++# A worker that loses its owner (a dropped socket, a restart, `hermes update` draining it or a new+# owner still starting) waits for it instead of exiting: the condition is the host's, not the card's,+# and a failed exit here spends the card's failure budget and lets the owner end the turn. Asking+# again is safe: ``kanban.run`` is idempotent per attempt and ``prompt.receipt`` is a read.+# ``OWNER_WAIT_S`` is the longest the owner may stay silent between answers before the worker+# gives up (the first ``kanban.run`` answer has its own wait, ``admission_answer_budget``).OWNER_WAIT_S = 180.0_TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})# In-band owner refusals that leave the attempt intact: a draining owner admits nothing new, and a# retired authority's successor holds the admission the next connection reads._TRANSIENT_RPC_REASONS = frozenset({'runtime_draining', 'runtime_retired'})+# Control-socket answers a restarting owner gives (discovery's ``inaccessible`` reason, or the+# ticket fetch's DiscoveryError): a reset or half-written answer, or a replaced owner refusing a+# stale instance. Unsafe nodes or peers, a refused authorization, a bad pointer or a failed start+# stay final.+_RESTART_CONTROL_REASONS = frozenset({'invalid_control_peer', 'incomplete_control_response',+ 'invalid_control_response'})# Past the board lock wait: the owner's session setup and receipt before and after the write.ADMISSION_ANSWER_MARGIN_S = 60.0@@ -35,21 +45,33 @@ def admission_answer_budget():return lock_wait + ADMISSION_ANSWER_MARGIN_S+@dataclass+class _Claim:+ """What the owner has said about this attempt, carried across connections."""+ answered: float # monotonic time of the owner's last answer+ session_id: Optional[str] = None+ admission_id: Optional[str] = None++ def silent_for(self):+ return time.monotonic() - self.answered++async def run(params, board_db):from agent.retry_utils import jittered_backofffrom hermes_cli.gateway_client import (connect_gateway, GatewayClientError, GatewayRPCError,GatewayUnavailableError)- from websockets.exceptions import ConnectionClosed+ from hermes_cli.gateway_runtime_discovery import DiscoveryError+ from websockets.exceptions import WebSocketExceptionif os.environ.get('HERMES_TUI_GATEWAY_URL'):raise GatewayClientError('Kanban requires the assigned local profile owner')# A lost socket is not a lost attempt, and exiting here would let ``_watch_submitter`` end the- # owner's in-flight turn. ``kanban.run`` is idempotent per attempt (one request id per card run)- # and ``prompt.receipt`` is a read, so the next connection resumes whatever the last one reached.- # The wait is bounded from the owner's last answer: an owner that keeps answering keeps the+ # owner's in-flight turn, so the next connection resumes whatever the last one reached. The+ # wait is bounded from the owner's last answer: an owner that keeps answering keeps the# worker, and one that stays silent for ``OWNER_WAIT_S`` fails the worker honestly.- claim = {'answered': time.monotonic()}- failures, answered = 0, claim['answered']+ claim = _Claim(answered=time.monotonic())+ failures = 0while True:+ seen = claim.answeredtry:async with connect_gateway() as client:# kanban.run subscribes this socket to the card's session, but only receipts matter@@ -61,20 +83,30 @@ async def run(params, board_db):finally:drain.cancel()except GatewayRPCError as exc:- if str(exc) not in _TRANSIENT_RPC_REASONS or time.monotonic() - claim['answered'] >= OWNER_WAIT_S:+ if str(exc) not in _TRANSIENT_RPC_REASONS or claim.silent_for() >= OWNER_WAIT_S:raiseexcept GatewayUnavailableError as exc:- if exc.state not in _TRANSIENT_OWNER_STATES or time.monotonic() - claim['answered'] >= OWNER_WAIT_S:+ restarting = exc.state in _TRANSIENT_OWNER_STATES or exc.reason_code in _RESTART_CONTROL_REASONS+ if not restarting or claim.silent_for() >= OWNER_WAIT_S:+ raise+ except DiscoveryError as exc:+ if exc.reason not in _RESTART_CONTROL_REASONS or claim.silent_for() >= OWNER_WAIT_S:raise- except (GatewayClientError, ConnectionClosed, OSError, TimeoutError) as exc:- if time.monotonic() - claim['answered'] >= OWNER_WAIT_S:- raise GatewayClientError(f'Kanban owner unreachable for {OWNER_WAIT_S:.0f}s: {exc}') from exc+ # A restart also surfaces as a refused or reset dial, a failed handshake, or an answer+ # that never came.+ except (GatewayClientError, WebSocketException, OSError, TimeoutError) as exc:+ if claim.silent_for() >= OWNER_WAIT_S:+ # WebSocket errors can embed remote bodies; only our own messages are quoted.+ status = getattr(getattr(exc, 'response', None), 'status_code', None)+ detail = exc if isinstance(exc, GatewayClientError) else type(exc).__name__ + (+ f' (HTTP {status})' if status else '')+ raise GatewayClientError(f'Kanban owner silent for {claim.silent_for():.0f}s: {detail}') from excelse:return _exit_code(receipt, params, board_db)- if claim['answered'] != answered:- failures, answered = 0, claim['answered']- failures += 1- await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0))+ failures = 1 if claim.answered != seen else failures + 1+ # Full-width jitter: after an owner dies, every worker redials at once, and each that finds+ # it absent starts a replacement.+ await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0, jitter_ratio=1.0))async def _discard(events):@@ -84,9 +116,8 @@ async def _discard(events):async def _await_receipt(client, params, board_db, claim):"""Submit the claim (or, on a later connection, re-read the admission it got) and return its- settled receipt. *claim* carries the owner's ``session_id``/``admission_id`` and the monotonic- time of its last answer across connections."""- if 'admission_id' not in claim:+ settled receipt, recording each owner answer on *claim*."""+ if claim.admission_id is None:db = str(await asyncio.to_thread(Path(board_db).resolve))# The owner answers only after it checks the claim and writes its ``owner_admitted`` marker# in one board write transaction. The board (the cross-profile dispatch bus) can make that@@ -94,18 +125,18 @@ async def _await_receipt(client, params, board_db, claim):# submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the# admission was left ``unknown``). The wait is still finite: once the marker commits,# ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner- # that stalls after admitting is bounded only here. A hang-up fails the wait sooner.+ # that stalls after admitting is bounded by this wait and ``OWNER_WAIT_S``.accepted = await client.rpc('kanban.run', _timeout=admission_answer_budget(), **dict(params, db=db))receipt = accepted['receipt']- claim.update(session_id=accepted['session_id'], admission_id=receipt['admission_id'])- print(f'Session: {claim["session_id"]}', file=sys.stderr)+ claim.session_id, claim.admission_id = accepted['session_id'], receipt['admission_id']+ print(f'Session: {claim.session_id}', file=sys.stderr)else:receipt = await _receipt(client, claim)- claim['answered'] = time.monotonic()+ claim.answered = time.monotonic()while receipt['status'] not in {'terminal', 'unknown'}:await asyncio.sleep(.2)receipt = await _receipt(client, claim)- claim['answered'] = time.monotonic()+ claim.answered = time.monotonic()return receipt@@ -122,7 +153,7 @@ def _exit_code(receipt, params, board_db):async def _receipt(client, claim):- return await client.rpc('prompt.receipt', session_id=claim['session_id'], admission_id=claim['admission_id'])+ return await client.rpc('prompt.receipt', session_id=claim.session_id, admission_id=claim.admission_id)def main():diff --git a/tests/hermes_cli/test_kanban_worker_admission_wait.py b/tests/hermes_cli/test_kanban_worker_admission_wait.pyindex 5ddc601f0a8..70bd4de9615 100644--- a/tests/hermes_cli/test_kanban_worker_admission_wait.py+++ b/tests/hermes_cli/test_kanban_worker_admission_wait.py@@ -49,7 +49,7 @@ async def test_a_slow_admission_answer_is_awaited_not_failed(monkeypatch, tmp_paowner = _Owner()params = {'board': 'b', 'task_id': 't1', 'run_id': 1, 'claim_lock': 'lock'}async with GatewayClient(owner) as client:- receipt = await kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), {})+ receipt = await kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), kanban_worker_client._Claim(answered=0.0))code = kanban_worker_client._exit_code(receipt, params, str(tmp_path / 'kanban.db'))assert code == 0# One submission: the wait never resends the claim, and the budget never goes on the wire.@@ -85,5 +85,5 @@ async def test_an_owner_that_stalls_after_admitting_fails_the_wait_within_the_loasync with GatewayClient(owner) as client:with pytest.raises(TimeoutError):# The outer 5 s guard only stops an unbounded wait from hanging the test.- await real_wait_for(kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), {}), 5)+ await real_wait_for(kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), kanban_worker_client._Claim(answered=0.0)), 5)assert budgets == [budget] and [r['method'] for r in owner.sent] == ['kanban.run']diff --git a/tests/hermes_cli/test_kanban_worker_socket_recovery.py b/tests/hermes_cli/test_kanban_worker_socket_recovery.pyindex aa81e1a1707..fcaa7f2f9da 100644--- a/tests/hermes_cli/test_kanban_worker_socket_recovery.py+++ b/tests/hermes_cli/test_kanban_worker_socket_recovery.py@@ -11,8 +11,10 @@ import asynciofrom contextlib import asynccontextmanagerimport pytest+from websockets.exceptions import InvalidMessagefrom hermes_cli import gateway_client, kanban_worker_client+from hermes_cli.gateway_runtime_discovery import DiscoveryErrorclass _Clock:@@ -40,7 +42,7 @@ class _Owner:@staticmethoddef _step(steps, default=None):step = steps.pop(0) if steps else default- if callable(step) and not isinstance(step, Exception):+ if callable(step):step = step()if isinstance(step, Exception):raise step@@ -108,7 +110,14 @@ async def _run(owner, monkeypatch, tmp_path):gateway_client.GatewayClientError("Gateway connection failed; no local fallback"),ConnectionRefusedError(61, "refused"),gateway_client.GatewayUnavailableError("Gateway starting: not_ready", "starting"),-], ids=["reconnects", "dial-refused", "ticket-oserror", "owner-starting"])+ gateway_client.GatewayUnavailableError("Gateway inaccessible: invalid_control_peer",+ "inaccessible", "invalid_control_peer"),+ gateway_client.GatewayUnavailableError("Gateway inaccessible: incomplete_control_response",+ "inaccessible", "incomplete_control_response"),+ DiscoveryError("invalid_control_response"),+ InvalidMessage("did not receive a valid HTTP response"),+], ids=["reconnects", "dial-refused", "ticket-oserror", "owner-starting", "control-reset", "control-eof",+ "owner-replaced", "handshake"])async def test_a_socket_lost_mid_turn_resumes_the_same_attempt(monkeypatch, _worker_env, clock, redial):owner = _Owner(connects=[None, redial, None], receipts=["pending", _disconnect(), "terminal"])@@ -118,6 +127,15 @@ async def test_a_socket_lost_mid_turn_resumes_the_same_attempt(monkeypatch, _worassert owner.dials == (2 if redial is None else 3)[email protected]("reason", ["runtime_draining", "runtime_retired"])+async def test_a_draining_or_retired_authority_mid_turn_is_asked_again(monkeypatch, _worker_env, clock, reason):+ owner = _Owner(connects=[None, None], receipts=["pending", gateway_client.GatewayRPCError(reason), "terminal"])++ assert await _run(owner, monkeypatch, _worker_env) == 0+ assert owner.dials == 2 and owner.calls.count("kanban.run") == 1++@pytest.mark.asyncio@pytest.mark.parametrize("lost", [_disconnect(), gateway_client.GatewayRPCError("runtime_draining")],ids=["answer-lost", "draining"])@@ -142,6 +160,19 @@ async def test_an_owner_refusal_is_not_retried(monkeypatch, _worker_env, clock,assert owner.dials == 1[email protected]("final", [+ gateway_client.GatewayUnavailableError("Gateway inaccessible: authorization", "inaccessible", "authorization"),+ DiscoveryError("unsafe_control_permissions"),+], ids=["refused-control", "unsafe-control"])+async def test_an_owner_that_cannot_be_trusted_fails_at_once(monkeypatch, _worker_env, clock, final):+ owner = _Owner(connects=[None, final], receipts=["pending", _disconnect()])++ with pytest.raises(type(final)):+ await _run(owner, monkeypatch, _worker_env)+ assert owner.dials == 2++@pytest.mark.asyncioasync def test_a_later_loss_gets_a_fresh_budget_from_the_owners_last_answer(monkeypatch, _worker_env, clock):def silent_then_lost():@@ -156,12 +187,47 @@ async def test_a_later_loss_gets_a_fresh_budget_from_the_owners_last_answer(monkassert owner.dials == 3 and owner.calls.count("kanban.run") == 1+async def test_connecting_is_not_an_answer(monkeypatch, _worker_env, clock):+ # Every redial connects, but every receipt read drops: only the owner's answers refresh the+ # budget, so this still ends at OWNER_WAIT_S instead of redialling forever.+ owner = _Owner(receipts=["pending"] + [_disconnect()] * 200)++ with pytest.raises(gateway_client.GatewayClientError, match=r"owner silent for 1[89]\ds"):+ await _run(owner, monkeypatch, _worker_env)+ assert owner.calls.count("kanban.run") == 1++@pytest.mark.asyncioasync def test_an_owner_silent_for_the_whole_budget_fails_the_worker(monkeypatch, _worker_env, clock):owner = _Owner(connects=[None], receipts=["pending", _disconnect()], refuse_dials_after=True)- with pytest.raises(gateway_client.GatewayClientError, match="unreachable for 180s"):+ with pytest.raises(gateway_client.GatewayClientError, match=r"owner silent for 1[89]\ds: Gateway connection failed"):await _run(owner, monkeypatch, _worker_env)assert clock.now >= kanban_worker_client.OWNER_WAIT_S- # Jittered exponential backoff (capped at 8-12 s) between dials, not one dial a second.+ # Jittered exponential backoff (capped at 8-16 s) between dials, not one dial a second.assert 3 < owner.dials < 40 and owner.calls.count("kanban.run") == 1+++async def test_giving_up_on_a_refused_handshake_names_the_status(monkeypatch, _worker_env, clock):+ from types import SimpleNamespace+ from websockets.exceptions import InvalidStatus+ refused = InvalidStatus(SimpleNamespace(status_code=403, headers={}, body=b"remote body"))+ owner = _Owner(connects=[None] + [refused] * 60, receipts=["pending", _disconnect()])++ with pytest.raises(gateway_client.GatewayClientError, match=r"InvalidStatus \(HTTP 403\)$"):+ await _run(owner, monkeypatch, _worker_env)+++async def test_discovery_hands_the_worker_its_reason(monkeypatch, _worker_env):+ """The real connect_gateway carries discovery's reason, which is what tells a restarting+ owner's reset control socket apart from a foreign one."""+ from hermes_cli import gateway_runtime+ monkeypatch.setattr(gateway_runtime, "ensure_gateway_runtime", lambda *_, **__: gateway_runtime.GatewayDiscovery(+ "inaccessible", reason_code="incomplete_control_response"))+ with pytest.raises(gateway_client.GatewayUnavailableError) as raised:+ async with gateway_client.connect_gateway():+ pass+ assert (raised.value.state, raised.value.reason_code) == ("inaccessible", "incomplete_control_response")