The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(kanban): one owner-refusal rule and one give-up message for a recovering worker
kshitijk4poorb2e0171c29e5
diff --git a/hermes_cli/gateway_client.py b/hermes_cli/gateway_client.pyindex a20e3fe661a..cbaf355d366 100644--- a/hermes_cli/gateway_client.py+++ b/hermes_cli/gateway_client.py@@ -276,6 +276,7 @@ async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 10from websockets.asyncio.client import connectfrom hermes_constants import get_hermes_homefrom hermes_cli.gateway_runtime import attachable, ensure_gateway_runtime, gateway_profile_home+ from hermes_cli.gateway_runtime_discovery import DiscoveryErrorfrom urllib.parse import urlsplitremote = "" if endpoint is not None else os.environ.get("HERMES_TUI_GATEWAY_URL", "").strip()@@ -304,7 +305,13 @@ async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 10raise GatewayUnavailableError(f"Gateway {result.state}: {result.reason_code or 'not_ready'}{detail}",result.state, result.reason_code)endpoint = result.endpoint- ticket = await asyncio.to_thread(_session_ticket, home, endpoint)+ try:+ ticket = await asyncio.to_thread(_session_ticket, home, endpoint)+ except DiscoveryError as exc:+ # The control socket refused the ticket: the verdict discovery gives the same refusal.+ detail = exc.bounded_detail(500)+ raise GatewayUnavailableError(f"Gateway inaccessible: {exc.reason}" + (f" ({detail})" if detail else ""),+ "inaccessible", exc.reason) from excurl, protocols = gateway_ws_target(endpoint, ticket)try:# The gateway is a loopback (or explicitly named) peer, never something to route through thediff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.pyindex 447f2808ada..d82dfe09827 100644--- a/hermes_cli/kanban_worker_client.py+++ b/hermes_cli/kanban_worker_client.py@@ -26,8 +26,8 @@ _TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})# In-band owner refusals that leave the attempt intact: a draining owner admits nothing new, and a# retired authority's successor holds the admission the next connection reads._TRANSIENT_RPC_REASONS = frozenset({'runtime_draining', 'runtime_retired'})-# Control-socket answers a restarting owner gives (discovery's ``inaccessible`` reason, or the-# ticket fetch's DiscoveryError): a reset or half-written answer, or a replaced owner refusing a+# Control-socket answers a restarting owner gives (an ``inaccessible`` owner's reason, from+# discovery or the ticket fetch): a reset or half-written answer, or a replaced owner refusing a# stale instance. Unsafe nodes or peers, a refused authorization, a bad pointer or a failed start# stay final._RESTART_CONTROL_REASONS = frozenset({'invalid_control_peer', 'incomplete_control_response',@@ -58,9 +58,7 @@ class _Claim:async def run(params, board_db):from agent.retry_utils import jittered_backoff- from hermes_cli.gateway_client import (connect_gateway, GatewayClientError, GatewayRPCError,- GatewayUnavailableError)- from hermes_cli.gateway_runtime_discovery import DiscoveryError+ from hermes_cli.gateway_client import connect_gateway, GatewayClientErrorfrom websockets.exceptions import WebSocketExceptionif os.environ.get('HERMES_TUI_GATEWAY_URL'):raise GatewayClientError('Kanban requires the assigned local profile owner')@@ -82,19 +80,11 @@ async def run(params, board_db):receipt = await _await_receipt(client, params, board_db, claim)finally:drain.cancel()- except GatewayRPCError as exc:- if str(exc) not in _TRANSIENT_RPC_REASONS or claim.silent_for() >= OWNER_WAIT_S:- raise- except GatewayUnavailableError as exc:- restarting = exc.state in _TRANSIENT_OWNER_STATES or exc.reason_code in _RESTART_CONTROL_REASONS- if not restarting or claim.silent_for() >= OWNER_WAIT_S:- raise- except DiscoveryError as exc:- if exc.reason not in _RESTART_CONTROL_REASONS or claim.silent_for() >= OWNER_WAIT_S:- raise# A restart also surfaces as a refused or reset dial, a failed handshake, or an answer# that never came.except (GatewayClientError, WebSocketException, OSError, TimeoutError) as exc:+ if not _restartable(exc):+ raiseif claim.silent_for() >= OWNER_WAIT_S:# WebSocket errors can embed remote bodies; only our own messages are quoted.status = getattr(getattr(exc, 'response', None), 'status_code', None)@@ -104,11 +94,21 @@ async def run(params, board_db):else:return _exit_code(receipt, params, board_db)failures = 1 if claim.answered != seen else failures + 1- # Full-width jitter: after an owner dies, every worker redials at once, and each that finds- # it absent starts a replacement.+ # Jittered (1-2 s first, up to 16 s): workers of one dead owner do not all redial, and so+ # start a replacement, in the same instant.await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0, jitter_ratio=1.0))+def _restartable(exc):+ """False when *exc* is the owner's own final answer; True for a restarting owner or a lost transport."""+ from hermes_cli.gateway_client import GatewayRPCError, GatewayUnavailableError+ if isinstance(exc, GatewayRPCError):+ return str(exc) in _TRANSIENT_RPC_REASONS+ if isinstance(exc, GatewayUnavailableError):+ return exc.state in _TRANSIENT_OWNER_STATES or exc.reason_code in _RESTART_CONTROL_REASONS+ return True++async def _discard(events):while True:await events.get()diff --git a/tests/hermes_cli/test_kanban_worker_socket_recovery.py b/tests/hermes_cli/test_kanban_worker_socket_recovery.pyindex fcaa7f2f9da..00b8fb1c3b2 100644--- a/tests/hermes_cli/test_kanban_worker_socket_recovery.py+++ b/tests/hermes_cli/test_kanban_worker_socket_recovery.py@@ -16,6 +16,8 @@ from websockets.exceptions import InvalidMessagefrom hermes_cli import gateway_client, kanban_worker_clientfrom hermes_cli.gateway_runtime_discovery import DiscoveryError+GUE = gateway_client.GatewayUnavailableError+class _Clock:"""Monotonic time that moves only when the client sleeps or a scripted step advances it."""@@ -30,12 +32,12 @@ class _Clock:class _Owner:"""Scripted owner. ``connects`` holds one step per dial, ``receipts`` one per ``prompt.receipt``and ``submits`` one per ``kanban.run``; a step is None (succeed), an exception (raised) or a- callable (run first, so it can advance the clock or raise). ``refuse_dials_after`` fails every- dial past the scripted ones with a refused connection."""+ callable (run first, so it can advance the clock or raise). ``after`` is the step for every+ dial past the scripted ones (None: the dial succeeds)."""- def __init__(self, connects=(), receipts=(), submits=(), refuse_dials_after=False):+ def __init__(self, connects=(), receipts=(), submits=(), after=None):self.connects, self.receipts, self.submits = list(connects), list(receipts), list(submits)- self.refuse_dials_after = refuse_dials_after+ self.after = afterself.calls = []self.dials = 0@@ -54,8 +56,7 @@ class _Owner:@asynccontextmanagerasync def _connect():owner.dials += 1- refused = gateway_client.GatewayClientError("Gateway connection failed; no local fallback")- owner._step(owner.connects, refused if owner.refuse_dials_after else None)+ owner._step(owner.connects, owner.after)class _Client:def __init__(self):@@ -114,7 +115,7 @@ async def _run(owner, monkeypatch, tmp_path):"inaccessible", "invalid_control_peer"),gateway_client.GatewayUnavailableError("Gateway inaccessible: incomplete_control_response","inaccessible", "incomplete_control_response"),- DiscoveryError("invalid_control_response"),+ GUE("Gateway inaccessible: invalid_control_response", "inaccessible", "invalid_control_response"),InvalidMessage("did not receive a valid HTTP response"),], ids=["reconnects", "dial-refused", "ticket-oserror", "owner-starting", "control-reset", "control-eof","owner-replaced", "handshake"])@@ -163,7 +164,7 @@ async def test_an_owner_refusal_is_not_retried(monkeypatch, _worker_env, clock,@pytest.mark.asyncio@pytest.mark.parametrize("final", [gateway_client.GatewayUnavailableError("Gateway inaccessible: authorization", "inaccessible", "authorization"),- DiscoveryError("unsafe_control_permissions"),+ GUE("Gateway inaccessible: unsafe_control_permissions", "inaccessible", "unsafe_control_permissions"),], ids=["refused-control", "unsafe-control"])async def test_an_owner_that_cannot_be_trusted_fails_at_once(monkeypatch, _worker_env, clock, final):owner = _Owner(connects=[None, final], receipts=["pending", _disconnect()])@@ -199,10 +200,17 @@ async def test_connecting_is_not_an_answer(monkeypatch, _worker_env, clock):@pytest.mark.asyncio-async def test_an_owner_silent_for_the_whole_budget_fails_the_worker(monkeypatch, _worker_env, clock):- owner = _Owner(connects=[None], receipts=["pending", _disconnect()], refuse_dials_after=True)-- with pytest.raises(gateway_client.GatewayClientError, match=r"owner silent for 1[89]\ds: Gateway connection failed"):[email protected]("fault", [+ gateway_client.GatewayClientError("Gateway connection failed; no local fallback"),+ GUE("Gateway starting: deadline", "starting", "deadline"),+ GUE("Gateway inaccessible: invalid_control_peer", "inaccessible", "invalid_control_peer"),+ gateway_client.GatewayRPCError("runtime_draining"),+], ids=["refused", "starting", "control-reset", "draining"])+async def test_an_owner_silent_for_the_whole_budget_fails_the_worker(monkeypatch, _worker_env, clock, fault):+ # Every restart shape still ends at OWNER_WAIT_S when it never clears.+ owner = _Owner(connects=[None], receipts=["pending", _disconnect()], after=fault)++ with pytest.raises(gateway_client.GatewayClientError, match=rf"owner silent for 1[89]\ds: {fault}"):await _run(owner, monkeypatch, _worker_env)assert clock.now >= kanban_worker_client.OWNER_WAIT_S# Jittered exponential backoff (capped at 8-16 s) between dials, not one dial a second.@@ -231,3 +239,20 @@ async def test_discovery_hands_the_worker_its_reason(monkeypatch, _worker_env):async with gateway_client.connect_gateway():passassert (raised.value.state, raised.value.reason_code) == ("inaccessible", "incomplete_control_response")+++async def test_a_refused_ticket_reads_like_a_refusing_owner(monkeypatch, _worker_env):+ """The ticket fetch's DiscoveryError reaches callers as the inaccessible verdict discovery gives."""+ from types import SimpleNamespace+ from hermes_cli import gateway_runtime+ monkeypatch.setattr(gateway_runtime, "ensure_gateway_runtime", lambda *_, **__: SimpleNamespace(endpoint=None))+ monkeypatch.setattr(gateway_runtime, "attachable", lambda result: True)++ def refuse(*_):+ raise DiscoveryError("invalid_control_response")+ monkeypatch.setattr(gateway_client, "_session_ticket", refuse)+ with pytest.raises(GUE) as raised:+ async with gateway_client.connect_gateway():+ pass+ assert (raised.value.state, raised.value.reason_code) == ("inaccessible", "invalid_control_response")