Skip to content

The agent that grows with you

fix(kanban): one owner-refusal rule and one give-up message for a recovering worker

kshitijk4poorb2e0171c29e5
diff --git a/hermes_cli/gateway_client.py b/hermes_cli/gateway_client.py
index a20e3fe661a..cbaf355d366 100644
--- a/hermes_cli/gateway_client.py
+++ b/hermes_cli/gateway_client.py
@@ -276,6 +276,7 @@ async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 10
from websockets.asyncio.client import connect
from hermes_constants import get_hermes_home
from hermes_cli.gateway_runtime import attachable, ensure_gateway_runtime, gateway_profile_home
+ from hermes_cli.gateway_runtime_discovery import DiscoveryError
from urllib.parse import urlsplit
remote = "" if endpoint is not None else os.environ.get("HERMES_TUI_GATEWAY_URL", "").strip()
@@ -304,7 +305,13 @@ async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 10
raise GatewayUnavailableError(f"Gateway {result.state}: {result.reason_code or 'not_ready'}{detail}",
result.state, result.reason_code)
endpoint = result.endpoint
- ticket = await asyncio.to_thread(_session_ticket, home, endpoint)
+ try:
+ ticket = await asyncio.to_thread(_session_ticket, home, endpoint)
+ except DiscoveryError as exc:
+ # The control socket refused the ticket: the verdict discovery gives the same refusal.
+ detail = exc.bounded_detail(500)
+ raise GatewayUnavailableError(f"Gateway inaccessible: {exc.reason}" + (f" ({detail})" if detail else ""),
+ "inaccessible", exc.reason) from exc
url, protocols = gateway_ws_target(endpoint, ticket)
try:
# The gateway is a loopback (or explicitly named) peer, never something to route through the
diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.py
index 447f2808ada..d82dfe09827 100644
--- a/hermes_cli/kanban_worker_client.py
+++ b/hermes_cli/kanban_worker_client.py
@@ -26,8 +26,8 @@ _TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})
# In-band owner refusals that leave the attempt intact: a draining owner admits nothing new, and a
# retired authority's successor holds the admission the next connection reads.
_TRANSIENT_RPC_REASONS = frozenset({'runtime_draining', 'runtime_retired'})
-# Control-socket answers a restarting owner gives (discovery's ``inaccessible`` reason, or the
-# ticket fetch's DiscoveryError): a reset or half-written answer, or a replaced owner refusing a
+# Control-socket answers a restarting owner gives (an ``inaccessible`` owner's reason, from
+# discovery or the ticket fetch): a reset or half-written answer, or a replaced owner refusing a
# stale instance. Unsafe nodes or peers, a refused authorization, a bad pointer or a failed start
# stay final.
_RESTART_CONTROL_REASONS = frozenset({'invalid_control_peer', 'incomplete_control_response',
@@ -58,9 +58,7 @@ class _Claim:
async def run(params, board_db):
from agent.retry_utils import jittered_backoff
- from hermes_cli.gateway_client import (connect_gateway, GatewayClientError, GatewayRPCError,
- GatewayUnavailableError)
- from hermes_cli.gateway_runtime_discovery import DiscoveryError
+ from hermes_cli.gateway_client import connect_gateway, GatewayClientError
from websockets.exceptions import WebSocketException
if os.environ.get('HERMES_TUI_GATEWAY_URL'):
raise GatewayClientError('Kanban requires the assigned local profile owner')
@@ -82,19 +80,11 @@ async def run(params, board_db):
receipt = await _await_receipt(client, params, board_db, claim)
finally:
drain.cancel()
- except GatewayRPCError as exc:
- if str(exc) not in _TRANSIENT_RPC_REASONS or claim.silent_for() >= OWNER_WAIT_S:
- raise
- except GatewayUnavailableError as exc:
- restarting = exc.state in _TRANSIENT_OWNER_STATES or exc.reason_code in _RESTART_CONTROL_REASONS
- if not restarting or claim.silent_for() >= OWNER_WAIT_S:
- raise
- except DiscoveryError as exc:
- if exc.reason not in _RESTART_CONTROL_REASONS or claim.silent_for() >= OWNER_WAIT_S:
- raise
# A restart also surfaces as a refused or reset dial, a failed handshake, or an answer
# that never came.
except (GatewayClientError, WebSocketException, OSError, TimeoutError) as exc:
+ if not _restartable(exc):
+ raise
if claim.silent_for() >= OWNER_WAIT_S:
# WebSocket errors can embed remote bodies; only our own messages are quoted.
status = getattr(getattr(exc, 'response', None), 'status_code', None)
@@ -104,11 +94,21 @@ async def run(params, board_db):
else:
return _exit_code(receipt, params, board_db)
failures = 1 if claim.answered != seen else failures + 1
- # Full-width jitter: after an owner dies, every worker redials at once, and each that finds
- # it absent starts a replacement.
+ # Jittered (1-2 s first, up to 16 s): workers of one dead owner do not all redial, and so
+ # start a replacement, in the same instant.
await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0, jitter_ratio=1.0))
+def _restartable(exc):
+ """False when *exc* is the owner's own final answer; True for a restarting owner or a lost transport."""
+ from hermes_cli.gateway_client import GatewayRPCError, GatewayUnavailableError
+ if isinstance(exc, GatewayRPCError):
+ return str(exc) in _TRANSIENT_RPC_REASONS
+ if isinstance(exc, GatewayUnavailableError):
+ return exc.state in _TRANSIENT_OWNER_STATES or exc.reason_code in _RESTART_CONTROL_REASONS
+ return True
+
+
async def _discard(events):
while True:
await events.get()
diff --git a/tests/hermes_cli/test_kanban_worker_socket_recovery.py b/tests/hermes_cli/test_kanban_worker_socket_recovery.py
index fcaa7f2f9da..00b8fb1c3b2 100644
--- a/tests/hermes_cli/test_kanban_worker_socket_recovery.py
+++ b/tests/hermes_cli/test_kanban_worker_socket_recovery.py
@@ -16,6 +16,8 @@ from websockets.exceptions import InvalidMessage
from hermes_cli import gateway_client, kanban_worker_client
from hermes_cli.gateway_runtime_discovery import DiscoveryError
+GUE = gateway_client.GatewayUnavailableError
+
class _Clock:
"""Monotonic time that moves only when the client sleeps or a scripted step advances it."""
@@ -30,12 +32,12 @@ class _Clock:
class _Owner:
"""Scripted owner. ``connects`` holds one step per dial, ``receipts`` one per ``prompt.receipt``
and ``submits`` one per ``kanban.run``; a step is None (succeed), an exception (raised) or a
- callable (run first, so it can advance the clock or raise). ``refuse_dials_after`` fails every
- dial past the scripted ones with a refused connection."""
+ callable (run first, so it can advance the clock or raise). ``after`` is the step for every
+ dial past the scripted ones (None: the dial succeeds)."""
- def __init__(self, connects=(), receipts=(), submits=(), refuse_dials_after=False):
+ def __init__(self, connects=(), receipts=(), submits=(), after=None):
self.connects, self.receipts, self.submits = list(connects), list(receipts), list(submits)
- self.refuse_dials_after = refuse_dials_after
+ self.after = after
self.calls = []
self.dials = 0
@@ -54,8 +56,7 @@ class _Owner:
@asynccontextmanager
async def _connect():
owner.dials += 1
- refused = gateway_client.GatewayClientError("Gateway connection failed; no local fallback")
- owner._step(owner.connects, refused if owner.refuse_dials_after else None)
+ owner._step(owner.connects, owner.after)
class _Client:
def __init__(self):
@@ -114,7 +115,7 @@ async def _run(owner, monkeypatch, tmp_path):
"inaccessible", "invalid_control_peer"),
gateway_client.GatewayUnavailableError("Gateway inaccessible: incomplete_control_response",
"inaccessible", "incomplete_control_response"),
- DiscoveryError("invalid_control_response"),
+ GUE("Gateway inaccessible: invalid_control_response", "inaccessible", "invalid_control_response"),
InvalidMessage("did not receive a valid HTTP response"),
], ids=["reconnects", "dial-refused", "ticket-oserror", "owner-starting", "control-reset", "control-eof",
"owner-replaced", "handshake"])
@@ -163,7 +164,7 @@ async def test_an_owner_refusal_is_not_retried(monkeypatch, _worker_env, clock,
@pytest.mark.asyncio
@pytest.mark.parametrize("final", [
gateway_client.GatewayUnavailableError("Gateway inaccessible: authorization", "inaccessible", "authorization"),
- DiscoveryError("unsafe_control_permissions"),
+ GUE("Gateway inaccessible: unsafe_control_permissions", "inaccessible", "unsafe_control_permissions"),
], ids=["refused-control", "unsafe-control"])
async def test_an_owner_that_cannot_be_trusted_fails_at_once(monkeypatch, _worker_env, clock, final):
owner = _Owner(connects=[None, final], receipts=["pending", _disconnect()])
@@ -199,10 +200,17 @@ async def test_connecting_is_not_an_answer(monkeypatch, _worker_env, clock):
@pytest.mark.asyncio
-async def test_an_owner_silent_for_the_whole_budget_fails_the_worker(monkeypatch, _worker_env, clock):
- owner = _Owner(connects=[None], receipts=["pending", _disconnect()], refuse_dials_after=True)
-
- with pytest.raises(gateway_client.GatewayClientError, match=r"owner silent for 1[89]\ds: Gateway connection failed"):
+ gateway_client.GatewayClientError("Gateway connection failed; no local fallback"),
+ GUE("Gateway starting: deadline", "starting", "deadline"),
+ GUE("Gateway inaccessible: invalid_control_peer", "inaccessible", "invalid_control_peer"),
+ gateway_client.GatewayRPCError("runtime_draining"),
+], ids=["refused", "starting", "control-reset", "draining"])
+async def test_an_owner_silent_for_the_whole_budget_fails_the_worker(monkeypatch, _worker_env, clock, fault):
+ # Every restart shape still ends at OWNER_WAIT_S when it never clears.
+ owner = _Owner(connects=[None], receipts=["pending", _disconnect()], after=fault)
+
+ with pytest.raises(gateway_client.GatewayClientError, match=rf"owner silent for 1[89]\ds: {fault}"):
await _run(owner, monkeypatch, _worker_env)
assert clock.now >= kanban_worker_client.OWNER_WAIT_S
# Jittered exponential backoff (capped at 8-16 s) between dials, not one dial a second.
@@ -231,3 +239,20 @@ async def test_discovery_hands_the_worker_its_reason(monkeypatch, _worker_env):
async with gateway_client.connect_gateway():
pass
assert (raised.value.state, raised.value.reason_code) == ("inaccessible", "incomplete_control_response")
+
+
+async def test_a_refused_ticket_reads_like_a_refusing_owner(monkeypatch, _worker_env):
+ """The ticket fetch's DiscoveryError reaches callers as the inaccessible verdict discovery gives."""
+ from types import SimpleNamespace
+ from hermes_cli import gateway_runtime
+ monkeypatch.setattr(gateway_runtime, "ensure_gateway_runtime", lambda *_, **__: SimpleNamespace(endpoint=None))
+ monkeypatch.setattr(gateway_runtime, "attachable", lambda result: True)
+
+ def refuse(*_):
+ raise DiscoveryError("invalid_control_response")
+ monkeypatch.setattr(gateway_client, "_session_ticket", refuse)
+ with pytest.raises(GUE) as raised:
+ async with gateway_client.connect_gateway():
+ pass
+ assert (raised.value.state, raised.value.reason_code) == ("inaccessible", "invalid_control_response")