Skip to content

The agent that grows with you

fix(kanban): a restarting owner's control socket or handshake failure during a redial also retries

kshitijk4poorf4e042e944df
diff --git a/hermes_cli/gateway_client.py b/hermes_cli/gateway_client.py
index cbb02af51f4..a20e3fe661a 100644
--- a/hermes_cli/gateway_client.py
+++ b/hermes_cli/gateway_client.py
@@ -98,9 +98,9 @@ class GatewayUnavailableError(GatewayClientError):
"""No ready owner to dial: ``state`` is ``ensure_gateway_runtime``'s verdict (``draining`` while an
update holds the install, ``starting`` past the deadline, ...). Nothing was submitted."""
- def __init__(self, message, state):
+ def __init__(self, message, state, reason_code=None):
super().__init__(message)
- self.state = state
+ self.state, self.reason_code = state, reason_code
class EventQueue(asyncio.Queue):
@@ -299,10 +299,10 @@ async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 10
detail = getattr(result, "detail", None)
if result.reason_code == "runtime_exited" and detail:
# A multi-line startup report (redacted traceback tail + where the full log is).
- raise GatewayUnavailableError(f"Gateway could not start: {detail}", result.state)
+ raise GatewayUnavailableError(f"Gateway could not start: {detail}", result.state, result.reason_code)
detail = f" ({detail})" if detail else ""
raise GatewayUnavailableError(f"Gateway {result.state}: {result.reason_code or 'not_ready'}{detail}",
- result.state)
+ result.state, result.reason_code)
endpoint = result.endpoint
ticket = await asyncio.to_thread(_session_ticket, home, endpoint)
url, protocols = gateway_ws_target(endpoint, ticket)
diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.py
index cd2f4d68fbe..447f2808ada 100644
--- a/hermes_cli/kanban_worker_client.py
+++ b/hermes_cli/kanban_worker_client.py
@@ -8,20 +8,30 @@ except ModuleNotFoundError as exc:
raise
import asyncio
+from dataclasses import dataclass
import os
from pathlib import Path
import sys
import time
-
-# A worker the dispatcher spawns while `hermes update` drains the owner (or while the new owner is
-# still starting) waits for it instead of exiting: the condition is the host's, not the card's, and
-# a failed exit here spends the card's failure budget on an update window. Nothing is submitted
-# until an owner is ready, so waiting can never run the claim twice.
+from typing import Optional
+
+# A worker that loses its owner (a dropped socket, a restart, `hermes update` draining it or a new
+# owner still starting) waits for it instead of exiting: the condition is the host's, not the card's,
+# and a failed exit here spends the card's failure budget and lets the owner end the turn. Asking
+# again is safe: ``kanban.run`` is idempotent per attempt and ``prompt.receipt`` is a read.
+# ``OWNER_WAIT_S`` is the longest the owner may stay silent between answers before the worker
+# gives up (the first ``kanban.run`` answer has its own wait, ``admission_answer_budget``).
OWNER_WAIT_S = 180.0
_TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})
# In-band owner refusals that leave the attempt intact: a draining owner admits nothing new, and a
# retired authority's successor holds the admission the next connection reads.
_TRANSIENT_RPC_REASONS = frozenset({'runtime_draining', 'runtime_retired'})
+# Control-socket answers a restarting owner gives (discovery's ``inaccessible`` reason, or the
+# ticket fetch's DiscoveryError): a reset or half-written answer, or a replaced owner refusing a
+# stale instance. Unsafe nodes or peers, a refused authorization, a bad pointer or a failed start
+# stay final.
+_RESTART_CONTROL_REASONS = frozenset({'invalid_control_peer', 'incomplete_control_response',
+ 'invalid_control_response'})
# Past the board lock wait: the owner's session setup and receipt before and after the write.
ADMISSION_ANSWER_MARGIN_S = 60.0
@@ -35,21 +45,33 @@ def admission_answer_budget():
return lock_wait + ADMISSION_ANSWER_MARGIN_S
+@dataclass
+class _Claim:
+ """What the owner has said about this attempt, carried across connections."""
+ answered: float # monotonic time of the owner's last answer
+ session_id: Optional[str] = None
+ admission_id: Optional[str] = None
+
+ def silent_for(self):
+ return time.monotonic() - self.answered
+
+
async def run(params, board_db):
from agent.retry_utils import jittered_backoff
from hermes_cli.gateway_client import (connect_gateway, GatewayClientError, GatewayRPCError,
GatewayUnavailableError)
- from websockets.exceptions import ConnectionClosed
+ from hermes_cli.gateway_runtime_discovery import DiscoveryError
+ from websockets.exceptions import WebSocketException
if os.environ.get('HERMES_TUI_GATEWAY_URL'):
raise GatewayClientError('Kanban requires the assigned local profile owner')
# A lost socket is not a lost attempt, and exiting here would let ``_watch_submitter`` end the
- # owner's in-flight turn. ``kanban.run`` is idempotent per attempt (one request id per card run)
- # and ``prompt.receipt`` is a read, so the next connection resumes whatever the last one reached.
- # The wait is bounded from the owner's last answer: an owner that keeps answering keeps the
+ # owner's in-flight turn, so the next connection resumes whatever the last one reached. The
+ # wait is bounded from the owner's last answer: an owner that keeps answering keeps the
# worker, and one that stays silent for ``OWNER_WAIT_S`` fails the worker honestly.
- claim = {'answered': time.monotonic()}
- failures, answered = 0, claim['answered']
+ claim = _Claim(answered=time.monotonic())
+ failures = 0
while True:
+ seen = claim.answered
try:
async with connect_gateway() as client:
# kanban.run subscribes this socket to the card's session, but only receipts matter
@@ -61,20 +83,30 @@ async def run(params, board_db):
finally:
drain.cancel()
except GatewayRPCError as exc:
- if str(exc) not in _TRANSIENT_RPC_REASONS or time.monotonic() - claim['answered'] >= OWNER_WAIT_S:
+ if str(exc) not in _TRANSIENT_RPC_REASONS or claim.silent_for() >= OWNER_WAIT_S:
raise
except GatewayUnavailableError as exc:
- if exc.state not in _TRANSIENT_OWNER_STATES or time.monotonic() - claim['answered'] >= OWNER_WAIT_S:
+ restarting = exc.state in _TRANSIENT_OWNER_STATES or exc.reason_code in _RESTART_CONTROL_REASONS
+ if not restarting or claim.silent_for() >= OWNER_WAIT_S:
+ raise
+ except DiscoveryError as exc:
+ if exc.reason not in _RESTART_CONTROL_REASONS or claim.silent_for() >= OWNER_WAIT_S:
raise
- except (GatewayClientError, ConnectionClosed, OSError, TimeoutError) as exc:
- if time.monotonic() - claim['answered'] >= OWNER_WAIT_S:
- raise GatewayClientError(f'Kanban owner unreachable for {OWNER_WAIT_S:.0f}s: {exc}') from exc
+ # A restart also surfaces as a refused or reset dial, a failed handshake, or an answer
+ # that never came.
+ except (GatewayClientError, WebSocketException, OSError, TimeoutError) as exc:
+ if claim.silent_for() >= OWNER_WAIT_S:
+ # WebSocket errors can embed remote bodies; only our own messages are quoted.
+ status = getattr(getattr(exc, 'response', None), 'status_code', None)
+ detail = exc if isinstance(exc, GatewayClientError) else type(exc).__name__ + (
+ f' (HTTP {status})' if status else '')
+ raise GatewayClientError(f'Kanban owner silent for {claim.silent_for():.0f}s: {detail}') from exc
else:
return _exit_code(receipt, params, board_db)
- if claim['answered'] != answered:
- failures, answered = 0, claim['answered']
- failures += 1
- await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0))
+ failures = 1 if claim.answered != seen else failures + 1
+ # Full-width jitter: after an owner dies, every worker redials at once, and each that finds
+ # it absent starts a replacement.
+ await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0, jitter_ratio=1.0))
async def _discard(events):
@@ -84,9 +116,8 @@ async def _discard(events):
async def _await_receipt(client, params, board_db, claim):
"""Submit the claim (or, on a later connection, re-read the admission it got) and return its
- settled receipt. *claim* carries the owner's ``session_id``/``admission_id`` and the monotonic
- time of its last answer across connections."""
- if 'admission_id' not in claim:
+ settled receipt, recording each owner answer on *claim*."""
+ if claim.admission_id is None:
db = str(await asyncio.to_thread(Path(board_db).resolve))
# The owner answers only after it checks the claim and writes its ``owner_admitted`` marker
# in one board write transaction. The board (the cross-profile dispatch bus) can make that
@@ -94,18 +125,18 @@ async def _await_receipt(client, params, board_db, claim):
# submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the
# admission was left ``unknown``). The wait is still finite: once the marker commits,
# ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner
- # that stalls after admitting is bounded only here. A hang-up fails the wait sooner.
+ # that stalls after admitting is bounded by this wait and ``OWNER_WAIT_S``.
accepted = await client.rpc('kanban.run', _timeout=admission_answer_budget(), **dict(params, db=db))
receipt = accepted['receipt']
- claim.update(session_id=accepted['session_id'], admission_id=receipt['admission_id'])
- print(f'Session: {claim["session_id"]}', file=sys.stderr)
+ claim.session_id, claim.admission_id = accepted['session_id'], receipt['admission_id']
+ print(f'Session: {claim.session_id}', file=sys.stderr)
else:
receipt = await _receipt(client, claim)
- claim['answered'] = time.monotonic()
+ claim.answered = time.monotonic()
while receipt['status'] not in {'terminal', 'unknown'}:
await asyncio.sleep(.2)
receipt = await _receipt(client, claim)
- claim['answered'] = time.monotonic()
+ claim.answered = time.monotonic()
return receipt
@@ -122,7 +153,7 @@ def _exit_code(receipt, params, board_db):
async def _receipt(client, claim):
- return await client.rpc('prompt.receipt', session_id=claim['session_id'], admission_id=claim['admission_id'])
+ return await client.rpc('prompt.receipt', session_id=claim.session_id, admission_id=claim.admission_id)
def main():
diff --git a/tests/hermes_cli/test_kanban_worker_admission_wait.py b/tests/hermes_cli/test_kanban_worker_admission_wait.py
index 5ddc601f0a8..70bd4de9615 100644
--- a/tests/hermes_cli/test_kanban_worker_admission_wait.py
+++ b/tests/hermes_cli/test_kanban_worker_admission_wait.py
@@ -49,7 +49,7 @@ async def test_a_slow_admission_answer_is_awaited_not_failed(monkeypatch, tmp_pa
owner = _Owner()
params = {'board': 'b', 'task_id': 't1', 'run_id': 1, 'claim_lock': 'lock'}
async with GatewayClient(owner) as client:
- receipt = await kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), {})
+ receipt = await kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), kanban_worker_client._Claim(answered=0.0))
code = kanban_worker_client._exit_code(receipt, params, str(tmp_path / 'kanban.db'))
assert code == 0
# One submission: the wait never resends the claim, and the budget never goes on the wire.
@@ -85,5 +85,5 @@ async def test_an_owner_that_stalls_after_admitting_fails_the_wait_within_the_lo
async with GatewayClient(owner) as client:
with pytest.raises(TimeoutError):
# The outer 5 s guard only stops an unbounded wait from hanging the test.
- await real_wait_for(kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), {}), 5)
+ await real_wait_for(kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), kanban_worker_client._Claim(answered=0.0)), 5)
assert budgets == [budget] and [r['method'] for r in owner.sent] == ['kanban.run']
diff --git a/tests/hermes_cli/test_kanban_worker_socket_recovery.py b/tests/hermes_cli/test_kanban_worker_socket_recovery.py
index aa81e1a1707..fcaa7f2f9da 100644
--- a/tests/hermes_cli/test_kanban_worker_socket_recovery.py
+++ b/tests/hermes_cli/test_kanban_worker_socket_recovery.py
@@ -11,8 +11,10 @@ import asyncio
from contextlib import asynccontextmanager
import pytest
+from websockets.exceptions import InvalidMessage
from hermes_cli import gateway_client, kanban_worker_client
+from hermes_cli.gateway_runtime_discovery import DiscoveryError
class _Clock:
@@ -40,7 +42,7 @@ class _Owner:
@staticmethod
def _step(steps, default=None):
step = steps.pop(0) if steps else default
- if callable(step) and not isinstance(step, Exception):
+ if callable(step):
step = step()
if isinstance(step, Exception):
raise step
@@ -108,7 +110,14 @@ async def _run(owner, monkeypatch, tmp_path):
gateway_client.GatewayClientError("Gateway connection failed; no local fallback"),
ConnectionRefusedError(61, "refused"),
gateway_client.GatewayUnavailableError("Gateway starting: not_ready", "starting"),
-], ids=["reconnects", "dial-refused", "ticket-oserror", "owner-starting"])
+ gateway_client.GatewayUnavailableError("Gateway inaccessible: invalid_control_peer",
+ "inaccessible", "invalid_control_peer"),
+ gateway_client.GatewayUnavailableError("Gateway inaccessible: incomplete_control_response",
+ "inaccessible", "incomplete_control_response"),
+ DiscoveryError("invalid_control_response"),
+ InvalidMessage("did not receive a valid HTTP response"),
+], ids=["reconnects", "dial-refused", "ticket-oserror", "owner-starting", "control-reset", "control-eof",
+ "owner-replaced", "handshake"])
async def test_a_socket_lost_mid_turn_resumes_the_same_attempt(monkeypatch, _worker_env, clock, redial):
owner = _Owner(connects=[None, redial, None], receipts=["pending", _disconnect(), "terminal"])
@@ -118,6 +127,15 @@ async def test_a_socket_lost_mid_turn_resumes_the_same_attempt(monkeypatch, _wor
assert owner.dials == (2 if redial is None else 3)
[email protected]("reason", ["runtime_draining", "runtime_retired"])
+async def test_a_draining_or_retired_authority_mid_turn_is_asked_again(monkeypatch, _worker_env, clock, reason):
+ owner = _Owner(connects=[None, None], receipts=["pending", gateway_client.GatewayRPCError(reason), "terminal"])
+
+ assert await _run(owner, monkeypatch, _worker_env) == 0
+ assert owner.dials == 2 and owner.calls.count("kanban.run") == 1
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("lost", [_disconnect(), gateway_client.GatewayRPCError("runtime_draining")],
ids=["answer-lost", "draining"])
@@ -142,6 +160,19 @@ async def test_an_owner_refusal_is_not_retried(monkeypatch, _worker_env, clock,
assert owner.dials == 1
+ gateway_client.GatewayUnavailableError("Gateway inaccessible: authorization", "inaccessible", "authorization"),
+ DiscoveryError("unsafe_control_permissions"),
+], ids=["refused-control", "unsafe-control"])
+async def test_an_owner_that_cannot_be_trusted_fails_at_once(monkeypatch, _worker_env, clock, final):
+ owner = _Owner(connects=[None, final], receipts=["pending", _disconnect()])
+
+ with pytest.raises(type(final)):
+ await _run(owner, monkeypatch, _worker_env)
+ assert owner.dials == 2
+
+
@pytest.mark.asyncio
async def test_a_later_loss_gets_a_fresh_budget_from_the_owners_last_answer(monkeypatch, _worker_env, clock):
def silent_then_lost():
@@ -156,12 +187,47 @@ async def test_a_later_loss_gets_a_fresh_budget_from_the_owners_last_answer(monk
assert owner.dials == 3 and owner.calls.count("kanban.run") == 1
+async def test_connecting_is_not_an_answer(monkeypatch, _worker_env, clock):
+ # Every redial connects, but every receipt read drops: only the owner's answers refresh the
+ # budget, so this still ends at OWNER_WAIT_S instead of redialling forever.
+ owner = _Owner(receipts=["pending"] + [_disconnect()] * 200)
+
+ with pytest.raises(gateway_client.GatewayClientError, match=r"owner silent for 1[89]\ds"):
+ await _run(owner, monkeypatch, _worker_env)
+ assert owner.calls.count("kanban.run") == 1
+
+
@pytest.mark.asyncio
async def test_an_owner_silent_for_the_whole_budget_fails_the_worker(monkeypatch, _worker_env, clock):
owner = _Owner(connects=[None], receipts=["pending", _disconnect()], refuse_dials_after=True)
- with pytest.raises(gateway_client.GatewayClientError, match="unreachable for 180s"):
+ with pytest.raises(gateway_client.GatewayClientError, match=r"owner silent for 1[89]\ds: Gateway connection failed"):
await _run(owner, monkeypatch, _worker_env)
assert clock.now >= kanban_worker_client.OWNER_WAIT_S
- # Jittered exponential backoff (capped at 8-12 s) between dials, not one dial a second.
+ # Jittered exponential backoff (capped at 8-16 s) between dials, not one dial a second.
assert 3 < owner.dials < 40 and owner.calls.count("kanban.run") == 1
+
+
+async def test_giving_up_on_a_refused_handshake_names_the_status(monkeypatch, _worker_env, clock):
+ from types import SimpleNamespace
+ from websockets.exceptions import InvalidStatus
+ refused = InvalidStatus(SimpleNamespace(status_code=403, headers={}, body=b"remote body"))
+ owner = _Owner(connects=[None] + [refused] * 60, receipts=["pending", _disconnect()])
+
+ with pytest.raises(gateway_client.GatewayClientError, match=r"InvalidStatus \(HTTP 403\)$"):
+ await _run(owner, monkeypatch, _worker_env)
+
+
+async def test_discovery_hands_the_worker_its_reason(monkeypatch, _worker_env):
+ """The real connect_gateway carries discovery's reason, which is what tells a restarting
+ owner's reset control socket apart from a foreign one."""
+ from hermes_cli import gateway_runtime
+ monkeypatch.setattr(gateway_runtime, "ensure_gateway_runtime", lambda *_, **__: gateway_runtime.GatewayDiscovery(
+ "inaccessible", reason_code="incomplete_control_response"))
+ with pytest.raises(gateway_client.GatewayUnavailableError) as raised:
+ async with gateway_client.connect_gateway():
+ pass
+ assert (raised.value.state, raised.value.reason_code) == ("inaccessible", "incomplete_control_response")