The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(kanban): drop the worker.adopt handshake retry from the lost-worker salvage
kshitijk4poorc91bd232c0d7
diff --git a/agent/runtime_session_store.py b/agent/runtime_session_store.pyindex b942bd21e24..f05c66459eb 100644--- a/agent/runtime_session_store.py+++ b/agent/runtime_session_store.py@@ -22,57 +22,6 @@ logger = logging.getLogger(__name__)_worker_process = False-# The ``worker.adopt`` handshake is a round trip through the OWNER's event loop, and a live-# gateway legitimately spends 1000 s+ inside one turn (fan/gateway.log 2026-10-22:-# ``time=1018.3s api_calls=46``). A 5 s budget lost that race and cost a whole dispatch-# cycle silently: the managed worker died in ``worker.adopt``, the card kept saying-# ``running``, and nothing retried. Generous by default, overridable, and a TIMEOUT (never-# an auth/protocol failure) is retried with backoff.-WORKER_RPC_OPEN_TIMEOUT_SECONDS = 60.0-WORKER_RPC_CONNECT_ATTEMPTS = 3-WORKER_RPC_RETRY_SLEEP_SECONDS = 1.0---def worker_rpc_open_timeout():- """The handshake budget: ``HERMES_WORKER_RPC_OPEN_TIMEOUT`` seconds, else 60 s."""- # The adopt handshake runs BEFORE the worker may import config (agent/managed_worker.py:- # "No agent/config imports before adopted process authority"), so no config.yaml key can- # size it; resolved per call, 60 s default.- raw = os.environ.get('HERMES_WORKER_RPC_OPEN_TIMEOUT', '').strip() # health: allow HX002 -- the adopt handshake runs before the worker may import config, so no config.yaml key can size it; read per call, 60 s default- if raw:- try:- value = float(raw)- except ValueError:- value = 0.0- if value > 0:- return value- return WORKER_RPC_OPEN_TIMEOUT_SECONDS---def connect_worker_link(connect, url, protocols, *, open_timeout=None, attempts=None, sleep=None):- """Open the owner link, retrying a handshake that TIMED OUT because the owner was busy.-- Only ``TimeoutError`` is retried: an authentication or protocol failure will not heal with- another attempt, and re-sending after a completed handshake is never safe. Each attempt- gets the full budget; ``sleep`` is a test seam.- """- import time as _time- budget = worker_rpc_open_timeout() if open_timeout is None else float(open_timeout)- tries = max(1, int(WORKER_RPC_CONNECT_ATTEMPTS if attempts is None else attempts))- pause = _time.sleep if sleep is None else sleep- last = None- for attempt in range(tries):- try:- return connect(url, subprotocols=protocols, open_timeout=budget, close_timeout=1,- max_size=8 * 1024 * 1024, proxy=None, ping_interval=None)- except TimeoutError as exc:- last = exc- if attempt + 1 < tries:- logger.warning('owner handshake timed out after %ss (attempt %d/%d); retrying',- budget, attempt + 1, tries)- pause(WORKER_RPC_RETRY_SLEEP_SECONDS * (2 ** attempt))- raise last-def is_worker_process():"""A successful self-registration makes this interpreter compute-only.@@ -206,7 +155,8 @@ class WorkerRPC:# The owner is loopback, like connect_gateway's peer: an inherited HTTPS_PROXY must not carry it.# No library keepalive: its pong deadline would run independently of (and could fail) a call# inside its receipt budget; that budget and the observed close are the liveness checks.- self.link = _OwnerLink(connect_worker_link(connect, url, protocols))+ self.link = _OwnerLink(connect(url, subprotocols=protocols, open_timeout=5, close_timeout=1,+ max_size=8 * 1024 * 1024, proxy=None, ping_interval=None))return self.linkdef _drop(self):diff --git a/tests/agent/test_worker_rpc_adopt_timeout.py b/tests/agent/test_worker_rpc_adopt_timeout.pydeleted file mode 100644index 81312d60600..00000000000--- a/tests/agent/test_worker_rpc_adopt_timeout.py+++ /dev/null@@ -1,116 +0,0 @@-"""The ``worker.adopt`` handshake budget must survive a busy owner, and say so when it doesn't.--2026-10-22: the fan gateway was mid-turn (``time=1018.3s api_calls=46``) when a kanban worker's-``worker.adopt`` dial hit ``open_timeout=5`` and raised-``TimeoutError: timed out while waiting for handshake response``. The managed worker died, the-card kept saying ``running`` for 49 minutes, and nothing retried. A 5 s budget against a-single-process gateway that legitimately serves 1000 s turns is a coin flip; the budget is now-60 s by default (``HERMES_WORKER_RPC_OPEN_TIMEOUT`` overrides it) and a TIMED-OUT handshake is-retried with backoff.-"""-from __future__ import annotations--import json-import threading-from types import SimpleNamespace--import pytest---def _serve_owner(monkeypatch):- """A real loopback owner whose only answer is the method name; discovery is stubbed."""- from websockets.sync.server import serve- from hermes_cli import gateway_client, gateway_runtime- import hermes_cli.gateway_runtime_discovery as discovery-- def owner(ws):- request = json.loads(ws.recv())- ws.send(json.dumps({'jsonrpc': '2.0', 'id': request['id'],- 'result': {'method': request['method']}}))-- server = serve(owner, '127.0.0.1', 0, subprotocols=['hermes-gateway-v1'])- threading.Thread(target=server.serve_forever, daemon=True).start()- endpoint = SimpleNamespace(api_origin=f'http://127.0.0.1:{server.socket.getsockname()[1]}',- control_home=None)- monkeypatch.setattr(gateway_runtime, 'discover_gateway_endpoint',- lambda home, timeout: SimpleNamespace(state='ready', endpoint=endpoint))- monkeypatch.setattr(discovery, 'query_identify', lambda home, timeout: {'pid': -1})- monkeypatch.setattr(gateway_client, '_session_ticket', lambda home, endpoint, purpose: 'ticket')- return server---def test_the_handshake_budget_defaults_to_60s_and_is_configurable(monkeypatch, tmp_path):- import websockets.sync.client as ws_client- from agent import runtime_session_store as store-- real_connect, seen = ws_client.connect, {}-- def spy(url, **kwargs):- seen.update(kwargs)- return real_connect(url, **kwargs)-- monkeypatch.setattr(ws_client, 'connect', spy)- server = _serve_owner(monkeypatch)- try:- assert store.WorkerRPC(tmp_path)('worker.persist') == {'method': 'worker.persist'}- assert seen['open_timeout'] == 60.0 # NOT the old 5 s coin flip- monkeypatch.setenv('HERMES_WORKER_RPC_OPEN_TIMEOUT', '90')- assert store.worker_rpc_open_timeout() == 90.0- seen.clear()- assert store.WorkerRPC(tmp_path)('worker.persist') == {'method': 'worker.persist'}- assert seen['open_timeout'] == 90.0- finally:- server.shutdown()---def test_a_timed_out_handshake_is_retried_and_the_call_still_lands(monkeypatch, tmp_path):- import websockets.sync.client as ws_client- from agent import runtime_session_store as store-- real_connect = ws_client.connect- calls = []-- def flaky(url, **kwargs):- calls.append(kwargs['open_timeout'])- if len(calls) < 3:- raise TimeoutError('timed out while waiting for handshake response')- return real_connect(url, **kwargs)-- monkeypatch.setattr(ws_client, 'connect', flaky)- monkeypatch.setattr(store, 'WORKER_RPC_RETRY_SLEEP_SECONDS', 0.0)- server = _serve_owner(monkeypatch)- try:- assert store.WorkerRPC(tmp_path)('worker.adopt') == {'method': 'worker.adopt'}- finally:- server.shutdown()- assert len(calls) == 3---def test_connect_worker_link_backs_off_only_on_timeouts(monkeypatch):- from agent import runtime_session_store as store- delays, attempts = [], []-- def sleeper(seconds):- delays.append(seconds)-- def timeout_twice_then_ok(url, **kwargs):- attempts.append(kwargs['open_timeout'])- if len(attempts) < 3:- raise TimeoutError('busy owner')- return 'link'-- assert store.connect_worker_link(timeout_twice_then_ok, 'ws://x', ['p'],- open_timeout=60, sleep=sleeper) == 'link'- assert delays == [1.0, 2.0]- assert attempts == [60.0, 60.0, 60.0]-- attempts.clear()- delays.clear()-- def auth_failure(url, **kwargs):- attempts.append(kwargs['open_timeout'])- raise OSError('handshake rejected')-- with pytest.raises(OSError):- store.connect_worker_link(auth_failure, 'ws://x', ['p'], open_timeout=60, sleep=sleeper)- assert len(attempts) == 1 and delays == [] # a rejected handshake never heals by retrying