Skip to content

The agent that grows with you

fix(kanban): drop the worker.adopt handshake retry from the lost-worker salvage

kshitijk4poorc91bd232c0d7
diff --git a/agent/runtime_session_store.py b/agent/runtime_session_store.py
index b942bd21e24..f05c66459eb 100644
--- a/agent/runtime_session_store.py
+++ b/agent/runtime_session_store.py
@@ -22,57 +22,6 @@ logger = logging.getLogger(__name__)
_worker_process = False
-# The ``worker.adopt`` handshake is a round trip through the OWNER's event loop, and a live
-# gateway legitimately spends 1000 s+ inside one turn (fan/gateway.log 2026-10-22:
-# ``time=1018.3s api_calls=46``). A 5 s budget lost that race and cost a whole dispatch
-# cycle silently: the managed worker died in ``worker.adopt``, the card kept saying
-# ``running``, and nothing retried. Generous by default, overridable, and a TIMEOUT (never
-# an auth/protocol failure) is retried with backoff.
-WORKER_RPC_OPEN_TIMEOUT_SECONDS = 60.0
-WORKER_RPC_CONNECT_ATTEMPTS = 3
-WORKER_RPC_RETRY_SLEEP_SECONDS = 1.0
-
-
-def worker_rpc_open_timeout():
- """The handshake budget: ``HERMES_WORKER_RPC_OPEN_TIMEOUT`` seconds, else 60 s."""
- # The adopt handshake runs BEFORE the worker may import config (agent/managed_worker.py:
- # "No agent/config imports before adopted process authority"), so no config.yaml key can
- # size it; resolved per call, 60 s default.
- raw = os.environ.get('HERMES_WORKER_RPC_OPEN_TIMEOUT', '').strip() # health: allow HX002 -- the adopt handshake runs before the worker may import config, so no config.yaml key can size it; read per call, 60 s default
- if raw:
- try:
- value = float(raw)
- except ValueError:
- value = 0.0
- if value > 0:
- return value
- return WORKER_RPC_OPEN_TIMEOUT_SECONDS
-
-
-def connect_worker_link(connect, url, protocols, *, open_timeout=None, attempts=None, sleep=None):
- """Open the owner link, retrying a handshake that TIMED OUT because the owner was busy.
-
- Only ``TimeoutError`` is retried: an authentication or protocol failure will not heal with
- another attempt, and re-sending after a completed handshake is never safe. Each attempt
- gets the full budget; ``sleep`` is a test seam.
- """
- import time as _time
- budget = worker_rpc_open_timeout() if open_timeout is None else float(open_timeout)
- tries = max(1, int(WORKER_RPC_CONNECT_ATTEMPTS if attempts is None else attempts))
- pause = _time.sleep if sleep is None else sleep
- last = None
- for attempt in range(tries):
- try:
- return connect(url, subprotocols=protocols, open_timeout=budget, close_timeout=1,
- max_size=8 * 1024 * 1024, proxy=None, ping_interval=None)
- except TimeoutError as exc:
- last = exc
- if attempt + 1 < tries:
- logger.warning('owner handshake timed out after %ss (attempt %d/%d); retrying',
- budget, attempt + 1, tries)
- pause(WORKER_RPC_RETRY_SLEEP_SECONDS * (2 ** attempt))
- raise last
-
def is_worker_process():
"""A successful self-registration makes this interpreter compute-only.
@@ -206,7 +155,8 @@ class WorkerRPC:
# The owner is loopback, like connect_gateway's peer: an inherited HTTPS_PROXY must not carry it.
# No library keepalive: its pong deadline would run independently of (and could fail) a call
# inside its receipt budget; that budget and the observed close are the liveness checks.
- self.link = _OwnerLink(connect_worker_link(connect, url, protocols))
+ self.link = _OwnerLink(connect(url, subprotocols=protocols, open_timeout=5, close_timeout=1,
+ max_size=8 * 1024 * 1024, proxy=None, ping_interval=None))
return self.link
def _drop(self):
diff --git a/tests/agent/test_worker_rpc_adopt_timeout.py b/tests/agent/test_worker_rpc_adopt_timeout.py
deleted file mode 100644
index 81312d60600..00000000000
--- a/tests/agent/test_worker_rpc_adopt_timeout.py
+++ /dev/null
@@ -1,116 +0,0 @@
-"""The ``worker.adopt`` handshake budget must survive a busy owner, and say so when it doesn't.
-
-2026-10-22: the fan gateway was mid-turn (``time=1018.3s api_calls=46``) when a kanban worker's
-``worker.adopt`` dial hit ``open_timeout=5`` and raised
-``TimeoutError: timed out while waiting for handshake response``. The managed worker died, the
-card kept saying ``running`` for 49 minutes, and nothing retried. A 5 s budget against a
-single-process gateway that legitimately serves 1000 s turns is a coin flip; the budget is now
-60 s by default (``HERMES_WORKER_RPC_OPEN_TIMEOUT`` overrides it) and a TIMED-OUT handshake is
-retried with backoff.
-"""
-from __future__ import annotations
-
-import json
-import threading
-from types import SimpleNamespace
-
-import pytest
-
-
-def _serve_owner(monkeypatch):
- """A real loopback owner whose only answer is the method name; discovery is stubbed."""
- from websockets.sync.server import serve
- from hermes_cli import gateway_client, gateway_runtime
- import hermes_cli.gateway_runtime_discovery as discovery
-
- def owner(ws):
- request = json.loads(ws.recv())
- ws.send(json.dumps({'jsonrpc': '2.0', 'id': request['id'],
- 'result': {'method': request['method']}}))
-
- server = serve(owner, '127.0.0.1', 0, subprotocols=['hermes-gateway-v1'])
- threading.Thread(target=server.serve_forever, daemon=True).start()
- endpoint = SimpleNamespace(api_origin=f'http://127.0.0.1:{server.socket.getsockname()[1]}',
- control_home=None)
- monkeypatch.setattr(gateway_runtime, 'discover_gateway_endpoint',
- lambda home, timeout: SimpleNamespace(state='ready', endpoint=endpoint))
- monkeypatch.setattr(discovery, 'query_identify', lambda home, timeout: {'pid': -1})
- monkeypatch.setattr(gateway_client, '_session_ticket', lambda home, endpoint, purpose: 'ticket')
- return server
-
-
-def test_the_handshake_budget_defaults_to_60s_and_is_configurable(monkeypatch, tmp_path):
- import websockets.sync.client as ws_client
- from agent import runtime_session_store as store
-
- real_connect, seen = ws_client.connect, {}
-
- def spy(url, **kwargs):
- seen.update(kwargs)
- return real_connect(url, **kwargs)
-
- monkeypatch.setattr(ws_client, 'connect', spy)
- server = _serve_owner(monkeypatch)
- try:
- assert store.WorkerRPC(tmp_path)('worker.persist') == {'method': 'worker.persist'}
- assert seen['open_timeout'] == 60.0 # NOT the old 5 s coin flip
- monkeypatch.setenv('HERMES_WORKER_RPC_OPEN_TIMEOUT', '90')
- assert store.worker_rpc_open_timeout() == 90.0
- seen.clear()
- assert store.WorkerRPC(tmp_path)('worker.persist') == {'method': 'worker.persist'}
- assert seen['open_timeout'] == 90.0
- finally:
- server.shutdown()
-
-
-def test_a_timed_out_handshake_is_retried_and_the_call_still_lands(monkeypatch, tmp_path):
- import websockets.sync.client as ws_client
- from agent import runtime_session_store as store
-
- real_connect = ws_client.connect
- calls = []
-
- def flaky(url, **kwargs):
- calls.append(kwargs['open_timeout'])
- if len(calls) < 3:
- raise TimeoutError('timed out while waiting for handshake response')
- return real_connect(url, **kwargs)
-
- monkeypatch.setattr(ws_client, 'connect', flaky)
- monkeypatch.setattr(store, 'WORKER_RPC_RETRY_SLEEP_SECONDS', 0.0)
- server = _serve_owner(monkeypatch)
- try:
- assert store.WorkerRPC(tmp_path)('worker.adopt') == {'method': 'worker.adopt'}
- finally:
- server.shutdown()
- assert len(calls) == 3
-
-
-def test_connect_worker_link_backs_off_only_on_timeouts(monkeypatch):
- from agent import runtime_session_store as store
- delays, attempts = [], []
-
- def sleeper(seconds):
- delays.append(seconds)
-
- def timeout_twice_then_ok(url, **kwargs):
- attempts.append(kwargs['open_timeout'])
- if len(attempts) < 3:
- raise TimeoutError('busy owner')
- return 'link'
-
- assert store.connect_worker_link(timeout_twice_then_ok, 'ws://x', ['p'],
- open_timeout=60, sleep=sleeper) == 'link'
- assert delays == [1.0, 2.0]
- assert attempts == [60.0, 60.0, 60.0]
-
- attempts.clear()
- delays.clear()
-
- def auth_failure(url, **kwargs):
- attempts.append(kwargs['open_timeout'])
- raise OSError('handshake rejected')
-
- with pytest.raises(OSError):
- store.connect_worker_link(auth_failure, 'ws://x', ['p'], open_timeout=60, sleep=sleeper)
- assert len(attempts) == 1 and delays == [] # a rejected handshake never heals by retrying