The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(kanban): any transport loss resumes the same attempt, bounded from the owner's last answer
kshitijk4poor89f13590e0e8
diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.pyindex 9a633970f9f..cd2f4d68fbe 100644--- a/hermes_cli/kanban_worker_client.py+++ b/hermes_cli/kanban_worker_client.py@@ -19,26 +19,13 @@ import time# until an owner is ready, so waiting can never run the claim twice.OWNER_WAIT_S = 180.0_TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})-# An owner that is up but draining refuses in band (JSON-RPC error), before admitting anything.-_TRANSIENT_RPC_REASONS = frozenset({'runtime_draining'})+# In-band owner refusals that leave the attempt intact: a draining owner admits nothing new, and a+# retired authority's successor holds the admission the next connection reads.+_TRANSIENT_RPC_REASONS = frozenset({'runtime_draining', 'runtime_retired'})# Past the board lock wait: the owner's session setup and receipt before and after the write.ADMISSION_ANSWER_MARGIN_S = 60.0-class _Redial(Exception):- """The socket died after the owner admitted the claim: reconnect and re-read the receipt.-- The admission id and session id are the durable handle for the attempt (the owner keeps the- admission across client connections), so a dead socket is a transport event rather than a turn- outcome. Exiting instead would let ``_watch_submitter`` end the owner's in-flight turn.- """-- def __init__(self, admission):- super().__init__('gateway connection lost after the owner admitted the claim')- self.admission = admission---def admission_answer_budget():"""Seconds to wait for ``kanban.run``'s answer: the longest the owner's admission write canwait on the board lock (``write_txn`` retries ``BEGIN IMMEDIATE`` ``_BUSY_MAX_RETRIES`` times,@@ -49,56 +36,45 @@ def admission_answer_budget():async def run(params, board_db):+ from agent.retry_utils import jittered_backofffrom hermes_cli.gateway_client import (connect_gateway, GatewayClientError, GatewayRPCError,GatewayUnavailableError)+ from websockets.exceptions import ConnectionClosedif os.environ.get('HERMES_TUI_GATEWAY_URL'):raise GatewayClientError('Kanban requires the assigned local profile owner')- deadline = time.monotonic() + OWNER_WAIT_S- # A lost socket is not a lost attempt: after the owner admits the claim, the admission id is- # what the next connection re-reads. Bounded once, so a flapping gateway cannot hold the worker- # for ever and a truly gone owner still fails honestly.- admission = None- redial_deadline = None- # A connection alone is not progress: receipt reads must succeed before a new- # recovery episode can receive a fresh bounded budget.- progress = [False]+ # A lost socket is not a lost attempt, and exiting here would let ``_watch_submitter`` end the+ # owner's in-flight turn. ``kanban.run`` is idempotent per attempt (one request id per card run)+ # and ``prompt.receipt`` is a read, so the next connection resumes whatever the last one reached.+ # The wait is bounded from the owner's last answer: an owner that keeps answering keeps the+ # worker, and one that stays silent for ``OWNER_WAIT_S`` fails the worker honestly.+ claim = {'answered': time.monotonic()}+ failures, answered = 0, claim['answered']while True:try:async with connect_gateway() as client:- progress[0] = False# kanban.run subscribes this socket to the card's session, but only receipts matter# here: discard the turn's events, or they fill the client's bounded queue and its# reader stops, with the next receipt behind it.drain = asyncio.create_task(_discard(client.events))try:- return await _submit_claim(client, params, board_db, admission, progress)+ receipt = await _await_receipt(client, params, board_db, claim)finally:drain.cancel()- except _Redial as exc:- admission = exc.admission- if progress[0]:- # A successful receipt read proves the prior recovery episode made progress.- redial_deadline = None- if redial_deadline is None:- redial_deadline = time.monotonic() + OWNER_WAIT_S- if time.monotonic() >= redial_deadline:- # The attempt stays readable for its owner; failing honestly beats resubmitting it.- raise GatewayClientError('Kanban owner connection lost after admission: the attempt '- 'is retained by the owner; do not resend the work') from None- await asyncio.sleep(1)except GatewayRPCError as exc:- # Nothing was submitted when the owner refused the claim, so this is the host's state.- if admission is not None or str(exc) not in _TRANSIENT_RPC_REASONS or time.monotonic() >= deadline:+ if str(exc) not in _TRANSIENT_RPC_REASONS or time.monotonic() - claim['answered'] >= OWNER_WAIT_S:raise- await asyncio.sleep(1)except GatewayUnavailableError as exc:- # Before admission, use the initial owner-startup deadline. While recovering an- # admitted attempt, use that recovery episode's deadline instead.- wait_deadline = (redial_deadline if admission is not None and redial_deadline is not None- else deadline)- if exc.state not in _TRANSIENT_OWNER_STATES or time.monotonic() >= wait_deadline:+ if exc.state not in _TRANSIENT_OWNER_STATES or time.monotonic() - claim['answered'] >= OWNER_WAIT_S:raise- await asyncio.sleep(1)+ except (GatewayClientError, ConnectionClosed, OSError, TimeoutError) as exc:+ if time.monotonic() - claim['answered'] >= OWNER_WAIT_S:+ raise GatewayClientError(f'Kanban owner unreachable for {OWNER_WAIT_S:.0f}s: {exc}') from exc+ else:+ return _exit_code(receipt, params, board_db)+ if claim['answered'] != answered:+ failures, answered = 0, claim['answered']+ failures += 1+ await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0))async def _discard(events):@@ -106,43 +82,35 @@ async def _discard(events):await events.get()-async def _receipt(client, session_id, admission_id):- """One ``prompt.receipt`` read. A transport loss becomes a redial; an owner's answer does not."""- from hermes_cli.gateway_client import GatewayClientError, GatewayRPCError- from websockets.exceptions import ConnectionClosed- try:- return await client.rpc('prompt.receipt', session_id=session_id, admission_id=admission_id)- except GatewayRPCError:- raise- except (GatewayClientError, ConnectionClosed, OSError, TimeoutError):- raise _Redial((session_id, admission_id)) from None---async def _submit_claim(client, params, board_db, admission=None, progress=None):- from hermes_cli.gateway_client import GatewayClientError- db = str(await asyncio.to_thread(Path(board_db).resolve))- # The owner answers only after it checks the claim and writes its ``owner_admitted`` marker- # in one board write transaction. The board (the cross-profile dispatch bus) can make that- # wait out its busy timeout, so the 30 s default left the owner admitting an attempt whose- # submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the- # admission was left ``unknown``). The wait is still finite: once the marker commits,- # ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner- # that stalls after admitting is bounded only here. A hang-up fails the wait sooner.- if admission is None:+async def _await_receipt(client, params, board_db, claim):+ """Submit the claim (or, on a later connection, re-read the admission it got) and return its+ settled receipt. *claim* carries the owner's ``session_id``/``admission_id`` and the monotonic+ time of its last answer across connections."""+ if 'admission_id' not in claim:+ db = str(await asyncio.to_thread(Path(board_db).resolve))+ # The owner answers only after it checks the claim and writes its ``owner_admitted`` marker+ # in one board write transaction. The board (the cross-profile dispatch bus) can make that+ # wait out its busy timeout, so the 30 s default left the owner admitting an attempt whose+ # submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the+ # admission was left ``unknown``). The wait is still finite: once the marker commits,+ # ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner+ # that stalls after admitting is bounded only here. A hang-up fails the wait sooner.accepted = await client.rpc('kanban.run', _timeout=admission_answer_budget(), **dict(params, db=db))- sid, receipt = accepted['session_id'], accepted['receipt']- print(f'Session: {sid}', file=sys.stderr)- admission = (sid, receipt['admission_id'])+ receipt = accepted['receipt']+ claim.update(session_id=accepted['session_id'], admission_id=receipt['admission_id'])+ print(f'Session: {claim["session_id"]}', file=sys.stderr)else:- # A redial re-reads the attempt the owner already holds; it never submits the claim again.- receipt = await _receipt(client, *admission)- if progress is not None:- progress[0] = True+ receipt = await _receipt(client, claim)+ claim['answered'] = time.monotonic()while receipt['status'] not in {'terminal', 'unknown'}:await asyncio.sleep(.2)- receipt = await _receipt(client, *admission)- if progress is not None:- progress[0] = True+ receipt = await _receipt(client, claim)+ claim['answered'] = time.monotonic()+ return receipt+++def _exit_code(receipt, params, board_db):+ from hermes_cli.gateway_client import GatewayClientErrorif receipt['status'] == 'unknown':raise GatewayClientError('Kanban execution unknown; retained by owner')from gateway.session_kanban import worker_result@@ -153,6 +121,10 @@ async def _submit_claim(client, params, board_db, admission=None, progress=None)return result.get('exit_code', 1)+async def _receipt(client, claim):+ return await client.rpc('prompt.receipt', session_id=claim['session_id'], admission_id=claim['admission_id'])++def main():# The dead-worker sweep books this process's exit from the trailer in its log (it never# reaps us), so every exit path writes it — the same contract as a quiet one-shot worker.diff --git a/tests/gateway/fixtures/kanban_recovery_probe.py b/tests/gateway/fixtures/kanban_recovery_probe.pyindex 721d1b82c11..53873b76c5b 100644--- a/tests/gateway/fixtures/kanban_recovery_probe.py+++ b/tests/gateway/fixtures/kanban_recovery_probe.py@@ -133,6 +133,9 @@ def main():wait_for(lambda: admission()['status'] == 'terminal')wait_for(lambda: kb.get_task(conn, tid).status == 'done')assert asyncio.run(retry(desc, params))['session_id'] == first['target_session_id']+ # The worker reads its terminal receipt from this owner: one still polling when the+ # owner stops redials, and the redial starts a replacement owner (a client's own start).+ wait_for(lambda: all(p.poll() is not None for p in clients))if mode == 'restart':cfg['model']['default'] = 'changed-default'(owner / 'config.yaml').write_text(json.dumps(cfg))diff --git a/tests/hermes_cli/test_kanban_worker_admission_wait.py b/tests/hermes_cli/test_kanban_worker_admission_wait.pyindex e46dc238b12..5ddc601f0a8 100644--- a/tests/hermes_cli/test_kanban_worker_admission_wait.py+++ b/tests/hermes_cli/test_kanban_worker_admission_wait.py@@ -49,7 +49,8 @@ async def test_a_slow_admission_answer_is_awaited_not_failed(monkeypatch, tmp_paowner = _Owner()params = {'board': 'b', 'task_id': 't1', 'run_id': 1, 'claim_lock': 'lock'}async with GatewayClient(owner) as client:- code = await kanban_worker_client._submit_claim(client, params, str(tmp_path / 'kanban.db'))+ receipt = await kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), {})+ code = kanban_worker_client._exit_code(receipt, params, str(tmp_path / 'kanban.db'))assert code == 0# One submission: the wait never resends the claim, and the budget never goes on the wire.assert [r['method'] for r in owner.sent] == ['kanban.run']@@ -84,5 +85,5 @@ async def test_an_owner_that_stalls_after_admitting_fails_the_wait_within_the_loasync with GatewayClient(owner) as client:with pytest.raises(TimeoutError):# The outer 5 s guard only stops an unbounded wait from hanging the test.- await real_wait_for(kanban_worker_client._submit_claim(client, params, str(tmp_path / 'kanban.db')), 5)+ await real_wait_for(kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), {}), 5)assert budgets == [budget] and [r['method'] for r in owner.sent] == ['kanban.run']diff --git a/tests/hermes_cli/test_kanban_worker_socket_recovery.py b/tests/hermes_cli/test_kanban_worker_socket_recovery.pyindex ebbcfa32dc6..aa81e1a1707 100644--- a/tests/hermes_cli/test_kanban_worker_socket_recovery.py+++ b/tests/hermes_cli/test_kanban_worker_socket_recovery.py@@ -1,10 +1,10 @@-"""A Kanban worker whose gateway socket dies mid-turn redials and re-reads the same receipt.+"""A Kanban worker whose gateway socket dies mid-turn resumes the same attempt on a new connection.-A dead socket is a transport event, not a turn outcome: the owner keeps the admission and answers-``prompt.receipt`` over the next connection, so exiting 1 there loses work that was already running-(``_watch_submitter`` then ends the owner's in-flight turn). Measured before the fix: 93 runs booked-``crashed`` on a gateway disconnect, 80 of them already admitted and bound, 0 with a-``worker_result``. An owner's own refusal is still an answer, and a genuine refusal still fails.+A dead socket is a transport event, not a turn outcome: the owner keeps the admission, answers+``prompt.receipt`` over the next connection and returns the same admission for a repeated+``kanban.run`` (one request id per attempt). Exiting 1 instead loses work that was already running,+because ``_watch_submitter`` then ends the owner's in-flight turn (#145062). An owner's own refusal+is still an answer, and an owner that stays silent still fails the worker."""import asyncio@@ -15,27 +15,45 @@ import pytestfrom hermes_cli import gateway_client, kanban_worker_client+class _Clock:+ """Monotonic time that moves only when the client sleeps or a scripted step advances it."""++ def __init__(self):+ self.now = 0.0++ def monotonic(self):+ return self.now++class _Owner:- """Scripted owner: ``connects`` holds one entry per dial (an exception fails the dial), and- ``receipts`` holds one entry per ``prompt.receipt`` read (an exception is a transport loss).- ``submit_failures`` holds entries consumed by ``kanban.run``."""-- def __init__(self, connects=None, receipts=None, submit_failures=None):- self.connects = list(connects or [])- self.receipts = list(receipts or [])- self.submit_failures = list(submit_failures or [])+ """Scripted owner. ``connects`` holds one step per dial, ``receipts`` one per ``prompt.receipt``+ and ``submits`` one per ``kanban.run``; a step is None (succeed), an exception (raised) or a+ callable (run first, so it can advance the clock or raise). ``refuse_dials_after`` fails every+ dial past the scripted ones with a refused connection."""++ def __init__(self, connects=(), receipts=(), submits=(), refuse_dials_after=False):+ self.connects, self.receipts, self.submits = list(connects), list(receipts), list(submits)+ self.refuse_dials_after = refuse_dials_afterself.calls = []self.dials = 0+ @staticmethod+ def _step(steps, default=None):+ step = steps.pop(0) if steps else default+ if callable(step) and not isinstance(step, Exception):+ step = step()+ if isinstance(step, Exception):+ raise step+ return step+def connect(self):owner = self@asynccontextmanagerasync def _connect():owner.dials += 1- step = owner.connects.pop(0) if owner.connects else None- if isinstance(step, Exception):- raise step+ refused = gateway_client.GatewayClientError("Gateway connection failed; no local fallback")+ owner._step(owner.connects, refused if owner.refuse_dials_after else None)class _Client:def __init__(self):@@ -44,14 +62,10 @@ class _Owner:async def rpc(self, method, **params):owner.calls.append(method)if method == "kanban.run":- failure = owner.submit_failures.pop(0) if owner.submit_failures else None- if failure is not None:- raise failure+ owner._step(owner.submits)return {"session_id": "s1", "receipt": {"status": "pending", "admission_id": "a1"}}- status = owner.receipts.pop(0) if owner.receipts else "terminal"- if isinstance(status, Exception):- raise status- return {"status": status}+ assert params == {"session_id": "s1", "admission_id": "a1"}+ return {"status": owner._step(owner.receipts, "terminal")}yield _Client()@@ -63,128 +77,91 @@ def _disconnect():"Gateway disconnected; turn outcome is unknown. Resume the printed session ID")+def clock(monkeypatch):+ clock = _Clock()+ monkeypatch.setattr(kanban_worker_client, "time", clock)++ async def _sleep(seconds):+ clock.now += seconds+ monkeypatch.setattr(kanban_worker_client.asyncio, "sleep", _sleep)+ return clock++@pytest.fixture(autouse=True)def _worker_env(monkeypatch, tmp_path):monkeypatch.delenv("HERMES_TUI_GATEWAY_URL", raising=False)- monkeypatch.setattr(kanban_worker_client.asyncio, "sleep", _no_sleep)from gateway import session_kanbanmonkeypatch.setattr(session_kanban, "worker_result",lambda *_: {"exit_code": 0, "last_output": "card closed"})return tmp_path-async def _no_sleep(_seconds):- return None+async def _run(owner, monkeypatch, tmp_path):+ monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())+ return await kanban_worker_client.run({"task_id": "t1"}, str(tmp_path / "kanban.db"))@pytest.mark.asyncio-async def test_a_socket_lost_mid_turn_redials_and_settles_the_same_attempt(monkeypatch, _worker_env):- owner = _Owner(connects=[None, None],- receipts=["pending", _disconnect(), "terminal"])- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())-- assert await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db")) == 0- # The claim is submitted once: the redial re-reads the receipt, it does not resubmit.[email protected]("redial", [+ None,+ gateway_client.GatewayClientError("Gateway connection failed; no local fallback"),+ ConnectionRefusedError(61, "refused"),+ gateway_client.GatewayUnavailableError("Gateway starting: not_ready", "starting"),+], ids=["reconnects", "dial-refused", "ticket-oserror", "owner-starting"])+async def test_a_socket_lost_mid_turn_resumes_the_same_attempt(monkeypatch, _worker_env, clock, redial):+ owner = _Owner(connects=[None, redial, None], receipts=["pending", _disconnect(), "terminal"])++ assert await _run(owner, monkeypatch, _worker_env) == 0+ # The claim is submitted once: every later connection re-reads the receipt.assert owner.calls.count("kanban.run") == 1- assert owner.dials == 2+ assert owner.dials == (2 if redial is None else 3)@pytest.mark.asyncio-async def test_a_draining_owner_is_retried_before_anything_is_admitted(monkeypatch, _worker_env):- owner = _Owner(connects=[None, None],- submit_failures=[gateway_client.GatewayRPCError("runtime_draining")],- receipts=["terminal"])- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())-- assert await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db")) == 0[email protected]("lost", [_disconnect(), gateway_client.GatewayRPCError("runtime_draining")],+ ids=["answer-lost", "draining"])+async def test_a_claim_with_no_answer_is_resubmitted(monkeypatch, _worker_env, clock, lost):+ # No admission id reached the worker, so it asks again; the owner answers a repeated+ # kanban.run for the same attempt with the same admission.+ owner = _Owner(connects=[None, None], submits=[lost], receipts=["terminal"])++ assert await _run(owner, monkeypatch, _worker_env) == 0assert owner.calls.count("kanban.run") == 2@pytest.mark.asyncio-async def test_an_owner_refusal_is_not_a_redial(monkeypatch, _worker_env):- owner = _Owner(connects=[None, None],- submit_failures=[gateway_client.GatewayRPCError("claim_mismatch")])- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())-- with pytest.raises(gateway_client.GatewayRPCError):- await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db"))[email protected]("where", ["kanban.run", "prompt.receipt"])+async def test_an_owner_refusal_is_not_retried(monkeypatch, _worker_env, clock, where):+ refusal = gateway_client.GatewayRPCError("claim_mismatch")+ owner = (_Owner(submits=[refusal]) if where == "kanban.run"+ else _Owner(receipts=["pending", refusal]))++ with pytest.raises(gateway_client.GatewayRPCError, match="claim_mismatch"):+ await _run(owner, monkeypatch, _worker_env)assert owner.dials == 1@pytest.mark.asyncio-async def test_a_gateway_that_never_comes_back_fails_honestly(monkeypatch, _worker_env):- owner = _Owner(connects=[None, gateway_client.GatewayUnavailableError("no owner", "inaccessible")],- receipts=["pending", _disconnect()])- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())-- with pytest.raises(gateway_client.GatewayClientError):- await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db"))---class _Clock:- def __init__(self, values):- self.values = iter(values)- self.last = values[-1]+async def test_a_later_loss_gets_a_fresh_budget_from_the_owners_last_answer(monkeypatch, _worker_env, clock):+ def silent_then_lost():+ clock.now += kanban_worker_client.OWNER_WAIT_S - 10+ return _disconnect()- def monotonic(self):- try:- self.last = next(self.values)- except StopIteration:- pass- return self.last---async def test_reconnect_starting_uses_redial_deadline_not_initial_deadline(monkeypatch, _worker_env):- # Initial wait expires at t=180, but the admitted attempt's redial budget runs until t=361.- monkeypatch.setattr(kanban_worker_client, "time", _Clock([0, 181, 181, 200]))- owner = _Owner(- connects=[None, gateway_client.GatewayUnavailableError("owner starting", "starting"), None],- receipts=["pending", _disconnect(), "terminal"],- )- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())-- assert await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db")) == 0- assert owner.calls.count("kanban.run") == 1- assert owner.dials == 3---async def test_a_later_disconnect_gets_a_fresh_redial_budget(monkeypatch, _worker_env):- # The second disconnect occurs at t=200, beyond the first episode's t=190 deadline.- monkeypatch.setattr(kanban_worker_client, "time", _Clock([0, 10, 10, 200, 200]))- owner = _Owner(- connects=[None, None, None],- receipts=["pending", _disconnect(), "pending", _disconnect(), "terminal"],- )- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())-- assert await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db")) == 0- assert owner.calls.count("kanban.run") == 1- assert owner.dials == 3---async def test_repeated_receipt_disconnects_cannot_reset_the_redial_budget(monkeypatch, _worker_env):- # Every dial succeeds, but receipt reads disconnect. Advancing monotonic time- # by 100 seconds per check must exhaust one recovery budget rather than reset it.- monkeypatch.setattr(kanban_worker_client, "time", _Clock([0, 100, 200, 300, 400]))+ # Each loss comes 170 s after the last answer; across both, far more than OWNER_WAIT_S passes.owner = _Owner(connects=[None, None, None],- receipts=[_disconnect(), _disconnect(), _disconnect()])- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())+ receipts=["pending", silent_then_lost, "pending", silent_then_lost, "terminal"])- with pytest.raises(gateway_client.GatewayClientError, match="after admission"):- await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db"))- assert owner.calls.count("kanban.run") == 1- assert owner.dials == 2+ assert await _run(owner, monkeypatch, _worker_env) == 0+ assert owner.dials == 3 and owner.calls.count("kanban.run") == 1@pytest.mark.asyncio-async def test_the_redial_budget_bounds_a_flapping_gateway(monkeypatch, _worker_env):- monkeypatch.setattr(kanban_worker_client, "OWNER_WAIT_S", 0.0)- owner = _Owner(connects=[None, None], receipts=["pending", _disconnect(), "terminal"])- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())-- with pytest.raises(gateway_client.GatewayClientError):- await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db"))- assert owner.calls.count("kanban.run") == 1+async def test_an_owner_silent_for_the_whole_budget_fails_the_worker(monkeypatch, _worker_env, clock):+ owner = _Owner(connects=[None], receipts=["pending", _disconnect()], refuse_dials_after=True)++ with pytest.raises(gateway_client.GatewayClientError, match="unreachable for 180s"):+ await _run(owner, monkeypatch, _worker_env)+ assert clock.now >= kanban_worker_client.OWNER_WAIT_S+ # Jittered exponential backoff (capped at 8-12 s) between dials, not one dial a second.+ assert 3 < owner.dials < 40 and owner.calls.count("kanban.run") == 1