Skip to content

The agent that grows with you

fix(kanban): any transport loss resumes the same attempt, bounded from the owner's last answer

kshitijk4poor89f13590e0e8
diff --git a/hermes_cli/kanban_worker_client.py b/hermes_cli/kanban_worker_client.py
index 9a633970f9f..cd2f4d68fbe 100644
--- a/hermes_cli/kanban_worker_client.py
+++ b/hermes_cli/kanban_worker_client.py
@@ -19,26 +19,13 @@ import time
# until an owner is ready, so waiting can never run the claim twice.
OWNER_WAIT_S = 180.0
_TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})
-# An owner that is up but draining refuses in band (JSON-RPC error), before admitting anything.
-_TRANSIENT_RPC_REASONS = frozenset({'runtime_draining'})
+# In-band owner refusals that leave the attempt intact: a draining owner admits nothing new, and a
+# retired authority's successor holds the admission the next connection reads.
+_TRANSIENT_RPC_REASONS = frozenset({'runtime_draining', 'runtime_retired'})
# Past the board lock wait: the owner's session setup and receipt before and after the write.
ADMISSION_ANSWER_MARGIN_S = 60.0
-class _Redial(Exception):
- """The socket died after the owner admitted the claim: reconnect and re-read the receipt.
-
- The admission id and session id are the durable handle for the attempt (the owner keeps the
- admission across client connections), so a dead socket is a transport event rather than a turn
- outcome. Exiting instead would let ``_watch_submitter`` end the owner's in-flight turn.
- """
-
- def __init__(self, admission):
- super().__init__('gateway connection lost after the owner admitted the claim')
- self.admission = admission
-
-
-
def admission_answer_budget():
"""Seconds to wait for ``kanban.run``'s answer: the longest the owner's admission write can
wait on the board lock (``write_txn`` retries ``BEGIN IMMEDIATE`` ``_BUSY_MAX_RETRIES`` times,
@@ -49,56 +36,45 @@ def admission_answer_budget():
async def run(params, board_db):
+ from agent.retry_utils import jittered_backoff
from hermes_cli.gateway_client import (connect_gateway, GatewayClientError, GatewayRPCError,
GatewayUnavailableError)
+ from websockets.exceptions import ConnectionClosed
if os.environ.get('HERMES_TUI_GATEWAY_URL'):
raise GatewayClientError('Kanban requires the assigned local profile owner')
- deadline = time.monotonic() + OWNER_WAIT_S
- # A lost socket is not a lost attempt: after the owner admits the claim, the admission id is
- # what the next connection re-reads. Bounded once, so a flapping gateway cannot hold the worker
- # for ever and a truly gone owner still fails honestly.
- admission = None
- redial_deadline = None
- # A connection alone is not progress: receipt reads must succeed before a new
- # recovery episode can receive a fresh bounded budget.
- progress = [False]
+ # A lost socket is not a lost attempt, and exiting here would let ``_watch_submitter`` end the
+ # owner's in-flight turn. ``kanban.run`` is idempotent per attempt (one request id per card run)
+ # and ``prompt.receipt`` is a read, so the next connection resumes whatever the last one reached.
+ # The wait is bounded from the owner's last answer: an owner that keeps answering keeps the
+ # worker, and one that stays silent for ``OWNER_WAIT_S`` fails the worker honestly.
+ claim = {'answered': time.monotonic()}
+ failures, answered = 0, claim['answered']
while True:
try:
async with connect_gateway() as client:
- progress[0] = False
# kanban.run subscribes this socket to the card's session, but only receipts matter
# here: discard the turn's events, or they fill the client's bounded queue and its
# reader stops, with the next receipt behind it.
drain = asyncio.create_task(_discard(client.events))
try:
- return await _submit_claim(client, params, board_db, admission, progress)
+ receipt = await _await_receipt(client, params, board_db, claim)
finally:
drain.cancel()
- except _Redial as exc:
- admission = exc.admission
- if progress[0]:
- # A successful receipt read proves the prior recovery episode made progress.
- redial_deadline = None
- if redial_deadline is None:
- redial_deadline = time.monotonic() + OWNER_WAIT_S
- if time.monotonic() >= redial_deadline:
- # The attempt stays readable for its owner; failing honestly beats resubmitting it.
- raise GatewayClientError('Kanban owner connection lost after admission: the attempt '
- 'is retained by the owner; do not resend the work') from None
- await asyncio.sleep(1)
except GatewayRPCError as exc:
- # Nothing was submitted when the owner refused the claim, so this is the host's state.
- if admission is not None or str(exc) not in _TRANSIENT_RPC_REASONS or time.monotonic() >= deadline:
+ if str(exc) not in _TRANSIENT_RPC_REASONS or time.monotonic() - claim['answered'] >= OWNER_WAIT_S:
raise
- await asyncio.sleep(1)
except GatewayUnavailableError as exc:
- # Before admission, use the initial owner-startup deadline. While recovering an
- # admitted attempt, use that recovery episode's deadline instead.
- wait_deadline = (redial_deadline if admission is not None and redial_deadline is not None
- else deadline)
- if exc.state not in _TRANSIENT_OWNER_STATES or time.monotonic() >= wait_deadline:
+ if exc.state not in _TRANSIENT_OWNER_STATES or time.monotonic() - claim['answered'] >= OWNER_WAIT_S:
raise
- await asyncio.sleep(1)
+ except (GatewayClientError, ConnectionClosed, OSError, TimeoutError) as exc:
+ if time.monotonic() - claim['answered'] >= OWNER_WAIT_S:
+ raise GatewayClientError(f'Kanban owner unreachable for {OWNER_WAIT_S:.0f}s: {exc}') from exc
+ else:
+ return _exit_code(receipt, params, board_db)
+ if claim['answered'] != answered:
+ failures, answered = 0, claim['answered']
+ failures += 1
+ await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0))
async def _discard(events):
@@ -106,43 +82,35 @@ async def _discard(events):
await events.get()
-async def _receipt(client, session_id, admission_id):
- """One ``prompt.receipt`` read. A transport loss becomes a redial; an owner's answer does not."""
- from hermes_cli.gateway_client import GatewayClientError, GatewayRPCError
- from websockets.exceptions import ConnectionClosed
- try:
- return await client.rpc('prompt.receipt', session_id=session_id, admission_id=admission_id)
- except GatewayRPCError:
- raise
- except (GatewayClientError, ConnectionClosed, OSError, TimeoutError):
- raise _Redial((session_id, admission_id)) from None
-
-
-async def _submit_claim(client, params, board_db, admission=None, progress=None):
- from hermes_cli.gateway_client import GatewayClientError
- db = str(await asyncio.to_thread(Path(board_db).resolve))
- # The owner answers only after it checks the claim and writes its ``owner_admitted`` marker
- # in one board write transaction. The board (the cross-profile dispatch bus) can make that
- # wait out its busy timeout, so the 30 s default left the owner admitting an attempt whose
- # submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the
- # admission was left ``unknown``). The wait is still finite: once the marker commits,
- # ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner
- # that stalls after admitting is bounded only here. A hang-up fails the wait sooner.
- if admission is None:
+async def _await_receipt(client, params, board_db, claim):
+ """Submit the claim (or, on a later connection, re-read the admission it got) and return its
+ settled receipt. *claim* carries the owner's ``session_id``/``admission_id`` and the monotonic
+ time of its last answer across connections."""
+ if 'admission_id' not in claim:
+ db = str(await asyncio.to_thread(Path(board_db).resolve))
+ # The owner answers only after it checks the claim and writes its ``owner_admitted`` marker
+ # in one board write transaction. The board (the cross-profile dispatch bus) can make that
+ # wait out its busy timeout, so the 30 s default left the owner admitting an attempt whose
+ # submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the
+ # admission was left ``unknown``). The wait is still finite: once the marker commits,
+ # ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner
+ # that stalls after admitting is bounded only here. A hang-up fails the wait sooner.
accepted = await client.rpc('kanban.run', _timeout=admission_answer_budget(), **dict(params, db=db))
- sid, receipt = accepted['session_id'], accepted['receipt']
- print(f'Session: {sid}', file=sys.stderr)
- admission = (sid, receipt['admission_id'])
+ receipt = accepted['receipt']
+ claim.update(session_id=accepted['session_id'], admission_id=receipt['admission_id'])
+ print(f'Session: {claim["session_id"]}', file=sys.stderr)
else:
- # A redial re-reads the attempt the owner already holds; it never submits the claim again.
- receipt = await _receipt(client, *admission)
- if progress is not None:
- progress[0] = True
+ receipt = await _receipt(client, claim)
+ claim['answered'] = time.monotonic()
while receipt['status'] not in {'terminal', 'unknown'}:
await asyncio.sleep(.2)
- receipt = await _receipt(client, *admission)
- if progress is not None:
- progress[0] = True
+ receipt = await _receipt(client, claim)
+ claim['answered'] = time.monotonic()
+ return receipt
+
+
+def _exit_code(receipt, params, board_db):
+ from hermes_cli.gateway_client import GatewayClientError
if receipt['status'] == 'unknown':
raise GatewayClientError('Kanban execution unknown; retained by owner')
from gateway.session_kanban import worker_result
@@ -153,6 +121,10 @@ async def _submit_claim(client, params, board_db, admission=None, progress=None)
return result.get('exit_code', 1)
+async def _receipt(client, claim):
+ return await client.rpc('prompt.receipt', session_id=claim['session_id'], admission_id=claim['admission_id'])
+
+
def main():
# The dead-worker sweep books this process's exit from the trailer in its log (it never
# reaps us), so every exit path writes it — the same contract as a quiet one-shot worker.
diff --git a/tests/gateway/fixtures/kanban_recovery_probe.py b/tests/gateway/fixtures/kanban_recovery_probe.py
index 721d1b82c11..53873b76c5b 100644
--- a/tests/gateway/fixtures/kanban_recovery_probe.py
+++ b/tests/gateway/fixtures/kanban_recovery_probe.py
@@ -133,6 +133,9 @@ def main():
wait_for(lambda: admission()['status'] == 'terminal')
wait_for(lambda: kb.get_task(conn, tid).status == 'done')
assert asyncio.run(retry(desc, params))['session_id'] == first['target_session_id']
+ # The worker reads its terminal receipt from this owner: one still polling when the
+ # owner stops redials, and the redial starts a replacement owner (a client's own start).
+ wait_for(lambda: all(p.poll() is not None for p in clients))
if mode == 'restart':
cfg['model']['default'] = 'changed-default'
(owner / 'config.yaml').write_text(json.dumps(cfg))
diff --git a/tests/hermes_cli/test_kanban_worker_admission_wait.py b/tests/hermes_cli/test_kanban_worker_admission_wait.py
index e46dc238b12..5ddc601f0a8 100644
--- a/tests/hermes_cli/test_kanban_worker_admission_wait.py
+++ b/tests/hermes_cli/test_kanban_worker_admission_wait.py
@@ -49,7 +49,8 @@ async def test_a_slow_admission_answer_is_awaited_not_failed(monkeypatch, tmp_pa
owner = _Owner()
params = {'board': 'b', 'task_id': 't1', 'run_id': 1, 'claim_lock': 'lock'}
async with GatewayClient(owner) as client:
- code = await kanban_worker_client._submit_claim(client, params, str(tmp_path / 'kanban.db'))
+ receipt = await kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), {})
+ code = kanban_worker_client._exit_code(receipt, params, str(tmp_path / 'kanban.db'))
assert code == 0
# One submission: the wait never resends the claim, and the budget never goes on the wire.
assert [r['method'] for r in owner.sent] == ['kanban.run']
@@ -84,5 +85,5 @@ async def test_an_owner_that_stalls_after_admitting_fails_the_wait_within_the_lo
async with GatewayClient(owner) as client:
with pytest.raises(TimeoutError):
# The outer 5 s guard only stops an unbounded wait from hanging the test.
- await real_wait_for(kanban_worker_client._submit_claim(client, params, str(tmp_path / 'kanban.db')), 5)
+ await real_wait_for(kanban_worker_client._await_receipt(client, params, str(tmp_path / 'kanban.db'), {}), 5)
assert budgets == [budget] and [r['method'] for r in owner.sent] == ['kanban.run']
diff --git a/tests/hermes_cli/test_kanban_worker_socket_recovery.py b/tests/hermes_cli/test_kanban_worker_socket_recovery.py
index ebbcfa32dc6..aa81e1a1707 100644
--- a/tests/hermes_cli/test_kanban_worker_socket_recovery.py
+++ b/tests/hermes_cli/test_kanban_worker_socket_recovery.py
@@ -1,10 +1,10 @@
-"""A Kanban worker whose gateway socket dies mid-turn redials and re-reads the same receipt.
+"""A Kanban worker whose gateway socket dies mid-turn resumes the same attempt on a new connection.
-A dead socket is a transport event, not a turn outcome: the owner keeps the admission and answers
-``prompt.receipt`` over the next connection, so exiting 1 there loses work that was already running
-(``_watch_submitter`` then ends the owner's in-flight turn). Measured before the fix: 93 runs booked
-``crashed`` on a gateway disconnect, 80 of them already admitted and bound, 0 with a
-``worker_result``. An owner's own refusal is still an answer, and a genuine refusal still fails.
+A dead socket is a transport event, not a turn outcome: the owner keeps the admission, answers
+``prompt.receipt`` over the next connection and returns the same admission for a repeated
+``kanban.run`` (one request id per attempt). Exiting 1 instead loses work that was already running,
+because ``_watch_submitter`` then ends the owner's in-flight turn (#145062). An owner's own refusal
+is still an answer, and an owner that stays silent still fails the worker.
"""
import asyncio
@@ -15,27 +15,45 @@ import pytest
from hermes_cli import gateway_client, kanban_worker_client
+class _Clock:
+ """Monotonic time that moves only when the client sleeps or a scripted step advances it."""
+
+ def __init__(self):
+ self.now = 0.0
+
+ def monotonic(self):
+ return self.now
+
+
class _Owner:
- """Scripted owner: ``connects`` holds one entry per dial (an exception fails the dial), and
- ``receipts`` holds one entry per ``prompt.receipt`` read (an exception is a transport loss).
- ``submit_failures`` holds entries consumed by ``kanban.run``."""
-
- def __init__(self, connects=None, receipts=None, submit_failures=None):
- self.connects = list(connects or [])
- self.receipts = list(receipts or [])
- self.submit_failures = list(submit_failures or [])
+ """Scripted owner. ``connects`` holds one step per dial, ``receipts`` one per ``prompt.receipt``
+ and ``submits`` one per ``kanban.run``; a step is None (succeed), an exception (raised) or a
+ callable (run first, so it can advance the clock or raise). ``refuse_dials_after`` fails every
+ dial past the scripted ones with a refused connection."""
+
+ def __init__(self, connects=(), receipts=(), submits=(), refuse_dials_after=False):
+ self.connects, self.receipts, self.submits = list(connects), list(receipts), list(submits)
+ self.refuse_dials_after = refuse_dials_after
self.calls = []
self.dials = 0
+ @staticmethod
+ def _step(steps, default=None):
+ step = steps.pop(0) if steps else default
+ if callable(step) and not isinstance(step, Exception):
+ step = step()
+ if isinstance(step, Exception):
+ raise step
+ return step
+
def connect(self):
owner = self
@asynccontextmanager
async def _connect():
owner.dials += 1
- step = owner.connects.pop(0) if owner.connects else None
- if isinstance(step, Exception):
- raise step
+ refused = gateway_client.GatewayClientError("Gateway connection failed; no local fallback")
+ owner._step(owner.connects, refused if owner.refuse_dials_after else None)
class _Client:
def __init__(self):
@@ -44,14 +62,10 @@ class _Owner:
async def rpc(self, method, **params):
owner.calls.append(method)
if method == "kanban.run":
- failure = owner.submit_failures.pop(0) if owner.submit_failures else None
- if failure is not None:
- raise failure
+ owner._step(owner.submits)
return {"session_id": "s1", "receipt": {"status": "pending", "admission_id": "a1"}}
- status = owner.receipts.pop(0) if owner.receipts else "terminal"
- if isinstance(status, Exception):
- raise status
- return {"status": status}
+ assert params == {"session_id": "s1", "admission_id": "a1"}
+ return {"status": owner._step(owner.receipts, "terminal")}
yield _Client()
@@ -63,128 +77,91 @@ def _disconnect():
"Gateway disconnected; turn outcome is unknown. Resume the printed session ID")
+def clock(monkeypatch):
+ clock = _Clock()
+ monkeypatch.setattr(kanban_worker_client, "time", clock)
+
+ async def _sleep(seconds):
+ clock.now += seconds
+ monkeypatch.setattr(kanban_worker_client.asyncio, "sleep", _sleep)
+ return clock
+
+
@pytest.fixture(autouse=True)
def _worker_env(monkeypatch, tmp_path):
monkeypatch.delenv("HERMES_TUI_GATEWAY_URL", raising=False)
- monkeypatch.setattr(kanban_worker_client.asyncio, "sleep", _no_sleep)
from gateway import session_kanban
monkeypatch.setattr(session_kanban, "worker_result",
lambda *_: {"exit_code": 0, "last_output": "card closed"})
return tmp_path
-async def _no_sleep(_seconds):
- return None
+async def _run(owner, monkeypatch, tmp_path):
+ monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
+ return await kanban_worker_client.run({"task_id": "t1"}, str(tmp_path / "kanban.db"))
@pytest.mark.asyncio
-async def test_a_socket_lost_mid_turn_redials_and_settles_the_same_attempt(monkeypatch, _worker_env):
- owner = _Owner(connects=[None, None],
- receipts=["pending", _disconnect(), "terminal"])
- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
-
- assert await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db")) == 0
- # The claim is submitted once: the redial re-reads the receipt, it does not resubmit.
[email protected]("redial", [
+ None,
+ gateway_client.GatewayClientError("Gateway connection failed; no local fallback"),
+ ConnectionRefusedError(61, "refused"),
+ gateway_client.GatewayUnavailableError("Gateway starting: not_ready", "starting"),
+], ids=["reconnects", "dial-refused", "ticket-oserror", "owner-starting"])
+async def test_a_socket_lost_mid_turn_resumes_the_same_attempt(monkeypatch, _worker_env, clock, redial):
+ owner = _Owner(connects=[None, redial, None], receipts=["pending", _disconnect(), "terminal"])
+
+ assert await _run(owner, monkeypatch, _worker_env) == 0
+ # The claim is submitted once: every later connection re-reads the receipt.
assert owner.calls.count("kanban.run") == 1
- assert owner.dials == 2
+ assert owner.dials == (2 if redial is None else 3)
@pytest.mark.asyncio
-async def test_a_draining_owner_is_retried_before_anything_is_admitted(monkeypatch, _worker_env):
- owner = _Owner(connects=[None, None],
- submit_failures=[gateway_client.GatewayRPCError("runtime_draining")],
- receipts=["terminal"])
- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
-
- assert await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db")) == 0
[email protected]("lost", [_disconnect(), gateway_client.GatewayRPCError("runtime_draining")],
+ ids=["answer-lost", "draining"])
+async def test_a_claim_with_no_answer_is_resubmitted(monkeypatch, _worker_env, clock, lost):
+ # No admission id reached the worker, so it asks again; the owner answers a repeated
+ # kanban.run for the same attempt with the same admission.
+ owner = _Owner(connects=[None, None], submits=[lost], receipts=["terminal"])
+
+ assert await _run(owner, monkeypatch, _worker_env) == 0
assert owner.calls.count("kanban.run") == 2
@pytest.mark.asyncio
-async def test_an_owner_refusal_is_not_a_redial(monkeypatch, _worker_env):
- owner = _Owner(connects=[None, None],
- submit_failures=[gateway_client.GatewayRPCError("claim_mismatch")])
- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
-
- with pytest.raises(gateway_client.GatewayRPCError):
- await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db"))
[email protected]("where", ["kanban.run", "prompt.receipt"])
+async def test_an_owner_refusal_is_not_retried(monkeypatch, _worker_env, clock, where):
+ refusal = gateway_client.GatewayRPCError("claim_mismatch")
+ owner = (_Owner(submits=[refusal]) if where == "kanban.run"
+ else _Owner(receipts=["pending", refusal]))
+
+ with pytest.raises(gateway_client.GatewayRPCError, match="claim_mismatch"):
+ await _run(owner, monkeypatch, _worker_env)
assert owner.dials == 1
@pytest.mark.asyncio
-async def test_a_gateway_that_never_comes_back_fails_honestly(monkeypatch, _worker_env):
- owner = _Owner(connects=[None, gateway_client.GatewayUnavailableError("no owner", "inaccessible")],
- receipts=["pending", _disconnect()])
- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
-
- with pytest.raises(gateway_client.GatewayClientError):
- await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db"))
-
-
-class _Clock:
- def __init__(self, values):
- self.values = iter(values)
- self.last = values[-1]
+async def test_a_later_loss_gets_a_fresh_budget_from_the_owners_last_answer(monkeypatch, _worker_env, clock):
+ def silent_then_lost():
+ clock.now += kanban_worker_client.OWNER_WAIT_S - 10
+ return _disconnect()
- def monotonic(self):
- try:
- self.last = next(self.values)
- except StopIteration:
- pass
- return self.last
-
-
-async def test_reconnect_starting_uses_redial_deadline_not_initial_deadline(monkeypatch, _worker_env):
- # Initial wait expires at t=180, but the admitted attempt's redial budget runs until t=361.
- monkeypatch.setattr(kanban_worker_client, "time", _Clock([0, 181, 181, 200]))
- owner = _Owner(
- connects=[None, gateway_client.GatewayUnavailableError("owner starting", "starting"), None],
- receipts=["pending", _disconnect(), "terminal"],
- )
- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
-
- assert await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db")) == 0
- assert owner.calls.count("kanban.run") == 1
- assert owner.dials == 3
-
-
-async def test_a_later_disconnect_gets_a_fresh_redial_budget(monkeypatch, _worker_env):
- # The second disconnect occurs at t=200, beyond the first episode's t=190 deadline.
- monkeypatch.setattr(kanban_worker_client, "time", _Clock([0, 10, 10, 200, 200]))
- owner = _Owner(
- connects=[None, None, None],
- receipts=["pending", _disconnect(), "pending", _disconnect(), "terminal"],
- )
- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
-
- assert await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db")) == 0
- assert owner.calls.count("kanban.run") == 1
- assert owner.dials == 3
-
-
-async def test_repeated_receipt_disconnects_cannot_reset_the_redial_budget(monkeypatch, _worker_env):
- # Every dial succeeds, but receipt reads disconnect. Advancing monotonic time
- # by 100 seconds per check must exhaust one recovery budget rather than reset it.
- monkeypatch.setattr(kanban_worker_client, "time", _Clock([0, 100, 200, 300, 400]))
+ # Each loss comes 170 s after the last answer; across both, far more than OWNER_WAIT_S passes.
owner = _Owner(connects=[None, None, None],
- receipts=[_disconnect(), _disconnect(), _disconnect()])
- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
+ receipts=["pending", silent_then_lost, "pending", silent_then_lost, "terminal"])
- with pytest.raises(gateway_client.GatewayClientError, match="after admission"):
- await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db"))
- assert owner.calls.count("kanban.run") == 1
- assert owner.dials == 2
+ assert await _run(owner, monkeypatch, _worker_env) == 0
+ assert owner.dials == 3 and owner.calls.count("kanban.run") == 1
@pytest.mark.asyncio
-async def test_the_redial_budget_bounds_a_flapping_gateway(monkeypatch, _worker_env):
- monkeypatch.setattr(kanban_worker_client, "OWNER_WAIT_S", 0.0)
- owner = _Owner(connects=[None, None], receipts=["pending", _disconnect(), "terminal"])
- monkeypatch.setattr(gateway_client, "connect_gateway", owner.connect())
-
- with pytest.raises(gateway_client.GatewayClientError):
- await kanban_worker_client.run({"task_id": "t1"}, str(_worker_env / "kanban.db"))
- assert owner.calls.count("kanban.run") == 1
+async def test_an_owner_silent_for_the_whole_budget_fails_the_worker(monkeypatch, _worker_env, clock):
+ owner = _Owner(connects=[None], receipts=["pending", _disconnect()], refuse_dials_after=True)
+
+ with pytest.raises(gateway_client.GatewayClientError, match="unreachable for 180s"):
+ await _run(owner, monkeypatch, _worker_env)
+ assert clock.now >= kanban_worker_client.OWNER_WAIT_S
+ # Jittered exponential backoff (capped at 8-12 s) between dials, not one dial a second.
+ assert 3 < owner.dials < 40 and owner.calls.count("kanban.run") == 1