Skip to content

The agent that grows with you

fix: a refused async completion converges to dropped instead of refunding forever

Tekniumcda47903ddfa
diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py
index 8d78521cc09..a4c943d836b 100644
--- a/gateway/run_notifications.py
+++ b/gateway/run_notifications.py
@@ -1241,7 +1241,8 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
self._mark_completions_delivered_locked((identity,))
return True
except WakeNotAccepted:
- refused = True
+ # Past the watcher's bounded retries a refusal spends its attempt (#141153).
+ refused = not evt.get("_spend_refusal")
return False
finally:
if identity_claimed and not accepted:
@@ -1589,9 +1590,14 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
self._enrich_async_delegation_routing(evt)
groups.setdefault(self._event_route_key(evt, self._ASYNC_GROUP_KEY_FIELDS), []).append(evt)
for group in groups.values():
+ if any(evt.get("_watch_next_retry_at", 0) > time.monotonic() for evt in group):
+ for evt in group:
+ _pr.completion_queue.put(evt)
+ continue
try:
delivered = await self._deliver_async_delegation_group(group)
if delivered is False:
+ self._back_off_refused_completions(group)
for evt in group:
_pr.completion_queue.put(evt)
except Exception as e:
@@ -1600,6 +1606,20 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
logger.error("Async delegation injection error: %s", e)
await asyncio.sleep(interval)
+ @staticmethod
+ def _back_off_refused_completions(group: list[dict]) -> None:
+ """Space out an undelivered async group's retries; past the cap its next refusal spends a
+ durable delivery attempt (``_spend_refusal``), so a refusal that never clears converges to
+ ``dropped`` instead of being refunded on every 2 s pass, across restarts (#141153). A single
+ refusal still refunds: it may clear once the owner's turn settles."""
+ for evt in group:
+ attempts = int(evt.get("_watch_injection_attempts") or 0) + 1
+ evt["_watch_injection_attempts"] = attempts
+ evt["_spend_refusal"] = attempts >= _MAX_COMPLETION_INJECTION_ATTEMPTS
+ evt["_watch_next_retry_at"] = time.monotonic() + jittered_backoff(
+ attempts, base_delay=_WATCH_INJECTION_RETRY_BASE_DELAY,
+ max_delay=_COMPLETION_INJECTION_RETRY_MAX_DELAY, jitter_ratio=0.0)
+
@staticmethod
def _redacted_output_tail(session, limit: int) -> str:
"""Last ``limit`` chars of process output through the secret redactors (unconditional floor)."""
diff --git a/tests/gateway/test_completion_delivery.py b/tests/gateway/test_completion_delivery.py
index af62378892f..bcdfd22a32d 100644
--- a/tests/gateway/test_completion_delivery.py
+++ b/tests/gateway/test_completion_delivery.py
@@ -95,12 +95,16 @@ def _completion_event(*, started_at, session_id="proc_reused"):
}
-def _stop_after_sleeps(monkeypatch, runner, count):
+def _stop_after_sleeps(monkeypatch, runner, count, *, minutes_apart=False):
+ """``minutes_apart``: each watcher tick is a minute later, so a refused retry's backoff is due."""
sleep_calls = 0
+ clock = install_module_clock(monkeypatch, run_notifications) if minutes_apart else None
async def _bounded_sleep(_delay):
nonlocal sleep_calls
sleep_calls += 1
+ if clock is not None:
+ clock.now += 60
if sleep_calls >= count:
runner._running = False
@@ -179,7 +183,7 @@ def test_failed_async_injection_is_retried_and_only_success_is_acked(
handle_message=AdmittingHandler(side_effect=[RuntimeError("temporary"), None])
)
runner = _runner(adapter)
- _stop_after_sleeps(monkeypatch, runner, count=3)
+ _stop_after_sleeps(monkeypatch, runner, count=3, minutes_apart=True)
from tools import async_delegation
@@ -852,7 +856,7 @@ def test_failed_coalesced_async_batch_releases_claims_and_retries(
handle_message=AdmittingHandler(side_effect=[RuntimeError("temporary"), None])
)
runner = _runner(adapter)
- _stop_after_sleeps(monkeypatch, runner, count=3)
+ _stop_after_sleeps(monkeypatch, runner, count=3, minutes_apart=True)
asyncio.run(runner._async_delegation_watcher(interval=0))
diff --git a/tests/gateway/test_watch_notification_reset_pin.py b/tests/gateway/test_watch_notification_reset_pin.py
index f2c0ad7b9aa..428bcab2e66 100644
--- a/tests/gateway/test_watch_notification_reset_pin.py
+++ b/tests/gateway/test_watch_notification_reset_pin.py
@@ -244,3 +244,43 @@ async def test_unproven_heartbeat_refusal_requeues_the_original_tick(monkeypatch
assert not _queued_admissions(authority, entry.session_id)
finally:
authority.db.close()
+
+
+async def test_durable_completion_refused_on_every_retry_converges_to_dropped(monkeypatch):
+ """The async watcher backs off a refused completion and, past its bounded retries, lets the
+ refusal spend a durable delivery attempt: the row reaches ``dropped`` instead of being refunded
+ on every 2 s pass and replayed after each restart (#141153)."""
+ import asyncio
+ import time
+ from gateway import run_notifications
+ from tools import async_delegation as delegation
+ from tools.process_registry import process_registry
+ runner, authority, entry = await _gateway(monkeypatch)
+ try:
+ evt = dict(_notice(entry, await _foreign_pin(runner), 'async_delegation'),
+ delegation_id='refused-1', summary='review findings', status='completed',
+ dispatched_at=time.time())
+ delegation._persist_dispatch(evt)
+ delegation._persist_completion(evt, {'status': 'completed', 'summary': 'review findings'})
+ monkeypatch.setattr(process_registry, 'completion_queue', queue.Queue())
+ process_registry.completion_queue.put(dict(evt))
+ clock = install_module_clock(monkeypatch, run_notifications)
+ real_sleep, ticks = asyncio.sleep, []
+
+ async def tick(delay):
+ ticks.append(delay)
+ clock.now += 1
+ runner._running = len(ticks) < 4000 and not process_registry.completion_queue.empty()
+ await real_sleep(0)
+ monkeypatch.setattr(asyncio, 'sleep', tick)
+ runner._running = True
+
+ await runner._async_delegation_watcher(interval=0)
+
+ row = delegation.get_durable_delegation('refused-1')
+ assert (row['delivery_state'], row['delivery_attempts']) == ('dropped', delegation._MAX_DELIVERY_ATTEMPTS)
+ # Eight refunded retries back off ~5 min, then each spent attempt waits a minute.
+ assert len(ticks) > 500, 'refused retries are spaced out, not re-attempted on every pass'
+ finally:
+ authority.db.close()