The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix: a refused async completion converges to dropped instead of refunding forever
Tekniumcda47903ddfa
diff --git a/gateway/run_notifications.py b/gateway/run_notifications.pyindex 8d78521cc09..a4c943d836b 100644--- a/gateway/run_notifications.py+++ b/gateway/run_notifications.py@@ -1241,7 +1241,8 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotificaself._mark_completions_delivered_locked((identity,))return Trueexcept WakeNotAccepted:- refused = True+ # Past the watcher's bounded retries a refusal spends its attempt (#141153).+ refused = not evt.get("_spend_refusal")return Falsefinally:if identity_claimed and not accepted:@@ -1589,9 +1590,14 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotificaself._enrich_async_delegation_routing(evt)groups.setdefault(self._event_route_key(evt, self._ASYNC_GROUP_KEY_FIELDS), []).append(evt)for group in groups.values():+ if any(evt.get("_watch_next_retry_at", 0) > time.monotonic() for evt in group):+ for evt in group:+ _pr.completion_queue.put(evt)+ continuetry:delivered = await self._deliver_async_delegation_group(group)if delivered is False:+ self._back_off_refused_completions(group)for evt in group:_pr.completion_queue.put(evt)except Exception as e:@@ -1600,6 +1606,20 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotificalogger.error("Async delegation injection error: %s", e)await asyncio.sleep(interval)+ @staticmethod+ def _back_off_refused_completions(group: list[dict]) -> None:+ """Space out an undelivered async group's retries; past the cap its next refusal spends a+ durable delivery attempt (``_spend_refusal``), so a refusal that never clears converges to+ ``dropped`` instead of being refunded on every 2 s pass, across restarts (#141153). A single+ refusal still refunds: it may clear once the owner's turn settles."""+ for evt in group:+ attempts = int(evt.get("_watch_injection_attempts") or 0) + 1+ evt["_watch_injection_attempts"] = attempts+ evt["_spend_refusal"] = attempts >= _MAX_COMPLETION_INJECTION_ATTEMPTS+ evt["_watch_next_retry_at"] = time.monotonic() + jittered_backoff(+ attempts, base_delay=_WATCH_INJECTION_RETRY_BASE_DELAY,+ max_delay=_COMPLETION_INJECTION_RETRY_MAX_DELAY, jitter_ratio=0.0)+@staticmethoddef _redacted_output_tail(session, limit: int) -> str:"""Last ``limit`` chars of process output through the secret redactors (unconditional floor)."""diff --git a/tests/gateway/test_completion_delivery.py b/tests/gateway/test_completion_delivery.pyindex af62378892f..bcdfd22a32d 100644--- a/tests/gateway/test_completion_delivery.py+++ b/tests/gateway/test_completion_delivery.py@@ -95,12 +95,16 @@ def _completion_event(*, started_at, session_id="proc_reused"):}-def _stop_after_sleeps(monkeypatch, runner, count):+def _stop_after_sleeps(monkeypatch, runner, count, *, minutes_apart=False):+ """``minutes_apart``: each watcher tick is a minute later, so a refused retry's backoff is due."""sleep_calls = 0+ clock = install_module_clock(monkeypatch, run_notifications) if minutes_apart else Noneasync def _bounded_sleep(_delay):nonlocal sleep_callssleep_calls += 1+ if clock is not None:+ clock.now += 60if sleep_calls >= count:runner._running = False@@ -179,7 +183,7 @@ def test_failed_async_injection_is_retried_and_only_success_is_acked(handle_message=AdmittingHandler(side_effect=[RuntimeError("temporary"), None]))runner = _runner(adapter)- _stop_after_sleeps(monkeypatch, runner, count=3)+ _stop_after_sleeps(monkeypatch, runner, count=3, minutes_apart=True)from tools import async_delegation@@ -852,7 +856,7 @@ def test_failed_coalesced_async_batch_releases_claims_and_retries(handle_message=AdmittingHandler(side_effect=[RuntimeError("temporary"), None]))runner = _runner(adapter)- _stop_after_sleeps(monkeypatch, runner, count=3)+ _stop_after_sleeps(monkeypatch, runner, count=3, minutes_apart=True)asyncio.run(runner._async_delegation_watcher(interval=0))diff --git a/tests/gateway/test_watch_notification_reset_pin.py b/tests/gateway/test_watch_notification_reset_pin.pyindex f2c0ad7b9aa..428bcab2e66 100644--- a/tests/gateway/test_watch_notification_reset_pin.py+++ b/tests/gateway/test_watch_notification_reset_pin.py@@ -244,3 +244,43 @@ async def test_unproven_heartbeat_refusal_requeues_the_original_tick(monkeypatchassert not _queued_admissions(authority, entry.session_id)finally:authority.db.close()+++async def test_durable_completion_refused_on_every_retry_converges_to_dropped(monkeypatch):+ """The async watcher backs off a refused completion and, past its bounded retries, lets the+ refusal spend a durable delivery attempt: the row reaches ``dropped`` instead of being refunded+ on every 2 s pass and replayed after each restart (#141153)."""+ import asyncio+ import time+ from gateway import run_notifications+ from tools import async_delegation as delegation+ from tools.process_registry import process_registry+ runner, authority, entry = await _gateway(monkeypatch)+ try:+ evt = dict(_notice(entry, await _foreign_pin(runner), 'async_delegation'),+ delegation_id='refused-1', summary='review findings', status='completed',+ dispatched_at=time.time())+ delegation._persist_dispatch(evt)+ delegation._persist_completion(evt, {'status': 'completed', 'summary': 'review findings'})+ monkeypatch.setattr(process_registry, 'completion_queue', queue.Queue())+ process_registry.completion_queue.put(dict(evt))+ clock = install_module_clock(monkeypatch, run_notifications)+ real_sleep, ticks = asyncio.sleep, []++ async def tick(delay):+ ticks.append(delay)+ clock.now += 1+ runner._running = len(ticks) < 4000 and not process_registry.completion_queue.empty()+ await real_sleep(0)+ monkeypatch.setattr(asyncio, 'sleep', tick)+ runner._running = True++ await runner._async_delegation_watcher(interval=0)++ row = delegation.get_durable_delegation('refused-1')+ assert (row['delivery_state'], row['delivery_attempts']) == ('dropped', delegation._MAX_DELIVERY_ATTEMPTS)+ # Eight refunded retries back off ~5 min, then each spent attempt waits a minute.+ assert len(ticks) > 500, 'refused retries are spaced out, not re-attempted on every pass'+ finally:+ authority.db.close()