Skip to content

The agent that grows with you

fix(gateway): say where a capped watch notice's output still lives

Kurt VanDusenf7fb21ac34e1
diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py
index 67a2e75c7c4..8d78521cc09 100644
--- a/gateway/run_notifications.py
+++ b/gateway/run_notifications.py
@@ -751,7 +751,8 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
else:
logger.warning(
"Watch notification for %s chat=%s (type=%s session_id=%s) was refused "
- "%d times; dropping the queued event",
+ "%d times; dropping the queued event (output remains available via "
+ "process(action='log'))",
evt.get("platform") or "?", evt.get("chat_id") or "?",
evt.get("type") or "?", evt.get("session_id", "unknown"), attempts,
)
diff --git a/tests/gateway/test_background_process_notifications.py b/tests/gateway/test_background_process_notifications.py
index d03460aa836..e0847aee885 100644
--- a/tests/gateway/test_background_process_notifications.py
+++ b/tests/gateway/test_background_process_notifications.py
@@ -336,6 +336,7 @@ async def test_post_turn_watch_drain_refused_event_backs_off_then_drops(monkeypa
from gateway import run_notifications
from gateway.run_notifications import (
+ _COMPLETION_INJECTION_RETRY_MAX_DELAY,
_MAX_COMPLETION_INJECTION_ATTEMPTS,
_WATCH_INJECTION_RETRY_BASE_DELAY,
)
@@ -360,10 +361,10 @@ async def test_post_turn_watch_drain_refused_event_backs_off_then_drops(monkeypa
break
assert len(attempt_times) == _MAX_COMPLETION_INJECTION_ATTEMPTS
- # Backoff, not one attempt per drain tick: successive retries are >= the base delay apart.
+ # Backoff, not one attempt per drain tick: each retry waits twice the last, up to the cap.
gaps = [b - a for a, b in itertools.pairwise(attempt_times)]
- assert gaps
- assert all(gap >= _WATCH_INJECTION_RETRY_BASE_DELAY for gap in gaps)
+ assert gaps == [min(_WATCH_INJECTION_RETRY_BASE_DELAY * 2 ** n, _COMPLETION_INJECTION_RETRY_MAX_DELAY)
+ for n in range(_MAX_COMPLETION_INJECTION_ATTEMPTS - 1)]
assert completion_queue.empty()
drops = [r for r in caplog.records
if r.levelno >= logging.WARNING and "dropping the queued event" in r.message]
diff --git a/tests/gateway/test_completion_delivery.py b/tests/gateway/test_completion_delivery.py
index e10d8468a20..af62378892f 100644
--- a/tests/gateway/test_completion_delivery.py
+++ b/tests/gateway/test_completion_delivery.py
@@ -18,8 +18,8 @@ import pytest
from gateway import run_notifications
from gateway.config import Platform
from gateway.run import GatewayRunner
-from tools.process_registry import ProcessRegistry, ProcessSession
from tests.module_clock import install_module_clock
+from tools.process_registry import ProcessRegistry, ProcessSession
class AdmittingHandler(AsyncMock):
@@ -988,8 +988,14 @@ def test_watch_drain_retries_transport_failure(monkeypatch, isolated_registry):
evt = dict(_completion_event(started_at=1), type="watch_match", pattern="READY", output="READY")
isolated_registry.completion_queue.put(evt)
_stop_after_sleeps(monkeypatch, runner, count=3)
- # The refused event backs off (#140899); each read is a minute later, so the retry is due.
- install_module_clock(monkeypatch, run_notifications, step=60)
+ # The refused event backs off (#140899): each watcher tick is a minute later, so the retry is due.
+ clock = install_module_clock(monkeypatch, run_notifications)
+ bounded_sleep = asyncio.sleep
+
+ async def minute_later(delay):
+ clock.now += 60
+ await bounded_sleep(delay)
+ monkeypatch.setattr(asyncio, "sleep", minute_later)
asyncio.run(runner._async_delegation_watcher(interval=0))
assert adapter.handle_message.await_count == 2
assert isolated_registry.completion_queue.empty()
diff --git a/tests/gateway/test_watch_notification_reset_pin.py b/tests/gateway/test_watch_notification_reset_pin.py
index 7c5a2b022ba..62a3ac3fa9a 100644
--- a/tests/gateway/test_watch_notification_reset_pin.py
+++ b/tests/gateway/test_watch_notification_reset_pin.py
@@ -176,6 +176,25 @@ async def test_unproven_refusal_requeues_with_backoff_until_the_cap(monkeypatch,
authority.db.close()
+async def test_unpinned_refusal_requeues(monkeypatch, caplog):
+ """A notice that names no pinned session proves nothing gone: its refusal is retried."""
+ runner, authority, entry = await _gateway(monkeypatch)
+ try:
+ notice = _notice(entry, '')
+ pending = queue.Queue()
+ pending.put(notice)
+
+ with caplog.at_level(logging.WARNING, logger='gateway.run'):
+ await runner._drain_watch_notifications(pending)
+
+ assert pending.get_nowait() is notice
+ assert notice['_watch_injection_attempts'] == 1
+ assert not [r for r in caplog.records if 'permanently gone' in r.getMessage()]
+ finally:
+ authority.db.close()
+
+
@pytest.mark.asyncio
async def test_unproven_heartbeat_refusal_requeues_the_original_tick(monkeypatch, caplog):
"""A liveness tick keeps its own lifecycle: a refusal the pre-flight cannot prove permanent puts
diff --git a/tests/module_clock.py b/tests/module_clock.py
index 25afbff5df3..d6cc2015fc6 100644
--- a/tests/module_clock.py
+++ b/tests/module_clock.py
@@ -6,21 +6,19 @@ import time
class ModuleClock:
- """``time`` for one module: ``monotonic()`` returns ``now``, advanced by ``step`` per read."""
+ """``time`` for one module, except that ``monotonic()`` returns ``now``."""
- def __init__(self, now: float, step: float):
- self.now, self.step = now, step
+ def __init__(self, now: float):
+ self.now = now
def monotonic(self) -> float:
- value = self.now
- self.now += self.step
- return value
+ return self.now
def __getattr__(self, name):
return getattr(time, name)
-def install_module_clock(monkeypatch, module, *, now: float = 1000.0, step: float = 0.0) -> ModuleClock:
- clock = ModuleClock(now, step)
+def install_module_clock(monkeypatch, module, *, now: float = 1000.0) -> ModuleClock:
+ clock = ModuleClock(now)
monkeypatch.setattr(module, "time", clock)
return clock