The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(mcp): back off failing event refreshes and drop refused ones
emozilla59e3b6ee3dcc
diff --git a/tests/tools/test_mcp_events.py b/tests/tools/test_mcp_events.pyindex 9cbfb33d557..b5f9e41f702 100644--- a/tests/tools/test_mcp_events.py+++ b/tests/tools/test_mcp_events.py@@ -168,16 +168,34 @@ class TestRefresh:next_due = asyncio.run(events.refresh_server_subscriptions(_server(session), force=False))assert session.sent == [] and next_due > time.time()- def test_a_forbidden_refresh_ends_the_subscription(self, home):+ @pytest.mark.parametrize("code", [-32024, -32023, -32602, -32026],+ ids=["forbidden", "not-found", "invalid-params", "unsupported"])+ def test_a_refusal_that_cannot_change_ends_the_subscription(self, home, code):_subscribe(FakeSession(_grant()))- session = FakeSession(MCPError(-32024, "Forbidden"))+ session = FakeSession(MCPError(code, "refused"))asyncio.run(events.refresh_server_subscriptions(_server(session), force=True))assert _routes(home) == {}+ def test_a_failing_refresh_backs_off(self, home):+ _subscribe(FakeSession(_grant()))+ failures = {}+ delays = []+ for _ in range(3):+ session = FakeSession(MCPError(-32027, "CallbackEndpointError", {"reason": "timeout"}))+ next_due = asyncio.run(events.refresh_server_subscriptions(_server(session), force=True, failures=failures))+ delays.append(next_due - time.time())+ assert [round(delay / events._REFRESH_MIN_SLEEP_S) for delay in delays] == [1, 2, 4]+ state = _state(home) # the grant has run out, so only the back-off holds the next attempt+ next(iter(state.values())).update(granted_at=time.time() - 7200, refresh_before=_grant(-3600)["refreshBefore"])+ (home / "mcp_event_subscriptions.json").write_text(json.dumps(state))+ session = FakeSession()+ next_due = asyncio.run(events.refresh_server_subscriptions(_server(session), force=False, failures=failures))+ assert session.sent == [] and 100 < next_due - time.time() <= 4 * events._REFRESH_MIN_SLEEP_S+def test_a_new_subscription_wakes_the_refresh_loop(self, home, monkeypatch):passes = []- async def recording_pass(server, *, force):+ async def recording_pass(server, *, force, failures=None):passes.append(force)return float("inf") # nothing due: without a wake-up the loop sleeps for an hourmonkeypatch.setattr(events, "refresh_server_subscriptions", recording_pass)diff --git a/tools/mcp_tool_events.py b/tools/mcp_tool_events.pyindex cb33724ff4b..7c0350f0708 100644--- a/tools/mcp_tool_events.py+++ b/tools/mcp_tool_events.py@@ -35,8 +35,11 @@ _REFRESH_MIN_SLEEP_S, _REFRESH_MAX_SLEEP_S = 30.0, 3600.0_ROUTE_SLUG_RE = re.compile(r"[^a-z0-9_-]+")# Per-route subscription state (grant, cursor, granted URL), kept out of the route config._STATE_FILENAME = "mcp_event_subscriptions.json"-# Error codes from the sketch's Error Codes table (SEP-3415 numbering) that end a subscription for good.-_FORBIDDEN, _NOT_FOUND = -32024, -32023+# Error codes from the sketch's Error Codes table (SEP-3415 numbering) plus JSON-RPC InvalidParams.+_FORBIDDEN, _NOT_FOUND, _UNSUPPORTED, _INVALID_PARAMS = -32024, -32023, -32026, -32602+# A refresh refused with one of these can never succeed as stored: access was revoked, the event type is+# gone, its arguments no longer validate, or the server no longer offers what the subscription asks for.+_TERMINAL_REFRESH_CODES = frozenset({_FORBIDDEN, _NOT_FOUND, _UNSUPPORTED, _INVALID_PARAMS})# Refresh loops by server name, each as (its event loop, its wake-up event). A new subscription wakes them# so its first refresh is scheduled from its own grant, not after the loop's current sleep (up to an hour)._refresh_wakers: dict[str, set[tuple[asyncio.AbstractEventLoop, asyncio.Event]]] = {}@@ -218,15 +221,21 @@ def _wake_refresh_loops(server_name: str) -> None:loop.call_soon_threadsafe(wake.set)-async def refresh_server_subscriptions(server, *, force: bool) -> float:+async def refresh_server_subscriptions(server, *, force: bool,+ failures: Optional[dict[str, tuple[int, float]]] = None) -> float:"""Re-subscribe this server's routes that are due (all of them when *force*, i.e. on a fresh- connection, where the server may have lost soft state). Returns the next due time."""+ connection, where the server may have lost soft state). A failed refresh is retried with exponential+ back-off: *failures* maps a route to (consecutive failures, retry time) across passes. Returns the next+ due time."""+ failures = {} if failures is None else failuresurl_base = _callback_base()routes = await asyncio.to_thread(_load_routes, server.name)states = await asyncio.to_thread(_load_state)next_due = float("inf")for route_name in routes:due = _refresh_due_at(states.get(route_name, {}))+ if route_name in failures:+ due = max(due, failures[route_name][1])if url_base and (force or due <= time.time()):try:async with server._rpc_lock:@@ -236,18 +245,24 @@ async def refresh_server_subscriptions(server, *, force: bool) -> float:if route is not None:stored = await _subscribe_route(server.session, route_name, route, url_base)if stored is None:+ failures.pop(route_name, None)continueexcept Exception as exc:error = _mcp_error(exc)- if error and error["code"] in (_FORBIDDEN, _NOT_FOUND):- logger.warning("MCP server '%s': event subscription %s ended by the server (%s); removing it",+ if error and error["code"] in _TERMINAL_REFRESH_CODES:+ logger.warning("MCP server '%s': event subscription %s refused by the server (%s); removing it",server.name, route_name, error)await asyncio.to_thread(_write_route, route_name, None)+ failures.pop(route_name, None)continue- logger.warning("MCP server '%s': refreshing event subscription %s failed: %s",- server.name, route_name, error or exc, exc_info=error is None)- due = time.time() + _REFRESH_MIN_SLEEP_S+ count = failures.get(route_name, (0, 0.0))[0] + 1+ delay = min(_REFRESH_MAX_SLEEP_S, _REFRESH_MIN_SLEEP_S * 2 ** min(count - 1, 10))+ due = time.time() + delay+ failures[route_name] = (count, due)+ logger.warning("MCP server '%s': refreshing event subscription %s failed (attempt %d, next in %.0fs): %s",+ server.name, route_name, count, delay, error or exc, exc_info=error is None)else:+ failures.pop(route_name, None)due = _refresh_due_at(stored)next_due = min(next_due, due)return next_due@@ -258,12 +273,13 @@ async def run_refresh_loop(server) -> None:wake = asyncio.Event()waker = (asyncio.get_running_loop(), wake)_refresh_wakers.setdefault(server.name, set()).add(waker)+ failures: dict[str, tuple[int, float]] = {}force = Truetry:while True:wake.clear() # before the pass, so a subscription made during it still wakes the next onetry:- next_due = await refresh_server_subscriptions(server, force=force)+ next_due = await refresh_server_subscriptions(server, force=force, failures=failures)except Exception as exc:logger.warning("MCP server '%s': event subscription refresh failed: %s", server.name, exc,exc_info=True)diff --git a/website/docs/user-guide/features/mcp.md b/website/docs/user-guide/features/mcp.mdindex 72ee30b5afd..f90c0881203 100644--- a/website/docs/user-guide/features/mcp.md+++ b/website/docs/user-guide/features/mcp.md@@ -1035,8 +1035,10 @@ Each subscription is stored as a dynamic webhook route (`hermes webhook list` sh`mcp_event_subscriptions.json`, so the route itself only changes when you resubscribe with differentinstructions or a different destination. Hermes answers the server's signed verification challenge, deduplicatesevents by `eventId`, and refreshes the subscription before the lifetime the server granted runs out-while the MCP connection is up (and again whenever it reconnects). If the server ends a-subscription, Hermes removes the route and tells the `deliver` target. Events only arrive while+while the MCP connection is up (and again whenever it reconnects). A refresh the server refuses+for good (access revoked, event type removed, arguments no longer valid) removes the route; other+failures are retried with back-off. If the server ends a subscription,+Hermes removes the route and tells the `deliver` target. Events only arrive while`hermes gateway run` is running. Event payloads are external data: event runs use the sameconstrained webhook toolset as other webhook routes.The listener's per-route rate limit (`platforms.webhook.extra.rate_limit`, 30 a minute by default)