Skip to content

The agent that grows with you

fix(mcp): back off failing event refreshes and drop refused ones

emozilla59e3b6ee3dcc
diff --git a/tests/tools/test_mcp_events.py b/tests/tools/test_mcp_events.py
index 9cbfb33d557..b5f9e41f702 100644
--- a/tests/tools/test_mcp_events.py
+++ b/tests/tools/test_mcp_events.py
@@ -168,16 +168,34 @@ class TestRefresh:
next_due = asyncio.run(events.refresh_server_subscriptions(_server(session), force=False))
assert session.sent == [] and next_due > time.time()
- def test_a_forbidden_refresh_ends_the_subscription(self, home):
+ @pytest.mark.parametrize("code", [-32024, -32023, -32602, -32026],
+ ids=["forbidden", "not-found", "invalid-params", "unsupported"])
+ def test_a_refusal_that_cannot_change_ends_the_subscription(self, home, code):
_subscribe(FakeSession(_grant()))
- session = FakeSession(MCPError(-32024, "Forbidden"))
+ session = FakeSession(MCPError(code, "refused"))
asyncio.run(events.refresh_server_subscriptions(_server(session), force=True))
assert _routes(home) == {}
+ def test_a_failing_refresh_backs_off(self, home):
+ _subscribe(FakeSession(_grant()))
+ failures = {}
+ delays = []
+ for _ in range(3):
+ session = FakeSession(MCPError(-32027, "CallbackEndpointError", {"reason": "timeout"}))
+ next_due = asyncio.run(events.refresh_server_subscriptions(_server(session), force=True, failures=failures))
+ delays.append(next_due - time.time())
+ assert [round(delay / events._REFRESH_MIN_SLEEP_S) for delay in delays] == [1, 2, 4]
+ state = _state(home) # the grant has run out, so only the back-off holds the next attempt
+ next(iter(state.values())).update(granted_at=time.time() - 7200, refresh_before=_grant(-3600)["refreshBefore"])
+ (home / "mcp_event_subscriptions.json").write_text(json.dumps(state))
+ session = FakeSession()
+ next_due = asyncio.run(events.refresh_server_subscriptions(_server(session), force=False, failures=failures))
+ assert session.sent == [] and 100 < next_due - time.time() <= 4 * events._REFRESH_MIN_SLEEP_S
+
def test_a_new_subscription_wakes_the_refresh_loop(self, home, monkeypatch):
passes = []
- async def recording_pass(server, *, force):
+ async def recording_pass(server, *, force, failures=None):
passes.append(force)
return float("inf") # nothing due: without a wake-up the loop sleeps for an hour
monkeypatch.setattr(events, "refresh_server_subscriptions", recording_pass)
diff --git a/tools/mcp_tool_events.py b/tools/mcp_tool_events.py
index cb33724ff4b..7c0350f0708 100644
--- a/tools/mcp_tool_events.py
+++ b/tools/mcp_tool_events.py
@@ -35,8 +35,11 @@ _REFRESH_MIN_SLEEP_S, _REFRESH_MAX_SLEEP_S = 30.0, 3600.0
_ROUTE_SLUG_RE = re.compile(r"[^a-z0-9_-]+")
# Per-route subscription state (grant, cursor, granted URL), kept out of the route config.
_STATE_FILENAME = "mcp_event_subscriptions.json"
-# Error codes from the sketch's Error Codes table (SEP-3415 numbering) that end a subscription for good.
-_FORBIDDEN, _NOT_FOUND = -32024, -32023
+# Error codes from the sketch's Error Codes table (SEP-3415 numbering) plus JSON-RPC InvalidParams.
+_FORBIDDEN, _NOT_FOUND, _UNSUPPORTED, _INVALID_PARAMS = -32024, -32023, -32026, -32602
+# A refresh refused with one of these can never succeed as stored: access was revoked, the event type is
+# gone, its arguments no longer validate, or the server no longer offers what the subscription asks for.
+_TERMINAL_REFRESH_CODES = frozenset({_FORBIDDEN, _NOT_FOUND, _UNSUPPORTED, _INVALID_PARAMS})
# Refresh loops by server name, each as (its event loop, its wake-up event). A new subscription wakes them
# so its first refresh is scheduled from its own grant, not after the loop's current sleep (up to an hour).
_refresh_wakers: dict[str, set[tuple[asyncio.AbstractEventLoop, asyncio.Event]]] = {}
@@ -218,15 +221,21 @@ def _wake_refresh_loops(server_name: str) -> None:
loop.call_soon_threadsafe(wake.set)
-async def refresh_server_subscriptions(server, *, force: bool) -> float:
+async def refresh_server_subscriptions(server, *, force: bool,
+ failures: Optional[dict[str, tuple[int, float]]] = None) -> float:
"""Re-subscribe this server's routes that are due (all of them when *force*, i.e. on a fresh
- connection, where the server may have lost soft state). Returns the next due time."""
+ connection, where the server may have lost soft state). A failed refresh is retried with exponential
+ back-off: *failures* maps a route to (consecutive failures, retry time) across passes. Returns the next
+ due time."""
+ failures = {} if failures is None else failures
url_base = _callback_base()
routes = await asyncio.to_thread(_load_routes, server.name)
states = await asyncio.to_thread(_load_state)
next_due = float("inf")
for route_name in routes:
due = _refresh_due_at(states.get(route_name, {}))
+ if route_name in failures:
+ due = max(due, failures[route_name][1])
if url_base and (force or due <= time.time()):
try:
async with server._rpc_lock:
@@ -236,18 +245,24 @@ async def refresh_server_subscriptions(server, *, force: bool) -> float:
if route is not None:
stored = await _subscribe_route(server.session, route_name, route, url_base)
if stored is None:
+ failures.pop(route_name, None)
continue
except Exception as exc:
error = _mcp_error(exc)
- if error and error["code"] in (_FORBIDDEN, _NOT_FOUND):
- logger.warning("MCP server '%s': event subscription %s ended by the server (%s); removing it",
+ if error and error["code"] in _TERMINAL_REFRESH_CODES:
+ logger.warning("MCP server '%s': event subscription %s refused by the server (%s); removing it",
server.name, route_name, error)
await asyncio.to_thread(_write_route, route_name, None)
+ failures.pop(route_name, None)
continue
- logger.warning("MCP server '%s': refreshing event subscription %s failed: %s",
- server.name, route_name, error or exc, exc_info=error is None)
- due = time.time() + _REFRESH_MIN_SLEEP_S
+ count = failures.get(route_name, (0, 0.0))[0] + 1
+ delay = min(_REFRESH_MAX_SLEEP_S, _REFRESH_MIN_SLEEP_S * 2 ** min(count - 1, 10))
+ due = time.time() + delay
+ failures[route_name] = (count, due)
+ logger.warning("MCP server '%s': refreshing event subscription %s failed (attempt %d, next in %.0fs): %s",
+ server.name, route_name, count, delay, error or exc, exc_info=error is None)
else:
+ failures.pop(route_name, None)
due = _refresh_due_at(stored)
next_due = min(next_due, due)
return next_due
@@ -258,12 +273,13 @@ async def run_refresh_loop(server) -> None:
wake = asyncio.Event()
waker = (asyncio.get_running_loop(), wake)
_refresh_wakers.setdefault(server.name, set()).add(waker)
+ failures: dict[str, tuple[int, float]] = {}
force = True
try:
while True:
wake.clear() # before the pass, so a subscription made during it still wakes the next one
try:
- next_due = await refresh_server_subscriptions(server, force=force)
+ next_due = await refresh_server_subscriptions(server, force=force, failures=failures)
except Exception as exc:
logger.warning("MCP server '%s': event subscription refresh failed: %s", server.name, exc,
exc_info=True)
diff --git a/website/docs/user-guide/features/mcp.md b/website/docs/user-guide/features/mcp.md
index 72ee30b5afd..f90c0881203 100644
--- a/website/docs/user-guide/features/mcp.md
+++ b/website/docs/user-guide/features/mcp.md
@@ -1035,8 +1035,10 @@ Each subscription is stored as a dynamic webhook route (`hermes webhook list` sh
`mcp_event_subscriptions.json`, so the route itself only changes when you resubscribe with different
instructions or a different destination. Hermes answers the server's signed verification challenge, deduplicates
events by `eventId`, and refreshes the subscription before the lifetime the server granted runs out
-while the MCP connection is up (and again whenever it reconnects). If the server ends a
-subscription, Hermes removes the route and tells the `deliver` target. Events only arrive while
+while the MCP connection is up (and again whenever it reconnects). A refresh the server refuses
+for good (access revoked, event type removed, arguments no longer valid) removes the route; other
+failures are retried with back-off. If the server ends a subscription,
+Hermes removes the route and tells the `deliver` target. Events only arrive while
`hermes gateway run` is running. Event payloads are external data: event runs use the same
constrained webhook toolset as other webhook routes.
The listener's per-route rate limit (`platforms.webhook.extra.rate_limit`, 30 a minute by default)