The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(mcp): keep event subscription state out of the webhook route
emozilla4608cdc3a7f7
diff --git a/gateway/platforms/webhook_mcp_events.py b/gateway/platforms/webhook_mcp_events.pyindex 4e717132a01..c723a109187 100644--- a/gateway/platforms/webhook_mcp_events.py+++ b/gateway/platforms/webhook_mcp_events.py@@ -33,19 +33,18 @@ def mcp_event_delivery_id(route_config: dict, payload: Any) -> Optional[str]:return f"mcp-event:{event_id}" if isinstance(event_id, str) and event_id else None-def _update_event_block(route_name: str, **changes: Any) -> None:- from hermes_cli.webhook import _mutate_subscriptions-- def mutate(subscriptions: dict) -> None:- route = subscriptions.get(route_name)- if isinstance(route, dict) and isinstance(route.get("mcp_event"), dict):- subscriptions[route_name] = {**route, "mcp_event": {**route["mcp_event"], **changes}}- _mutate_subscriptions(mutate)+def _store_cursor(route_name: str, cursor: str) -> None:+ # Subscription state lives beside the route, never in it: an admitted run's reply is checked against+ # a digest of the route config, so a cursor written there would strand every run still in flight.+ from tools.mcp_tool_events import store_cursor+ store_cursor(route_name, cursor)def _remove_route(route_name: str) -> None:from hermes_cli.webhook import _mutate_subscriptions+ from tools.mcp_tool_events import drop_state_mutate_subscriptions(lambda subscriptions: subscriptions.pop(route_name, None))+ drop_state(route_name)async def _verification(adapter, route_name: str, route_config: dict, payload: dict, profile) -> Any:@@ -74,7 +73,7 @@ async def _terminated(adapter, route_name: str, route_config: dict, payload: dicasync def _gap(adapter, route_name: str, route_config: dict, payload: dict, profile) -> Any:cursor = payload.get("cursor")if isinstance(cursor, str) and cursor:- await asyncio.to_thread(_update_event_block, route_name, cursor=cursor)+ await asyncio.to_thread(_store_cursor, route_name, cursor)logger.warning("[webhook] MCP event route %s: the server skipped events (gap); resuming from a fresh cursor",route_name)return _json({"status": "gap"})@@ -93,6 +92,6 @@ async def handle_mcp_event_control(adapter, route_name: str, route_config: dict,if handler is not None:return await handler(adapter, route_name, route_config, payload, profile)cursor = payload.get("cursor")- if isinstance(cursor, str) and cursor and cursor != route_config["mcp_event"].get("cursor"):- await asyncio.to_thread(_update_event_block, route_name, cursor=cursor)+ if isinstance(cursor, str) and cursor:+ await asyncio.to_thread(_store_cursor, route_name, cursor)return Nonediff --git a/tests/gateway/test_webhook_mcp_events.py b/tests/gateway/test_webhook_mcp_events.pyindex 96ca155f8b0..72585b12dd9 100644--- a/tests/gateway/test_webhook_mcp_events.py+++ b/tests/gateway/test_webhook_mcp_events.py@@ -90,11 +90,13 @@ async def test_occurrence_runs_once_per_event_id(home):@pytest.mark.asyncioasync def test_terminated_drops_the_route(home):+ (home / "mcp_event_subscriptions.json").write_text(json.dumps({ROUTE: {"cursor": "c-1"}}))adapter = _adapter()async with TestClient(TestServer(_app(adapter))) as cli:resp = await _post(cli, {"type": "terminated", "error": {"code": -32024, "message": "Forbidden"}})assert resp.status == 200assert ROUTE not in _routes(home)+ assert ROUTE not in json.loads((home / "mcp_event_subscriptions.json").read_text())adapter.handle_message.assert_not_called()@@ -104,5 +106,31 @@ async def test_gap_remembers_the_fresh_cursor(home):async with TestClient(TestServer(_app(adapter))) as cli:resp = await _post(cli, {"type": "gap", "cursor": "c-9"})assert resp.status == 200- assert _routes(home)[ROUTE]["mcp_event"]["cursor"] == "c-9"+ assert json.loads((home / "mcp_event_subscriptions.json").read_text())[ROUTE]["cursor"] == "c-9"adapter.handle_message.assert_not_called()+++async def test_subscription_state_changes_do_not_strand_an_admitted_run(home):+ # The durable admission pins a digest of the route config, and the reply leg refuses a route+ # that changed since. The cursor and the grant change all the time, so they must live outside it.+ from gateway.platforms.webhook_delivery import retained_destination+ from tools import mcp_tool_events as events+ routes = _routes(home)+ routes[ROUTE].update(deliver="telegram", deliver_extra={"chat_id": "42"})+ (home / "webhook_subscriptions.json").write_text(json.dumps(routes))+ adapter = _adapter()+ occurrence = {"name": "feed.emission.created", "timestamp": "2026-10-16T00:00:00Z", "data": {"feed_id": "f-1"}}+ async with TestClient(TestServer(_app(adapter))) as cli:+ assert (await _post(cli, {**occurrence, "eventId": "e-1", "cursor": "c-1"})).status == 202+ chat_id = adapter.handle_message.await_args_list[0].args[0].source.chat_id+ destination = retained_destination(adapter, chat_id)+ assert (await _post(cli, {**occurrence, "eventId": "e-2", "cursor": "c-2"})).status == 202+ assert (await _post(cli, {"type": "gap", "cursor": "c-3"})).status == 200+ events._store_grant(ROUTE, {"id": "sub_1", "refreshBefore": "2026-10-17T00:00:00Z", "cursor": "c-4"},+ "https://hermes.example.com/webhooks/" + ROUTE)+ assert retained_destination(adapter, chat_id) == destination+ retry = await _post(cli, {**occurrence, "eventId": "e-1", "cursor": "c-4"}) # watermark moved on+ assert retry.status == 200 and (await retry.json())["status"] == "duplicate"+ assert _routes(home)[ROUTE] == routes[ROUTE]+diff --git a/tests/tools/test_mcp_events.py b/tests/tools/test_mcp_events.pyindex 3b121a01eec..f09f32f6d18 100644--- a/tests/tools/test_mcp_events.py+++ b/tests/tools/test_mcp_events.py@@ -32,6 +32,11 @@ def _routes(home):return json.loads(path.read_text()) if path.exists() else {}+def _state(home):+ path = home / "mcp_event_subscriptions.json"+ return json.loads(path.read_text()) if path.exists() else {}++def _grant(seconds=86400):expires = datetime.now(UTC) + timedelta(seconds=seconds)return {"id": "sub_1", "refreshBefore": expires.isoformat().replace("+00:00", "Z"), "cursor": None,@@ -93,15 +98,27 @@ class TestSubscribe:assert route["secret"].startswith("whsec_")assert len(base64.b64decode(route["secret"].removeprefix("whsec_"))) == 32assert route["deliver"] == "telegram" and "Explain the deal." in route["prompt"]- assert route["mcp_event"]["url"] == sent["params"]["delivery"]["url"]- assert route["mcp_event"]["subscription_id"] == "sub_1"- assert result["subscribed"]["refreshBefore"] == route["mcp_event"]["refresh_before"]+ state = _state(home)[name]+ assert state["url"] == sent["params"]["delivery"]["url"]+ assert state["subscription_id"] == "sub_1"+ assert result["subscribed"]["refreshBefore"] == state["refresh_before"]+ assert set(route["mcp_event"]) == {"server", "name", "arguments"} # the route holds no changing statedef test_resubscribing_keeps_the_route_secret(self, home):_subscribe(FakeSession(_grant()))- first = next(iter(_routes(home).values()))["secret"]+ data = _routes(home)+ next(iter(data.values()))["created_at"] = "2026-01-01T00:00:00Z"+ (home / "webhook_subscriptions.json").write_text(json.dumps(data))+ first = next(iter(data.values()))_subscribe(FakeSession(_grant()))- assert [r["secret"] for r in _routes(home).values()] == [first]+ assert list(_routes(home).values()) == [first] # identical: in-flight runs keep their route digest++ def test_a_fresh_subscription_ignores_state_left_by_a_removed_route(self, home):+ name = events._route_name("ambush", "feed.emission.created", ARGS) # e.g. after `hermes webhook remove`+ (home / "mcp_event_subscriptions.json").write_text(json.dumps({name: {"cursor": "stale"}}))+ session = FakeSession(_grant())+ _subscribe(session)+ assert "cursor" not in session.sent[0]["params"]def test_a_rejected_subscription_leaves_no_route(self, home):failed = MCPError(-32027, "CallbackEndpointError", {"reason": "challenge_failed"})@@ -120,14 +137,14 @@ class TestSubscribe:class TestUnsubscribe:def test_unsubscribe_sends_the_granted_url_and_drops_the_route(self, home):_subscribe(FakeSession(_grant()))- url = next(iter(_routes(home).values()))["mcp_event"]["url"]+ url = next(iter(_state(home).values()))["url"](home / "config.yaml").write_text("{}\n") # the subscription key is the URL that was grantedsession = FakeSession({})result = asyncio.run(events._unsubscribe_event(session, {"name": "feed.emission.created", "arguments": ARGS}, "ambush"))assert session.sent[0]["params"] == {"name": "feed.emission.created", "arguments": ARGS,"delivery": {"url": url}}- assert "unsubscribed" in result and _routes(home) == {}+ assert "unsubscribed" in result and _routes(home) == {} and _state(home) == {}class TestRefresh:diff --git a/tools/mcp_tool_events.py b/tools/mcp_tool_events.pyindex b04d9944d02..59c8461ef74 100644--- a/tools/mcp_tool_events.py+++ b/tools/mcp_tool_events.py@@ -4,10 +4,10 @@ WG: modelcontextprotocol/experimental-ext-triggers-events ``docs/design-sketch-pOpt-in per server (``mcp_servers.<name>.events: true``): the SDK drops the server's ``events``capability while parsing, so it cannot gate these tools the way ``resources``/``prompts`` do. Eachsubscription is a dynamic webhook route (``webhook_subscriptions.json``) with an ``mcp_event`` block-(server, event name, arguments, the server's grant); the gateway's webhook adapter receives the-deliveries (``gateway/platforms/webhook_mcp_events.py``) and the server task keeps every route's-subscription alive with :func:`run_refresh_loop`. Delivery modes ``poll`` and ``push`` are not-implemented.+(server, event name, arguments); its grant and replay cursor live in ``mcp_event_subscriptions.json``.+The gateway's webhook adapter receives the deliveries (``gateway/platforms/webhook_mcp_events.py``) and+the server task keeps every route's subscription alive with :func:`run_refresh_loop`. Delivery modes+``poll`` and ``push`` are not implemented."""import asyncio@@ -32,6 +32,8 @@ _DEFAULT_TTL_MS = 24 * 3600 * 1000_REFRESH_AT_FRACTION = 0.8_REFRESH_MIN_SLEEP_S, _REFRESH_MAX_SLEEP_S = 30.0, 3600.0_ROUTE_SLUG_RE = re.compile(r"[^a-z0-9_-]+")+# Per-route subscription state (grant, cursor, granted URL), kept out of the route config.+_STATE_FILENAME = "mcp_event_subscriptions.json"# Error codes from the sketch's Error Codes table (SEP-3415 numbering) that end a subscription for good._FORBIDDEN, _NOT_FOUND = -32024, -32023@@ -91,30 +93,84 @@ def _mcp_error(exc: BaseException) -> Optional[dict]:return {"code": exc.code, "message": exc.message, **({"data": exc.data} if exc.data is not None else {})}-def _subscribe_params(route_name: str, route: dict, url_base: str) -> dict:+def _subscribe_params(route_name: str, route: dict, url_base: str, state: dict) -> dict:event = route["mcp_event"]params = {"name": event["name"], "arguments": event["arguments"],"delivery": {"mode": "webhook", "url": _callback_url(url_base, route_name), "secret": route["secret"]},"ttlMs": event.get("ttl_ms", _DEFAULT_TTL_MS)}- if event.get("cursor"):- params["cursor"] = event["cursor"]+ if state.get("cursor"):+ params["cursor"] = state["cursor"]return params-def _record_grant(event: dict, result: dict, url: str) -> dict:- """The route's ``mcp_event`` block updated with a subscribe/refresh result (``url`` is half of the- server's subscription key, so unsubscribe must send the one that was granted)."""- granted = dict(event, subscription_id=result.get("id"), refresh_before=result.get("refreshBefore"),- granted_at=time.time(), url=url)- if result.get("cursor"):- granted["cursor"] = result["cursor"]- return granted+# ---- subscription state ----+# The route holds the subscription's fixed contract (server, event name, arguments, secret, prompt,+# destination). What changes while it lives (the grant, the cursor, the granted callback URL) is kept+# here instead, keyed by route name: an admitted run pins a digest of its route config and its reply is+# refused if the route changed, so per-event cursor writes on the route would strand in-flight runs.+def _state_path():+ from hermes_constants import get_hermes_home+ return get_hermes_home() / _STATE_FILENAME-def _refresh_due_at(event: dict) -> float:++def _load_state() -> dict[str, dict]:+ """Lock-free read; writers publish by atomic rename."""+ try:+ data = json.loads(_state_path().read_text(encoding="utf-8-sig"))+ except (OSError, ValueError):+ return {}+ return {k: v for k, v in data.items() if isinstance(v, dict)} if isinstance(data, dict) else {}+++def _mutate_state(mutate, *, route_must_exist: Optional[str] = None) -> Any:+ """Apply *mutate* to the state under the webhook store's lock (never nested inside+ ``_mutate_subscriptions``: the lock is not reentrant). With *route_must_exist*, nothing is written+ and None is returned when that route is gone."""+ from hermes_cli.webhook import _load_subscriptions_unlocked, _subscriptions_lock+ from utils import atomic_json_write+ with _subscriptions_lock():+ if route_must_exist is not None:+ route = _load_subscriptions_unlocked().get(route_must_exist)+ if not (isinstance(route, dict) and isinstance(route.get("mcp_event"), dict)):+ return None+ state = _load_state()+ result = mutate(state)+ atomic_json_write(_state_path(), state, mode=0o600)+ return result+++def _store_grant(route_name: str, result: dict, url: str) -> Optional[dict]:+ """Record a subscribe/refresh result for a route that still exists (an unsubscribe made while the+ request was in flight wins); returns the stored state, None when the route is gone. ``url`` is half+ of the server's subscription key, so unsubscribe must send the one that was granted."""+ def mutate(state: dict) -> dict:+ entry = {**state.get(route_name, {}), "subscription_id": result.get("id"),+ "refresh_before": result.get("refreshBefore"), "granted_at": time.time(), "url": url}+ if result.get("cursor"):+ entry["cursor"] = result["cursor"]+ state[route_name] = entry+ return entry+ return _mutate_state(mutate, route_must_exist=route_name)+++def store_cursor(route_name: str, cursor: str) -> None:+ """Persist the newest replay cursor the server sent for a route (occurrences and ``gap``)."""+ def mutate(state: dict) -> None:+ state[route_name] = {**state.get(route_name, {}), "cursor": cursor}+ if _load_state().get(route_name, {}).get("cursor") != cursor:+ _mutate_state(mutate, route_must_exist=route_name)+++def drop_state(route_name: str) -> None:+ if route_name in _load_state():+ _mutate_state(lambda state: state.pop(route_name, None))+++def _refresh_due_at(state: dict) -> float:"""Epoch seconds at which the route must be refreshed; 0 when it never got a grant on record,``inf`` for a no-expiry grant."""- granted_at, refresh_before = event.get("granted_at"), event.get("refresh_before")+ granted_at, refresh_before = state.get("granted_at"), state.get("refresh_before")if not isinstance(granted_at, (int, float)):return 0.0if refresh_before is None:@@ -135,20 +191,8 @@ def _write_route(route_name: str, route: Optional[dict]) -> None:else:subscriptions[route_name] = route_mutate_subscriptions(mutate)---def _merge_grant(route_name: str, event: dict) -> Optional[dict]:- """Store a grant on the route as it is NOW (an unsubscribe or a dashboard edit made while the RPC- was in flight wins); returns the published route, None when it is gone."""- from hermes_cli.webhook import _mutate_subscriptions-- def mutate(subscriptions: dict) -> Optional[dict]:- route = subscriptions.get(route_name)- if not isinstance(route, dict):- return None- subscriptions[route_name] = {**route, "mcp_event": event}- return subscriptions[route_name]- return _mutate_subscriptions(mutate)+ if route is None:+ drop_state(route_name)def _load_routes(server_name: str) -> dict[str, dict]:@@ -157,11 +201,12 @@ def _load_routes(server_name: str) -> dict[str, dict]:async def _subscribe_route(session, route_name: str, route: dict, url_base: str) -> Optional[dict]:- """``events/subscribe`` for one route and persist the grant; returns the published route."""- params = _subscribe_params(route_name, route, url_base)+ """``events/subscribe`` for one route and record the grant; returns the stored state, None when the+ route was removed while the request was in flight."""+ state = (await asyncio.to_thread(_load_state)).get(route_name, {})+ params = _subscribe_params(route_name, route, url_base, state)result = await _send(session, "events/subscribe", params)- event = _record_grant(route["mcp_event"], result, params["delivery"]["url"])- return await asyncio.to_thread(_merge_grant, route_name, event)+ return await asyncio.to_thread(_store_grant, route_name, result, params["delivery"]["url"])async def refresh_server_subscriptions(server, *, force: bool) -> float:@@ -169,17 +214,19 @@ async def refresh_server_subscriptions(server, *, force: bool) -> float:connection, where the server may have lost soft state). Returns the next due time."""url_base = _callback_base()routes = await asyncio.to_thread(_load_routes, server.name)+ states = await asyncio.to_thread(_load_state)next_due = float("inf")- for route_name, route in routes.items():- due = _refresh_due_at(route["mcp_event"])+ for route_name in routes:+ due = _refresh_due_at(states.get(route_name, {}))if url_base and (force or due <= time.time()):try:async with server._rpc_lock:# Re-read under the lock: an unsubscribe that ran while we waited must not be undone.route = (await asyncio.to_thread(_load_routes, server.name)).get(route_name)+ stored = Noneif route is not None:- route = await _subscribe_route(server.session, route_name, route, url_base)- if route is None:+ stored = await _subscribe_route(server.session, route_name, route, url_base)+ if stored is None:continueexcept Exception as exc:error = _mcp_error(exc)@@ -192,7 +239,7 @@ async def refresh_server_subscriptions(server, *, force: bool) -> float:server.name, route_name, error or exc, exc_info=error is None)due = time.time() + _REFRESH_MIN_SLEEP_Selse:- due = _refresh_due_at(route["mcp_event"])+ due = _refresh_due_at(stored)next_due = min(next_due, due)return next_due@@ -225,9 +272,10 @@ async def _list_events(session, args: dict, server_name: str) -> dict:if not cursor:breakroutes = await asyncio.to_thread(_load_routes, server_name)+ states = await asyncio.to_thread(_load_state)subscribed = [{"name": r["mcp_event"]["name"], "arguments": r["mcp_event"]["arguments"],- "deliver": r.get("deliver", "log"), "refreshBefore": r["mcp_event"].get("refresh_before")}- for r in routes.values()]+ "deliver": r.get("deliver", "log"), "refreshBefore": states.get(n, {}).get("refresh_before")}+ for n, r in routes.items()]return {"events": events, "subscriptions": subscribed}@@ -240,20 +288,23 @@ async def _subscribe_event(session, args: dict, server_name: str) -> dict:route_name = _route_name(server_name, args["name"], arguments)existing = (await asyncio.to_thread(_load_routes, server_name)).get(route_name)deliver_extra = {"chat_id": args["deliver_chat_id"]} if args.get("deliver_chat_id") else {}+ # The route is the subscription's fixed contract: an admitted run's reply is checked against a digest+ # of it, so nothing that changes while the subscription lives (grant, cursor) is stored here, and an+ # unchanged resubscribe republishes identical bytes (created_at is kept).route = {"description": f"MCP event {args['name']} from {server_name}","secret": (existing or {}).get("secret") or _new_secret(),"prompt": _event_prompt(server_name, args["name"], args.get("instructions") or ""),"deliver": args.get("deliver") or "log", "profile": "default",**({"deliver_extra": deliver_extra} if deliver_extra else {}),- "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),- "mcp_event": {"server": server_name, "name": args["name"], "arguments": arguments,- **({"cursor": existing["mcp_event"]["cursor"]}- if existing and existing["mcp_event"].get("cursor") else {})}}+ "created_at": (existing or {}).get("created_at") or time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),+ "mcp_event": {"server": server_name, "name": args["name"], "arguments": arguments}}+ if existing is None:+ await asyncio.to_thread(drop_state, route_name) # a fresh subscription replays nothing stale# The route must exist before subscribing: the server verifies the callback with a signed challenge.await asyncio.to_thread(_write_route, route_name, route)try:- published = await _subscribe_route(session, route_name, route, url_base) or route+ stored = await _subscribe_route(session, route_name, route, url_base)except Exception as exc:await asyncio.to_thread(_write_route, route_name, existing)error = _mcp_error(exc)@@ -261,7 +312,7 @@ async def _subscribe_event(session, args: dict, server_name: str) -> dict:raisereturn {"error": error}return {"subscribed": {"name": args["name"], "arguments": arguments, "deliver": route["deliver"],- "refreshBefore": published["mcp_event"].get("refresh_before")}}+ "refreshBefore": (stored or {}).get("refresh_before")}}async def _unsubscribe_event(session, args: dict, server_name: str) -> dict:@@ -270,7 +321,8 @@ async def _unsubscribe_event(session, args: dict, server_name: str) -> dict:route = (await asyncio.to_thread(_load_routes, server_name)).get(route_name)if route is None:return {"error": f"No subscription to '{args['name']}' with these arguments on '{server_name}'"}- url = route["mcp_event"].get("url") or _callback_url(_callback_base() or "", route_name)+ state = (await asyncio.to_thread(_load_state)).get(route_name, {})+ url = state.get("url") or _callback_url(_callback_base() or "", route_name)try:await _send(session, "events/unsubscribe",{"name": args["name"], "arguments": arguments, "delivery": {"url": url}})diff --git a/website/docs/user-guide/features/mcp.md b/website/docs/user-guide/features/mcp.mdindex 9bd24589101..72ee30b5afd 100644--- a/website/docs/user-guide/features/mcp.md+++ b/website/docs/user-guide/features/mcp.md@@ -1031,7 +1031,9 @@ The server then gets three more utility tools:- `unsubscribe_event`: cancel a subscriptionEach subscription is stored as a dynamic webhook route (`hermes webhook list` shows it) with its own-`whsec_` signing secret. Hermes answers the server's signed verification challenge, deduplicates+`whsec_` signing secret; the grant and the replay cursor are kept beside it in+`mcp_event_subscriptions.json`, so the route itself only changes when you resubscribe with different+instructions or a different destination. Hermes answers the server's signed verification challenge, deduplicatesevents by `eventId`, and refreshes the subscription before the lifetime the server granted runs outwhile the MCP connection is up (and again whenever it reconnects). If the server ends asubscription, Hermes removes the route and tells the `deliver` target. Events only arrive while