Skip to content

The agent that grows with you

fix(mcp): keep event subscription state out of the webhook route

emozilla4608cdc3a7f7
diff --git a/gateway/platforms/webhook_mcp_events.py b/gateway/platforms/webhook_mcp_events.py
index 4e717132a01..c723a109187 100644
--- a/gateway/platforms/webhook_mcp_events.py
+++ b/gateway/platforms/webhook_mcp_events.py
@@ -33,19 +33,18 @@ def mcp_event_delivery_id(route_config: dict, payload: Any) -> Optional[str]:
return f"mcp-event:{event_id}" if isinstance(event_id, str) and event_id else None
-def _update_event_block(route_name: str, **changes: Any) -> None:
- from hermes_cli.webhook import _mutate_subscriptions
-
- def mutate(subscriptions: dict) -> None:
- route = subscriptions.get(route_name)
- if isinstance(route, dict) and isinstance(route.get("mcp_event"), dict):
- subscriptions[route_name] = {**route, "mcp_event": {**route["mcp_event"], **changes}}
- _mutate_subscriptions(mutate)
+def _store_cursor(route_name: str, cursor: str) -> None:
+ # Subscription state lives beside the route, never in it: an admitted run's reply is checked against
+ # a digest of the route config, so a cursor written there would strand every run still in flight.
+ from tools.mcp_tool_events import store_cursor
+ store_cursor(route_name, cursor)
def _remove_route(route_name: str) -> None:
from hermes_cli.webhook import _mutate_subscriptions
+ from tools.mcp_tool_events import drop_state
_mutate_subscriptions(lambda subscriptions: subscriptions.pop(route_name, None))
+ drop_state(route_name)
async def _verification(adapter, route_name: str, route_config: dict, payload: dict, profile) -> Any:
@@ -74,7 +73,7 @@ async def _terminated(adapter, route_name: str, route_config: dict, payload: dic
async def _gap(adapter, route_name: str, route_config: dict, payload: dict, profile) -> Any:
cursor = payload.get("cursor")
if isinstance(cursor, str) and cursor:
- await asyncio.to_thread(_update_event_block, route_name, cursor=cursor)
+ await asyncio.to_thread(_store_cursor, route_name, cursor)
logger.warning("[webhook] MCP event route %s: the server skipped events (gap); resuming from a fresh cursor",
route_name)
return _json({"status": "gap"})
@@ -93,6 +92,6 @@ async def handle_mcp_event_control(adapter, route_name: str, route_config: dict,
if handler is not None:
return await handler(adapter, route_name, route_config, payload, profile)
cursor = payload.get("cursor")
- if isinstance(cursor, str) and cursor and cursor != route_config["mcp_event"].get("cursor"):
- await asyncio.to_thread(_update_event_block, route_name, cursor=cursor)
+ if isinstance(cursor, str) and cursor:
+ await asyncio.to_thread(_store_cursor, route_name, cursor)
return None
diff --git a/tests/gateway/test_webhook_mcp_events.py b/tests/gateway/test_webhook_mcp_events.py
index 96ca155f8b0..72585b12dd9 100644
--- a/tests/gateway/test_webhook_mcp_events.py
+++ b/tests/gateway/test_webhook_mcp_events.py
@@ -90,11 +90,13 @@ async def test_occurrence_runs_once_per_event_id(home):
@pytest.mark.asyncio
async def test_terminated_drops_the_route(home):
+ (home / "mcp_event_subscriptions.json").write_text(json.dumps({ROUTE: {"cursor": "c-1"}}))
adapter = _adapter()
async with TestClient(TestServer(_app(adapter))) as cli:
resp = await _post(cli, {"type": "terminated", "error": {"code": -32024, "message": "Forbidden"}})
assert resp.status == 200
assert ROUTE not in _routes(home)
+ assert ROUTE not in json.loads((home / "mcp_event_subscriptions.json").read_text())
adapter.handle_message.assert_not_called()
@@ -104,5 +106,31 @@ async def test_gap_remembers_the_fresh_cursor(home):
async with TestClient(TestServer(_app(adapter))) as cli:
resp = await _post(cli, {"type": "gap", "cursor": "c-9"})
assert resp.status == 200
- assert _routes(home)[ROUTE]["mcp_event"]["cursor"] == "c-9"
+ assert json.loads((home / "mcp_event_subscriptions.json").read_text())[ROUTE]["cursor"] == "c-9"
adapter.handle_message.assert_not_called()
+
+
+async def test_subscription_state_changes_do_not_strand_an_admitted_run(home):
+ # The durable admission pins a digest of the route config, and the reply leg refuses a route
+ # that changed since. The cursor and the grant change all the time, so they must live outside it.
+ from gateway.platforms.webhook_delivery import retained_destination
+ from tools import mcp_tool_events as events
+ routes = _routes(home)
+ routes[ROUTE].update(deliver="telegram", deliver_extra={"chat_id": "42"})
+ (home / "webhook_subscriptions.json").write_text(json.dumps(routes))
+ adapter = _adapter()
+ occurrence = {"name": "feed.emission.created", "timestamp": "2026-10-16T00:00:00Z", "data": {"feed_id": "f-1"}}
+ async with TestClient(TestServer(_app(adapter))) as cli:
+ assert (await _post(cli, {**occurrence, "eventId": "e-1", "cursor": "c-1"})).status == 202
+ chat_id = adapter.handle_message.await_args_list[0].args[0].source.chat_id
+ destination = retained_destination(adapter, chat_id)
+ assert (await _post(cli, {**occurrence, "eventId": "e-2", "cursor": "c-2"})).status == 202
+ assert (await _post(cli, {"type": "gap", "cursor": "c-3"})).status == 200
+ events._store_grant(ROUTE, {"id": "sub_1", "refreshBefore": "2026-10-17T00:00:00Z", "cursor": "c-4"},
+ "https://hermes.example.com/webhooks/" + ROUTE)
+ assert retained_destination(adapter, chat_id) == destination
+ retry = await _post(cli, {**occurrence, "eventId": "e-1", "cursor": "c-4"}) # watermark moved on
+ assert retry.status == 200 and (await retry.json())["status"] == "duplicate"
+ assert _routes(home)[ROUTE] == routes[ROUTE]
+
diff --git a/tests/tools/test_mcp_events.py b/tests/tools/test_mcp_events.py
index 3b121a01eec..f09f32f6d18 100644
--- a/tests/tools/test_mcp_events.py
+++ b/tests/tools/test_mcp_events.py
@@ -32,6 +32,11 @@ def _routes(home):
return json.loads(path.read_text()) if path.exists() else {}
+def _state(home):
+ path = home / "mcp_event_subscriptions.json"
+ return json.loads(path.read_text()) if path.exists() else {}
+
+
def _grant(seconds=86400):
expires = datetime.now(UTC) + timedelta(seconds=seconds)
return {"id": "sub_1", "refreshBefore": expires.isoformat().replace("+00:00", "Z"), "cursor": None,
@@ -93,15 +98,27 @@ class TestSubscribe:
assert route["secret"].startswith("whsec_")
assert len(base64.b64decode(route["secret"].removeprefix("whsec_"))) == 32
assert route["deliver"] == "telegram" and "Explain the deal." in route["prompt"]
- assert route["mcp_event"]["url"] == sent["params"]["delivery"]["url"]
- assert route["mcp_event"]["subscription_id"] == "sub_1"
- assert result["subscribed"]["refreshBefore"] == route["mcp_event"]["refresh_before"]
+ state = _state(home)[name]
+ assert state["url"] == sent["params"]["delivery"]["url"]
+ assert state["subscription_id"] == "sub_1"
+ assert result["subscribed"]["refreshBefore"] == state["refresh_before"]
+ assert set(route["mcp_event"]) == {"server", "name", "arguments"} # the route holds no changing state
def test_resubscribing_keeps_the_route_secret(self, home):
_subscribe(FakeSession(_grant()))
- first = next(iter(_routes(home).values()))["secret"]
+ data = _routes(home)
+ next(iter(data.values()))["created_at"] = "2026-01-01T00:00:00Z"
+ (home / "webhook_subscriptions.json").write_text(json.dumps(data))
+ first = next(iter(data.values()))
_subscribe(FakeSession(_grant()))
- assert [r["secret"] for r in _routes(home).values()] == [first]
+ assert list(_routes(home).values()) == [first] # identical: in-flight runs keep their route digest
+
+ def test_a_fresh_subscription_ignores_state_left_by_a_removed_route(self, home):
+ name = events._route_name("ambush", "feed.emission.created", ARGS) # e.g. after `hermes webhook remove`
+ (home / "mcp_event_subscriptions.json").write_text(json.dumps({name: {"cursor": "stale"}}))
+ session = FakeSession(_grant())
+ _subscribe(session)
+ assert "cursor" not in session.sent[0]["params"]
def test_a_rejected_subscription_leaves_no_route(self, home):
failed = MCPError(-32027, "CallbackEndpointError", {"reason": "challenge_failed"})
@@ -120,14 +137,14 @@ class TestSubscribe:
class TestUnsubscribe:
def test_unsubscribe_sends_the_granted_url_and_drops_the_route(self, home):
_subscribe(FakeSession(_grant()))
- url = next(iter(_routes(home).values()))["mcp_event"]["url"]
+ url = next(iter(_state(home).values()))["url"]
(home / "config.yaml").write_text("{}\n") # the subscription key is the URL that was granted
session = FakeSession({})
result = asyncio.run(events._unsubscribe_event(
session, {"name": "feed.emission.created", "arguments": ARGS}, "ambush"))
assert session.sent[0]["params"] == {"name": "feed.emission.created", "arguments": ARGS,
"delivery": {"url": url}}
- assert "unsubscribed" in result and _routes(home) == {}
+ assert "unsubscribed" in result and _routes(home) == {} and _state(home) == {}
class TestRefresh:
diff --git a/tools/mcp_tool_events.py b/tools/mcp_tool_events.py
index b04d9944d02..59c8461ef74 100644
--- a/tools/mcp_tool_events.py
+++ b/tools/mcp_tool_events.py
@@ -4,10 +4,10 @@ WG: modelcontextprotocol/experimental-ext-triggers-events ``docs/design-sketch-p
Opt-in per server (``mcp_servers.<name>.events: true``): the SDK drops the server's ``events``
capability while parsing, so it cannot gate these tools the way ``resources``/``prompts`` do. Each
subscription is a dynamic webhook route (``webhook_subscriptions.json``) with an ``mcp_event`` block
-(server, event name, arguments, the server's grant); the gateway's webhook adapter receives the
-deliveries (``gateway/platforms/webhook_mcp_events.py``) and the server task keeps every route's
-subscription alive with :func:`run_refresh_loop`. Delivery modes ``poll`` and ``push`` are not
-implemented.
+(server, event name, arguments); its grant and replay cursor live in ``mcp_event_subscriptions.json``.
+The gateway's webhook adapter receives the deliveries (``gateway/platforms/webhook_mcp_events.py``) and
+the server task keeps every route's subscription alive with :func:`run_refresh_loop`. Delivery modes
+``poll`` and ``push`` are not implemented.
"""
import asyncio
@@ -32,6 +32,8 @@ _DEFAULT_TTL_MS = 24 * 3600 * 1000
_REFRESH_AT_FRACTION = 0.8
_REFRESH_MIN_SLEEP_S, _REFRESH_MAX_SLEEP_S = 30.0, 3600.0
_ROUTE_SLUG_RE = re.compile(r"[^a-z0-9_-]+")
+# Per-route subscription state (grant, cursor, granted URL), kept out of the route config.
+_STATE_FILENAME = "mcp_event_subscriptions.json"
# Error codes from the sketch's Error Codes table (SEP-3415 numbering) that end a subscription for good.
_FORBIDDEN, _NOT_FOUND = -32024, -32023
@@ -91,30 +93,84 @@ def _mcp_error(exc: BaseException) -> Optional[dict]:
return {"code": exc.code, "message": exc.message, **({"data": exc.data} if exc.data is not None else {})}
-def _subscribe_params(route_name: str, route: dict, url_base: str) -> dict:
+def _subscribe_params(route_name: str, route: dict, url_base: str, state: dict) -> dict:
event = route["mcp_event"]
params = {"name": event["name"], "arguments": event["arguments"],
"delivery": {"mode": "webhook", "url": _callback_url(url_base, route_name), "secret": route["secret"]},
"ttlMs": event.get("ttl_ms", _DEFAULT_TTL_MS)}
- if event.get("cursor"):
- params["cursor"] = event["cursor"]
+ if state.get("cursor"):
+ params["cursor"] = state["cursor"]
return params
-def _record_grant(event: dict, result: dict, url: str) -> dict:
- """The route's ``mcp_event`` block updated with a subscribe/refresh result (``url`` is half of the
- server's subscription key, so unsubscribe must send the one that was granted)."""
- granted = dict(event, subscription_id=result.get("id"), refresh_before=result.get("refreshBefore"),
- granted_at=time.time(), url=url)
- if result.get("cursor"):
- granted["cursor"] = result["cursor"]
- return granted
+# ---- subscription state ----
+# The route holds the subscription's fixed contract (server, event name, arguments, secret, prompt,
+# destination). What changes while it lives (the grant, the cursor, the granted callback URL) is kept
+# here instead, keyed by route name: an admitted run pins a digest of its route config and its reply is
+# refused if the route changed, so per-event cursor writes on the route would strand in-flight runs.
+def _state_path():
+ from hermes_constants import get_hermes_home
+ return get_hermes_home() / _STATE_FILENAME
-def _refresh_due_at(event: dict) -> float:
+
+def _load_state() -> dict[str, dict]:
+ """Lock-free read; writers publish by atomic rename."""
+ try:
+ data = json.loads(_state_path().read_text(encoding="utf-8-sig"))
+ except (OSError, ValueError):
+ return {}
+ return {k: v for k, v in data.items() if isinstance(v, dict)} if isinstance(data, dict) else {}
+
+
+def _mutate_state(mutate, *, route_must_exist: Optional[str] = None) -> Any:
+ """Apply *mutate* to the state under the webhook store's lock (never nested inside
+ ``_mutate_subscriptions``: the lock is not reentrant). With *route_must_exist*, nothing is written
+ and None is returned when that route is gone."""
+ from hermes_cli.webhook import _load_subscriptions_unlocked, _subscriptions_lock
+ from utils import atomic_json_write
+ with _subscriptions_lock():
+ if route_must_exist is not None:
+ route = _load_subscriptions_unlocked().get(route_must_exist)
+ if not (isinstance(route, dict) and isinstance(route.get("mcp_event"), dict)):
+ return None
+ state = _load_state()
+ result = mutate(state)
+ atomic_json_write(_state_path(), state, mode=0o600)
+ return result
+
+
+def _store_grant(route_name: str, result: dict, url: str) -> Optional[dict]:
+ """Record a subscribe/refresh result for a route that still exists (an unsubscribe made while the
+ request was in flight wins); returns the stored state, None when the route is gone. ``url`` is half
+ of the server's subscription key, so unsubscribe must send the one that was granted."""
+ def mutate(state: dict) -> dict:
+ entry = {**state.get(route_name, {}), "subscription_id": result.get("id"),
+ "refresh_before": result.get("refreshBefore"), "granted_at": time.time(), "url": url}
+ if result.get("cursor"):
+ entry["cursor"] = result["cursor"]
+ state[route_name] = entry
+ return entry
+ return _mutate_state(mutate, route_must_exist=route_name)
+
+
+def store_cursor(route_name: str, cursor: str) -> None:
+ """Persist the newest replay cursor the server sent for a route (occurrences and ``gap``)."""
+ def mutate(state: dict) -> None:
+ state[route_name] = {**state.get(route_name, {}), "cursor": cursor}
+ if _load_state().get(route_name, {}).get("cursor") != cursor:
+ _mutate_state(mutate, route_must_exist=route_name)
+
+
+def drop_state(route_name: str) -> None:
+ if route_name in _load_state():
+ _mutate_state(lambda state: state.pop(route_name, None))
+
+
+def _refresh_due_at(state: dict) -> float:
"""Epoch seconds at which the route must be refreshed; 0 when it never got a grant on record,
``inf`` for a no-expiry grant."""
- granted_at, refresh_before = event.get("granted_at"), event.get("refresh_before")
+ granted_at, refresh_before = state.get("granted_at"), state.get("refresh_before")
if not isinstance(granted_at, (int, float)):
return 0.0
if refresh_before is None:
@@ -135,20 +191,8 @@ def _write_route(route_name: str, route: Optional[dict]) -> None:
else:
subscriptions[route_name] = route
_mutate_subscriptions(mutate)
-
-
-def _merge_grant(route_name: str, event: dict) -> Optional[dict]:
- """Store a grant on the route as it is NOW (an unsubscribe or a dashboard edit made while the RPC
- was in flight wins); returns the published route, None when it is gone."""
- from hermes_cli.webhook import _mutate_subscriptions
-
- def mutate(subscriptions: dict) -> Optional[dict]:
- route = subscriptions.get(route_name)
- if not isinstance(route, dict):
- return None
- subscriptions[route_name] = {**route, "mcp_event": event}
- return subscriptions[route_name]
- return _mutate_subscriptions(mutate)
+ if route is None:
+ drop_state(route_name)
def _load_routes(server_name: str) -> dict[str, dict]:
@@ -157,11 +201,12 @@ def _load_routes(server_name: str) -> dict[str, dict]:
async def _subscribe_route(session, route_name: str, route: dict, url_base: str) -> Optional[dict]:
- """``events/subscribe`` for one route and persist the grant; returns the published route."""
- params = _subscribe_params(route_name, route, url_base)
+ """``events/subscribe`` for one route and record the grant; returns the stored state, None when the
+ route was removed while the request was in flight."""
+ state = (await asyncio.to_thread(_load_state)).get(route_name, {})
+ params = _subscribe_params(route_name, route, url_base, state)
result = await _send(session, "events/subscribe", params)
- event = _record_grant(route["mcp_event"], result, params["delivery"]["url"])
- return await asyncio.to_thread(_merge_grant, route_name, event)
+ return await asyncio.to_thread(_store_grant, route_name, result, params["delivery"]["url"])
async def refresh_server_subscriptions(server, *, force: bool) -> float:
@@ -169,17 +214,19 @@ async def refresh_server_subscriptions(server, *, force: bool) -> float:
connection, where the server may have lost soft state). Returns the next due time."""
url_base = _callback_base()
routes = await asyncio.to_thread(_load_routes, server.name)
+ states = await asyncio.to_thread(_load_state)
next_due = float("inf")
- for route_name, route in routes.items():
- due = _refresh_due_at(route["mcp_event"])
+ for route_name in routes:
+ due = _refresh_due_at(states.get(route_name, {}))
if url_base and (force or due <= time.time()):
try:
async with server._rpc_lock:
# Re-read under the lock: an unsubscribe that ran while we waited must not be undone.
route = (await asyncio.to_thread(_load_routes, server.name)).get(route_name)
+ stored = None
if route is not None:
- route = await _subscribe_route(server.session, route_name, route, url_base)
- if route is None:
+ stored = await _subscribe_route(server.session, route_name, route, url_base)
+ if stored is None:
continue
except Exception as exc:
error = _mcp_error(exc)
@@ -192,7 +239,7 @@ async def refresh_server_subscriptions(server, *, force: bool) -> float:
server.name, route_name, error or exc, exc_info=error is None)
due = time.time() + _REFRESH_MIN_SLEEP_S
else:
- due = _refresh_due_at(route["mcp_event"])
+ due = _refresh_due_at(stored)
next_due = min(next_due, due)
return next_due
@@ -225,9 +272,10 @@ async def _list_events(session, args: dict, server_name: str) -> dict:
if not cursor:
break
routes = await asyncio.to_thread(_load_routes, server_name)
+ states = await asyncio.to_thread(_load_state)
subscribed = [{"name": r["mcp_event"]["name"], "arguments": r["mcp_event"]["arguments"],
- "deliver": r.get("deliver", "log"), "refreshBefore": r["mcp_event"].get("refresh_before")}
- for r in routes.values()]
+ "deliver": r.get("deliver", "log"), "refreshBefore": states.get(n, {}).get("refresh_before")}
+ for n, r in routes.items()]
return {"events": events, "subscriptions": subscribed}
@@ -240,20 +288,23 @@ async def _subscribe_event(session, args: dict, server_name: str) -> dict:
route_name = _route_name(server_name, args["name"], arguments)
existing = (await asyncio.to_thread(_load_routes, server_name)).get(route_name)
deliver_extra = {"chat_id": args["deliver_chat_id"]} if args.get("deliver_chat_id") else {}
+ # The route is the subscription's fixed contract: an admitted run's reply is checked against a digest
+ # of it, so nothing that changes while the subscription lives (grant, cursor) is stored here, and an
+ # unchanged resubscribe republishes identical bytes (created_at is kept).
route = {
"description": f"MCP event {args['name']} from {server_name}",
"secret": (existing or {}).get("secret") or _new_secret(),
"prompt": _event_prompt(server_name, args["name"], args.get("instructions") or ""),
"deliver": args.get("deliver") or "log", "profile": "default",
**({"deliver_extra": deliver_extra} if deliver_extra else {}),
- "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
- "mcp_event": {"server": server_name, "name": args["name"], "arguments": arguments,
- **({"cursor": existing["mcp_event"]["cursor"]}
- if existing and existing["mcp_event"].get("cursor") else {})}}
+ "created_at": (existing or {}).get("created_at") or time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+ "mcp_event": {"server": server_name, "name": args["name"], "arguments": arguments}}
+ if existing is None:
+ await asyncio.to_thread(drop_state, route_name) # a fresh subscription replays nothing stale
# The route must exist before subscribing: the server verifies the callback with a signed challenge.
await asyncio.to_thread(_write_route, route_name, route)
try:
- published = await _subscribe_route(session, route_name, route, url_base) or route
+ stored = await _subscribe_route(session, route_name, route, url_base)
except Exception as exc:
await asyncio.to_thread(_write_route, route_name, existing)
error = _mcp_error(exc)
@@ -261,7 +312,7 @@ async def _subscribe_event(session, args: dict, server_name: str) -> dict:
raise
return {"error": error}
return {"subscribed": {"name": args["name"], "arguments": arguments, "deliver": route["deliver"],
- "refreshBefore": published["mcp_event"].get("refresh_before")}}
+ "refreshBefore": (stored or {}).get("refresh_before")}}
async def _unsubscribe_event(session, args: dict, server_name: str) -> dict:
@@ -270,7 +321,8 @@ async def _unsubscribe_event(session, args: dict, server_name: str) -> dict:
route = (await asyncio.to_thread(_load_routes, server_name)).get(route_name)
if route is None:
return {"error": f"No subscription to '{args['name']}' with these arguments on '{server_name}'"}
- url = route["mcp_event"].get("url") or _callback_url(_callback_base() or "", route_name)
+ state = (await asyncio.to_thread(_load_state)).get(route_name, {})
+ url = state.get("url") or _callback_url(_callback_base() or "", route_name)
try:
await _send(session, "events/unsubscribe",
{"name": args["name"], "arguments": arguments, "delivery": {"url": url}})
diff --git a/website/docs/user-guide/features/mcp.md b/website/docs/user-guide/features/mcp.md
index 9bd24589101..72ee30b5afd 100644
--- a/website/docs/user-guide/features/mcp.md
+++ b/website/docs/user-guide/features/mcp.md
@@ -1031,7 +1031,9 @@ The server then gets three more utility tools:
- `unsubscribe_event`: cancel a subscription
Each subscription is stored as a dynamic webhook route (`hermes webhook list` shows it) with its own
-`whsec_` signing secret. Hermes answers the server's signed verification challenge, deduplicates
+`whsec_` signing secret; the grant and the replay cursor are kept beside it in
+`mcp_event_subscriptions.json`, so the route itself only changes when you resubscribe with different
+instructions or a different destination. Hermes answers the server's signed verification challenge, deduplicates
events by `eventId`, and refreshes the subscription before the lifetime the server granted runs out
while the MCP connection is up (and again whenever it reconnects). If the server ends a
subscription, Hermes removes the route and tells the `deliver` target. Events only arrive while