Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(events): Visibility-only admin WS handlers (Wave 12 D3)
trohitgaebfb353dad5
diff --git a/docs-internal/event_framework.md b/docs-internal/event_framework.mdindex fc95188f..af4d353b 100644--- a/docs-internal/event_framework.md+++ b/docs-internal/event_framework.md@@ -21,7 +21,7 @@ phase plan lives in `~/.claude/plans/properly-fix-the-tech-dreamy-tarjan.md`.| D1 — Shared `_retry_policies` + `NodeUserError` non-retryable on workflow callsites | ✅ shipped 2026-05-15 (11 tests) || D2 — Custom `event_dlq` SQLModel table | ❌ dropped 2026-05-15 — Temporal Event History + Visibility queries cover the ops-inspection use case; no separate table needed (matches the plan's "What Temporal eliminates" rationale) || D2b — Retire `event_waiter.py` Redis-Streams branch | ⏸ deferred — gated on `event_framework_enabled` flipping to default-on |-| D3 — Visibility admin WS handlers (`list_canary_listeners` / `list_canary_schedules` / `get_workflow_failure_history`) | ⏳ pending — reframed to Visibility-only after D2 drop |+| D3 — Visibility admin WS handlers (`list_canary_listeners` / `list_canary_schedules` / `get_workflow_failure_history`) | ✅ shipped 2026-05-15 (13 tests) || D4 — Drain `_LEGACY_RAW_DICT_BROADCASTS` | ⏸ blocked on B11 FE migration || D5 — Auto-gen `DEFAULT_TOOL_NAMES` from `ToolNode` ClassVars | ⏳ pending — large-scope refactor || C2 — Polling triggers as long-lived workflows | ⏳ pending |diff --git a/server/services/events/__init__.py b/server/services/events/__init__.pyindex d9b2f0d5..53dbd76c 100644--- a/server/services/events/__init__.py+++ b/server/services/events/__init__.py@@ -32,6 +32,17 @@ from .verifiers import (GitHubVerifier,)+# Wave 12 D3: publish the Visibility admin WS handlers into the central+# WS dispatcher on package import. Even though ``admin_handlers.py`` is+# framework code (not plugin code), routing it through the same+# ws_handler_registry keeps the router's dispatch surface uniform —+# the registry queries don't care whether the caller is plugin or+# framework.+from .admin_handlers import WS_HANDLERS as _ADMIN_WS_HANDLERS # noqa: E402+from services.ws_handler_registry import register_ws_handlers as _register_ws_handlers # noqa: E402++_register_ws_handlers(_ADMIN_WS_HANDLERS)+__all__ = ["BaseTriggerParams","DaemonEventSource",diff --git a/server/services/events/admin_handlers.py b/server/services/events/admin_handlers.pynew file mode 100644index 00000000..7c51b0de--- /dev/null+++ b/server/services/events/admin_handlers.py@@ -0,0 +1,373 @@+"""Wave 12 D3: Visibility admin WS handlers.++Thin wrappers around Temporal's Visibility API so the MachinaOs admin+UI can inspect canary listener / schedule state without leaving the+app. No new DB tables, no custom persistence — Temporal Event History++ Visibility queries ARE the registry (see+``docs-internal/event_framework.md`` § "Failure inspection").++Three WS message types:++ - ``list_canary_listeners`` →+ ``client.list_workflows(query="EventWorkflowId='X' AND WorkflowType IN (...)")``+ Returns running TriggerListenerWorkflow + PollingTriggerWorkflow+ instances for a deployment.++ - ``list_canary_schedules`` →+ ``client.list_schedules(query="EventWorkflowId='X' AND EventTriggerKind='cron'")``+ Returns Temporal Schedules created by the deployment's cron+ triggers (Wave 12 C3).++ - ``get_workflow_failure_history`` →+ ``client.get_workflow_handle(...).fetch_history()`` filtered to+ ActivityTaskFailed events for ops inspection of failed runs.++All three are read-only — no replay, no mutation. Cancel + delete are+already covered by ``DeploymentManager.cancel()`` (the production+cancel path). These handlers exist purely for visibility.+"""++from __future__ import annotations++from typing import Any, Dict, List, Optional++from fastapi import WebSocket++from core.logging import get_logger+from services.plugin.ws import ws_response++logger = get_logger(__name__)+++# Mirrors the workflow-type names declared in+# services.deployment.manager._LISTENER_WORKFLOW_TYPES. Re-declared+# here to keep this module's dependency graph minimal — admin code+# shouldn't pull in DeploymentManager.+_LISTENER_WORKFLOW_TYPES: tuple[str, ...] = (+ "TriggerListenerWorkflow",+ "PollingTriggerWorkflow",+)+++@ws_response+async def handle_list_canary_listeners(+ data: Dict[str, Any], websocket: WebSocket,+) -> Dict[str, Any]:+ """List Temporal-durable canary listener workflows.++ Request shape::++ {"type": "list_canary_listeners", "workflow_id": "<deployment-id>"}++ Response::++ {+ "listeners": [+ {"id": "trigger-listener-<wf>-<node>",+ "type": "TriggerListenerWorkflow",+ "status": "RUNNING",+ "start_time": "<ISO>",+ "trigger_node_id": "<node>",+ "event_type": "<com.machinaos. ... >"},+ ...+ ],+ "count": <int>,+ }+ """+ from core.container import container++ workflow_id = data.get("workflow_id")+ if not workflow_id:+ return {"success": False, "error": "workflow_id is required"}++ wrapper = container.temporal_client()+ if wrapper is None or wrapper.client is None:+ return {"success": True, "listeners": [], "count": 0,+ "note": "Temporal not connected"}++ wf_types_in = ", ".join(f"'{t}'" for t in _LISTENER_WORKFLOW_TYPES)+ query = (+ f"EventWorkflowId='{workflow_id}' "+ f"AND WorkflowType IN ({wf_types_in}) "+ f"AND ExecutionStatus='Running'"+ )++ listeners: List[Dict[str, Any]] = []+ try:+ async for wf in wrapper.client.list_workflows(query=query):+ listeners.append(_describe_workflow(wf))+ except Exception as exc: # noqa: BLE001+ logger.warning(+ f"list_canary_listeners Visibility query failed: {exc} "+ f"(query={query!r})",+ workflow_id=workflow_id,+ )+ return {"success": False, "error": str(exc), "query": query}++ return {"success": True, "listeners": listeners, "count": len(listeners)}+++@ws_response+async def handle_list_canary_schedules(+ data: Dict[str, Any], websocket: WebSocket,+) -> Dict[str, Any]:+ """List Temporal Schedules created by this deployment's cron triggers.++ Request shape::++ {"type": "list_canary_schedules", "workflow_id": "<deployment-id>"}++ Response::++ {+ "schedules": [+ {"id": "cron-schedule-<wf>-<node>",+ "trigger_node_id": "<node>"},+ ...+ ],+ "count": <int>,+ }+ """+ from core.container import container++ workflow_id = data.get("workflow_id")+ if not workflow_id:+ return {"success": False, "error": "workflow_id is required"}++ wrapper = container.temporal_client()+ if wrapper is None or wrapper.client is None:+ return {"success": True, "schedules": [], "count": 0,+ "note": "Temporal not connected"}++ query = (+ f"EventWorkflowId='{workflow_id}' "+ f"AND EventTriggerKind='cron'"+ )++ schedules: List[Dict[str, Any]] = []+ try:+ async for desc in wrapper.client.list_schedules(query=query):+ schedules.append(_describe_schedule(desc))+ except Exception as exc: # noqa: BLE001+ logger.warning(+ f"list_canary_schedules Visibility query failed: {exc} "+ f"(query={query!r})",+ workflow_id=workflow_id,+ )+ return {"success": False, "error": str(exc), "query": query}++ return {"success": True, "schedules": schedules, "count": len(schedules)}+++@ws_response+async def handle_get_workflow_failure_history(+ data: Dict[str, Any], websocket: WebSocket,+) -> Dict[str, Any]:+ """Return ActivityTaskFailed events from a workflow's Event History.++ Request shape::++ {"type": "get_workflow_failure_history",+ "workflow_id": "<temporal-workflow-id>",+ "run_id": "<optional-run-id>"}++ Response::++ {+ "workflow_id": "<id>",+ "failures": [+ {"event_id": <int>,+ "event_type": "ActivityTaskFailed",+ "event_time": "<ISO>",+ "activity_id": "<id>",+ "activity_type": "<type>",+ "message": "<failure detail>"},+ ...+ ],+ "count": <int>,+ }++ Ops-only — no replay action available here. For more detail use+ Temporal Web UI at http://localhost:8233 or the temporal CLI:++ temporal workflow show --workflow-id <id> --output json+ """+ from core.container import container++ target_workflow_id = data.get("workflow_id")+ if not target_workflow_id:+ return {"success": False, "error": "workflow_id is required"}++ wrapper = container.temporal_client()+ if wrapper is None or wrapper.client is None:+ return {"success": False, "error": "Temporal not connected"}++ run_id: Optional[str] = data.get("run_id")+ try:+ handle = wrapper.client.get_workflow_handle(+ target_workflow_id, run_id=run_id,+ )+ except Exception as exc: # noqa: BLE001+ return {"success": False, "error": f"workflow lookup failed: {exc}"}++ failures: List[Dict[str, Any]] = []+ try:+ async for event in handle.fetch_history_events():+ failure = _maybe_describe_failure(event)+ if failure is not None:+ failures.append(failure)+ except Exception as exc: # noqa: BLE001+ logger.warning(+ f"get_workflow_failure_history history fetch failed: {exc}",+ workflow_id=target_workflow_id,+ )+ return {"success": False, "error": str(exc)}++ return {+ "success": True,+ "workflow_id": target_workflow_id,+ "failures": failures,+ "count": len(failures),+ }+++# ---------------------------------------------------------------------------+# Internal helpers+# ---------------------------------------------------------------------------+++def _describe_workflow(wf_execution: Any) -> Dict[str, Any]:+ """Best-effort dict view of a WorkflowExecution from the Visibility list.++ Search-attribute access on ``WorkflowExecutionDescription`` varies+ by SDK version (some expose ``.search_attributes`` as a plain dict,+ others as TypedSearchAttributes). The helper tries the typed form+ first + falls back to plain dict lookup.+ """+ sa_view = _search_attributes_dict(getattr(wf_execution, "search_attributes", None))+ return {+ "id": getattr(wf_execution, "id", None),+ "run_id": getattr(wf_execution, "run_id", None),+ "type": _value_or_name(getattr(wf_execution, "workflow_type", None)),+ "status": _value_or_name(getattr(wf_execution, "status", None)),+ "start_time": _iso(getattr(wf_execution, "start_time", None)),+ "close_time": _iso(getattr(wf_execution, "close_time", None)),+ "trigger_node_id": sa_view.get("TriggerNodeId"),+ "event_type": sa_view.get("EventType"),+ "event_trigger_kind": sa_view.get("EventTriggerKind"),+ }+++def _describe_schedule(schedule_desc: Any) -> Dict[str, Any]:+ """Best-effort dict view of a ScheduleListDescription."""+ sa_view = _search_attributes_dict(getattr(schedule_desc, "search_attributes", None))+ return {+ "id": getattr(schedule_desc, "id", None),+ "trigger_node_id": sa_view.get("TriggerNodeId"),+ "event_trigger_kind": sa_view.get("EventTriggerKind"),+ }+++def _maybe_describe_failure(event: Any) -> Optional[Dict[str, Any]]:+ """Return the ActivityTaskFailed shape, or None when ``event`` isn't a failure.++ Temporal's HistoryEvent is a protobuf message; the failure detail+ lives at ``event.activity_task_failed_event_attributes.failure``.+ The protobuf API can change between SDK versions so we guard every+ attribute access.+ """+ event_type_attr = getattr(event, "event_type", None)+ event_type_name = _value_or_name(event_type_attr) or ""+ if "ActivityTaskFailed" not in str(event_type_name):+ return None++ attrs = getattr(event, "activity_task_failed_event_attributes", None)+ if attrs is None:+ return None++ failure = getattr(attrs, "failure", None)+ message = getattr(failure, "message", None) if failure is not None else None++ activity_id = None+ activity_type = None+ # Activity identification lives on the scheduled-event lookup but+ # the protobuf surface keeps it on the failure event too.+ activity_id = getattr(attrs, "activity_id", None)+ activity_type = getattr(attrs, "activity_type", None)+ if activity_type is not None:+ activity_type = getattr(activity_type, "name", str(activity_type))++ return {+ "event_id": getattr(event, "event_id", None),+ "event_type": "ActivityTaskFailed",+ "event_time": _iso(getattr(event, "event_time", None)),+ "activity_id": activity_id,+ "activity_type": activity_type,+ "message": message,+ }+++def _search_attributes_dict(raw: Any) -> Dict[str, Any]:+ """Coerce TypedSearchAttributes / plain-dict / None into a flat dict view."""+ if raw is None:+ return {}+ # TypedSearchAttributes is iterable over SearchAttributePair-like+ # objects with ``key.name`` + ``value`` fields.+ if hasattr(raw, "__iter__") and not isinstance(raw, dict):+ result: Dict[str, Any] = {}+ for pair in raw:+ key = getattr(getattr(pair, "key", None), "name", None)+ if key is None:+ continue+ result[key] = getattr(pair, "value", None)+ if result:+ return result+ if isinstance(raw, dict):+ # Untyped search-attribute map from older SDK: name -> [value, ...]+ return {+ k: (v[0] if isinstance(v, list) and v else v)+ for k, v in raw.items()+ }+ return {}+++def _value_or_name(obj: Any) -> Any:+ """Return ``.name`` for enum-like objects, otherwise the value itself."""+ if obj is None:+ return None+ name = getattr(obj, "name", None)+ if name is not None:+ return name+ return obj+++def _iso(ts: Any) -> Optional[str]:+ """Format a datetime/Timestamp protobuf as ISO-8601."""+ if ts is None:+ return None+ # datetime objects: have .isoformat()+ isofmt = getattr(ts, "isoformat", None)+ if callable(isofmt):+ try:+ return isofmt()+ except Exception: # noqa: BLE001+ pass+ # Fallback: stringify.+ return str(ts)+++# Wire-key → handler dispatch table the WS router picks up via+# services.ws_handler_registry.register_ws_handlers.+WS_HANDLERS: Dict[str, Any] = {+ "list_canary_listeners": handle_list_canary_listeners,+ "list_canary_schedules": handle_list_canary_schedules,+ "get_workflow_failure_history": handle_get_workflow_failure_history,+}+++__all__ = [+ "WS_HANDLERS",+ "handle_list_canary_listeners",+ "handle_list_canary_schedules",+ "handle_get_workflow_failure_history",+]diff --git a/server/tests/test_admin_handlers.py b/server/tests/test_admin_handlers.pynew file mode 100644index 00000000..f3334847--- /dev/null+++ b/server/tests/test_admin_handlers.py@@ -0,0 +1,328 @@+"""Wave 12 D3: Visibility admin WS handler tests.++Unit-style — no live Temporal cluster. The container's ``temporal_client``+is monkeypatched per test. Three handlers, each tested against four+contracts:++1. ``workflow_id`` required (input validation).+2. Visibility query shape (correct filter clauses).+3. Fail-soft on Visibility errors (don't crash the admin UI).+4. ``temporal_client`` not connected → graceful empty result, not error.++Plus helper tests for the search-attribute coercion + failure event+extraction.+"""++from __future__ import annotations++import sys+import types+from typing import Any, Dict, List+from unittest.mock import AsyncMock, MagicMock++import pytest+++if "machina" not in sys.modules:+ _machina = types.ModuleType("machina")+ _machina.__path__ = []+ sys.modules["machina"] = _machina+ _machina_tcp = types.ModuleType("machina.tcp")+ _machina_tcp.probe_tcp_port = MagicMock(return_value=False)+ sys.modules["machina.tcp"] = _machina_tcp+++# ---------------------------------------------------------------------------+# list_canary_listeners+# ---------------------------------------------------------------------------+++class TestListCanaryListeners:+ @pytest.mark.asyncio+ async def test_workflow_id_required(self):+ from services.events.admin_handlers import handle_list_canary_listeners++ result = await handle_list_canary_listeners({}, MagicMock())+ assert result["success"] is False+ assert "workflow_id" in result["error"]++ @pytest.mark.asyncio+ async def test_temporal_not_connected_returns_empty(self, monkeypatch):+ from services.events.admin_handlers import handle_list_canary_listeners++ wrapper = MagicMock()+ wrapper.client = None+ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ result = await handle_list_canary_listeners(+ {"workflow_id": "wf-1"}, MagicMock(),+ )+ assert result["success"] is True+ assert result["listeners"] == []+ assert result["count"] == 0++ @pytest.mark.asyncio+ async def test_visibility_query_filters_both_listener_types(self, monkeypatch):+ from services.events.admin_handlers import handle_list_canary_listeners++ recorded_queries: List[str] = []++ async def fake_list(query):+ recorded_queries.append(query)+ yield MagicMock(+ id="trigger-listener-wf-1-wh-1",+ run_id="run-1",+ workflow_type="TriggerListenerWorkflow",+ status=MagicMock(name="Status", spec_set=["name"]),+ start_time=None,+ close_time=None,+ search_attributes=None,+ )++ client = MagicMock()+ client.list_workflows = fake_list++ wrapper = MagicMock()+ wrapper.client = client++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ result = await handle_list_canary_listeners(+ {"workflow_id": "wf-1"}, MagicMock(),+ )+ assert result["success"] is True+ assert result["count"] == 1++ q = recorded_queries[0]+ assert "EventWorkflowId='wf-1'" in q+ assert "'TriggerListenerWorkflow'" in q+ assert "'PollingTriggerWorkflow'" in q+ assert "WorkflowType IN" in q+ assert "ExecutionStatus='Running'" in q++ @pytest.mark.asyncio+ async def test_failsoft_on_visibility_error(self, monkeypatch):+ from services.events.admin_handlers import handle_list_canary_listeners++ async def fake_list_raises(query):+ if False:+ yield None+ raise RuntimeError("Visibility unavailable")++ client = MagicMock()+ client.list_workflows = fake_list_raises++ wrapper = MagicMock()+ wrapper.client = client++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ result = await handle_list_canary_listeners(+ {"workflow_id": "wf-1"}, MagicMock(),+ )+ assert result["success"] is False+ assert "Visibility unavailable" in result["error"]+++# ---------------------------------------------------------------------------+# list_canary_schedules+# ---------------------------------------------------------------------------+++class TestListCanarySchedules:+ @pytest.mark.asyncio+ async def test_workflow_id_required(self):+ from services.events.admin_handlers import handle_list_canary_schedules++ result = await handle_list_canary_schedules({}, MagicMock())+ assert result["success"] is False++ @pytest.mark.asyncio+ async def test_visibility_query_filters_cron_kind(self, monkeypatch):+ from services.events.admin_handlers import handle_list_canary_schedules++ recorded_queries: List[str] = []++ async def fake_list(query):+ recorded_queries.append(query)+ for sched_id in ["cron-schedule-wf-1-a", "cron-schedule-wf-1-b"]:+ yield MagicMock(id=sched_id, search_attributes=None)++ client = MagicMock()+ client.list_schedules = fake_list++ wrapper = MagicMock()+ wrapper.client = client++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ result = await handle_list_canary_schedules(+ {"workflow_id": "wf-1"}, MagicMock(),+ )+ assert result["success"] is True+ assert result["count"] == 2++ q = recorded_queries[0]+ assert "EventWorkflowId='wf-1'" in q+ assert "EventTriggerKind='cron'" in q+++# ---------------------------------------------------------------------------+# get_workflow_failure_history+# ---------------------------------------------------------------------------+++class TestGetWorkflowFailureHistory:+ @pytest.mark.asyncio+ async def test_workflow_id_required(self):+ from services.events.admin_handlers import handle_get_workflow_failure_history++ result = await handle_get_workflow_failure_history({}, MagicMock())+ assert result["success"] is False++ @pytest.mark.asyncio+ async def test_temporal_not_connected_returns_error(self, monkeypatch):+ from services.events.admin_handlers import handle_get_workflow_failure_history++ wrapper = MagicMock()+ wrapper.client = None++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ result = await handle_get_workflow_failure_history(+ {"workflow_id": "wf-1"}, MagicMock(),+ )+ assert result["success"] is False++ @pytest.mark.asyncio+ async def test_extracts_only_failure_events(self, monkeypatch):+ from services.events.admin_handlers import handle_get_workflow_failure_history++ # 3 events: WorkflowStarted (no), ActivityTaskFailed (yes),+ # ActivityTaskCompleted (no).+ events = [+ MagicMock(+ event_type=MagicMock(name="WorkflowExecutionStarted", spec=["name"]),+ ),+ MagicMock(+ event_type=MagicMock(name="ActivityTaskFailed", spec=["name"]),+ event_id=42,+ event_time=None,+ activity_task_failed_event_attributes=MagicMock(+ failure=MagicMock(message="connection refused"),+ activity_id="poll-1",+ activity_type=MagicMock(name="poll.googleGmailReceive.v1"),+ ),+ ),+ MagicMock(+ event_type=MagicMock(name="ActivityTaskCompleted", spec=["name"]),+ ),+ ]+ # Need to fix the .name on the type enum since MagicMock spec+ # doesn't auto-stringify in our `_value_or_name` helper.+ for ev, type_name in zip(+ events,+ ["WorkflowExecutionStarted", "ActivityTaskFailed", "ActivityTaskCompleted"],+ ):+ ev.event_type.name = type_name++ async def fake_fetch():+ for e in events:+ yield e++ handle = MagicMock()+ handle.fetch_history_events = fake_fetch++ client = MagicMock()+ client.get_workflow_handle = MagicMock(return_value=handle)++ wrapper = MagicMock()+ wrapper.client = client++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ result = await handle_get_workflow_failure_history(+ {"workflow_id": "wf-temporal-1"}, MagicMock(),+ )+ assert result["success"] is True+ assert result["count"] == 1+ failure = result["failures"][0]+ assert failure["event_type"] == "ActivityTaskFailed"+ assert failure["event_id"] == 42+ assert failure["message"] == "connection refused"+ assert failure["activity_id"] == "poll-1"+++# ---------------------------------------------------------------------------+# Search-attribute coercion helper+# ---------------------------------------------------------------------------+++class TestSearchAttributesDict:+ def test_none_returns_empty_dict(self):+ from services.events.admin_handlers import _search_attributes_dict++ assert _search_attributes_dict(None) == {}++ def test_plain_dict_with_list_values(self):+ """Older Temporal SDK: untyped attributes are dict[str, List]."""+ from services.events.admin_handlers import _search_attributes_dict++ raw = {"EventWorkflowId": ["wf-1"], "TriggerNodeId": ["node-a"]}+ view = _search_attributes_dict(raw)+ assert view == {"EventWorkflowId": "wf-1", "TriggerNodeId": "node-a"}++ def test_typed_search_attributes_iterable(self):+ """Newer SDK: TypedSearchAttributes is iterable over pairs."""+ from services.events.admin_handlers import _search_attributes_dict+ from temporalio.common import (+ SearchAttributeKey,+ SearchAttributePair,+ TypedSearchAttributes,+ )++ typed = TypedSearchAttributes([+ SearchAttributePair(+ SearchAttributeKey.for_keyword("EventWorkflowId"), "wf-1",+ ),+ SearchAttributePair(+ SearchAttributeKey.for_keyword("EventTriggerKind"), "cron",+ ),+ ])+ view = _search_attributes_dict(typed)+ assert view == {"EventWorkflowId": "wf-1", "EventTriggerKind": "cron"}+++# ---------------------------------------------------------------------------+# Registration smoke+# ---------------------------------------------------------------------------+++class TestWsHandlerRegistration:+ """Importing services.events registers the three admin handlers+ via ws_handler_registry — the WS router dispatches by message type+ without per-handler imports."""++ def test_all_three_handlers_registered(self):+ # Force re-import path-of-least-resistance: just import the+ # package and confirm the registry has the wire keys.+ import services.events # noqa: F401 — side-effect import+ from services.ws_handler_registry import get_ws_handlers++ handlers = get_ws_handlers()+ for wire_key in (+ "list_canary_listeners",+ "list_canary_schedules",+ "get_workflow_failure_history",+ ):+ assert wire_key in handlers, (+ f"WS handler {wire_key!r} not registered. Importing "+ "services.events should call register_ws_handlers with "+ "the admin_handlers.WS_HANDLERS dict."+ )