Skip to content

Self-improving AI Employees for everyone

69.8 KB
Raw Blame
"""WebSocket router for real-time bidirectional communication.

Handles WebSocket connections from frontend clients for ALL operations:
- Node parameters (get, save, delete)
- Node execution
- AI execution and model fetching
- API key validation and storage
- Android device operations
- Google Maps key validation
- Status broadcasts
"""

import time
import asyncio
import uuid
import weakref
from typing import Dict, Any, Callable, Awaitable, Optional, Set

from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from services.status_broadcaster import get_status_broadcaster
from core.container import container
from core.logging import get_logger


def get_auth_service():
    """Get auth service from DI container."""
    return container.auth_service()


logger = get_logger(__name__)

router = APIRouter(tags=["websocket"])

# =============================================================================
# Concurrent Send Protection
# =============================================================================
# Use WeakKeyDictionary to auto-cleanup when WebSocket is garbage collected
_send_locks: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()

# Track running handler tasks per WebSocket for cleanup on disconnect
_handler_tasks: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()


async def _safe_send(websocket: WebSocket, data: dict):
    """Thread-safe WebSocket send with lock to prevent concurrent writes."""
    # Guard against sending on closed/disconnected WebSocket
    if websocket.client_state.name != "CONNECTED":
        return
    if websocket not in _send_locks:
        _send_locks[websocket] = asyncio.Lock()
    async with _send_locks[websocket]:
        try:
            await websocket.send_json(data)
        except Exception as e:
            logger.debug("[WebSocket] Send skipped (connection closed): %s", e)


# How long a disconnecting socket waits for its in-flight handlers before
# cancelling them. The connect-time init burst is a handful of short DB
# reads; a client that reconnects ~100 ms later (React Strict Mode double
# mount, a health probe) used to cancel those mid-statement, which
# SQLAlchemy reported as "Exception during reset or similar" with a
# traceback per handler. Long handlers (execute_node parked on an event
# waiter) are still cancelled once the grace expires.
_HANDLER_DRAIN_GRACE_SECONDS = 1.0


async def _drain_handler_tasks(handler_tasks: Set[asyncio.Task], grace: float = _HANDLER_DRAIN_GRACE_SECONDS) -> None:
    """Let in-flight handler tasks finish for ``grace`` seconds, then cancel the rest."""
    tasks = [task for task in handler_tasks if not task.done()]
    if not tasks:
        return
    _done, pending = await asyncio.wait(tasks, timeout=grace)
    for task in pending:
        task.cancel()
    await asyncio.gather(*tasks, return_exceptions=True)


# Type for message handlers
MessageHandler = Callable[[Dict[str, Any], WebSocket], Awaitable[Dict[str, Any]]]


def ws_handler(*required_fields: str):
    """Simple decorator for WebSocket handlers. Validates required fields and wraps errors."""
    import functools

    def decorator(func: MessageHandler) -> MessageHandler:
        @functools.wraps(func)
        async def wrapper(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
            for field in required_fields:
                if not data.get(field):
                    return {"success": False, "error": f"{field} required"}
            try:
                result = await func(data, websocket)
                if "success" not in result:
                    result = {"success": True, **result}
                return result
            except Exception as e:
                logger.error(f"Handler error: {e}", exc_info=True)
                return {"success": False, "error": str(e)}

        return wrapper

    return decorator


# ============================================================================
# Message Handlers
# ============================================================================


async def handle_ping(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Handle ping request."""
    return {"type": "pong", "timestamp": time.time()}


async def handle_get_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get current full status."""
    broadcaster = get_status_broadcaster()
    return {"type": "full_status", "data": broadcaster.get_status()}


# handle_get_android_status moved to nodes/android/_handlers.py (Wave 13.9).


async def handle_get_node_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get specific node status."""
    broadcaster = get_status_broadcaster()
    node_id = data.get("node_id")
    if node_id:
        status = broadcaster.get_node_status(node_id)
        return {"type": "node_status", "node_id": node_id, "data": status}
    return {"type": "error", "message": "node_id required"}


async def handle_get_variable(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get variable value."""
    broadcaster = get_status_broadcaster()
    name = data.get("name")
    if name:
        value = broadcaster.get_variable(name)
        return {"type": "variable_update", "name": name, "value": value}
    return {"type": "error", "message": "name required"}


# ============================================================================
# Node Parameters Handlers
# ============================================================================


@ws_handler("node_id")
async def handle_get_node_parameters(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get parameters for a specific node."""
    database = container.database()
    node_id = data["node_id"]
    parameters = await database.get_node_parameters(node_id)
    logger.debug(f"[GET_PARAMS] Node ID: {node_id}")
    logger.debug(
        "[GET_PARAMS] Parameter keys: %s",
        sorted(parameters) if isinstance(parameters, dict) else [],
    )
    logger.debug(f"[GET_PARAMS] Code length: {len(parameters.get('code', '')) if parameters and 'code' in parameters else 'no code field'}")
    return {"node_id": node_id, "parameters": parameters or {}, "version": 1, "timestamp": time.time()}


@ws_handler()
async def handle_get_all_node_parameters(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get parameters for multiple nodes."""
    database = container.database()
    node_ids = data.get("node_ids", [])
    if not isinstance(node_ids, list) or len(node_ids) > 1000:
        raise ValueError("node_ids must be a list of at most 1000 ids")

    export_mode = data.get("purpose") == "export"
    if export_mode:
        workflow_id = str(data.get("workflow_id") or "")
        if not workflow_id:
            raise ValueError("workflow_id is required for export")
        workflow = await database.get_workflow(workflow_id)
        if workflow is None:
            raise ValueError("Workflow not found")
        graph = workflow.data if isinstance(workflow.data, dict) else {}
        authenticated_owner = str(
            getattr(getattr(websocket, "state", None), "user_id", None)
            or "owner"
        )
        stored_owner = str(graph.get("owner_id") or "")
        if stored_owner and stored_owner != authenticated_owner:
            raise ValueError("Workflow access denied")
        workflow_node_ids = {
            str(node.get("id"))
            for node in graph.get("nodes", [])
            if isinstance(node, dict) and node.get("id")
        }
        requested_ids = {str(node_id) for node_id in node_ids}
        if not requested_ids.issubset(workflow_node_ids):
            raise ValueError("Export contains nodes outside the workflow")
        from services.workflow_sanitizer import sanitize_runtime_payload

    result = {}
    for node_id in node_ids:
        parameters = await database.get_node_parameters(node_id)
        if parameters:
            result[node_id] = {
                "parameters": (
                    sanitize_runtime_payload(parameters)
                    if export_mode
                    else parameters
                ),
                "version": 1,
            }
    return {"parameters": result, "timestamp": time.time()}


@ws_handler("node_id")
async def handle_save_node_parameters(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Save node parameters and broadcast to all clients."""
    database = container.database()
    broadcaster = get_status_broadcaster()
    node_id, parameters = data["node_id"], data.get("parameters", {})
    from services.agent_runtime_policy import preserve_runtime_version
    parameters = preserve_runtime_version(await database.get_node_parameters(node_id) or {}, parameters)
    from services.browser_resources import validate_browser_parameters
    await validate_browser_parameters(database, node_id, parameters)

    logger.debug(
        f"[SAVE_PARAMS] Node ID: {node_id}, has_code: {'code' in parameters}, code_len: {len(parameters.get('code', '')) if 'code' in parameters else 0}"
    )
    await database.save_node_parameters(node_id, parameters)
    # CloudEvents v1.0 envelope (RFC §6.4) — type is
    # ``com.opencompany.node.parameters.updated``; ``source_hint="user"``
    # because this handler fires from the parameter-panel save flow.
    await broadcaster.broadcast_node_parameters_updated(
        node_id,
        parameters=parameters,
        source_hint="user",
    )
    return {"node_id": node_id, "parameters": parameters, "version": 1, "timestamp": time.time()}


@ws_handler("node_id")
async def handle_delete_node_parameters(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Delete node parameters."""
    database = container.database()
    await database.delete_node_parameters(data["node_id"])
    return {"node_id": data["node_id"]}


# ============================================================================
# Tool Schema Handlers (Source of truth for tool node configurations)
# ============================================================================


@ws_handler("node_id")
async def handle_get_tool_schema(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get tool schema for a node."""
    database = container.database()
    schema = await database.get_tool_schema(data["node_id"])
    return {"node_id": data["node_id"], "schema": schema}


@ws_handler("node_id", "tool_name", "tool_description", "schema_config")
async def handle_save_tool_schema(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Save tool schema for a node. Used by Android Toolkit to update connected service schemas."""
    database = container.database()
    broadcaster = get_status_broadcaster()

    node_id = data["node_id"]
    tool_name = data["tool_name"]
    tool_description = data["tool_description"]
    schema_config = data["schema_config"]
    connected_services = data.get("connected_services")

    success = await database.save_tool_schema(
        node_id=node_id,
        tool_name=tool_name,
        tool_description=tool_description,
        schema_config=schema_config,
        connected_services=connected_services,
    )

    if success:
        # Broadcast schema update to all clients
        await broadcaster.broadcast({"type": "tool_schema_updated", "node_id": node_id, "tool_name": tool_name, "timestamp": time.time()})

    return {"node_id": node_id, "tool_name": tool_name, "saved": success}


@ws_handler("node_id")
async def handle_delete_tool_schema(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Delete tool schema for a node."""
    database = container.database()
    await database.delete_tool_schema(data["node_id"])
    return {"node_id": data["node_id"]}


async def handle_get_all_tool_schemas(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get all tool schemas."""
    database = container.database()
    schemas = await database.get_all_tool_schemas()
    return {"success": True, "schemas": schemas}


# ============================================================================
# Node Output Schemas (n8n-style: backend as single source of truth for
# the drag-drop variable panel's "before first run" fallback shape).
# See docs-internal/ARCHIVE/schema_source_of_truth_rfc.md.
# ============================================================================


@ws_handler("node_type")
async def handle_get_node_output_schema(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Return the JSON Schema for a node type's runtime output, or
    ``{schema: null}`` when no schema is declared. Frontend caches the
    result per node type in-memory (mirrors n8n's schemaPreview.store)."""
    from services.node_output_schemas import get_node_output_schema

    schema = get_node_output_schema(data["node_type"])
    return {"node_type": data["node_type"], "schema": schema}


@ws_handler("node_type")
async def handle_get_node_spec(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Return the unified NodeSpec (input schema + output schema +
    display metadata) for a node type, or ``{spec: null}`` when the
    type is unknown. Wave 6 Phase 2 WS mirror of the REST endpoint
    GET /api/schemas/nodes/{type}/spec.json."""
    from services.node_spec import get_node_spec

    spec = get_node_spec(data["node_type"])
    return {"node_type": data["node_type"], "spec": spec}


@ws_handler()
async def handle_list_node_specs(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Return the sorted list of node types that have a NodeSpec, plus
    a content-hash revision the editor uses to invalidate its persisted
    spec cache when the backend catalogue changes between deploys."""
    from services.node_spec import list_node_types_with_spec, node_spec_revision

    return {
        "node_types": list_node_types_with_spec(),
        "revision": node_spec_revision(),
    }


@ws_handler("method")
async def handle_load_options(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Wave 6 Phase 4: unified loadOptionsMethod dispatcher.

    Replaces the per-method WS handlers (whatsapp_groups, whatsapp_newsletters,
    etc.) with a single registry-driven endpoint. ``loadOptionsMethod`` strings
    declared via Pydantic ``Field(json_schema_extra={"loadOptionsMethod": "..."})``
    on the backend NodeSpec are resolved here.

    Body: ``{"method": "...", "params": {...}}``
    Response: ``{"options": [{"value": ..., "label": ...}]}``

    The caller's identity comes from the socket (an empty payload goes to
    ``execution_principal`` so a client-sent ``user_id`` is never trusted).
    """
    from services.ws_handler_registry import dispatch_load_options

    options = await dispatch_load_options(
        data["method"], data.get("params", {}), principal=execution_principal({}, websocket)
    )
    return {"method": data["method"], "options": options}


@ws_handler()
async def handle_list_load_options_methods(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Return registered loadOptionsMethod names. Editor uses this to
    know which dynamic-option loaders are wired."""
    from services.ws_handler_registry import list_load_options_methods

    return {"methods": list_load_options_methods()}


@ws_handler()
async def handle_get_node_groups(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Wave 6 Phase 5: {group_name: [node_type, ...]} index derived from
    every NodeSpec's ``group`` array. Replaces the 34 hand-rolled
    ``*_NODE_TYPES`` arrays scattered across the frontend."""
    from services.node_spec import list_node_groups

    return {"groups": list_node_groups()}


# ============================================================================
# Credential Registry Handler (Nango-style bulk fetch for 20 -> 5000 providers)
# ============================================================================


@ws_handler()
async def handle_get_credential_catalogue(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Return the full credential provider catalogue with live stored-key status.

    Response shape: {providers, categories, consumer_categories, version}.
    Each provider includes a `stored: boolean` field indicating whether a
    key/token is present in the encrypted credentials database. The frontend
    renders this directly — no client-side credential checks needed. Cards a
    plugin builds from what was saved (`Credential.catalogue_entries`, one
    per custom MCP connector) follow the static providers.
    """
    from services.credential_registry import get_credential_registry, provider_connection_state

    registry = get_credential_registry()
    auth_service = container.auth_service()
    principal = execution_principal({}, websocket)
    since = data.get("since")
    # Includes live pairing state, which changes without a credential
    # mutation (see CredentialRegistry.get_live_version).
    version = registry.get_live_version() + "+" + await auth_service.credential_source_version(principal)
    if since and since == version:
        return {"unchanged": True, "version": version}

    catalogue = registry.get_catalogue()
    catalogue["version"] = version

    # Enrich each provider with live credential state from AuthService
    # (``stored`` / ``connected`` / ``account_label`` plus any plugin
    # extras). Credential state stays a backend concern: the frontend
    # renders these flags and never checks key existence itself. The rules
    # live in services.credential_registry.provider_connection_state, which
    # the Normal-mode employee summaries share.
    for provider in catalogue.get("providers", []):
        provider.update(await provider_connection_state(provider, auth_service, principal=principal))

    # Cards a plugin builds from what was saved (one per custom connector),
    # each with its own state.
    from services.plugin.credential import CREDENTIAL_REGISTRY

    scope = {"principal": principal} if principal is not None else {}
    for cred_cls in list(CREDENTIAL_REGISTRY.values()):
        catalogue["providers"].extend(await cred_cls.catalogue_entries(**scope))

    return catalogue


# ============================================================================
# Node Execution Handlers
# ============================================================================


@ws_handler("node_id", "node_type")
async def handle_execute_node(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Execute a workflow node with per-workflow status scoping (n8n pattern).

    Generates an `execution_id` correlation token (uuid4 hex) and propagates
    it through every broadcast for this run plus the response payload.
    Frontend uses it for idempotent dedup -- two executions with identical
    output payloads were previously collapsing because the dedup key was
    JSON.stringify(outputs). This is the standard request-id / trace-id
    pattern (OpenTelemetry, AWS X-Ray, etc.).
    """
    workflow_service = container.workflow_service()
    broadcaster = get_status_broadcaster()
    node_id, node_type = data["node_id"], data["node_type"]
    workflow_id = data.get("workflow_id")  # Per-workflow isolation
    execution_id = str(data.get("execution_id") or uuid.uuid4().hex)
    invocation_extras = {
        key: data[key]
        for key in (
            "auto_rebind_tools",
            "invoking_agent_node_id",
            "agent_iteration",
            "tool_call_index",
            "tool_call_id",
        )
        if key in data
    }
    # A chat run's scope links what a node produces to the owner's chat, so
    # only the worker relaying MachinaWorkflow's context may set it.
    socket_scope = getattr(websocket, "scope", None) or {}
    if isinstance(socket_scope, dict) and socket_scope.get("path") == "/ws/internal":
        if data.get("pi_harness_version") == 1:
            from services.temporal.pi_inline_delegation import forward_inline_scope
            invocation_extras.update(forward_inline_scope(data))
        else:
            invocation_extras["coding_native_version"] = 1
    if "run_scope" in data and isinstance(socket_scope, dict) and socket_scope.get("path") == "/ws/internal":
        invocation_extras["run_scope"] = data["run_scope"]
    user_id = execution_principal(data, websocket)

    await broadcaster.update_node_status(
        node_id,
        "executing",
        {"execution_id": execution_id},
        workflow_id=workflow_id,
    )
    # Mark this workflow active so the toolbar Start->Stop reflects ad-hoc runs.
    # finally: ensures the counter rolls back even on crash.
    await broadcaster.workflow_run_started(workflow_id)
    result: Dict[str, Any]
    try:
        result = await workflow_service.execute_node(
            node_id=node_id,
            node_type=node_type,
            parameters=data.get("parameters", {}),
            nodes=data.get("nodes", []),
            edges=data.get("edges", []),
            session_id=data.get("session_id", "default"),
            execution_id=execution_id,
            workflow_id=workflow_id,
            outputs=data.get("outputs", {}),  # Upstream node outputs for data flow
            extras=invocation_extras or None,
            user_id=user_id,
        )

        if result.get("success"):
            raw_result = result.get("result")
            success_payload = (
                {**raw_result, "execution_id": execution_id}
                if isinstance(raw_result, dict)
                else {
                    **({"result": raw_result} if raw_result is not None else {}),
                    "execution_id": execution_id,
                }
            )
            await broadcaster.update_node_status(node_id, "success", success_payload, workflow_id=workflow_id)
            await broadcaster.update_node_output(node_id, success_payload, workflow_id=workflow_id)
        elif result.get("error") == "Cancelled by user":
            # Cancelled trigger nodes go back to idle, not error
            await broadcaster.update_node_status(
                node_id,
                "idle",
                {"message": "Cancelled", "execution_id": execution_id},
                workflow_id=workflow_id,
            )
        else:
            await broadcaster.update_node_status(
                node_id,
                "error",
                {"error": result.get("error"), "execution_id": execution_id},
                workflow_id=workflow_id,
            )
    except Exception:
        # Mark the node as errored so the UI doesn't keep glowing on crash
        await broadcaster.update_node_status(
            node_id,
            "error",
            {"error": "execution crashed", "execution_id": execution_id},
            workflow_id=workflow_id,
        )
        raise
    finally:
        await broadcaster.workflow_run_ended(workflow_id)

    # Explicitly pass through success status (don't let decorator default to True)
    ws_result = {
        "success": result.get("success", False),
        "node_id": node_id,
        "execution_id": execution_id,
        "result": result.get("result"),
        "error": result.get("error"),
        "execution_time": result.get("execution_time"),
        "timestamp": time.time(),
    }
    # Debug: Log what we're returning to WebSocket
    result_data = result.get("result")
    logger.debug(
        f"[WS execute_node] Returning: success={ws_result['success']}, result.response={repr(result_data.get('response', 'MISSING')[:100] if result_data and result_data.get('response') else 'None')}"
    )
    return ws_result


@ws_handler()
async def handle_cancel_execution(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Cancel ad-hoc node execution(s).

    Two modes:
      * `node_id` — reset that single node to idle (legacy single-node cancel).
      * `workflow_id` only — reset every node in this workflow that's
        currently `executing` or `waiting`, and reset the active-run counter.
        Used by the toolbar Stop button when no deployment is active.

    Either argument is sufficient; if both are present the node-specific
    reset still happens and the workflow-level cleanup runs as well.
    """
    broadcaster = get_status_broadcaster()
    workflow_id = data.get("workflow_id")
    node_id = data.get("node_id")

    if node_id:
        await broadcaster.update_node_status(node_id, "idle", workflow_id=workflow_id)

    cleared = 0
    if workflow_id:
        # Reset run counter to zero (broadcasts executing=false + clears stuck nodes)
        async with broadcaster._workflow_active_lock:
            broadcaster._workflow_active_runs.pop(workflow_id, None)
        await broadcaster.update_workflow_status(executing=False, workflow_id=workflow_id)
        # Explicit user cancel -- include `waiting` so the user sees every
        # indicator go quiet. (Default sweep on run-end skips `waiting` to
        # protect deployment trigger listeners.)
        cleared = await broadcaster._clear_stuck_node_statuses(
            workflow_id,
            include_waiting=True,
        )

    return {
        "node_id": node_id,
        "workflow_id": workflow_id,
        "cleared_nodes": cleared,
        "message": "Execution cancelled",
    }


@ws_handler()
async def handle_get_workflow_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Return cached per-workflow execution status for resync (reconnect / workflow switch)."""
    broadcaster = get_status_broadcaster()
    workflow_id = data.get("workflow_id")
    if not workflow_id:
        return {"success": False, "error": "workflow_id required"}
    return {"workflow_id": workflow_id, "data": broadcaster.get_workflow_status(workflow_id)}


@ws_handler()
async def handle_cancel_event_wait(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Cancel an active event waiter (for trigger nodes).

    Can cancel by waiter_id or node_id.
    Note: Status update to "idle" happens in handle_execute_node when it catches CancelledError.
    """
    from services import event_waiter

    waiter_id = data.get("waiter_id")
    node_id = data.get("node_id")

    logger.debug(f"[WebSocket] handle_cancel_event_wait called: waiter_id={waiter_id}, node_id={node_id}")

    if waiter_id:
        success = event_waiter.cancel(waiter_id)
        logger.debug(f"[WebSocket] cancel by waiter_id result: success={success}")
        return {"success": success, "waiter_id": waiter_id, "message": "Cancelled" if success else "Not found"}
    elif node_id:
        count = event_waiter.cancel_for_node(node_id)
        logger.debug(f"[WebSocket] cancel by node_id result: cancelled_count={count}")
        return {"success": count > 0, "node_id": node_id, "cancelled_count": count}
    else:
        return {"success": False, "error": "waiter_id or node_id required"}


@ws_handler()
async def handle_get_active_waiters(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get list of active event waiters (for debugging/UI)."""
    from services import event_waiter

    return {"waiters": event_waiter.get_active_waiters()}


# ============================================================================
# Dead Letter Queue (DLQ) Handlers
# ============================================================================


@ws_handler()
async def handle_get_dlq_entries(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get DLQ entries with optional filtering.

    Optional params:
        workflow_id: Filter by workflow ID
        node_type: Filter by node type
        limit: Max entries to return (default 100)

    Returns:
        List of DLQ entries
    """
    from services.execution import ExecutionCache

    cache_service = container.cache()
    execution_cache = ExecutionCache(cache_service)

    entries = await execution_cache.get_dlq_entries(
        workflow_id=data.get("workflow_id"), node_type=data.get("node_type"), limit=data.get("limit", 100)
    )

    return {"entries": [entry.to_dict() for entry in entries], "count": len(entries), "timestamp": time.time()}


@ws_handler("entry_id")
async def handle_get_dlq_entry(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get a single DLQ entry by ID.

    Required:
        entry_id: DLQ entry ID

    Returns:
        DLQ entry details
    """
    from services.execution import ExecutionCache

    cache_service = container.cache()
    execution_cache = ExecutionCache(cache_service)

    entry = await execution_cache.get_dlq_entry(data["entry_id"])

    if entry:
        return {"entry": entry.to_dict(), "timestamp": time.time()}
    else:
        return {"success": False, "error": "DLQ entry not found"}


@ws_handler()
async def handle_get_dlq_stats(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get DLQ statistics.

    Returns:
        Total count, breakdown by node type and workflow
    """
    from services.execution import ExecutionCache

    cache_service = container.cache()
    execution_cache = ExecutionCache(cache_service)

    stats = await execution_cache.get_dlq_stats()
    return {"stats": stats, "timestamp": time.time()}


@ws_handler("entry_id")
async def handle_replay_dlq_entry(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Replay a failed node from the DLQ.

    Required:
        entry_id: DLQ entry ID to replay
        nodes: Workflow nodes
        edges: Workflow edges

    Returns:
        Replay execution result
    """
    from services.execution import ExecutionCache, WorkflowExecutor

    cache_service = container.cache()
    execution_cache = ExecutionCache(cache_service)
    workflow_service = container.workflow_service()
    broadcaster = get_status_broadcaster()

    entry_id = data["entry_id"]
    nodes = data.get("nodes", [])
    edges = data.get("edges", [])

    # Get the entry to find the node_id
    entry = await execution_cache.get_dlq_entry(entry_id)
    if not entry:
        return {"success": False, "error": "DLQ entry not found"}

    # Update status
    await broadcaster.update_node_status(entry.node_id, "executing", {"message": "Replaying from DLQ"})

    # Create executor with node adapter
    async def node_executor(node_id: str, node_type: str, params: dict, context: dict) -> dict:
        return await workflow_service.execute_node(
            node_id=node_id,
            node_type=node_type,
            parameters=params,
            nodes=context.get("nodes", []),
            edges=context.get("edges", []),
            session_id=context.get("session_id", "dlq_replay"),
            execution_id=context.get("execution_id"),
        )

    async def status_callback(node_id: str, status: str, status_data: dict):
        await broadcaster.update_node_status(node_id, status, status_data)

    # DLQ replay needs DLQ enabled to re-add on failure
    settings = container.settings()
    executor = WorkflowExecutor(
        cache=execution_cache, node_executor=node_executor, status_callback=status_callback, dlq_enabled=settings.dlq_enabled
    )

    result = await executor.replay_dlq_entry(entry_id, nodes, edges)

    # Update final status
    if result.get("success"):
        await broadcaster.update_node_status(entry.node_id, "success", result.get("result"))
    else:
        await broadcaster.update_node_status(entry.node_id, "error", {"error": result.get("error")})

    return result


@ws_handler("entry_id")
async def handle_remove_dlq_entry(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Remove an entry from the DLQ without replaying.

    Required:
        entry_id: DLQ entry ID to remove

    Returns:
        Success status
    """
    from services.execution import ExecutionCache

    cache_service = container.cache()
    execution_cache = ExecutionCache(cache_service)

    success = await execution_cache.remove_from_dlq(data["entry_id"])
    return {"removed": success, "entry_id": data["entry_id"]}


@ws_handler()
async def handle_purge_dlq(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Purge entries from the DLQ.

    Optional params:
        workflow_id: Only purge entries for this workflow
        node_type: Only purge entries for this node type
        older_than_hours: Only purge entries older than X hours

    Returns:
        Number of entries purged
    """
    from services.execution import ExecutionCache

    cache_service = container.cache()
    execution_cache = ExecutionCache(cache_service)

    older_than = None
    if data.get("older_than_hours"):
        older_than = time.time() - (data["older_than_hours"] * 3600)

    purged = await execution_cache.purge_dlq(workflow_id=data.get("workflow_id"), node_type=data.get("node_type"), older_than=older_than)

    return {"purged": purged, "timestamp": time.time()}


@ws_handler("node_id")
async def handle_get_node_output(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get output data for a specific node."""
    workflow_service = container.workflow_service()
    node_id = data["node_id"]
    output_name = data.get("output_name", "output_0")
    output_data = await workflow_service.get_node_output(data.get("session_id", "default"), node_id, output_name)
    return {"node_id": node_id, "output_name": output_name, "data": output_data, "timestamp": time.time()}


@ws_handler("node_id")
async def handle_clear_node_output(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Clear output data for a specific node from memory, database, and broadcaster cache."""
    workflow_service = container.workflow_service()
    database = container.database()
    broadcaster = get_status_broadcaster()
    node_id = data["node_id"]

    # Clear from memory - find keys ending with _{node_id}
    memory_cleared = 0
    keys_to_delete = [key for key in workflow_service.node_outputs.keys() if key.endswith(f"_{node_id}")]
    for key in keys_to_delete:
        del workflow_service.node_outputs[key]
        memory_cleared += 1

    # Clear from database (persisted storage)
    db_cleared = await database.delete_node_output(node_id)

    # Clear from broadcaster's status cache (prevents reload from showing old data)
    broadcaster_cleared = await broadcaster.clear_node_status(node_id)

    logger.info(
        "Cleared node output",
        node_id=node_id,
        memory_cleared=memory_cleared,
        db_cleared=db_cleared,
        broadcaster_cleared=broadcaster_cleared,
    )

    return {
        "node_id": node_id,
        "cleared": True,
        "memory_cleared": memory_cleared,
        "db_cleared": db_cleared,
        "broadcaster_cleared": broadcaster_cleared,
    }


@ws_handler()
async def handle_validate_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Run pre-execution validation on a workflow graph.

    Serves three call sites: editor live-lint (debounced), pre-execute gate
    (called internally from ``handle_execute_workflow``), and import dry-run
    (called by the frontend after ``importWorkflowFromFile``).

    Expects:
        nodes: List of workflow nodes with {id, type, data}
        edges: List of edges with {id, source, target}
        parameters_by_id: Optional map of node_id -> parameters for INVALID_PARAM
            check. Defaults to ``node.data.parameters`` (rarely populated).

    Returns:
        ``{"success": True, "report": {"errors": [...], "warnings": [...]}}``.
    """
    from services.workflow_validator import validate_workflow

    report = await validate_workflow(
        nodes=data.get("nodes", []),
        edges=data.get("edges", []),
        parameters_by_id=data.get("parameters_by_id"),
    )
    return {"report": report}


@ws_handler()
async def handle_execute_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Execute entire workflow from start node to end.

    Expects:
        workflow_id: Workflow identifier for per-workflow status scoping
        nodes: List of workflow nodes with {id, type, data}
        edges: List of edges with {id, source, target}
        session_id: Optional session identifier
        force: Optional bool — bypass validation errors. When False/missing,
            validation errors short-circuit execution and the response carries
            the report so the frontend can prompt the user.

    Returns:
        Workflow execution result with all node outputs
    """
    workflow_service = container.workflow_service()
    broadcaster = get_status_broadcaster()

    workflow_id = data.get("workflow_id")  # Per-workflow isolation (n8n pattern)
    nodes = data.get("nodes", [])
    edges = data.get("edges", [])
    session_id = data.get("session_id", "default")
    force = bool(data.get("force"))

    if not nodes:
        return {"success": False, "error": "No nodes provided"}

    # Pre-execute validation gate. force=True overrides errors (matches the
    # "Run anyway" UX in Windmill); warnings never block.
    if not force:
        from services.workflow_validator import validate_workflow

        report = await validate_workflow(
            nodes=nodes,
            edges=edges,
            parameters_by_id=data.get("parameters_by_id"),
        )
        if report["errors"]:
            return {
                "success": False,
                "error": "validation_failed",
                "report": report,
            }

    # Mark this workflow active so the toolbar Start->Stop reflects whole-workflow runs
    await broadcaster.workflow_run_started(workflow_id)

    # Create status callback with workflow_id for per-workflow scoping (n8n pattern)
    async def status_callback(node_id: str, status: str, node_data: Optional[Dict] = None):
        await broadcaster.update_node_status(node_id, status, node_data, workflow_id=workflow_id)
        if status == "executing":
            position = node_data.get("position", 0) if node_data else 0
            total = node_data.get("total", 1) if node_data else 1
            progress = int((position / total) * 100) if total > 0 else 0
            await broadcaster.update_workflow_status(
                executing=True,
                current_node=node_id,
                progress=progress,
                workflow_id=workflow_id,
            )

    result: Dict[str, Any]
    try:
        result = await workflow_service.execute_workflow(
            nodes=nodes,
            edges=edges,
            session_id=session_id,
            status_callback=status_callback,
            workflow_id=workflow_id,
            user_id=execution_principal(data, websocket),
        )
    finally:
        # Always release the active-run counter so the button never gets stuck
        await broadcaster.workflow_run_ended(workflow_id)

    return {
        "success": result.get("success", False),
        "nodes_executed": result.get("nodes_executed", []),
        "node_results": result.get("node_results", {}),
        "execution_order": result.get("execution_order", []),
        "errors": result.get("errors", []),
        "error": result.get("error"),
        "total_nodes": result.get("total_nodes", 0),
        "completed_nodes": result.get("completed_nodes", 0),
        "execution_id": result.get("execution_id"),
        "execution_time": result.get("execution_time", 0),
        "timestamp": time.time(),
    }


# ============================================================================
# Deployment handlers — extracted to services/deployment/handlers.py (Wave 13.2)
# ============================================================================
# 5 handlers (deploy_workflow / cancel_deployment / get_deployment_status /
# get_workflow_lock / update_deployment_settings) self-register into the
# central ws_handler_registry via services/deployment/__init__.py.


# ============================================================================
# AI Handlers
# ============================================================================


@ws_handler("node_id", "node_type")
async def handle_execute_ai_node(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Execute an AI node (chat model or agent)."""
    workflow_service = container.workflow_service()
    broadcaster = get_status_broadcaster()
    node_id, node_type = data["node_id"], data["node_type"]
    workflow_id = data.get("workflow_id")  # Per-workflow isolation for tool node glowing
    execution_id = str(data.get("execution_id") or uuid.uuid4().hex)
    user_id = execution_principal(data, websocket)

    await broadcaster.update_node_status(node_id, "executing", {"execution_id": execution_id}, workflow_id=workflow_id)
    await broadcaster.workflow_run_started(workflow_id)
    result: Dict[str, Any]
    try:
        result = await workflow_service.execute_node(
            node_id=node_id,
            node_type=node_type,
            parameters=data.get("parameters", {}),
            nodes=data.get("nodes", []),
            edges=data.get("edges", []),
            session_id=data.get("session_id", "default"),
            execution_id=execution_id,
            workflow_id=workflow_id,
            user_id=user_id,
        )

        if result.get("success"):
            raw_result = result.get("result")
            success_payload = {
                **(
                    raw_result
                    if isinstance(raw_result, dict)
                    else ({"result": raw_result} if raw_result is not None else {})
                ),
                "execution_id": execution_id,
            }
            await broadcaster.update_node_status(node_id, "success", success_payload, workflow_id=workflow_id)
            await broadcaster.update_node_output(node_id, success_payload, workflow_id=workflow_id)
        else:
            await broadcaster.update_node_status(
                node_id,
                "error",
                {"error": result.get("error"), "execution_id": execution_id},
                workflow_id=workflow_id,
            )
    except Exception:
        await broadcaster.update_node_status(
            node_id,
            "error",
            {"error": "execution crashed", "execution_id": execution_id},
            workflow_id=workflow_id,
        )
        raise
    finally:
        await broadcaster.workflow_run_ended(workflow_id)

    return {
        "success": result.get("success", False),
        "node_id": node_id,
        "execution_id": execution_id,
        "result": result.get("result"),
        "error": result.get("error"),
        "execution_time": result.get("execution_time"),
        "timestamp": time.time(),
    }


@ws_handler("provider", "api_key")
async def handle_get_ai_models(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get available AI models for a provider."""
    ai_service = container.ai_service()
    models = await ai_service.fetch_models(data["provider"], data["api_key"])
    return {"provider": data["provider"], "models": models, "timestamp": time.time()}


# ============================================================================
# API Key Handlers — extracted to services/credentials/handlers.py (Wave 13.5)
# ============================================================================
# 4 handlers (validate_api_key / get_stored_api_key / save_api_key /
# delete_api_key) self-register via services/credentials/__init__.py.


# ============================================================================
# Workflow Storage Operations — extracted to services/workflow_storage/handlers.py (Wave 13.7)
# ============================================================================
# 5 handlers (save_workflow / import_workflow / get_workflow /
# get_all_workflows / delete_workflow) self-register via
# services/workflow_storage/__init__.py.


# ============================================================================
# Chat — moved to services/chat/handlers.py with the chat run runtime
# ============================================================================
# send_chat_message / get_chat_messages / chat_subscribe / chat_unsubscribe /
# get_chat_run / clear_chat_messages / save_chat_message self-register via
# services/chat/__init__.py.


@ws_handler()
async def handle_get_console_logs(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get console logs from database, optionally scoped to a workflow."""
    limit = data.get("limit", 100)
    workflow_id = data.get("workflow_id")

    database = container.database()
    control = await database.get_latest_workflow_control(workflow_id) if workflow_id else None
    if control is not None and control.status == "reset":
        logs = []
    else:
        logs = await database.get_console_logs(
            limit, workflow_id=workflow_id,
            execution_id=control.root_execution_id if control is not None else None,
        )

    return {
        "success": True,
        "logs": logs,
        "workflow_id": workflow_id,
    }


@ws_handler()
async def handle_clear_console_logs(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Clear console logs from database and in-memory broadcaster cache.

    With ``workflow_id`` set, only that workflow's history is cleared
    (DB rows + in-memory entries). Without it, the legacy "clear all"
    behaviour is preserved. Broadcasts ``console_logs_cleared`` carrying
    the workflow_id so other tabs viewing the same workflow drop their
    local list, while tabs on a different workflow are unaffected
    (the frontend filter already keys on this).
    """
    workflow_id = data.get("workflow_id")
    database = container.database()
    count = await database.clear_console_logs(workflow_id=workflow_id)

    # Also clear / filter in-memory logs
    broadcaster = get_status_broadcaster()
    if "console_logs" in broadcaster._status:
        if workflow_id:
            broadcaster._status["console_logs"] = [
                log for log in broadcaster._status["console_logs"] if log.get("workflow_id") != workflow_id
            ]
        else:
            broadcaster._status["console_logs"] = []

    # Tell connected clients to drop their local copy for this scope.
    await broadcaster.broadcast(
        {
            "type": "console_logs_cleared",
            "workflow_id": workflow_id,
        }
    )

    return {
        "success": True,
        "message": f"Cleared {count} console logs",
        "cleared_count": count,
        "workflow_id": workflow_id,
    }


# ============================================================================
# Terminal Logs Handlers
# ============================================================================


@ws_handler()
async def handle_get_terminal_logs(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get terminal log history."""
    broadcaster = get_status_broadcaster()
    logs = broadcaster.get_terminal_logs()
    return {"success": True, "logs": logs}


@ws_handler()
async def handle_clear_terminal_logs(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Clear terminal log history."""
    broadcaster = get_status_broadcaster()
    await broadcaster.clear_terminal_logs()
    return {"success": True, "message": "Terminal logs cleared"}


# =============================================================================
# Process Manager Handlers
# =============================================================================


@ws_handler()
async def handle_process_list(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """List running processes."""
    from services.process_service import get_process_service

    svc = get_process_service()
    workflow_id = data.get("workflow_id", "default")
    return {"success": True, "processes": svc.list_processes(workflow_id), "max_processes": svc.max_processes}


@ws_handler("name")
async def handle_process_get_output(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get output from a process's log file."""
    from services.process_service import get_process_service

    name = data["name"]
    workflow_id = data.get("workflow_id", "default")
    stream = data.get("stream", "stdout")
    tail = int(data.get("tail", 50))
    offset = int(data.get("offset", 0))
    return {"success": True, **get_process_service().get_output(name, workflow_id, stream, tail, offset)}


@ws_handler("name", "text")
async def handle_process_send_input(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Send stdin text to a running process."""
    from services.process_service import get_process_service

    name = data["name"]
    text = data["text"]
    workflow_id = data.get("workflow_id", "default")
    return await get_process_service().send_input(name, workflow_id, text)


@ws_handler("name")
async def handle_process_stop(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Stop one workflow-scoped managed process."""
    from services.process_service import get_process_service

    return await get_process_service().stop(data["name"], data.get("workflow_id", "default"))


@ws_handler("name")
async def handle_process_restart(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Restart one workflow-scoped managed process after port preflight."""
    from services.process_service import get_process_service

    return await get_process_service().restart(data["name"], data.get("workflow_id", "default"))


# ============================================================================
# User Skills Handlers — extracted to services/skills/handlers.py (Wave 13.1)
# ============================================================================
# The 13 handlers (get_skill_content / save_skill_content / scan_skill_folder
# / list_skill_folders / lookup_skill_metadata / evaluate_auto_skill /
# get_user_skills / get_user_skill / create_user_skill / update_user_skill /
# delete_user_skill / clear_memory / reset_skill) self-register into the
# central ws_handler_registry via services/skills/__init__.py. The WS
# router resolves dispatch from that registry, so this comment block is
# the only trace they leave in routers/websocket.py.


# ============================================================================
# Settings handlers — extracted to services/settings/handlers.py (Wave 13.3)
# ============================================================================
# 6 of 8 handlers live there: get_user_settings / save_user_settings /
# get_provider_defaults / save_provider_defaults / get_validated_ai_providers /
# save_global_model. The remaining 2 (get_compaction_stats / configure_compaction)
# also moved out — see the Compaction Handlers comment below.


# ============================================================================
# Pricing Config Handlers — extracted to services/pricing_handlers.py (Wave 13.8)
# ============================================================================
# 3 handlers (get_pricing_config / save_pricing_config / get_api_usage_summary)
# self-register on services/pricing_handlers.py import.


@ws_handler()
async def handle_get_node_allowlist(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Return the node allowlist that controls which nodes appear in the palette."""
    from services.node_allowlist import get_node_allowlist_service

    return get_node_allowlist_service().get_config()


# ============================================================================
# Compaction Handlers — extracted to services/settings/handlers.py (Wave 13.3)
# ============================================================================
# get_compaction_stats + configure_compaction moved, and so did
# get_provider_usage_summary; all three self-register from services/settings.


# ============================================================================
# Agent Team Handlers — extracted to services/agent_teams/handlers.py (Wave 13.4)
# ============================================================================
# 10 handlers (create_team / get_team / get_team_status / dissolve_team /
# add_team_task / claim_team_task / complete_team_task / get_team_tasks /
# send_team_message / get_team_messages) self-register via
# services/agent_teams/__init__.py.


# ============================================================================
# Model Registry Handlers
# ============================================================================


@ws_handler()
async def handle_get_model_constraints(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Get model constraints (max_output_tokens, temperature range, thinking support, etc.)."""
    from services.model_registry import get_model_registry

    registry = get_model_registry()
    model = data.get("model", "")
    provider = data.get("provider", "")
    if not model or not provider:
        return {"success": False, "error": "model and provider are required"}
    constraints = registry.get_model_constraints(model, provider)
    return {"success": True, **constraints}


@ws_handler()
async def handle_refresh_model_registry(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
    """Force refresh model registry from OpenRouter."""
    from services.model_registry import get_model_registry

    registry = get_model_registry()
    try:
        count = await registry.refresh()
        return {"success": True, "model_count": count}
    except Exception as e:
        return {"success": False, "error": str(e)}


# ============================================================================
# Message Router
# ============================================================================

# Plugin packages (nodes/<group>/__init__.py) self-register their WS
# handlers into ``services.ws_handler_registry`` at import time. This
# router doesn't know any plugin's message-type names — it consults the
# registry at dispatch time via :func:`_resolve_handler`.
from services.ws_handler_registry import get_ws_handlers


from services.authz import (  # noqa: E402
    admit_internal_ws,
    authenticate_ws,
    execution_principal,
    resolve_internal_handler,
)


def _resolve_handler(msg_type: str):
    """Resolve a WS message_type to its handler.

    Looks first in the legacy core ``MESSAGE_HANDLERS`` table, then
    falls back to the plugin-owned registry. Plugin handlers are
    discovered at import time (via the side-effect ``import nodes`` in
    ``main.lifespan``), so by the time messages flow they're registered.
    """
    handler = MESSAGE_HANDLERS.get(msg_type)
    if handler is None:
        handler = get_ws_handlers().get(msg_type)
    return handler


MESSAGE_HANDLERS: Dict[str, MessageHandler] = {
    # Status/ping
    "ping": handle_ping,
    "get_status": handle_get_status,
    # get_android_status moved to nodes/android/_handlers.py (Wave 13.9)
    "get_node_status": handle_get_node_status,
    "get_variable": handle_get_variable,
    # Node parameters
    "get_node_parameters": handle_get_node_parameters,
    "get_all_node_parameters": handle_get_all_node_parameters,
    "save_node_parameters": handle_save_node_parameters,
    "delete_node_parameters": handle_delete_node_parameters,
    # Tool schemas (source of truth for tool configurations)
    "get_tool_schema": handle_get_tool_schema,
    "save_tool_schema": handle_save_tool_schema,
    "delete_tool_schema": handle_delete_tool_schema,
    "get_all_tool_schemas": handle_get_all_tool_schemas,
    # Node output schemas (Pydantic-backed registry; see
    # docs-internal/ARCHIVE/schema_source_of_truth_rfc.md).
    "get_node_output_schema": handle_get_node_output_schema,
    # Wave 6 Phase 2: unified NodeSpec (input + output + metadata).
    "get_node_spec": handle_get_node_spec,
    "list_node_specs": handle_list_node_specs,
    # Wave 6 Phase 4: generic loadOptionsMethod dispatcher.
    "load_options": handle_load_options,
    "list_load_options_methods": handle_list_load_options_methods,
    # Wave 6 Phase 5: node-groups index (replaces *_NODE_TYPES helpers).
    "get_node_groups": handle_get_node_groups,
    # Credential registry (Nango-style bulk catalogue for credentials panel)
    "get_credential_catalogue": handle_get_credential_catalogue,
    # Node execution
    "execute_node": handle_execute_node,
    "execute_workflow": handle_execute_workflow,
    "validate_workflow": handle_validate_workflow,
    "cancel_execution": handle_cancel_execution,
    "get_workflow_status": handle_get_workflow_status,
    "get_node_output": handle_get_node_output,
    "clear_node_output": handle_clear_node_output,
    # Trigger/event waiting
    "cancel_event_wait": handle_cancel_event_wait,
    "get_active_waiters": handle_get_active_waiters,
    # Dead Letter Queue (DLQ) operations
    "get_dlq_entries": handle_get_dlq_entries,
    "get_dlq_entry": handle_get_dlq_entry,
    "get_dlq_stats": handle_get_dlq_stats,
    "replay_dlq_entry": handle_replay_dlq_entry,
    "remove_dlq_entry": handle_remove_dlq_entry,
    "purge_dlq": handle_purge_dlq,
    # Deployment operations — extracted to services/deployment/handlers.py
    # (Wave 13.2); registered via ws_handler_registry on package import.
    # AI operations
    "execute_ai_node": handle_execute_ai_node,
    "get_ai_models": handle_get_ai_models,
    # API key operations — extracted to services/credentials/handlers.py
    # (Wave 13.5); registered via ws_handler_registry on package import.
    # Claude/Codex login handlers live in the plugin folders + framework:
    # - ``claude_code_login`` / ``claude_code_logout`` registered by
    #   ``nodes/agent/claude_code_agent/__init__.py``.
    # - ``codex_cli_login`` / ``codex_cli_logout`` registered by
    #   ``services/cli_agent/__init__.py``.
    # Both go through ``services.ws_handler_registry``; nothing to
    # mount in MESSAGE_HANDLERS here.
    # Twitter OAuth operations
    # Google Workspace OAuth operations
    # Android operations
    # Maps + Apify validation now flow through ``handle_validate_api_key``
    # (which dispatches via ``CREDENTIAL_REGISTRY`` to
    # ``GoogleMapsCredential._probe`` / ``ApifyCredential._probe``).
    # The legacy ``validate_maps_key`` / ``validate_apify_key`` WS message
    # types are no longer needed — the frontend already uses
    # ``validate_api_key`` for all providers.
    # WhatsApp and Telegram operations live in nodes/<plugin>/_handlers.py and
    # self-register via services.ws_handler_registry. Dispatch hits them
    # via _resolve_handler() defined above.
    # Workflow storage operations — extracted to services/workflow_storage/handlers.py
    # (Wave 13.7); registered via ws_handler_registry on package import.
    # Chat commands live in services/chat/handlers.py (registered on import).
    # Console logs (for Console nodes)
    "get_console_logs": handle_get_console_logs,
    "clear_console_logs": handle_clear_console_logs,
    # Terminal logs
    "get_terminal_logs": handle_get_terminal_logs,
    "clear_terminal_logs": handle_clear_terminal_logs,
    # Process Manager
    "process_list": handle_process_list,
    "process_get_output": handle_process_get_output,
    "process_send_input": handle_process_send_input,
    "process_stop": handle_process_stop,
    "process_restart": handle_process_restart,
    # User Skills + Built-in Skill Content + Memory/Reset — extracted to
    # services/skills/handlers.py (Wave 13.1); register via the shared
    # ws_handler_registry on package import. No entries below.
    # User Settings + Provider Defaults + Compaction + Provider Usage Summary
    # — extracted to services/settings/handlers.py (Wave 13.3); register
    # via ws_handler_registry on package import.
    # Pricing Config — extracted to services/pricing_handlers.py (Wave 13.8);
    # registered via ws_handler_registry on package import.
    # Node Allowlist (UI palette filter)
    "get_node_allowlist": handle_get_node_allowlist,
    # Model Registry
    "get_model_constraints": handle_get_model_constraints,
    "refresh_model_registry": handle_refresh_model_registry,
    # Agent Teams — extracted to services/agent_teams/handlers.py (Wave 13.4);
    # registered via ws_handler_registry on package import.
}


async def _execute_handler(handler: MessageHandler, data: Dict[str, Any], websocket: WebSocket, msg_type: str, request_id: Optional[str]):
    """Execute handler and send response using safe send."""
    try:
        result = await handler(data, websocket)

        if request_id:
            # request_id belongs to the socket's pending-request registry.
            # Hiring and delivery also return durable operation IDs; those
            # must not orphan the browser request after the operation saves.
            response = {"type": f"{msg_type}_result", **result, "request_id": request_id}
            operation_request_id = result.get("request_id")
            if operation_request_id is not None and operation_request_id != request_id:
                response["operation_request_id"] = operation_request_id
            await _safe_send(websocket, response)
        else:
            await _safe_send(websocket, result)

    except asyncio.CancelledError:
        # Task was cancelled (e.g., WebSocket disconnected)
        logger.debug(f"[WebSocket] Handler cancelled: {msg_type}")
        raise
    except Exception as e:
        logger.error("Handler error", msg_type=msg_type, error=str(e))
        if request_id:
            await _safe_send(websocket, {"type": f"{msg_type}_result", "request_id": request_id, "success": False, "error": str(e)})


@router.websocket("/ws/status")
async def websocket_status_endpoint(websocket: WebSocket):
    """WebSocket endpoint for real-time bidirectional communication.

    Uses decoupled receive/process pattern with asyncio.Queue:
    - Receive task: continuously receives messages into queue (never blocks)
    - Process task: reads from queue and spawns handler tasks (can be long-running)

    This ensures cancel messages are always processed immediately, even when
    long-running handlers (like trigger node execution) are active.

    All client requests include a request_id for correlation.
    The server responds with the same request_id for request/response matching.
    Broadcasts (without request_id) are sent to all connected clients.
    """
    # Origin, then the session cookie (unless login is off), before accepting.
    authenticated_user_id = await authenticate_ws(
        websocket,
        settings=container.settings(),
        user_auth_service=container.user_auth_service,
    )
    if authenticated_user_id is None:
        return

    # Plugin-owned handlers resolve namespace ownership from trusted
    # connection state. Client payloads cannot choose a Memory/Context owner.
    websocket.state.user_id = authenticated_user_id

    broadcaster = get_status_broadcaster()
    try:
        await broadcaster.connect(websocket)
    except BaseException:
        # connect registers the socket before delivering initial snapshots.
        # A cancellation there must not leave a connection outside the
        # receive/process cleanup guard below. disconnect is idempotent if
        # acceptance itself failed before registration.
        await broadcaster.disconnect(websocket)
        raise

    # Message queue for decoupling receive from processing
    message_queue: asyncio.Queue = asyncio.Queue()

    # Track handler tasks for this WebSocket
    handler_tasks: Set[asyncio.Task] = set()
    _handler_tasks[websocket] = handler_tasks

    async def receive_loop():
        """Receives messages and puts them in queue - never blocks on handlers."""
        try:
            while True:
                data = await websocket.receive_json()
                await message_queue.put(data)
        except WebSocketDisconnect as exc:
            # Routine tab closes and server restarts should not look like
            # failures, but abnormal closes need enough context to diagnose.
            log = logger.debug if exc.code in (1000, 1001) else logger.warning
            reason = "".join(character if character.isprintable() else "?" for character in str(exc.reason or "")[:123])
            log("[WebSocket] Client disconnected", path="/ws/status", close_code=exc.code, reason=reason)
            await message_queue.put(None)  # Signal shutdown
        except asyncio.CancelledError:
            # Task cancelled during shutdown - this is expected
            await message_queue.put(None)
            raise
        except Exception as e:
            # Only log if it's not a shutdown-related error
            if not isinstance(e, (KeyboardInterrupt, SystemExit)):
                logger.error(f"[WebSocket] Receive error: {e}")
            await message_queue.put(None)

    async def process_loop():
        """Processes messages from queue - spawns handler tasks that can run concurrently."""
        while True:
            data = await message_queue.get()

            if data is None:  # Shutdown signal
                break

            msg_type = data.get("type", "")
            request_id = data.get("request_id")

            logger.debug("WebSocket message received", msg_type=msg_type, has_request_id=bool(request_id))

            handler = _resolve_handler(msg_type)

            if handler:
                # Run handler as task so it doesn't block queue processing
                # This allows cancel_event_wait to be processed while execute_node is waiting
                task = asyncio.create_task(_execute_handler(handler, data, websocket, msg_type, request_id))
                handler_tasks.add(task)
                task.add_done_callback(handler_tasks.discard)
            else:
                logger.warning("Unknown message type", msg_type=msg_type)
                if request_id:
                    await _safe_send(
                        websocket,
                        {
                            "type": "error",
                            "request_id": request_id,
                            "code": "UNKNOWN_MESSAGE_TYPE",
                            "message": f"Unknown message type: {msg_type}",
                        },
                    )

    try:
        # Run receive and process loops concurrently using TaskGroup (Python 3.11+)
        async with asyncio.TaskGroup() as tg:
            tg.create_task(receive_loop())
            tg.create_task(process_loop())

    except* WebSocketDisconnect:
        pass  # Normal disconnect - don't log during shutdown
    except* asyncio.CancelledError:
        pass  # Task cancelled during shutdown - expected
    except* (KeyboardInterrupt, SystemExit):
        pass  # Server shutdown - don't log
    except* Exception as eg:
        for exc in eg.exceptions:
            if not isinstance(exc, (WebSocketDisconnect, asyncio.CancelledError, KeyboardInterrupt, SystemExit)):
                logger.error(f"[WebSocket] TaskGroup error: {exc}")
    finally:
        # Let short handlers finish, then cancel stragglers (see
        # _drain_handler_tasks for why this is not an immediate cancel).
        await _drain_handler_tasks(handler_tasks)

        # Cleanup
        _handler_tasks.pop(websocket, None)
        await broadcaster.disconnect(websocket)


@router.websocket("/ws/internal")
async def websocket_internal_endpoint(websocket: WebSocket):
    """Internal WebSocket endpoint for Temporal workers.

    Service-to-service only (Temporal activity -> OpenCompany). It bypasses
    the cookie gate, so the worker must present the ``SECRET_KEY``-derived
    token (``services.authz.internal_socket_token``) before the handshake is
    accepted, and even then it reaches only ``INTERNAL_SOCKET_HANDLERS``.
    """
    if not await admit_internal_ws(websocket, settings=container.settings()):
        return

    get_status_broadcaster()
    await websocket.accept()

    logger.info("[WebSocket Internal] Temporal worker connected")

    # Message queue for decoupling receive from processing
    message_queue: asyncio.Queue = asyncio.Queue()

    # Track handler tasks for this WebSocket
    handler_tasks: Set[asyncio.Task] = set()

    async def receive_loop():
        """Receives messages and puts them in queue."""
        try:
            while True:
                data = await websocket.receive_json()
                await message_queue.put(data)
        except WebSocketDisconnect:
            await message_queue.put(None)
        except asyncio.CancelledError:
            await message_queue.put(None)
            raise
        except Exception as e:
            if not isinstance(e, (KeyboardInterrupt, SystemExit)):
                logger.error(f"[WebSocket Internal] Receive error: {e}")
            await message_queue.put(None)

    async def process_loop():
        """Processes messages from queue."""
        while True:
            data = await message_queue.get()

            if data is None:
                break

            msg_type = data.get("type", "")
            request_id = data.get("request_id")

            # Deny-by-default: this socket is unauthenticated, so it may
            # reach only the handlers the activity worker actually needs.
            handler = resolve_internal_handler(msg_type, _resolve_handler)

            if handler:
                task = asyncio.create_task(_execute_handler(handler, data, websocket, msg_type, request_id))
                handler_tasks.add(task)
                task.add_done_callback(handler_tasks.discard)
            else:
                logger.warning(f"[WebSocket Internal] Unknown message type: {msg_type}")
                if request_id:
                    await _safe_send(
                        websocket,
                        {
                            "type": "error",
                            "request_id": request_id,
                            "code": "UNKNOWN_MESSAGE_TYPE",
                            "message": f"Unknown message type: {msg_type}",
                        },
                    )

    try:
        async with asyncio.TaskGroup() as tg:
            tg.create_task(receive_loop())
            tg.create_task(process_loop())

    except* WebSocketDisconnect:
        pass  # Normal disconnect
    except* asyncio.CancelledError:
        pass  # Task cancelled during shutdown
    except* (KeyboardInterrupt, SystemExit):
        pass  # Server shutdown
    except* Exception as eg:
        for exc in eg.exceptions:
            if not isinstance(exc, (WebSocketDisconnect, asyncio.CancelledError, KeyboardInterrupt, SystemExit)):
                logger.error(f"[WebSocket Internal] TaskGroup error: {exc}")
    finally:
        await _drain_handler_tasks(handler_tasks)


@router.get("/ws/info")
async def websocket_info():
    """Get WebSocket connection info."""
    broadcaster = get_status_broadcaster()
    return {
        "endpoint": "/ws/status",
        "connected_clients": broadcaster.connection_count,
        "current_status": broadcaster.get_status(),
        "supported_message_types": sorted(set(MESSAGE_HANDLERS.keys()) | set(get_ws_handlers().keys())),
    }