Skip to content

Self-improving AI Employees for everyone

27.6 KB
Raw Blame
"""Temporal activities for distributed node execution.

Uses class-based activity pattern recommended by Temporal docs for sharing
resources like aiohttp.ClientSession across activity invocations.

References:
- https://docs.temporal.io/develop/python/python-sdk-sync-vs-async
- https://docs.temporal.io/develop/python/core-application

Architecture:
- NodeExecutionActivities class holds shared aiohttp.ClientSession
- Session is passed via constructor, avoiding recreation per activity
- Each activity call gets its own WebSocket connection from the session pool
"""

from datetime import datetime
from typing import Any, Dict, Optional

import aiohttp
from temporalio import activity

from core.logging import get_logger
from core.config import Settings

logger = get_logger(__name__)


def _resolve_urls() -> tuple[str, str]:
    """Resolve ``(http_base, ws_url)`` from current ``Settings``.

    Deferred to first call so module import doesn't require the full env
    surface (``HOST``, ``PORT``, ``JWT_SECRET_KEY``, ...). Module-level
    ``Settings()`` made the temporal package un-importable in any test
    process that hadn't loaded ``.env`` -- conftest stubs got wiped by
    sibling test suites and the validation error masked real bugs.
    """
    settings = Settings()
    return (
        f"http://{settings.host}:{settings.port}",
        f"ws://{settings.host}:{settings.port}/ws/internal",
    )


class NodeExecutionActivities:
    """Activity class for node execution with shared aiohttp session.

    Following Temporal's recommended pattern for dependency injection:
    - aiohttp.ClientSession is passed via constructor
    - Session provides connection pooling for concurrent activities
    - Each activity call gets its own WebSocket connection from the pool

    Reference: https://docs.temporal.io/develop/python/python-sdk-sync-vs-async
    """

    def __init__(self, session: aiohttp.ClientSession):
        """Initialize with shared aiohttp session.

        Args:
            session: aiohttp.ClientSession with connection pooling configured
        """
        self.session = session
        http_base, ws_url = _resolve_urls()
        self.ws_url = ws_url
        self.ws_headers: Optional[Dict[str, str]] = None
        self.http_url = f"{http_base}/api/workflow/node/execute"
        self.broadcast_url = f"{http_base}/api/workflow/broadcast-status"

    @activity.defn
    async def execute_node_activity(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Execute a single workflow node with isolated context.

        This activity delegates execution over WebSocket back to the
        originating FastAPI server (ws://{host}:{port}/ws/internal per
        _resolve_urls), so any worker running it must be able to reach
        that server -- it is not freely relocatable across a cluster.

        Each node execution is independent - if it fails, Temporal will retry
        on the same or different worker without affecting other nodes.

        Args:
            context: Immutable context containing:
                - node_id: Unique node identifier
                - node_type: Type of node (aiAgent, console, timer, etc.)
                - node_data: Node configuration from React Flow
                - inputs: Outputs from upstream nodes (dependencies)
                - workflow_id: Parent workflow ID for tracking
                - tenant_id: Tenant identifier for multi-tenancy
                - session_id: Session identifier
                - nodes: Full node list (for tool/memory detection by handlers)
                - edges: Full edge list (for tool/memory detection by handlers)

        Returns:
            Dict with success, result, node_id, and metadata
        """
        node_id = context["node_id"]
        node_type = context["node_type"]
        node_data = context.get("node_data", {})
        workflow_id = context.get("workflow_id")
        execution_id = context.get("execution_id")
        tenant_id = context.get("tenant_id")

        activity.logger.info(
            f"Executing node activity: {node_id} ({node_type})",
            extra={"tenant_id": tenant_id, "workflow_id": workflow_id},
        )

        # Heartbeat at start to signal activity is alive
        activity.heartbeat(f"Starting {node_type}: {node_id}")

        # Handle pre-executed trigger nodes (already have their output)
        if context.get("pre_executed"):
            activity.logger.debug(f"Node {node_id} is pre-executed, returning cached result")
            result = {
                "success": True,
                "node_id": node_id,
                "node_type": node_type,
                "result": context.get("trigger_output", {}),
                "pre_executed": True,
                "execution_id": execution_id,
                "timestamp": datetime.now().isoformat(),
            }
            await self._broadcast_status(node_id, "success", result, workflow_id)
            return result

        # Handle disabled nodes
        if node_data.get("disabled"):
            activity.logger.debug(f"Node {node_id} is disabled, skipping")
            result = {
                "success": True,
                "node_id": node_id,
                "node_type": node_type,
                "skipped": True,
                "reason": "disabled",
                "execution_id": execution_id,
                "timestamp": datetime.now().isoformat(),
            }
            await self._broadcast_status(
                node_id,
                "skipped",
                {"disabled": True, "execution_id": execution_id},
                workflow_id,
            )
            return result

        # A call runs at most once unless it is safe to repeat
        # (services/approvals/tool_calls.py): a later attempt answers that it
        # may already have happened, and the owner gets a card.
        from services.approvals import tool_calls as approval_calls
        from services.node_registry import get_node_class

        node_cls = get_node_class(node_type)
        redo = await approval_calls.redo_refusal(context, node_cls, node_data)
        if redo is not None:
            result = {
                "node_id": node_id,
                "node_type": node_type,
                "execution_id": execution_id,
                "timestamp": datetime.now().isoformat(),
                **redo,
            }
            await approval_calls.record_unknown(context, node_cls, node_data, result)
            await self._broadcast_status(
                node_id=node_id,
                status="error",
                data={"error": redo["error"], "execution_id": execution_id,
                      **{key: redo[key] for key in ("hint", "requires_user_action", "error_type") if key in redo}},
                workflow_id=workflow_id,
            )
            return result

        # Broadcast "executing" status for UI updates
        await self._broadcast_status(
            node_id=node_id,
            status="executing",
            data={"node_type": node_type, "execution_id": execution_id},
            workflow_id=workflow_id,
        )

        try:
            # Heartbeat before potentially long WebSocket operation
            activity.heartbeat(f"Executing via WebSocket: {node_id}")

            # Execute node via WebSocket (each call gets own connection from pool)
            result = await self._execute_via_websocket(context)

            # Add metadata
            result["node_id"] = node_id
            result["node_type"] = node_type
            result["timestamp"] = datetime.now().isoformat()

            # Status is already broadcast by the WebSocket handler (handle_execute_node)
            # which calls both update_node_status and update_node_output.
            # Do NOT duplicate here - late-arriving broadcasts overwrite output data.
            if result.get("success"):
                activity.logger.info(f"Node {node_id} completed successfully")
            else:
                activity.logger.warning(f"Node {node_id} failed: {result.get('error')}")

            # Heartbeat for activity liveness
            activity.heartbeat(f"Node {node_id} completed")

            return result

        except Exception as e:
            error_msg = f"{type(e).__name__}: {str(e)}"
            activity.logger.error(f"Node {node_id} execution failed: {error_msg}")

            # Broadcast error status
            await self._broadcast_status(
                node_id=node_id,
                status="error",
                data={"error": error_msg, "execution_id": execution_id},
                workflow_id=workflow_id,
            )

            # Raise to trigger Temporal retry mechanism
            raise

    async def _execute_via_websocket(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Execute node via WebSocket using shared session's connection pool.

        Each call creates a new WebSocket connection from the session's pool,
        avoiding race conditions when multiple activities run concurrently.

        Heartbeats are sent both on incoming non-matching messages AND on a
        30-second timer. This is critical for long-running nodes (browser,
        AI multi-tool, claude_code_agent) where the backend may be processing internally
        without broadcasting any WS messages for minutes. Without the periodic
        timer, the 2-minute heartbeat_timeout would cancel the activity even
        though the node is actively executing.
        """
        import asyncio
        import json
        import uuid

        node_id = context["node_id"]
        node_type = context["node_type"]
        request_id = str(uuid.uuid4())

        message = {
            "type": "execute_node",
            "request_id": request_id,
            "node_id": node_id,
            "node_type": node_type,
            "parameters": context.get("node_data", {}),
            "nodes": context.get("nodes", []),
            "edges": context.get("edges", []),
            "session_id": context.get("session_id", "default"),
            "workflow_id": context.get("workflow_id"),
            "execution_id": context.get("execution_id"),
            "user_id": context.get("user_id", "owner"),
            # CRITICAL: Pass upstream node outputs for downstream nodes to access
            # This enables taskTrigger -> chatAgent data flow via input-task handle
            "outputs": context.get("inputs", {}),
        }
        if context.get("pi_harness_version") == 1:
            from services.temporal.pi_inline_delegation import forward_inline_scope
            message.update(forward_inline_scope(context))
        for key in (
            "auto_rebind_tools",
            "pi_harness_version",
            "parameter_snapshot",
            "invoking_agent_node_id",
            "agent_iteration",
            "tool_call_index",
            "tool_call_id",
            "run_scope",
        ):
            if key in context:
                message[key] = context[key]

        activity.logger.debug(f"WebSocket execute for {node_id}")

        try:
            # Each activity gets its own WebSocket connection from the pool
            async with self.session.ws_connect(
                self.ws_url,
                headers=self._internal_socket_headers(),
                heartbeat=30,
                receive_timeout=None,  # No receive timeout — we handle liveness via heartbeats
            ) as ws:
                await ws.send_json(message)

                # Read loop with periodic heartbeat timer.
                # ws.receive() may block for minutes if the backend is doing
                # heavy processing without broadcasting. The 30s timeout ensures
                # we heartbeat at least every 30s regardless of message traffic.
                while True:
                    try:
                        msg = await asyncio.wait_for(ws.receive(), timeout=30.0)
                    except asyncio.TimeoutError:
                        # No message in 30s — node is still running, send heartbeat
                        activity.heartbeat(f"Waiting for {node_id} ({node_type})")
                        continue

                    if msg.type == aiohttp.WSMsgType.TEXT:
                        response = json.loads(msg.data)
                        if response.get("request_id") == request_id:
                            activity.logger.debug(f"Got response for {node_id}: success={response.get('success')}")
                            return response
                        # Non-matching message (broadcast to other clients) — heartbeat
                        activity.heartbeat(f"Waiting for {node_id}")
                    elif msg.type == aiohttp.WSMsgType.ERROR:
                        raise Exception(f"WebSocket error: {ws.exception()}")
                    elif msg.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.CLOSING):
                        raise Exception("WebSocket closed unexpectedly")

        except aiohttp.ClientError as e:
            raise Exception(f"WebSocket connection error: {e}")

    def _internal_socket_headers(self) -> Dict[str, str]:
        """The token ``/ws/internal`` requires in the handshake.

        Resolved on first connect rather than in ``__init__``, for the same
        reason as ``_resolve_urls``: building the activities must not need
        the full env surface.
        """
        if self.ws_headers is None:
            from services.authz import internal_socket_headers

            self.ws_headers = internal_socket_headers(Settings().secret_key)
        return self.ws_headers

    async def _broadcast_status(
        self,
        node_id: str,
        status: str,
        data: dict,
        workflow_id: str = None,
    ) -> None:
        """Broadcast node status for real-time UI updates.

        Non-fatal - execution continues even if broadcast fails.
        """
        try:
            async with self.session.post(
                self.broadcast_url,
                json={
                    "node_id": node_id,
                    "status": status,
                    "data": data or {},
                    "workflow_id": workflow_id,
                },
                timeout=aiohttp.ClientTimeout(total=5),
            ) as response:
                if response.status == 200:
                    logger.debug(f"Broadcast: {node_id} -> {status}")
        except Exception as e:
            # Non-fatal - don't fail execution if broadcast fails
            logger.warning(f"Broadcast failed for {node_id}: {e}")


# =============================================================================
# Factory function for creating activity instance with session
# =============================================================================


def create_node_activities(session: aiohttp.ClientSession) -> NodeExecutionActivities:
    """Factory function to create activity instance with shared session.

    This follows Temporal's recommended pattern for dependency injection.
    The session should be created once when the worker starts and reused.

    Args:
        session: aiohttp.ClientSession with connection pooling

    Returns:
        NodeExecutionActivities instance ready for worker registration
    """
    return NodeExecutionActivities(session)


async def create_shared_session(pool_size: int = 100) -> aiohttp.ClientSession:
    """Create a shared aiohttp session with connection pooling.

    Args:
        pool_size: Maximum number of concurrent connections

    Returns:
        Configured aiohttp.ClientSession
    """
    connector = aiohttp.TCPConnector(
        limit=pool_size,
        limit_per_host=pool_size,
        enable_cleanup_closed=True,
    )
    timeout = aiohttp.ClientTimeout(
        total=300,  # 5 min total
        connect=10,  # 10 sec connect
    )
    session = aiohttp.ClientSession(
        connector=connector,
        timeout=timeout,
    )
    logger.info(f"Created shared session with pool_size={pool_size}")
    return session


# ---------------------------------------------------------------------------
# Wave 12 A8: emit_event_activity
#
# Thin Temporal-activity wrapper around ``services.events.dispatch.emit``.
# Used by ``PollingTriggerWorkflow`` (Phase C2) when a poll cycle produces
# new items — the workflow can't directly do I/O (the dispatch helper
# uses asyncio + a network round-trip to Temporal Visibility), so it
# offloads to this activity.
#
# Activity scheduling timeouts (``start_to_close_timeout``, retry policy,
# heartbeat) are set by the CALLING workflow at scheduling time —
# ``workflow.execute_activity(emit_event_activity, ...,
# start_to_close_timeout=timedelta(seconds=2))``. Defaults are not
# hardcoded here.


@activity.defn
async def emit_event_activity(event_payload: Dict[str, Any]) -> Dict[str, Any]:
    """Re-enter :func:`services.events.dispatch.emit` from a workflow context.

    Args:
        event_payload: The serialised :class:`WorkflowEvent` (output of
            ``event.model_dump(mode="json")``). The activity reconstructs
            the envelope and routes it through the dispatch helper.

    Returns:
        Status dict:
            {
              "delivered": True | False,    # whether dispatch completed
              "event_id": str,
              "event_type": str,
            }
    """
    from services.events.envelope import WorkflowEvent
    from services.events.dispatch import emit

    try:
        event = WorkflowEvent.model_validate(event_payload)
    except Exception as exc:  # noqa: BLE001
        activity.logger.warning(f"emit_event_activity: envelope rejected: {exc}")
        return {
            "delivered": False,
            "event_id": event_payload.get("id", "?"),
            "event_type": event_payload.get("type", "?"),
            "error": f"{type(exc).__name__}: {exc}",
        }

    await emit(event, broadcast=False)
    return {
        "delivered": True,
        "event_id": event.id,
        "event_type": event.type,
    }


# ---------------------------------------------------------------------------
# Trigger-node status broadcast activity
#
# Used by :class:`services.temporal.trigger_listener_workflow.TriggerListenerWorkflow`
# and :class:`services.temporal.polling_trigger_workflow.PollingTriggerWorkflow`
# to surface firing/waiting transitions to FE. Workflows can't broadcast
# directly (sandboxed, deterministic) so they offload to this activity.
#
# Mirrors the legacy ``services/deployment/triggers.py`` collector/processor
# transitions: ``"waiting" → "idle (processing)" → "waiting"`` per event.
# Without this, the canary path would keep the trigger node visually
# stuck on "waiting" with no firing pulse — a UX regression vs legacy.


@activity.defn
async def load_persisted_workflow_graph_activity(payload: Dict[str, Any]) -> Dict[str, Any]:
    """Resolve the current saved graph for a durable trigger firing.

    Listener workflow inputs are immutable deployment snapshots. Dynamic
    canvas operations can add tools after deployment, so every event resolves
    the authoritative graph before selecting downstream/config nodes.
    """
    from core.container import container
    from services.workflow_migrations import (
        normalize_edge_handles,
        normalize_legacy_android_toolkit,
        normalize_legacy_browser_nodes,
    )

    workflow_id = str(payload.get("workflow_id") or "")
    if not workflow_id:
        return {"found": False, "nodes": [], "edges": []}
    saved = await container.database().get_workflow(workflow_id)
    if saved is None:
        return {"found": False, "nodes": [], "edges": []}
    data = saved.data or {}
    nodes, edges, _params, _warnings = normalize_legacy_android_toolkit(
        data.get("nodes") or [], data.get("edges") or []
    )
    nodes, edges, _params, _warnings = normalize_legacy_browser_nodes(nodes, edges)
    return {
        "found": True,
        "graphVersion": int(data.get("graphVersion") or 0),
        "nodes": nodes,
        "edges": normalize_edge_handles(edges),
    }


@activity.defn
async def store_node_output_activity(payload: Dict[str, Any]) -> None:
    """Persist a node's output to the workflow output store so
    :class:`services.parameter_resolver.ParameterResolver` can resolve
    ``{{nodeName.field}}`` templates in downstream nodes.

    Used by :class:`services.temporal.workflow.MachinaWorkflow` for
    pre-executed trigger nodes — they bypass ``NodeExecutor.execute``
    (which is what normally persists outputs) because the workflow
    received their output via the parent ``TriggerListenerWorkflow`` /
    ``PollingTriggerWorkflow`` signal, not via the standard activity
    dispatch. Without this persist step, ``ParameterResolver`` reads
    ``None`` from the store and downstream templates like
    ``{{chatTrigger.message}}`` silently resolve to empty.

    Mirrors the per-handle write pattern in
    :func:`services.temporal.agent_activities.store_agent_output`.

    Payload shape::

        {
            "node_id": str,
            "session_id": str,
            "result": dict,  # the trigger output payload
        }
    """
    from core.container import container

    workflow_service = container.workflow_service()
    node_id = payload["node_id"]
    session_id = payload.get("session_id", "default")
    data = payload.get("result") or {}
    # Mirror NodeExecutor — write to every standard output handle so
    # the resolver finds the data regardless of which handle name the
    # downstream edge targets.
    for output_name in ("output_main", "output_top", "output_0"):
        try:
            await workflow_service.store_node_output(
                session_id,
                node_id,
                output_name,
                data,
            )
        except Exception as exc:  # noqa: BLE001 — non-fatal
            activity.logger.warning(f"store_node_output_activity failed for " f"node={node_id!r} handle={output_name!r}: {exc}")


@activity.defn
async def broadcast_trigger_status_activity(payload: Dict[str, Any]) -> None:
    """Broadcast a trigger node status update for FE animation.

    Payload shape:
        {
            "node_id": str,
            "status": "waiting" | "idle" | ...,
            "data": dict,             # optional, defaults to {}
            "workflow_id": str | None,
        }

    Embedded-worker mode (default): direct in-process call to the
    status_broadcaster singleton — same event loop as FastAPI handlers.
    Non-fatal: broadcast failures are logged at WARNING and swallowed
    so they don't fail the calling workflow.
    """
    from services.status_broadcaster import get_status_broadcaster

    try:
        broadcaster = get_status_broadcaster()
        await broadcaster.update_node_status(
            node_id=payload["node_id"],
            status=payload.get("status", "waiting"),
            data=payload.get("data", {}) or {},
            workflow_id=payload.get("workflow_id"),
        )
    except Exception as exc:  # noqa: BLE001 — non-fatal
        activity.logger.warning(f"broadcast_trigger_status_activity failed for " f"node={payload.get('node_id')!r}: {exc}")


@activity.defn
async def evaluate_trigger_filter_activity(payload: Dict[str, Any]) -> bool:
    """Apply a trigger node's own filter to one event.

    Payload shape:
        {
            "node_type": str,          # e.g. "webhookTrigger"
            "filter_params": dict,     # the node's saved parameters
            "event_data": dict,        # the CloudEvents ``data`` member
        }

    Why this is an activity and not inline workflow code: filter builders
    live in plugin folders (``nodes/whatsapp/_filters.py`` and friends) and
    reach imports that are not workflow-sandbox safe. Activities run outside
    the sandbox, and the boolean is recorded in history so replay stays
    deterministic.

    **The predicate takes the CloudEvents ``data`` member, not the envelope.**
    ``event_waiter.dispatch`` unpacks to ``(event_type, data)`` and hands
    filters the inner payload, so a webhook filter reads ``data["path"]``.
    Passing the envelope would make every filter silently reject.

    Fails OPEN. An unknown node type, a malformed payload or a raising
    builder all return True, which preserves the pre-fix behaviour of
    spawning the run: over-firing is visible and recoverable, whereas a
    silently dropped trigger event looks like the product is broken.
    """
    node_type = payload.get("node_type") or ""
    data = payload.get("event_data")
    if not isinstance(data, dict):
        return True

    try:
        from services import event_waiter

        predicate = event_waiter.build_filter(node_type, payload.get("filter_params") or {})
        return bool(predicate(data))
    except Exception as exc:  # noqa: BLE001 — fail open, see docstring
        activity.logger.warning(
            f"evaluate_trigger_filter_activity failed for node_type={node_type!r}; "
            f"admitting the event rather than dropping it: {exc}"
        )
        return True


@activity.defn(name="workflow_control.pause_on_failure")
async def pause_workflow_on_failure_activity(payload: Dict[str, Any]) -> Dict[str, Any]:
    """Circuit breaker: pause a controlled deployment after a failed run.

    Scheduled by MachinaWorkflow's failure path (no ``workflow.patched``
    guard: the schedule is unconditional for trigger-spawned runs). All policy
    lives in ``services.deployment.handlers.pause_generation_on_failure``
    — including the WORKFLOW_CONTROL_PAUSE_ON_FAILURE knob, evaluated on
    the activity side so flipping it never touches recorded workflow
    commands. Never raises; the result dict lands in workflow history
    for ops inspection.

    Payload shape:
        {
            "workflow_id": str,       # OpenCompany deployment workflow_id
            "reason": str,            # first node error of the failed run
        }
    """
    from services.deployment.handlers import pause_generation_on_failure

    try:
        return await pause_generation_on_failure(
            workflow_id=str(payload.get("workflow_id") or ""),
            reason=str(payload.get("reason") or "run_failed"),
            **({
                "requires_user_action": True,
                "hint": payload.get("hint"),
                "generation": payload.get("generation"),
            } if payload.get("requires_user_action") is True else {}),
        )
    except Exception as exc:  # noqa: BLE001 — non-fatal
        activity.logger.warning(f"pause_workflow_on_failure_activity failed for " f"workflow={payload.get('workflow_id')!r}: {exc}")
        return {"paused": False, "error": str(exc)}


@activity.defn(name="workflow_runs.record_completion")
async def record_run_completion_activity(payload: Dict[str, Any]) -> Dict[str, Any]:
    """Record a finished trigger-spawned run (Normal mode's "N done today").

    Scheduled at the end of MachinaWorkflow.run behind the
    ``machina-run-record-v1`` patch. Idempotent: ``run_id`` is the Temporal
    workflow id plus run id, unique per run, so a retry records nothing new.
    Never raises; a missed record only undercounts the day.

    Payload shape:
        {
            "workflow_id": str,   # OpenCompany workflow id
            "run_id": str,        # "<temporal workflow id>:<temporal run id>"
            "generation": int,
            "status": "success" | "failed",
        }
    """
    from core.container import container
    from services.employees.runs import record_run

    try:
        recorded = await record_run(
            container.database(),
            workflow_id=str(payload.get("workflow_id") or ""),
            run_id=str(payload.get("run_id") or ""),
            status=str(payload.get("status") or "failed"),
            runtime="temporal",
            generation=int(payload.get("generation") or 0),
        )
        return {"recorded": recorded}
    except Exception as exc:  # noqa: BLE001 — non-fatal
        activity.logger.warning(f"record_run_completion_activity failed for workflow={payload.get('workflow_id')!r}: {exc}")
        return {"recorded": False, "error": str(exc)}