Skip to content

Self-improving AI Employees for everyone

Refactor Temporal to distributed node execution with WebSocket-based activities

trohitg726503ce94dc
diff --git a/server/middleware/auth.py b/server/middleware/auth.py
index 13dbe908..e48b09b6 100644
--- a/server/middleware/auth.py
+++ b/server/middleware/auth.py
@@ -19,6 +19,7 @@ PUBLIC_PATHS = frozenset([
"/api/auth/login",
"/api/auth/register",
"/api/auth/logout",
+ "/ws/internal", # Internal WebSocket for Temporal workers
])
# Path prefixes that are public
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index 91d94ba1..23b3db4f 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -1720,6 +1720,90 @@ async def websocket_status_endpoint(websocket: WebSocket):
await broadcaster.disconnect(websocket)
[email protected]("/ws/internal")
+async def websocket_internal_endpoint(websocket: WebSocket):
+ """Internal WebSocket endpoint for Temporal workers.
+
+ This endpoint bypasses authentication and is intended for internal
+ service-to-service communication (e.g., Temporal activity -> MachinaOs).
+
+ Security: Should only be exposed on localhost/internal network.
+ """
+ broadcaster = get_status_broadcaster()
+ await websocket.accept()
+
+ logger.info("[WebSocket Internal] Temporal worker connected")
+
+ # Message queue for decoupling receive from processing
+ message_queue: asyncio.Queue = asyncio.Queue()
+
+ # Track handler tasks for this WebSocket
+ handler_tasks: Set[asyncio.Task] = set()
+
+ async def receive_loop():
+ """Receives messages and puts them in queue."""
+ try:
+ while True:
+ data = await websocket.receive_json()
+ await message_queue.put(data)
+ except WebSocketDisconnect:
+ logger.info("[WebSocket Internal] Client disconnected")
+ await message_queue.put(None)
+ except Exception as e:
+ logger.error(f"[WebSocket Internal] Receive error: {e}")
+ await message_queue.put(None)
+
+ async def process_loop():
+ """Processes messages from queue."""
+ while True:
+ data = await message_queue.get()
+
+ if data is None:
+ break
+
+ msg_type = data.get("type", "")
+ request_id = data.get("request_id")
+
+ handler = MESSAGE_HANDLERS.get(msg_type)
+
+ if handler:
+ task = asyncio.create_task(
+ _execute_handler(handler, data, websocket, msg_type, request_id)
+ )
+ handler_tasks.add(task)
+ task.add_done_callback(handler_tasks.discard)
+ else:
+ logger.warning(f"[WebSocket Internal] Unknown message type: {msg_type}")
+ if request_id:
+ await _safe_send(websocket, {
+ "type": "error",
+ "request_id": request_id,
+ "code": "UNKNOWN_MESSAGE_TYPE",
+ "message": f"Unknown message type: {msg_type}"
+ })
+
+ try:
+ async with asyncio.TaskGroup() as tg:
+ tg.create_task(receive_loop())
+ tg.create_task(process_loop())
+
+ except* WebSocketDisconnect:
+ logger.info("[WebSocket Internal] Client disconnected (TaskGroup)")
+ except* Exception as eg:
+ for exc in eg.exceptions:
+ if not isinstance(exc, WebSocketDisconnect):
+ logger.error(f"[WebSocket Internal] TaskGroup error: {exc}")
+ finally:
+ for task in list(handler_tasks):
+ if not task.done():
+ task.cancel()
+
+ if handler_tasks:
+ await asyncio.gather(*handler_tasks, return_exceptions=True)
+
+ logger.info("[WebSocket Internal] Connection closed")
+
+
@router.get("/ws/info")
async def websocket_info():
"""Get WebSocket connection info."""
diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.py
index f99d69c4..5135b197 100644
--- a/server/services/deployment/manager.py
+++ b/server/services/deployment/manager.py
@@ -558,6 +558,7 @@ class DeploymentManager:
logger.debug(f"[Run] filtered_edges: {len(filtered_edges)} edges")
# Execute filtered graph with deployment's workflow_id for scoped status
+ # Use Temporal for proper parallel branch execution
status_callback = self._status_callbacks.get(workflow_id)
result = await self._execute_workflow(
nodes=filtered_nodes,
@@ -566,6 +567,7 @@ class DeploymentManager:
status_callback=status_callback,
skip_clear_outputs=True,
workflow_id=workflow_id, # Pass deployment's workflow_id for status scoping
+ use_temporal=True, # Force Temporal for parallel node execution
)
result["run_id"] = run_id
diff --git a/server/services/temporal/__init__.py b/server/services/temporal/__init__.py
index 96fa5bec..9cd29db2 100644
--- a/server/services/temporal/__init__.py
+++ b/server/services/temporal/__init__.py
@@ -1,16 +1,20 @@
"""Temporal workflow orchestration service.
-This module provides Temporal integration for durable workflow execution.
-LangGraph runs inside activities (outside Temporal's sandboxed event loop)
-to use its native asyncio parallel execution.
+This module provides Temporal integration for durable distributed workflow execution.
+
+Architecture:
+- Each workflow node executes as an independent Temporal activity
+- Activities can run on ANY worker in the cluster for horizontal scaling
+- Workflow only orchestrates - schedules activities and routes outputs
+- WebSocket connection to MachinaOs for low-latency node execution
When TEMPORAL_ENABLED=true:
-- Workflows are executed via Temporal for durability
-- LangGraph handles parallel execution inside activities
-- Activities call back to MachinaOs node handlers
+- Workflows are executed via Temporal for durability and distribution
+- Each node is a separate activity with its own retry policy
+- Parallel branches execute concurrently on available workers
When TEMPORAL_ENABLED=false (default):
-- Falls back to the existing Redis-based executor
+- Falls back to the existing parallel/sequential executor
"""
from .executor import TemporalExecutor
diff --git a/server/services/temporal/activities.py b/server/services/temporal/activities.py
index 37e4c81b..e2423f0d 100644
--- a/server/services/temporal/activities.py
+++ b/server/services/temporal/activities.py
@@ -1,23 +1,22 @@
-"""Temporal activities with continuous scheduling orchestration.
+"""Temporal activities for distributed node execution.
-Uses asyncio.wait(FIRST_COMPLETED) pattern for immediate downstream execution
-when dependencies complete - no waiting for sibling branches.
+Uses class-based activity pattern recommended by Temporal docs for sharing
+resources like aiohttp.ClientSession across activity invocations.
-This replaces LangGraph's Pregel/BSP model which uses supersteps where all
-parallel nodes must complete before downstream nodes can start.
+References:
+- https://docs.temporal.io/develop/python/python-sdk-sync-vs-async
+- https://docs.temporal.io/develop/python/core-application
-Pattern borrowed from services/execution/executor.py WorkflowExecutor.
-
-Activities call back to MachinaOs node handlers via HTTP and broadcast
-status updates via the StatusBroadcaster for real-time UI updates.
+Architecture:
+- NodeExecutionActivities class holds shared aiohttp.ClientSession
+- Session is passed via constructor, avoiding recreation per activity
+- Each activity call gets its own WebSocket connection from the session pool
"""
-import asyncio
-import httpx
-from collections import defaultdict
from datetime import datetime
-from typing import Any, Dict, List, Set, Optional
+from typing import Any, Dict, Optional
+import aiohttp
from temporalio import activity
from core.logging import get_logger
@@ -28,515 +27,312 @@ logger = get_logger(__name__)
# Load settings to get the correct server port
_settings = Settings()
MACHINA_URL = f"http://{_settings.host}:{_settings.port}"
+WS_URL = f"ws://{_settings.host}:{_settings.port}/ws/internal"
print(f"[Temporal Activities] MACHINA_URL configured: {MACHINA_URL}")
+print(f"[Temporal Activities] WS_URL configured: {WS_URL}")
-# =============================================================================
-# STATUS BROADCASTING
-# =============================================================================
+class NodeExecutionActivities:
+ """Activity class for node execution with shared aiohttp session.
-async def broadcast_node_status(
- node_id: str,
- status: str,
- data: dict = None,
- workflow_id: str = None,
-) -> None:
- """Broadcast node status update via HTTP to StatusBroadcaster.
+ Following Temporal's recommended pattern for dependency injection:
+ - aiohttp.ClientSession is passed via constructor
+ - Session provides connection pooling for concurrent activities
+ - Each activity call gets its own WebSocket connection from the pool
- This allows the Temporal activity to send real-time status updates
- to connected WebSocket clients.
+ Reference: https://docs.temporal.io/develop/python/python-sdk-sync-vs-async
"""
- url = f"{MACHINA_URL}/api/workflow/broadcast-status"
- payload = {
- "node_id": node_id,
- "status": status,
- "data": data or {},
- "workflow_id": workflow_id,
- }
-
- async with httpx.AsyncClient(timeout=5.0) as client:
- try:
- await client.post(url, json=payload)
- print(f"[Temporal] Broadcast status: {node_id} -> {status}")
- except Exception as e:
- # Don't fail execution if broadcast fails
- print(f"[Temporal] Broadcast failed (non-fatal): {str(e)}")
-
-# =============================================================================
-# NODE EXECUTION VIA HTTP
-# =============================================================================
-
-async def execute_node_via_http(
- node_id: str,
- node_type: str,
- data: dict,
- context: dict,
-) -> dict:
- """Execute a node by calling MachinaOs HTTP endpoint.
-
- This allows the activity to leverage all existing node handlers
- without duplicating code.
- """
- url = f"{MACHINA_URL}/api/workflow/node/execute"
- print(f"[Temporal] Executing node {node_id} ({node_type}) via {url}")
+ def __init__(self, session: aiohttp.ClientSession):
+ """Initialize with shared aiohttp session.
+
+ Args:
+ session: aiohttp.ClientSession with connection pooling configured
+ """
+ self.session = session
+ self.ws_url = WS_URL
+ self.http_url = f"{MACHINA_URL}/api/workflow/node/execute"
+ self.broadcast_url = f"{MACHINA_URL}/api/workflow/broadcast-status"
+
+ @activity.defn
+ async def execute_node_activity(self, context: Dict[str, Any]) -> Dict[str, Any]:
+ """Execute a single workflow node with isolated context.
+
+ This activity can run on ANY worker in the cluster, enabling
+ horizontal scaling and multi-tenant distribution.
+
+ Each node execution is independent - if it fails, Temporal will retry
+ on the same or different worker without affecting other nodes.
+
+ Args:
+ context: Immutable context containing:
+ - node_id: Unique node identifier
+ - node_type: Type of node (aiAgent, console, timer, etc.)
+ - node_data: Node configuration from React Flow
+ - inputs: Outputs from upstream nodes (dependencies)
+ - workflow_id: Parent workflow ID for tracking
+ - tenant_id: Tenant identifier for multi-tenancy
+ - session_id: Session identifier
+ - nodes: Full node list (for tool/memory detection by handlers)
+ - edges: Full edge list (for tool/memory detection by handlers)
+
+ Returns:
+ Dict with success, result, node_id, and metadata
+ """
+ node_id = context["node_id"]
+ node_type = context["node_type"]
+ node_data = context.get("node_data", {})
+ workflow_id = context.get("workflow_id")
+ tenant_id = context.get("tenant_id")
+
+ activity.logger.info(
+ f"Executing node activity: {node_id} ({node_type})",
+ extra={"tenant_id": tenant_id, "workflow_id": workflow_id},
+ )
+ print(f"[Activity] Starting node: {node_id} (type={node_type})")
- async with httpx.AsyncClient(timeout=120.0) as client:
- try:
- response = await client.post(
- url,
- json={
- "node_id": node_id,
- "node_type": node_type,
- "data": data,
- "context": context,
- },
- )
- response.raise_for_status()
- result = response.json()
- print(f"[Temporal] Node {node_id} completed: success={result.get('success', 'N/A')}")
- return result
- except httpx.HTTPStatusError as e:
- print(f"[Temporal] HTTP ERROR {e.response.status_code} for node {node_id}: {e.response.text}")
- logger.error(
- f"Node execution HTTP error: {e.response.status_code}",
- node_id=node_id,
- node_type=node_type,
- )
- return {
- "success": False,
+ # Handle pre-executed trigger nodes (already have their output)
+ if context.get("pre_executed"):
+ print(f"[Activity] Node {node_id} is pre-executed, returning cached result")
+ result = {
+ "success": True,
"node_id": node_id,
"node_type": node_type,
- "error": f"HTTP {e.response.status_code}: {e.response.text}",
+ "result": context.get("trigger_output", {}),
+ "pre_executed": True,
+ "timestamp": datetime.now().isoformat(),
}
- except Exception as e:
- print(f"[Temporal] EXCEPTION for node {node_id}: {str(e)}")
- logger.error(
- f"Node execution failed: {str(e)}",
- node_id=node_id,
- node_type=node_type,
- )
- return {
- "success": False,
+ await self._broadcast_status(node_id, "success", result, workflow_id)
+ return result
+
+ # Handle disabled nodes
+ if node_data.get("disabled"):
+ print(f"[Activity] Node {node_id} is disabled, skipping")
+ result = {
+ "success": True,
"node_id": node_id,
"node_type": node_type,
- "error": str(e),
+ "skipped": True,
+ "reason": "disabled",
+ "timestamp": datetime.now().isoformat(),
}
+ await self._broadcast_status(node_id, "skipped", {"disabled": True}, workflow_id)
+ return result
+ # Broadcast "executing" status for UI updates
+ await self._broadcast_status(
+ node_id=node_id,
+ status="executing",
+ data={"node_type": node_type},
+ workflow_id=workflow_id,
+ )
-# =============================================================================
-# GRAPH UTILITIES (from WorkflowExecutor pattern)
-# =============================================================================
+ try:
+ # Execute node via WebSocket (each call gets own connection from pool)
+ result = await self._execute_via_websocket(context)
-def build_dependency_maps(
- nodes: List[Dict],
- edges: List[Dict],
-) -> tuple[Dict[str, Set[str]], Dict[str, Set[str]], Dict[str, Dict]]:
- """Build dependency graph from nodes and edges.
+ # Add metadata
+ result["node_id"] = node_id
+ result["node_type"] = node_type
+ result["timestamp"] = datetime.now().isoformat()
- Returns:
- Tuple of (dependencies, dependents, node_map)
- - dependencies: node_id -> set of nodes it depends on (must complete first)
- - dependents: node_id -> set of nodes that depend on it (to schedule next)
- - node_map: node_id -> node definition
- """
- node_map = {n["id"]: n for n in nodes}
- node_ids = set(node_map.keys())
+ # Broadcast result status
+ if result.get("success"):
+ await self._broadcast_status(
+ node_id=node_id,
+ status="success",
+ data={
+ "result": result.get("result"),
+ "execution_time": result.get("execution_time"),
+ },
+ workflow_id=workflow_id,
+ )
+ print(f"[Activity] Node {node_id} completed successfully")
+ else:
+ await self._broadcast_status(
+ node_id=node_id,
+ status="error",
+ data={"error": result.get("error")},
+ workflow_id=workflow_id,
+ )
+ print(f"[Activity] Node {node_id} failed: {result.get('error')}")
+
+ # Heartbeat for activity liveness
+ activity.heartbeat(f"Node {node_id} completed")
- # dependencies[node_id] = set of nodes that must complete before this node
- dependencies: Dict[str, Set[str]] = {node_id: set() for node_id in node_ids}
+ return result
- # dependents[node_id] = set of nodes that depend on this node
- dependents: Dict[str, Set[str]] = {node_id: set() for node_id in node_ids}
+ except Exception as e:
+ error_msg = f"{type(e).__name__}: {str(e)}"
+ logger.error(f"Node {node_id} execution failed: {error_msg}")
+ print(f"[Activity] Node {node_id} EXCEPTION: {error_msg}")
- for edge in edges:
- source = edge.get("source")
- target = edge.get("target")
- if source in node_ids and target in node_ids:
- dependencies[target].add(source)
- dependents[source].add(target)
+ # Broadcast error status
+ await self._broadcast_status(
+ node_id=node_id,
+ status="error",
+ data={"error": error_msg},
+ workflow_id=workflow_id,
+ )
- return dependencies, dependents, node_map
+ # Raise to trigger Temporal retry mechanism
+ raise
+ async def _execute_via_websocket(self, context: Dict[str, Any]) -> Dict[str, Any]:
+ """Execute node via WebSocket using shared session's connection pool.
-def find_ready_nodes(
- dependencies: Dict[str, Set[str]],
- completed: Set[str],
- running: Set[str],
- all_nodes: Set[str],
-) -> List[str]:
- """Find nodes ready to execute (all dependencies met, not running/completed).
+ Each call creates a new WebSocket connection from the session's pool,
+ avoiding race conditions when multiple activities run concurrently.
+ """
+ import json
+ import uuid
- Args:
- dependencies: node_id -> set of dependencies
- completed: Set of completed node IDs
- running: Set of currently running node IDs
- all_nodes: Set of all node IDs
+ node_id = context["node_id"]
+ node_type = context["node_type"]
+ request_id = str(uuid.uuid4())
- Returns:
- List of node IDs ready to execute
- """
- ready = []
- for node_id in all_nodes:
- if node_id in completed or node_id in running:
- continue
- # Check if all dependencies are completed
- if dependencies[node_id] <= completed:
- ready.append(node_id)
- return ready
+ message = {
+ "type": "execute_node",
+ "request_id": request_id,
+ "node_id": node_id,
+ "node_type": node_type,
+ "parameters": context.get("node_data", {}),
+ "nodes": context.get("nodes", []),
+ "edges": context.get("edges", []),
+ "session_id": context.get("session_id", "default"),
+ "workflow_id": context.get("workflow_id"),
+ }
+
+ print(f"[Activity] WebSocket execute for {node_id}")
+
+ try:
+ # Each activity gets its own WebSocket connection from the pool
+ async with self.session.ws_connect(
+ self.ws_url,
+ heartbeat=20,
+ receive_timeout=120,
+ ) as ws:
+ await ws.send_json(message)
+ print(f"[Activity] Sent request for {node_id}")
+
+ # Wait for response with matching request_id
+ async for msg in ws:
+ if msg.type == aiohttp.WSMsgType.TEXT:
+ response = json.loads(msg.data)
+ if response.get("request_id") == request_id:
+ print(f"[Activity] Got response for {node_id}: success={response.get('success')}")
+ return response
+ elif msg.type == aiohttp.WSMsgType.ERROR:
+ raise Exception(f"WebSocket error: {ws.exception()}")
+ elif msg.type == aiohttp.WSMsgType.CLOSED:
+ raise Exception("WebSocket closed unexpectedly")
+
+ raise Exception(f"No response for request {request_id}")
+
+ except aiohttp.ClientError as e:
+ raise Exception(f"WebSocket connection error: {e}")
+
+ async def _broadcast_status(
+ self,
+ node_id: str,
+ status: str,
+ data: dict,
+ workflow_id: str = None,
+ ) -> None:
+ """Broadcast node status for real-time UI updates.
+
+ Non-fatal - execution continues even if broadcast fails.
+ """
+ try:
+ async with self.session.post(
+ self.broadcast_url,
+ json={
+ "node_id": node_id,
+ "status": status,
+ "data": data or {},
+ "workflow_id": workflow_id,
+ },
+ timeout=aiohttp.ClientTimeout(total=5),
+ ) as response:
+ if response.status == 200:
+ print(f"[Activity] Broadcast: {node_id} -> {status}")
+ except Exception as e:
+ # Non-fatal - don't fail execution if broadcast fails
+ logger.warning(f"Broadcast failed for {node_id}: {e}")
+ print(f"[Activity] Broadcast failed (non-fatal): {e}")
# =============================================================================
-# SINGLE NODE EXECUTION
+# Factory function for creating activity instance with session
# =============================================================================
-async def execute_single_node(
- node: Dict,
- nodes: List[Dict],
- edges: List[Dict],
- outputs: Dict[str, Any],
- session_id: str,
- workflow_id: Optional[str],
-) -> Dict[str, Any]:
- """Execute a single node, handling pre-executed triggers and disabled nodes.
+def create_node_activities(session: aiohttp.ClientSession) -> NodeExecutionActivities:
+ """Factory function to create activity instance with shared session.
+
+ This follows Temporal's recommended pattern for dependency injection.
+ The session should be created once when the worker starts and reused.
Args:
- node: Node definition
- nodes: All nodes in workflow
- edges: All edges in workflow
- outputs: Current outputs from completed nodes
- session_id: Session identifier
- workflow_id: Workflow ID for tracking
+ session: aiohttp.ClientSession with connection pooling
Returns:
- Execution result dict
+ NodeExecutionActivities instance ready for worker registration
"""
- node_id = node["id"]
- node_type = node.get("type", "unknown")
-
- print(f"[Temporal] Processing node: {node_id} (type={node_type}, _pre_executed={node.get('_pre_executed')})")
-
- # Check if this is a pre-executed trigger node
- if node.get("_pre_executed"):
- print(f"[Temporal] Skipping pre-executed trigger node: {node_id}")
- trigger_output = node.get("_trigger_output", {})
- result = {
- "success": True,
- "node_id": node_id,
- "node_type": node_type,
- "result": trigger_output,
- "skipped": True,
- "reason": "pre-executed trigger",
- "timestamp": datetime.now().isoformat(),
- }
+ return NodeExecutionActivities(session)
- await broadcast_node_status(
- node_id=node_id,
- status="success",
- data={"result": trigger_output, "pre_executed": True},
- workflow_id=workflow_id,
- )
-
- return result
- # Check if node is disabled
- if node.get("data", {}).get("disabled"):
- print(f"[Temporal] Skipping disabled node: {node_id}")
- result = {
- "success": True,
- "node_id": node_id,
- "node_type": node_type,
- "skipped": True,
- "reason": "disabled",
- "timestamp": datetime.now().isoformat(),
- }
+async def create_shared_session(pool_size: int = 100) -> aiohttp.ClientSession:
+ """Create a shared aiohttp session with connection pooling.
- await broadcast_node_status(
- node_id=node_id,
- status="skipped",
- data={"disabled": True},
- workflow_id=workflow_id,
- )
+ Args:
+ pool_size: Maximum number of concurrent connections
- return result
-
- # Build context for node execution
- context = {
- "outputs": outputs,
- "nodes": nodes,
- "edges": edges,
- "session_id": session_id,
- "workflow_id": workflow_id,
- }
-
- # Broadcast "executing" status BEFORE execution
- await broadcast_node_status(
- node_id=node_id,
- status="executing",
- data={"node_type": node_type},
- workflow_id=workflow_id,
+ Returns:
+ Configured aiohttp.ClientSession
+ """
+ connector = aiohttp.TCPConnector(
+ limit=pool_size,
+ limit_per_host=pool_size,
+ enable_cleanup_closed=True,
)
-
- # Execute node via MachinaOs
- result = await execute_node_via_http(
- node_id=node_id,
- node_type=node_type,
- data={}, # Let server load from DB
- context=context,
+ timeout = aiohttp.ClientTimeout(
+ total=300, # 5 min total
+ connect=10, # 10 sec connect
)
-
- # Add timestamp
- result["timestamp"] = datetime.now().isoformat()
-
- # Broadcast result status AFTER execution
- if result.get("success"):
- await broadcast_node_status(
- node_id=node_id,
- status="success",
- data={"result": result.get("result"), "execution_time": result.get("execution_time")},
- workflow_id=workflow_id,
- )
- else:
- await broadcast_node_status(
- node_id=node_id,
- status="error",
- data={"error": result.get("error")},
- workflow_id=workflow_id,
- )
-
- return result
+ session = aiohttp.ClientSession(
+ connector=connector,
+ timeout=timeout,
+ )
+ print(f"[Activities] Created shared session with pool_size={pool_size}")
+ return session
# =============================================================================
-# CONTINUOUS SCHEDULING ACTIVITY
+# Standalone activity function (for backwards compatibility)
# =============================================================================
-async def execute_workflow_with_continuous_scheduling(
- nodes: List[Dict],
- edges: List[Dict],
- session_id: str = "default",
- workflow_id: Optional[str] = None,
-) -> Dict[str, Any]:
- """Execute workflow with continuous scheduling (FIRST_COMPLETED pattern).
-
- When any node completes, immediately check for and start newly-ready
- dependent nodes. This enables true parallel pipelines where each path
- progresses independently without waiting for sibling branches.
-
- Pattern from WorkflowExecutor._execute_with_continuous_scheduling
-
- Args:
- nodes: List of node definitions (may include _pre_executed trigger nodes)
- edges: List of edge definitions
- session_id: Session identifier
- workflow_id: Workflow ID for tracking
-
- Returns:
- Dict with success, outputs, and execution_trace
- """
- activity.logger.info(
- f"Executing workflow with {len(nodes)} nodes, {len(edges)} edges (continuous scheduling)"
- )
- print(f"[Temporal] Starting continuous scheduling execution")
-
- try:
- # Build dependency graph
- dependencies, dependents, node_map = build_dependency_maps(nodes, edges)
- all_node_ids = set(node_map.keys())
-
- # Track execution state
- outputs: Dict[str, Any] = {}
- execution_trace: List[str] = []
- completed: Set[str] = set()
- running: Set[str] = set()
- errors: List[Dict] = []
-
- # Collect pre-executed trigger outputs
- for node in nodes:
- if node.get("_pre_executed") and node.get("_trigger_output"):
- node_id = node["id"]
- outputs[node_id] = {
- "success": True,
- "node_id": node_id,
- "node_type": node.get("type", "unknown"),
- "result": node["_trigger_output"],
- "pre_executed": True,
- }
- completed.add(node_id)
- execution_trace.append(node_id)
- print(f"[Temporal] Pre-loaded trigger output: {node_id}")
-
- # Broadcast success for pre-executed trigger
- await broadcast_node_status(
- node_id=node_id,
- status="success",
- data={"result": node["_trigger_output"], "pre_executed": True},
- workflow_id=workflow_id,
- )
+# Global session for standalone function
+_global_session: Optional[aiohttp.ClientSession] = None
+_global_activities: Optional[NodeExecutionActivities] = None
- # Find initial ready nodes
- ready_node_ids = find_ready_nodes(dependencies, completed, running, all_node_ids)
- if not ready_node_ids:
- print(f"[Temporal] No ready nodes found, workflow complete or stuck")
- return {
- "success": len(completed) == len(all_node_ids),
- "outputs": outputs,
- "execution_trace": execution_trace,
- }
+async def _get_global_activities() -> NodeExecutionActivities:
+ """Get or create global activities instance."""
+ global _global_session, _global_activities
- print(f"[Temporal] Initial ready nodes: {ready_node_ids}")
-
- # Track running tasks: task -> node_id
- task_to_node_id: Dict[asyncio.Task, str] = {}
- pending_tasks: Set[asyncio.Task] = set()
- workflow_failed = False
-
- def create_node_task(node_id: str) -> asyncio.Task:
- """Create and track a task for node execution."""
- node = node_map[node_id]
- running.add(node_id)
-
- task = asyncio.create_task(
- execute_single_node(
- node=node,
- nodes=nodes,
- edges=edges,
- outputs=outputs,
- session_id=session_id,
- workflow_id=workflow_id,
- ),
- name=f"node_{node_id}"
- )
- task_to_node_id[task] = node_id
- pending_tasks.add(task)
- return task
-
- # Start initial ready nodes
- for node_id in ready_node_ids:
- create_node_task(node_id)
- print(f"[Temporal] Scheduled initial node: {node_id}")
-
- # Send heartbeat
- activity.heartbeat(f"Started {len(ready_node_ids)} initial nodes")
-
- # Continuous scheduling loop
- while pending_tasks and not workflow_failed:
- # Wait for ANY task to complete (FIRST_COMPLETED pattern)
- done, pending_tasks = await asyncio.wait(
- pending_tasks,
- return_when=asyncio.FIRST_COMPLETED
- )
+ if _global_session is None or _global_session.closed:
+ _global_session = await create_shared_session()
+ _global_activities = NodeExecutionActivities(_global_session)
- # Process each completed task
- for task in done:
- node_id = task_to_node_id[task]
- running.discard(node_id)
-
- try:
- result = task.result()
-
- if result.get("success"):
- # Success - store output and find newly ready nodes
- outputs[node_id] = result
- completed.add(node_id)
- execution_trace.append(node_id)
-
- print(f"[Temporal] Node {node_id} completed successfully")
- activity.heartbeat(f"Completed: {node_id}")
-
- # Find nodes that are now ready (their dependencies just completed)
- newly_ready = find_ready_nodes(dependencies, completed, running, all_node_ids)
-
- # Schedule newly ready nodes immediately
- for ready_id in newly_ready:
- create_node_task(ready_id)
- print(f"[Temporal] Scheduled dependent: {ready_id} (triggered by {node_id})")
-
- else:
- # Node failed
- error_msg = result.get("error", "Unknown error")
- outputs[node_id] = result
- errors.append({
- "node_id": node_id,
- "error": error_msg,
- "timestamp": datetime.now().isoformat(),
- })
- print(f"[Temporal] Node {node_id} failed: {error_msg}")
- workflow_failed = True
-
- except asyncio.CancelledError:
- print(f"[Temporal] Node {node_id} was cancelled")
- outputs[node_id] = {
- "success": False,
- "error": "Cancelled",
- "node_id": node_id,
- }
-
- except Exception as e:
- error_msg = str(e)
- print(f"[Temporal] Node {node_id} exception: {error_msg}")
- outputs[node_id] = {
- "success": False,
- "error": error_msg,
- "node_id": node_id,
- }
- errors.append({
- "node_id": node_id,
- "error": error_msg,
- "timestamp": datetime.now().isoformat(),
- })
- workflow_failed = True
-
- # Handle workflow failure - cancel remaining tasks
- if workflow_failed and pending_tasks:
- print(f"[Temporal] Workflow failed, cancelling {len(pending_tasks)} remaining tasks")
-
- for task in pending_tasks:
- task.cancel()
-
- # Wait for cancelled tasks
- if pending_tasks:
- cancelled_done, _ = await asyncio.wait(
- pending_tasks,
- return_when=asyncio.ALL_COMPLETED
- )
+ return _global_activities
- for task in cancelled_done:
- node_id = task_to_node_id.get(task)
- if node_id and node_id not in outputs:
- outputs[node_id] = {
- "success": False,
- "error": "Cancelled due to workflow failure",
- "node_id": node_id,
- }
-
- # Final heartbeat
- activity.heartbeat(f"Workflow complete: {len(execution_trace)} nodes executed")
-
- success = not workflow_failed and len(completed) == len(all_node_ids)
- print(f"[Temporal] Workflow finished: success={success}, executed={len(execution_trace)}/{len(all_node_ids)}")
-
- return {
- "success": success,
- "outputs": outputs,
- "execution_trace": execution_trace,
- "errors": errors if errors else None,
- }
- except Exception as e:
- import traceback
- tb = traceback.format_exc()
- error_details = f"{type(e).__name__}: {str(e)}"
- activity.logger.error(f"Workflow failed: {error_details}\n{tb}")
- print(f"[Temporal] Workflow exception: {error_details}")
- print(f"[Temporal] Traceback:\n{tb}")
- return {
- "success": False,
- "error": error_details,
- "outputs": {},
- "execution_trace": [],
- }
+async def execute_node_activity(context: Dict[str, Any]) -> Dict[str, Any]:
+ """Standalone activity function for backwards compatibility.
+ For new code, use NodeExecutionActivities class with shared session.
+ """
+ activities = await _get_global_activities()
+ return await activities.execute_node_activity(context)
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index 8d5f03dd..2958c38c 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -1,40 +1,66 @@
-"""Temporal worker for MachinaOs.
+"""Temporal worker for distributed node execution.
-The worker polls the task queue and executes workflows and activities.
-It can be run as a standalone process or embedded in the main server.
+Uses class-based activities with shared aiohttp session for proper
+connection pooling across concurrent activity executions.
+
+The worker polls the task queue and executes:
+- MachinaWorkflow: Orchestrates the graph, schedules node activities
+- NodeExecutionActivities: Executes individual nodes with shared session
+
+Multiple workers can be started on different machines for horizontal scaling.
+Each node activity can execute on any available worker in the cluster.
+
+References:
+- https://docs.temporal.io/develop/python/python-sdk-sync-vs-async
+- https://docs.temporal.io/develop/worker-performance
"""
import asyncio
from typing import Optional
+import aiohttp
from temporalio.client import Client
from temporalio.worker import Worker
from core.logging import get_logger
from .workflow import MachinaWorkflow
-from .activities import execute_workflow_with_continuous_scheduling
+print(f"[Worker Import] MachinaWorkflow loaded from: {MachinaWorkflow.__module__}")
+from .activities import (
+ NodeExecutionActivities,
+ create_shared_session,
+ execute_node_activity,
+)
logger = get_logger(__name__)
class TemporalWorkerManager:
- """Manages the Temporal worker lifecycle."""
+ """Manages the Temporal worker lifecycle with shared resources.
+
+ Creates a shared aiohttp.ClientSession that is passed to the activity
+ class, following Temporal's recommended dependency injection pattern.
+ """
def __init__(
self,
client: Client,
task_queue: str = "machina-tasks",
+ pool_size: int = 100,
):
"""Initialize the worker manager.
Args:
client: Connected Temporal client
task_queue: Task queue name to poll
+ pool_size: Connection pool size for aiohttp session
"""
self.client = client
self.task_queue = task_queue
+ self.pool_size = pool_size
self._worker: Optional[Worker] = None
self._worker_task: Optional[asyncio.Task] = None
+ self._session: Optional[aiohttp.ClientSession] = None
+ self._activities: Optional[NodeExecutionActivities] = None
@property
def is_running(self) -> bool:
@@ -47,17 +73,30 @@ class TemporalWorkerManager:
logger.warning("Temporal worker already running")
return
+ # Create shared aiohttp session with connection pooling
+ self._session = await create_shared_session(self.pool_size)
+
+ # Create activity instance with shared session
+ self._activities = NodeExecutionActivities(self._session)
+
+ # Create worker with class-based activity
+ # For class-based activities, pass the bound method (instance.method)
self._worker = Worker(
self.client,
task_queue=self.task_queue,
workflows=[MachinaWorkflow],
- activities=[execute_workflow_with_continuous_scheduling],
+ activities=[self._activities.execute_node_activity], # Pass bound method
+ # Allow concurrent activity execution for parallel branches
+ max_concurrent_activities=self.pool_size,
+ max_concurrent_workflow_tasks=10,
)
logger.info(
"Starting Temporal worker",
task_queue=self.task_queue,
+ pool_size=self.pool_size,
)
+ print(f"[Worker] Starting with pool_size={self.pool_size}")
# Run worker in background task
self._worker_task = asyncio.create_task(
@@ -76,7 +115,7 @@ class TemporalWorkerManager:
raise
async def stop(self) -> None:
- """Stop the Temporal worker."""
+ """Stop the Temporal worker and cleanup resources."""
if not self.is_running:
return
@@ -90,7 +129,14 @@ class TemporalWorkerManager:
pass
self._worker_task = None
+ # Close shared session
+ if self._session and not self._session.closed:
+ await self._session.close()
+ print("[Worker] Closed shared session")
+
self._worker = None
+ self._session = None
+ self._activities = None
logger.info("Temporal worker stopped")
@@ -98,15 +144,22 @@ async def run_standalone_worker(
server_address: str = "localhost:7233",
namespace: str = "default",
task_queue: str = "machina-tasks",
+ pool_size: int = 100,
) -> None:
"""Run the Temporal worker as a standalone process.
- This can be used for running workers separately from the main server.
+ This can be used for running workers separately from the main server,
+ enabling horizontal scaling across multiple machines.
+
+ Example:
+ # Start multiple workers for horizontal scaling
+ python -m services.temporal.worker
Args:
server_address: Temporal server address
namespace: Temporal namespace
task_queue: Task queue to poll
+ pool_size: Connection pool size
"""
logger.info(
"Starting standalone Temporal worker",
@@ -115,19 +168,69 @@ async def run_standalone_worker(
task_queue=task_queue,
)
+ print(f"[Worker] Connecting to {server_address}")
+ print(f"[Worker] Namespace: {namespace}")
+ print(f"[Worker] Task Queue: {task_queue}")
+ print(f"[Worker] Pool Size: {pool_size}")
+
client = await Client.connect(server_address, namespace=namespace)
- worker = Worker(
+ # Create shared session and activities
+ session = await create_shared_session(pool_size)
+ activities = NodeExecutionActivities(session)
+
+ try:
+ worker = Worker(
+ client,
+ task_queue=task_queue,
+ workflows=[MachinaWorkflow],
+ activities=[activities.execute_node_activity], # Pass bound method
+ max_concurrent_activities=pool_size,
+ max_concurrent_workflow_tasks=10,
+ )
+
+ print("[Worker] Running. Press Ctrl+C to stop.")
+ logger.info("Worker running. Press Ctrl+C to stop.")
+ await worker.run()
+
+ finally:
+ # Cleanup session on shutdown
+ if not session.closed:
+ await session.close()
+ print("[Worker] Session closed")
+
+
+async def create_worker(
+ client: Client,
+ task_queue: str = "machina-tasks",
+ session: Optional[aiohttp.ClientSession] = None,
+) -> Worker:
+ """Create a worker instance for use in tests or custom setups.
+
+ Args:
+ client: Connected Temporal client
+ task_queue: Task queue name
+ session: Optional shared aiohttp session (created if not provided)
+
+ Returns:
+ Configured Worker instance (not started)
+ """
+ if session is None:
+ session = await create_shared_session()
+
+ activities = NodeExecutionActivities(session)
+
+ return Worker(
client,
task_queue=task_queue,
workflows=[MachinaWorkflow],
- activities=[execute_workflow_with_continuous_scheduling],
+ activities=[activities.execute_node_activity], # Pass bound method
+ max_concurrent_activities=100,
+ max_concurrent_workflow_tasks=10,
)
- logger.info("Worker running. Press Ctrl+C to stop.")
- await worker.run()
-
if __name__ == "__main__":
# Allow running worker standalone
+ # Usage: python -m services.temporal.worker
asyncio.run(run_standalone_worker())
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index 4bf27c6c..1f0009fc 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -1,52 +1,72 @@
-"""Temporal workflow definition for MachinaOs.
+"""Temporal workflow - Pure orchestrator for distributed node execution.
-The workflow is a thin wrapper that delegates execution to a continuous
-scheduling activity. Uses asyncio.wait(FIRST_COMPLETED) pattern for immediate
-downstream execution when dependencies complete - no waiting for sibling branches.
+The workflow ONLY orchestrates:
+- Parses graph structure
+- Filters config nodes (tools, memory, services)
+- Determines execution order based on dependencies
+- Schedules node activities (can run on ANY worker)
+- Collects results and routes outputs to dependent nodes
-This replaces LangGraph's Pregel/BSP model which uses supersteps where all
-parallel nodes must complete before downstream nodes can start.
+NO business logic in workflow - all execution happens in activities.
+This enables massive horizontal scaling and multi-tenant distribution.
"""
from datetime import timedelta
-from typing import Any, Dict
+from typing import Any, Dict, List, Set
from temporalio import workflow
from temporalio.common import RetryPolicy
-# Import activities through the unsafe imports mechanism
-# This is required because Temporal sandboxes workflow imports
-with workflow.unsafe.imports_passed_through():
- from .activities import execute_workflow_with_continuous_scheduling
+# Config handles - nodes connecting via these are config nodes (not executed)
+CONFIG_HANDLES = {"input-tools", "input-memory", "input-model"}
+# Android service types (connect to androidTool, not executed directly)
+ANDROID_SERVICE_TYPES = {
+ "batteryMonitor", "locationService", "deviceState",
+ "systemInfo", "appList", "appLauncher",
+}
@workflow.defn(sandboxed=False)
class MachinaWorkflow:
- """Temporal workflow with continuous scheduling orchestration.
+ """Distributed workflow orchestrator.
- The actual graph execution happens in an activity using the
- asyncio.wait(FIRST_COMPLETED) pattern for true parallel pipelines
- where each path progresses independently.
+ This workflow ONLY orchestrates - all execution happens in activities
+ that can run on any worker in the cluster.
+
+ Features:
+ - Continuous scheduling (FIRST_COMPLETED pattern)
+ - Per-node retry policies
+ - Config node filtering (tools, memory, services)
+ - Multi-tenant support via tenant_id in context
"""
@workflow.run
async def run(self, workflow_data: Dict[str, Any]) -> Dict[str, Any]:
- """Execute workflow via continuous scheduling activity.
+ print("[Workflow] ========== RUN METHOD CALLED ==========")
+ """Execute workflow by orchestrating node activities.
Args:
workflow_data: Dict containing:
- - nodes: List of node definitions
- - edges: List of edge definitions
+ - nodes: List of node definitions from React Flow
+ - edges: List of edge definitions from React Flow
- session_id: Session identifier
- workflow_id: Workflow ID for tracking
+ - tenant_id: Tenant identifier for multi-tenancy
Returns:
- Dict with success, outputs, and execution_trace
+ Dict with success, outputs, execution_trace, and errors
"""
nodes = workflow_data.get("nodes", [])
edges = workflow_data.get("edges", [])
session_id = workflow_data.get("session_id", "default")
workflow_id = workflow_data.get("workflow_id")
+ tenant_id = workflow_data.get("tenant_id")
+
+ workflow.logger.info(
+ f"Starting workflow orchestration: {len(nodes)} nodes, {len(edges)} edges"
+ )
+ # Debug print for console visibility
+ print(f"[Workflow] Starting: {len(nodes)} nodes, {len(edges)} edges")
if not nodes:
return {
@@ -56,16 +76,268 @@ class MachinaWorkflow:
"execution_trace": [],
}
- # Execute the entire workflow in a single activity
- # Continuous scheduling handles parallel execution inside the activity
- return await workflow.execute_activity(
- execute_workflow_with_continuous_scheduling,
- args=[nodes, edges, session_id, workflow_id],
- start_to_close_timeout=timedelta(minutes=30),
- heartbeat_timeout=timedelta(minutes=5),
- retry_policy=RetryPolicy(
- initial_interval=timedelta(seconds=1),
- maximum_interval=timedelta(seconds=60),
- maximum_attempts=3,
- ),
+ # 1. Filter out config nodes (tools, memory, services)
+ exec_nodes, exec_edges = self._filter_executable_graph(nodes, edges)
+
+ workflow.logger.info(
+ f"After filtering: {len(exec_nodes)} executable nodes "
+ f"(filtered {len(nodes) - len(exec_nodes)} config nodes)"
+ )
+ print(f"[Workflow] Executable: {len(exec_nodes)}, Config filtered: {len(nodes) - len(exec_nodes)}")
+
+ # 2. Build dependency maps
+ deps, node_map = self._build_dependency_maps(exec_nodes, exec_edges)
+
+ # 3. Initialize state
+ outputs: Dict[str, Any] = {} # node_id -> result
+ completed: Set[str] = set()
+ running: Dict[str, Any] = {} # node_id -> activity handle
+ errors: List[Dict] = []
+ execution_trace: List[str] = []
+
+ # 4. Handle pre-executed triggers (already have their output)
+ pre_executed_count = 0
+ for node in exec_nodes:
+ if node.get("_pre_executed"):
+ node_id = node["id"]
+ outputs[node_id] = {
+ "success": True,
+ "result": node.get("_trigger_output", {}),
+ "pre_executed": True,
+ }
+ completed.add(node_id)
+ execution_trace.append(node_id)
+ pre_executed_count += 1
+ workflow.logger.info(f"Pre-executed trigger: {node_id}")
+
+ workflow.logger.info(f"Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")
+ print(f"[Workflow] Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")
+
+ # 5. Retry policy for node activities
+ retry_policy = RetryPolicy(
+ initial_interval=timedelta(seconds=1),
+ maximum_interval=timedelta(seconds=30),
+ maximum_attempts=3,
)
+
+ # 6. Continuous scheduling loop
+ loop_count = 0
+ while True:
+ loop_count += 1
+ # Find ready nodes (all deps completed, not running/completed)
+ ready = self._find_ready_nodes(deps, completed, running, node_map)
+ workflow.logger.info(f"Loop {loop_count}: ready={len(ready)}, running={len(running)}, completed={len(completed)}")
+ print(f"[Workflow] Loop {loop_count}: ready={len(ready)}, running={len(running)}, completed={len(completed)}")
+
+ # Start activities for ready nodes
+ for node_id in ready:
+ node = node_map[node_id]
+
+ # Build immutable context for this node
+ context = {
+ "node_id": node_id,
+ "node_type": node.get("type", "unknown"),
+ "node_data": node.get("data", {}),
+ "inputs": self._get_node_inputs(node_id, deps, outputs),
+ "workflow_id": workflow_id,
+ "tenant_id": tenant_id,
+ "session_id": session_id,
+ "nodes": nodes, # Full list for tool/memory detection
+ "edges": edges, # Full list for tool/memory detection
+ # Include pre-executed info if applicable
+ "pre_executed": node.get("_pre_executed", False),
+ "trigger_output": node.get("_trigger_output"),
+ }
+
+ # Schedule activity by name (for class-based activities)
+ # The activity is registered as NodeExecutionActivities.execute_node_activity
+ handle = workflow.start_activity(
+ "execute_node_activity",
+ args=[context],
+ start_to_close_timeout=timedelta(minutes=10),
+ heartbeat_timeout=timedelta(minutes=2),
+ retry_policy=retry_policy,
+ )
+ running[node_id] = handle
+
+ workflow.logger.info(f"Scheduled activity for node: {node_id}")
+ print(f"[Workflow] Scheduled: {node_id}")
+
+ # Exit if nothing running and nothing ready
+ if not running:
+ break
+
+ # Wait for ANY activity to complete (FIRST_COMPLETED pattern)
+ done_id, result = await self._wait_any_complete(running)
+
+ if result.get("success"):
+ outputs[done_id] = result
+ completed.add(done_id)
+ execution_trace.append(done_id)
+ workflow.logger.info(f"Node completed: {done_id}")
+ else:
+ # Node failed after all retries
+ error_info = {
+ "node_id": done_id,
+ "error": result.get("error", "Unknown error"),
+ }
+ errors.append(error_info)
+ workflow.logger.error(f"Node failed: {done_id} - {error_info['error']}")
+
+ # Stop workflow on failure
+ # TODO: Could add option to continue with partial results
+ break
+
+ # Build final result
+ success = len(errors) == 0 and len(completed) == len(node_map)
+
+ workflow.logger.info(
+ f"Workflow complete: success={success}, "
+ f"executed={len(execution_trace)}/{len(node_map)}"
+ )
+ print(f"[Workflow] Complete: success={success}, executed={len(execution_trace)}/{len(node_map)}")
+
+ return {
+ "success": success,
+ "outputs": outputs,
+ "execution_trace": execution_trace,
+ "errors": errors if errors else None,
+ }
+
+ def _get_node_inputs(
+ self,
+ node_id: str,
+ deps: Dict[str, Set[str]],
+ outputs: Dict[str, Any],
+ ) -> Dict[str, Any]:
+ """Get outputs from upstream nodes as inputs for this node."""
+ inputs = {}
+ for dep_id in deps.get(node_id, set()):
+ if dep_id in outputs:
+ inputs[dep_id] = outputs[dep_id].get("result", {})
+ return inputs
+
+ async def _wait_any_complete(self, running: Dict[str, Any]) -> tuple:
+ """Wait for any activity to complete, return (node_id, result).
+
+ Uses Temporal's native wait mechanism for efficient polling.
+ """
+ # Convert to list for iteration
+ items = list(running.items())
+
+ # Check if any already done
+ for node_id, handle in items:
+ if handle.done():
+ del running[node_id]
+ try:
+ result = await handle
+ return node_id, result
+ except Exception as e:
+ return node_id, {"success": False, "error": str(e)}
+
+ # Wait for first completion using Temporal's wait
+ handles = [h for _, h in items]
+
+ # Use asyncio.wait pattern via workflow.wait
+ await workflow.wait_condition(
+ lambda: any(h.done() for _, h in items)
+ )
+
+ # Find the completed one
+ for node_id, handle in items:
+ if handle.done():
+ del running[node_id]
+ try:
+ result = await handle
+ return node_id, result
+ except Exception as e:
+ return node_id, {"success": False, "error": str(e)}
+
+ # Should not reach here
+ raise RuntimeError("No activity completed after wait")
+
+ def _filter_executable_graph(
+ self,
+ nodes: List[Dict],
+ edges: List[Dict],
+ ) -> tuple:
+ """Filter out config nodes based on edge handles.
+
+ Config nodes (tools, memory, model configs) connect via special handles
+ and are consumed by their target nodes, not executed independently.
+
+ Returns:
+ Tuple of (executable_nodes, executable_edges)
+ """
+ node_map = {n["id"]: n for n in nodes}
+ config_ids = set()
+
+ for edge in edges:
+ handle = edge.get("targetHandle", "")
+ source_id = edge.get("source")
+
+ # Edges to config handles mean source is a config node
+ if handle in CONFIG_HANDLES:
+ config_ids.add(source_id)
+
+ # Android services connecting to androidTool
+ source_node = node_map.get(source_id, {})
+ if source_node.get("type") in ANDROID_SERVICE_TYPES:
+ config_ids.add(source_id)
+
+ # Filter nodes and edges
+ exec_nodes = [n for n in nodes if n["id"] not in config_ids]
+ exec_edges = [
+ e for e in edges
+ if e.get("source") not in config_ids
+ and e.get("target") not in config_ids
+ and e.get("targetHandle", "") not in CONFIG_HANDLES
+ ]
+
+ return exec_nodes, exec_edges
+
+ def _build_dependency_maps(
+ self,
+ nodes: List[Dict],
+ edges: List[Dict],
+ ) -> tuple:
+ """Build dependency graph from nodes and edges.
+
+ Returns:
+ Tuple of (dependencies_map, node_map)
+ - dependencies_map: node_id -> set of node IDs it depends on
+ - node_map: node_id -> node definition
+ """
+ node_map = {n["id"]: n for n in nodes}
+ node_ids = set(node_map.keys())
+
+ deps = {nid: set() for nid in node_ids}
+
+ for edge in edges:
+ src, tgt = edge.get("source"), edge.get("target")
+ if src in node_ids and tgt in node_ids:
+ deps[tgt].add(src)
+
+ return deps, node_map
+
+ def _find_ready_nodes(
+ self,
+ deps: Dict[str, Set[str]],
+ completed: Set[str],
+ running: Dict[str, Any],
+ node_map: Dict[str, Dict],
+ ) -> List[str]:
+ """Find nodes ready to execute.
+
+ A node is ready when:
+ - All its dependencies have completed
+ - It's not already running
+ - It's not already completed
+ """
+ ready = []
+ for node_id in node_map:
+ if node_id in completed or node_id in running:
+ continue
+ if deps[node_id] <= completed:
+ ready.append(node_id)
+ return ready
diff --git a/server/services/temporal/ws_client.py b/server/services/temporal/ws_client.py
new file mode 100644
index 00000000..35caa1ff
--- /dev/null
+++ b/server/services/temporal/ws_client.py
@@ -0,0 +1,236 @@
+"""WebSocket client for Temporal activities.
+
+Uses aiohttp ClientSession for proper connection pooling and concurrent
+request handling. Each activity creates its own WebSocket connection
+to avoid race conditions, but the session manages connection reuse.
+
+References:
+- https://docs.aiohttp.org/en/stable/client_quickstart.html
+- https://websockets.readthedocs.io/en/stable/reference/asyncio/client.html
+"""
+
+import asyncio
+import json
+import uuid
+from typing import Any, Dict, Optional
+from contextlib import asynccontextmanager
+
+import aiohttp
+
+from core.logging import get_logger
+from core.config import Settings
+
+logger = get_logger(__name__)
+
+# Load settings
+_settings = Settings()
+WS_URL = f"ws://{_settings.host}:{_settings.port}/ws/internal"
+
+print(f"[Temporal WS Client] WebSocket URL configured: {WS_URL}")
+
+
+class WSConnectionPool:
+ """WebSocket connection pool using aiohttp ClientSession.
+
+ aiohttp ClientSession provides built-in connection pooling with:
+ - Connection reuse and keep-alive
+ - Configurable connection limits
+ - Proper concurrent request handling
+
+ Each execute_node call gets its own WebSocket connection from the pool,
+ avoiding the ConcurrencyError that occurs when sharing a single connection.
+ """
+
+ def __init__(self, url: str = WS_URL, pool_size: int = 100):
+ self.url = url
+ self.pool_size = pool_size
+ self._session: Optional[aiohttp.ClientSession] = None
+ self._lock = asyncio.Lock()
+
+ async def _get_session(self) -> aiohttp.ClientSession:
+ """Get or create the aiohttp session with connection pooling."""
+ if self._session is None or self._session.closed:
+ async with self._lock:
+ if self._session is None or self._session.closed:
+ connector = aiohttp.TCPConnector(
+ limit=self.pool_size,
+ limit_per_host=self.pool_size,
+ enable_cleanup_closed=True,
+ )
+ timeout = aiohttp.ClientTimeout(
+ total=300, # 5 min total timeout
+ connect=10, # 10 sec connect timeout
+ )
+ self._session = aiohttp.ClientSession(
+ connector=connector,
+ timeout=timeout,
+ )
+ print(f"[WS Pool] Created session with pool_size={self.pool_size}")
+ return self._session
+
+ @asynccontextmanager
+ async def connection(self):
+ """Get a WebSocket connection from the pool.
+
+ Usage:
+ async with pool.connection() as ws:
+ await ws.send_json(request)
+ response = await ws.receive_json()
+ """
+ session = await self._get_session()
+ async with session.ws_connect(
+ self.url,
+ heartbeat=20,
+ receive_timeout=120,
+ ) as ws:
+ yield ws
+
+ async def execute_node(
+ self,
+ node_id: str,
+ node_type: str,
+ data: Dict[str, Any],
+ context: Dict[str, Any],
+ timeout: float = 120.0,
+ ) -> Dict[str, Any]:
+ """Execute a node via WebSocket.
+
+ Each call gets its own connection from the pool, allowing
+ concurrent execution without race conditions.
+ """
+ request_id = str(uuid.uuid4())
+
+ message = {
+ "type": "execute_node",
+ "request_id": request_id,
+ "node_id": node_id,
+ "node_type": node_type,
+ "parameters": data,
+ "nodes": context.get("nodes", []),
+ "edges": context.get("edges", []),
+ "session_id": context.get("session_id", "default"),
+ "workflow_id": context.get("workflow_id"),
+ }
+
+ try:
+ async with self.connection() as ws:
+ await ws.send_json(message)
+ print(f"[WS Pool] Sent execute_node for {node_id}")
+
+ # Wait for response with matching request_id
+ async with asyncio.timeout(timeout):
+ async for msg in ws:
+ if msg.type == aiohttp.WSMsgType.TEXT:
+ response = json.loads(msg.data)
+ if response.get("request_id") == request_id:
+ print(f"[WS Pool] Received response for {node_id}: success={response.get('success')}")
+ return response
+ elif msg.type == aiohttp.WSMsgType.ERROR:
+ raise Exception(f"WebSocket error: {ws.exception()}")
+ elif msg.type == aiohttp.WSMsgType.CLOSED:
+ raise Exception("WebSocket closed unexpectedly")
+
+ raise Exception(f"No response received for request {request_id}")
+
+ except asyncio.TimeoutError:
+ raise Exception(f"WebSocket request timeout ({timeout}s) for node {node_id}")
+ except aiohttp.ClientError as e:
+ raise Exception(f"WebSocket connection error: {e}")
+
+ async def close(self):
+ """Close the connection pool."""
+ if self._session and not self._session.closed:
+ await self._session.close()
+ print("[WS Pool] Session closed")
+
+
+# Global connection pool instance
+_pool: Optional[WSConnectionPool] = None
+_pool_lock = asyncio.Lock()
+
+
+async def get_ws_pool() -> WSConnectionPool:
+ """Get or create the global WebSocket connection pool."""
+ global _pool
+
+ async with _pool_lock:
+ if _pool is None:
+ _pool = WSConnectionPool()
+ return _pool
+
+
+async def execute_node_ws(
+ node_id: str,
+ node_type: str,
+ data: Dict[str, Any],
+ context: Dict[str, Any],
+ timeout: float = 120.0,
+) -> Dict[str, Any]:
+ """Execute a node via WebSocket using the connection pool.
+
+ This is the recommended way to execute nodes from activities.
+ Each call gets its own connection from the pool.
+ """
+ pool = await get_ws_pool()
+ return await pool.execute_node(
+ node_id=node_id,
+ node_type=node_type,
+ data=data,
+ context=context,
+ timeout=timeout,
+ )
+
+
+async def close_ws_pool() -> None:
+ """Close the global WebSocket connection pool."""
+ global _pool
+
+ async with _pool_lock:
+ if _pool:
+ await _pool.close()
+ _pool = None
+
+
+# Backwards compatibility aliases
+async def get_ws_client() -> WSConnectionPool:
+ """Alias for get_ws_pool() for backwards compatibility."""
+ return await get_ws_pool()
+
+
+async def close_ws_client() -> None:
+ """Alias for close_ws_pool() for backwards compatibility."""
+ await close_ws_pool()
+
+
+# For backwards compatibility with old code that expects TemporalWSClient
+class TemporalWSClient:
+ """Backwards compatibility wrapper around WSConnectionPool."""
+
+ def __init__(self, url: str = WS_URL):
+ self._pool = WSConnectionPool(url=url)
+
+ @property
+ def connected(self) -> bool:
+ return True # Pool manages connections
+
+ async def connect(self) -> None:
+ pass # Pool connects on demand
+
+ async def disconnect(self) -> None:
+ await self._pool.close()
+
+ async def execute_node(
+ self,
+ node_id: str,
+ node_type: str,
+ data: Dict[str, Any],
+ context: Dict[str, Any],
+ timeout: float = 120.0,
+ ) -> Dict[str, Any]:
+ return await self._pool.execute_node(
+ node_id=node_id,
+ node_type=node_type,
+ data=data,
+ context=context,
+ timeout=timeout,
+ )
diff --git a/server/services/workflow.py b/server/services/workflow.py
index 6f2f0de1..e69245a5 100644
--- a/server/services/workflow.py
+++ b/server/services/workflow.py
@@ -232,6 +232,14 @@ class WorkflowService:
if use_temporal and self._temporal_executor is not None:
return await self._execute_temporal(nodes, edges, session_id, status_callback, start_time, workflow_id)
+ # Log warning if Temporal was requested but not available
+ if use_temporal and self._temporal_executor is None:
+ logger.warning(
+ "Temporal execution requested but executor not configured. "
+ "Falling back to parallel/sequential execution. "
+ "Check TEMPORAL_ENABLED and Temporal server connection."
+ )
+
# Use parallel executor if enabled and Redis available
if use_parallel and self.settings.redis_enabled:
return await self._execute_parallel(nodes, edges, session_id, status_callback, start_time, workflow_id)