Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
Refactor Temporal to distributed node execution with WebSocket-based activities
trohitg726503ce94dc
diff --git a/server/middleware/auth.py b/server/middleware/auth.pyindex 13dbe908..e48b09b6 100644--- a/server/middleware/auth.py+++ b/server/middleware/auth.py@@ -19,6 +19,7 @@ PUBLIC_PATHS = frozenset(["/api/auth/login","/api/auth/register","/api/auth/logout",+ "/ws/internal", # Internal WebSocket for Temporal workers])# Path prefixes that are publicdiff --git a/server/routers/websocket.py b/server/routers/websocket.pyindex 91d94ba1..23b3db4f 100644--- a/server/routers/websocket.py+++ b/server/routers/websocket.py@@ -1720,6 +1720,90 @@ async def websocket_status_endpoint(websocket: WebSocket):await broadcaster.disconnect(websocket)[email protected]("/ws/internal")+async def websocket_internal_endpoint(websocket: WebSocket):+ """Internal WebSocket endpoint for Temporal workers.++ This endpoint bypasses authentication and is intended for internal+ service-to-service communication (e.g., Temporal activity -> MachinaOs).++ Security: Should only be exposed on localhost/internal network.+ """+ broadcaster = get_status_broadcaster()+ await websocket.accept()++ logger.info("[WebSocket Internal] Temporal worker connected")++ # Message queue for decoupling receive from processing+ message_queue: asyncio.Queue = asyncio.Queue()++ # Track handler tasks for this WebSocket+ handler_tasks: Set[asyncio.Task] = set()++ async def receive_loop():+ """Receives messages and puts them in queue."""+ try:+ while True:+ data = await websocket.receive_json()+ await message_queue.put(data)+ except WebSocketDisconnect:+ logger.info("[WebSocket Internal] Client disconnected")+ await message_queue.put(None)+ except Exception as e:+ logger.error(f"[WebSocket Internal] Receive error: {e}")+ await message_queue.put(None)++ async def process_loop():+ """Processes messages from queue."""+ while True:+ data = await message_queue.get()++ if data is None:+ break++ msg_type = data.get("type", "")+ request_id = data.get("request_id")++ handler = MESSAGE_HANDLERS.get(msg_type)++ if handler:+ task = asyncio.create_task(+ _execute_handler(handler, data, websocket, msg_type, request_id)+ )+ handler_tasks.add(task)+ task.add_done_callback(handler_tasks.discard)+ else:+ logger.warning(f"[WebSocket Internal] Unknown message type: {msg_type}")+ if request_id:+ await _safe_send(websocket, {+ "type": "error",+ "request_id": request_id,+ "code": "UNKNOWN_MESSAGE_TYPE",+ "message": f"Unknown message type: {msg_type}"+ })++ try:+ async with asyncio.TaskGroup() as tg:+ tg.create_task(receive_loop())+ tg.create_task(process_loop())++ except* WebSocketDisconnect:+ logger.info("[WebSocket Internal] Client disconnected (TaskGroup)")+ except* Exception as eg:+ for exc in eg.exceptions:+ if not isinstance(exc, WebSocketDisconnect):+ logger.error(f"[WebSocket Internal] TaskGroup error: {exc}")+ finally:+ for task in list(handler_tasks):+ if not task.done():+ task.cancel()++ if handler_tasks:+ await asyncio.gather(*handler_tasks, return_exceptions=True)++ logger.info("[WebSocket Internal] Connection closed")++@router.get("/ws/info")async def websocket_info():"""Get WebSocket connection info."""diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.pyindex f99d69c4..5135b197 100644--- a/server/services/deployment/manager.py+++ b/server/services/deployment/manager.py@@ -558,6 +558,7 @@ class DeploymentManager:logger.debug(f"[Run] filtered_edges: {len(filtered_edges)} edges")# Execute filtered graph with deployment's workflow_id for scoped status+ # Use Temporal for proper parallel branch executionstatus_callback = self._status_callbacks.get(workflow_id)result = await self._execute_workflow(nodes=filtered_nodes,@@ -566,6 +567,7 @@ class DeploymentManager:status_callback=status_callback,skip_clear_outputs=True,workflow_id=workflow_id, # Pass deployment's workflow_id for status scoping+ use_temporal=True, # Force Temporal for parallel node execution)result["run_id"] = run_iddiff --git a/server/services/temporal/__init__.py b/server/services/temporal/__init__.pyindex 96fa5bec..9cd29db2 100644--- a/server/services/temporal/__init__.py+++ b/server/services/temporal/__init__.py@@ -1,16 +1,20 @@"""Temporal workflow orchestration service.-This module provides Temporal integration for durable workflow execution.-LangGraph runs inside activities (outside Temporal's sandboxed event loop)-to use its native asyncio parallel execution.+This module provides Temporal integration for durable distributed workflow execution.++Architecture:+- Each workflow node executes as an independent Temporal activity+- Activities can run on ANY worker in the cluster for horizontal scaling+- Workflow only orchestrates - schedules activities and routes outputs+- WebSocket connection to MachinaOs for low-latency node executionWhen TEMPORAL_ENABLED=true:-- Workflows are executed via Temporal for durability-- LangGraph handles parallel execution inside activities-- Activities call back to MachinaOs node handlers+- Workflows are executed via Temporal for durability and distribution+- Each node is a separate activity with its own retry policy+- Parallel branches execute concurrently on available workersWhen TEMPORAL_ENABLED=false (default):-- Falls back to the existing Redis-based executor+- Falls back to the existing parallel/sequential executor"""from .executor import TemporalExecutordiff --git a/server/services/temporal/activities.py b/server/services/temporal/activities.pyindex 37e4c81b..e2423f0d 100644--- a/server/services/temporal/activities.py+++ b/server/services/temporal/activities.py@@ -1,23 +1,22 @@-"""Temporal activities with continuous scheduling orchestration.+"""Temporal activities for distributed node execution.-Uses asyncio.wait(FIRST_COMPLETED) pattern for immediate downstream execution-when dependencies complete - no waiting for sibling branches.+Uses class-based activity pattern recommended by Temporal docs for sharing+resources like aiohttp.ClientSession across activity invocations.-This replaces LangGraph's Pregel/BSP model which uses supersteps where all-parallel nodes must complete before downstream nodes can start.+References:+- https://docs.temporal.io/develop/python/python-sdk-sync-vs-async+- https://docs.temporal.io/develop/python/core-application-Pattern borrowed from services/execution/executor.py WorkflowExecutor.--Activities call back to MachinaOs node handlers via HTTP and broadcast-status updates via the StatusBroadcaster for real-time UI updates.+Architecture:+- NodeExecutionActivities class holds shared aiohttp.ClientSession+- Session is passed via constructor, avoiding recreation per activity+- Each activity call gets its own WebSocket connection from the session pool"""-import asyncio-import httpx-from collections import defaultdictfrom datetime import datetime-from typing import Any, Dict, List, Set, Optional+from typing import Any, Dict, Optional+import aiohttpfrom temporalio import activityfrom core.logging import get_logger@@ -28,515 +27,312 @@ logger = get_logger(__name__)# Load settings to get the correct server port_settings = Settings()MACHINA_URL = f"http://{_settings.host}:{_settings.port}"+WS_URL = f"ws://{_settings.host}:{_settings.port}/ws/internal"print(f"[Temporal Activities] MACHINA_URL configured: {MACHINA_URL}")+print(f"[Temporal Activities] WS_URL configured: {WS_URL}")-# =============================================================================-# STATUS BROADCASTING-# =============================================================================+class NodeExecutionActivities:+ """Activity class for node execution with shared aiohttp session.-async def broadcast_node_status(- node_id: str,- status: str,- data: dict = None,- workflow_id: str = None,-) -> None:- """Broadcast node status update via HTTP to StatusBroadcaster.+ Following Temporal's recommended pattern for dependency injection:+ - aiohttp.ClientSession is passed via constructor+ - Session provides connection pooling for concurrent activities+ - Each activity call gets its own WebSocket connection from the pool- This allows the Temporal activity to send real-time status updates- to connected WebSocket clients.+ Reference: https://docs.temporal.io/develop/python/python-sdk-sync-vs-async"""- url = f"{MACHINA_URL}/api/workflow/broadcast-status"- payload = {- "node_id": node_id,- "status": status,- "data": data or {},- "workflow_id": workflow_id,- }-- async with httpx.AsyncClient(timeout=5.0) as client:- try:- await client.post(url, json=payload)- print(f"[Temporal] Broadcast status: {node_id} -> {status}")- except Exception as e:- # Don't fail execution if broadcast fails- print(f"[Temporal] Broadcast failed (non-fatal): {str(e)}")--# =============================================================================-# NODE EXECUTION VIA HTTP-# =============================================================================--async def execute_node_via_http(- node_id: str,- node_type: str,- data: dict,- context: dict,-) -> dict:- """Execute a node by calling MachinaOs HTTP endpoint.-- This allows the activity to leverage all existing node handlers- without duplicating code.- """- url = f"{MACHINA_URL}/api/workflow/node/execute"- print(f"[Temporal] Executing node {node_id} ({node_type}) via {url}")+ def __init__(self, session: aiohttp.ClientSession):+ """Initialize with shared aiohttp session.++ Args:+ session: aiohttp.ClientSession with connection pooling configured+ """+ self.session = session+ self.ws_url = WS_URL+ self.http_url = f"{MACHINA_URL}/api/workflow/node/execute"+ self.broadcast_url = f"{MACHINA_URL}/api/workflow/broadcast-status"++ @activity.defn+ async def execute_node_activity(self, context: Dict[str, Any]) -> Dict[str, Any]:+ """Execute a single workflow node with isolated context.++ This activity can run on ANY worker in the cluster, enabling+ horizontal scaling and multi-tenant distribution.++ Each node execution is independent - if it fails, Temporal will retry+ on the same or different worker without affecting other nodes.++ Args:+ context: Immutable context containing:+ - node_id: Unique node identifier+ - node_type: Type of node (aiAgent, console, timer, etc.)+ - node_data: Node configuration from React Flow+ - inputs: Outputs from upstream nodes (dependencies)+ - workflow_id: Parent workflow ID for tracking+ - tenant_id: Tenant identifier for multi-tenancy+ - session_id: Session identifier+ - nodes: Full node list (for tool/memory detection by handlers)+ - edges: Full edge list (for tool/memory detection by handlers)++ Returns:+ Dict with success, result, node_id, and metadata+ """+ node_id = context["node_id"]+ node_type = context["node_type"]+ node_data = context.get("node_data", {})+ workflow_id = context.get("workflow_id")+ tenant_id = context.get("tenant_id")++ activity.logger.info(+ f"Executing node activity: {node_id} ({node_type})",+ extra={"tenant_id": tenant_id, "workflow_id": workflow_id},+ )+ print(f"[Activity] Starting node: {node_id} (type={node_type})")- async with httpx.AsyncClient(timeout=120.0) as client:- try:- response = await client.post(- url,- json={- "node_id": node_id,- "node_type": node_type,- "data": data,- "context": context,- },- )- response.raise_for_status()- result = response.json()- print(f"[Temporal] Node {node_id} completed: success={result.get('success', 'N/A')}")- return result- except httpx.HTTPStatusError as e:- print(f"[Temporal] HTTP ERROR {e.response.status_code} for node {node_id}: {e.response.text}")- logger.error(- f"Node execution HTTP error: {e.response.status_code}",- node_id=node_id,- node_type=node_type,- )- return {- "success": False,+ # Handle pre-executed trigger nodes (already have their output)+ if context.get("pre_executed"):+ print(f"[Activity] Node {node_id} is pre-executed, returning cached result")+ result = {+ "success": True,"node_id": node_id,"node_type": node_type,- "error": f"HTTP {e.response.status_code}: {e.response.text}",+ "result": context.get("trigger_output", {}),+ "pre_executed": True,+ "timestamp": datetime.now().isoformat(),}- except Exception as e:- print(f"[Temporal] EXCEPTION for node {node_id}: {str(e)}")- logger.error(- f"Node execution failed: {str(e)}",- node_id=node_id,- node_type=node_type,- )- return {- "success": False,+ await self._broadcast_status(node_id, "success", result, workflow_id)+ return result++ # Handle disabled nodes+ if node_data.get("disabled"):+ print(f"[Activity] Node {node_id} is disabled, skipping")+ result = {+ "success": True,"node_id": node_id,"node_type": node_type,- "error": str(e),+ "skipped": True,+ "reason": "disabled",+ "timestamp": datetime.now().isoformat(),}+ await self._broadcast_status(node_id, "skipped", {"disabled": True}, workflow_id)+ return result+ # Broadcast "executing" status for UI updates+ await self._broadcast_status(+ node_id=node_id,+ status="executing",+ data={"node_type": node_type},+ workflow_id=workflow_id,+ )-# =============================================================================-# GRAPH UTILITIES (from WorkflowExecutor pattern)-# =============================================================================+ try:+ # Execute node via WebSocket (each call gets own connection from pool)+ result = await self._execute_via_websocket(context)-def build_dependency_maps(- nodes: List[Dict],- edges: List[Dict],-) -> tuple[Dict[str, Set[str]], Dict[str, Set[str]], Dict[str, Dict]]:- """Build dependency graph from nodes and edges.+ # Add metadata+ result["node_id"] = node_id+ result["node_type"] = node_type+ result["timestamp"] = datetime.now().isoformat()- Returns:- Tuple of (dependencies, dependents, node_map)- - dependencies: node_id -> set of nodes it depends on (must complete first)- - dependents: node_id -> set of nodes that depend on it (to schedule next)- - node_map: node_id -> node definition- """- node_map = {n["id"]: n for n in nodes}- node_ids = set(node_map.keys())+ # Broadcast result status+ if result.get("success"):+ await self._broadcast_status(+ node_id=node_id,+ status="success",+ data={+ "result": result.get("result"),+ "execution_time": result.get("execution_time"),+ },+ workflow_id=workflow_id,+ )+ print(f"[Activity] Node {node_id} completed successfully")+ else:+ await self._broadcast_status(+ node_id=node_id,+ status="error",+ data={"error": result.get("error")},+ workflow_id=workflow_id,+ )+ print(f"[Activity] Node {node_id} failed: {result.get('error')}")++ # Heartbeat for activity liveness+ activity.heartbeat(f"Node {node_id} completed")- # dependencies[node_id] = set of nodes that must complete before this node- dependencies: Dict[str, Set[str]] = {node_id: set() for node_id in node_ids}+ return result- # dependents[node_id] = set of nodes that depend on this node- dependents: Dict[str, Set[str]] = {node_id: set() for node_id in node_ids}+ except Exception as e:+ error_msg = f"{type(e).__name__}: {str(e)}"+ logger.error(f"Node {node_id} execution failed: {error_msg}")+ print(f"[Activity] Node {node_id} EXCEPTION: {error_msg}")- for edge in edges:- source = edge.get("source")- target = edge.get("target")- if source in node_ids and target in node_ids:- dependencies[target].add(source)- dependents[source].add(target)+ # Broadcast error status+ await self._broadcast_status(+ node_id=node_id,+ status="error",+ data={"error": error_msg},+ workflow_id=workflow_id,+ )- return dependencies, dependents, node_map+ # Raise to trigger Temporal retry mechanism+ raise+ async def _execute_via_websocket(self, context: Dict[str, Any]) -> Dict[str, Any]:+ """Execute node via WebSocket using shared session's connection pool.-def find_ready_nodes(- dependencies: Dict[str, Set[str]],- completed: Set[str],- running: Set[str],- all_nodes: Set[str],-) -> List[str]:- """Find nodes ready to execute (all dependencies met, not running/completed).+ Each call creates a new WebSocket connection from the session's pool,+ avoiding race conditions when multiple activities run concurrently.+ """+ import json+ import uuid- Args:- dependencies: node_id -> set of dependencies- completed: Set of completed node IDs- running: Set of currently running node IDs- all_nodes: Set of all node IDs+ node_id = context["node_id"]+ node_type = context["node_type"]+ request_id = str(uuid.uuid4())- Returns:- List of node IDs ready to execute- """- ready = []- for node_id in all_nodes:- if node_id in completed or node_id in running:- continue- # Check if all dependencies are completed- if dependencies[node_id] <= completed:- ready.append(node_id)- return ready+ message = {+ "type": "execute_node",+ "request_id": request_id,+ "node_id": node_id,+ "node_type": node_type,+ "parameters": context.get("node_data", {}),+ "nodes": context.get("nodes", []),+ "edges": context.get("edges", []),+ "session_id": context.get("session_id", "default"),+ "workflow_id": context.get("workflow_id"),+ }++ print(f"[Activity] WebSocket execute for {node_id}")++ try:+ # Each activity gets its own WebSocket connection from the pool+ async with self.session.ws_connect(+ self.ws_url,+ heartbeat=20,+ receive_timeout=120,+ ) as ws:+ await ws.send_json(message)+ print(f"[Activity] Sent request for {node_id}")++ # Wait for response with matching request_id+ async for msg in ws:+ if msg.type == aiohttp.WSMsgType.TEXT:+ response = json.loads(msg.data)+ if response.get("request_id") == request_id:+ print(f"[Activity] Got response for {node_id}: success={response.get('success')}")+ return response+ elif msg.type == aiohttp.WSMsgType.ERROR:+ raise Exception(f"WebSocket error: {ws.exception()}")+ elif msg.type == aiohttp.WSMsgType.CLOSED:+ raise Exception("WebSocket closed unexpectedly")++ raise Exception(f"No response for request {request_id}")++ except aiohttp.ClientError as e:+ raise Exception(f"WebSocket connection error: {e}")++ async def _broadcast_status(+ self,+ node_id: str,+ status: str,+ data: dict,+ workflow_id: str = None,+ ) -> None:+ """Broadcast node status for real-time UI updates.++ Non-fatal - execution continues even if broadcast fails.+ """+ try:+ async with self.session.post(+ self.broadcast_url,+ json={+ "node_id": node_id,+ "status": status,+ "data": data or {},+ "workflow_id": workflow_id,+ },+ timeout=aiohttp.ClientTimeout(total=5),+ ) as response:+ if response.status == 200:+ print(f"[Activity] Broadcast: {node_id} -> {status}")+ except Exception as e:+ # Non-fatal - don't fail execution if broadcast fails+ logger.warning(f"Broadcast failed for {node_id}: {e}")+ print(f"[Activity] Broadcast failed (non-fatal): {e}")# =============================================================================-# SINGLE NODE EXECUTION+# Factory function for creating activity instance with session# =============================================================================-async def execute_single_node(- node: Dict,- nodes: List[Dict],- edges: List[Dict],- outputs: Dict[str, Any],- session_id: str,- workflow_id: Optional[str],-) -> Dict[str, Any]:- """Execute a single node, handling pre-executed triggers and disabled nodes.+def create_node_activities(session: aiohttp.ClientSession) -> NodeExecutionActivities:+ """Factory function to create activity instance with shared session.++ This follows Temporal's recommended pattern for dependency injection.+ The session should be created once when the worker starts and reused.Args:- node: Node definition- nodes: All nodes in workflow- edges: All edges in workflow- outputs: Current outputs from completed nodes- session_id: Session identifier- workflow_id: Workflow ID for tracking+ session: aiohttp.ClientSession with connection poolingReturns:- Execution result dict+ NodeExecutionActivities instance ready for worker registration"""- node_id = node["id"]- node_type = node.get("type", "unknown")-- print(f"[Temporal] Processing node: {node_id} (type={node_type}, _pre_executed={node.get('_pre_executed')})")-- # Check if this is a pre-executed trigger node- if node.get("_pre_executed"):- print(f"[Temporal] Skipping pre-executed trigger node: {node_id}")- trigger_output = node.get("_trigger_output", {})- result = {- "success": True,- "node_id": node_id,- "node_type": node_type,- "result": trigger_output,- "skipped": True,- "reason": "pre-executed trigger",- "timestamp": datetime.now().isoformat(),- }+ return NodeExecutionActivities(session)- await broadcast_node_status(- node_id=node_id,- status="success",- data={"result": trigger_output, "pre_executed": True},- workflow_id=workflow_id,- )-- return result- # Check if node is disabled- if node.get("data", {}).get("disabled"):- print(f"[Temporal] Skipping disabled node: {node_id}")- result = {- "success": True,- "node_id": node_id,- "node_type": node_type,- "skipped": True,- "reason": "disabled",- "timestamp": datetime.now().isoformat(),- }+async def create_shared_session(pool_size: int = 100) -> aiohttp.ClientSession:+ """Create a shared aiohttp session with connection pooling.- await broadcast_node_status(- node_id=node_id,- status="skipped",- data={"disabled": True},- workflow_id=workflow_id,- )+ Args:+ pool_size: Maximum number of concurrent connections- return result-- # Build context for node execution- context = {- "outputs": outputs,- "nodes": nodes,- "edges": edges,- "session_id": session_id,- "workflow_id": workflow_id,- }-- # Broadcast "executing" status BEFORE execution- await broadcast_node_status(- node_id=node_id,- status="executing",- data={"node_type": node_type},- workflow_id=workflow_id,+ Returns:+ Configured aiohttp.ClientSession+ """+ connector = aiohttp.TCPConnector(+ limit=pool_size,+ limit_per_host=pool_size,+ enable_cleanup_closed=True,)-- # Execute node via MachinaOs- result = await execute_node_via_http(- node_id=node_id,- node_type=node_type,- data={}, # Let server load from DB- context=context,+ timeout = aiohttp.ClientTimeout(+ total=300, # 5 min total+ connect=10, # 10 sec connect)-- # Add timestamp- result["timestamp"] = datetime.now().isoformat()-- # Broadcast result status AFTER execution- if result.get("success"):- await broadcast_node_status(- node_id=node_id,- status="success",- data={"result": result.get("result"), "execution_time": result.get("execution_time")},- workflow_id=workflow_id,- )- else:- await broadcast_node_status(- node_id=node_id,- status="error",- data={"error": result.get("error")},- workflow_id=workflow_id,- )-- return result+ session = aiohttp.ClientSession(+ connector=connector,+ timeout=timeout,+ )+ print(f"[Activities] Created shared session with pool_size={pool_size}")+ return session# =============================================================================-# CONTINUOUS SCHEDULING ACTIVITY+# Standalone activity function (for backwards compatibility)# =============================================================================-async def execute_workflow_with_continuous_scheduling(- nodes: List[Dict],- edges: List[Dict],- session_id: str = "default",- workflow_id: Optional[str] = None,-) -> Dict[str, Any]:- """Execute workflow with continuous scheduling (FIRST_COMPLETED pattern).-- When any node completes, immediately check for and start newly-ready- dependent nodes. This enables true parallel pipelines where each path- progresses independently without waiting for sibling branches.-- Pattern from WorkflowExecutor._execute_with_continuous_scheduling-- Args:- nodes: List of node definitions (may include _pre_executed trigger nodes)- edges: List of edge definitions- session_id: Session identifier- workflow_id: Workflow ID for tracking-- Returns:- Dict with success, outputs, and execution_trace- """- activity.logger.info(- f"Executing workflow with {len(nodes)} nodes, {len(edges)} edges (continuous scheduling)"- )- print(f"[Temporal] Starting continuous scheduling execution")-- try:- # Build dependency graph- dependencies, dependents, node_map = build_dependency_maps(nodes, edges)- all_node_ids = set(node_map.keys())-- # Track execution state- outputs: Dict[str, Any] = {}- execution_trace: List[str] = []- completed: Set[str] = set()- running: Set[str] = set()- errors: List[Dict] = []-- # Collect pre-executed trigger outputs- for node in nodes:- if node.get("_pre_executed") and node.get("_trigger_output"):- node_id = node["id"]- outputs[node_id] = {- "success": True,- "node_id": node_id,- "node_type": node.get("type", "unknown"),- "result": node["_trigger_output"],- "pre_executed": True,- }- completed.add(node_id)- execution_trace.append(node_id)- print(f"[Temporal] Pre-loaded trigger output: {node_id}")-- # Broadcast success for pre-executed trigger- await broadcast_node_status(- node_id=node_id,- status="success",- data={"result": node["_trigger_output"], "pre_executed": True},- workflow_id=workflow_id,- )+# Global session for standalone function+_global_session: Optional[aiohttp.ClientSession] = None+_global_activities: Optional[NodeExecutionActivities] = None- # Find initial ready nodes- ready_node_ids = find_ready_nodes(dependencies, completed, running, all_node_ids)- if not ready_node_ids:- print(f"[Temporal] No ready nodes found, workflow complete or stuck")- return {- "success": len(completed) == len(all_node_ids),- "outputs": outputs,- "execution_trace": execution_trace,- }+async def _get_global_activities() -> NodeExecutionActivities:+ """Get or create global activities instance."""+ global _global_session, _global_activities- print(f"[Temporal] Initial ready nodes: {ready_node_ids}")-- # Track running tasks: task -> node_id- task_to_node_id: Dict[asyncio.Task, str] = {}- pending_tasks: Set[asyncio.Task] = set()- workflow_failed = False-- def create_node_task(node_id: str) -> asyncio.Task:- """Create and track a task for node execution."""- node = node_map[node_id]- running.add(node_id)-- task = asyncio.create_task(- execute_single_node(- node=node,- nodes=nodes,- edges=edges,- outputs=outputs,- session_id=session_id,- workflow_id=workflow_id,- ),- name=f"node_{node_id}"- )- task_to_node_id[task] = node_id- pending_tasks.add(task)- return task-- # Start initial ready nodes- for node_id in ready_node_ids:- create_node_task(node_id)- print(f"[Temporal] Scheduled initial node: {node_id}")-- # Send heartbeat- activity.heartbeat(f"Started {len(ready_node_ids)} initial nodes")-- # Continuous scheduling loop- while pending_tasks and not workflow_failed:- # Wait for ANY task to complete (FIRST_COMPLETED pattern)- done, pending_tasks = await asyncio.wait(- pending_tasks,- return_when=asyncio.FIRST_COMPLETED- )+ if _global_session is None or _global_session.closed:+ _global_session = await create_shared_session()+ _global_activities = NodeExecutionActivities(_global_session)- # Process each completed task- for task in done:- node_id = task_to_node_id[task]- running.discard(node_id)-- try:- result = task.result()-- if result.get("success"):- # Success - store output and find newly ready nodes- outputs[node_id] = result- completed.add(node_id)- execution_trace.append(node_id)-- print(f"[Temporal] Node {node_id} completed successfully")- activity.heartbeat(f"Completed: {node_id}")-- # Find nodes that are now ready (their dependencies just completed)- newly_ready = find_ready_nodes(dependencies, completed, running, all_node_ids)-- # Schedule newly ready nodes immediately- for ready_id in newly_ready:- create_node_task(ready_id)- print(f"[Temporal] Scheduled dependent: {ready_id} (triggered by {node_id})")-- else:- # Node failed- error_msg = result.get("error", "Unknown error")- outputs[node_id] = result- errors.append({- "node_id": node_id,- "error": error_msg,- "timestamp": datetime.now().isoformat(),- })- print(f"[Temporal] Node {node_id} failed: {error_msg}")- workflow_failed = True-- except asyncio.CancelledError:- print(f"[Temporal] Node {node_id} was cancelled")- outputs[node_id] = {- "success": False,- "error": "Cancelled",- "node_id": node_id,- }-- except Exception as e:- error_msg = str(e)- print(f"[Temporal] Node {node_id} exception: {error_msg}")- outputs[node_id] = {- "success": False,- "error": error_msg,- "node_id": node_id,- }- errors.append({- "node_id": node_id,- "error": error_msg,- "timestamp": datetime.now().isoformat(),- })- workflow_failed = True-- # Handle workflow failure - cancel remaining tasks- if workflow_failed and pending_tasks:- print(f"[Temporal] Workflow failed, cancelling {len(pending_tasks)} remaining tasks")-- for task in pending_tasks:- task.cancel()-- # Wait for cancelled tasks- if pending_tasks:- cancelled_done, _ = await asyncio.wait(- pending_tasks,- return_when=asyncio.ALL_COMPLETED- )+ return _global_activities- for task in cancelled_done:- node_id = task_to_node_id.get(task)- if node_id and node_id not in outputs:- outputs[node_id] = {- "success": False,- "error": "Cancelled due to workflow failure",- "node_id": node_id,- }-- # Final heartbeat- activity.heartbeat(f"Workflow complete: {len(execution_trace)} nodes executed")-- success = not workflow_failed and len(completed) == len(all_node_ids)- print(f"[Temporal] Workflow finished: success={success}, executed={len(execution_trace)}/{len(all_node_ids)}")-- return {- "success": success,- "outputs": outputs,- "execution_trace": execution_trace,- "errors": errors if errors else None,- }- except Exception as e:- import traceback- tb = traceback.format_exc()- error_details = f"{type(e).__name__}: {str(e)}"- activity.logger.error(f"Workflow failed: {error_details}\n{tb}")- print(f"[Temporal] Workflow exception: {error_details}")- print(f"[Temporal] Traceback:\n{tb}")- return {- "success": False,- "error": error_details,- "outputs": {},- "execution_trace": [],- }+async def execute_node_activity(context: Dict[str, Any]) -> Dict[str, Any]:+ """Standalone activity function for backwards compatibility.+ For new code, use NodeExecutionActivities class with shared session.+ """+ activities = await _get_global_activities()+ return await activities.execute_node_activity(context)diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex 8d5f03dd..2958c38c 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -1,40 +1,66 @@-"""Temporal worker for MachinaOs.+"""Temporal worker for distributed node execution.-The worker polls the task queue and executes workflows and activities.-It can be run as a standalone process or embedded in the main server.+Uses class-based activities with shared aiohttp session for proper+connection pooling across concurrent activity executions.++The worker polls the task queue and executes:+- MachinaWorkflow: Orchestrates the graph, schedules node activities+- NodeExecutionActivities: Executes individual nodes with shared session++Multiple workers can be started on different machines for horizontal scaling.+Each node activity can execute on any available worker in the cluster.++References:+- https://docs.temporal.io/develop/python/python-sdk-sync-vs-async+- https://docs.temporal.io/develop/worker-performance"""import asynciofrom typing import Optional+import aiohttpfrom temporalio.client import Clientfrom temporalio.worker import Workerfrom core.logging import get_loggerfrom .workflow import MachinaWorkflow-from .activities import execute_workflow_with_continuous_scheduling+print(f"[Worker Import] MachinaWorkflow loaded from: {MachinaWorkflow.__module__}")+from .activities import (+ NodeExecutionActivities,+ create_shared_session,+ execute_node_activity,+)logger = get_logger(__name__)class TemporalWorkerManager:- """Manages the Temporal worker lifecycle."""+ """Manages the Temporal worker lifecycle with shared resources.++ Creates a shared aiohttp.ClientSession that is passed to the activity+ class, following Temporal's recommended dependency injection pattern.+ """def __init__(self,client: Client,task_queue: str = "machina-tasks",+ pool_size: int = 100,):"""Initialize the worker manager.Args:client: Connected Temporal clienttask_queue: Task queue name to poll+ pool_size: Connection pool size for aiohttp session"""self.client = clientself.task_queue = task_queue+ self.pool_size = pool_sizeself._worker: Optional[Worker] = Noneself._worker_task: Optional[asyncio.Task] = None+ self._session: Optional[aiohttp.ClientSession] = None+ self._activities: Optional[NodeExecutionActivities] = None@propertydef is_running(self) -> bool:@@ -47,17 +73,30 @@ class TemporalWorkerManager:logger.warning("Temporal worker already running")return+ # Create shared aiohttp session with connection pooling+ self._session = await create_shared_session(self.pool_size)++ # Create activity instance with shared session+ self._activities = NodeExecutionActivities(self._session)++ # Create worker with class-based activity+ # For class-based activities, pass the bound method (instance.method)self._worker = Worker(self.client,task_queue=self.task_queue,workflows=[MachinaWorkflow],- activities=[execute_workflow_with_continuous_scheduling],+ activities=[self._activities.execute_node_activity], # Pass bound method+ # Allow concurrent activity execution for parallel branches+ max_concurrent_activities=self.pool_size,+ max_concurrent_workflow_tasks=10,)logger.info("Starting Temporal worker",task_queue=self.task_queue,+ pool_size=self.pool_size,)+ print(f"[Worker] Starting with pool_size={self.pool_size}")# Run worker in background taskself._worker_task = asyncio.create_task(@@ -76,7 +115,7 @@ class TemporalWorkerManager:raiseasync def stop(self) -> None:- """Stop the Temporal worker."""+ """Stop the Temporal worker and cleanup resources."""if not self.is_running:return@@ -90,7 +129,14 @@ class TemporalWorkerManager:passself._worker_task = None+ # Close shared session+ if self._session and not self._session.closed:+ await self._session.close()+ print("[Worker] Closed shared session")+self._worker = None+ self._session = None+ self._activities = Nonelogger.info("Temporal worker stopped")@@ -98,15 +144,22 @@ async def run_standalone_worker(server_address: str = "localhost:7233",namespace: str = "default",task_queue: str = "machina-tasks",+ pool_size: int = 100,) -> None:"""Run the Temporal worker as a standalone process.- This can be used for running workers separately from the main server.+ This can be used for running workers separately from the main server,+ enabling horizontal scaling across multiple machines.++ Example:+ # Start multiple workers for horizontal scaling+ python -m services.temporal.workerArgs:server_address: Temporal server addressnamespace: Temporal namespacetask_queue: Task queue to poll+ pool_size: Connection pool size"""logger.info("Starting standalone Temporal worker",@@ -115,19 +168,69 @@ async def run_standalone_worker(task_queue=task_queue,)+ print(f"[Worker] Connecting to {server_address}")+ print(f"[Worker] Namespace: {namespace}")+ print(f"[Worker] Task Queue: {task_queue}")+ print(f"[Worker] Pool Size: {pool_size}")+client = await Client.connect(server_address, namespace=namespace)- worker = Worker(+ # Create shared session and activities+ session = await create_shared_session(pool_size)+ activities = NodeExecutionActivities(session)++ try:+ worker = Worker(+ client,+ task_queue=task_queue,+ workflows=[MachinaWorkflow],+ activities=[activities.execute_node_activity], # Pass bound method+ max_concurrent_activities=pool_size,+ max_concurrent_workflow_tasks=10,+ )++ print("[Worker] Running. Press Ctrl+C to stop.")+ logger.info("Worker running. Press Ctrl+C to stop.")+ await worker.run()++ finally:+ # Cleanup session on shutdown+ if not session.closed:+ await session.close()+ print("[Worker] Session closed")+++async def create_worker(+ client: Client,+ task_queue: str = "machina-tasks",+ session: Optional[aiohttp.ClientSession] = None,+) -> Worker:+ """Create a worker instance for use in tests or custom setups.++ Args:+ client: Connected Temporal client+ task_queue: Task queue name+ session: Optional shared aiohttp session (created if not provided)++ Returns:+ Configured Worker instance (not started)+ """+ if session is None:+ session = await create_shared_session()++ activities = NodeExecutionActivities(session)++ return Worker(client,task_queue=task_queue,workflows=[MachinaWorkflow],- activities=[execute_workflow_with_continuous_scheduling],+ activities=[activities.execute_node_activity], # Pass bound method+ max_concurrent_activities=100,+ max_concurrent_workflow_tasks=10,)- logger.info("Worker running. Press Ctrl+C to stop.")- await worker.run()-if __name__ == "__main__":# Allow running worker standalone+ # Usage: python -m services.temporal.workerasyncio.run(run_standalone_worker())diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex 4bf27c6c..1f0009fc 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -1,52 +1,72 @@-"""Temporal workflow definition for MachinaOs.+"""Temporal workflow - Pure orchestrator for distributed node execution.-The workflow is a thin wrapper that delegates execution to a continuous-scheduling activity. Uses asyncio.wait(FIRST_COMPLETED) pattern for immediate-downstream execution when dependencies complete - no waiting for sibling branches.+The workflow ONLY orchestrates:+- Parses graph structure+- Filters config nodes (tools, memory, services)+- Determines execution order based on dependencies+- Schedules node activities (can run on ANY worker)+- Collects results and routes outputs to dependent nodes-This replaces LangGraph's Pregel/BSP model which uses supersteps where all-parallel nodes must complete before downstream nodes can start.+NO business logic in workflow - all execution happens in activities.+This enables massive horizontal scaling and multi-tenant distribution."""from datetime import timedelta-from typing import Any, Dict+from typing import Any, Dict, List, Setfrom temporalio import workflowfrom temporalio.common import RetryPolicy-# Import activities through the unsafe imports mechanism-# This is required because Temporal sandboxes workflow imports-with workflow.unsafe.imports_passed_through():- from .activities import execute_workflow_with_continuous_scheduling+# Config handles - nodes connecting via these are config nodes (not executed)+CONFIG_HANDLES = {"input-tools", "input-memory", "input-model"}+# Android service types (connect to androidTool, not executed directly)+ANDROID_SERVICE_TYPES = {+ "batteryMonitor", "locationService", "deviceState",+ "systemInfo", "appList", "appLauncher",+}@workflow.defn(sandboxed=False)class MachinaWorkflow:- """Temporal workflow with continuous scheduling orchestration.+ """Distributed workflow orchestrator.- The actual graph execution happens in an activity using the- asyncio.wait(FIRST_COMPLETED) pattern for true parallel pipelines- where each path progresses independently.+ This workflow ONLY orchestrates - all execution happens in activities+ that can run on any worker in the cluster.++ Features:+ - Continuous scheduling (FIRST_COMPLETED pattern)+ - Per-node retry policies+ - Config node filtering (tools, memory, services)+ - Multi-tenant support via tenant_id in context"""@workflow.runasync def run(self, workflow_data: Dict[str, Any]) -> Dict[str, Any]:- """Execute workflow via continuous scheduling activity.+ print("[Workflow] ========== RUN METHOD CALLED ==========")+ """Execute workflow by orchestrating node activities.Args:workflow_data: Dict containing:- - nodes: List of node definitions- - edges: List of edge definitions+ - nodes: List of node definitions from React Flow+ - edges: List of edge definitions from React Flow- session_id: Session identifier- workflow_id: Workflow ID for tracking+ - tenant_id: Tenant identifier for multi-tenancyReturns:- Dict with success, outputs, and execution_trace+ Dict with success, outputs, execution_trace, and errors"""nodes = workflow_data.get("nodes", [])edges = workflow_data.get("edges", [])session_id = workflow_data.get("session_id", "default")workflow_id = workflow_data.get("workflow_id")+ tenant_id = workflow_data.get("tenant_id")++ workflow.logger.info(+ f"Starting workflow orchestration: {len(nodes)} nodes, {len(edges)} edges"+ )+ # Debug print for console visibility+ print(f"[Workflow] Starting: {len(nodes)} nodes, {len(edges)} edges")if not nodes:return {@@ -56,16 +76,268 @@ class MachinaWorkflow:"execution_trace": [],}- # Execute the entire workflow in a single activity- # Continuous scheduling handles parallel execution inside the activity- return await workflow.execute_activity(- execute_workflow_with_continuous_scheduling,- args=[nodes, edges, session_id, workflow_id],- start_to_close_timeout=timedelta(minutes=30),- heartbeat_timeout=timedelta(minutes=5),- retry_policy=RetryPolicy(- initial_interval=timedelta(seconds=1),- maximum_interval=timedelta(seconds=60),- maximum_attempts=3,- ),+ # 1. Filter out config nodes (tools, memory, services)+ exec_nodes, exec_edges = self._filter_executable_graph(nodes, edges)++ workflow.logger.info(+ f"After filtering: {len(exec_nodes)} executable nodes "+ f"(filtered {len(nodes) - len(exec_nodes)} config nodes)"+ )+ print(f"[Workflow] Executable: {len(exec_nodes)}, Config filtered: {len(nodes) - len(exec_nodes)}")++ # 2. Build dependency maps+ deps, node_map = self._build_dependency_maps(exec_nodes, exec_edges)++ # 3. Initialize state+ outputs: Dict[str, Any] = {} # node_id -> result+ completed: Set[str] = set()+ running: Dict[str, Any] = {} # node_id -> activity handle+ errors: List[Dict] = []+ execution_trace: List[str] = []++ # 4. Handle pre-executed triggers (already have their output)+ pre_executed_count = 0+ for node in exec_nodes:+ if node.get("_pre_executed"):+ node_id = node["id"]+ outputs[node_id] = {+ "success": True,+ "result": node.get("_trigger_output", {}),+ "pre_executed": True,+ }+ completed.add(node_id)+ execution_trace.append(node_id)+ pre_executed_count += 1+ workflow.logger.info(f"Pre-executed trigger: {node_id}")++ workflow.logger.info(f"Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")+ print(f"[Workflow] Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")++ # 5. Retry policy for node activities+ retry_policy = RetryPolicy(+ initial_interval=timedelta(seconds=1),+ maximum_interval=timedelta(seconds=30),+ maximum_attempts=3,)++ # 6. Continuous scheduling loop+ loop_count = 0+ while True:+ loop_count += 1+ # Find ready nodes (all deps completed, not running/completed)+ ready = self._find_ready_nodes(deps, completed, running, node_map)+ workflow.logger.info(f"Loop {loop_count}: ready={len(ready)}, running={len(running)}, completed={len(completed)}")+ print(f"[Workflow] Loop {loop_count}: ready={len(ready)}, running={len(running)}, completed={len(completed)}")++ # Start activities for ready nodes+ for node_id in ready:+ node = node_map[node_id]++ # Build immutable context for this node+ context = {+ "node_id": node_id,+ "node_type": node.get("type", "unknown"),+ "node_data": node.get("data", {}),+ "inputs": self._get_node_inputs(node_id, deps, outputs),+ "workflow_id": workflow_id,+ "tenant_id": tenant_id,+ "session_id": session_id,+ "nodes": nodes, # Full list for tool/memory detection+ "edges": edges, # Full list for tool/memory detection+ # Include pre-executed info if applicable+ "pre_executed": node.get("_pre_executed", False),+ "trigger_output": node.get("_trigger_output"),+ }++ # Schedule activity by name (for class-based activities)+ # The activity is registered as NodeExecutionActivities.execute_node_activity+ handle = workflow.start_activity(+ "execute_node_activity",+ args=[context],+ start_to_close_timeout=timedelta(minutes=10),+ heartbeat_timeout=timedelta(minutes=2),+ retry_policy=retry_policy,+ )+ running[node_id] = handle++ workflow.logger.info(f"Scheduled activity for node: {node_id}")+ print(f"[Workflow] Scheduled: {node_id}")++ # Exit if nothing running and nothing ready+ if not running:+ break++ # Wait for ANY activity to complete (FIRST_COMPLETED pattern)+ done_id, result = await self._wait_any_complete(running)++ if result.get("success"):+ outputs[done_id] = result+ completed.add(done_id)+ execution_trace.append(done_id)+ workflow.logger.info(f"Node completed: {done_id}")+ else:+ # Node failed after all retries+ error_info = {+ "node_id": done_id,+ "error": result.get("error", "Unknown error"),+ }+ errors.append(error_info)+ workflow.logger.error(f"Node failed: {done_id} - {error_info['error']}")++ # Stop workflow on failure+ # TODO: Could add option to continue with partial results+ break++ # Build final result+ success = len(errors) == 0 and len(completed) == len(node_map)++ workflow.logger.info(+ f"Workflow complete: success={success}, "+ f"executed={len(execution_trace)}/{len(node_map)}"+ )+ print(f"[Workflow] Complete: success={success}, executed={len(execution_trace)}/{len(node_map)}")++ return {+ "success": success,+ "outputs": outputs,+ "execution_trace": execution_trace,+ "errors": errors if errors else None,+ }++ def _get_node_inputs(+ self,+ node_id: str,+ deps: Dict[str, Set[str]],+ outputs: Dict[str, Any],+ ) -> Dict[str, Any]:+ """Get outputs from upstream nodes as inputs for this node."""+ inputs = {}+ for dep_id in deps.get(node_id, set()):+ if dep_id in outputs:+ inputs[dep_id] = outputs[dep_id].get("result", {})+ return inputs++ async def _wait_any_complete(self, running: Dict[str, Any]) -> tuple:+ """Wait for any activity to complete, return (node_id, result).++ Uses Temporal's native wait mechanism for efficient polling.+ """+ # Convert to list for iteration+ items = list(running.items())++ # Check if any already done+ for node_id, handle in items:+ if handle.done():+ del running[node_id]+ try:+ result = await handle+ return node_id, result+ except Exception as e:+ return node_id, {"success": False, "error": str(e)}++ # Wait for first completion using Temporal's wait+ handles = [h for _, h in items]++ # Use asyncio.wait pattern via workflow.wait+ await workflow.wait_condition(+ lambda: any(h.done() for _, h in items)+ )++ # Find the completed one+ for node_id, handle in items:+ if handle.done():+ del running[node_id]+ try:+ result = await handle+ return node_id, result+ except Exception as e:+ return node_id, {"success": False, "error": str(e)}++ # Should not reach here+ raise RuntimeError("No activity completed after wait")++ def _filter_executable_graph(+ self,+ nodes: List[Dict],+ edges: List[Dict],+ ) -> tuple:+ """Filter out config nodes based on edge handles.++ Config nodes (tools, memory, model configs) connect via special handles+ and are consumed by their target nodes, not executed independently.++ Returns:+ Tuple of (executable_nodes, executable_edges)+ """+ node_map = {n["id"]: n for n in nodes}+ config_ids = set()++ for edge in edges:+ handle = edge.get("targetHandle", "")+ source_id = edge.get("source")++ # Edges to config handles mean source is a config node+ if handle in CONFIG_HANDLES:+ config_ids.add(source_id)++ # Android services connecting to androidTool+ source_node = node_map.get(source_id, {})+ if source_node.get("type") in ANDROID_SERVICE_TYPES:+ config_ids.add(source_id)++ # Filter nodes and edges+ exec_nodes = [n for n in nodes if n["id"] not in config_ids]+ exec_edges = [+ e for e in edges+ if e.get("source") not in config_ids+ and e.get("target") not in config_ids+ and e.get("targetHandle", "") not in CONFIG_HANDLES+ ]++ return exec_nodes, exec_edges++ def _build_dependency_maps(+ self,+ nodes: List[Dict],+ edges: List[Dict],+ ) -> tuple:+ """Build dependency graph from nodes and edges.++ Returns:+ Tuple of (dependencies_map, node_map)+ - dependencies_map: node_id -> set of node IDs it depends on+ - node_map: node_id -> node definition+ """+ node_map = {n["id"]: n for n in nodes}+ node_ids = set(node_map.keys())++ deps = {nid: set() for nid in node_ids}++ for edge in edges:+ src, tgt = edge.get("source"), edge.get("target")+ if src in node_ids and tgt in node_ids:+ deps[tgt].add(src)++ return deps, node_map++ def _find_ready_nodes(+ self,+ deps: Dict[str, Set[str]],+ completed: Set[str],+ running: Dict[str, Any],+ node_map: Dict[str, Dict],+ ) -> List[str]:+ """Find nodes ready to execute.++ A node is ready when:+ - All its dependencies have completed+ - It's not already running+ - It's not already completed+ """+ ready = []+ for node_id in node_map:+ if node_id in completed or node_id in running:+ continue+ if deps[node_id] <= completed:+ ready.append(node_id)+ return readydiff --git a/server/services/temporal/ws_client.py b/server/services/temporal/ws_client.pynew file mode 100644index 00000000..35caa1ff--- /dev/null+++ b/server/services/temporal/ws_client.py@@ -0,0 +1,236 @@+"""WebSocket client for Temporal activities.++Uses aiohttp ClientSession for proper connection pooling and concurrent+request handling. Each activity creates its own WebSocket connection+to avoid race conditions, but the session manages connection reuse.++References:+- https://docs.aiohttp.org/en/stable/client_quickstart.html+- https://websockets.readthedocs.io/en/stable/reference/asyncio/client.html+"""++import asyncio+import json+import uuid+from typing import Any, Dict, Optional+from contextlib import asynccontextmanager++import aiohttp++from core.logging import get_logger+from core.config import Settings++logger = get_logger(__name__)++# Load settings+_settings = Settings()+WS_URL = f"ws://{_settings.host}:{_settings.port}/ws/internal"++print(f"[Temporal WS Client] WebSocket URL configured: {WS_URL}")+++class WSConnectionPool:+ """WebSocket connection pool using aiohttp ClientSession.++ aiohttp ClientSession provides built-in connection pooling with:+ - Connection reuse and keep-alive+ - Configurable connection limits+ - Proper concurrent request handling++ Each execute_node call gets its own WebSocket connection from the pool,+ avoiding the ConcurrencyError that occurs when sharing a single connection.+ """++ def __init__(self, url: str = WS_URL, pool_size: int = 100):+ self.url = url+ self.pool_size = pool_size+ self._session: Optional[aiohttp.ClientSession] = None+ self._lock = asyncio.Lock()++ async def _get_session(self) -> aiohttp.ClientSession:+ """Get or create the aiohttp session with connection pooling."""+ if self._session is None or self._session.closed:+ async with self._lock:+ if self._session is None or self._session.closed:+ connector = aiohttp.TCPConnector(+ limit=self.pool_size,+ limit_per_host=self.pool_size,+ enable_cleanup_closed=True,+ )+ timeout = aiohttp.ClientTimeout(+ total=300, # 5 min total timeout+ connect=10, # 10 sec connect timeout+ )+ self._session = aiohttp.ClientSession(+ connector=connector,+ timeout=timeout,+ )+ print(f"[WS Pool] Created session with pool_size={self.pool_size}")+ return self._session++ @asynccontextmanager+ async def connection(self):+ """Get a WebSocket connection from the pool.++ Usage:+ async with pool.connection() as ws:+ await ws.send_json(request)+ response = await ws.receive_json()+ """+ session = await self._get_session()+ async with session.ws_connect(+ self.url,+ heartbeat=20,+ receive_timeout=120,+ ) as ws:+ yield ws++ async def execute_node(+ self,+ node_id: str,+ node_type: str,+ data: Dict[str, Any],+ context: Dict[str, Any],+ timeout: float = 120.0,+ ) -> Dict[str, Any]:+ """Execute a node via WebSocket.++ Each call gets its own connection from the pool, allowing+ concurrent execution without race conditions.+ """+ request_id = str(uuid.uuid4())++ message = {+ "type": "execute_node",+ "request_id": request_id,+ "node_id": node_id,+ "node_type": node_type,+ "parameters": data,+ "nodes": context.get("nodes", []),+ "edges": context.get("edges", []),+ "session_id": context.get("session_id", "default"),+ "workflow_id": context.get("workflow_id"),+ }++ try:+ async with self.connection() as ws:+ await ws.send_json(message)+ print(f"[WS Pool] Sent execute_node for {node_id}")++ # Wait for response with matching request_id+ async with asyncio.timeout(timeout):+ async for msg in ws:+ if msg.type == aiohttp.WSMsgType.TEXT:+ response = json.loads(msg.data)+ if response.get("request_id") == request_id:+ print(f"[WS Pool] Received response for {node_id}: success={response.get('success')}")+ return response+ elif msg.type == aiohttp.WSMsgType.ERROR:+ raise Exception(f"WebSocket error: {ws.exception()}")+ elif msg.type == aiohttp.WSMsgType.CLOSED:+ raise Exception("WebSocket closed unexpectedly")++ raise Exception(f"No response received for request {request_id}")++ except asyncio.TimeoutError:+ raise Exception(f"WebSocket request timeout ({timeout}s) for node {node_id}")+ except aiohttp.ClientError as e:+ raise Exception(f"WebSocket connection error: {e}")++ async def close(self):+ """Close the connection pool."""+ if self._session and not self._session.closed:+ await self._session.close()+ print("[WS Pool] Session closed")+++# Global connection pool instance+_pool: Optional[WSConnectionPool] = None+_pool_lock = asyncio.Lock()+++async def get_ws_pool() -> WSConnectionPool:+ """Get or create the global WebSocket connection pool."""+ global _pool++ async with _pool_lock:+ if _pool is None:+ _pool = WSConnectionPool()+ return _pool+++async def execute_node_ws(+ node_id: str,+ node_type: str,+ data: Dict[str, Any],+ context: Dict[str, Any],+ timeout: float = 120.0,+) -> Dict[str, Any]:+ """Execute a node via WebSocket using the connection pool.++ This is the recommended way to execute nodes from activities.+ Each call gets its own connection from the pool.+ """+ pool = await get_ws_pool()+ return await pool.execute_node(+ node_id=node_id,+ node_type=node_type,+ data=data,+ context=context,+ timeout=timeout,+ )+++async def close_ws_pool() -> None:+ """Close the global WebSocket connection pool."""+ global _pool++ async with _pool_lock:+ if _pool:+ await _pool.close()+ _pool = None+++# Backwards compatibility aliases+async def get_ws_client() -> WSConnectionPool:+ """Alias for get_ws_pool() for backwards compatibility."""+ return await get_ws_pool()+++async def close_ws_client() -> None:+ """Alias for close_ws_pool() for backwards compatibility."""+ await close_ws_pool()+++# For backwards compatibility with old code that expects TemporalWSClient+class TemporalWSClient:+ """Backwards compatibility wrapper around WSConnectionPool."""++ def __init__(self, url: str = WS_URL):+ self._pool = WSConnectionPool(url=url)++ @property+ def connected(self) -> bool:+ return True # Pool manages connections++ async def connect(self) -> None:+ pass # Pool connects on demand++ async def disconnect(self) -> None:+ await self._pool.close()++ async def execute_node(+ self,+ node_id: str,+ node_type: str,+ data: Dict[str, Any],+ context: Dict[str, Any],+ timeout: float = 120.0,+ ) -> Dict[str, Any]:+ return await self._pool.execute_node(+ node_id=node_id,+ node_type=node_type,+ data=data,+ context=context,+ timeout=timeout,+ )diff --git a/server/services/workflow.py b/server/services/workflow.pyindex 6f2f0de1..e69245a5 100644--- a/server/services/workflow.py+++ b/server/services/workflow.py@@ -232,6 +232,14 @@ class WorkflowService:if use_temporal and self._temporal_executor is not None:return await self._execute_temporal(nodes, edges, session_id, status_callback, start_time, workflow_id)+ # Log warning if Temporal was requested but not available+ if use_temporal and self._temporal_executor is None:+ logger.warning(+ "Temporal execution requested but executor not configured. "+ "Falling back to parallel/sequential execution. "+ "Check TEMPORAL_ENABLED and Temporal server connection."+ )+# Use parallel executor if enabled and Redis availableif use_parallel and self.settings.redis_enabled:return await self._execute_parallel(nodes, edges, session_id, status_callback, start_time, workflow_id)