Skip to content

Self-improving AI Employees for everyone

Temporal Server.

trohitg981698d228b7
diff --git a/server/.env.template b/server/.env.template
index 54d4d387..4967624c 100644
--- a/server/.env.template
+++ b/server/.env.template
@@ -35,6 +35,13 @@ DATABASE_MAX_OVERFLOW=10
REDIS_ENABLED=false
CACHE_TTL=300
+# Temporal Configuration (Disabled by default)
+# Enable for durable workflow execution with automatic retries
+TEMPORAL_ENABLED=false
+TEMPORAL_SERVER_ADDRESS=localhost:7233
+TEMPORAL_NAMESPACE=default
+TEMPORAL_TASK_QUEUE=machina-tasks
+
# Logging
LOG_LEVEL=DEBUG
LOG_FORMAT=text
diff --git a/server/core/config.py b/server/core/config.py
index dc10714d..3cfbc815 100644
--- a/server/core/config.py
+++ b/server/core/config.py
@@ -47,6 +47,12 @@ class Settings(BaseSettings):
# Execution Engine
dlq_enabled: bool = Field(default=False, env="DLQ_ENABLED")
+ # Temporal Configuration
+ temporal_enabled: bool = Field(default=False, env="TEMPORAL_ENABLED")
+ temporal_server_address: str = Field(default="localhost:7233", env="TEMPORAL_SERVER_ADDRESS")
+ temporal_namespace: str = Field(default="default", env="TEMPORAL_NAMESPACE")
+ temporal_task_queue: str = Field(default="machina-tasks", env="TEMPORAL_TASK_QUEUE")
+
# API Keys (all optional, injected at runtime)
google_maps_api_key: Optional[str] = Field(default=None, env="GOOGLE_MAPS_API_KEY")
openai_api_key: Optional[str] = Field(default=None, env="OPENAI_API_KEY")
diff --git a/server/core/container.py b/server/core/container.py
index 8425f6ea..c995037b 100644
--- a/server/core/container.py
+++ b/server/core/container.py
@@ -13,6 +13,7 @@ from services.auth import AuthService
from services.text import TextService
from services.android_service import AndroidService
from services.user_auth import UserAuthService
+from services.temporal import TemporalClientWrapper
class Container(containers.DeclarativeContainer):
@@ -39,6 +40,13 @@ class Container(containers.DeclarativeContainer):
database=database
)
+ # Temporal client (lazy - only created when temporal_enabled)
+ temporal_client = providers.Singleton(
+ TemporalClientWrapper,
+ server_address=settings.provided.temporal_server_address,
+ namespace=settings.provided.temporal_namespace,
+ )
+
# Services
auth_service = providers.Factory(
AuthService,
diff --git a/server/main.py b/server/main.py
index 2d2e19bc..4d0cc764 100644
--- a/server/main.py
+++ b/server/main.py
@@ -134,6 +134,44 @@ async def lifespan(app: FastAPI):
setup_websocket_logging(loop)
logger.info("WebSocket logging handler started")
+ # Initialize Temporal if enabled
+ temporal_worker_manager = None
+ if settings.temporal_enabled:
+ try:
+ from services.temporal import TemporalClientWrapper, TemporalExecutor
+ from services.temporal.worker import TemporalWorkerManager
+
+ logger.info(
+ "Initializing Temporal integration",
+ server_address=settings.temporal_server_address,
+ namespace=settings.temporal_namespace,
+ task_queue=settings.temporal_task_queue,
+ )
+
+ # Connect Temporal client
+ temporal_client_wrapper = container.temporal_client()
+ temporal_client = await temporal_client_wrapper.connect()
+
+ # Create and set the Temporal executor on WorkflowService
+ temporal_executor = TemporalExecutor(
+ client=temporal_client,
+ task_queue=settings.temporal_task_queue,
+ )
+ container.workflow_service().set_temporal_executor(temporal_executor)
+
+ # Start embedded Temporal worker
+ temporal_worker_manager = TemporalWorkerManager(
+ client=temporal_client,
+ task_queue=settings.temporal_task_queue,
+ )
+ await temporal_worker_manager.start()
+
+ logger.info("Temporal integration initialized successfully")
+
+ except Exception as e:
+ logger.error(f"Failed to initialize Temporal: {str(e)}")
+ logger.warning("Falling back to Redis/sequential execution")
+
logger.info("Services started successfully")
yield
@@ -141,6 +179,18 @@ async def lifespan(app: FastAPI):
# Stop WebSocket logging handler
shutdown_websocket_logging()
+ # Stop Temporal worker if running
+ if temporal_worker_manager is not None:
+ await temporal_worker_manager.stop()
+ logger.info("Temporal worker stopped")
+
+ # Disconnect Temporal client if connected
+ if settings.temporal_enabled:
+ try:
+ await container.temporal_client().disconnect()
+ except Exception:
+ pass
+
# Close Android relay client (prevents "Unclosed client session" warning)
from services.android.manager import close_relay_client
await close_relay_client(clear_stored_session=False)
@@ -225,10 +275,23 @@ async def health_check():
sweeper = get_recovery_sweeper()
+ # Check Temporal status
+ temporal_status = {
+ "enabled": settings.temporal_enabled,
+ "connected": False,
+ }
+ if settings.temporal_enabled:
+ try:
+ temporal_status["connected"] = container.temporal_client().is_connected
+ temporal_status["server_address"] = settings.temporal_server_address
+ temporal_status["task_queue"] = settings.temporal_task_queue
+ except Exception:
+ pass
+
return {
"status": "OK",
"service": "python",
- "version": "3.1.0", # Bumped for execution engine
+ "version": "3.2.0", # Bumped for Temporal integration
"environment": "development" if settings.debug else "production",
"redis_enabled": settings.redis_enabled,
"event_waiter_mode": event_waiter.get_backend_mode(),
@@ -236,6 +299,7 @@ async def health_check():
"enabled": settings.redis_enabled,
"recovery_sweeper": sweeper is not None and sweeper._running,
},
+ "temporal": temporal_status,
"timestamp": datetime.now().isoformat()
}
diff --git a/server/requirements.txt b/server/requirements.txt
index 7a2312b1..e67d8f5b 100644
--- a/server/requirements.txt
+++ b/server/requirements.txt
@@ -50,6 +50,9 @@ langchain-cerebras>=0.1.0
# LangGraph - State machine for AI agents
langgraph>=1.0.6
+# Temporal - Durable workflow execution (optional, requires TEMPORAL_ENABLED=true)
+temporalio>=1.21.1
+
# Google Services
googlemaps>=4.10.0
diff --git a/server/routers/workflow.py b/server/routers/workflow.py
index 6fda0291..fc583360 100644
--- a/server/routers/workflow.py
+++ b/server/routers/workflow.py
@@ -138,6 +138,34 @@ class TemporalNodeExecuteRequest(BaseModel):
context: Dict[str, Any] = {}
+class BroadcastStatusRequest(BaseModel):
+ """Request to broadcast node status from Temporal activity."""
+ node_id: str
+ status: str
+ data: Dict[str, Any] = {}
+ workflow_id: str = None
+
+
[email protected]("/broadcast-status")
+async def broadcast_status_for_temporal(request: BroadcastStatusRequest):
+ """Broadcast node status - called by Temporal activities for real-time updates.
+
+ This endpoint allows Temporal activities to send status updates
+ to connected WebSocket clients during workflow execution.
+ """
+ print(f"[Broadcast] {request.node_id} -> {request.status} (workflow={request.workflow_id})")
+
+ broadcaster = get_status_broadcaster()
+ await broadcaster.update_node_status(
+ node_id=request.node_id,
+ status=request.status,
+ data=request.data,
+ workflow_id=request.workflow_id,
+ )
+
+ return {"success": True}
+
+
@router.post("/node/execute")
async def execute_node_for_temporal(
request: TemporalNodeExecuteRequest,
@@ -148,16 +176,25 @@ async def execute_node_for_temporal(
This is a simplified endpoint for Temporal activities to call.
It extracts context and delegates to the existing execute_node method.
"""
- logger.info(f"[Temporal] Executing node: {request.node_id} (type={request.node_type})")
+ print(f"[Temporal Endpoint] Received request for node: {request.node_id} (type={request.node_type})")
+ print(f"[Temporal Endpoint] Data: {request.data}")
+ print(f"[Temporal Endpoint] Context keys: {list(request.context.keys())}")
context = request.context
- result = await workflow_service.execute_node(
- node_id=request.node_id,
- node_type=request.node_type,
- parameters=request.data,
- nodes=context.get("nodes", []),
- edges=context.get("edges", []),
- session_id=context.get("session_id", "temporal"),
- )
-
- return result
\ No newline at end of file
+ try:
+ result = await workflow_service.execute_node(
+ node_id=request.node_id,
+ node_type=request.node_type,
+ parameters=request.data,
+ nodes=context.get("nodes", []),
+ edges=context.get("edges", []),
+ session_id=context.get("session_id", "temporal"),
+ workflow_id=context.get("workflow_id"),
+ )
+ print(f"[Temporal Endpoint] Node {request.node_id} result: success={result.get('success')}, error={result.get('error')}")
+ return result
+ except Exception as e:
+ print(f"[Temporal Endpoint] Node {request.node_id} EXCEPTION: {str(e)}")
+ import traceback
+ traceback.print_exc()
+ raise
\ No newline at end of file
diff --git a/server/services/temporal/__init__.py b/server/services/temporal/__init__.py
new file mode 100644
index 00000000..96fa5bec
--- /dev/null
+++ b/server/services/temporal/__init__.py
@@ -0,0 +1,19 @@
+"""Temporal workflow orchestration service.
+
+This module provides Temporal integration for durable workflow execution.
+LangGraph runs inside activities (outside Temporal's sandboxed event loop)
+to use its native asyncio parallel execution.
+
+When TEMPORAL_ENABLED=true:
+- Workflows are executed via Temporal for durability
+- LangGraph handles parallel execution inside activities
+- Activities call back to MachinaOs node handlers
+
+When TEMPORAL_ENABLED=false (default):
+- Falls back to the existing Redis-based executor
+"""
+
+from .executor import TemporalExecutor
+from .client import TemporalClientWrapper
+
+__all__ = ["TemporalExecutor", "TemporalClientWrapper"]
diff --git a/server/services/temporal/activities.py b/server/services/temporal/activities.py
new file mode 100644
index 00000000..37e4c81b
--- /dev/null
+++ b/server/services/temporal/activities.py
@@ -0,0 +1,542 @@
+"""Temporal activities with continuous scheduling orchestration.
+
+Uses asyncio.wait(FIRST_COMPLETED) pattern for immediate downstream execution
+when dependencies complete - no waiting for sibling branches.
+
+This replaces LangGraph's Pregel/BSP model which uses supersteps where all
+parallel nodes must complete before downstream nodes can start.
+
+Pattern borrowed from services/execution/executor.py WorkflowExecutor.
+
+Activities call back to MachinaOs node handlers via HTTP and broadcast
+status updates via the StatusBroadcaster for real-time UI updates.
+"""
+
+import asyncio
+import httpx
+from collections import defaultdict
+from datetime import datetime
+from typing import Any, Dict, List, Set, Optional
+
+from temporalio import activity
+
+from core.logging import get_logger
+from core.config import Settings
+
+logger = get_logger(__name__)
+
+# Load settings to get the correct server port
+_settings = Settings()
+MACHINA_URL = f"http://{_settings.host}:{_settings.port}"
+
+print(f"[Temporal Activities] MACHINA_URL configured: {MACHINA_URL}")
+
+
+# =============================================================================
+# STATUS BROADCASTING
+# =============================================================================
+
+async def broadcast_node_status(
+ node_id: str,
+ status: str,
+ data: dict = None,
+ workflow_id: str = None,
+) -> None:
+ """Broadcast node status update via HTTP to StatusBroadcaster.
+
+ This allows the Temporal activity to send real-time status updates
+ to connected WebSocket clients.
+ """
+ url = f"{MACHINA_URL}/api/workflow/broadcast-status"
+ payload = {
+ "node_id": node_id,
+ "status": status,
+ "data": data or {},
+ "workflow_id": workflow_id,
+ }
+
+ async with httpx.AsyncClient(timeout=5.0) as client:
+ try:
+ await client.post(url, json=payload)
+ print(f"[Temporal] Broadcast status: {node_id} -> {status}")
+ except Exception as e:
+ # Don't fail execution if broadcast fails
+ print(f"[Temporal] Broadcast failed (non-fatal): {str(e)}")
+
+
+# =============================================================================
+# NODE EXECUTION VIA HTTP
+# =============================================================================
+
+async def execute_node_via_http(
+ node_id: str,
+ node_type: str,
+ data: dict,
+ context: dict,
+) -> dict:
+ """Execute a node by calling MachinaOs HTTP endpoint.
+
+ This allows the activity to leverage all existing node handlers
+ without duplicating code.
+ """
+ url = f"{MACHINA_URL}/api/workflow/node/execute"
+ print(f"[Temporal] Executing node {node_id} ({node_type}) via {url}")
+
+ async with httpx.AsyncClient(timeout=120.0) as client:
+ try:
+ response = await client.post(
+ url,
+ json={
+ "node_id": node_id,
+ "node_type": node_type,
+ "data": data,
+ "context": context,
+ },
+ )
+ response.raise_for_status()
+ result = response.json()
+ print(f"[Temporal] Node {node_id} completed: success={result.get('success', 'N/A')}")
+ return result
+ except httpx.HTTPStatusError as e:
+ print(f"[Temporal] HTTP ERROR {e.response.status_code} for node {node_id}: {e.response.text}")
+ logger.error(
+ f"Node execution HTTP error: {e.response.status_code}",
+ node_id=node_id,
+ node_type=node_type,
+ )
+ return {
+ "success": False,
+ "node_id": node_id,
+ "node_type": node_type,
+ "error": f"HTTP {e.response.status_code}: {e.response.text}",
+ }
+ except Exception as e:
+ print(f"[Temporal] EXCEPTION for node {node_id}: {str(e)}")
+ logger.error(
+ f"Node execution failed: {str(e)}",
+ node_id=node_id,
+ node_type=node_type,
+ )
+ return {
+ "success": False,
+ "node_id": node_id,
+ "node_type": node_type,
+ "error": str(e),
+ }
+
+
+# =============================================================================
+# GRAPH UTILITIES (from WorkflowExecutor pattern)
+# =============================================================================
+
+def build_dependency_maps(
+ nodes: List[Dict],
+ edges: List[Dict],
+) -> tuple[Dict[str, Set[str]], Dict[str, Set[str]], Dict[str, Dict]]:
+ """Build dependency graph from nodes and edges.
+
+ Returns:
+ Tuple of (dependencies, dependents, node_map)
+ - dependencies: node_id -> set of nodes it depends on (must complete first)
+ - dependents: node_id -> set of nodes that depend on it (to schedule next)
+ - node_map: node_id -> node definition
+ """
+ node_map = {n["id"]: n for n in nodes}
+ node_ids = set(node_map.keys())
+
+ # dependencies[node_id] = set of nodes that must complete before this node
+ dependencies: Dict[str, Set[str]] = {node_id: set() for node_id in node_ids}
+
+ # dependents[node_id] = set of nodes that depend on this node
+ dependents: Dict[str, Set[str]] = {node_id: set() for node_id in node_ids}
+
+ for edge in edges:
+ source = edge.get("source")
+ target = edge.get("target")
+ if source in node_ids and target in node_ids:
+ dependencies[target].add(source)
+ dependents[source].add(target)
+
+ return dependencies, dependents, node_map
+
+
+def find_ready_nodes(
+ dependencies: Dict[str, Set[str]],
+ completed: Set[str],
+ running: Set[str],
+ all_nodes: Set[str],
+) -> List[str]:
+ """Find nodes ready to execute (all dependencies met, not running/completed).
+
+ Args:
+ dependencies: node_id -> set of dependencies
+ completed: Set of completed node IDs
+ running: Set of currently running node IDs
+ all_nodes: Set of all node IDs
+
+ Returns:
+ List of node IDs ready to execute
+ """
+ ready = []
+ for node_id in all_nodes:
+ if node_id in completed or node_id in running:
+ continue
+ # Check if all dependencies are completed
+ if dependencies[node_id] <= completed:
+ ready.append(node_id)
+ return ready
+
+
+# =============================================================================
+# SINGLE NODE EXECUTION
+# =============================================================================
+
+async def execute_single_node(
+ node: Dict,
+ nodes: List[Dict],
+ edges: List[Dict],
+ outputs: Dict[str, Any],
+ session_id: str,
+ workflow_id: Optional[str],
+) -> Dict[str, Any]:
+ """Execute a single node, handling pre-executed triggers and disabled nodes.
+
+ Args:
+ node: Node definition
+ nodes: All nodes in workflow
+ edges: All edges in workflow
+ outputs: Current outputs from completed nodes
+ session_id: Session identifier
+ workflow_id: Workflow ID for tracking
+
+ Returns:
+ Execution result dict
+ """
+ node_id = node["id"]
+ node_type = node.get("type", "unknown")
+
+ print(f"[Temporal] Processing node: {node_id} (type={node_type}, _pre_executed={node.get('_pre_executed')})")
+
+ # Check if this is a pre-executed trigger node
+ if node.get("_pre_executed"):
+ print(f"[Temporal] Skipping pre-executed trigger node: {node_id}")
+ trigger_output = node.get("_trigger_output", {})
+ result = {
+ "success": True,
+ "node_id": node_id,
+ "node_type": node_type,
+ "result": trigger_output,
+ "skipped": True,
+ "reason": "pre-executed trigger",
+ "timestamp": datetime.now().isoformat(),
+ }
+
+ await broadcast_node_status(
+ node_id=node_id,
+ status="success",
+ data={"result": trigger_output, "pre_executed": True},
+ workflow_id=workflow_id,
+ )
+
+ return result
+
+ # Check if node is disabled
+ if node.get("data", {}).get("disabled"):
+ print(f"[Temporal] Skipping disabled node: {node_id}")
+ result = {
+ "success": True,
+ "node_id": node_id,
+ "node_type": node_type,
+ "skipped": True,
+ "reason": "disabled",
+ "timestamp": datetime.now().isoformat(),
+ }
+
+ await broadcast_node_status(
+ node_id=node_id,
+ status="skipped",
+ data={"disabled": True},
+ workflow_id=workflow_id,
+ )
+
+ return result
+
+ # Build context for node execution
+ context = {
+ "outputs": outputs,
+ "nodes": nodes,
+ "edges": edges,
+ "session_id": session_id,
+ "workflow_id": workflow_id,
+ }
+
+ # Broadcast "executing" status BEFORE execution
+ await broadcast_node_status(
+ node_id=node_id,
+ status="executing",
+ data={"node_type": node_type},
+ workflow_id=workflow_id,
+ )
+
+ # Execute node via MachinaOs
+ result = await execute_node_via_http(
+ node_id=node_id,
+ node_type=node_type,
+ data={}, # Let server load from DB
+ context=context,
+ )
+
+ # Add timestamp
+ result["timestamp"] = datetime.now().isoformat()
+
+ # Broadcast result status AFTER execution
+ if result.get("success"):
+ await broadcast_node_status(
+ node_id=node_id,
+ status="success",
+ data={"result": result.get("result"), "execution_time": result.get("execution_time")},
+ workflow_id=workflow_id,
+ )
+ else:
+ await broadcast_node_status(
+ node_id=node_id,
+ status="error",
+ data={"error": result.get("error")},
+ workflow_id=workflow_id,
+ )
+
+ return result
+
+
+# =============================================================================
+# CONTINUOUS SCHEDULING ACTIVITY
+# =============================================================================
+
+async def execute_workflow_with_continuous_scheduling(
+ nodes: List[Dict],
+ edges: List[Dict],
+ session_id: str = "default",
+ workflow_id: Optional[str] = None,
+) -> Dict[str, Any]:
+ """Execute workflow with continuous scheduling (FIRST_COMPLETED pattern).
+
+ When any node completes, immediately check for and start newly-ready
+ dependent nodes. This enables true parallel pipelines where each path
+ progresses independently without waiting for sibling branches.
+
+ Pattern from WorkflowExecutor._execute_with_continuous_scheduling
+
+ Args:
+ nodes: List of node definitions (may include _pre_executed trigger nodes)
+ edges: List of edge definitions
+ session_id: Session identifier
+ workflow_id: Workflow ID for tracking
+
+ Returns:
+ Dict with success, outputs, and execution_trace
+ """
+ activity.logger.info(
+ f"Executing workflow with {len(nodes)} nodes, {len(edges)} edges (continuous scheduling)"
+ )
+ print(f"[Temporal] Starting continuous scheduling execution")
+
+ try:
+ # Build dependency graph
+ dependencies, dependents, node_map = build_dependency_maps(nodes, edges)
+ all_node_ids = set(node_map.keys())
+
+ # Track execution state
+ outputs: Dict[str, Any] = {}
+ execution_trace: List[str] = []
+ completed: Set[str] = set()
+ running: Set[str] = set()
+ errors: List[Dict] = []
+
+ # Collect pre-executed trigger outputs
+ for node in nodes:
+ if node.get("_pre_executed") and node.get("_trigger_output"):
+ node_id = node["id"]
+ outputs[node_id] = {
+ "success": True,
+ "node_id": node_id,
+ "node_type": node.get("type", "unknown"),
+ "result": node["_trigger_output"],
+ "pre_executed": True,
+ }
+ completed.add(node_id)
+ execution_trace.append(node_id)
+ print(f"[Temporal] Pre-loaded trigger output: {node_id}")
+
+ # Broadcast success for pre-executed trigger
+ await broadcast_node_status(
+ node_id=node_id,
+ status="success",
+ data={"result": node["_trigger_output"], "pre_executed": True},
+ workflow_id=workflow_id,
+ )
+
+ # Find initial ready nodes
+ ready_node_ids = find_ready_nodes(dependencies, completed, running, all_node_ids)
+
+ if not ready_node_ids:
+ print(f"[Temporal] No ready nodes found, workflow complete or stuck")
+ return {
+ "success": len(completed) == len(all_node_ids),
+ "outputs": outputs,
+ "execution_trace": execution_trace,
+ }
+
+ print(f"[Temporal] Initial ready nodes: {ready_node_ids}")
+
+ # Track running tasks: task -> node_id
+ task_to_node_id: Dict[asyncio.Task, str] = {}
+ pending_tasks: Set[asyncio.Task] = set()
+ workflow_failed = False
+
+ def create_node_task(node_id: str) -> asyncio.Task:
+ """Create and track a task for node execution."""
+ node = node_map[node_id]
+ running.add(node_id)
+
+ task = asyncio.create_task(
+ execute_single_node(
+ node=node,
+ nodes=nodes,
+ edges=edges,
+ outputs=outputs,
+ session_id=session_id,
+ workflow_id=workflow_id,
+ ),
+ name=f"node_{node_id}"
+ )
+ task_to_node_id[task] = node_id
+ pending_tasks.add(task)
+ return task
+
+ # Start initial ready nodes
+ for node_id in ready_node_ids:
+ create_node_task(node_id)
+ print(f"[Temporal] Scheduled initial node: {node_id}")
+
+ # Send heartbeat
+ activity.heartbeat(f"Started {len(ready_node_ids)} initial nodes")
+
+ # Continuous scheduling loop
+ while pending_tasks and not workflow_failed:
+ # Wait for ANY task to complete (FIRST_COMPLETED pattern)
+ done, pending_tasks = await asyncio.wait(
+ pending_tasks,
+ return_when=asyncio.FIRST_COMPLETED
+ )
+
+ # Process each completed task
+ for task in done:
+ node_id = task_to_node_id[task]
+ running.discard(node_id)
+
+ try:
+ result = task.result()
+
+ if result.get("success"):
+ # Success - store output and find newly ready nodes
+ outputs[node_id] = result
+ completed.add(node_id)
+ execution_trace.append(node_id)
+
+ print(f"[Temporal] Node {node_id} completed successfully")
+ activity.heartbeat(f"Completed: {node_id}")
+
+ # Find nodes that are now ready (their dependencies just completed)
+ newly_ready = find_ready_nodes(dependencies, completed, running, all_node_ids)
+
+ # Schedule newly ready nodes immediately
+ for ready_id in newly_ready:
+ create_node_task(ready_id)
+ print(f"[Temporal] Scheduled dependent: {ready_id} (triggered by {node_id})")
+
+ else:
+ # Node failed
+ error_msg = result.get("error", "Unknown error")
+ outputs[node_id] = result
+ errors.append({
+ "node_id": node_id,
+ "error": error_msg,
+ "timestamp": datetime.now().isoformat(),
+ })
+ print(f"[Temporal] Node {node_id} failed: {error_msg}")
+ workflow_failed = True
+
+ except asyncio.CancelledError:
+ print(f"[Temporal] Node {node_id} was cancelled")
+ outputs[node_id] = {
+ "success": False,
+ "error": "Cancelled",
+ "node_id": node_id,
+ }
+
+ except Exception as e:
+ error_msg = str(e)
+ print(f"[Temporal] Node {node_id} exception: {error_msg}")
+ outputs[node_id] = {
+ "success": False,
+ "error": error_msg,
+ "node_id": node_id,
+ }
+ errors.append({
+ "node_id": node_id,
+ "error": error_msg,
+ "timestamp": datetime.now().isoformat(),
+ })
+ workflow_failed = True
+
+ # Handle workflow failure - cancel remaining tasks
+ if workflow_failed and pending_tasks:
+ print(f"[Temporal] Workflow failed, cancelling {len(pending_tasks)} remaining tasks")
+
+ for task in pending_tasks:
+ task.cancel()
+
+ # Wait for cancelled tasks
+ if pending_tasks:
+ cancelled_done, _ = await asyncio.wait(
+ pending_tasks,
+ return_when=asyncio.ALL_COMPLETED
+ )
+
+ for task in cancelled_done:
+ node_id = task_to_node_id.get(task)
+ if node_id and node_id not in outputs:
+ outputs[node_id] = {
+ "success": False,
+ "error": "Cancelled due to workflow failure",
+ "node_id": node_id,
+ }
+
+ # Final heartbeat
+ activity.heartbeat(f"Workflow complete: {len(execution_trace)} nodes executed")
+
+ success = not workflow_failed and len(completed) == len(all_node_ids)
+ print(f"[Temporal] Workflow finished: success={success}, executed={len(execution_trace)}/{len(all_node_ids)}")
+
+ return {
+ "success": success,
+ "outputs": outputs,
+ "execution_trace": execution_trace,
+ "errors": errors if errors else None,
+ }
+
+ except Exception as e:
+ import traceback
+ tb = traceback.format_exc()
+ error_details = f"{type(e).__name__}: {str(e)}"
+ activity.logger.error(f"Workflow failed: {error_details}\n{tb}")
+ print(f"[Temporal] Workflow exception: {error_details}")
+ print(f"[Temporal] Traceback:\n{tb}")
+ return {
+ "success": False,
+ "error": error_details,
+ "outputs": {},
+ "execution_trace": [],
+ }
+
diff --git a/server/services/temporal/client.py b/server/services/temporal/client.py
new file mode 100644
index 00000000..084263af
--- /dev/null
+++ b/server/services/temporal/client.py
@@ -0,0 +1,67 @@
+"""Temporal client wrapper for MachinaOs.
+
+Manages the Temporal client connection lifecycle.
+"""
+
+from typing import Optional
+from temporalio.client import Client
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class TemporalClientWrapper:
+ """Wrapper around Temporal client for lifecycle management."""
+
+ def __init__(self, server_address: str, namespace: str = "default"):
+ """Initialize the client wrapper.
+
+ Args:
+ server_address: Temporal server address (e.g., "localhost:7233")
+ namespace: Temporal namespace to use
+ """
+ self.server_address = server_address
+ self.namespace = namespace
+ self._client: Optional[Client] = None
+
+ @property
+ def client(self) -> Optional[Client]:
+ """Get the underlying Temporal client."""
+ return self._client
+
+ @property
+ def is_connected(self) -> bool:
+ """Check if client is connected."""
+ return self._client is not None
+
+ async def connect(self) -> Client:
+ """Connect to the Temporal server.
+
+ Returns:
+ The connected Temporal client
+ """
+ if self._client is not None:
+ return self._client
+
+ logger.info(
+ "Connecting to Temporal server",
+ server_address=self.server_address,
+ namespace=self.namespace,
+ )
+
+ self._client = await Client.connect(
+ self.server_address,
+ namespace=self.namespace,
+ )
+
+ logger.info("Connected to Temporal server")
+ return self._client
+
+ async def disconnect(self) -> None:
+ """Disconnect from the Temporal server."""
+ if self._client is not None:
+ # Temporal client doesn't have an explicit close method,
+ # but we clear the reference
+ self._client = None
+ logger.info("Disconnected from Temporal server")
diff --git a/server/services/temporal/executor.py b/server/services/temporal/executor.py
new file mode 100644
index 00000000..aa60c0f2
--- /dev/null
+++ b/server/services/temporal/executor.py
@@ -0,0 +1,147 @@
+"""Temporal executor for MachinaOs workflow execution.
+
+Provides the same interface as WorkflowExecutor but delegates
+execution to Temporal for durable workflow orchestration.
+"""
+
+import time
+import uuid
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Callable
+
+from temporalio.client import Client
+
+from core.logging import get_logger
+from .workflow import MachinaWorkflow
+
+logger = get_logger(__name__)
+
+
+class TemporalExecutor:
+ """Workflow executor that uses Temporal for durable execution.
+
+ Provides a compatible interface with the existing WorkflowExecutor
+ so it can be used as a drop-in replacement when Temporal is enabled.
+ """
+
+ def __init__(
+ self,
+ client: Client,
+ task_queue: str = "machina-tasks",
+ status_callback: Optional[Callable] = None,
+ ):
+ """Initialize the Temporal executor.
+
+ Args:
+ client: Connected Temporal client
+ task_queue: Temporal task queue name
+ status_callback: Optional callback for node status updates
+ """
+ self.client = client
+ self.task_queue = task_queue
+ self.status_callback = status_callback
+
+ async def execute_workflow(
+ self,
+ workflow_id: str,
+ nodes: List[Dict],
+ edges: List[Dict],
+ session_id: str = "default",
+ enable_caching: bool = True,
+ ) -> Dict[str, Any]:
+ """Execute a workflow using Temporal.
+
+ Args:
+ workflow_id: Unique workflow identifier
+ nodes: List of node definitions
+ edges: List of edge definitions
+ session_id: Session identifier
+ enable_caching: Whether to enable result caching (passed to activity)
+
+ Returns:
+ Dict with success, outputs, execution_trace, and timing info
+ """
+ start_time = time.time()
+ execution_id = f"temporal-{uuid.uuid4().hex[:8]}"
+
+ logger.info(
+ "Starting Temporal workflow execution",
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ node_count=len(nodes),
+ edge_count=len(edges),
+ )
+
+ try:
+ # Execute workflow via Temporal
+ result = await self.client.execute_workflow(
+ MachinaWorkflow.run,
+ {
+ "nodes": nodes,
+ "edges": edges,
+ "session_id": session_id,
+ "workflow_id": workflow_id,
+ },
+ id=execution_id,
+ task_queue=self.task_queue,
+ )
+
+ execution_time = time.time() - start_time
+
+ # Notify status callback for completed nodes
+ if self.status_callback and result.get("success"):
+ for node_id in result.get("execution_trace", []):
+ try:
+ await self.status_callback(
+ node_id,
+ "completed",
+ result.get("outputs", {}).get(node_id, {}),
+ )
+ except Exception as e:
+ logger.warning(
+ f"Status callback error for node {node_id}: {e}"
+ )
+
+ logger.info(
+ "Temporal workflow completed",
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ success=result.get("success"),
+ nodes_executed=len(result.get("execution_trace", [])),
+ execution_time=execution_time,
+ )
+
+ return {
+ "success": result.get("success", False),
+ "execution_id": execution_id,
+ "nodes_executed": result.get("execution_trace", []),
+ "outputs": result.get("outputs", {}),
+ "errors": [result.get("error")] if result.get("error") else [],
+ "execution_time": execution_time,
+ "temporal_execution": True,
+ "timestamp": datetime.now().isoformat(),
+ }
+
+ except Exception as e:
+ import traceback
+ execution_time = time.time() - start_time
+ error_details = f"{type(e).__name__}: {str(e)}"
+ tb = traceback.format_exc()
+ logger.error(
+ f"Temporal workflow failed: {error_details}",
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ traceback=tb,
+ )
+ print(f"[Temporal] Workflow failed with exception:\n{tb}")
+
+ return {
+ "success": False,
+ "execution_id": execution_id,
+ "nodes_executed": [],
+ "outputs": {},
+ "errors": [error_details],
+ "execution_time": execution_time,
+ "temporal_execution": True,
+ "timestamp": datetime.now().isoformat(),
+ }
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
new file mode 100644
index 00000000..8d5f03dd
--- /dev/null
+++ b/server/services/temporal/worker.py
@@ -0,0 +1,133 @@
+"""Temporal worker for MachinaOs.
+
+The worker polls the task queue and executes workflows and activities.
+It can be run as a standalone process or embedded in the main server.
+"""
+
+import asyncio
+from typing import Optional
+
+from temporalio.client import Client
+from temporalio.worker import Worker
+
+from core.logging import get_logger
+from .workflow import MachinaWorkflow
+from .activities import execute_workflow_with_continuous_scheduling
+
+logger = get_logger(__name__)
+
+
+class TemporalWorkerManager:
+ """Manages the Temporal worker lifecycle."""
+
+ def __init__(
+ self,
+ client: Client,
+ task_queue: str = "machina-tasks",
+ ):
+ """Initialize the worker manager.
+
+ Args:
+ client: Connected Temporal client
+ task_queue: Task queue name to poll
+ """
+ self.client = client
+ self.task_queue = task_queue
+ self._worker: Optional[Worker] = None
+ self._worker_task: Optional[asyncio.Task] = None
+
+ @property
+ def is_running(self) -> bool:
+ """Check if the worker is running."""
+ return self._worker_task is not None and not self._worker_task.done()
+
+ async def start(self) -> None:
+ """Start the Temporal worker in the background."""
+ if self.is_running:
+ logger.warning("Temporal worker already running")
+ return
+
+ self._worker = Worker(
+ self.client,
+ task_queue=self.task_queue,
+ workflows=[MachinaWorkflow],
+ activities=[execute_workflow_with_continuous_scheduling],
+ )
+
+ logger.info(
+ "Starting Temporal worker",
+ task_queue=self.task_queue,
+ )
+
+ # Run worker in background task
+ self._worker_task = asyncio.create_task(
+ self._run_worker(),
+ name="temporal-worker",
+ )
+
+ async def _run_worker(self) -> None:
+ """Run the worker (background task)."""
+ try:
+ await self._worker.run()
+ except asyncio.CancelledError:
+ logger.info("Temporal worker cancelled")
+ except Exception as e:
+ logger.error(f"Temporal worker error: {str(e)}")
+ raise
+
+ async def stop(self) -> None:
+ """Stop the Temporal worker."""
+ if not self.is_running:
+ return
+
+ logger.info("Stopping Temporal worker")
+
+ if self._worker_task:
+ self._worker_task.cancel()
+ try:
+ await self._worker_task
+ except asyncio.CancelledError:
+ pass
+ self._worker_task = None
+
+ self._worker = None
+ logger.info("Temporal worker stopped")
+
+
+async def run_standalone_worker(
+ server_address: str = "localhost:7233",
+ namespace: str = "default",
+ task_queue: str = "machina-tasks",
+) -> None:
+ """Run the Temporal worker as a standalone process.
+
+ This can be used for running workers separately from the main server.
+
+ Args:
+ server_address: Temporal server address
+ namespace: Temporal namespace
+ task_queue: Task queue to poll
+ """
+ logger.info(
+ "Starting standalone Temporal worker",
+ server_address=server_address,
+ namespace=namespace,
+ task_queue=task_queue,
+ )
+
+ client = await Client.connect(server_address, namespace=namespace)
+
+ worker = Worker(
+ client,
+ task_queue=task_queue,
+ workflows=[MachinaWorkflow],
+ activities=[execute_workflow_with_continuous_scheduling],
+ )
+
+ logger.info("Worker running. Press Ctrl+C to stop.")
+ await worker.run()
+
+
+if __name__ == "__main__":
+ # Allow running worker standalone
+ asyncio.run(run_standalone_worker())
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
new file mode 100644
index 00000000..4bf27c6c
--- /dev/null
+++ b/server/services/temporal/workflow.py
@@ -0,0 +1,71 @@
+"""Temporal workflow definition for MachinaOs.
+
+The workflow is a thin wrapper that delegates execution to a continuous
+scheduling activity. Uses asyncio.wait(FIRST_COMPLETED) pattern for immediate
+downstream execution when dependencies complete - no waiting for sibling branches.
+
+This replaces LangGraph's Pregel/BSP model which uses supersteps where all
+parallel nodes must complete before downstream nodes can start.
+"""
+
+from datetime import timedelta
+from typing import Any, Dict
+
+from temporalio import workflow
+from temporalio.common import RetryPolicy
+
+# Import activities through the unsafe imports mechanism
+# This is required because Temporal sandboxes workflow imports
+with workflow.unsafe.imports_passed_through():
+ from .activities import execute_workflow_with_continuous_scheduling
+
+
[email protected](sandboxed=False)
+class MachinaWorkflow:
+ """Temporal workflow with continuous scheduling orchestration.
+
+ The actual graph execution happens in an activity using the
+ asyncio.wait(FIRST_COMPLETED) pattern for true parallel pipelines
+ where each path progresses independently.
+ """
+
+ @workflow.run
+ async def run(self, workflow_data: Dict[str, Any]) -> Dict[str, Any]:
+ """Execute workflow via continuous scheduling activity.
+
+ Args:
+ workflow_data: Dict containing:
+ - nodes: List of node definitions
+ - edges: List of edge definitions
+ - session_id: Session identifier
+ - workflow_id: Workflow ID for tracking
+
+ Returns:
+ Dict with success, outputs, and execution_trace
+ """
+ nodes = workflow_data.get("nodes", [])
+ edges = workflow_data.get("edges", [])
+ session_id = workflow_data.get("session_id", "default")
+ workflow_id = workflow_data.get("workflow_id")
+
+ if not nodes:
+ return {
+ "success": False,
+ "error": "No nodes provided",
+ "outputs": {},
+ "execution_trace": [],
+ }
+
+ # Execute the entire workflow in a single activity
+ # Continuous scheduling handles parallel execution inside the activity
+ return await workflow.execute_activity(
+ execute_workflow_with_continuous_scheduling,
+ args=[nodes, edges, session_id, workflow_id],
+ start_to_close_timeout=timedelta(minutes=30),
+ heartbeat_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(
+ initial_interval=timedelta(seconds=1),
+ maximum_interval=timedelta(seconds=60),
+ maximum_attempts=3,
+ ),
+ )
diff --git a/server/services/workflow.py b/server/services/workflow.py
index 2f2ab4d9..6f2f0de1 100644
--- a/server/services/workflow.py
+++ b/server/services/workflow.py
@@ -5,6 +5,7 @@ This is a thin facade that delegates to specialized modules:
- ParameterResolver: Template variable resolution
- DeploymentManager: Event-driven deployment lifecycle
- WorkflowExecutor: Parallel/sequential orchestration
+- TemporalExecutor: Durable workflow execution (optional)
Following n8n/Conductor patterns for clean separation of concerns.
"""
@@ -28,6 +29,7 @@ if TYPE_CHECKING:
from services.maps import MapsService
from services.text import TextService
from services.android_service import AndroidService
+ from services.temporal import TemporalExecutor
logger = get_logger(__name__)
@@ -39,7 +41,7 @@ class WorkflowService:
- Node execution (NodeExecutor)
- Parameter resolution (ParameterResolver)
- Deployment lifecycle (DeploymentManager)
- - Workflow orchestration (WorkflowExecutor)
+ - Workflow orchestration (WorkflowExecutor or TemporalExecutor)
"""
def __init__(
@@ -79,6 +81,9 @@ class WorkflowService:
self._execution_cache = ExecutionCache(cache)
self._workflow_executor: Optional[WorkflowExecutor] = None
+ # Temporal executor (set via set_temporal_executor when enabled)
+ self._temporal_executor: Optional["TemporalExecutor"] = None
+
# Initialize DeploymentManager (lazy - needs broadcaster)
self._deployment_manager: Optional[DeploymentManager] = None
self._broadcaster = None
@@ -90,6 +95,15 @@ class WorkflowService:
"use_parallel_executor": True,
}
+ def set_temporal_executor(self, executor: "TemporalExecutor") -> None:
+ """Set the Temporal executor for durable workflow execution.
+
+ Args:
+ executor: Configured TemporalExecutor instance
+ """
+ self._temporal_executor = executor
+ logger.info("Temporal executor configured for workflow execution")
+
def _get_deployment_manager(self) -> DeploymentManager:
"""Get or create DeploymentManager."""
if self._deployment_manager is None:
@@ -178,6 +192,7 @@ class WorkflowService:
use_parallel: bool = None,
skip_clear_outputs: bool = False,
workflow_id: Optional[str] = None,
+ use_temporal: bool = None,
) -> Dict[str, Any]:
"""Execute entire workflow.
@@ -189,6 +204,7 @@ class WorkflowService:
use_parallel: Force parallel/sequential execution
skip_clear_outputs: Skip clearing outputs (for deployment runs)
workflow_id: Workflow ID for per-workflow status scoping (n8n pattern)
+ use_temporal: Force Temporal execution (None = use settings default)
"""
start_time = time.time()
@@ -208,6 +224,14 @@ class WorkflowService:
if use_parallel is None:
use_parallel = self._settings.get("use_parallel_executor", True)
+ # Check if Temporal execution is requested
+ if use_temporal is None:
+ use_temporal = self.settings.temporal_enabled
+
+ # Use Temporal if enabled and executor is configured
+ if use_temporal and self._temporal_executor is not None:
+ return await self._execute_temporal(nodes, edges, session_id, status_callback, start_time, workflow_id)
+
# Use parallel executor if enabled and Redis available
if use_parallel and self.settings.redis_enabled:
return await self._execute_parallel(nodes, edges, session_id, status_callback, start_time, workflow_id)
@@ -215,6 +239,49 @@ class WorkflowService:
# Fall back to sequential
return await self._execute_sequential(nodes, edges, session_id, status_callback, start_time, workflow_id)
+ async def _execute_temporal(self, nodes, edges, session_id, status_callback, start_time, workflow_id: Optional[str] = None) -> Dict:
+ """Execute with Temporal for durable workflow orchestration."""
+ # Use passed workflow_id (from deployment) or generate new one
+ if not workflow_id:
+ workflow_id = f"temporal_{session_id}_{int(time.time() * 1000)}"
+
+ logger.info(
+ "Executing workflow via Temporal",
+ workflow_id=workflow_id,
+ node_count=len(nodes),
+ )
+
+ result = await self._temporal_executor.execute_workflow(
+ workflow_id=workflow_id,
+ nodes=nodes,
+ edges=edges,
+ session_id=session_id,
+ enable_caching=True,
+ )
+
+ # Notify status callback for completed nodes if provided
+ if status_callback and result.get("success"):
+ for node_id in result.get("nodes_executed", []):
+ try:
+ await status_callback(
+ node_id,
+ "completed",
+ result.get("outputs", {}).get(node_id, {}),
+ )
+ except Exception:
+ pass
+
+ return {
+ "success": result.get("success", False),
+ "execution_id": result.get("execution_id"),
+ "nodes_executed": result.get("nodes_executed", []),
+ "outputs": result.get("outputs", {}),
+ "errors": result.get("errors", []),
+ "execution_time": result.get("execution_time", time.time() - start_time),
+ "temporal_execution": True,
+ "timestamp": datetime.now().isoformat(),
+ }
+
async def _execute_parallel(self, nodes, edges, session_id, status_callback, start_time, workflow_id: Optional[str] = None) -> Dict:
"""Execute with parallel orchestration engine."""
# Use passed workflow_id (from deployment) or generate new one