Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix: graceful Temporal connection with retry, Docker Temporal service
trohitg247718afe588
diff --git a/docker-compose.prod.yml b/docker-compose.prod.ymlindex 5b588e44..55a33542 100644--- a/docker-compose.prod.yml+++ b/docker-compose.prod.yml@@ -15,6 +15,27 @@ services:retries: 3command: redis-server --appendonly yes+ # Temporal Server (durable workflow execution)+ temporal:+ image: temporalio/auto-setup:latest+ container_name: machina-temporal+ networks:+ - app-network+ restart: always+ healthcheck:+ test: ["CMD", "tctl", "--address", "temporal:7233", "cluster", "health"]+ interval: 15s+ timeout: 5s+ retries: 5+ start_period: 30s+ environment:+ - DB=sqlite+ - SQLITE_DB_PATH=/data/temporal.db+ - TEMPORAL_ADDRESS=temporal:7233+ - TZ=UTC+ volumes:+ - temporal-data:/data+# Python FastAPI Backendbackend:image: machinaos-backend:latest@@ -45,6 +66,8 @@ services:- ENV=production- REDIS_ENABLED=true- REDIS_URL=redis://redis:6379+ - TEMPORAL_ENABLED=true+ - TEMPORAL_SERVER_ADDRESS=temporal:7233- WHATSAPP_RPC_URL=ws://whatsapp:${WHATSAPP_RPC_PORT:-9400}/ws/rpc- CREDENTIALS_DB_PATH=/app/credentials/credentials.db- CREDENTIAL_BACKEND=${CREDENTIAL_BACKEND:-fernet}@@ -52,6 +75,8 @@ services:depends_on:redis:condition: service_healthy+ temporal:+ condition: service_healthywhatsapp:condition: service_healthy@@ -110,6 +135,8 @@ networks:volumes:redis-data:driver: local+ temporal-data:+ driver: localbackend-data:driver: localcredentials-data:diff --git a/docker-compose.yml b/docker-compose.ymlindex da0251a1..3e9e8a08 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -19,6 +19,32 @@ services:retries: 3command: redis-server --appendonly yes+ # Temporal Server (durable workflow execution)+ temporal:+ image: temporalio/auto-setup:latest+ container_name: machina-temporal+ profiles:+ - temporal+ ports:+ - "7233:7233" # gRPC frontend+ - "8233:8233" # Web UI+ networks:+ - app-network+ restart: unless-stopped+ healthcheck:+ test: ["CMD", "tctl", "--address", "temporal:7233", "cluster", "health"]+ interval: 15s+ timeout: 5s+ retries: 5+ start_period: 30s+ environment:+ - DB=sqlite+ - SQLITE_DB_PATH=/data/temporal.db+ - TEMPORAL_ADDRESS=temporal:7233+ - TZ=UTC+ volumes:+ - temporal-data:/data+# Python FastAPI Backendbackend:build:@@ -47,6 +73,7 @@ services:# Override specific vars for Docker networking- PORT=${PYTHON_BACKEND_PORT:-3010}- REDIS_URL=redis://redis:6379+ - TEMPORAL_SERVER_ADDRESS=temporal:7233- WHATSAPP_RPC_URL=ws://whatsapp:${WHATSAPP_RPC_PORT:-9400}/ws/rpc- CREDENTIALS_DB_PATH=/app/credentials/credentials.db- TZ=UTC@@ -101,6 +128,8 @@ networks:volumes:redis-data:driver: local+ temporal-data:+ driver: localbackend-data:driver: localcredentials-data:diff --git a/scripts/docker.js b/scripts/docker.jsindex d6ddbd96..b5df8dea 100755--- a/scripts/docker.js+++ b/scripts/docker.js@@ -39,6 +39,14 @@ if (config.redisEnabled) {console.log('[Docker] Redis profile disabled (REDIS_ENABLED=false in .env)');}+// Add Temporal profile if enabled in .env+if (config.temporalEnabled) {+ composeArgs.push('--profile', 'temporal');+ console.log('[Docker] Temporal profile enabled (TEMPORAL_ENABLED=true in .env)');+} else {+ console.log('[Docker] Temporal profile disabled (TEMPORAL_ENABLED=false in .env)');+}+// Add commandcomposeArgs.push(command);diff --git a/server/core/container.py b/server/core/container.pyindex d4124cc0..f647b69f 100644--- a/server/core/container.py+++ b/server/core/container.py@@ -30,19 +30,11 @@ from services.user_auth import UserAuthServicefrom services.compaction import init_compaction_service_clog("all service imports done")-# Temporal is optional - import with fallback-try:- from services.temporal import TemporalClientWrapper- _TEMPORAL_AVAILABLE = True-except ImportError:- TemporalClientWrapper = None- _TEMPORAL_AVAILABLE = False+from services.temporal import TemporalClientWrapperdef _create_temporal_client(server_address: str, namespace: str):- """Factory function for temporal client - returns None if not available."""- if not _TEMPORAL_AVAILABLE or TemporalClientWrapper is None:- return None+ """Factory function for temporal client."""return TemporalClientWrapper(server_address=server_address, namespace=namespace)@@ -82,7 +74,7 @@ class Container(containers.DeclarativeContainer):encryption=encryption_service)- # Temporal client (lazy - returns None if temporalio not installed)+ # Temporal clienttemporal_client = providers.Singleton(_create_temporal_client,server_address=settings.provided.temporal_server_address,diff --git a/server/main.py b/server/main.pyindex 5fc48a75..2b4d2375 100644--- a/server/main.py+++ b/server/main.py@@ -189,27 +189,21 @@ async def lifespan(app: FastAPI):# Initialize Temporal if enabledtemporal_worker_manager = Noneif settings.temporal_enabled:- try:- from services.temporal import TemporalClientWrapper, TemporalExecutor- from services.temporal.worker import TemporalWorkerManager-- # Check if temporal is actually available (temporalio installed)- if TemporalClientWrapper is None:- raise ImportError("temporalio not installed")-- logger.info(- "Initializing Temporal integration",- server_address=settings.temporal_server_address,- namespace=settings.temporal_namespace,- task_queue=settings.temporal_task_queue,- )+ from services.temporal import TemporalClientWrapper, TemporalExecutor+ from services.temporal.worker import TemporalWorkerManager++ logger.info(+ "Initializing Temporal integration",+ server_address=settings.temporal_server_address,+ namespace=settings.temporal_namespace,+ task_queue=settings.temporal_task_queue,+ )- # Connect Temporal client- temporal_client_wrapper = container.temporal_client()- if temporal_client_wrapper is None:- raise ImportError("temporalio not installed")- temporal_client = await temporal_client_wrapper.connect()+ # Connect Temporal client (retries 3 times, then falls back)+ temporal_client_wrapper = container.temporal_client()+ temporal_client = await temporal_client_wrapper.connect(retries=3, delay=2.0)+ if temporal_client is not None:# Create and set the Temporal executor on WorkflowServicetemporal_executor = TemporalExecutor(client=temporal_client,@@ -225,10 +219,14 @@ async def lifespan(app: FastAPI):await temporal_worker_manager.start()logger.info("Temporal integration initialized successfully")-- except Exception as e:- logger.warning(f"Temporal not available (optional): {str(e)}")- logger.info("Continuing without Temporal - using local parallel/sequential execution")+ else:+ logger.warning(+ f"Temporal server not reachable at {settings.temporal_server_address}. "+ "Falling back to local parallel/sequential execution. "+ "Start Temporal server and restart to enable durable workflows."+ )+ # Start background reconnect loop (tries every 30s)+ await temporal_client_wrapper.start_background_reconnect(interval=30.0)_startup_log("All services initialized")print("Application startup complete", flush=True)@@ -243,7 +241,7 @@ async def lifespan(app: FastAPI):await temporal_worker_manager.stop()logger.info("Temporal worker stopped")- # Disconnect Temporal client if connected+ # Disconnect Temporal client (also cancels background reconnect)if settings.temporal_enabled:try:temporal_client_wrapper = container.temporal_client()diff --git a/server/services/temporal/__init__.py b/server/services/temporal/__init__.pyindex 757c23c4..c3d517af 100644--- a/server/services/temporal/__init__.py+++ b/server/services/temporal/__init__.py@@ -13,21 +13,11 @@ When TEMPORAL_ENABLED=true:- Each node is a separate activity with its own retry policy- Parallel branches execute concurrently on available workers-When TEMPORAL_ENABLED=false (default):+When TEMPORAL_ENABLED=false:- Falls back to the existing parallel/sequential executor--Note: temporalio is an optional dependency. Import with try/except."""__all__ = ["TemporalExecutor", "TemporalClientWrapper"]-# Lazy imports - temporalio is optional-TemporalExecutor = None-TemporalClientWrapper = None--try:- from .executor import TemporalExecutor- from .client import TemporalClientWrapper-except ImportError:- # temporalio not installed - Temporal features disabled- pass+from .executor import TemporalExecutor+from .client import TemporalClientWrapperdiff --git a/server/services/temporal/client.py b/server/services/temporal/client.pyindex d9b55d67..892612e0 100644--- a/server/services/temporal/client.py+++ b/server/services/temporal/client.py@@ -1,8 +1,9 @@"""Temporal client wrapper for MachinaOs.-Manages the Temporal client connection lifecycle.+Manages the Temporal client connection lifecycle with retry support."""+import asynciofrom typing import Optionalfrom temporalio.client import Clientfrom temporalio.runtime import Runtime, TelemetryConfig@@ -16,16 +17,11 @@ class TemporalClientWrapper:"""Wrapper around Temporal client for lifecycle management."""def __init__(self, server_address: str, namespace: str = "default"):- """Initialize the client wrapper.-- Args:- server_address: Temporal server address (e.g., "localhost:7233")- namespace: Temporal namespace to use- """self.server_address = server_addressself.namespace = namespaceself._client: Optional[Client] = Noneself._runtime: Optional[Runtime] = None+ self._reconnect_task: Optional[asyncio.Task] = None@propertydef client(self) -> Optional[Client]:@@ -37,40 +33,71 @@ class TemporalClientWrapper:"""Check if client is connected."""return self._client is not None- async def connect(self) -> Client:- """Connect to the Temporal server.+ async def connect(self, retries: int = 3, delay: float = 2.0) -> Optional[Client]:+ """Connect to the Temporal server with retries.Returns:- The connected Temporal client+ The connected Temporal client, or None if connection failed."""if self._client is not None:return self._client- logger.info(- "Connecting to Temporal server",- server_address=self.server_address,- namespace=self.namespace,- )-- # Create runtime with worker heartbeating disabled to avoid warning on older servers- self._runtime = Runtime(- telemetry=TelemetryConfig(),- worker_heartbeat_interval=None,+ # Create runtime once (reusable across reconnects)+ if self._runtime is None:+ self._runtime = Runtime(+ telemetry=TelemetryConfig(),+ worker_heartbeat_interval=None,+ )++ for attempt in range(1, retries + 1):+ try:+ logger.info(+ f"Connecting to Temporal server (attempt {attempt}/{retries})",+ server_address=self.server_address,+ namespace=self.namespace,+ )+ self._client = await Client.connect(+ self.server_address,+ namespace=self.namespace,+ runtime=self._runtime,+ )+ logger.info("Connected to Temporal server")+ return self._client+ except Exception as e:+ logger.warning(+ f"Temporal connection attempt {attempt}/{retries} failed: {e}"+ )+ if attempt < retries:+ await asyncio.sleep(delay)++ logger.error(+ f"Failed to connect to Temporal server at {self.server_address} after {retries} attempts")-- self._client = await Client.connect(- self.server_address,- namespace=self.namespace,- runtime=self._runtime,- )-- logger.info("Connected to Temporal server")- return self._client+ return None++ async def start_background_reconnect(self, interval: float = 30.0) -> None:+ """Start a background task that periodically tries to reconnect."""+ if self._reconnect_task is not None:+ return++ async def _reconnect_loop():+ while True:+ await asyncio.sleep(interval)+ if self._client is not None:+ continue+ logger.info("Attempting Temporal reconnect...")+ client = await self.connect(retries=1, delay=0)+ if client:+ logger.info("Temporal reconnected successfully")+ break++ self._reconnect_task = asyncio.create_task(_reconnect_loop())async def disconnect(self) -> None:"""Disconnect from the Temporal server."""+ if self._reconnect_task is not None:+ self._reconnect_task.cancel()+ self._reconnect_task = Noneif self._client is not None:- # Temporal client doesn't have an explicit close method,- # but we clear the referenceself._client = Nonelogger.info("Disconnected from Temporal server")