Skip to content

Self-improving AI Employees for everyone

fix: graceful Temporal connection with retry, Docker Temporal service

trohitg247718afe588
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 5b588e44..55a33542 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -15,6 +15,27 @@ services:
retries: 3
command: redis-server --appendonly yes
+ # Temporal Server (durable workflow execution)
+ temporal:
+ image: temporalio/auto-setup:latest
+ container_name: machina-temporal
+ networks:
+ - app-network
+ restart: always
+ healthcheck:
+ test: ["CMD", "tctl", "--address", "temporal:7233", "cluster", "health"]
+ interval: 15s
+ timeout: 5s
+ retries: 5
+ start_period: 30s
+ environment:
+ - DB=sqlite
+ - SQLITE_DB_PATH=/data/temporal.db
+ - TEMPORAL_ADDRESS=temporal:7233
+ - TZ=UTC
+ volumes:
+ - temporal-data:/data
+
# Python FastAPI Backend
backend:
image: machinaos-backend:latest
@@ -45,6 +66,8 @@ services:
- ENV=production
- REDIS_ENABLED=true
- REDIS_URL=redis://redis:6379
+ - TEMPORAL_ENABLED=true
+ - TEMPORAL_SERVER_ADDRESS=temporal:7233
- WHATSAPP_RPC_URL=ws://whatsapp:${WHATSAPP_RPC_PORT:-9400}/ws/rpc
- CREDENTIALS_DB_PATH=/app/credentials/credentials.db
- CREDENTIAL_BACKEND=${CREDENTIAL_BACKEND:-fernet}
@@ -52,6 +75,8 @@ services:
depends_on:
redis:
condition: service_healthy
+ temporal:
+ condition: service_healthy
whatsapp:
condition: service_healthy
@@ -110,6 +135,8 @@ networks:
volumes:
redis-data:
driver: local
+ temporal-data:
+ driver: local
backend-data:
driver: local
credentials-data:
diff --git a/docker-compose.yml b/docker-compose.yml
index da0251a1..3e9e8a08 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -19,6 +19,32 @@ services:
retries: 3
command: redis-server --appendonly yes
+ # Temporal Server (durable workflow execution)
+ temporal:
+ image: temporalio/auto-setup:latest
+ container_name: machina-temporal
+ profiles:
+ - temporal
+ ports:
+ - "7233:7233" # gRPC frontend
+ - "8233:8233" # Web UI
+ networks:
+ - app-network
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD", "tctl", "--address", "temporal:7233", "cluster", "health"]
+ interval: 15s
+ timeout: 5s
+ retries: 5
+ start_period: 30s
+ environment:
+ - DB=sqlite
+ - SQLITE_DB_PATH=/data/temporal.db
+ - TEMPORAL_ADDRESS=temporal:7233
+ - TZ=UTC
+ volumes:
+ - temporal-data:/data
+
# Python FastAPI Backend
backend:
build:
@@ -47,6 +73,7 @@ services:
# Override specific vars for Docker networking
- PORT=${PYTHON_BACKEND_PORT:-3010}
- REDIS_URL=redis://redis:6379
+ - TEMPORAL_SERVER_ADDRESS=temporal:7233
- WHATSAPP_RPC_URL=ws://whatsapp:${WHATSAPP_RPC_PORT:-9400}/ws/rpc
- CREDENTIALS_DB_PATH=/app/credentials/credentials.db
- TZ=UTC
@@ -101,6 +128,8 @@ networks:
volumes:
redis-data:
driver: local
+ temporal-data:
+ driver: local
backend-data:
driver: local
credentials-data:
diff --git a/scripts/docker.js b/scripts/docker.js
index d6ddbd96..b5df8dea 100755
--- a/scripts/docker.js
+++ b/scripts/docker.js
@@ -39,6 +39,14 @@ if (config.redisEnabled) {
console.log('[Docker] Redis profile disabled (REDIS_ENABLED=false in .env)');
}
+// Add Temporal profile if enabled in .env
+if (config.temporalEnabled) {
+ composeArgs.push('--profile', 'temporal');
+ console.log('[Docker] Temporal profile enabled (TEMPORAL_ENABLED=true in .env)');
+} else {
+ console.log('[Docker] Temporal profile disabled (TEMPORAL_ENABLED=false in .env)');
+}
+
// Add command
composeArgs.push(command);
diff --git a/server/core/container.py b/server/core/container.py
index d4124cc0..f647b69f 100644
--- a/server/core/container.py
+++ b/server/core/container.py
@@ -30,19 +30,11 @@ from services.user_auth import UserAuthService
from services.compaction import init_compaction_service
_clog("all service imports done")
-# Temporal is optional - import with fallback
-try:
- from services.temporal import TemporalClientWrapper
- _TEMPORAL_AVAILABLE = True
-except ImportError:
- TemporalClientWrapper = None
- _TEMPORAL_AVAILABLE = False
+from services.temporal import TemporalClientWrapper
def _create_temporal_client(server_address: str, namespace: str):
- """Factory function for temporal client - returns None if not available."""
- if not _TEMPORAL_AVAILABLE or TemporalClientWrapper is None:
- return None
+ """Factory function for temporal client."""
return TemporalClientWrapper(server_address=server_address, namespace=namespace)
@@ -82,7 +74,7 @@ class Container(containers.DeclarativeContainer):
encryption=encryption_service
)
- # Temporal client (lazy - returns None if temporalio not installed)
+ # Temporal client
temporal_client = providers.Singleton(
_create_temporal_client,
server_address=settings.provided.temporal_server_address,
diff --git a/server/main.py b/server/main.py
index 5fc48a75..2b4d2375 100644
--- a/server/main.py
+++ b/server/main.py
@@ -189,27 +189,21 @@ async def lifespan(app: FastAPI):
# Initialize Temporal if enabled
temporal_worker_manager = None
if settings.temporal_enabled:
- try:
- from services.temporal import TemporalClientWrapper, TemporalExecutor
- from services.temporal.worker import TemporalWorkerManager
-
- # Check if temporal is actually available (temporalio installed)
- if TemporalClientWrapper is None:
- raise ImportError("temporalio not installed")
-
- logger.info(
- "Initializing Temporal integration",
- server_address=settings.temporal_server_address,
- namespace=settings.temporal_namespace,
- task_queue=settings.temporal_task_queue,
- )
+ from services.temporal import TemporalClientWrapper, TemporalExecutor
+ from services.temporal.worker import TemporalWorkerManager
+
+ logger.info(
+ "Initializing Temporal integration",
+ server_address=settings.temporal_server_address,
+ namespace=settings.temporal_namespace,
+ task_queue=settings.temporal_task_queue,
+ )
- # Connect Temporal client
- temporal_client_wrapper = container.temporal_client()
- if temporal_client_wrapper is None:
- raise ImportError("temporalio not installed")
- temporal_client = await temporal_client_wrapper.connect()
+ # Connect Temporal client (retries 3 times, then falls back)
+ temporal_client_wrapper = container.temporal_client()
+ temporal_client = await temporal_client_wrapper.connect(retries=3, delay=2.0)
+ if temporal_client is not None:
# Create and set the Temporal executor on WorkflowService
temporal_executor = TemporalExecutor(
client=temporal_client,
@@ -225,10 +219,14 @@ async def lifespan(app: FastAPI):
await temporal_worker_manager.start()
logger.info("Temporal integration initialized successfully")
-
- except Exception as e:
- logger.warning(f"Temporal not available (optional): {str(e)}")
- logger.info("Continuing without Temporal - using local parallel/sequential execution")
+ else:
+ logger.warning(
+ f"Temporal server not reachable at {settings.temporal_server_address}. "
+ "Falling back to local parallel/sequential execution. "
+ "Start Temporal server and restart to enable durable workflows."
+ )
+ # Start background reconnect loop (tries every 30s)
+ await temporal_client_wrapper.start_background_reconnect(interval=30.0)
_startup_log("All services initialized")
print("Application startup complete", flush=True)
@@ -243,7 +241,7 @@ async def lifespan(app: FastAPI):
await temporal_worker_manager.stop()
logger.info("Temporal worker stopped")
- # Disconnect Temporal client if connected
+ # Disconnect Temporal client (also cancels background reconnect)
if settings.temporal_enabled:
try:
temporal_client_wrapper = container.temporal_client()
diff --git a/server/services/temporal/__init__.py b/server/services/temporal/__init__.py
index 757c23c4..c3d517af 100644
--- a/server/services/temporal/__init__.py
+++ b/server/services/temporal/__init__.py
@@ -13,21 +13,11 @@ When TEMPORAL_ENABLED=true:
- Each node is a separate activity with its own retry policy
- Parallel branches execute concurrently on available workers
-When TEMPORAL_ENABLED=false (default):
+When TEMPORAL_ENABLED=false:
- Falls back to the existing parallel/sequential executor
-
-Note: temporalio is an optional dependency. Import with try/except.
"""
__all__ = ["TemporalExecutor", "TemporalClientWrapper"]
-# Lazy imports - temporalio is optional
-TemporalExecutor = None
-TemporalClientWrapper = None
-
-try:
- from .executor import TemporalExecutor
- from .client import TemporalClientWrapper
-except ImportError:
- # temporalio not installed - Temporal features disabled
- pass
+from .executor import TemporalExecutor
+from .client import TemporalClientWrapper
diff --git a/server/services/temporal/client.py b/server/services/temporal/client.py
index d9b55d67..892612e0 100644
--- a/server/services/temporal/client.py
+++ b/server/services/temporal/client.py
@@ -1,8 +1,9 @@
"""Temporal client wrapper for MachinaOs.
-Manages the Temporal client connection lifecycle.
+Manages the Temporal client connection lifecycle with retry support.
"""
+import asyncio
from typing import Optional
from temporalio.client import Client
from temporalio.runtime import Runtime, TelemetryConfig
@@ -16,16 +17,11 @@ class TemporalClientWrapper:
"""Wrapper around Temporal client for lifecycle management."""
def __init__(self, server_address: str, namespace: str = "default"):
- """Initialize the client wrapper.
-
- Args:
- server_address: Temporal server address (e.g., "localhost:7233")
- namespace: Temporal namespace to use
- """
self.server_address = server_address
self.namespace = namespace
self._client: Optional[Client] = None
self._runtime: Optional[Runtime] = None
+ self._reconnect_task: Optional[asyncio.Task] = None
@property
def client(self) -> Optional[Client]:
@@ -37,40 +33,71 @@ class TemporalClientWrapper:
"""Check if client is connected."""
return self._client is not None
- async def connect(self) -> Client:
- """Connect to the Temporal server.
+ async def connect(self, retries: int = 3, delay: float = 2.0) -> Optional[Client]:
+ """Connect to the Temporal server with retries.
Returns:
- The connected Temporal client
+ The connected Temporal client, or None if connection failed.
"""
if self._client is not None:
return self._client
- logger.info(
- "Connecting to Temporal server",
- server_address=self.server_address,
- namespace=self.namespace,
- )
-
- # Create runtime with worker heartbeating disabled to avoid warning on older servers
- self._runtime = Runtime(
- telemetry=TelemetryConfig(),
- worker_heartbeat_interval=None,
+ # Create runtime once (reusable across reconnects)
+ if self._runtime is None:
+ self._runtime = Runtime(
+ telemetry=TelemetryConfig(),
+ worker_heartbeat_interval=None,
+ )
+
+ for attempt in range(1, retries + 1):
+ try:
+ logger.info(
+ f"Connecting to Temporal server (attempt {attempt}/{retries})",
+ server_address=self.server_address,
+ namespace=self.namespace,
+ )
+ self._client = await Client.connect(
+ self.server_address,
+ namespace=self.namespace,
+ runtime=self._runtime,
+ )
+ logger.info("Connected to Temporal server")
+ return self._client
+ except Exception as e:
+ logger.warning(
+ f"Temporal connection attempt {attempt}/{retries} failed: {e}"
+ )
+ if attempt < retries:
+ await asyncio.sleep(delay)
+
+ logger.error(
+ f"Failed to connect to Temporal server at {self.server_address} after {retries} attempts"
)
-
- self._client = await Client.connect(
- self.server_address,
- namespace=self.namespace,
- runtime=self._runtime,
- )
-
- logger.info("Connected to Temporal server")
- return self._client
+ return None
+
+ async def start_background_reconnect(self, interval: float = 30.0) -> None:
+ """Start a background task that periodically tries to reconnect."""
+ if self._reconnect_task is not None:
+ return
+
+ async def _reconnect_loop():
+ while True:
+ await asyncio.sleep(interval)
+ if self._client is not None:
+ continue
+ logger.info("Attempting Temporal reconnect...")
+ client = await self.connect(retries=1, delay=0)
+ if client:
+ logger.info("Temporal reconnected successfully")
+ break
+
+ self._reconnect_task = asyncio.create_task(_reconnect_loop())
async def disconnect(self) -> None:
"""Disconnect from the Temporal server."""
+ if self._reconnect_task is not None:
+ self._reconnect_task.cancel()
+ self._reconnect_task = None
if self._client is not None:
- # Temporal client doesn't have an explicit close method,
- # but we clear the reference
self._client = None
logger.info("Disconnected from Temporal server")