Skip to content

Self-improving AI Employees for everyone

feat(temporal): lifecycle module with dev-server watchdog and worker crash rebuilds

trohitg73830a331bcc
diff --git a/server/main.py b/server/main.py
index d104ff4e..7c7eb967 100644
--- a/server/main.py
+++ b/server/main.py
@@ -295,19 +295,23 @@ async def lifespan(app: FastAPI):
set_startup_time()
# Initialize Temporal in the background - do NOT block lifespan startup.
- # scripts/start.js launches the Python backend and temporal-server concurrently,
- # so on fresh startup temporal-server may take several seconds to become reachable.
- # Blocking the lifespan here would delay FastAPI HTTP serving and cascade into
- # frontend ERR_CONNECTION_REFUSED on /api/auth/status. Instead, yield fast and
- # let Temporal init happen in a background task. WorkflowService falls back to
- # parallel/sequential execution until Temporal is ready.
+ # This lifespan owns the Temporal dev server (spawned below via
+ # TemporalServerRuntime.ensure_started), and on fresh startup it may take
+ # several seconds to become reachable. Blocking the lifespan here would
+ # delay FastAPI HTTP serving and cascade into frontend
+ # ERR_CONNECTION_REFUSED on /api/auth/status. Instead, yield fast and
+ # let Temporal init happen in a background task. WorkflowService falls back
+ # to parallel/sequential execution until Temporal is ready.
app.state.temporal_worker_manager = None
app.state.temporal_pool = None
temporal_init_task: asyncio.Task | None = None
if settings.temporal_enabled:
- from services.temporal import TemporalExecutor
- from services.temporal.worker import TemporalWorkerManager
+ # The whole Temporal runtime story (dev-server supervision,
+ # connect loop, executor/worker wiring, boot-time control
+ # reconcile, resident dev-server watchdog) lives in
+ # services.temporal.lifecycle — main.py only schedules it.
+ from services.temporal.lifecycle import run_temporal_lifecycle
logger.info(
"Scheduling Temporal initialization in background",
@@ -316,111 +320,10 @@ async def lifespan(app: FastAPI):
task_queue=settings.temporal_task_queue,
)
_startup_log(f"[Temporal] Init scheduled for {settings.temporal_server_address}")
-
- async def _init_temporal_background() -> None:
- """Connect, wire executor, start worker. Retries every 3s until connected.
-
- This task is the single source of truth for Temporal lifecycle during
- startup. It replaces the previous blocking-connect + background-reconnect
- split, which blocked the lifespan for up to 30s and left the executor
- unwired after an initial failed connect.
- """
- temporal_client_wrapper = container.temporal_client()
- # Loopback address => this process owns the SQLite dev server,
- # via the same BaseSupervisor singleton pattern as the WhatsApp
- # runtime (teardown is covered by shutdown_all_supervisors()).
- # Remote/production clusters are never spawned at.
- _tp_host = settings.temporal_server_address.rsplit(":", 1)[0].strip("[]")
- _owns_dev_server = _tp_host in ("localhost", "127.0.0.1", "::1")
- attempt = 0
- while True:
- attempt += 1
- if _owns_dev_server:
- from services.temporal._runtime import get_temporal_server_runtime
-
- try:
- await get_temporal_server_runtime().ensure_started()
- except Exception as dev_exc: # noqa: BLE001 — connect loop retries
- _startup_log(f"[Temporal] Dev server start failed (attempt {attempt}): {dev_exc}")
- client = await temporal_client_wrapper.connect(retries=1, delay=0)
- if client is None:
- # Surface every failed attempt to stdout so users can see
- # the retry loop is still alive when "Temporal is up" but
- # the Python client can't connect (server-up != client-up).
- _startup_log(
- f"[Temporal] Connect attempt {attempt} failed for "
- f"{settings.temporal_server_address} (ns={settings.temporal_namespace}); "
- "retrying in 3s"
- )
- else:
- try:
- # Disable workflow auto-resumption while
- # DeploymentManager has no boot-time reconcile
- # against Temporal Visibility. History stays in
- # the SQLite db; UI shows workflows as
- # ``Terminated`` rather than continuing to run
- # invisibly to OpenCompany. Toggle off via
- # TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false.
- if settings.temporal_terminate_running_on_startup:
- try:
- has_controls = await container.database().has_active_workflow_controls()
- terminated = 0
- if has_controls:
- logger.info("Skipping startup Temporal termination sweep; durable workflow controls are active")
- else:
- terminated = await temporal_client_wrapper.terminate_running_workflows()
- if terminated:
- _startup_log(
- f"[Temporal] Terminated {terminated} running workflow(s) " "at startup (history preserved)"
- )
- except Exception as term_exc: # noqa: BLE001 — non-fatal
- logger.warning(
- f"Startup terminate-running sweep failed: {term_exc}",
- )
-
- temporal_executor = TemporalExecutor(
- client=client,
- task_queue=settings.temporal_task_queue,
- )
- container.workflow_service().set_temporal_executor(temporal_executor)
-
- worker_manager = TemporalWorkerManager(
- client=client,
- task_queue=settings.temporal_task_queue,
- )
- await worker_manager.start()
- app.state.temporal_worker_manager = worker_manager
-
- # Wave 16: per-queue activity worker pool (default-on
- # since 16.4; TEMPORAL_WORKER_POOL_ENABLED=false is
- # the rollback channel). Starts AFTER the manager so
- # workflow registration is in place before
- # specialised activity workers poll.
- if settings.temporal_worker_pool_enabled:
- from services.temporal.worker import TemporalWorkerPool
-
- pool = TemporalWorkerPool(client=client)
- await pool.start()
- app.state.temporal_pool = pool
- _startup_log(f"[Temporal] Worker pool started ({len(pool.queues)} queues)")
-
- _startup_log(f"[Temporal] Worker started, execution engine ready (attempt {attempt})")
- logger.info(
- "Temporal integration initialized successfully",
- attempts=attempt,
- )
- return
- except Exception as exc:
- _startup_log(f"[Temporal] Executor/worker setup failed (attempt {attempt}): {exc}; will retry")
- logger.error(
- "Temporal executor/worker setup failed; will retry",
- error=str(exc),
- )
- # Drop the client so the next iteration reconnects cleanly.
- await temporal_client_wrapper.disconnect()
- await asyncio.sleep(3.0)
-
- temporal_init_task = asyncio.create_task(_init_temporal_background(), name="temporal-init")
+ temporal_init_task = asyncio.create_task(
+ run_temporal_lifecycle(app.state, settings, startup_log=_startup_log),
+ name="temporal-init",
+ )
else:
_startup_log("[Temporal] Disabled")
@@ -462,7 +365,8 @@ async def lifespan(app: FastAPI):
# Stop WebSocket logging handler
shutdown_websocket_logging()
- # Cancel Temporal init task if it's still trying to connect.
+ # Cancel the Temporal lifecycle task (connect loop while starting;
+ # resident dev-server watchdog once the engine is up).
if temporal_init_task is not None and not temporal_init_task.done():
temporal_init_task.cancel()
try:
@@ -520,9 +424,9 @@ async def lifespan(app: FastAPI):
await shutdown_process_service()
- # Stop every plugin supervisor that registered itself via
- # services._supervisor.register_supervisor() (currently: WhatsApp;
- # other plugins migrate in PR 2).
+ # Stop every supervisor that registered itself via
+ # services._supervisor.register_supervisor() (WhatsApp runtime,
+ # Node.js executor runtime, Temporal dev server).
from services._supervisor import shutdown_all_supervisors
await shutdown_all_supervisors()
diff --git a/server/services/_supervisor/process.py b/server/services/_supervisor/process.py
index e20e9b53..6ea8f9ff 100644
--- a/server/services/_supervisor/process.py
+++ b/server/services/_supervisor/process.py
@@ -64,6 +64,19 @@ class BaseProcessSupervisor(BaseSupervisor):
async def _pre_spawn(self) -> None:
"""Hook before ``anyio.open_process`` (e.g., write a config file)."""
+ def stdout_log(self, line: str) -> None:
+ """Sink for one child-stdout line (INFO). Override to reclassify."""
+ self._logger.info(line)
+
+ def stderr_log(self, line: str) -> None:
+ """Sink for one child-stderr line (ERROR by default).
+
+ Override to downgrade known-benign child noise — e.g. the
+ Temporal dev server logs ERROR for routinely-canceled long-polls
+ (see :class:`services.temporal._runtime.TemporalServerRuntime`).
+ """
+ self._logger.error(line)
+
# ---- lifecycle implementation ---------------------------------------
def is_running(self) -> bool:
@@ -104,10 +117,10 @@ class BaseProcessSupervisor(BaseSupervisor):
if self.pipe_streams:
self._drain_tasks = [
asyncio.create_task(
- drain_stream(self._proc.stdout, self._logger.info, prefix=f"[{self.label}] "),
+ drain_stream(self._proc.stdout, self.stdout_log, prefix=f"[{self.label}] "),
),
asyncio.create_task(
- drain_stream(self._proc.stderr, self._logger.error, prefix=f"[{self.label}] "),
+ drain_stream(self._proc.stderr, self.stderr_log, prefix=f"[{self.label}] "),
),
]
diff --git a/server/services/temporal/_runtime.py b/server/services/temporal/_runtime.py
index a2e382a5..8f1d6306 100644
--- a/server/services/temporal/_runtime.py
+++ b/server/services/temporal/_runtime.py
@@ -109,12 +109,12 @@ class TemporalServerRuntime(BaseProcessSupervisor):
# https://docs.temporal.io/cli/server (subset we use):
# --port frontend gRPC port (gates ready-probe)
# --ui-port Web UI port (default ``--port + 1000``)
- # --db-filename SQLite file — preserves history across
- # restarts. Running workflows are not
- # auto-resumed; see ``TemporalClientWrapper.
- # terminate_running_workflows`` (run from
- # ``main.py`` lifespan) for the boot-time
- # terminate-but-preserve-history behaviour.
+ # --db-filename SQLite file — preserves history AND running
+ # workflow state across restarts. Running and
+ # paused deployments resume when the server
+ # and workers come back; the debug-only
+ # ``TEMPORAL_TERMINATE_RUNNING_ON_STARTUP``
+ # sweep (default false) is the one exception.
# --metrics-port 0 disables the Prometheus endpoint
# --log-level warn keeps the supervisor log readable
# --namespace default namespace bootstrapped at start
@@ -169,6 +169,24 @@ class TemporalServerRuntime(BaseProcessSupervisor):
# inherit parent env only.
return {**os.environ}
+ # Known-benign dev-server noise: the matching/persistence layer logs
+ # ERROR when a poller's long-poll context is canceled mid-read (worker
+ # restart, poller autoscaling settling, idle partition unload, or the
+ # server's own idle background jobs). Staff-confirmed benign upstream
+ # (community.temporal.io/t/18635; temporalio/cli#633): tasks are
+ # persisted and redelivered, nothing is lost. Both markers must match
+ # so genuine dev-server errors keep surfacing at ERROR.
+ _BENIGN_STDERR_MARKERS = (
+ "Operation failed with internal error.",
+ "GetTaskQueue operation failed",
+ )
+
+ def stderr_log(self, line: str) -> None:
+ if "context canceled" in line and any(marker in line for marker in self._BENIGN_STDERR_MARKERS):
+ self._logger.debug(line)
+ return
+ super().stderr_log(line)
+
async def health_check(self) -> bool:
if not self.is_running():
return False
diff --git a/server/services/temporal/client.py b/server/services/temporal/client.py
index 7107c94e..fcd67529 100644
--- a/server/services/temporal/client.py
+++ b/server/services/temporal/client.py
@@ -149,6 +149,24 @@ class TemporalClientWrapper:
self._client = None
logger.info("Disconnected from Temporal server")
+ async def check_health(self) -> bool:
+ """One gRPC WorkflowService health probe against the connected client.
+
+ False when disconnected or the frontend is not SERVING. Used by
+ the lifecycle watchdog (``services.temporal.lifecycle``); the
+ bounded readiness gate inside :meth:`connect` is separate.
+ """
+ if self._client is None:
+ return False
+ try:
+ return bool(
+ await self._client.service_client.check_health(
+ timeout=timedelta(seconds=self._health_check_timeout_seconds),
+ )
+ )
+ except Exception: # noqa: BLE001 — unreachable == not serving
+ return False
+
async def terminate_running_workflows(
self,
*,
diff --git a/server/services/temporal/lifecycle.py b/server/services/temporal/lifecycle.py
new file mode 100644
index 00000000..ad947bff
--- /dev/null
+++ b/server/services/temporal/lifecycle.py
@@ -0,0 +1,262 @@
+"""Temporal lifecycle owner: bootstrap, wiring, and resident supervision.
+
+``main.py``'s lifespan schedules exactly one background task —
+:func:`run_temporal_lifecycle` — and this module owns the entire
+Temporal runtime story for the process:
+
+1. Dev-server supervision (loopback deployments only) through the
+ :class:`~services.temporal._runtime.TemporalServerRuntime`
+ ``BaseSupervisor`` singleton, registered with the supervisor
+ registry so ``shutdown_all_supervisors()`` reaches it at teardown.
+2. Client connect loop — retries forever on a fixed cadence; the
+ backend serves HTTP immediately while this converges.
+3. The debug-only startup terminate sweep (config-gated, default off,
+ vetoed by any active durable workflow control).
+4. Execution-engine wiring: ``TemporalExecutor`` into the workflow
+ service, then ``TemporalWorkerManager`` + ``TemporalWorkerPool``.
+5. Boot-time reconcile of durable workflow-control generations so
+ running/paused deployments survive a backend restart.
+6. A resident dev-server watchdog: a child that dies or wedges while
+ the backend stays up is respawned, because deployed workflows must
+ keep executing for months without operator attention.
+
+Remote/production clusters are never spawned at, never restarted, and
+never swept beyond the config-gated sweep — only loopback addresses
+are treated as backend-owned.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Callable, Optional
+
+from core.config import Settings
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+# Cadence of the connect loop while Temporal is still unreachable.
+_CONNECT_RETRY_SECONDS = 3.0
+
+# Consecutive failed gRPC health probes before the watchdog restarts a
+# supervisor-owned dev-server child that still holds the port ("wedged").
+_WATCHDOG_UNHEALTHY_THRESHOLD = 4
+
+# No-op default so ``startup_log`` is always callable.
+_NULL_LOG: Callable[[str], None] = lambda line: None # noqa: E731
+
+
+def owns_dev_server(server_address: str) -> bool:
+ """True when ``server_address`` is loopback — this process owns the
+ SQLite dev server. Remote/production clusters are never spawned at."""
+ host = server_address.rsplit(":", 1)[0].strip("[]")
+ return host in ("localhost", "127.0.0.1", "::1")
+
+
+async def run_temporal_lifecycle(
+ app_state: Any,
+ settings: Settings,
+ startup_log: Optional[Callable[[str], None]] = None,
+) -> None:
+ """Connect, wire the execution engine, then stay resident supervising.
+
+ Runs as one background task for the process lifetime (cancelled by
+ the lifespan at shutdown). Retries the connect until it succeeds;
+ after success it reconciles durable workflow controls and — for
+ backend-owned (loopback) deployments — remains alive as the
+ dev-server watchdog.
+ """
+ log = startup_log or _NULL_LOG
+ from core.container import container
+
+ wrapper = container.temporal_client()
+ owned = owns_dev_server(settings.temporal_server_address)
+ if owned:
+ # Same BaseSupervisor singleton pattern as the WhatsApp and
+ # Node.js-executor runtimes; registering makes lifespan teardown
+ # (shutdown_all_supervisors) actually stop the child.
+ from services._supervisor import register_supervisor
+ from services.temporal._runtime import get_temporal_server_runtime
+
+ register_supervisor(get_temporal_server_runtime())
+
+ attempt = 0
+ while True:
+ attempt += 1
+ if owned:
+ await _ensure_dev_server(attempt, log)
+ client = await wrapper.connect(retries=1, delay=0)
+ if client is None:
+ # Surface every failed attempt to stdout so users can see the
+ # retry loop is alive when "Temporal is up" but the Python
+ # client can't connect (server-up != client-up).
+ log(
+ f"[Temporal] Connect attempt {attempt} failed for "
+ f"{settings.temporal_server_address} (ns={settings.temporal_namespace}); "
+ f"retrying in {_CONNECT_RETRY_SECONDS:g}s"
+ )
+ else:
+ try:
+ await _startup_sweep(wrapper, settings, log)
+ await _start_execution_engine(client, app_state, settings, log)
+ log(f"[Temporal] Worker started, execution engine ready (attempt {attempt})")
+ logger.info(
+ "Temporal integration initialized successfully",
+ attempts=attempt,
+ )
+ break
+ except Exception as exc: # noqa: BLE001 — loop retries
+ log(f"[Temporal] Executor/worker setup failed (attempt {attempt}): {exc}; will retry")
+ logger.error(
+ "Temporal executor/worker setup failed; will retry",
+ error=str(exc),
+ )
+ # Drop the client so the next iteration reconnects cleanly.
+ await wrapper.disconnect()
+ await asyncio.sleep(_CONNECT_RETRY_SECONDS)
+
+ await _boot_reconcile(log)
+ if owned:
+ await _watch_dev_server(wrapper, settings)
+
+
+async def _ensure_dev_server(attempt: int, log: Callable[[str], None]) -> None:
+ """Probe-or-spawn the backend-owned dev server (idempotent)."""
+ from services.temporal._runtime import get_temporal_server_runtime
+
+ try:
+ await get_temporal_server_runtime().ensure_started()
+ except Exception as exc: # noqa: BLE001 — connect loop retries
+ log(f"[Temporal] Dev server start failed (attempt {attempt}): {exc}")
+
+
+async def _startup_sweep(
+ wrapper: Any,
+ settings: Settings,
+ log: Callable[[str], None],
+) -> None:
+ """Debug-only escape hatch (default false): terminate Running workflows.
+
+ Running and paused deployments must survive backend restarts —
+ durable workflow-control generations are reconciled after the
+ workers start. When the sweep IS enabled, any active control row
+ still vetoes it so a live deployment is never killed.
+ """
+ if not settings.temporal_terminate_running_on_startup:
+ return
+ from core.container import container
+
+ try:
+ if await container.database().has_active_workflow_controls():
+ logger.info("Skipping startup Temporal termination sweep; durable workflow controls are active")
+ return
+ terminated = await wrapper.terminate_running_workflows()
+ if terminated:
+ log(f"[Temporal] Terminated {terminated} running workflow(s) at startup (history preserved)")
+ except Exception as exc: # noqa: BLE001 — non-fatal
+ logger.warning(f"Startup terminate-running sweep failed: {exc}")
+
+
+async def _start_execution_engine(
+ client: Any,
+ app_state: Any,
+ settings: Settings,
+ log: Callable[[str], None],
+) -> None:
+ """Wire the executor and start the worker manager (+ optional pool)."""
+ from core.container import container
+ from services.temporal import TemporalExecutor
+ from services.temporal.worker import TemporalWorkerManager
+
+ executor = TemporalExecutor(
+ client=client,
+ task_queue=settings.temporal_task_queue,
+ )
+ container.workflow_service().set_temporal_executor(executor)
+
+ manager = TemporalWorkerManager(
+ client=client,
+ task_queue=settings.temporal_task_queue,
+ )
+ await manager.start()
+ app_state.temporal_worker_manager = manager
+
+ # Wave 16: per-queue activity worker pool (default-on since 16.4;
+ # TEMPORAL_WORKER_POOL_ENABLED=false is the rollback channel).
+ # Starts AFTER the manager so workflow registration is in place
+ # before specialised activity workers poll.
+ if settings.temporal_worker_pool_enabled:
+ from services.temporal.worker import TemporalWorkerPool
+
+ pool = TemporalWorkerPool(client=client)
+ await pool.start()
+ app_state.temporal_pool = pool
+ log(f"[Temporal] Worker pool started ({len(pool.queues)} queues)")
+
+
+async def _boot_reconcile(log: Callable[[str], None]) -> None:
+ """Converge durable control rows left transitional by a crash/restart.
+
+ Non-fatal — every status/pause/resume request also reconciles
+ lazily; this pass just closes the unattended-server window where
+ durable intent and runtime behaviour could diverge indefinitely.
+ """
+ try:
+ from services.deployment.handlers import reconcile_active_controls_on_boot
+
+ count = await reconcile_active_controls_on_boot()
+ if count:
+ log(f"[Temporal] Reconciled {count} active workflow control(s)")
+ except Exception as exc: # noqa: BLE001 — non-fatal
+ logger.warning(f"Boot-time workflow-control reconcile failed: {exc}")
+
+
+async def _watch_dev_server(wrapper: Any, settings: Settings) -> None:
+ """Resident watchdog for the backend-owned dev server.
+
+ Deployed workflows must keep executing for months, so a dev-server
+ child that dies (or wedges) while the backend stays up is respawned
+ here — previously nothing probed it again after startup and every
+ deployment froze until an operator intervened.
+ """
+ from services.temporal._runtime import get_temporal_server_runtime
+
+ runtime = get_temporal_server_runtime()
+ interval = settings.temporal_health_monitor_interval_seconds
+ unhealthy = 0
+ while True:
+ await asyncio.sleep(interval)
+ try:
+ # Respawn if the child died and freed the port; no-op while
+ # anything is listening on it.
+ await runtime.ensure_started()
+ if await wrapper.check_health():
+ unhealthy = 0
+ continue
+ unhealthy += 1
+ if unhealthy < _WATCHDOG_UNHEALTHY_THRESHOLD:
+ continue
+ # Port bound but gRPC not SERVING. Restart a wedged child we
+ # own; an orphan we didn't spawn is only reported.
+ if runtime.is_running():
+ logger.warning(
+ f"Temporal dev server unhealthy for {unhealthy} consecutive probes; restarting owned child"
+ )
+ await runtime.stop()
+ await runtime.start()
+ else:
+ logger.warning(
+ "Temporal gRPC port is bound but not SERVING and the process "
+ "is not supervisor-owned; cannot restart it safely"
+ )
+ unhealthy = 0
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc: # noqa: BLE001 — keep the watchdog alive
+ logger.warning(f"Temporal health monitor iteration failed: {exc}")
+
+
+__all__ = [
+ "owns_dev_server",
+ "run_temporal_lifecycle",
+]
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index 3b0e65a0..07b42c60 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -148,119 +148,7 @@ class TemporalWorkerManager:
# Create activity instance with shared session
self._activities = NodeExecutionActivities(self._session)
- # F4.A: register per-type activities alongside the legacy
- # `execute_node_activity` dispatcher. The orchestrator at
- # workflow.py:_resolve_activity picks one of the two paths per node
- # based on the temporal_per_type_dispatch flag; both must be served
- # by the worker. Per-type activities register WITHOUT a task_queue
- # filter here (cls.task_queue is the *declared* preference; this
- # framework worker polls self.task_queue regardless). Per-queue
- # filtering lives in TemporalWorkerPool (below in this file),
- # which is default-on (Wave 16.4) and runs one activity-only
- # worker per declared queue. Registration cost: ~1.6s startup;
- # zero runtime cost when the flag is off (orchestrator routes to
- # execute_node_activity, per-type entries sit idle).
- #
- # F4.B: register AgentWorkflow + its three activities
- # (execute_llm_step / persist_turn / compact_memory). The
- # orchestrator schedules AgentWorkflow as a child workflow
- # for the 15 migrating agent types when
- # ``temporal_agent_workflow_enabled`` is on.
- from services.temporal.plugin_activities import (
- collect_plugin_activities,
- collect_polling_activities,
- )
- from services.temporal.agent_activities import collect_agent_activities
- from services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflow
- from services.temporal.activities import (
- broadcast_trigger_status_activity,
- load_persisted_workflow_graph_activity,
- store_node_output_activity,
- )
-
- per_type = collect_plugin_activities() # no queue filter; all plugins
- agent_activities = collect_agent_activities()
- polling_activities = collect_polling_activities()
- # Plugin-owned workflow classes (e.g. cron's
- # CronTriggerWorkflow) self-register a SimplePlugin via
- # services.temporal.plugin_registry.register_temporal_plugin
- # from their plugin __init__.py. The Temporal SDK's plugin
- # chain merges each registered plugin's workflows / activities
- # / interceptors into the effective worker configuration —
- # the framework worker stays plugin-agnostic.
- plugin_list = temporal_plugins()
-
- # The Worker() constructor derives its default build id by
- # MD5-hashing the bytecode of every module in sys.modules
- # (disk reads included) — a synchronous ~3s stall at our
- # module count (152 plugins + LLM SDK stack) that freezes
- # the event loop and starves concurrent boot work (e.g.
- # broadcaster refreshes). The result is memoized in an SDK
- # module global, so one off-loop call here makes this
- # Worker AND every TemporalWorkerPool worker construct
- # cheaply. load_default_build_id is not re-exported from
- # temporalio.worker (private module in 1.30); if an SDK
- # upgrade moves it, skip the pre-warm and pay the
- # synchronous cost again rather than fail startup.
- try:
- from temporalio.worker._worker import load_default_build_id
- except ImportError:
- logger.warning(
- "temporalio private API moved: load_default_build_id "
- "unavailable, skipping off-loop build-id pre-warm"
- )
- else:
- await asyncio.to_thread(load_default_build_id)
-
- self._worker = Worker(
- self.client,
- task_queue=self.task_queue,
- plugins=plugin_list,
- workflows=[
- MachinaWorkflow,
- AgentWorkflow,
- DelegatedTaskWorkflow,
- TriggerListenerWorkflow,
- PollingTriggerWorkflow,
- WorkflowControlWorkflow,
- ],
- activities=[
- self._activities.execute_node_activity,
- broadcast_trigger_status_activity,
- load_persisted_workflow_graph_activity,
- store_node_output_activity,
- *per_type,
- *agent_activities,
- *polling_activities,
- ],
- # Allow concurrent activity execution for parallel branches
- max_concurrent_activities=self.pool_size,
- max_concurrent_workflow_tasks=10,
- graceful_shutdown_timeout=_graceful_shutdown_timeout(),
- identity=_worker_identity(self.task_queue),
- # ``TracingInterceptor`` is registered once on the shared client;
- # the SDK prepends client interceptors to every worker, so
- # repeating it here double-instruments each Signal / Query /
- # Update / Activity and doubles the emitted spans.
- interceptors=[ObservabilityWorkerInterceptor()],
- # Wave 18.2: sticky cache sized by deployment mode so
- # cached workflows skip Event-History replay without
- # blowing laptop RAM.
- max_cached_workflows=_sticky_cache_size(),
- # Wave 18.3: autoscaling pollers replace fixed counts —
- # scale between min/max on demand (SDK-recommended over
- # manual poller tuning). Pollers stay well below the
- # executor slot counts per the worker-performance docs.
- activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=10),
- workflow_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=20),
- )
- logger.info(
- "Registered Temporal activities",
- legacy=1,
- per_type=len(per_type),
- agent=len(agent_activities),
- task_queue=self.task_queue,
- )
+ self._worker = await self._build_worker()
span.set_attribute("task_queue", self.task_queue)
span.set_attribute("pool_size", self.pool_size)
@@ -276,6 +164,129 @@ class TemporalWorkerManager:
name="temporal-worker",
)
+ async def _build_worker(self) -> Worker:
+ """Construct a fresh, ready-to-run Worker instance.
+
+ Called from :meth:`start` AND from every crash-restart attempt in
+ :meth:`_run_worker` — the SDK ``Worker`` is single-use (a second
+ ``run()`` on the same instance raises ``RuntimeError("Already
+ started")``), so restarting must always rebuild.
+ """
+ # F4.A: register per-type activities alongside the legacy
+ # `execute_node_activity` dispatcher. The orchestrator at
+ # workflow.py:_resolve_activity picks one of the two paths per node
+ # based on the temporal_per_type_dispatch flag; both must be served
+ # by the worker. Per-type activities register WITHOUT a task_queue
+ # filter here (cls.task_queue is the *declared* preference; this
+ # framework worker polls self.task_queue regardless). Per-queue
+ # filtering lives in TemporalWorkerPool (below in this file),
+ # which is default-on (Wave 16.4) and runs one activity-only
+ # worker per declared queue. Registration cost: ~1.6s startup;
+ # zero runtime cost when the flag is off (orchestrator routes to
+ # execute_node_activity, per-type entries sit idle).
+ #
+ # F4.B: register AgentWorkflow + its three activities
+ # (execute_llm_step / persist_turn / compact_memory). The
+ # orchestrator schedules AgentWorkflow as a child workflow
+ # for the 15 migrating agent types when
+ # ``temporal_agent_workflow_enabled`` is on.
+ from services.temporal.plugin_activities import (
+ collect_plugin_activities,
+ collect_polling_activities,
+ )
+ from services.temporal.agent_activities import collect_agent_activities
+ from services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflow
+ from services.temporal.activities import (
+ broadcast_trigger_status_activity,
+ load_persisted_workflow_graph_activity,
+ store_node_output_activity,
+ )
+
+ per_type = collect_plugin_activities() # no queue filter; all plugins
+ agent_activities = collect_agent_activities()
+ polling_activities = collect_polling_activities()
+ # Plugin-owned workflow classes (e.g. cron's
+ # CronTriggerWorkflow) self-register a SimplePlugin via
+ # services.temporal.plugin_registry.register_temporal_plugin
+ # from their plugin __init__.py. The Temporal SDK's plugin
+ # chain merges each registered plugin's workflows / activities
+ # / interceptors into the effective worker configuration —
+ # the framework worker stays plugin-agnostic.
+ plugin_list = temporal_plugins()
+
+ # The Worker() constructor derives its default build id by
+ # MD5-hashing the bytecode of every module in sys.modules
+ # (disk reads included) — a synchronous ~3s stall at our
+ # module count (152 plugins + LLM SDK stack) that freezes
+ # the event loop and starves concurrent boot work (e.g.
+ # broadcaster refreshes). The result is memoized in an SDK
+ # module global, so one off-loop call here makes this
+ # Worker AND every TemporalWorkerPool worker construct
+ # cheaply. load_default_build_id is not re-exported from
+ # temporalio.worker (private module in 1.30); if an SDK
+ # upgrade moves it, skip the pre-warm and pay the
+ # synchronous cost again rather than fail startup.
+ try:
+ from temporalio.worker._worker import load_default_build_id
+ except ImportError:
+ logger.warning(
+ "temporalio private API moved: load_default_build_id "
+ "unavailable, skipping off-loop build-id pre-warm"
+ )
+ else:
+ await asyncio.to_thread(load_default_build_id)
+
+ worker = Worker(
+ self.client,
+ task_queue=self.task_queue,
+ plugins=plugin_list,
+ workflows=[
+ MachinaWorkflow,
+ AgentWorkflow,
+ DelegatedTaskWorkflow,
+ TriggerListenerWorkflow,
+ PollingTriggerWorkflow,
+ WorkflowControlWorkflow,
+ ],
+ activities=[
+ self._activities.execute_node_activity,
+ broadcast_trigger_status_activity,
+ load_persisted_workflow_graph_activity,
+ store_node_output_activity,
+ *per_type,
+ *agent_activities,
+ *polling_activities,
+ ],
+ # Allow concurrent activity execution for parallel branches
+ max_concurrent_activities=self.pool_size,
+ max_concurrent_workflow_tasks=10,
+ graceful_shutdown_timeout=_graceful_shutdown_timeout(),
+ identity=_worker_identity(self.task_queue),
+ # ``TracingInterceptor`` is registered once on the shared client;
+ # the SDK prepends client interceptors to every worker, so
+ # repeating it here double-instruments each Signal / Query /
+ # Update / Activity and doubles the emitted spans.
+ interceptors=[ObservabilityWorkerInterceptor()],
+ # Wave 18.2: sticky cache sized by deployment mode so
+ # cached workflows skip Event-History replay without
+ # blowing laptop RAM.
+ max_cached_workflows=_sticky_cache_size(),
+ # Wave 18.3: autoscaling pollers replace fixed counts —
+ # scale between min/max on demand (SDK-recommended over
+ # manual poller tuning). Pollers stay well below the
+ # executor slot counts per the worker-performance docs.
+ activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=10),
+ workflow_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=20),
+ )
+ logger.info(
+ "Registered Temporal activities",
+ legacy=1,
+ per_type=len(per_type),
+ agent=len(agent_activities),
+ task_queue=self.task_queue,
+ )
+ return worker
+
async def _run_worker(self) -> None:
"""Run the worker (background task), self-restarting on crash.
@@ -284,8 +295,13 @@ class TemporalWorkerManager:
detached — the startup retry loop in ``main.py`` has already
returned by the time it runs. So a transient crash (e.g. the
server briefly unavailable mid-poll) would otherwise leave the
- worker permanently dead until a process restart. Re-run the SAME
- worker instance with doubling backoff; cancellation (from
+ worker permanently dead until a process restart.
+
+ Every restart attempt REBUILDS the worker via
+ :meth:`_build_worker`: the SDK ``Worker`` is single-use, so
+ re-running the same instance raises ``RuntimeError("Already
+ started")`` on the first retry and every one after it — the old
+ loop spun forever without ever polling again. Cancellation (from
``stop()``) always wins so shutdown is never delayed by a restart.
Backoff knobs are env-driven (canonical defaults in .env.template).
"""
@@ -303,11 +319,18 @@ class TemporalWorkerManager:
raise
except Exception as e:
logger.error(f"Temporal worker crashed; restarting in {backoff:.1f}s: {e}")
- try:
- await asyncio.sleep(backoff)
- except asyncio.CancelledError:
- raise # cancellation during backoff still wins
- backoff = min(backoff * 2, backoff_max)
+ try:
+ await asyncio.sleep(backoff)
+ except asyncio.CancelledError:
+ raise # cancellation during backoff still wins
+ backoff = min(backoff * 2, backoff_max)
+ try:
+ self._worker = await self._build_worker()
+ except Exception as build_exc: # noqa: BLE001 — retried next loop
+ # Next iteration's run() on the stale single-use instance
+ # fails fast and re-enters this backoff, so rebuild
+ # failures are retried rather than fatal.
+ logger.error(f"Temporal worker rebuild failed; will retry: {build_exc}")
async def stop(self) -> None:
"""Stop the Temporal worker and cleanup resources."""
@@ -484,60 +507,113 @@ class TemporalWorkerPool:
def is_running(self) -> bool:
return any(t is not None and not t.done() for t in self._tasks)
- async def start(self) -> None:
+ def _build_queue_worker(self, queue: str) -> Optional[Worker]:
+ """Construct a fresh Worker for one queue.
+
+ Returns ``None`` when the queue has no registered activities.
+ Called from :meth:`start` AND from every crash-restart attempt in
+ :meth:`_run_queue_worker` — the SDK ``Worker`` is single-use, so
+ restarting must always rebuild.
+ """
from services.temporal.plugin_activities import (
collect_plugin_activities,
)
+ activities = collect_plugin_activities(task_queue=queue)
+ if not activities:
+ return None
+ concurrency = self._concurrency_for(queue)
+ rate_limit = self._rate_limit_for(queue)
+ tuner = self._tuner_for(queue, concurrency)
+ worker_kwargs: dict = dict(
+ task_queue=queue,
+ activities=activities,
+ graceful_shutdown_timeout=_graceful_shutdown_timeout(),
+ identity=_worker_identity(queue),
+ # ``TracingInterceptor`` is registered once on the shared client;
+ # the SDK prepends client interceptors to every worker, so
+ # repeating it here double-instruments each Signal / Query /
+ # Update / Activity and doubles the emitted spans.
+ interceptors=[ObservabilityWorkerInterceptor()],
+ # Wave 18.1: per-queue activities/second ceiling.
+ max_activities_per_second=rate_limit,
+ # Wave 18.3: activity-only workers need a small
+ # autoscaling poller budget — specialised queues see
+ # lower task volume than the manager's default queue.
+ activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=1, minimum=1, maximum=5),
+ )
+ # Wave 18.4: the Worker rejects `tuner` alongside ANY of
+ # `max_concurrent_workflow_tasks` / `max_concurrent_activities` /
+ # `max_concurrent_local_activities` / `max_concurrent_nexus_tasks`
+ # — the SDK enforces "size via tuner OR via max_concurrent_*,
+ # never both" (temporalio.worker.Worker.__init__). Resource-tuned
+ # queues get the tuner (which already carries a
+ # FixedSizeSlotSupplier(10) for workflow slots via
+ # `_tuner_for`); the rest keep fixed sizing.
+ if tuner is not None:
+ worker_kwargs["tuner"] = tuner
+ else:
+ worker_kwargs["max_concurrent_activities"] = concurrency
+ worker_kwargs["max_concurrent_workflow_tasks"] = 10
+
+ worker = Worker(self.client, **worker_kwargs)
+ logger.info(
+ f"[Pool] Built worker queue={queue!r} "
+ f"activities={len(activities)} "
+ f"slots={'resource-tuned<=' + str(max(concurrency, 2)) if tuner else concurrency} "
+ f"rate_limit={rate_limit if rate_limit is not None else 'unthrottled'}"
+ )
+ return worker
+
+ async def _run_queue_worker(self, queue: str, worker: Worker) -> None:
+ """Run one queue's worker, rebuilding + restarting on crash.
+
+ Previously each queue worker ran as a bare ``worker.run()`` task:
+ a crash silently killed the ONLY worker for that queue and every
+ activity routed there pended forever (its workflows stalled)
+ until a backend restart. Same backoff + rebuild contract as
+ :meth:`TemporalWorkerManager._run_worker`; cancellation (from
+ :meth:`stop`) always wins.
+ """
+ from core.config import Settings
+
+ _s = Settings()
+ backoff = _s.temporal_worker_restart_backoff_seconds
+ backoff_max = _s.temporal_worker_restart_backoff_max_seconds
+ current = worker
+ while True:
+ try:
+ await current.run()
+ return # clean shutdown only
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc: # noqa: BLE001 — supervised restart
+ logger.error(f"[Pool] Worker for queue {queue!r} crashed; restarting in {backoff:.1f}s: {exc}")
+ try:
+ await asyncio.sleep(backoff)
+ except asyncio.CancelledError:
+ raise # cancellation during backoff still wins
+ backoff = min(backoff * 2, backoff_max)
+ rebuilt = self._build_queue_worker(queue)
+ if rebuilt is None:
+ logger.error(f"[Pool] Queue {queue!r} has no activities on rebuild; worker not restarted")
+ return
+ current = rebuilt
+
+ async def start(self) -> None:
for queue in self.queues:
- activities = collect_plugin_activities(task_queue=queue)
- if not activities:
+ worker = self._build_queue_worker(queue)
+ if worker is None:
logger.info(f"[Pool] Skipping empty queue {queue!r}")
continue
- concurrency = self._concurrency_for(queue)
- rate_limit = self._rate_limit_for(queue)
- tuner = self._tuner_for(queue, concurrency)
- worker_kwargs: dict = dict(
- task_queue=queue,
- activities=activities,
- graceful_shutdown_timeout=_graceful_shutdown_timeout(),
- identity=_worker_identity(queue),
- # ``TracingInterceptor`` is registered once on the shared client;
- # the SDK prepends client interceptors to every worker, so
- # repeating it here double-instruments each Signal / Query /
- # Update / Activity and doubles the emitted spans.
- interceptors=[ObservabilityWorkerInterceptor()],
- # Wave 18.1: per-queue activities/second ceiling.
- max_activities_per_second=rate_limit,
- # Wave 18.3: activity-only workers need a small
- # autoscaling poller budget — specialised queues see
- # lower task volume than the manager's default queue.
- activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=1, minimum=1, maximum=5),
+ task = asyncio.create_task(
+ self._run_queue_worker(queue, worker),
+ name=f"worker-{queue}",
)
- # Wave 18.4: the Worker rejects `tuner` alongside ANY of
- # `max_concurrent_workflow_tasks` / `max_concurrent_activities` /
- # `max_concurrent_local_activities` / `max_concurrent_nexus_tasks`
- # — the SDK enforces "size via tuner OR via max_concurrent_*,
- # never both" (temporalio.worker.Worker.__init__). Resource-tuned
- # queues get the tuner (which already carries a
- # FixedSizeSlotSupplier(10) for workflow slots via
- # `_tuner_for`); the rest keep fixed sizing.
- if tuner is not None:
- worker_kwargs["tuner"] = tuner
- else:
- worker_kwargs["max_concurrent_activities"] = concurrency
- worker_kwargs["max_concurrent_workflow_tasks"] = 10
-
- worker = Worker(self.client, **worker_kwargs)
- task = asyncio.create_task(worker.run(), name=f"worker-{queue}")
+ # Initial instances only — after a crash-restart the live
+ # instance is owned by the supervising task.
self._workers.append(worker)
self._tasks.append(task)
- logger.info(
- f"[Pool] Started worker queue={queue!r} "
- f"activities={len(activities)} "
- f"slots={'resource-tuned<=' + str(max(concurrency, 2)) if tuner else concurrency} "
- f"rate_limit={rate_limit if rate_limit is not None else 'unthrottled'}"
- )
async def stop(self) -> None:
for task in self._tasks:
diff --git a/server/tests/temporal/test_worker_restart.py b/server/tests/temporal/test_worker_restart.py
index 6276d8bd..b3d0b306 100644
--- a/server/tests/temporal/test_worker_restart.py
+++ b/server/tests/temporal/test_worker_restart.py
@@ -1,12 +1,19 @@
-"""Worker self-restart in TemporalWorkerManager._run_worker.
+"""Worker self-restart in TemporalWorkerManager / TemporalWorkerPool.
The Temporal worker shuts down on a transient poll failure rather than
auto-retrying, and the run task is detached (main.py's startup retry loop
-has already returned). ``_run_worker`` now re-runs the same worker instance
-with doubling backoff on a crash, while cancellation (from ``stop()``)
-always wins so shutdown is never delayed by a restart. Backoff knobs come
-from Settings (env-driven); the conftest stubs Settings, so we patch it to
-a SimpleNamespace with tiny real backoffs.
+has already returned). The SDK ``Worker`` is single-use — a second
+``run()`` on the same instance raises ``RuntimeError("Already started")``
+— so every restart attempt must REBUILD a fresh worker. The previous
+restart loop re-ran the same instance, which meant one crash left the
+task queue permanently unpolled while the loop logged "restarting"
+forever; the previous version of this test masked that by mocking a
+re-runnable worker.
+
+Cancellation (from ``stop()``) always wins so shutdown is never delayed
+by a restart. Backoff knobs come from Settings (env-driven); the conftest
+stubs Settings, so we patch it to a SimpleNamespace with tiny real
+backoffs.
"""
from __future__ import annotations
@@ -17,7 +24,7 @@ from unittest.mock import MagicMock, patch
import pytest
-from services.temporal.worker import TemporalWorkerManager
+from services.temporal.worker import TemporalWorkerManager, TemporalWorkerPool
def _patch_backoff():
@@ -30,29 +37,82 @@ def _patch_backoff():
)
-def _manager_with_run(fake_run) -> TemporalWorkerManager:
+class _SingleUseWorker:
+ """Mimics the SDK contract: a second run() on one instance raises."""
+
+ def __init__(self, run_impl):
+ self._run_impl = run_impl
+ self._started = False
+
+ async def run(self):
+ if self._started:
+ raise RuntimeError("Already started")
+ self._started = True
+ await self._run_impl()
+
+
+def _manager_with_builds(run_impls) -> tuple[TemporalWorkerManager, dict]:
+ """Manager whose _build_worker yields single-use workers in sequence."""
mgr = TemporalWorkerManager(client=MagicMock(), task_queue="q")
- worker = MagicMock()
- worker.run = fake_run
- mgr._worker = worker
- return mgr
+ state = {"builds": 0, "runs": 0}
+ impls = list(run_impls)
+ async def _tracked(impl):
+ state["runs"] += 1
+ await impl()
-async def test_worker_restarts_after_transient_crash():
- calls = {"n": 0}
+ async def fake_build_worker():
+ impl = impls[min(state["builds"], len(impls) - 1)]
+ state["builds"] += 1
+ return _SingleUseWorker(lambda impl=impl: _tracked(impl))
- async def fake_run():
- calls["n"] += 1
- if calls["n"] == 1:
- raise RuntimeError("transient poll failure")
+ mgr._build_worker = fake_build_worker # type: ignore[method-assign]
+ return mgr, state
+
+
+async def test_worker_restarts_with_fresh_instance_after_transient_crash():
+ async def crash():
+ raise RuntimeError("transient poll failure")
+
+ async def healthy():
await asyncio.Event().wait() # healthy run blocks
- mgr = _manager_with_run(fake_run)
+ mgr, state = _manager_with_builds([crash, healthy])
+ mgr._worker = await mgr._build_worker()
with _patch_backoff():
task = asyncio.create_task(mgr._run_worker())
- await asyncio.sleep(0.05) # allow crash + restart
- assert calls["n"] == 2 # restarted exactly once
+ await asyncio.sleep(0.05) # allow crash + rebuild + restart
+ # First build happened before the loop; the restart rebuilt once.
+ assert state["builds"] == 2
+ assert state["runs"] == 2
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+
+async def test_restart_never_reuses_the_single_use_instance():
+ """Locks the regression the old mock hid: re-running the same SDK
+ worker raises "Already started" instantly. The restart loop must
+ rebuild before every retry, so the crashed instance is run once."""
+ crashed = _SingleUseWorker(run_impl=None)
+
+ async def crash():
+ raise RuntimeError("transient poll failure")
+
+ async def healthy():
+ await asyncio.Event().wait()
+
+ mgr, state = _manager_with_builds([healthy])
+ crashed._run_impl = crash
+ mgr._worker = crashed
+
+ with _patch_backoff():
+ task = asyncio.create_task(mgr._run_worker())
+ await asyncio.sleep(0.05)
+ # The crashed single-use instance was replaced, not re-run.
+ assert mgr._worker is not crashed
+ assert state["builds"] == 1
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
@@ -61,11 +121,12 @@ async def test_worker_restarts_after_transient_crash():
async def test_cancellation_does_not_restart():
calls = {"n": 0}
- async def fake_run():
+ async def healthy():
calls["n"] += 1
await asyncio.Event().wait() # healthy run blocks until cancelled
- mgr = _manager_with_run(fake_run)
+ mgr, state = _manager_with_builds([healthy])
+ mgr._worker = await mgr._build_worker()
with _patch_backoff():
task = asyncio.create_task(mgr._run_worker())
@@ -76,3 +137,50 @@ async def test_cancellation_does_not_restart():
await task
assert calls["n"] == 1 # cancellation wins; not restarted
+ assert state["builds"] == 1 # and nothing was rebuilt
+
+
+async def test_pool_queue_worker_restarts_with_fresh_instance():
+ """Pool workers previously ran as bare worker.run() tasks with no
+ restart machinery — a crash silently killed the only worker for the
+ queue and its activities pended forever."""
+ pool = TemporalWorkerPool(client=MagicMock(), queues=["code-exec"])
+ state = {"builds": 0}
+
+ async def crash():
+ raise RuntimeError("poll failure")
+
+ async def healthy():
+ await asyncio.Event().wait()
+
+ def fake_build(queue):
+ impl = crash if state["builds"] == 0 else healthy
+ state["builds"] += 1
+ return _SingleUseWorker(impl)
+
+ pool._build_queue_worker = fake_build # type: ignore[method-assign]
+
+ with _patch_backoff():
+ first = pool._build_queue_worker("code-exec")
+ task = asyncio.create_task(pool._run_queue_worker("code-exec", first))
+ await asyncio.sleep(0.05)
+ assert state["builds"] == 2 # crash triggered one rebuild
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+
+async def test_pool_queue_worker_stops_when_queue_has_no_activities():
+ pool = TemporalWorkerPool(client=MagicMock(), queues=["code-exec"])
+
+ async def crash():
+ raise RuntimeError("poll failure")
+
+ pool._build_queue_worker = lambda queue: None # type: ignore[method-assign]
+
+ with _patch_backoff():
+ task = asyncio.create_task(
+ pool._run_queue_worker("code-exec", _SingleUseWorker(crash))
+ )
+ # Crash, then rebuild returns None -> the supervisor task ends.
+ await asyncio.wait_for(task, timeout=1.0)