Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(temporal): lifecycle module with dev-server watchdog and worker crash rebuilds
trohitg73830a331bcc
diff --git a/server/main.py b/server/main.pyindex d104ff4e..7c7eb967 100644--- a/server/main.py+++ b/server/main.py@@ -295,19 +295,23 @@ async def lifespan(app: FastAPI):set_startup_time()# Initialize Temporal in the background - do NOT block lifespan startup.- # scripts/start.js launches the Python backend and temporal-server concurrently,- # so on fresh startup temporal-server may take several seconds to become reachable.- # Blocking the lifespan here would delay FastAPI HTTP serving and cascade into- # frontend ERR_CONNECTION_REFUSED on /api/auth/status. Instead, yield fast and- # let Temporal init happen in a background task. WorkflowService falls back to- # parallel/sequential execution until Temporal is ready.+ # This lifespan owns the Temporal dev server (spawned below via+ # TemporalServerRuntime.ensure_started), and on fresh startup it may take+ # several seconds to become reachable. Blocking the lifespan here would+ # delay FastAPI HTTP serving and cascade into frontend+ # ERR_CONNECTION_REFUSED on /api/auth/status. Instead, yield fast and+ # let Temporal init happen in a background task. WorkflowService falls back+ # to parallel/sequential execution until Temporal is ready.app.state.temporal_worker_manager = Noneapp.state.temporal_pool = Nonetemporal_init_task: asyncio.Task | None = Noneif settings.temporal_enabled:- from services.temporal import TemporalExecutor- from services.temporal.worker import TemporalWorkerManager+ # The whole Temporal runtime story (dev-server supervision,+ # connect loop, executor/worker wiring, boot-time control+ # reconcile, resident dev-server watchdog) lives in+ # services.temporal.lifecycle — main.py only schedules it.+ from services.temporal.lifecycle import run_temporal_lifecyclelogger.info("Scheduling Temporal initialization in background",@@ -316,111 +320,10 @@ async def lifespan(app: FastAPI):task_queue=settings.temporal_task_queue,)_startup_log(f"[Temporal] Init scheduled for {settings.temporal_server_address}")-- async def _init_temporal_background() -> None:- """Connect, wire executor, start worker. Retries every 3s until connected.-- This task is the single source of truth for Temporal lifecycle during- startup. It replaces the previous blocking-connect + background-reconnect- split, which blocked the lifespan for up to 30s and left the executor- unwired after an initial failed connect.- """- temporal_client_wrapper = container.temporal_client()- # Loopback address => this process owns the SQLite dev server,- # via the same BaseSupervisor singleton pattern as the WhatsApp- # runtime (teardown is covered by shutdown_all_supervisors()).- # Remote/production clusters are never spawned at.- _tp_host = settings.temporal_server_address.rsplit(":", 1)[0].strip("[]")- _owns_dev_server = _tp_host in ("localhost", "127.0.0.1", "::1")- attempt = 0- while True:- attempt += 1- if _owns_dev_server:- from services.temporal._runtime import get_temporal_server_runtime-- try:- await get_temporal_server_runtime().ensure_started()- except Exception as dev_exc: # noqa: BLE001 — connect loop retries- _startup_log(f"[Temporal] Dev server start failed (attempt {attempt}): {dev_exc}")- client = await temporal_client_wrapper.connect(retries=1, delay=0)- if client is None:- # Surface every failed attempt to stdout so users can see- # the retry loop is still alive when "Temporal is up" but- # the Python client can't connect (server-up != client-up).- _startup_log(- f"[Temporal] Connect attempt {attempt} failed for "- f"{settings.temporal_server_address} (ns={settings.temporal_namespace}); "- "retrying in 3s"- )- else:- try:- # Disable workflow auto-resumption while- # DeploymentManager has no boot-time reconcile- # against Temporal Visibility. History stays in- # the SQLite db; UI shows workflows as- # ``Terminated`` rather than continuing to run- # invisibly to OpenCompany. Toggle off via- # TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false.- if settings.temporal_terminate_running_on_startup:- try:- has_controls = await container.database().has_active_workflow_controls()- terminated = 0- if has_controls:- logger.info("Skipping startup Temporal termination sweep; durable workflow controls are active")- else:- terminated = await temporal_client_wrapper.terminate_running_workflows()- if terminated:- _startup_log(- f"[Temporal] Terminated {terminated} running workflow(s) " "at startup (history preserved)"- )- except Exception as term_exc: # noqa: BLE001 — non-fatal- logger.warning(- f"Startup terminate-running sweep failed: {term_exc}",- )-- temporal_executor = TemporalExecutor(- client=client,- task_queue=settings.temporal_task_queue,- )- container.workflow_service().set_temporal_executor(temporal_executor)-- worker_manager = TemporalWorkerManager(- client=client,- task_queue=settings.temporal_task_queue,- )- await worker_manager.start()- app.state.temporal_worker_manager = worker_manager-- # Wave 16: per-queue activity worker pool (default-on- # since 16.4; TEMPORAL_WORKER_POOL_ENABLED=false is- # the rollback channel). Starts AFTER the manager so- # workflow registration is in place before- # specialised activity workers poll.- if settings.temporal_worker_pool_enabled:- from services.temporal.worker import TemporalWorkerPool-- pool = TemporalWorkerPool(client=client)- await pool.start()- app.state.temporal_pool = pool- _startup_log(f"[Temporal] Worker pool started ({len(pool.queues)} queues)")-- _startup_log(f"[Temporal] Worker started, execution engine ready (attempt {attempt})")- logger.info(- "Temporal integration initialized successfully",- attempts=attempt,- )- return- except Exception as exc:- _startup_log(f"[Temporal] Executor/worker setup failed (attempt {attempt}): {exc}; will retry")- logger.error(- "Temporal executor/worker setup failed; will retry",- error=str(exc),- )- # Drop the client so the next iteration reconnects cleanly.- await temporal_client_wrapper.disconnect()- await asyncio.sleep(3.0)-- temporal_init_task = asyncio.create_task(_init_temporal_background(), name="temporal-init")+ temporal_init_task = asyncio.create_task(+ run_temporal_lifecycle(app.state, settings, startup_log=_startup_log),+ name="temporal-init",+ )else:_startup_log("[Temporal] Disabled")@@ -462,7 +365,8 @@ async def lifespan(app: FastAPI):# Stop WebSocket logging handlershutdown_websocket_logging()- # Cancel Temporal init task if it's still trying to connect.+ # Cancel the Temporal lifecycle task (connect loop while starting;+ # resident dev-server watchdog once the engine is up).if temporal_init_task is not None and not temporal_init_task.done():temporal_init_task.cancel()try:@@ -520,9 +424,9 @@ async def lifespan(app: FastAPI):await shutdown_process_service()- # Stop every plugin supervisor that registered itself via- # services._supervisor.register_supervisor() (currently: WhatsApp;- # other plugins migrate in PR 2).+ # Stop every supervisor that registered itself via+ # services._supervisor.register_supervisor() (WhatsApp runtime,+ # Node.js executor runtime, Temporal dev server).from services._supervisor import shutdown_all_supervisorsawait shutdown_all_supervisors()diff --git a/server/services/_supervisor/process.py b/server/services/_supervisor/process.pyindex e20e9b53..6ea8f9ff 100644--- a/server/services/_supervisor/process.py+++ b/server/services/_supervisor/process.py@@ -64,6 +64,19 @@ class BaseProcessSupervisor(BaseSupervisor):async def _pre_spawn(self) -> None:"""Hook before ``anyio.open_process`` (e.g., write a config file)."""+ def stdout_log(self, line: str) -> None:+ """Sink for one child-stdout line (INFO). Override to reclassify."""+ self._logger.info(line)++ def stderr_log(self, line: str) -> None:+ """Sink for one child-stderr line (ERROR by default).++ Override to downgrade known-benign child noise — e.g. the+ Temporal dev server logs ERROR for routinely-canceled long-polls+ (see :class:`services.temporal._runtime.TemporalServerRuntime`).+ """+ self._logger.error(line)+# ---- lifecycle implementation ---------------------------------------def is_running(self) -> bool:@@ -104,10 +117,10 @@ class BaseProcessSupervisor(BaseSupervisor):if self.pipe_streams:self._drain_tasks = [asyncio.create_task(- drain_stream(self._proc.stdout, self._logger.info, prefix=f"[{self.label}] "),+ drain_stream(self._proc.stdout, self.stdout_log, prefix=f"[{self.label}] "),),asyncio.create_task(- drain_stream(self._proc.stderr, self._logger.error, prefix=f"[{self.label}] "),+ drain_stream(self._proc.stderr, self.stderr_log, prefix=f"[{self.label}] "),),]diff --git a/server/services/temporal/_runtime.py b/server/services/temporal/_runtime.pyindex a2e382a5..8f1d6306 100644--- a/server/services/temporal/_runtime.py+++ b/server/services/temporal/_runtime.py@@ -109,12 +109,12 @@ class TemporalServerRuntime(BaseProcessSupervisor):# https://docs.temporal.io/cli/server (subset we use):# --port frontend gRPC port (gates ready-probe)# --ui-port Web UI port (default ``--port + 1000``)- # --db-filename SQLite file — preserves history across- # restarts. Running workflows are not- # auto-resumed; see ``TemporalClientWrapper.- # terminate_running_workflows`` (run from- # ``main.py`` lifespan) for the boot-time- # terminate-but-preserve-history behaviour.+ # --db-filename SQLite file — preserves history AND running+ # workflow state across restarts. Running and+ # paused deployments resume when the server+ # and workers come back; the debug-only+ # ``TEMPORAL_TERMINATE_RUNNING_ON_STARTUP``+ # sweep (default false) is the one exception.# --metrics-port 0 disables the Prometheus endpoint# --log-level warn keeps the supervisor log readable# --namespace default namespace bootstrapped at start@@ -169,6 +169,24 @@ class TemporalServerRuntime(BaseProcessSupervisor):# inherit parent env only.return {**os.environ}+ # Known-benign dev-server noise: the matching/persistence layer logs+ # ERROR when a poller's long-poll context is canceled mid-read (worker+ # restart, poller autoscaling settling, idle partition unload, or the+ # server's own idle background jobs). Staff-confirmed benign upstream+ # (community.temporal.io/t/18635; temporalio/cli#633): tasks are+ # persisted and redelivered, nothing is lost. Both markers must match+ # so genuine dev-server errors keep surfacing at ERROR.+ _BENIGN_STDERR_MARKERS = (+ "Operation failed with internal error.",+ "GetTaskQueue operation failed",+ )++ def stderr_log(self, line: str) -> None:+ if "context canceled" in line and any(marker in line for marker in self._BENIGN_STDERR_MARKERS):+ self._logger.debug(line)+ return+ super().stderr_log(line)+async def health_check(self) -> bool:if not self.is_running():return Falsediff --git a/server/services/temporal/client.py b/server/services/temporal/client.pyindex 7107c94e..fcd67529 100644--- a/server/services/temporal/client.py+++ b/server/services/temporal/client.py@@ -149,6 +149,24 @@ class TemporalClientWrapper:self._client = Nonelogger.info("Disconnected from Temporal server")+ async def check_health(self) -> bool:+ """One gRPC WorkflowService health probe against the connected client.++ False when disconnected or the frontend is not SERVING. Used by+ the lifecycle watchdog (``services.temporal.lifecycle``); the+ bounded readiness gate inside :meth:`connect` is separate.+ """+ if self._client is None:+ return False+ try:+ return bool(+ await self._client.service_client.check_health(+ timeout=timedelta(seconds=self._health_check_timeout_seconds),+ )+ )+ except Exception: # noqa: BLE001 — unreachable == not serving+ return False+async def terminate_running_workflows(self,*,diff --git a/server/services/temporal/lifecycle.py b/server/services/temporal/lifecycle.pynew file mode 100644index 00000000..ad947bff--- /dev/null+++ b/server/services/temporal/lifecycle.py@@ -0,0 +1,262 @@+"""Temporal lifecycle owner: bootstrap, wiring, and resident supervision.++``main.py``'s lifespan schedules exactly one background task —+:func:`run_temporal_lifecycle` — and this module owns the entire+Temporal runtime story for the process:++1. Dev-server supervision (loopback deployments only) through the+ :class:`~services.temporal._runtime.TemporalServerRuntime`+ ``BaseSupervisor`` singleton, registered with the supervisor+ registry so ``shutdown_all_supervisors()`` reaches it at teardown.+2. Client connect loop — retries forever on a fixed cadence; the+ backend serves HTTP immediately while this converges.+3. The debug-only startup terminate sweep (config-gated, default off,+ vetoed by any active durable workflow control).+4. Execution-engine wiring: ``TemporalExecutor`` into the workflow+ service, then ``TemporalWorkerManager`` + ``TemporalWorkerPool``.+5. Boot-time reconcile of durable workflow-control generations so+ running/paused deployments survive a backend restart.+6. A resident dev-server watchdog: a child that dies or wedges while+ the backend stays up is respawned, because deployed workflows must+ keep executing for months without operator attention.++Remote/production clusters are never spawned at, never restarted, and+never swept beyond the config-gated sweep — only loopback addresses+are treated as backend-owned.+"""++from __future__ import annotations++import asyncio+from typing import Any, Callable, Optional++from core.config import Settings+from core.logging import get_logger++logger = get_logger(__name__)++# Cadence of the connect loop while Temporal is still unreachable.+_CONNECT_RETRY_SECONDS = 3.0++# Consecutive failed gRPC health probes before the watchdog restarts a+# supervisor-owned dev-server child that still holds the port ("wedged").+_WATCHDOG_UNHEALTHY_THRESHOLD = 4++# No-op default so ``startup_log`` is always callable.+_NULL_LOG: Callable[[str], None] = lambda line: None # noqa: E731+++def owns_dev_server(server_address: str) -> bool:+ """True when ``server_address`` is loopback — this process owns the+ SQLite dev server. Remote/production clusters are never spawned at."""+ host = server_address.rsplit(":", 1)[0].strip("[]")+ return host in ("localhost", "127.0.0.1", "::1")+++async def run_temporal_lifecycle(+ app_state: Any,+ settings: Settings,+ startup_log: Optional[Callable[[str], None]] = None,+) -> None:+ """Connect, wire the execution engine, then stay resident supervising.++ Runs as one background task for the process lifetime (cancelled by+ the lifespan at shutdown). Retries the connect until it succeeds;+ after success it reconciles durable workflow controls and — for+ backend-owned (loopback) deployments — remains alive as the+ dev-server watchdog.+ """+ log = startup_log or _NULL_LOG+ from core.container import container++ wrapper = container.temporal_client()+ owned = owns_dev_server(settings.temporal_server_address)+ if owned:+ # Same BaseSupervisor singleton pattern as the WhatsApp and+ # Node.js-executor runtimes; registering makes lifespan teardown+ # (shutdown_all_supervisors) actually stop the child.+ from services._supervisor import register_supervisor+ from services.temporal._runtime import get_temporal_server_runtime++ register_supervisor(get_temporal_server_runtime())++ attempt = 0+ while True:+ attempt += 1+ if owned:+ await _ensure_dev_server(attempt, log)+ client = await wrapper.connect(retries=1, delay=0)+ if client is None:+ # Surface every failed attempt to stdout so users can see the+ # retry loop is alive when "Temporal is up" but the Python+ # client can't connect (server-up != client-up).+ log(+ f"[Temporal] Connect attempt {attempt} failed for "+ f"{settings.temporal_server_address} (ns={settings.temporal_namespace}); "+ f"retrying in {_CONNECT_RETRY_SECONDS:g}s"+ )+ else:+ try:+ await _startup_sweep(wrapper, settings, log)+ await _start_execution_engine(client, app_state, settings, log)+ log(f"[Temporal] Worker started, execution engine ready (attempt {attempt})")+ logger.info(+ "Temporal integration initialized successfully",+ attempts=attempt,+ )+ break+ except Exception as exc: # noqa: BLE001 — loop retries+ log(f"[Temporal] Executor/worker setup failed (attempt {attempt}): {exc}; will retry")+ logger.error(+ "Temporal executor/worker setup failed; will retry",+ error=str(exc),+ )+ # Drop the client so the next iteration reconnects cleanly.+ await wrapper.disconnect()+ await asyncio.sleep(_CONNECT_RETRY_SECONDS)++ await _boot_reconcile(log)+ if owned:+ await _watch_dev_server(wrapper, settings)+++async def _ensure_dev_server(attempt: int, log: Callable[[str], None]) -> None:+ """Probe-or-spawn the backend-owned dev server (idempotent)."""+ from services.temporal._runtime import get_temporal_server_runtime++ try:+ await get_temporal_server_runtime().ensure_started()+ except Exception as exc: # noqa: BLE001 — connect loop retries+ log(f"[Temporal] Dev server start failed (attempt {attempt}): {exc}")+++async def _startup_sweep(+ wrapper: Any,+ settings: Settings,+ log: Callable[[str], None],+) -> None:+ """Debug-only escape hatch (default false): terminate Running workflows.++ Running and paused deployments must survive backend restarts —+ durable workflow-control generations are reconciled after the+ workers start. When the sweep IS enabled, any active control row+ still vetoes it so a live deployment is never killed.+ """+ if not settings.temporal_terminate_running_on_startup:+ return+ from core.container import container++ try:+ if await container.database().has_active_workflow_controls():+ logger.info("Skipping startup Temporal termination sweep; durable workflow controls are active")+ return+ terminated = await wrapper.terminate_running_workflows()+ if terminated:+ log(f"[Temporal] Terminated {terminated} running workflow(s) at startup (history preserved)")+ except Exception as exc: # noqa: BLE001 — non-fatal+ logger.warning(f"Startup terminate-running sweep failed: {exc}")+++async def _start_execution_engine(+ client: Any,+ app_state: Any,+ settings: Settings,+ log: Callable[[str], None],+) -> None:+ """Wire the executor and start the worker manager (+ optional pool)."""+ from core.container import container+ from services.temporal import TemporalExecutor+ from services.temporal.worker import TemporalWorkerManager++ executor = TemporalExecutor(+ client=client,+ task_queue=settings.temporal_task_queue,+ )+ container.workflow_service().set_temporal_executor(executor)++ manager = TemporalWorkerManager(+ client=client,+ task_queue=settings.temporal_task_queue,+ )+ await manager.start()+ app_state.temporal_worker_manager = manager++ # Wave 16: per-queue activity worker pool (default-on since 16.4;+ # TEMPORAL_WORKER_POOL_ENABLED=false is the rollback channel).+ # Starts AFTER the manager so workflow registration is in place+ # before specialised activity workers poll.+ if settings.temporal_worker_pool_enabled:+ from services.temporal.worker import TemporalWorkerPool++ pool = TemporalWorkerPool(client=client)+ await pool.start()+ app_state.temporal_pool = pool+ log(f"[Temporal] Worker pool started ({len(pool.queues)} queues)")+++async def _boot_reconcile(log: Callable[[str], None]) -> None:+ """Converge durable control rows left transitional by a crash/restart.++ Non-fatal — every status/pause/resume request also reconciles+ lazily; this pass just closes the unattended-server window where+ durable intent and runtime behaviour could diverge indefinitely.+ """+ try:+ from services.deployment.handlers import reconcile_active_controls_on_boot++ count = await reconcile_active_controls_on_boot()+ if count:+ log(f"[Temporal] Reconciled {count} active workflow control(s)")+ except Exception as exc: # noqa: BLE001 — non-fatal+ logger.warning(f"Boot-time workflow-control reconcile failed: {exc}")+++async def _watch_dev_server(wrapper: Any, settings: Settings) -> None:+ """Resident watchdog for the backend-owned dev server.++ Deployed workflows must keep executing for months, so a dev-server+ child that dies (or wedges) while the backend stays up is respawned+ here — previously nothing probed it again after startup and every+ deployment froze until an operator intervened.+ """+ from services.temporal._runtime import get_temporal_server_runtime++ runtime = get_temporal_server_runtime()+ interval = settings.temporal_health_monitor_interval_seconds+ unhealthy = 0+ while True:+ await asyncio.sleep(interval)+ try:+ # Respawn if the child died and freed the port; no-op while+ # anything is listening on it.+ await runtime.ensure_started()+ if await wrapper.check_health():+ unhealthy = 0+ continue+ unhealthy += 1+ if unhealthy < _WATCHDOG_UNHEALTHY_THRESHOLD:+ continue+ # Port bound but gRPC not SERVING. Restart a wedged child we+ # own; an orphan we didn't spawn is only reported.+ if runtime.is_running():+ logger.warning(+ f"Temporal dev server unhealthy for {unhealthy} consecutive probes; restarting owned child"+ )+ await runtime.stop()+ await runtime.start()+ else:+ logger.warning(+ "Temporal gRPC port is bound but not SERVING and the process "+ "is not supervisor-owned; cannot restart it safely"+ )+ unhealthy = 0+ except asyncio.CancelledError:+ raise+ except Exception as exc: # noqa: BLE001 — keep the watchdog alive+ logger.warning(f"Temporal health monitor iteration failed: {exc}")+++__all__ = [+ "owns_dev_server",+ "run_temporal_lifecycle",+]diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex 3b0e65a0..07b42c60 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -148,119 +148,7 @@ class TemporalWorkerManager:# Create activity instance with shared sessionself._activities = NodeExecutionActivities(self._session)- # F4.A: register per-type activities alongside the legacy- # `execute_node_activity` dispatcher. The orchestrator at- # workflow.py:_resolve_activity picks one of the two paths per node- # based on the temporal_per_type_dispatch flag; both must be served- # by the worker. Per-type activities register WITHOUT a task_queue- # filter here (cls.task_queue is the *declared* preference; this- # framework worker polls self.task_queue regardless). Per-queue- # filtering lives in TemporalWorkerPool (below in this file),- # which is default-on (Wave 16.4) and runs one activity-only- # worker per declared queue. Registration cost: ~1.6s startup;- # zero runtime cost when the flag is off (orchestrator routes to- # execute_node_activity, per-type entries sit idle).- #- # F4.B: register AgentWorkflow + its three activities- # (execute_llm_step / persist_turn / compact_memory). The- # orchestrator schedules AgentWorkflow as a child workflow- # for the 15 migrating agent types when- # ``temporal_agent_workflow_enabled`` is on.- from services.temporal.plugin_activities import (- collect_plugin_activities,- collect_polling_activities,- )- from services.temporal.agent_activities import collect_agent_activities- from services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflow- from services.temporal.activities import (- broadcast_trigger_status_activity,- load_persisted_workflow_graph_activity,- store_node_output_activity,- )-- per_type = collect_plugin_activities() # no queue filter; all plugins- agent_activities = collect_agent_activities()- polling_activities = collect_polling_activities()- # Plugin-owned workflow classes (e.g. cron's- # CronTriggerWorkflow) self-register a SimplePlugin via- # services.temporal.plugin_registry.register_temporal_plugin- # from their plugin __init__.py. The Temporal SDK's plugin- # chain merges each registered plugin's workflows / activities- # / interceptors into the effective worker configuration —- # the framework worker stays plugin-agnostic.- plugin_list = temporal_plugins()-- # The Worker() constructor derives its default build id by- # MD5-hashing the bytecode of every module in sys.modules- # (disk reads included) — a synchronous ~3s stall at our- # module count (152 plugins + LLM SDK stack) that freezes- # the event loop and starves concurrent boot work (e.g.- # broadcaster refreshes). The result is memoized in an SDK- # module global, so one off-loop call here makes this- # Worker AND every TemporalWorkerPool worker construct- # cheaply. load_default_build_id is not re-exported from- # temporalio.worker (private module in 1.30); if an SDK- # upgrade moves it, skip the pre-warm and pay the- # synchronous cost again rather than fail startup.- try:- from temporalio.worker._worker import load_default_build_id- except ImportError:- logger.warning(- "temporalio private API moved: load_default_build_id "- "unavailable, skipping off-loop build-id pre-warm"- )- else:- await asyncio.to_thread(load_default_build_id)-- self._worker = Worker(- self.client,- task_queue=self.task_queue,- plugins=plugin_list,- workflows=[- MachinaWorkflow,- AgentWorkflow,- DelegatedTaskWorkflow,- TriggerListenerWorkflow,- PollingTriggerWorkflow,- WorkflowControlWorkflow,- ],- activities=[- self._activities.execute_node_activity,- broadcast_trigger_status_activity,- load_persisted_workflow_graph_activity,- store_node_output_activity,- *per_type,- *agent_activities,- *polling_activities,- ],- # Allow concurrent activity execution for parallel branches- max_concurrent_activities=self.pool_size,- max_concurrent_workflow_tasks=10,- graceful_shutdown_timeout=_graceful_shutdown_timeout(),- identity=_worker_identity(self.task_queue),- # ``TracingInterceptor`` is registered once on the shared client;- # the SDK prepends client interceptors to every worker, so- # repeating it here double-instruments each Signal / Query /- # Update / Activity and doubles the emitted spans.- interceptors=[ObservabilityWorkerInterceptor()],- # Wave 18.2: sticky cache sized by deployment mode so- # cached workflows skip Event-History replay without- # blowing laptop RAM.- max_cached_workflows=_sticky_cache_size(),- # Wave 18.3: autoscaling pollers replace fixed counts —- # scale between min/max on demand (SDK-recommended over- # manual poller tuning). Pollers stay well below the- # executor slot counts per the worker-performance docs.- activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=10),- workflow_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=20),- )- logger.info(- "Registered Temporal activities",- legacy=1,- per_type=len(per_type),- agent=len(agent_activities),- task_queue=self.task_queue,- )+ self._worker = await self._build_worker()span.set_attribute("task_queue", self.task_queue)span.set_attribute("pool_size", self.pool_size)@@ -276,6 +164,129 @@ class TemporalWorkerManager:name="temporal-worker",)+ async def _build_worker(self) -> Worker:+ """Construct a fresh, ready-to-run Worker instance.++ Called from :meth:`start` AND from every crash-restart attempt in+ :meth:`_run_worker` — the SDK ``Worker`` is single-use (a second+ ``run()`` on the same instance raises ``RuntimeError("Already+ started")``), so restarting must always rebuild.+ """+ # F4.A: register per-type activities alongside the legacy+ # `execute_node_activity` dispatcher. The orchestrator at+ # workflow.py:_resolve_activity picks one of the two paths per node+ # based on the temporal_per_type_dispatch flag; both must be served+ # by the worker. Per-type activities register WITHOUT a task_queue+ # filter here (cls.task_queue is the *declared* preference; this+ # framework worker polls self.task_queue regardless). Per-queue+ # filtering lives in TemporalWorkerPool (below in this file),+ # which is default-on (Wave 16.4) and runs one activity-only+ # worker per declared queue. Registration cost: ~1.6s startup;+ # zero runtime cost when the flag is off (orchestrator routes to+ # execute_node_activity, per-type entries sit idle).+ #+ # F4.B: register AgentWorkflow + its three activities+ # (execute_llm_step / persist_turn / compact_memory). The+ # orchestrator schedules AgentWorkflow as a child workflow+ # for the 15 migrating agent types when+ # ``temporal_agent_workflow_enabled`` is on.+ from services.temporal.plugin_activities import (+ collect_plugin_activities,+ collect_polling_activities,+ )+ from services.temporal.agent_activities import collect_agent_activities+ from services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflow+ from services.temporal.activities import (+ broadcast_trigger_status_activity,+ load_persisted_workflow_graph_activity,+ store_node_output_activity,+ )++ per_type = collect_plugin_activities() # no queue filter; all plugins+ agent_activities = collect_agent_activities()+ polling_activities = collect_polling_activities()+ # Plugin-owned workflow classes (e.g. cron's+ # CronTriggerWorkflow) self-register a SimplePlugin via+ # services.temporal.plugin_registry.register_temporal_plugin+ # from their plugin __init__.py. The Temporal SDK's plugin+ # chain merges each registered plugin's workflows / activities+ # / interceptors into the effective worker configuration —+ # the framework worker stays plugin-agnostic.+ plugin_list = temporal_plugins()++ # The Worker() constructor derives its default build id by+ # MD5-hashing the bytecode of every module in sys.modules+ # (disk reads included) — a synchronous ~3s stall at our+ # module count (152 plugins + LLM SDK stack) that freezes+ # the event loop and starves concurrent boot work (e.g.+ # broadcaster refreshes). The result is memoized in an SDK+ # module global, so one off-loop call here makes this+ # Worker AND every TemporalWorkerPool worker construct+ # cheaply. load_default_build_id is not re-exported from+ # temporalio.worker (private module in 1.30); if an SDK+ # upgrade moves it, skip the pre-warm and pay the+ # synchronous cost again rather than fail startup.+ try:+ from temporalio.worker._worker import load_default_build_id+ except ImportError:+ logger.warning(+ "temporalio private API moved: load_default_build_id "+ "unavailable, skipping off-loop build-id pre-warm"+ )+ else:+ await asyncio.to_thread(load_default_build_id)++ worker = Worker(+ self.client,+ task_queue=self.task_queue,+ plugins=plugin_list,+ workflows=[+ MachinaWorkflow,+ AgentWorkflow,+ DelegatedTaskWorkflow,+ TriggerListenerWorkflow,+ PollingTriggerWorkflow,+ WorkflowControlWorkflow,+ ],+ activities=[+ self._activities.execute_node_activity,+ broadcast_trigger_status_activity,+ load_persisted_workflow_graph_activity,+ store_node_output_activity,+ *per_type,+ *agent_activities,+ *polling_activities,+ ],+ # Allow concurrent activity execution for parallel branches+ max_concurrent_activities=self.pool_size,+ max_concurrent_workflow_tasks=10,+ graceful_shutdown_timeout=_graceful_shutdown_timeout(),+ identity=_worker_identity(self.task_queue),+ # ``TracingInterceptor`` is registered once on the shared client;+ # the SDK prepends client interceptors to every worker, so+ # repeating it here double-instruments each Signal / Query /+ # Update / Activity and doubles the emitted spans.+ interceptors=[ObservabilityWorkerInterceptor()],+ # Wave 18.2: sticky cache sized by deployment mode so+ # cached workflows skip Event-History replay without+ # blowing laptop RAM.+ max_cached_workflows=_sticky_cache_size(),+ # Wave 18.3: autoscaling pollers replace fixed counts —+ # scale between min/max on demand (SDK-recommended over+ # manual poller tuning). Pollers stay well below the+ # executor slot counts per the worker-performance docs.+ activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=10),+ workflow_task_poller_behavior=PollerBehaviorAutoscaling(initial=2, minimum=1, maximum=20),+ )+ logger.info(+ "Registered Temporal activities",+ legacy=1,+ per_type=len(per_type),+ agent=len(agent_activities),+ task_queue=self.task_queue,+ )+ return worker+async def _run_worker(self) -> None:"""Run the worker (background task), self-restarting on crash.@@ -284,8 +295,13 @@ class TemporalWorkerManager:detached — the startup retry loop in ``main.py`` has alreadyreturned by the time it runs. So a transient crash (e.g. theserver briefly unavailable mid-poll) would otherwise leave the- worker permanently dead until a process restart. Re-run the SAME- worker instance with doubling backoff; cancellation (from+ worker permanently dead until a process restart.++ Every restart attempt REBUILDS the worker via+ :meth:`_build_worker`: the SDK ``Worker`` is single-use, so+ re-running the same instance raises ``RuntimeError("Already+ started")`` on the first retry and every one after it — the old+ loop spun forever without ever polling again. Cancellation (from``stop()``) always wins so shutdown is never delayed by a restart.Backoff knobs are env-driven (canonical defaults in .env.template)."""@@ -303,11 +319,18 @@ class TemporalWorkerManager:raiseexcept Exception as e:logger.error(f"Temporal worker crashed; restarting in {backoff:.1f}s: {e}")- try:- await asyncio.sleep(backoff)- except asyncio.CancelledError:- raise # cancellation during backoff still wins- backoff = min(backoff * 2, backoff_max)+ try:+ await asyncio.sleep(backoff)+ except asyncio.CancelledError:+ raise # cancellation during backoff still wins+ backoff = min(backoff * 2, backoff_max)+ try:+ self._worker = await self._build_worker()+ except Exception as build_exc: # noqa: BLE001 — retried next loop+ # Next iteration's run() on the stale single-use instance+ # fails fast and re-enters this backoff, so rebuild+ # failures are retried rather than fatal.+ logger.error(f"Temporal worker rebuild failed; will retry: {build_exc}")async def stop(self) -> None:"""Stop the Temporal worker and cleanup resources."""@@ -484,60 +507,113 @@ class TemporalWorkerPool:def is_running(self) -> bool:return any(t is not None and not t.done() for t in self._tasks)- async def start(self) -> None:+ def _build_queue_worker(self, queue: str) -> Optional[Worker]:+ """Construct a fresh Worker for one queue.++ Returns ``None`` when the queue has no registered activities.+ Called from :meth:`start` AND from every crash-restart attempt in+ :meth:`_run_queue_worker` — the SDK ``Worker`` is single-use, so+ restarting must always rebuild.+ """from services.temporal.plugin_activities import (collect_plugin_activities,)+ activities = collect_plugin_activities(task_queue=queue)+ if not activities:+ return None+ concurrency = self._concurrency_for(queue)+ rate_limit = self._rate_limit_for(queue)+ tuner = self._tuner_for(queue, concurrency)+ worker_kwargs: dict = dict(+ task_queue=queue,+ activities=activities,+ graceful_shutdown_timeout=_graceful_shutdown_timeout(),+ identity=_worker_identity(queue),+ # ``TracingInterceptor`` is registered once on the shared client;+ # the SDK prepends client interceptors to every worker, so+ # repeating it here double-instruments each Signal / Query /+ # Update / Activity and doubles the emitted spans.+ interceptors=[ObservabilityWorkerInterceptor()],+ # Wave 18.1: per-queue activities/second ceiling.+ max_activities_per_second=rate_limit,+ # Wave 18.3: activity-only workers need a small+ # autoscaling poller budget — specialised queues see+ # lower task volume than the manager's default queue.+ activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=1, minimum=1, maximum=5),+ )+ # Wave 18.4: the Worker rejects `tuner` alongside ANY of+ # `max_concurrent_workflow_tasks` / `max_concurrent_activities` /+ # `max_concurrent_local_activities` / `max_concurrent_nexus_tasks`+ # — the SDK enforces "size via tuner OR via max_concurrent_*,+ # never both" (temporalio.worker.Worker.__init__). Resource-tuned+ # queues get the tuner (which already carries a+ # FixedSizeSlotSupplier(10) for workflow slots via+ # `_tuner_for`); the rest keep fixed sizing.+ if tuner is not None:+ worker_kwargs["tuner"] = tuner+ else:+ worker_kwargs["max_concurrent_activities"] = concurrency+ worker_kwargs["max_concurrent_workflow_tasks"] = 10++ worker = Worker(self.client, **worker_kwargs)+ logger.info(+ f"[Pool] Built worker queue={queue!r} "+ f"activities={len(activities)} "+ f"slots={'resource-tuned<=' + str(max(concurrency, 2)) if tuner else concurrency} "+ f"rate_limit={rate_limit if rate_limit is not None else 'unthrottled'}"+ )+ return worker++ async def _run_queue_worker(self, queue: str, worker: Worker) -> None:+ """Run one queue's worker, rebuilding + restarting on crash.++ Previously each queue worker ran as a bare ``worker.run()`` task:+ a crash silently killed the ONLY worker for that queue and every+ activity routed there pended forever (its workflows stalled)+ until a backend restart. Same backoff + rebuild contract as+ :meth:`TemporalWorkerManager._run_worker`; cancellation (from+ :meth:`stop`) always wins.+ """+ from core.config import Settings++ _s = Settings()+ backoff = _s.temporal_worker_restart_backoff_seconds+ backoff_max = _s.temporal_worker_restart_backoff_max_seconds+ current = worker+ while True:+ try:+ await current.run()+ return # clean shutdown only+ except asyncio.CancelledError:+ raise+ except Exception as exc: # noqa: BLE001 — supervised restart+ logger.error(f"[Pool] Worker for queue {queue!r} crashed; restarting in {backoff:.1f}s: {exc}")+ try:+ await asyncio.sleep(backoff)+ except asyncio.CancelledError:+ raise # cancellation during backoff still wins+ backoff = min(backoff * 2, backoff_max)+ rebuilt = self._build_queue_worker(queue)+ if rebuilt is None:+ logger.error(f"[Pool] Queue {queue!r} has no activities on rebuild; worker not restarted")+ return+ current = rebuilt++ async def start(self) -> None:for queue in self.queues:- activities = collect_plugin_activities(task_queue=queue)- if not activities:+ worker = self._build_queue_worker(queue)+ if worker is None:logger.info(f"[Pool] Skipping empty queue {queue!r}")continue- concurrency = self._concurrency_for(queue)- rate_limit = self._rate_limit_for(queue)- tuner = self._tuner_for(queue, concurrency)- worker_kwargs: dict = dict(- task_queue=queue,- activities=activities,- graceful_shutdown_timeout=_graceful_shutdown_timeout(),- identity=_worker_identity(queue),- # ``TracingInterceptor`` is registered once on the shared client;- # the SDK prepends client interceptors to every worker, so- # repeating it here double-instruments each Signal / Query /- # Update / Activity and doubles the emitted spans.- interceptors=[ObservabilityWorkerInterceptor()],- # Wave 18.1: per-queue activities/second ceiling.- max_activities_per_second=rate_limit,- # Wave 18.3: activity-only workers need a small- # autoscaling poller budget — specialised queues see- # lower task volume than the manager's default queue.- activity_task_poller_behavior=PollerBehaviorAutoscaling(initial=1, minimum=1, maximum=5),+ task = asyncio.create_task(+ self._run_queue_worker(queue, worker),+ name=f"worker-{queue}",)- # Wave 18.4: the Worker rejects `tuner` alongside ANY of- # `max_concurrent_workflow_tasks` / `max_concurrent_activities` /- # `max_concurrent_local_activities` / `max_concurrent_nexus_tasks`- # — the SDK enforces "size via tuner OR via max_concurrent_*,- # never both" (temporalio.worker.Worker.__init__). Resource-tuned- # queues get the tuner (which already carries a- # FixedSizeSlotSupplier(10) for workflow slots via- # `_tuner_for`); the rest keep fixed sizing.- if tuner is not None:- worker_kwargs["tuner"] = tuner- else:- worker_kwargs["max_concurrent_activities"] = concurrency- worker_kwargs["max_concurrent_workflow_tasks"] = 10-- worker = Worker(self.client, **worker_kwargs)- task = asyncio.create_task(worker.run(), name=f"worker-{queue}")+ # Initial instances only — after a crash-restart the live+ # instance is owned by the supervising task.self._workers.append(worker)self._tasks.append(task)- logger.info(- f"[Pool] Started worker queue={queue!r} "- f"activities={len(activities)} "- f"slots={'resource-tuned<=' + str(max(concurrency, 2)) if tuner else concurrency} "- f"rate_limit={rate_limit if rate_limit is not None else 'unthrottled'}"- )async def stop(self) -> None:for task in self._tasks:diff --git a/server/tests/temporal/test_worker_restart.py b/server/tests/temporal/test_worker_restart.pyindex 6276d8bd..b3d0b306 100644--- a/server/tests/temporal/test_worker_restart.py+++ b/server/tests/temporal/test_worker_restart.py@@ -1,12 +1,19 @@-"""Worker self-restart in TemporalWorkerManager._run_worker.+"""Worker self-restart in TemporalWorkerManager / TemporalWorkerPool.The Temporal worker shuts down on a transient poll failure rather thanauto-retrying, and the run task is detached (main.py's startup retry loop-has already returned). ``_run_worker`` now re-runs the same worker instance-with doubling backoff on a crash, while cancellation (from ``stop()``)-always wins so shutdown is never delayed by a restart. Backoff knobs come-from Settings (env-driven); the conftest stubs Settings, so we patch it to-a SimpleNamespace with tiny real backoffs.+has already returned). The SDK ``Worker`` is single-use — a second+``run()`` on the same instance raises ``RuntimeError("Already started")``+— so every restart attempt must REBUILD a fresh worker. The previous+restart loop re-ran the same instance, which meant one crash left the+task queue permanently unpolled while the loop logged "restarting"+forever; the previous version of this test masked that by mocking a+re-runnable worker.++Cancellation (from ``stop()``) always wins so shutdown is never delayed+by a restart. Backoff knobs come from Settings (env-driven); the conftest+stubs Settings, so we patch it to a SimpleNamespace with tiny real+backoffs."""from __future__ import annotations@@ -17,7 +24,7 @@ from unittest.mock import MagicMock, patchimport pytest-from services.temporal.worker import TemporalWorkerManager+from services.temporal.worker import TemporalWorkerManager, TemporalWorkerPooldef _patch_backoff():@@ -30,29 +37,82 @@ def _patch_backoff():)-def _manager_with_run(fake_run) -> TemporalWorkerManager:+class _SingleUseWorker:+ """Mimics the SDK contract: a second run() on one instance raises."""++ def __init__(self, run_impl):+ self._run_impl = run_impl+ self._started = False++ async def run(self):+ if self._started:+ raise RuntimeError("Already started")+ self._started = True+ await self._run_impl()+++def _manager_with_builds(run_impls) -> tuple[TemporalWorkerManager, dict]:+ """Manager whose _build_worker yields single-use workers in sequence."""mgr = TemporalWorkerManager(client=MagicMock(), task_queue="q")- worker = MagicMock()- worker.run = fake_run- mgr._worker = worker- return mgr+ state = {"builds": 0, "runs": 0}+ impls = list(run_impls)+ async def _tracked(impl):+ state["runs"] += 1+ await impl()-async def test_worker_restarts_after_transient_crash():- calls = {"n": 0}+ async def fake_build_worker():+ impl = impls[min(state["builds"], len(impls) - 1)]+ state["builds"] += 1+ return _SingleUseWorker(lambda impl=impl: _tracked(impl))- async def fake_run():- calls["n"] += 1- if calls["n"] == 1:- raise RuntimeError("transient poll failure")+ mgr._build_worker = fake_build_worker # type: ignore[method-assign]+ return mgr, state+++async def test_worker_restarts_with_fresh_instance_after_transient_crash():+ async def crash():+ raise RuntimeError("transient poll failure")++ async def healthy():await asyncio.Event().wait() # healthy run blocks- mgr = _manager_with_run(fake_run)+ mgr, state = _manager_with_builds([crash, healthy])+ mgr._worker = await mgr._build_worker()with _patch_backoff():task = asyncio.create_task(mgr._run_worker())- await asyncio.sleep(0.05) # allow crash + restart- assert calls["n"] == 2 # restarted exactly once+ await asyncio.sleep(0.05) # allow crash + rebuild + restart+ # First build happened before the loop; the restart rebuilt once.+ assert state["builds"] == 2+ assert state["runs"] == 2+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+++async def test_restart_never_reuses_the_single_use_instance():+ """Locks the regression the old mock hid: re-running the same SDK+ worker raises "Already started" instantly. The restart loop must+ rebuild before every retry, so the crashed instance is run once."""+ crashed = _SingleUseWorker(run_impl=None)++ async def crash():+ raise RuntimeError("transient poll failure")++ async def healthy():+ await asyncio.Event().wait()++ mgr, state = _manager_with_builds([healthy])+ crashed._run_impl = crash+ mgr._worker = crashed++ with _patch_backoff():+ task = asyncio.create_task(mgr._run_worker())+ await asyncio.sleep(0.05)+ # The crashed single-use instance was replaced, not re-run.+ assert mgr._worker is not crashed+ assert state["builds"] == 1task.cancel()with pytest.raises(asyncio.CancelledError):await task@@ -61,11 +121,12 @@ async def test_worker_restarts_after_transient_crash():async def test_cancellation_does_not_restart():calls = {"n": 0}- async def fake_run():+ async def healthy():calls["n"] += 1await asyncio.Event().wait() # healthy run blocks until cancelled- mgr = _manager_with_run(fake_run)+ mgr, state = _manager_with_builds([healthy])+ mgr._worker = await mgr._build_worker()with _patch_backoff():task = asyncio.create_task(mgr._run_worker())@@ -76,3 +137,50 @@ async def test_cancellation_does_not_restart():await taskassert calls["n"] == 1 # cancellation wins; not restarted+ assert state["builds"] == 1 # and nothing was rebuilt+++async def test_pool_queue_worker_restarts_with_fresh_instance():+ """Pool workers previously ran as bare worker.run() tasks with no+ restart machinery — a crash silently killed the only worker for the+ queue and its activities pended forever."""+ pool = TemporalWorkerPool(client=MagicMock(), queues=["code-exec"])+ state = {"builds": 0}++ async def crash():+ raise RuntimeError("poll failure")++ async def healthy():+ await asyncio.Event().wait()++ def fake_build(queue):+ impl = crash if state["builds"] == 0 else healthy+ state["builds"] += 1+ return _SingleUseWorker(impl)++ pool._build_queue_worker = fake_build # type: ignore[method-assign]++ with _patch_backoff():+ first = pool._build_queue_worker("code-exec")+ task = asyncio.create_task(pool._run_queue_worker("code-exec", first))+ await asyncio.sleep(0.05)+ assert state["builds"] == 2 # crash triggered one rebuild+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+++async def test_pool_queue_worker_stops_when_queue_has_no_activities():+ pool = TemporalWorkerPool(client=MagicMock(), queues=["code-exec"])++ async def crash():+ raise RuntimeError("poll failure")++ pool._build_queue_worker = lambda queue: None # type: ignore[method-assign]++ with _patch_backoff():+ task = asyncio.create_task(+ pool._run_queue_worker("code-exec", _SingleUseWorker(crash))+ )+ # Crash, then rebuild returns None -> the supervisor task ends.+ await asyncio.wait_for(task, timeout=1.0)