Skip to content

Self-improving AI Employees for everyone

fix(temporal): never terminate running deployments on boot

trohitg532236e3ae02
diff --git a/.env b/.env
index f7faab5d..1fda4fc8 100644
--- a/.env
+++ b/.env
@@ -74,19 +74,13 @@ TEMPORAL_UI_PORT=5680
# below for the resumption knob.
TEMPORAL_SQLITE_PATH=temporal.db
# On server boot, terminate every workflow still in ``Running`` state.
-# History is preserved (workflows show as ``Terminated`` in the UI,
-# NOT deleted) but resumption is disabled, because OpenCompany's
-# ``DeploymentManager`` has no boot-time reconcile against Temporal
-# Visibility yet — resumed workflows would otherwise keep executing
-# but be invisible to the OpenCompany UI. Flip to ``false`` once the
-# reconcile lands to re-enable durable resumption.
-TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=true
+# Keep false so running and paused deployments survive an application
+# restart — the durable workflow-control plane reconciles them. Setting
+# true converts every backend boot into a namespace-wide terminate
+# sweep (debug-only escape hatch).
+TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false
# Graceful shutdown window for the Temporal child process.
TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=30
-# Supervisor readiness-probe window. The CLI binary may need to
-# download (~114 MB zip) on first ``company start`` if ``company build``
-# didn't pre-cache it.
-TEMPORAL_SERVER_READY_TIMEOUT_SECONDS=120
# Wave 12 event framework. ``true`` (default) routes trigger events
# through TriggerListenerWorkflow + Temporal Signal fan-out for the
diff --git a/.env.template b/.env.template
index bb8b8a4b..b02863e8 100644
--- a/.env.template
+++ b/.env.template
@@ -107,20 +107,19 @@ TEMPORAL_UI_PORT=5680
TEMPORAL_SQLITE_PATH=temporal.db
# On server boot, terminate every workflow still in ``Running`` state.
# History is preserved (workflows show as ``Terminated`` in the UI,
-# NOT deleted) but resumption is disabled, because OpenCompany's
-# ``DeploymentManager`` has no boot-time reconcile against Temporal
-# Visibility yet — resumed workflows would otherwise keep executing
-# but be invisible to the OpenCompany UI. Flip to ``false`` once the
-# reconcile lands to re-enable durable resumption.
-# Durable workflow-control generations reconcile active Temporal executions.
-# Keep false so running and paused deployments survive an application restart.
+# NOT deleted) but the live executions stop. Durable workflow-control
+# generations reconcile active Temporal executions on boot, so keep
+# false — running and paused deployments must survive an application
+# restart and keep executing for months. Setting true is a debug-only
+# escape hatch that converts every boot into a namespace-wide sweep.
TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false
# Graceful shutdown window for the Temporal child process.
TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=30
-# Supervisor readiness-probe window. The CLI binary may need to
-# download (~114 MB zip) on first ``company start`` if ``company build``
-# didn't pre-cache it.
-TEMPORAL_SERVER_READY_TIMEOUT_SECONDS=120
+# Backend-owned dev-server health monitor: after startup the lifespan
+# keeps probing the Temporal dev server at this interval and respawns
+# it if the child process dies while the backend stays up. Loopback
+# (backend-owned) servers only; remote clusters are never touched.
+TEMPORAL_HEALTH_MONITOR_INTERVAL_SECONDS=15
# --- Startup resilience (server-ready-before-workers) ---
# After the gRPC port binds, the client polls the WorkflowService gRPC
diff --git a/server/core/config.py b/server/core/config.py
index 4550bbb5..2c877cd8 100644
--- a/server/core/config.py
+++ b/server/core/config.py
@@ -184,12 +184,13 @@ class Settings(BaseSettings):
# (``Settings._resolve_under_data``) unless absolute. Sourced
# from ``.env.template`` (canonical default lives there).
temporal_sqlite_path: str = Field(env="TEMPORAL_SQLITE_PATH")
- # Terminate every workflow in ``Running`` state on server boot.
- # Preserves history (workflows show as ``Terminated`` in the
- # Temporal UI, not deleted) but prevents auto-resumption, since
- # OpenCompany's ``DeploymentManager`` has no boot-time reconcile
- # against Temporal Visibility yet — resumed workflows would
- # otherwise be invisible to the UI.
+ # Debug-only escape hatch: terminate every workflow in ``Running``
+ # state on server boot. MUST stay false in normal operation — the
+ # durability guarantee is that running and paused deployments
+ # survive backend restarts and keep executing for months. The
+ # durable workflow-control plane reconciles active generations on
+ # boot; a live control row in any active state also skips the sweep
+ # entirely (``has_active_workflow_controls``).
temporal_terminate_running_on_startup: bool = Field(
env="TEMPORAL_TERMINATE_RUNNING_ON_STARTUP",
)
@@ -239,6 +240,15 @@ class Settings(BaseSettings):
env="TEMPORAL_WORKER_RESTART_BACKOFF_MAX_SECONDS",
gt=0,
)
+ # Backend-owned dev-server health monitor: after startup the lifespan
+ # keeps probing the Temporal dev server at this interval and respawns
+ # it if the child process dies while the backend stays up. Loopback
+ # (backend-owned) servers only; remote clusters are never touched.
+ temporal_health_monitor_interval_seconds: float = Field(
+ default=15.0,
+ env="TEMPORAL_HEALTH_MONITOR_INTERVAL_SECONDS",
+ gt=0,
+ )
# API Keys (all optional, injected at runtime)
google_maps_api_key: Optional[str] = Field(default=None, env="GOOGLE_MAPS_API_KEY")
diff --git a/server/core/database.py b/server/core/database.py
index 63464bdd..79d66f2d 100644
--- a/server/core/database.py
+++ b/server/core/database.py
@@ -2533,14 +2533,33 @@ class Database:
return result.scalar_one_or_none()
async def has_active_workflow_controls(self) -> bool:
+ from models.database import WORKFLOW_CONTROL_ACTIVE_STATES
+
async with self.get_session() as session:
result = await session.execute(
select(func.count()).select_from(WorkflowControlExecution).where(
- WorkflowControlExecution.status.in_({"starting", "running", "pausing", "paused", "resuming"})
+ WorkflowControlExecution.status.in_(WORKFLOW_CONTROL_ACTIVE_STATES)
)
)
return bool(result.scalar_one())
+ async def list_active_workflow_controls(self) -> List[WorkflowControlExecution]:
+ """Every control row in a non-terminal status (boot-time reconcile).
+
+ At most one generation per workflow can be active (``begin_generation``
+ requires the previous one to be ``reset``), so this is effectively
+ latest-generation-per-workflow filtered to the active set.
+ """
+ from models.database import WORKFLOW_CONTROL_ACTIVE_STATES
+
+ async with self.get_session() as session:
+ result = await session.execute(
+ select(WorkflowControlExecution)
+ .where(WorkflowControlExecution.status.in_(WORKFLOW_CONTROL_ACTIVE_STATES))
+ .order_by(WorkflowControlExecution.workflow_id, WorkflowControlExecution.generation.desc())
+ )
+ return list(result.scalars().all())
+
async def get_workflow_control_by_idempotency_key(
self, workflow_id: str, idempotency_key: str
) -> Optional[WorkflowControlExecution]:
diff --git a/server/models/database.py b/server/models/database.py
index 3d24b43b..93f6ab2a 100644
--- a/server/models/database.py
+++ b/server/models/database.py
@@ -460,6 +460,16 @@ class AgentTeam(SQLModel, table=True):
completed_at: Optional[datetime] = Field(default=None)
+# Canonical set of non-terminal WorkflowControlExecution statuses. Single
+# source of truth shared by the control plane (services/deployment/control.py)
+# and the startup terminate-sweep guard (core/database.py) — the two used to
+# carry independently maintained copies that drifted ("resetting" was missing
+# from the sweep guard, so a boot mid-reset could sweep the whole namespace).
+WORKFLOW_CONTROL_ACTIVE_STATES = frozenset(
+ {"starting", "running", "pausing", "paused", "resuming", "resetting"}
+)
+
+
class WorkflowControlExecution(SQLModel, table=True):
"""Durable control generation for one saved workflow deployment."""
diff --git a/server/services/deployment/control.py b/server/services/deployment/control.py
index 2d00adf9..3d3c795b 100644
--- a/server/services/deployment/control.py
+++ b/server/services/deployment/control.py
@@ -7,11 +7,13 @@ import json
from datetime import datetime, timezone
from typing import Any, Dict, Optional
-from models.database import WorkflowControlExecution
+from models.database import WORKFLOW_CONTROL_ACTIVE_STATES, WorkflowControlExecution
from sqlalchemy.exc import IntegrityError
-ACTIVE_STATES = {"starting", "running", "pausing", "paused", "resuming", "resetting"}
+# Re-exported name kept for existing importers; the canonical definition
+# lives beside the model so the terminate-sweep guard shares it.
+ACTIVE_STATES = WORKFLOW_CONTROL_ACTIVE_STATES
def _graph_hash(nodes: list[dict], edges: list[dict]) -> str: