Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(temporal): never terminate running deployments on boot
trohitg532236e3ae02
diff --git a/.env b/.envindex f7faab5d..1fda4fc8 100644--- a/.env+++ b/.env@@ -74,19 +74,13 @@ TEMPORAL_UI_PORT=5680# below for the resumption knob.TEMPORAL_SQLITE_PATH=temporal.db# On server boot, terminate every workflow still in ``Running`` state.-# History is preserved (workflows show as ``Terminated`` in the UI,-# NOT deleted) but resumption is disabled, because OpenCompany's-# ``DeploymentManager`` has no boot-time reconcile against Temporal-# Visibility yet — resumed workflows would otherwise keep executing-# but be invisible to the OpenCompany UI. Flip to ``false`` once the-# reconcile lands to re-enable durable resumption.-TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=true+# Keep false so running and paused deployments survive an application+# restart — the durable workflow-control plane reconciles them. Setting+# true converts every backend boot into a namespace-wide terminate+# sweep (debug-only escape hatch).+TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false# Graceful shutdown window for the Temporal child process.TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=30-# Supervisor readiness-probe window. The CLI binary may need to-# download (~114 MB zip) on first ``company start`` if ``company build``-# didn't pre-cache it.-TEMPORAL_SERVER_READY_TIMEOUT_SECONDS=120# Wave 12 event framework. ``true`` (default) routes trigger events# through TriggerListenerWorkflow + Temporal Signal fan-out for thediff --git a/.env.template b/.env.templateindex bb8b8a4b..b02863e8 100644--- a/.env.template+++ b/.env.template@@ -107,20 +107,19 @@ TEMPORAL_UI_PORT=5680TEMPORAL_SQLITE_PATH=temporal.db# On server boot, terminate every workflow still in ``Running`` state.# History is preserved (workflows show as ``Terminated`` in the UI,-# NOT deleted) but resumption is disabled, because OpenCompany's-# ``DeploymentManager`` has no boot-time reconcile against Temporal-# Visibility yet — resumed workflows would otherwise keep executing-# but be invisible to the OpenCompany UI. Flip to ``false`` once the-# reconcile lands to re-enable durable resumption.-# Durable workflow-control generations reconcile active Temporal executions.-# Keep false so running and paused deployments survive an application restart.+# NOT deleted) but the live executions stop. Durable workflow-control+# generations reconcile active Temporal executions on boot, so keep+# false — running and paused deployments must survive an application+# restart and keep executing for months. Setting true is a debug-only+# escape hatch that converts every boot into a namespace-wide sweep.TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false# Graceful shutdown window for the Temporal child process.TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=30-# Supervisor readiness-probe window. The CLI binary may need to-# download (~114 MB zip) on first ``company start`` if ``company build``-# didn't pre-cache it.-TEMPORAL_SERVER_READY_TIMEOUT_SECONDS=120+# Backend-owned dev-server health monitor: after startup the lifespan+# keeps probing the Temporal dev server at this interval and respawns+# it if the child process dies while the backend stays up. Loopback+# (backend-owned) servers only; remote clusters are never touched.+TEMPORAL_HEALTH_MONITOR_INTERVAL_SECONDS=15# --- Startup resilience (server-ready-before-workers) ---# After the gRPC port binds, the client polls the WorkflowService gRPCdiff --git a/server/core/config.py b/server/core/config.pyindex 4550bbb5..2c877cd8 100644--- a/server/core/config.py+++ b/server/core/config.py@@ -184,12 +184,13 @@ class Settings(BaseSettings):# (``Settings._resolve_under_data``) unless absolute. Sourced# from ``.env.template`` (canonical default lives there).temporal_sqlite_path: str = Field(env="TEMPORAL_SQLITE_PATH")- # Terminate every workflow in ``Running`` state on server boot.- # Preserves history (workflows show as ``Terminated`` in the- # Temporal UI, not deleted) but prevents auto-resumption, since- # OpenCompany's ``DeploymentManager`` has no boot-time reconcile- # against Temporal Visibility yet — resumed workflows would- # otherwise be invisible to the UI.+ # Debug-only escape hatch: terminate every workflow in ``Running``+ # state on server boot. MUST stay false in normal operation — the+ # durability guarantee is that running and paused deployments+ # survive backend restarts and keep executing for months. The+ # durable workflow-control plane reconciles active generations on+ # boot; a live control row in any active state also skips the sweep+ # entirely (``has_active_workflow_controls``).temporal_terminate_running_on_startup: bool = Field(env="TEMPORAL_TERMINATE_RUNNING_ON_STARTUP",)@@ -239,6 +240,15 @@ class Settings(BaseSettings):env="TEMPORAL_WORKER_RESTART_BACKOFF_MAX_SECONDS",gt=0,)+ # Backend-owned dev-server health monitor: after startup the lifespan+ # keeps probing the Temporal dev server at this interval and respawns+ # it if the child process dies while the backend stays up. Loopback+ # (backend-owned) servers only; remote clusters are never touched.+ temporal_health_monitor_interval_seconds: float = Field(+ default=15.0,+ env="TEMPORAL_HEALTH_MONITOR_INTERVAL_SECONDS",+ gt=0,+ )# API Keys (all optional, injected at runtime)google_maps_api_key: Optional[str] = Field(default=None, env="GOOGLE_MAPS_API_KEY")diff --git a/server/core/database.py b/server/core/database.pyindex 63464bdd..79d66f2d 100644--- a/server/core/database.py+++ b/server/core/database.py@@ -2533,14 +2533,33 @@ class Database:return result.scalar_one_or_none()async def has_active_workflow_controls(self) -> bool:+ from models.database import WORKFLOW_CONTROL_ACTIVE_STATES+async with self.get_session() as session:result = await session.execute(select(func.count()).select_from(WorkflowControlExecution).where(- WorkflowControlExecution.status.in_({"starting", "running", "pausing", "paused", "resuming"})+ WorkflowControlExecution.status.in_(WORKFLOW_CONTROL_ACTIVE_STATES)))return bool(result.scalar_one())+ async def list_active_workflow_controls(self) -> List[WorkflowControlExecution]:+ """Every control row in a non-terminal status (boot-time reconcile).++ At most one generation per workflow can be active (``begin_generation``+ requires the previous one to be ``reset``), so this is effectively+ latest-generation-per-workflow filtered to the active set.+ """+ from models.database import WORKFLOW_CONTROL_ACTIVE_STATES++ async with self.get_session() as session:+ result = await session.execute(+ select(WorkflowControlExecution)+ .where(WorkflowControlExecution.status.in_(WORKFLOW_CONTROL_ACTIVE_STATES))+ .order_by(WorkflowControlExecution.workflow_id, WorkflowControlExecution.generation.desc())+ )+ return list(result.scalars().all())+async def get_workflow_control_by_idempotency_key(self, workflow_id: str, idempotency_key: str) -> Optional[WorkflowControlExecution]:diff --git a/server/models/database.py b/server/models/database.pyindex 3d24b43b..93f6ab2a 100644--- a/server/models/database.py+++ b/server/models/database.py@@ -460,6 +460,16 @@ class AgentTeam(SQLModel, table=True):completed_at: Optional[datetime] = Field(default=None)+# Canonical set of non-terminal WorkflowControlExecution statuses. Single+# source of truth shared by the control plane (services/deployment/control.py)+# and the startup terminate-sweep guard (core/database.py) — the two used to+# carry independently maintained copies that drifted ("resetting" was missing+# from the sweep guard, so a boot mid-reset could sweep the whole namespace).+WORKFLOW_CONTROL_ACTIVE_STATES = frozenset(+ {"starting", "running", "pausing", "paused", "resuming", "resetting"}+)++class WorkflowControlExecution(SQLModel, table=True):"""Durable control generation for one saved workflow deployment."""diff --git a/server/services/deployment/control.py b/server/services/deployment/control.pyindex 2d00adf9..3d3c795b 100644--- a/server/services/deployment/control.py+++ b/server/services/deployment/control.py@@ -7,11 +7,13 @@ import jsonfrom datetime import datetime, timezonefrom typing import Any, Dict, Optional-from models.database import WorkflowControlExecution+from models.database import WORKFLOW_CONTROL_ACTIVE_STATES, WorkflowControlExecutionfrom sqlalchemy.exc import IntegrityError-ACTIVE_STATES = {"starting", "running", "pausing", "paused", "resuming", "resetting"}+# Re-exported name kept for existing importers; the canonical definition+# lives beside the model so the terminate-sweep guard shares it.+ACTIVE_STATES = WORKFLOW_CONTROL_ACTIVE_STATESdef _graph_hash(nodes: list[dict], edges: list[dict]) -> str: