Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Deployment domain WebSocket handlers.
Extracted from ``routers/websocket.py`` (Wave 13.2). The 5 handlers
below cover the deployment lifecycle:
- ``deploy_workflow`` — start a continuously-running workflow with
triggers + per-workflow locking.
- ``cancel_deployment`` — cancel a running deployment, drain its
listeners, unlock the workflow.
- ``get_deployment_status`` — snapshot of in-flight deployments.
- ``get_workflow_lock`` — current lock state.
- ``update_deployment_settings`` — mutate runtime settings without
re-deploying.
All handlers preserve their pre-Wave-13 wire shape. The module-level
``_deployment_tasks`` dict (workflow_id -> asyncio.Task) moves here
too; it was process-local in ``routers/websocket.py`` and stays the
same shape — only the import path changes.
"""
from __future__ import annotations
import asyncio
import json
import time
import uuid
import weakref
from datetime import datetime, timezone
from typing import Any, Dict, Optional
from fastapi import WebSocket
from core.logging import get_logger
from services.ws_handler_registry import ws_handler
from services.deployment.control import (
ACTIVE_STATES,
CLEAR_PAUSE_REASON,
PAUSE_REASON_CONTROLLER_MISSING,
PAUSE_REASON_FAILURES,
PAUSE_REASON_RECOVERY,
WorkflowControlService,
serialize_control,
)
# ``core.container`` and ``services.status_broadcaster`` are lazy-imported
# inside each handler body. This module is imported transitively via
# ``services.workflow`` during ``core.container`` initialization (the
# container wires ``WorkflowService`` which imports ``services.workflow``
# which imports ``services.deployment``). Eager imports at module scope
# would deadlock the partially-initialized container module.
logger = get_logger(__name__)
class TemporalControlUnavailable(RuntimeError):
"""A lifecycle Update could not start because no Temporal client exists."""
class TemporalControlAckMismatch(RuntimeError):
"""Temporal completed an Update without returning the requested state."""
class ControllerExecutionMissing(RuntimeError):
"""The controller execution this generation names no longer exists.
Distinct from :class:`TemporalControlUnavailable`, and the distinction is
the whole point: "unavailable" is transient and worth waiting out, whereas
a deleted execution never comes back. Treating the second as the first is
what let a generation sit in ``pausing`` forever, which in turn blocked
every future Start on that workflow.
"""
# Per-workflow deployment tasks for proper cancellation (Temporal/n8n pattern).
# Maps workflow_id -> asyncio.Task for parallel workflow deployments.
_deployment_tasks: Dict[str, asyncio.Task] = {}
# A socket only waits on these tasks, and at most _TRANSITION_ANSWER_SECONDS.
# Disconnecting a waiter must not undo a durable Stop, and status polling must
# never start a second checkpoint drain.
_control_transition_tasks: Dict[tuple[str, int], asyncio.Task] = {}
_boot_reconciliation_tasks: Dict[tuple[str, int], asyncio.Task] = {}
_control_completion_locks: weakref.WeakValueDictionary = weakref.WeakValueDictionary()
# Counts and Queries are UI observations, never checkpoint acknowledgements.
# An unavailable worker must not keep Reset behind an admission broadcast.
_CONTROL_OBSERVATION_TIMEOUT_SECONDS = 1.0
_BOOT_REARM_TIMEOUT_SECONDS = 30.0
_BOOT_RETRY_SECONDS = 1.0
# A Stop or Resume request answers within this, pending while its drain runs
# on; the transition's broadcast carries the outcome (1b.15).
_TRANSITION_ANSWER_SECONDS = 10.0
def _control_completion_lock(workflow_id: str) -> asyncio.Lock:
lock = _control_completion_locks.get(workflow_id)
if lock is None:
lock = asyncio.Lock()
_control_completion_locks[workflow_id] = lock
return lock
def _schedule_versioned_transition(service: WorkflowControlService, control) -> asyncio.Task:
key = (control.id, int(control.revision))
task = _control_transition_tasks.get(key)
if task is not None and not task.done():
return task
task = asyncio.create_task(_finish_versioned_transition(service, control),
name=f"workflow-control:{control.id}:{control.revision}")
_control_transition_tasks[key] = task
def completed(done: asyncio.Task) -> None:
if _control_transition_tasks.get(key) is done:
_control_transition_tasks.pop(key, None)
if not done.cancelled() and (exc := done.exception()) is not None:
logger.warning("Workflow control reconciliation remains pending",
workflow_id=control.workflow_id, revision=control.revision,
error_type=type(exc).__name__, error=str(exc))
task.add_done_callback(completed)
return task
def _supersede_versioned_transitions(control) -> None:
for (control_id, revision), task in list(_control_transition_tasks.items()):
if control_id == control.id and revision < control.revision and not task.done():
# Cancel local polling only. Reset's termination sweep is the
# authoritative cancellation of Temporal executions.
task.cancel()
_cancel_boot_reconciliation(control)
def _cancel_boot_reconciliation(control) -> None:
for (control_id, generation), task in list(_boot_reconciliation_tasks.items()):
if control_id == control.id and generation == control.generation and not task.done():
task.cancel()
async def _await_versioned_transition(service: WorkflowControlService, control, *, task=None) -> Dict[str, Any]:
"""Wait for the shared transition for at most ``_TRANSITION_ANSWER_SECONDS``.
A drain still running answers ``transition_pending``; it goes on in its
task, whose broadcast carries the outcome."""
intent_revision, intent_status = control.revision, control.status
intent_identity = (control.id, control.generation)
task = task or _schedule_versioned_transition(service, control)
try:
# asyncio.wait never cancels the task, and never mistakes a TimeoutError
# the transition itself raised for its own bound.
await asyncio.wait({task}, timeout=_TRANSITION_ANSWER_SECONDS)
if task.done():
current, controller_status = task.result()
else:
current = await service.database.get_latest_workflow_control(control.workflow_id) or control
controller_status = None
except asyncio.CancelledError:
# A Reset can retire a scheduled poll before its coroutine starts.
# Cancellation of just this socket waiter leaves task alive instead.
if not task.cancelled():
raise
current = await service.database.get_latest_workflow_control(control.workflow_id)
if current is None or (current.id, current.generation, current.revision, current.status) == (
intent_identity[0], intent_identity[1], intent_revision, intent_status
):
raise
controller_status = None
target_status = "paused" if intent_status == "pausing" else "running"
return {
"success": True,
"superseded": ((current.id, current.generation) != intent_identity
or current.status not in {intent_status, target_status}
or current.revision not in {intent_revision, intent_revision + 1}),
"transition_pending": current.status in {"pausing", "resuming"},
**await _control_payload(current, controller_status=controller_status),
}
@ws_handler()
async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Deploy workflow to run continuously until cancelled.
Expects:
workflow_id: Workflow identifier (required for locking)
nodes: List of workflow nodes with {id, type, data}
edges: List of edges with {id, source, target}
session_id: Optional session identifier
delay_between_runs: Optional delay in seconds between iterations (default: 1.0)
Returns:
Deployment start confirmation (deployment runs in background)
"""
global _deployment_tasks
from core.container import container
from services.status_broadcaster import get_status_broadcaster
workflow_service = container.workflow_service()
broadcaster = get_status_broadcaster()
workflow_id = data.get("workflow_id")
nodes = data.get("nodes", [])
edges = data.get("edges", [])
from services.workflow_context_migration import (
import_legacy_context_receipts,
load_node_parameters,
persist_parameter_aliases,
)
from services.employees.team_runtime import admit_team_plan
from services.workflow_migrations import normalize_workflow_graph
from services.workflow_sanitizer import sanitize_runtime_payload, sanitize_workflow_graph
parameters_by_id = data.get("parameters_by_id")
if not isinstance(parameters_by_id, dict):
parameters_by_id = await load_node_parameters(
container.database(),
nodes,
)
normalization = normalize_workflow_graph(
str(workflow_id or ""),
nodes,
edges,
parameters_by_id,
)
safe_graph = sanitize_workflow_graph(normalization.graph_data())
# A hired team's plan is saved beside the graph, not in it. Start and the
# restart re-arm deploy through here, so their listeners carry it too.
nodes = await admit_team_plan(container.database(), str(workflow_id or ""), list(safe_graph["nodes"]))
edges = list(safe_graph["edges"])
normalized_parameters = normalization.node_parameters
await import_legacy_context_receipts(
container.database(),
normalization.state_imports,
)
graph_version = normalization.graph_version
graph_aliases = normalization.aliases
if normalization.warnings:
logger.warning("[Deploy] %s", "; ".join(normalization.warnings))
session_id = data.get("session_id", "default")
logger.debug(f"[Deploy] Received {len(edges)} edges for workflow {workflow_id}")
for e in edges:
target_handle = e.get("targetHandle")
if target_handle and target_handle.startswith("input-") and target_handle != "input-main":
logger.debug(f"[Deploy] Config edge: {e.get('source')} -> {e.get('target')} (handle={target_handle})")
tool_edges = [e for e in edges if e.get("targetHandle") == "input-tools"]
if tool_edges:
logger.debug(f"[Deploy] Tool edges found: {len(tool_edges)}")
for te in tool_edges:
logger.debug(f"[Deploy] Tool edge: source={te.get('source')} -> target={te.get('target')}")
else:
logger.debug("[Deploy] No input-tools edges found")
if not nodes:
return {"success": False, "error": "No nodes provided"}
if not workflow_id:
return {"success": False, "error": "workflow_id is required for deployment"}
# Pre-deploy validation gate. Deploy never honors a force-override —
# a broken workflow running on a schedule is far worse than a failed
# one-shot manual run.
from services.workflow_validator import validate_workflow
deploy_report = await validate_workflow(
nodes=nodes,
edges=edges,
parameters_by_id=normalized_parameters,
enforce_context=True,
)
if deploy_report["errors"]:
return {
"success": False,
"error": "validation_failed",
"report": deploy_report,
}
# Rekey parameter rows onto canonical ids only after the graph is admitted.
# Running this before the gate orphaned configuration on a failed deploy:
# the rows were renamed and the originals deleted, while the stored graph
# kept its old ids, so the next read looked up ids that no longer existed.
# A generation's deploy writes no rows. Start saved them before it created
# the generation, and the restart re-arm passes the generation's sanitized
# snapshot, which would replace each saved row and erase its secrets (a
# webhook's header value among them) and any edit saved while paused.
if int(data.get("generation") or 0) <= 0:
await persist_parameter_aliases(
container.database(),
aliases=graph_aliases,
parameters=normalized_parameters,
)
if workflow_service.is_workflow_deployed(workflow_id):
status = workflow_service.get_deployment_status(workflow_id)
return {
"success": False,
"error": f"Workflow {workflow_id} is already deployed. Cancel it first.",
"workflow_id": workflow_id,
"is_running": True,
"run_counter": status.get("run_counter", 0),
}
lock_acquired = await broadcaster.lock_workflow(workflow_id, reason="deployment")
if not lock_acquired:
lock_info = broadcaster.get_workflow_lock(workflow_id)
return {
"success": False,
"error": f"Workflow {workflow_id} is already locked for {lock_info.get('reason', 'deployment')}",
"locked_by": lock_info.get("workflow_id"),
"locked_at": lock_info.get("locked_at"),
}
await broadcaster.update_workflow_status(
executing=True,
current_node=None,
progress=0,
workflow_id=workflow_id,
)
await broadcaster.update_deployment_status(
is_running=True,
status="starting",
active_runs=0,
workflow_id=workflow_id,
)
async def status_callback(node_id: str, status: str, node_data: Optional[Dict] = None):
if node_id == "__deployment__":
active_runs = node_data.get("active_runs", 0) if node_data else 0
await broadcaster.update_deployment_status(
is_running=True,
status=status,
active_runs=active_runs,
workflow_id=workflow_id,
data=node_data,
)
else:
await broadcaster.update_node_status(node_id, status, node_data, workflow_id=workflow_id)
if status == "executing":
position = node_data.get("position", 0) if node_data else 0
total = node_data.get("total", 1) if node_data else 1
progress = int((position / total) * 100) if total > 0 else 0
await broadcaster.update_workflow_status(
executing=True,
current_node=node_id,
progress=progress,
workflow_id=workflow_id,
)
async def run_deployment():
try:
result = await workflow_service.deploy_workflow(
nodes=nodes,
edges=edges,
session_id=session_id,
status_callback=status_callback,
workflow_id=workflow_id,
graph_version=graph_version,
generation=int(data.get("generation") or 0),
# No secret rides the snapshot into Temporal: a node gets its
# saved ones back when it runs (plugin.base.saved_settings).
**(
{"parameter_snapshot": sanitize_runtime_payload(data.get("parameters_by_id") or {})}
if int(data.get("generation") or 0) > 0
else {}
),
user_id=str(
(
getattr(
getattr(websocket, "state", None),
"user_id",
None,
)
if websocket is not None
else data.get("user_id")
)
or "owner"
),
)
if not result.get("success"):
logger.error("Deployment setup failed", error=result.get("error"), workflow_id=workflow_id)
# Clear BOTH UI-facing flags: the deploy handler already
# broadcast executing=True, and deployments never touch
# the run counter, so nothing else would ever emit
# executing=False for this workflow (toolbar stuck).
await broadcaster.update_workflow_status(
executing=False,
current_node=None,
progress=0,
workflow_id=workflow_id,
)
await broadcaster.update_deployment_status(
is_running=False,
status="error",
active_runs=0,
workflow_id=workflow_id,
error=result.get("error"),
)
await broadcaster.unlock_workflow(workflow_id)
_deployment_tasks.pop(workflow_id, None)
return result
else:
await broadcaster.update_deployment_status(
is_running=True,
status="running",
active_runs=0,
workflow_id=workflow_id,
data={
"triggers_setup": result.get("triggers_setup", []),
"deployment_id": result.get("deployment_id"),
},
)
logger.info(
"[Deployment] Event-driven deployment active",
deployment_id=result.get("deployment_id"),
workflow_id=workflow_id,
triggers=len(result.get("triggers_setup", [])),
)
return result
except asyncio.CancelledError:
# CancelledError is BaseException — the branch below cannot
# catch it, and a deploy task cancelled outside
# handle_cancel_deployment (which unlocks itself) previously
# left the workflow lock dangling forever.
await broadcaster.unlock_workflow(workflow_id)
_deployment_tasks.pop(workflow_id, None)
raise
except Exception as e:
logger.error("Deployment task error", workflow_id=workflow_id, error=str(e))
await broadcaster.update_workflow_status(
executing=False,
current_node=None,
progress=0,
workflow_id=workflow_id,
)
await broadcaster.update_deployment_status(
is_running=False,
status="error",
active_runs=0,
workflow_id=workflow_id,
error=str(e),
)
await broadcaster.unlock_workflow(workflow_id)
_deployment_tasks.pop(workflow_id, None)
return {"success": False, "error": str(e), "workflow_id": workflow_id}
_deployment_tasks[workflow_id] = asyncio.create_task(run_deployment())
return {
"success": True,
"message": "Deployment started",
"workflow_id": workflow_id,
"is_running": True,
"locked": True,
"timestamp": time.time(),
"graph": {
"graphVersion": graph_version,
"nodes": nodes,
"edges": edges,
},
"aliases": graph_aliases,
"migration_warnings": normalization.warnings,
}
@ws_handler()
async def handle_cancel_deployment(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Cancel running deployment for a specific workflow (Temporal/n8n pattern).
Expects:
workflow_id: Workflow to cancel (required).
Also cancels any active event waiters (trigger nodes) and unlocks the workflow.
Returns:
Cancellation result with iterations completed
"""
global _deployment_tasks
from core.container import container
from services.status_broadcaster import get_status_broadcaster
workflow_service = container.workflow_service()
broadcaster = get_status_broadcaster()
workflow_id = data.get("workflow_id")
if not workflow_id:
return {"success": False, "error": "workflow_id is required for cancellation"}
result = await workflow_service.cancel_deployment(workflow_id)
cancelled_waiters = 0
if result.get("success"):
cancelled_waiters = result.get("waiters_cancelled", 0)
task = _deployment_tasks.pop(workflow_id, None)
if task and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
logger.info("[Deployment] Deployment task cancelled", workflow_id=workflow_id)
if workflow_id:
await broadcaster.unlock_workflow(workflow_id)
if result.get("success"):
for node_id in result.get("cancelled_listener_node_ids", []):
await broadcaster.clear_node_status(node_id)
# Terminal UI-facing state is emitted UNCONDITIONALLY: cancelling a
# workflow that is not locally deployed (post-restart before re-arm,
# or a Reset after a crash) previously emitted nothing, leaving the
# FE showing executing/running forever. The idempotent terminal
# broadcast is harmless when nothing was running.
await broadcaster.update_workflow_status(
executing=False,
current_node=None,
progress=0,
workflow_id=workflow_id,
)
await broadcaster.update_deployment_status(
is_running=False,
status="cancelled",
active_runs=0,
workflow_id=workflow_id,
data={
"iterations_completed": result.get("iterations_completed", 0),
},
)
return {
"success": result.get("success", False),
"message": result.get("message", result.get("error")),
"workflow_id": workflow_id,
"was_running": result.get("was_running", False),
"iterations_completed": result.get("iterations_completed", 0),
"cancelled_waiters": cancelled_waiters,
"unlocked": workflow_id is not None,
"timestamp": time.time(),
}
@ws_handler()
async def handle_get_deployment_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Get current deployment status including workflow lock info."""
from core.container import container
from services.status_broadcaster import get_status_broadcaster
workflow_service = container.workflow_service()
broadcaster = get_status_broadcaster()
workflow_id = data.get("workflow_id")
status = workflow_service.get_deployment_status(workflow_id)
return {
"is_running": workflow_service.is_deployment_running(workflow_id),
"run_counter": status.get("run_counter", 0),
"active_runs": status.get("active_runs", 0),
"settings": workflow_service.get_deployment_settings(),
"workflow_id": workflow_id or status.get("workflow_id"),
"deployed_workflows": status.get("deployed_workflows", []),
"lock": broadcaster.get_workflow_lock(),
"timestamp": time.time(),
}
@ws_handler()
async def handle_get_workflow_lock(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Get current workflow lock status."""
from services.status_broadcaster import get_status_broadcaster
broadcaster = get_status_broadcaster()
return {
"lock": broadcaster.get_workflow_lock(),
"timestamp": time.time(),
}
@ws_handler()
async def handle_update_deployment_settings(
data: Dict[str, Any],
websocket: WebSocket,
) -> Dict[str, Any]:
"""Update deployment settings (can be called during active deployment)."""
from core.container import container
from services.status_broadcaster import get_status_broadcaster
workflow_service = container.workflow_service()
broadcaster = get_status_broadcaster()
settings_to_update = {}
if "delay_between_runs" in data:
settings_to_update["delay_between_runs"] = data["delay_between_runs"]
if "stop_on_error" in data:
settings_to_update["stop_on_error"] = data["stop_on_error"]
if "max_iterations" in data:
settings_to_update["max_iterations"] = data["max_iterations"]
updated_settings = await workflow_service.update_deployment_settings(settings_to_update)
status = workflow_service.get_deployment_status()
await broadcaster.broadcast(
{
"type": "deployment_settings_updated",
"settings": updated_settings,
"is_running": workflow_service.is_deployment_running(),
"run_counter": status.get("run_counter", 0),
}
)
return {
"success": True,
"settings": updated_settings,
"is_running": workflow_service.is_deployment_running(),
"run_counter": status.get("run_counter", 0),
"active_runs": status.get("active_runs", 0),
"timestamp": time.time(),
}
def _control_service():
from core.container import container
return WorkflowControlService(container.database())
def _versioned_control(control) -> bool:
return int((getattr(control, "resource_manifest", None) or {}).get("execution_control_version", 0)) >= 1
def _control_scope(control) -> Dict[str, Any]:
return {
"execution_control_version": 1 if _versioned_control(control) else 0,
"controller_workflow_id": control.controller_workflow_id,
}
async def _start_controller(control, *, use_existing: bool = False) -> Optional[str]:
"""Start the durable controller, or use local mode when Temporal is disabled.
``use_existing=True`` is the rebuild path (Resume after the controller
was killed): per the documented WorkflowIdConflictPolicy semantics,
``USE_EXISTING`` adopts a still-running controller (returning its run
id) instead of erroring, while the default reuse policy permits a
fresh run when the previous execution was Terminated/Failed — so the
rebuild is race-safe against mis-detection.
"""
from core.container import container
from temporalio.common import (
SearchAttributeKey,
SearchAttributePair,
TypedSearchAttributes,
WorkflowIDConflictPolicy,
)
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
if container.settings().temporal_enabled:
raise RuntimeError("temporal_control_unavailable")
return None
start_kwargs: Dict[str, Any] = {}
if use_existing:
start_kwargs["id_conflict_policy"] = WorkflowIDConflictPolicy.USE_EXISTING
handle = await wrapper.client.start_workflow(
"WorkflowControlWorkflow",
args=[{
"workflow_id": control.workflow_id,
"generation": control.generation,
"execution_id": control.execution_id,
"root_execution_id": control.root_execution_id,
"data_scope_id": control.data_scope_id or control.execution_id,
"state": "running",
"revision": control.revision,
**_control_scope(control),
}],
id=control.controller_workflow_id,
task_queue=container.settings().temporal_task_queue,
search_attributes=TypedSearchAttributes([
SearchAttributePair(
SearchAttributeKey.for_keyword("EventWorkflowId"), control.workflow_id,
)
]),
**start_kwargs,
)
return getattr(handle, "result_run_id", None) or getattr(handle, "first_execution_run_id", None)
def _controller_handle(control):
"""Handle addressed by workflow id only — never pinned to a run id.
The controller keeps its event history bounded via continue-as-new,
which mints a new run id under the same workflow id. A handle pinned
to ``control.controller_run_id`` (the FIRST run) would target a
closed run after the first rollover and every pause/resume/update/
query would fail. The workflow id already carries the generation
(``workflow-control-<wf>-g<N>``), so unpinned addressing cannot
reach a different generation's controller.
"""
from core.container import container
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None or not control.controller_workflow_id:
return None
return wrapper.client.get_workflow_handle(control.controller_workflow_id)
async def _signal_controller(control, signal_name: str) -> None:
handle = _controller_handle(control)
if handle is not None:
await handle.signal(signal_name)
async def _update_controller_state(
control,
requested_state: str,
*,
update_id: str,
) -> Optional[Dict[str, Any]]:
"""Apply and await a durable controller state change.
Temporal-disabled installations retain the local deployment path. When
Temporal is enabled, losing its client is a failed mutation rather than a
false-positive pause/resume acknowledgement.
"""
from core.container import container
handle = _controller_handle(control)
if handle is None:
if container.settings().temporal_enabled:
raise TemporalControlUnavailable("temporal_control_unavailable")
return None
result = await handle.execute_update(
"set_control_state",
requested_state,
id=update_id,
)
expected_state = "paused" if requested_state in {"pause", "paused"} else "running"
if not isinstance(result, dict) or result.get("state") != expected_state:
raise TemporalControlAckMismatch(
f"temporal_control_ack_mismatch:{result.get('state') if isinstance(result, dict) else 'missing'}"
)
return result
async def _query_controller_state(control) -> Optional[Dict[str, Any]]:
"""Return controller state when reachable; status reads remain resilient.
Raises :class:`ControllerExecutionMissing` when Temporal reports the
execution does not exist, rather than folding that into the ``None``
that means "could not reach it". Callers need to tell the two apart:
one resolves itself, the other never will.
"""
handle = _controller_handle(control)
if handle is None:
return None
try:
result = await asyncio.wait_for(handle.query("status"), timeout=_CONTROL_OBSERVATION_TIMEOUT_SECONDS)
return result if isinstance(result, dict) else None
except Exception as exc:
if _temporal_target_already_gone(exc):
# Expected and unremarkable: a controller closes when its
# generation ends, and Temporal deletes closed executions once
# the namespace retention window passes. Not a warning.
logger.debug(
"Workflow controller execution no longer exists",
workflow_id=control.workflow_id,
controller_workflow_id=control.controller_workflow_id,
status=control.status,
)
raise ControllerExecutionMissing(
str(control.controller_workflow_id)
) from exc
logger.warning(
"Workflow controller status query failed",
workflow_id=control.workflow_id,
controller_workflow_id=control.controller_workflow_id,
error=str(exc),
)
return None
# A missing controller is only actionable for a generation the database still
# believes is alive. ``resetting`` is excluded deliberately: it already has an
# explicit retry path through Reset, and auto-failing it here could race a
# reset running concurrently in another request.
_MISSING_CONTROLLER_FAILS = frozenset(
{"starting", "running", "pausing", "paused", "resuming"}
)
# The subset of live states that converge to ``paused`` (user-resumable)
# under WORKFLOW_CONTROL_MISSING_CONTROLLER=pause. ``starting`` always
# fails instead — nothing durable is running yet, and Reset + Start
# rebuilds a fresh generation cleanly.
_MISSING_CONTROLLER_PAUSES = frozenset({"running", "pausing", "paused", "resuming"})
async def _fail_missing_controller(service: WorkflowControlService, control):
"""Converge a generation whose controller execution has vanished.
Without this the row keeps whatever live status it had, and because
``begin_generation`` refuses to open a new generation unless the latest
one is ``reset``, the workflow could never be started again.
Under the default ``WORKFLOW_CONTROL_MISSING_CONTROLLER=pause``, live
states converge to ``paused``: a killed workflow (terminated in the
Temporal UI, crashed server, retention-deleted execution) stays
user-recoverable — Resume rebuilds the controller from the durable
row + graph snapshot — whereas ``failed`` forces a Reset that
archives conversation state. ``fail`` preserves the legacy
Reset-only behaviour; ``starting`` rows always fail (nothing durable
is running yet).
"""
if control.status not in _MISSING_CONTROLLER_FAILS:
return control
if _versioned_control(control):
_close_local_admission(control.workflow_id)
await _set_cron_pause(control.workflow_id, paused=True, strict=True)
if (
not _versioned_control(control)
and
control.status in _MISSING_CONTROLLER_PAUSES
and _missing_controller_policy() == "pause"
):
if control.status == "paused":
# Already user-resumable; Resume performs the rebuild.
return control
logger.warning(
"Workflow controller execution is gone; pausing the generation so "
"the user can resume it (Resume rebuilds the controller)",
workflow_id=control.workflow_id,
controller_workflow_id=control.controller_workflow_id,
status=control.status,
generation=control.generation,
)
try:
recovered = await service.transition(
control,
expected_revision=control.revision,
from_statuses={control.status},
status="paused",
values={
"pause_reason": PAUSE_REASON_CONTROLLER_MISSING,
"pause_detail": "Paused because its background process stopped. Resume to restart it.",
},
)
except ValueError:
# Lost the CAS to a concurrent writer; their transition wins.
return control
await _broadcast_control(recovered, extra={"recovery": "controller_missing"})
return recovered
logger.warning(
"Workflow controller execution is gone; failing the generation so it "
"can be reset",
workflow_id=control.workflow_id,
controller_workflow_id=control.controller_workflow_id,
status=control.status,
generation=control.generation,
)
try:
failed = await service.fail(control, "controller_execution_missing")
except ValueError:
# Lost the CAS to a concurrent writer, which means someone else has
# already moved this row. Their transition wins; report what we have.
return control
await _broadcast_control(failed)
return failed
def _generation_visibility_query(control) -> str:
"""Visibility query for controller descendants and standalone trigger roots."""
if control.controller_workflow_id:
root_id = str(control.controller_workflow_id).replace("'", "''")
workflow_id = str(control.workflow_id).replace("'", "''")
return (
f"(RootWorkflowId='{root_id}' OR "
f"EventWorkflowId='{workflow_id}') "
"AND ExecutionStatus='Running'"
)
workflow_id = str(control.workflow_id).replace("'", "''")
return f"EventWorkflowId='{workflow_id}' AND ExecutionStatus='Running'"
def _visibility_literal(value: Any) -> str:
return str(value).replace("'", "''")
async def _list_generation_workflows(client, control) -> list[Any]:
"""Resolve tagged roots and every running descendant in those trees.
Temporal Search Attributes are not inherited by child workflows. The
first query therefore discovers the controller tree plus tagged standalone
trigger/graph roots; a second batched RootWorkflowId query expands those
roots to active Agent/DelegatedTask descendants.
"""
targets: Dict[tuple[str, str], Any] = {}
root_ids: set[str] = set()
async for execution in client.list_workflows(
query=_generation_visibility_query(control)
):
execution_id = str(execution.id)
run_id = str(getattr(execution, "run_id", "") or "")
targets[(execution_id, run_id)] = execution
root_ids.add(
str(getattr(execution, "root_id", None) or execution_id)
)
ordered_roots = sorted(root_ids)
batch_size = 40
for offset in range(0, len(ordered_roots), batch_size):
batch = ordered_roots[offset : offset + batch_size]
root_values = ", ".join(
f"'{_visibility_literal(root_id)}'"
for root_id in batch
)
query = (
f"RootWorkflowId IN ({root_values}) "
"AND ExecutionStatus='Running'"
)
async for execution in client.list_workflows(query=query):
execution_id = str(execution.id)
run_id = str(getattr(execution, "run_id", "") or "")
targets[(execution_id, run_id)] = execution
return list(targets.values())
def _temporal_target_already_gone(exc: Exception) -> bool:
try:
from temporalio.service import RPCError, RPCStatusCode
if isinstance(exc, RPCError) and exc.status == RPCStatusCode.NOT_FOUND:
return True
except Exception:
pass
message = str(exc).lower()
return "not found" in message or "already completed" in message
async def _signal_generation_workflows(
control,
signal_name: str,
*,
strict: bool = False,
) -> int:
"""Best-effort cooperative fan-out to this deployment's live executions.
Visibility is discovery only; durable control state remains authoritative.
Every matching workflow receives an idempotent pause/resume flag mutation.
"""
from core.container import container
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
if strict and container.settings().temporal_enabled:
raise TemporalControlUnavailable("temporal_control_unavailable")
return 0
signalled = 0
failures: list[Exception] = []
try:
executions = await _list_generation_workflows(
wrapper.client,
control,
)
for execution in executions:
try:
await wrapper.client.get_workflow_handle(
execution.id, run_id=execution.run_id
).signal(signal_name)
signalled += 1
except Exception as exc:
if not _temporal_target_already_gone(exc):
failures.append(exc)
logger.warning(
"Workflow control signal failed",
workflow_id=control.workflow_id,
temporal_workflow_id=execution.id,
signal=signal_name,
error=str(exc),
)
except Exception as exc:
logger.warning(
"Workflow control visibility fan-out failed",
workflow_id=control.workflow_id,
signal=signal_name,
error=str(exc),
)
if strict:
raise RuntimeError(
"workflow_signal_visibility_failed"
) from exc
if strict and failures:
raise RuntimeError(
f"workflow_signal_failed:{len(failures)}"
) from failures[0]
return signalled
async def _terminate_generation_workflows(control, *, strict: bool = False) -> int:
"""Immediately terminate every visible execution in one application tree."""
from core.container import container
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
if strict and container.settings().temporal_enabled:
raise TemporalControlUnavailable("temporal_control_unavailable")
return 0
terminated = 0
failures: list[Exception] = []
try:
executions = await _list_generation_workflows(
wrapper.client,
control,
)
for execution in executions:
try:
await wrapper.client.get_workflow_handle(
execution.id, run_id=execution.run_id
).terminate(reason="workflow_reset")
terminated += 1
except Exception as exc:
if not _temporal_target_already_gone(exc):
failures.append(exc)
logger.warning(
"Workflow reset termination failed",
workflow_id=control.workflow_id,
temporal_workflow_id=execution.id,
error=str(exc),
)
except Exception as exc:
logger.warning(
"Workflow reset visibility scan failed",
workflow_id=control.workflow_id,
error=str(exc),
)
if strict:
raise RuntimeError("workflow_visibility_cleanup_failed") from exc
if strict and failures:
raise RuntimeError(
f"workflow_termination_failed:{len(failures)}"
) from failures[0]
return terminated
def _expected_revision(data: Dict[str, Any], control) -> int:
supplied = data.get("expected_revision")
if supplied is None:
raise ValueError("expected_revision_required")
return int(supplied)
_AUTOMATIC_PAUSE_REASONS = frozenset({PAUSE_REASON_FAILURES, PAUSE_REASON_RECOVERY, PAUSE_REASON_CONTROLLER_MISSING})
def _automatic_pause_values(data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""``pause_reason`` / ``pause_detail`` for a pause the server started on
its own (``_pause_reason`` in a server-side call), or None."""
reason = data.get("_pause_reason")
if reason not in _AUTOMATIC_PAUSE_REASONS:
return None
detail = str(data.get("_pause_detail") or "").strip()[:500] or None
return {"pause_reason": reason, "pause_detail": detail}
def _clear_pause_reason_kwargs(control) -> Dict[str, Any]:
"""Transition kwargs that clear an automatic pause's reason, when the
row has one (running again means the reason no longer applies)."""
if getattr(control, "pause_reason", None) or getattr(control, "pause_detail", None):
return {"values": dict(CLEAR_PAUSE_REASON)}
return {}
def _failure_pause_detail(reason: str) -> str:
"""What the owner reads on a deployment the circuit breaker paused."""
error = " ".join(str(reason or "").split())[:200]
base = "Paused after repeated errors. Resume when it's fixed."
return f"{base} Last error: {error}" if error else base
async def _set_cron_pause(
workflow_id: str,
*,
paused: bool,
strict: bool = False,
) -> int:
from core.container import container
from services.temporal.schedules import set_cron_schedules_paused
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
if strict and container.settings().temporal_enabled:
raise TemporalControlUnavailable("temporal_control_unavailable")
return 0
return await set_cron_schedules_paused(
wrapper.client,
workflow_id,
paused=paused,
strict=strict,
)
async def _delete_cron_schedules(
workflow_id: str,
*,
strict: bool = False,
) -> int:
from core.container import container
from services.temporal.schedules import (
delete_cron_schedules_for_deployment,
)
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
if strict and container.settings().temporal_enabled:
raise TemporalControlUnavailable("temporal_control_unavailable")
return 0
return await delete_cron_schedules_for_deployment(
wrapper.client,
workflow_id,
strict=strict,
)
async def _with_runtime_counts(payload: Dict[str, Any], workflow_id: str) -> Dict[str, Any]:
from core.container import container
status = container.workflow_service().get_deployment_status(workflow_id)
workspace = await _workspace_runtime_status(workflow_id)
active = payload.get("generation_in_flight_count", status.get("active_runs", 0))
return {
**payload,
**{key: value for key, value in workspace.items() if key != "workspace_active_count"},
"active_count": active + workspace.get("workspace_active_count", 0),
"in_flight_count": active + workspace.get("workspace_active_count", 0),
"queued_count": (
int(payload.get("queued_count", 0) or 0)
+ int(status.get("queued_events", 0) or 0)
),
}
async def _workspace_runtime_status(workflow_id: str) -> Dict[str, Any]:
from core.container import container
from services.node_invocations import controller_status, workspace_task_nodes
database = container.database()
_, nodes = await workspace_task_nodes(database, workflow_id)
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
return {"can_reset": True, "workspace_available": False} if nodes else {}
try:
state = await asyncio.wait_for(controller_status(workflow_id, client=wrapper.client),
timeout=_CONTROL_OBSERVATION_TIMEOUT_SECONDS)
except Exception as exc:
logger.warning("Workspace task status unavailable", workflow_id=workflow_id, error_type=type(exc).__name__)
return {"can_reset": True, "workspace_available": False} if nodes else {}
if state is None:
return {"can_reset": True, "workspace_epoch": 0} if nodes else {}
result = {
"can_reset": True, "workspace_available": True,
"workspace_active_count": state["active_count"], "workspace_epoch": state["epoch"],
"workspace_resetting": state["resetting"],
"workspace_reset_request_id": state.get("reset_request_id") or state.get("last_reset_request_id"),
}
if state["resetting"]:
result.update(state="resetting", can_start=False, can_pause=False, can_resume=False, can_edit=False)
return result
async def _reset_workspace_tasks(workflow_id: str, database, control, data: Dict[str, Any]) -> Dict[str, Any]:
from services.node_invocations import controller_status, reset, workspace_task_nodes
from core.container import container
_, nodes = await workspace_task_nodes(database, workflow_id, control)
wrapper = container.temporal_client()
state = None
if wrapper is not None and wrapper.client is not None:
state = await controller_status(workflow_id, client=wrapper.client)
if not nodes and state is None:
return {"cancelled": 0, "reset_nodes": [], "present": False}
# Phone-only resets do not advance the graph revision. Deriving an ID
# from that revision would make every later Reset replay the first one.
request_id = str(data.get("idempotency_key") or f"workflow-reset:{uuid.uuid4().hex}")
return {**await reset(workflow_id, request_id), "present": True}
def _close_local_admission(workflow_id: str) -> None:
"""Synchronously gate legacy trigger callbacks before durable cleanup."""
from core.container import container
container.workflow_service().pause_deployment(workflow_id)
async def _control_payload(
control,
*,
extra: Optional[Dict[str, Any]] = None,
controller_status: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
payload = serialize_control(control)
if controller_status is not None:
payload.update({
"temporal_state": controller_status.get("state"),
"temporal_revision": controller_status.get("revision"),
"queued_count": controller_status.get("queued_events", 0),
"temporal_available": True,
})
if _versioned_control(control):
from core.container import container
from services.deployment.execution_control import count_generation_actions
wrapper = container.temporal_client()
if wrapper is not None and wrapper.client is not None:
try:
payload["generation_in_flight_count"] = await asyncio.wait_for(count_generation_actions(
wrapper.client, control.controller_workflow_id, controller_status),
timeout=_CONTROL_OBSERVATION_TIMEOUT_SECONDS)
except Exception as exc:
logger.debug("Generation activity count unavailable", workflow_id=control.workflow_id,
error=str(exc))
payload.update(extra or {})
return await _with_runtime_counts(payload, control.workflow_id)
async def _broadcast_control(
control,
*,
extra: Optional[Dict[str, Any]] = None,
controller_status: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
from services.status_broadcaster import get_status_broadcaster
payload = await _control_payload(
control,
extra=extra,
controller_status=controller_status,
)
await get_status_broadcaster().broadcast({
"type": "workflow_control_status",
"workflow_id": control.workflow_id,
"data": payload,
})
from services.deployment.control import notify_control_changed
notify_control_changed(control.workflow_id)
return payload
async def _reconcile_control(service: WorkflowControlService, control):
"""Finish an interrupted DB transition from acknowledged Temporal state."""
from core.container import container
# A generation that has been reset (or failed) closed its controller on
# purpose, and Temporal deletes closed executions once the namespace
# retention window passes. Probing one is therefore a guaranteed-failing
# RPC on every single status read, forever. Terminal generations are
# answered from the database, which is authoritative for them anyway.
if control.status not in ACTIVE_STATES:
return control, None
if _versioned_control(control) and control.status in {"pausing", "resuming"}:
# A blocked admitted action can keep a checkpoint Update open for
# minutes. Status is a snapshot, never a waiter on that Update.
_schedule_versioned_transition(service, control)
return control, None
try:
controller_status = await _query_controller_state(control)
except ControllerExecutionMissing:
return await _fail_missing_controller(service, control), None
transition_target = {
"pausing": ("paused", "paused"),
"resuming": ("running", "running"),
}.get(control.status)
if transition_target is None:
return control, controller_status
requested_state, stable_state = transition_target
if controller_status is None:
if container.settings().temporal_enabled:
return control, None
if control.status == "pausing":
container.workflow_service().pause_deployment(control.workflow_id)
if controller_status is not None and controller_status.get("state") != stable_state:
controller_status = await _update_controller_state(
control,
requested_state,
update_id=(
f"reconcile:{control.id}:{control.revision}:{stable_state}"
),
)
if control.status == "pausing":
paused_schedules = await _set_cron_pause(
control.workflow_id,
paused=True,
strict=True,
)
paused_triggers = await container.workflow_service().update_trigger_pause_status(
control.workflow_id,
paused=True,
)
signalled = await _signal_generation_workflows(
control,
"pause",
strict=True,
)
transition_details = {
"signalled_executions": signalled,
"paused_schedules": paused_schedules,
"paused_triggers": paused_triggers,
}
else:
resumed_schedules = await _set_cron_pause(
control.workflow_id,
paused=False,
strict=True,
)
signalled = await _signal_generation_workflows(
control,
"resume",
strict=True,
)
queued = await container.workflow_service().resume_deployment(
control.workflow_id,
)
resumed_triggers = await container.workflow_service().update_trigger_pause_status(
control.workflow_id,
paused=False,
)
transition_details = {
"resumed_queued_events": queued,
"signalled_executions": signalled,
"resumed_schedules": resumed_schedules,
"resumed_triggers": resumed_triggers,
}
try:
control = await service.transition(
control,
expected_revision=control.revision,
from_statuses={control.status},
status=stable_state,
**(_clear_pause_reason_kwargs(control) if stable_state == "running" else {}),
)
await _broadcast_control(
control,
controller_status=controller_status,
extra=transition_details,
)
except ValueError as exc:
if str(exc) != "control_revision_conflict":
raise
latest = await service.database.get_latest_workflow_control(control.workflow_id)
if latest is not None and latest.generation == control.generation:
control = latest
return control, controller_status
async def _finish_versioned_transition(service: WorkflowControlService, control):
"""Drain the admitted work while allowing a newer Reset to take over."""
from core.container import container
from temporalio.client import WorkflowUpdateRPCTimeoutOrCancelledError
from services.deployment.execution_control import GenerationTransitionSuperseded, transition_generation
intent = (control.id, control.generation, control.revision, control.status)
paused = intent[3] == "pausing"
async def latest_control():
latest = await service.database.get_latest_workflow_control(control.workflow_id)
if latest is None:
raise RuntimeError("workflow_control_missing")
return latest
def owns(latest) -> bool:
return (latest.id, latest.generation, latest.revision, latest.status) == intent
async def ensure_current() -> None:
if not owns(await latest_control()):
raise GenerationTransitionSuperseded()
try:
runtime = container.workflow_service()
# Reset shares this short completion lock. It never waits for the
# activity drain; an older transition cannot gate or reopen a later
# generation's producers after Reset has claimed the durable row.
async with _control_completion_lock(control.workflow_id):
await ensure_current()
runtime.pause_deployment(control.workflow_id)
if paused:
schedules = await _set_cron_pause(control.workflow_id, paused=True, strict=True)
await ensure_current()
triggers = await runtime.update_trigger_pause_status(control.workflow_id, paused=True)
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
if container.settings().temporal_enabled:
raise TemporalControlUnavailable("temporal_control_unavailable")
result = {"controller_status": None, "controlled_executions": 0}
else:
async def admitted(status):
async with _control_completion_lock(control.workflow_id):
await ensure_current()
try:
await _broadcast_control(control, controller_status=status)
except Exception as exc:
logger.warning("Control admission status broadcast failed", workflow_id=control.workflow_id,
error=str(exc))
result = await transition_generation(wrapper.client, control, paused=paused,
on_admission=admitted, ensure_current=ensure_current)
async with _control_completion_lock(control.workflow_id):
await ensure_current()
details = {"controlled_executions": result["controlled_executions"]}
values: Dict[str, Any] = {}
if paused:
details.update(paused_schedules=schedules, paused_triggers=triggers)
else:
# Existing continuations are released before producers, and
# Reset cannot interleave with these producer mutations.
details["resumed_schedules"] = await _set_cron_pause(control.workflow_id, paused=False, strict=True)
await ensure_current()
details["resumed_queued_events"] = await runtime.resume_deployment(control.workflow_id)
await ensure_current()
details["resumed_triggers"] = await runtime.update_trigger_pause_status(control.workflow_id, paused=False)
await ensure_current()
values.update(CLEAR_PAUSE_REASON)
values["resource_manifest"] = {**(control.resource_manifest or {}),
"last_resumed_at": datetime.now(timezone.utc).isoformat()}
control = await service.transition(control, expected_revision=intent[2],
from_statuses={intent[3]}, status="paused" if paused else "running", values=values or None)
if not paused:
await _clear_failure_streak(service.database, control)
payload = await _broadcast_control(control, controller_status=result["controller_status"], extra=details)
from services.status_broadcaster import get_status_broadcaster
broadcaster = get_status_broadcaster()
await broadcaster.update_workflow_status(executing=bool(not paused and payload.get("in_flight_count")),
current_node=None, progress=0, workflow_id=control.workflow_id)
await broadcaster.update_deployment_status(is_running=True, status=control.status,
active_runs=payload.get("generation_in_flight_count", 0), workflow_id=control.workflow_id)
return control, result["controller_status"]
except (Exception, asyncio.CancelledError) as exc:
latest = await latest_control()
if not owns(latest):
# Reset may terminate the execution or cancel this local poll
# before an Update produces an outcome. That is supersession,
# never a failed Reset or a successful pause acknowledgement.
logger.debug("Workflow control transition superseded", workflow_id=control.workflow_id,
revision=intent[2], current_revision=latest.revision, state=latest.status)
return latest, None
if isinstance(exc, WorkflowUpdateRPCTimeoutOrCancelledError):
if isinstance(exc.__cause__, asyncio.CancelledError):
raise asyncio.CancelledError() from exc
# The SDK ended its wait without rejecting the durable Update (its
# own RPC deadline). Keep the intent and the closed admission; a
# later status read drives it again.
logger.info("Workflow control acknowledgement remains pending", workflow_id=control.workflow_id,
revision=intent[2], error_type=type(exc).__name__)
return latest, None
if str(exc) == "controller_execution_missing":
async with _control_completion_lock(control.workflow_id):
latest = await latest_control()
if not owns(latest):
return latest, None
return await _fail_missing_controller(service, latest), None
raise
# ---------------------------------------------------------------------------
# Boot-time reconcile — called once from services.temporal.lifecycle after
# the workers start. Request-time reconciliation stays lazy; this pass closes
# the unattended-server window where durable intent (running/paused rows) and
# runtime behaviour could diverge indefinitely after a backend restart.
# ---------------------------------------------------------------------------
# Dirty-bit marker for crash detection. Boot stamps "running"; the graceful
# lifespan teardown stamps "clean" (registered as a shutdown hook at the
# bottom of this module). A boot that finds "running" already stamped knows
# the previous process was killed or crashed mid-flight.
_SHUTDOWN_STATE_CACHE_KEY = "workflow_control:shutdown_state"
def _crash_recovery_policy() -> str:
"""Normalized WORKFLOW_CONTROL_CRASH_RECOVERY: ``pause`` | ``resume``."""
from core.container import container
value = str(
getattr(container.settings(), "workflow_control_crash_recovery", "pause") or "pause"
).strip().lower()
if value not in {"pause", "resume"}:
logger.warning(f"Unknown WORKFLOW_CONTROL_CRASH_RECOVERY={value!r}; using 'pause'")
return "pause"
return value
def _missing_controller_policy() -> str:
"""Normalized WORKFLOW_CONTROL_MISSING_CONTROLLER: ``pause`` | ``fail``."""
from core.container import container
value = str(
getattr(container.settings(), "workflow_control_missing_controller", "pause") or "pause"
).strip().lower()
if value not in {"pause", "fail"}:
logger.warning(f"Unknown WORKFLOW_CONTROL_MISSING_CONTROLLER={value!r}; using 'pause'")
return "pause"
return value
async def _consume_shutdown_state() -> bool:
"""True when the previous backend process exited cleanly.
A missing marker (first boot / pre-feature database) counts as clean
so a fresh install never boots into a recovery pause.
``Database.get_cache_entry`` returns the stored string directly.
"""
from core.container import container
database = container.database()
previous = await database.get_cache_entry(_SHUTDOWN_STATE_CACHE_KEY)
await database.set_cache_entry(_SHUTDOWN_STATE_CACHE_KEY, "running")
return previous is None or previous == "clean"
async def mark_clean_shutdown() -> None:
"""Stamp the clean-shutdown marker (lifespan teardown hook)."""
from core.container import container
tasks = [*_control_transition_tasks.values(), *_boot_reconciliation_tasks.values()]
for task in tasks:
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
try:
await container.database().set_cache_entry(_SHUTDOWN_STATE_CACHE_KEY, "clean")
except Exception as exc: # noqa: BLE001 — best-effort during teardown
logger.warning(f"Failed to record clean shutdown marker: {exc}")
async def _pause_for_recovery(service: WorkflowControlService, control, *, reason: str):
"""Durably pause a recovered generation so the user resumes it.
After a kill or crash the deployment must come back ``paused`` rather
than silently continuing — the user decides when it is safe to
resume. Reuses the full pause ceremony (controller update, cron
schedule pause, trigger flags, local admission) via
:func:`handle_pause_workflow` so recovery lands in exactly the same
posture a user-initiated pause produces. Temporal's native
Pause/Unpause (server 1.28+) is deliberately NOT used here: it is an
operational control without Python SDK client methods, and it halts
workflow-task dispatch entirely — a natively-paused controller could
not process the ``set_control_state`` Update that Resume relies on.
"""
logger.warning(
"Pausing recovered generation after unclean shutdown",
workflow_id=control.workflow_id,
generation=control.generation,
reason=reason,
)
try:
result = await handle_pause_workflow(
{
"workflow_id": control.workflow_id,
"expected_revision": control.revision,
"idempotency_key": f"recovery:{control.id}:{control.revision}",
"_pause_reason": PAUSE_REASON_RECOVERY,
"_pause_detail": "Paused after OpenCompany stopped unexpectedly. Resume when you're ready.",
},
None,
)
if not (isinstance(result, dict) and result.get("success")):
logger.warning(
"Recovery pause did not complete",
workflow_id=control.workflow_id,
error=(result or {}).get("error") if isinstance(result, dict) else None,
)
except Exception as exc: # noqa: BLE001 — lazy reconcile retries later
logger.warning(
"Recovery pause failed",
workflow_id=control.workflow_id,
error=str(exc),
)
latest = await service.database.get_latest_workflow_control(control.workflow_id)
return latest if latest is not None else control
async def reconcile_active_controls_on_boot() -> int:
"""Converge every active durable control after a backend restart.
Responsibilities per active row:
1. Run the same lazy :func:`_reconcile_control` used by status reads
(finishes interrupted pause/resume transitions, converges rows
whose controller vanished).
2. Converge ``starting`` rows — the one state the lazy path never
touches while its controller exists, so a crash mid-start used to
wedge the workflow behind a permanent ``workflow_start_pending``.
Boot is the only moment guaranteed free of a concurrent in-process
start, which is why this lives here and not in the lazy path.
3. Re-arm the process-local half of running/paused generations from
the persisted graph snapshot: the controller and its Temporal
children survive a restart on their own, but DeploymentManager
state (runtime counts, admission, pause fan-in) and in-process
collectors for non-canary trigger types die with the process and
previously stayed dead while the row reported ``running`` forever.
4. After an UNCLEAN shutdown (kill / crash — dirty-bit marker), pause
every generation still ``running`` so the user consciously
resumes it (``WORKFLOW_CONTROL_CRASH_RECOVERY=pause``, the
default; ``resume`` restores them running). A clean
``company stop`` + start always restores deployments as they were.
Returns the number of rows processed; per-row failures are logged and
skipped so one broken generation cannot block the rest.
"""
from core.container import container
database = container.database()
service = WorkflowControlService(database)
try:
clean_shutdown = await _consume_shutdown_state()
except Exception as exc: # noqa: BLE001 — marker is best-effort
logger.warning(f"Shutdown-state marker read failed; assuming clean: {exc}")
clean_shutdown = True
recovery_pause = not clean_shutdown and _crash_recovery_policy() == "pause"
controls = await database.list_active_workflow_controls()
processed = 0
for control in controls:
try:
if _versioned_control(control) and control.status in {"pausing", "resuming"}:
# A checkpoint can remain open on admitted work for months.
# Its eventual completion must still restore local collectors,
# without blocking this boot pass or other generations.
_schedule_boot_reconciliation(service, control, recovery_pause=recovery_pause)
processed += 1
continue
control, controller_status = await _reconcile_control(service, control)
if control.status == "starting":
control = await _converge_interrupted_start(
service, control, controller_status
)
if control.status in {"running", "paused"}:
await _rearm_generation(control)
if recovery_pause and control.status == "running":
control = await _pause_for_recovery(
service, control, reason="unclean_shutdown"
)
processed += 1
except Exception as exc: # noqa: BLE001 — per-row isolation
logger.warning(
"Boot reconcile failed for workflow control",
workflow_id=control.workflow_id,
status=control.status,
error=str(exc),
)
return processed
def _schedule_boot_reconciliation(service: WorkflowControlService, control, *, recovery_pause: bool) -> asyncio.Task:
key = (control.id, int(control.generation))
existing = _boot_reconciliation_tasks.get(key)
if existing is not None and not existing.done():
return existing
task = asyncio.create_task(_restore_after_boot_transition(service, control, recovery_pause=recovery_pause),
name=f"workflow-boot:{control.id}:{control.generation}")
_boot_reconciliation_tasks[key] = task
def completed(done: asyncio.Task) -> None:
if _boot_reconciliation_tasks.get(key) is done:
_boot_reconciliation_tasks.pop(key, None)
if not done.cancelled() and (exc := done.exception()) is not None:
logger.warning("Boot workflow restoration failed", workflow_id=control.workflow_id,
generation=control.generation, error=str(exc))
task.add_done_callback(completed)
return task
async def _restore_after_boot_transition(service: WorkflowControlService, control, *, recovery_pause: bool) -> None:
"""Restore a pending generation's local half after its durable drain.
Transient Update/setup failures retry independently of status requests.
Reset shares only the bounded setup critical section, never the
checkpoint wait. Re-reading the owning generation prevents an old boot
continuation from deploying the next Start's graph or changing its gate.
"""
identity = (control.id, control.generation)
while True:
try:
current = await service.database.get_latest_workflow_control(control.workflow_id)
if (current is None or (current.id, current.generation) != identity
or current.status not in {"pausing", "resuming", "paused", "running"}):
return
if current.status in {"pausing", "resuming"}:
await asyncio.shield(_schedule_versioned_transition(service, current))
latest = await service.database.get_latest_workflow_control(control.workflow_id)
if latest is not None and (latest.id, latest.generation) == identity and latest.status in {"pausing", "resuming"}:
await asyncio.sleep(_BOOT_RETRY_SECONDS)
continue
if recovery_pause and current.status == "running":
# A Resume interrupted by a crash follows the same recovery
# policy as an already-running row. Fence it before rebuilding
# collectors, then let its acknowledged pause settle normally.
await _pause_for_recovery(service, current, reason="unclean_shutdown")
await asyncio.sleep(_BOOT_RETRY_SECONDS)
continue
async with _control_completion_lock(control.workflow_id):
latest = await service.database.get_latest_workflow_control(control.workflow_id)
if (latest is None or (latest.id, latest.generation) != identity
or latest.status not in {"paused", "running"}):
continue
if latest.revision != current.revision or latest.status != current.status:
continue
try:
async def restore_local():
await _rearm_generation(latest)
if latest.status == "running":
# A failed/timed-out attempt kept its local gate
# closed. Release it after setup is known complete.
from core.container import container
await container.workflow_service().resume_deployment(control.workflow_id)
await asyncio.wait_for(restore_local(), timeout=_BOOT_REARM_TIMEOUT_SECONDS)
except Exception:
_close_local_admission(control.workflow_id)
raise
return
except asyncio.CancelledError:
# Reset owns the subsequent local teardown and execution sweep.
# Shutdown likewise leaves the durable intent for the next boot.
raise
except Exception as exc:
logger.warning("Boot workflow restoration remains pending", workflow_id=control.workflow_id,
generation=control.generation, error_type=type(exc).__name__, error=str(exc))
await asyncio.sleep(_BOOT_RETRY_SECONDS)
async def _converge_interrupted_start(
service: WorkflowControlService,
control,
controller_status: Optional[Dict[str, Any]],
):
"""Decide what a crash left behind for a row stuck in ``starting``.
A vanished controller was already failed by ``_reconcile_control``.
With the controller alive: triggers registered (or a triggerless
graph) means the deploy effectively completed — commit ``running``.
A live-but-empty controller for a graph that declares triggers means
the crash landed before registration — fail the generation (Reset +
Start rebuilds cleanly) and close the orphan controller so dispatch
stops signalling it.
"""
if controller_status is None:
# Controller unreachable (not missing) — leave transitional; the
# lazy path retries when Temporal comes back.
return control
from constants import WORKFLOW_TRIGGER_TYPES
triggers_registered = len(controller_status.get("triggers") or {})
graph_nodes = (control.graph_snapshot or {}).get("nodes") or []
graph_has_triggers = any(
node.get("type") in WORKFLOW_TRIGGER_TYPES for node in graph_nodes
)
if triggers_registered or not graph_has_triggers:
try:
control = await service.transition(
control,
expected_revision=control.revision,
from_statuses={"starting"},
status="running",
)
await _broadcast_control(control)
except ValueError:
# Concurrent writer won the CAS; report what the DB has.
latest = await service.database.get_latest_workflow_control(control.workflow_id)
if latest is not None and latest.generation == control.generation:
control = latest
return control
try:
await _signal_controller(control, "reset")
except Exception as exc: # noqa: BLE001 — best-effort close
logger.warning(
"Failed to close orphan controller for interrupted start",
workflow_id=control.workflow_id,
error=str(exc),
)
try:
failed = await service.fail(control, "interrupted_start")
except ValueError:
return control
await _broadcast_control(failed)
return failed
async def _rearm_generation(control) -> None:
"""Re-establish process-local deployment state for a durable generation.
Idempotent by construction: canary triggers re-register with the
controller keyed by listener id (a known listener takes only the
re-sent, sanitized parameter snapshot), legacy
listener starts use ``id_conflict_policy=USE_EXISTING``, and cron
schedule creation preserves server-owned paused state on redeploy.
In-process collectors for non-canary trigger types are re-armed
fresh (they died with the previous process).
"""
from core.container import container
workflow_service = container.workflow_service()
if workflow_service.is_workflow_deployed(control.workflow_id):
if control.workflow_id in _deployment_tasks:
# A previous bounded rearm may still be waiting on its shielded
# setup task. The manager installs local state before collectors
# finish starting, so "deployed" alone is not a setup receipt.
setup = await _await_deployment_setup(control.workflow_id)
if not setup.get("success"):
raise RuntimeError(str(setup.get("error", "deployment_setup_failed")))
if control.status == "paused":
await _restore_rearmed_pause(control)
return
snapshot = control.graph_snapshot or {}
nodes = snapshot.get("nodes") or []
edges = snapshot.get("edges") or []
if not nodes:
logger.warning(
"Cannot re-arm generation without a graph snapshot",
workflow_id=control.workflow_id,
generation=control.generation,
)
return
deploy_data = {
"workflow_id": control.workflow_id,
"nodes": nodes,
"edges": edges,
"parameters_by_id": snapshot.get("parameters") or {},
"generation": control.generation,
**_control_scope(control),
# Snapshots created before the Context topology deliberately retain version 0,
# so a process restart cannot mutate their Temporal command sequence.
"graphVersion": int(
snapshot.get("graphVersion")
or snapshot.get("graph_version")
or 0
),
# Runtime persistence is generation-scoped (same contract as
# handle_start_workflow's deploy call).
"session_id": control.data_scope_id or control.execution_id,
"execution_id": control.execution_id,
"root_execution_id": control.root_execution_id,
"user_id": str(snapshot.get("owner_id") or "owner"),
}
deployed = await handle_deploy_workflow(deploy_data, None)
if not deployed.get("success"):
raise RuntimeError(str(deployed.get("error", "deployment_failed")))
setup = await _await_deployment_setup(control.workflow_id)
if not setup.get("success"):
raise RuntimeError(str(setup.get("error", "deployment_setup_failed")))
if control.status == "paused":
await _restore_rearmed_pause(control)
logger.info(
"Re-armed durable generation after restart",
workflow_id=control.workflow_id,
generation=control.generation,
status=control.status,
)
async def _restore_rearmed_pause(control) -> None:
from core.container import container
from services.status_broadcaster import get_status_broadcaster
workflow_service = container.workflow_service()
workflow_service.pause_deployment(control.workflow_id)
await workflow_service.update_trigger_pause_status(control.workflow_id, paused=True)
await _set_cron_pause(control.workflow_id, paused=True, strict=False)
# Setup emits running status; publish the durable paused posture once its
# receipt arrives, including when an earlier rearm waiter timed out.
broadcaster = get_status_broadcaster()
await broadcaster.update_workflow_status(
executing=False, current_node=None, progress=0, workflow_id=control.workflow_id,
)
await broadcaster.update_deployment_status(
is_running=True, status="paused", active_runs=0, workflow_id=control.workflow_id,
)
async def _await_deployment_setup(workflow_id: str) -> Dict[str, Any]:
"""Wait for trigger/listener setup started by the legacy deploy handler."""
task = _deployment_tasks.get(workflow_id)
if task is None:
return {
"success": False,
"error": "deployment_setup_task_missing",
"workflow_id": workflow_id,
}
result = await asyncio.shield(task)
if isinstance(result, dict):
return result
return {
"success": False,
"error": "deployment_setup_did_not_return_status",
"workflow_id": workflow_id,
}
async def _restore_control_after_failed_update(
service: WorkflowControlService,
control,
*,
transitional_state: str,
stable_state: str,
):
"""Undo the DB projection when Temporal rejected a lifecycle update."""
try:
restored = await service.transition(
control,
expected_revision=control.revision,
from_statuses={transitional_state},
status=stable_state,
# A pause that did not happen leaves no reason behind.
**(_clear_pause_reason_kwargs(control) if stable_state == "running" else {}),
)
except ValueError:
latest = await service.database.get_latest_workflow_control(control.workflow_id)
restored = latest if latest is not None else control
await _broadcast_control(restored)
async def _rebuild_missing_controller(service: WorkflowControlService, control):
"""Start a fresh controller for a live generation whose previous one died.
Resume path for a killed workflow: the generation converged to
``paused`` when its controller vanished (terminated in the Temporal
UI, crashed server, retention-deleted execution); Resume calls this
to rebuild before applying the running state. Same generation-scoped
controller workflow id, fresh run chain (``USE_EXISTING`` conflict
policy makes it race-safe against a mis-detected live controller),
then the local re-arm re-registers canary triggers with the fresh
controller from the persisted graph snapshot.
"""
from core.container import container
if _versioned_control(control):
raise ControllerExecutionMissing("Versioned execution registry cannot be reconstructed; Reset is required")
logger.warning(
"Rebuilding missing workflow controller",
workflow_id=control.workflow_id,
controller_workflow_id=control.controller_workflow_id,
generation=control.generation,
)
run_id = await _start_controller(control, use_existing=True)
if run_id:
control = await service.transition(
control,
expected_revision=control.revision,
from_statuses={control.status},
status=control.status,
values={"controller_run_id": run_id},
)
await service.database.update_workflow_run_data_scope(
control.data_scope_id or control.execution_id,
temporal_run_id=run_id,
)
# Local trigger state may still be armed against the dead controller —
# tear it down so the re-arm registers with the fresh one. The
# "not deployed" error from a cold process is expected and benign.
workflow_service = container.workflow_service()
if workflow_service.is_workflow_deployed(control.workflow_id):
try:
await handle_cancel_deployment({"workflow_id": control.workflow_id}, None)
except Exception as exc: # noqa: BLE001 — re-arm still proceeds
logger.warning(
"Local cancel before controller rebuild re-arm failed",
workflow_id=control.workflow_id,
error=str(exc),
)
await _rearm_generation(control)
await _broadcast_control(control, extra={"recovery": "controller_rebuilt"})
return control
async def _duplicate_start_response(
control,
*,
controller_status: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Report the durable outcome of a retried Start idempotency key."""
payload = await _control_payload(
control,
controller_status=controller_status,
)
if control.status == "starting":
return {
"success": False,
"error": "workflow_start_pending",
"idempotent": True,
**payload,
}
if control.status == "failed":
return {
"success": False,
"error": control.terminal_reason or "workflow_start_failed",
"idempotent": True,
**payload,
}
return {"success": True, "idempotent": True, **payload}
@ws_handler("workflow_id")
async def handle_get_workflow_control_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
service = _control_service()
control = await service.database.get_latest_workflow_control(data["workflow_id"])
if control is None:
return await _with_runtime_counts(
serialize_control(None),
data["workflow_id"],
)
control, controller_status = await _reconcile_control(service, control)
return await _control_payload(
control,
controller_status=controller_status,
extra={
"temporal_available": (
controller_status is not None
if control.controller_run_id
else False
),
},
)
@ws_handler("workflow_id")
async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Create generation one and retain deploy_workflow wire compatibility."""
workflow_id = data["workflow_id"]
# A socket's identity wins; a server-side call (no socket, e.g.
# start_saved_workflow) names the owner in the payload.
owner_id = str(
(getattr(getattr(websocket, "state", None), "user_id", None) if websocket is not None else data.get("user_id"))
or "owner"
)
key = data.get("idempotency_key") or f"start:{workflow_id}:{uuid.uuid4().hex}"
service = _control_service()
existing = await service.database.get_workflow_control_by_idempotency_key(
workflow_id,
key,
)
if existing is not None:
existing, controller_status = await _reconcile_control(service, existing)
return await _duplicate_start_response(
existing,
controller_status=controller_status,
)
latest = await service.database.get_latest_workflow_control(workflow_id)
if data.get("expected_revision") is None:
raise ValueError("expected_revision_required")
expected_revision = int(data["expected_revision"])
if expected_revision != (latest.revision if latest else 0):
# Hire may have started the generation after the caller read its
# summary. Keep the revision fence, but return the authoritative
# snapshot so the client can reconcile this expected conflict.
logger.warning(
"Start rejected for stale control revision",
workflow_id=workflow_id,
expected_revision=expected_revision,
current_revision=latest.revision if latest else 0,
generation=latest.generation if latest else 0,
state=latest.status if latest else "never_started",
)
payload = (
await _control_payload(latest)
if latest is not None
else await _with_runtime_counts(serialize_control(None), workflow_id)
)
return {"success": False, "error": "control_revision_conflict", **payload}
if latest is not None and latest.status != "reset":
raise ValueError("workflow_already_started")
# Admit exactly the normalized, sanitized V2 graph. Doing this after the
# control row was created would leave restart recovery with a legacy
# snapshot/hash even though the live deployment had already migrated.
from services.workflow_context_migration import (
import_legacy_context_receipts,
load_node_parameters,
persist_parameter_aliases,
)
from services.employees.team_runtime import admit_team_plan
from services.workflow_migrations import normalize_workflow_graph
from services.workflow_sanitizer import sanitize_workflow_graph
raw_nodes = list(data.get("nodes") or [])
raw_edges = list(data.get("edges") or [])
parameters_by_id = data.get("parameters_by_id")
if not isinstance(parameters_by_id, dict):
parameters_by_id = await load_node_parameters(
service.database,
raw_nodes,
)
normalization = normalize_workflow_graph(
workflow_id,
raw_nodes,
raw_edges,
parameters_by_id,
)
safe_graph = sanitize_workflow_graph(normalization.graph_data())
# A hired team's plan is saved beside the graph, not in it.
admitted_nodes = await admit_team_plan(service.database, workflow_id, list(safe_graph["nodes"]))
admitted_edges = list(safe_graph["edges"])
from services.workflow_validator import validate_workflow
validation = await validate_workflow(
nodes=admitted_nodes,
edges=admitted_edges,
parameters_by_id=normalization.node_parameters,
)
if validation["errors"]:
return {
"success": False,
"error": "validation_failed",
"report": validation,
"graph": safe_graph,
"aliases": normalization.aliases,
"migration_warnings": normalization.warnings,
}
await import_legacy_context_receipts(
service.database,
normalization.state_imports,
)
await persist_parameter_aliases(
service.database,
aliases=normalization.aliases,
parameters=normalization.node_parameters,
)
control, created = await service.begin_generation(
workflow_id=workflow_id,
nodes=admitted_nodes,
edges=admitted_edges,
session_id=data.get("session_id", "default"),
idempotency_key=key,
graph_version=normalization.graph_version,
owner_id=owner_id,
parameters=normalization.node_parameters,
)
if not created:
control, controller_status = await _reconcile_control(service, control)
return await _duplicate_start_response(
control,
controller_status=controller_status,
)
try:
await _broadcast_control(control)
run_id = await _start_controller(control)
if run_id:
control = await service.transition(
control, expected_revision=control.revision, from_statuses={"starting"}, status="starting",
values={"controller_run_id": run_id},
)
await service.database.update_workflow_run_data_scope(
control.data_scope_id or control.execution_id, temporal_run_id=run_id,
)
await _broadcast_control(control)
# Runtime persistence is generation-scoped. The caller's session is
# retained on the scope for provenance, but must never namespace node
# outputs for a controlled run.
deploy_data = {
**data,
"nodes": admitted_nodes,
"edges": admitted_edges,
"parameters_by_id": normalization.node_parameters,
"graphVersion": normalization.graph_version,
"generation": control.generation,
**_control_scope(control),
"session_id": control.data_scope_id or control.execution_id,
"execution_id": control.execution_id,
"root_execution_id": control.root_execution_id,
"user_id": owner_id,
}
deployed = await handle_deploy_workflow(deploy_data, websocket)
if not deployed.get("success"):
raise RuntimeError(str(deployed.get("error", "deployment_failed")))
deployed = await _await_deployment_setup(workflow_id)
if not deployed.get("success"):
raise RuntimeError(str(deployed.get("error", "deployment_failed")))
control = await service.transition(control, expected_revision=control.revision, from_statuses={"starting"}, status="running")
except Exception as exc:
latest = await service.database.get_latest_workflow_control(workflow_id)
if latest is not None and latest.generation == control.generation:
control = latest
if control.status == "starting":
control = await service.fail(control, str(exc))
try:
await _signal_controller(control, "reset")
except Exception as reset_exc:
logger.warning(
"Failed to reset controller after deployment setup error",
workflow_id=workflow_id,
error=str(reset_exc),
)
await _terminate_generation_workflows(control)
# Tear down the local half too: the deploy may have armed
# listeners and taken the workflow lock, and leaving them meant a
# locked, armed, invisible deployment that blocked every future
# Start AND the boot re-arm for this workflow. Cancel unlocks and
# emits the terminal UI-facing state; "not deployed" is benign.
try:
await handle_cancel_deployment({"workflow_id": workflow_id}, None)
except Exception as cancel_exc: # noqa: BLE001 — best-effort teardown
logger.warning(
"Local teardown after failed start did not complete",
workflow_id=workflow_id,
error=str(cancel_exc),
)
await _broadcast_control(control)
raise
# Reaching ``running`` commits successful deployment setup. Projection
# failures after that point must not tear down a live durable generation;
# the client can recover the committed state through its status resync.
try:
controller_status = await _query_controller_state(control)
except ControllerExecutionMissing:
# Same rule as the comment above: this is a projection read taken
# after the generation committed. Reporting no controller state is
# acceptable; propagating and unwinding a live generation is not.
controller_status = None
payload = await _broadcast_control(
control,
controller_status=controller_status,
)
return {
"success": True,
**payload,
"graph": {
"graphVersion": normalization.graph_version,
"nodes": admitted_nodes,
"edges": admitted_edges,
},
"aliases": normalization.aliases,
"migration_warnings": normalization.warnings,
}
async def start_saved_workflow(
workflow_id: str,
*,
owner_id: str,
expected_revision: Optional[int] = None,
idempotency_key: Optional[str] = None,
reset_if_failed: bool = False,
) -> Dict[str, Any]:
"""Start a saved workflow from the server, without the editor.
Loads the saved graph and goes through ``handle_start_workflow`` exactly
as the editor's Start does (normalize, validate, admit a generation,
deploy), so both starts produce the same generation. With no
``expected_revision`` the latest control revision is used. A generation
that failed must be Reset before it can start again; ``reset_if_failed``
does that first. Returns the start handler's envelope.
"""
from core.container import container
workflow = await container.database().get_workflow(workflow_id)
if workflow is None:
return {"success": False, "error": "workflow_not_found"}
graph = workflow.data if isinstance(workflow.data, dict) else {}
service = _control_service()
latest = await service.database.get_latest_workflow_control(workflow_id)
if latest is not None and latest.status == "failed" and reset_if_failed:
reset = await handle_reset_workflow(
{
"workflow_id": workflow_id,
"expected_revision": latest.revision,
"idempotency_key": f"{idempotency_key or uuid.uuid4().hex}:reset",
},
None,
)
if not reset.get("success"):
return reset
latest = await service.database.get_latest_workflow_control(workflow_id)
revision = expected_revision if expected_revision is not None else (latest.revision if latest else 0)
return await handle_start_workflow(
{
"workflow_id": workflow_id,
"nodes": list(graph.get("nodes") or []),
"edges": list(graph.get("edges") or []),
"expected_revision": revision,
"idempotency_key": idempotency_key or f"start:{workflow_id}:{uuid.uuid4().hex}",
"user_id": owner_id,
},
None,
)
@ws_handler("workflow_id")
async def handle_pause_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
from core.container import container
workflow_id = data["workflow_id"]
service = _control_service()
control = await service.database.get_latest_workflow_control(workflow_id)
if control is None:
return {"success": False, "error": "workflow_never_started"}
if data.get("expected_root_execution_id") and data["expected_root_execution_id"] != control.root_execution_id:
raise ValueError("control_generation_conflict")
control, controller_status = await _reconcile_control(service, control)
if control.status == "paused":
return {
"success": True,
"idempotent": True,
**await _control_payload(control, controller_status=controller_status),
}
if control.status == "pausing":
if _versioned_control(control):
return await _await_versioned_transition(service, control)
return {
"success": False,
"error": "workflow_control_transition_pending",
**await _control_payload(control, controller_status=controller_status),
}
# Only server-side callers (no socket) may say why a pause happened: a
# client must not be able to dress its own pause up as an automatic one.
reason_values = _automatic_pause_values(data) if websocket is None else None
transition_kwargs: Dict[str, Any] = {"values": reason_values} if reason_values else {}
async with _control_completion_lock(workflow_id):
control = await service.transition(
control,
expected_revision=_expected_revision(data, control),
from_statuses={"running"},
status="pausing",
**transition_kwargs,
)
transition_task = _schedule_versioned_transition(service, control) if _versioned_control(control) else None
await _broadcast_control(control)
container.workflow_service().pause_deployment(workflow_id)
if _versioned_control(control):
return await _await_versioned_transition(service, control, task=transition_task)
try:
controller_status = await _update_controller_state(
control,
"paused",
update_id=(
f"{control.id}:{control.revision}:"
f"{data.get('idempotency_key') or uuid.uuid4().hex}:paused"
),
)
except (TemporalControlUnavailable, TemporalControlAckMismatch):
await container.workflow_service().resume_deployment(workflow_id)
await _restore_control_after_failed_update(
service,
control,
transitional_state="pausing",
stable_state="running",
)
raise
paused_schedules = await _set_cron_pause(
workflow_id,
paused=True,
strict=True,
)
paused_triggers = await container.workflow_service().update_trigger_pause_status(workflow_id, paused=True)
signalled = await _signal_generation_workflows(
control,
"pause",
strict=True,
)
control = await service.transition(control, expected_revision=control.revision, from_statuses={"pausing"}, status="paused")
# UI-facing execution flags follow the pause: the deployment stays
# armed but nothing new executes, so the canvas edge animation and
# toolbar executing indicator must stop. The pause family previously
# emitted only workflow_control_status, leaving `executing=true` /
# `status=running` stale on every connected client.
from services.status_broadcaster import get_status_broadcaster
broadcaster = get_status_broadcaster()
await broadcaster.update_workflow_status(
executing=False, current_node=None, progress=0, workflow_id=workflow_id,
)
await broadcaster.update_deployment_status(
is_running=True, status="paused", active_runs=0, workflow_id=workflow_id,
)
payload = await _broadcast_control(control, controller_status=controller_status, extra={
"signalled_executions": signalled,
"paused_schedules": paused_schedules,
"paused_triggers": paused_triggers,
})
return {
"success": True, "signalled_executions": signalled,
"paused_schedules": paused_schedules, "paused_triggers": paused_triggers,
**payload,
}
@ws_handler("workflow_id")
async def handle_resume_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
from core.container import container
workflow_id = data["workflow_id"]
service = _control_service()
control = await service.database.get_latest_workflow_control(workflow_id)
if control is None:
return {"success": False, "error": "workflow_never_started"}
if data.get("expected_root_execution_id") and data["expected_root_execution_id"] != control.root_execution_id:
raise ValueError("control_generation_conflict")
control, controller_status = await _reconcile_control(service, control)
if control.status == "running":
return {
"success": True,
"idempotent": True,
**await _control_payload(control, controller_status=controller_status),
}
if control.status == "resuming":
if _versioned_control(control):
return await _await_versioned_transition(service, control)
return {
"success": False,
"error": "workflow_control_transition_pending",
**await _control_payload(control, controller_status=controller_status),
}
async with _control_completion_lock(workflow_id):
control = await service.transition(
control, expected_revision=_expected_revision(data, control), from_statuses={"paused"}, status="resuming"
)
transition_task = _schedule_versioned_transition(service, control) if _versioned_control(control) else None
await _broadcast_control(control)
if _versioned_control(control):
return await _await_versioned_transition(service, control, task=transition_task)
resume_update_key = data.get("idempotency_key") or uuid.uuid4().hex
async def _apply_resume_update(target):
return await _update_controller_state(
target,
"running",
update_id=f"{target.id}:{target.revision}:{resume_update_key}:running",
)
try:
controller_status = await _apply_resume_update(control)
except (TemporalControlUnavailable, TemporalControlAckMismatch):
await _restore_control_after_failed_update(
service,
control,
transitional_state="resuming",
stable_state="paused",
)
raise
except Exception as exc:
if not _temporal_target_already_gone(exc):
raise
# The controller execution is gone (killed in the Temporal UI /
# crashed / retention-deleted). Under the missing-controller pause
# policy the row converged to paused precisely so this Resume can
# rebuild the controller from the durable row + graph snapshot,
# then re-apply the running state against the fresh execution.
try:
control = await _rebuild_missing_controller(service, control)
controller_status = await _apply_resume_update(control)
except Exception as rebuild_exc: # noqa: BLE001 — restore + surface
logger.warning(
"Controller rebuild during resume failed",
workflow_id=workflow_id,
error=str(rebuild_exc),
)
await _restore_control_after_failed_update(
service,
control,
transitional_state="resuming",
stable_state="paused",
)
raise
resumed_schedules = await _set_cron_pause(
workflow_id,
paused=False,
strict=True,
)
signalled = await _signal_generation_workflows(
control,
"resume",
strict=True,
)
queued = await container.workflow_service().resume_deployment(workflow_id)
resumed_triggers = await container.workflow_service().update_trigger_pause_status(workflow_id, paused=False)
control = await service.transition(
control,
expected_revision=control.revision,
from_statuses={"resuming"},
status="running",
**_clear_pause_reason_kwargs(control),
)
# Operator intervention resets the circuit-breaker streak — the next
# failure after a resume starts a fresh count, not a near-tripped one.
await _clear_failure_streak(service.database, control)
# Mirror of the pause-side emission: flip the UI-facing execution
# flags back so the canvas and toolbar re-animate on every client.
from services.status_broadcaster import get_status_broadcaster
broadcaster = get_status_broadcaster()
await broadcaster.update_workflow_status(
executing=True, current_node=None, progress=0, workflow_id=workflow_id,
)
await broadcaster.update_deployment_status(
is_running=True, status="running", active_runs=0, workflow_id=workflow_id,
)
payload = await _broadcast_control(control, controller_status=controller_status, extra={
"resumed_queued_events": queued,
"signalled_executions": signalled,
"resumed_schedules": resumed_schedules,
"resumed_triggers": resumed_triggers,
})
return {
"success": True, "resumed_queued_events": queued, "signalled_executions": signalled,
"resumed_schedules": resumed_schedules, "resumed_triggers": resumed_triggers,
**payload,
}
@ws_handler("workflow_id")
async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Reset a workflow. When the Workspace controller has not answered its
reset in time, the answer is ``workflow_control_transition_pending``; the
row stays ``resetting``, and the same Reset again finishes it."""
from services.temporal.update_calls import UpdatePending
try:
return await _reset_workflow(data, websocket)
except UpdatePending as exc:
workflow_id = data["workflow_id"]
service = _control_service()
logger.warning("Workspace cleanup has not answered; the Reset stays pending",
workflow_id=workflow_id, error=str(exc))
latest = await service.database.get_latest_workflow_control(workflow_id)
payload = (await _control_payload(latest) if latest
else await _with_runtime_counts(await service.get_status(workflow_id), workflow_id))
return {"success": False, "error": "workflow_control_transition_pending", **payload}
async def _reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
workflow_id = data["workflow_id"]
service = _control_service()
current = await service.database.get_latest_workflow_control(workflow_id)
if data.get("expected_root_execution_id") and (
current is None or data["expected_root_execution_id"] != current.root_execution_id
):
raise ValueError("control_generation_conflict")
if current is None or current.status == "reset":
# Workspace tasks also exist before Start and after a generation was
# reset. Their own Temporal controller supplies the cleanup barrier.
revision = current.revision if current else 0
if int(data.get("expected_revision", revision)) != revision:
if current is not None and current.status == "reset":
return {"success": True, "idempotent": True, **await _control_payload(current)}
raise ValueError("control_revision_conflict")
workspace = await _reset_workspace_tasks(workflow_id, service.database, current, data)
if not workspace.get("present", True):
payload = await _control_payload(current) if current else await service.get_status(workflow_id)
return {"success": True, "idempotent": True, **payload}
from services.status_broadcaster import get_status_broadcaster
await get_status_broadcaster().broadcast({
"type": "workflow_runtime_reset", "workflow_id": workflow_id,
"reset_nodes": workspace.get("reset_nodes", []),
})
payload = await _with_runtime_counts(await service.get_status(workflow_id), workflow_id)
await get_status_broadcaster().broadcast({"type": "workflow_control_status", "workflow_id": workflow_id, "data": payload})
return {"success": True, "idempotent": not workspace.get("cancelled"), **payload}
# The completed-generation branch above only resets direct Workspace
# tasks. Generation-wide cleanup below must not sweep a later Start.
if data.get("expected_root_execution_id"):
# A generation-scoped Reset can preempt a blocked boot setup waiter
# before acquiring the completion lock. The root identity was checked
# above and is rechecked by the owning-row CAS below; revision-only
# callers retain their strict conflict contract and bounded lock wait.
_cancel_boot_reconciliation(current)
async with _control_completion_lock(workflow_id):
# Completion can win while Reset waits for this short lock. Reload
# before CAS and never let a delayed Reset sweep a later generation.
latest = await service.database.get_latest_workflow_control(workflow_id)
if latest is None or latest.id != current.id or latest.generation != current.generation:
raise ValueError("control_generation_conflict")
current = latest
if current.status != "resetting":
# A generation-scoped Reset supersedes Stop/Resume even when
# their stable acknowledgement advanced the revision in flight.
# Older clients keep their strict revision-only CAS contract.
expected_revision = current.revision if data.get("expected_root_execution_id") else _expected_revision(data, current)
current = await service.transition(
current, expected_revision=expected_revision,
from_statuses={"starting", "running", "pausing", "paused", "resuming", "failed"}, status="resetting",
values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
)
# The durable Reset revision supersedes socket-independent Stop
# polling. Admission closes before any cleanup/broadcast yields.
_close_local_admission(workflow_id)
_supersede_versioned_transitions(current)
await _broadcast_control(current)
try:
await _signal_controller(current, "reset")
except Exception as exc:
logger.warning(
"Workflow reset signal failed; continuing with termination",
workflow_id=workflow_id,
error=str(exc),
)
# Remove cron producers before terminating executions; otherwise a firing
# between the execution scan and schedule deletion could survive Reset.
deleted_schedules = await _delete_cron_schedules(
workflow_id,
strict=True,
)
cancelled = await handle_cancel_deployment(
{"workflow_id": workflow_id},
websocket,
)
# Durable Temporal cleanup above is authoritative. The process-local
# deployment may legitimately be absent after an API-server restart.
# Any other local teardown failure can leave a listener/admission path
# alive, so keep the durable control in ``resetting`` for a safe retry.
local_cleanup_completed = bool(cancelled.get("success"))
if not local_cleanup_completed:
# ``handle_cancel_deployment`` retains its historical envelope and
# exposes manager errors as ``message``. Accept ``error`` as well for
# direct/internal callers and future wire compatibility.
local_error = str(
cancelled.get("error")
or cancelled.get("message")
or "unknown"
)
expected_absent_error = f"Workflow {workflow_id} is not deployed"
if local_error != expected_absent_error:
raise RuntimeError(
f"workflow_local_cleanup_failed:{local_error}"
)
# Controller, cron, and legacy local admission paths are now closed. This
# final strict sweep therefore observes a fixed set of generation runs.
terminated = await _terminate_generation_workflows(current, strict=True)
from services.approvals.waiting import cancel_waits
await cancel_waits(service.database, workflow_id, current.generation)
from core.container import container
from services.browser_reset import cleanup_generation_browser_tasks
wrapper = container.temporal_client()
cleaned_browser_tasks = await cleanup_generation_browser_tasks(
service.database, wrapper.client if wrapper is not None else None, current,
)
# A terminated run closes none of its own records: close the generation's
# team work and open requests here, or the next Apply waits on them. Late,
# so a delivery activity the sweep stopped has had time to see its cancel.
from services.employees.safe_apply import OPEN_JOB_STATES, close_open_work
await close_open_work(service.database, workflow_id, reason="The workflow was reset",
job_states=(*OPEN_JOB_STATES, "queued"))
workspace = await _reset_workspace_tasks(workflow_id, service.database, current, data)
archived = await service.database.update_workflow_run_data_scope(
current.data_scope_id or current.execution_id,
status="archived", archived_at=datetime.now(timezone.utc),
)
if not archived:
raise RuntimeError("workflow_data_scope_archive_failed")
from services.status_broadcaster import get_status_broadcaster
from services.deployment.runtime_state import archive_and_reset_node_state
broadcaster = get_status_broadcaster()
node_state = await archive_and_reset_node_state(
current, service.database, broadcaster,
)
current = await service.transition(
current, expected_revision=current.revision, from_statuses={"resetting"}, status="reset",
values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
)
await broadcaster.broadcast({
"type": "workflow_runtime_reset",
"workflow_id": workflow_id,
"generation": current.generation,
"data_scope_id": current.data_scope_id or current.execution_id,
"archived_nodes": node_state["archived_nodes"],
"reset_nodes": node_state["reset_nodes"],
"cancelled_workspace_tasks": workspace.get("cancelled", 0),
})
payload = await _broadcast_control(current, extra={
"terminated_executions": terminated,
"deleted_schedules": deleted_schedules,
"cleaned_browser_tasks": cleaned_browser_tasks,
"local_cleanup_completed": local_cleanup_completed,
"archived_nodes": node_state["archived_nodes"],
"reset_nodes": node_state["reset_nodes"],
})
return {
"success": True,
"idempotent": False,
"terminated_executions": terminated,
"deleted_schedules": deleted_schedules,
"cleaned_browser_tasks": cleaned_browser_tasks,
"local_cleanup_completed": local_cleanup_completed,
"archived_nodes": node_state["archived_nodes"],
"reset_nodes": node_state["reset_nodes"],
**payload,
}
def _pause_on_failure_threshold() -> int:
"""Failures within the rolling window required to trip the breaker.
Default 3 — a single node hiccup on one firing (missing config,
transient API error) must never pause a live deployment; only a
sustained failure streak does. 1 restores pause-on-first-failure.
"""
from core.container import container
try:
value = int(getattr(container.settings(), "workflow_control_pause_on_failure_threshold", 3))
except (TypeError, ValueError):
value = 3
return max(1, value)
def _pause_on_failure_window_seconds() -> float:
"""Rolling window for the failure streak; older failures age out."""
from core.container import container
try:
value = float(
getattr(container.settings(), "workflow_control_pause_on_failure_window_seconds", 600.0)
)
except (TypeError, ValueError):
value = 600.0
return max(1.0, value)
def _failure_streak_key(control) -> str:
# Keyed by the control row id, which embeds workflow + generation —
# a Reset/Start naturally begins with a fresh streak.
return f"workflow_control:failure_streak:{control.id}"
async def _bump_failure_streak(database, control, *, window_seconds: float) -> int:
"""Increment the rolling-window failure counter and return it.
State lives in the durable cache table (value is a JSON blob; the
TTL doubles as the window so stale streaks age out even if the
timestamp check is bypassed). Concurrent failed runs can race the
read-modify-write and miscount by one — acceptable slack for a
breaker threshold.
"""
key = _failure_streak_key(control)
now = time.time()
count = 0
raw = await database.get_cache_entry(key)
if raw:
try:
state = json.loads(raw)
if now - float(state.get("last_at") or 0.0) <= window_seconds:
count = int(state.get("count") or 0)
except (TypeError, ValueError):
count = 0
count += 1
await database.set_cache_entry(
key,
json.dumps({"count": count, "last_at": now}),
int(window_seconds),
)
return count
async def _clear_failure_streak(database, control) -> None:
try:
await database.delete_cache_entry(_failure_streak_key(control))
except Exception as exc: # noqa: BLE001 — the TTL ages it out anyway
logger.debug(f"Failed to clear failure streak: {exc}")
async def pause_generation_on_failure(
*, workflow_id: str, reason: str, requires_user_action: bool = False,
hint: Optional[str] = None, generation: Optional[int] = None,
) -> Dict[str, Any]:
"""Circuit breaker: pause a controlled deployment after failed runs.
Called by the ``workflow_control.pause_on_failure.v1`` activity that
MachinaWorkflow schedules on its failure path. Without it a trigger
keeps firing new runs into the same error indefinitely; with it the
deployment lands ``paused`` so the user fixes the cause and Resumes.
Trips only after ``WORKFLOW_CONTROL_PAUSE_ON_FAILURE_THRESHOLD``
failed runs inside the rolling window — small things (one node
failing on one firing) never pause the deployment. Gated on
``WORKFLOW_CONTROL_PAUSE_ON_FAILURE`` (default true) and applies
only to a generation currently ``running`` — direct manual runs,
legacy uncontrolled deployments, and already-paused generations are
untouched. All knobs are evaluated HERE (activity side) rather than
inside workflow code so config flips never touch recorded commands.
"""
from core.container import container
if not workflow_id:
return {"paused": False, "reason": "no_workflow_id"}
if not bool(getattr(container.settings(), "workflow_control_pause_on_failure", True)):
return {"paused": False, "reason": "disabled"}
service = _control_service()
control = await service.database.get_latest_workflow_control(workflow_id)
if control is None or control.status != "running":
return {
"paused": False,
"reason": "not_running",
"status": getattr(control, "status", None),
}
if generation is not None and int(generation) != control.generation:
return {"paused": False, "reason": "stale_generation"}
# Account/configuration blockers cannot heal by spending two more runs.
# Ordinary node/input failures retain the existing circuit breaker.
threshold = 1 if requires_user_action else _pause_on_failure_threshold()
if threshold > 1:
streak = await _bump_failure_streak(
service.database,
control,
window_seconds=_pause_on_failure_window_seconds(),
)
if streak < threshold:
logger.info(
"Run failure below circuit-breaker threshold; not pausing",
workflow_id=workflow_id,
streak=streak,
threshold=threshold,
failure=reason,
)
return {
"paused": False,
"reason": "below_threshold",
"streak": streak,
"threshold": threshold,
}
await _clear_failure_streak(service.database, control)
logger.warning(
"Pausing deployment after failed run (circuit breaker)",
workflow_id=workflow_id,
generation=control.generation,
failure=reason,
)
try:
result = await handle_pause_workflow(
{
"workflow_id": workflow_id,
"expected_revision": control.revision,
"idempotency_key": f"pause-on-failure:{control.id}:{control.revision}",
"_pause_reason": PAUSE_REASON_FAILURES,
"_pause_detail": (
" ".join(f"{reason} {hint or ''}".split())[:1000]
if requires_user_action else _failure_pause_detail(reason)
),
},
None,
)
except Exception as exc: # noqa: BLE001 — activity reports, never raises
logger.warning(
"Pause-on-failure did not complete",
workflow_id=workflow_id,
error=str(exc),
)
return {"paused": False, "error": str(exc)}
paused = bool(isinstance(result, dict) and result.get("success"))
if not paused:
logger.warning(
"Pause-on-failure did not complete",
workflow_id=workflow_id,
error=(result or {}).get("error") if isinstance(result, dict) else None,
)
return {"paused": paused, "failure": reason}
WS_HANDLERS: Dict[str, Any] = {
"deploy_workflow": handle_deploy_workflow,
"cancel_deployment": handle_cancel_deployment,
"get_deployment_status": handle_get_deployment_status,
"get_workflow_lock": handle_get_workflow_lock,
"update_deployment_settings": handle_update_deployment_settings,
"start_workflow": handle_start_workflow,
"pause_workflow": handle_pause_workflow,
"resume_workflow": handle_resume_workflow,
"reset_workflow": handle_reset_workflow,
"get_workflow_control_status": handle_get_workflow_control_status,
}
# Clean-shutdown marker for crash detection (see _consume_shutdown_state).
# Registered through the generic lifespan shutdown-hook registry so main.py
# never learns about the control plane's internals.
from services.plugin.shutdown_hooks import register_shutdown_hook # noqa: E402
register_shutdown_hook("workflow_control_clean_shutdown", mark_clean_shutdown)
__all__ = [
"WS_HANDLERS",
"handle_cancel_deployment",
"handle_deploy_workflow",
"handle_get_deployment_status",
"handle_get_workflow_lock",
"handle_update_deployment_settings",
"mark_clean_shutdown",
"pause_generation_on_failure",
"reconcile_active_controls_on_boot",
]