Skip to content

Self-improving AI Employees for everyone

fix(control): continue-as-new and boot reconcile keep deployments alive for months

trohitg34fa9afd1b1f
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index 5f753305..eecf8f36 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -452,15 +452,22 @@ async def _start_controller(control) -> Optional[str]:
def _controller_handle(control):
+ """Handle addressed by workflow id only — never pinned to a run id.
+
+ The controller keeps its event history bounded via continue-as-new,
+ which mints a new run id under the same workflow id. A handle pinned
+ to ``control.controller_run_id`` (the FIRST run) would target a
+ closed run after the first rollover and every pause/resume/update/
+ query would fail. The workflow id already carries the generation
+ (``workflow-control-<wf>-g<N>``), so unpinned addressing cannot
+ reach a different generation's controller.
+ """
from core.container import container
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None or not control.controller_workflow_id:
return None
- return wrapper.client.get_workflow_handle(
- control.controller_workflow_id,
- run_id=control.controller_run_id,
- )
+ return wrapper.client.get_workflow_handle(control.controller_workflow_id)
async def _signal_controller(control, signal_name: str) -> None:
@@ -968,6 +975,179 @@ async def _reconcile_control(service: WorkflowControlService, control):
return control, controller_status
+# ---------------------------------------------------------------------------
+# Boot-time reconcile — called once from services.temporal.lifecycle after
+# the workers start. Request-time reconciliation stays lazy; this pass closes
+# the unattended-server window where durable intent (running/paused rows) and
+# runtime behaviour could diverge indefinitely after a backend restart.
+# ---------------------------------------------------------------------------
+
+
+async def reconcile_active_controls_on_boot() -> int:
+ """Converge every active durable control after a backend restart.
+
+ Three responsibilities per active row:
+
+ 1. Run the same lazy :func:`_reconcile_control` used by status reads
+ (finishes interrupted pause/resume transitions, fails rows whose
+ controller vanished).
+ 2. Converge ``starting`` rows — the one state the lazy path never
+ touches while its controller exists, so a crash mid-start used to
+ wedge the workflow behind a permanent ``workflow_start_pending``.
+ Boot is the only moment guaranteed free of a concurrent in-process
+ start, which is why this lives here and not in the lazy path.
+ 3. Re-arm the process-local half of running/paused generations from
+ the persisted graph snapshot: the controller and its Temporal
+ children survive a restart on their own, but DeploymentManager
+ state (runtime counts, admission, pause fan-in) and in-process
+ collectors for non-canary trigger types die with the process and
+ previously stayed dead while the row reported ``running`` forever.
+
+ Returns the number of rows processed; per-row failures are logged and
+ skipped so one broken generation cannot block the rest.
+ """
+ from core.container import container
+
+ database = container.database()
+ service = WorkflowControlService(database)
+ controls = await database.list_active_workflow_controls()
+ processed = 0
+ for control in controls:
+ try:
+ control, controller_status = await _reconcile_control(service, control)
+ if control.status == "starting":
+ control = await _converge_interrupted_start(
+ service, control, controller_status
+ )
+ if control.status in {"running", "paused"}:
+ await _rearm_generation(control)
+ processed += 1
+ except Exception as exc: # noqa: BLE001 — per-row isolation
+ logger.warning(
+ "Boot reconcile failed for workflow control",
+ workflow_id=control.workflow_id,
+ status=control.status,
+ error=str(exc),
+ )
+ return processed
+
+
+async def _converge_interrupted_start(
+ service: WorkflowControlService,
+ control,
+ controller_status: Optional[Dict[str, Any]],
+):
+ """Decide what a crash left behind for a row stuck in ``starting``.
+
+ A vanished controller was already failed by ``_reconcile_control``.
+ With the controller alive: triggers registered (or a triggerless
+ graph) means the deploy effectively completed — commit ``running``.
+ A live-but-empty controller for a graph that declares triggers means
+ the crash landed before registration — fail the generation (Reset +
+ Start rebuilds cleanly) and close the orphan controller so dispatch
+ stops signalling it.
+ """
+ if controller_status is None:
+ # Controller unreachable (not missing) — leave transitional; the
+ # lazy path retries when Temporal comes back.
+ return control
+
+ from constants import WORKFLOW_TRIGGER_TYPES
+
+ triggers_registered = len(controller_status.get("triggers") or {})
+ graph_nodes = (control.graph_snapshot or {}).get("nodes") or []
+ graph_has_triggers = any(
+ node.get("type") in WORKFLOW_TRIGGER_TYPES for node in graph_nodes
+ )
+ if triggers_registered or not graph_has_triggers:
+ try:
+ control = await service.transition(
+ control,
+ expected_revision=control.revision,
+ from_statuses={"starting"},
+ status="running",
+ )
+ await _broadcast_control(control)
+ except ValueError:
+ # Concurrent writer won the CAS; report what the DB has.
+ latest = await service.database.get_latest_workflow_control(control.workflow_id)
+ if latest is not None and latest.generation == control.generation:
+ control = latest
+ return control
+
+ try:
+ await _signal_controller(control, "reset")
+ except Exception as exc: # noqa: BLE001 — best-effort close
+ logger.warning(
+ "Failed to close orphan controller for interrupted start",
+ workflow_id=control.workflow_id,
+ error=str(exc),
+ )
+ try:
+ failed = await service.fail(control, "interrupted_start")
+ except ValueError:
+ return control
+ await _broadcast_control(failed)
+ return failed
+
+
+async def _rearm_generation(control) -> None:
+ """Re-establish process-local deployment state for a durable generation.
+
+ Idempotent by construction: canary triggers re-register with the
+ controller keyed by listener id (no-op when already known), legacy
+ listener starts use ``id_conflict_policy=USE_EXISTING``, and cron
+ schedule creation preserves server-owned paused state on redeploy.
+ In-process collectors for non-canary trigger types are re-armed
+ fresh (they died with the previous process).
+ """
+ from core.container import container
+
+ workflow_service = container.workflow_service()
+ if workflow_service.is_workflow_deployed(control.workflow_id):
+ return
+ snapshot = control.graph_snapshot or {}
+ nodes = snapshot.get("nodes") or []
+ edges = snapshot.get("edges") or []
+ if not nodes:
+ logger.warning(
+ "Cannot re-arm generation without a graph snapshot",
+ workflow_id=control.workflow_id,
+ generation=control.generation,
+ )
+ return
+ deploy_data = {
+ "workflow_id": control.workflow_id,
+ "nodes": nodes,
+ "edges": edges,
+ # Runtime persistence is generation-scoped (same contract as
+ # handle_start_workflow's deploy call).
+ "session_id": control.data_scope_id or control.execution_id,
+ "execution_id": control.execution_id,
+ "root_execution_id": control.root_execution_id,
+ }
+ deployed = await handle_deploy_workflow(deploy_data, None)
+ if not deployed.get("success"):
+ raise RuntimeError(str(deployed.get("error", "deployment_failed")))
+ setup = await _await_deployment_setup(control.workflow_id)
+ if not setup.get("success"):
+ raise RuntimeError(str(setup.get("error", "deployment_setup_failed")))
+ if control.status == "paused":
+ # Restore the paused posture on the freshly re-armed local half.
+ workflow_service.pause_deployment(control.workflow_id)
+ await workflow_service.update_trigger_pause_status(
+ control.workflow_id,
+ paused=True,
+ )
+ await _set_cron_pause(control.workflow_id, paused=True, strict=False)
+ logger.info(
+ "Re-armed durable generation after restart",
+ workflow_id=control.workflow_id,
+ generation=control.generation,
+ status=control.status,
+ )
+
+
async def _await_deployment_setup(workflow_id: str) -> Dict[str, Any]:
"""Wait for trigger/listener setup started by the legacy deploy handler."""
task = _deployment_tasks.get(workflow_id)
diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.py
index a2eada8b..864d23bf 100644
--- a/server/services/deployment/manager.py
+++ b/server/services/deployment/manager.py
@@ -823,8 +823,11 @@ class DeploymentManager:
# The controller owns listeners as child workflows. This keeps
# trigger activity in the deployment's Temporal execution tree
# instead of presenting each listener as an unrelated root trace.
+ # Addressed by workflow id only (never run_id-pinned): the
+ # controller continue-as-news to bound its history, and a
+ # pinned handle would target a closed run after rollover.
await wrapper.client.get_workflow_handle(
- control.controller_workflow_id, run_id=control.controller_run_id,
+ control.controller_workflow_id,
).signal("register_trigger", {
"listener_id": listener_id,
"workflow_type": workflow_type_name,
diff --git a/server/services/events/dispatch.py b/server/services/events/dispatch.py
index d7815a56..b3edcda3 100644
--- a/server/services/events/dispatch.py
+++ b/server/services/events/dispatch.py
@@ -138,9 +138,29 @@ async def _signal_running_consumers(event: WorkflowEvent) -> None:
logger.warning(f"emit: Visibility query failed (query={query!r}): {exc}")
return
- if not consumers:
+ # Every signal is immutable history in the receiving workflow —
+ # controllers used to receive EVERY platform event (matching or not)
+ # and filter in-workflow, so one busy deployment burned down every
+ # other controller's continue-as-new budget. Controllers advertise
+ # their push event types via the ControlEventTypes Search Attribute
+ # (upserted as triggers register); skip the ones with no match.
+ # Controllers without the attribute (pre-upgrade histories) keep the
+ # legacy match-all behaviour.
+ matched_types = {str(t) for t in event_types}
+ targets = []
+ skipped = 0
+ for wf in consumers:
+ if _is_controller(wf):
+ declared = _controller_event_types(wf)
+ if declared is not None and not (declared & matched_types):
+ skipped += 1
+ continue
+ targets.append(wf)
+
+ if not targets:
logger.debug(
- f"emit: 0 consumers for event.type={event.type!r} " f"(event.id={event.id})",
+ f"emit: 0 consumers for event.type={event.type!r} "
+ f"(event.id={event.id}, controllers_skipped={skipped})",
)
return
@@ -148,17 +168,60 @@ async def _signal_running_consumers(event: WorkflowEvent) -> None:
# block other consumers (return_exceptions=True surfaces failures
# as exceptions in the result list).
signal_results = await asyncio.gather(
- *[_signal_one(client, wf.id, event) for wf in consumers],
+ *[_signal_one(client, wf.id, event) for wf in targets],
return_exceptions=True,
)
delivered = sum(1 for r in signal_results if not isinstance(r, Exception))
failed = len(signal_results) - delivered
logger.info(
- f"emit: signalled {delivered}/{len(consumers)} consumers for " f"event.type={event.type!r} (failed={failed}, event.id={event.id})",
+ f"emit: signalled {delivered}/{len(targets)} consumers for "
+ f"event.type={event.type!r} (failed={failed}, "
+ f"controllers_skipped={skipped}, event.id={event.id})",
)
+def _is_controller(wf: object) -> bool:
+ """True when a Visibility-listed execution is a WorkflowControlWorkflow."""
+ wf_type = getattr(wf, "workflow_type", None)
+ wf_type = getattr(wf_type, "name", None) or wf_type
+ return str(wf_type) == "WorkflowControlWorkflow"
+
+
+def _controller_event_types(wf: object) -> Optional[set]:
+ """Read the ControlEventTypes keyword-list Search Attribute off a
+ Visibility-listed execution.
+
+ ``None`` means the attribute is absent (pre-upgrade controller or a
+ controller whose upsert failed) — callers treat that as match-all so
+ narrowing can never silently disconnect a live deployment. The SDK's
+ listed-execution shape varies across versions (typed pairs vs plain
+ dict), so both forms are read defensively.
+ """
+ try:
+ typed = getattr(wf, "typed_search_attributes", None)
+ if typed is not None:
+ for pair in typed:
+ key = getattr(getattr(pair, "key", None), "name", None)
+ if key != "ControlEventTypes":
+ continue
+ value = getattr(pair, "value", None)
+ if value is None:
+ return None
+ values = value if isinstance(value, (list, tuple)) else [value]
+ return {str(v) for v in values}
+ raw = getattr(wf, "search_attributes", None)
+ if isinstance(raw, dict) and "ControlEventTypes" in raw:
+ value = raw.get("ControlEventTypes")
+ if value is None:
+ return None
+ values = value if isinstance(value, (list, tuple)) else [value]
+ return {str(v) for v in values}
+ except Exception: # noqa: BLE001 — unreadable attribute == match-all
+ return None
+ return None
+
+
async def _signal_one(client, workflow_id: str, event: WorkflowEvent) -> None:
"""Send the ``on_event`` signal to one workflow.
diff --git a/server/services/temporal/polling_trigger_workflow.py b/server/services/temporal/polling_trigger_workflow.py
index 2fa18be2..7b2c9acd 100644
--- a/server/services/temporal/polling_trigger_workflow.py
+++ b/server/services/temporal/polling_trigger_workflow.py
@@ -60,6 +60,27 @@ _ACTIVITY_TIMEOUT_MULT = 4
COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
"polling-trigger-cooperative-pause-scheduling-v1"
)
+# Same contract as trigger_listener_workflow.UNBOUNDED_CHILD_RUNS_PATCH:
+# child runs may execute or stay paused for months, so new executions
+# start them without lifetime caps; the patch keeps replay
+# compatibility for histories recorded with the old 1h caps.
+UNBOUNDED_CHILD_RUNS_PATCH = "polling-unbounded-child-runs-v1"
+# _processed_count above increments only per EMITTED event, but every
+# poll cycle burns ~8-11 history events (sleep timer + activity +
+# workflow tasks) regardless — a quiet mailbox at the 60s default
+# reached the server's ~51,200-event hard termination in ~3 days with
+# the counter frozen at 0 and continue-as-new never firing. The patched
+# path checks history pressure every cycle (server suggestion + the
+# soft cap below), clamps user-supplied poll intervals to a sane floor,
+# and carries the pause flag across the rollover instead of blocking it.
+HISTORY_BOUNDED_CAN_PATCH = "polling-history-bounded-can-v1"
+_HISTORY_SOFT_CAP = 10_000
+# Defensive floor for user-supplied poll intervals: a few-second
+# interval burns ~100K+ history events/day. Mirrors the plugin-side
+# clamp (PollingTriggerNode.poll_interval_clamp) that the legacy
+# asyncio path applies but workflow payloads historically did not.
+# Timer durations are recorded commands, so the clamp rides the patch.
+_MIN_POLL_INTERVAL_S = 30
@workflow.defn(name="PollingTriggerWorkflow", sandboxed=False)
@@ -121,13 +142,22 @@ class PollingTriggerWorkflow:
use_cooperative_pause_schedule_gate = workflow.patched(
COOPERATIVE_PAUSE_SCHEDULING_PATCH
)
+ use_history_bounded_can = workflow.patched(HISTORY_BOUNDED_CAN_PATCH)
except Exception: # direct unit invocation outside Temporal runtime
if workflow.in_workflow():
raise
use_cooperative_pause_schedule_gate = False
+ use_history_bounded_can = True
+
+ # Rehydrate rollover-carried pause state. Command-free, so
+ # pre-patch histories (whose payloads lack the key) replay
+ # identically.
+ self._control_paused = bool(listener_data.get("control_paused"))
params = listener_data.get("filter_params", {}) or {}
poll_interval = int(params.get("poll_interval") or _DEFAULT_POLL_INTERVAL_S)
+ if use_history_bounded_can:
+ poll_interval = max(_MIN_POLL_INTERVAL_S, poll_interval)
activity_timeout_s = max(30, poll_interval * _ACTIVITY_TIMEOUT_MULT)
# Carry seen_ids across continueAsNew. First-start payload has
@@ -223,9 +253,21 @@ class PollingTriggerWorkflow:
workflow.logger.error(f"PollingTriggerWorkflow spawn failed for event.id={event_id}: {spawn_exc}")
self._processed_count += 1
- if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:
+ should_rollover = self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW
+ if use_history_bounded_can and not should_rollover:
+ # Poll cycles burn history even with zero emitted events —
+ # check pressure every cycle, not only per processed event.
+ from services.temporal.trigger_listener_workflow import _history_pressure
+
+ should_rollover = _history_pressure(_HISTORY_SOFT_CAP)
+ if should_rollover:
workflow.logger.info(f"PollingTriggerWorkflow continue_as_new: processed={self._processed_count}")
- if use_cooperative_pause_schedule_gate:
+ if use_history_bounded_can:
+ # Carry the pause flag instead of blocking the rollover
+ # behind a resume that may be months away (signal
+ # traffic would overflow the history mid-pause).
+ listener_data["control_paused"] = self._control_paused
+ elif use_cooperative_pause_schedule_gate:
# A pause may arrive during the final waiting-status
# broadcast. Continue only after Resume so the next run
# cannot reset a still-active pause flag.
@@ -252,7 +294,9 @@ class PollingTriggerWorkflow:
nodes) stay single-source. Mirrors
:meth:`TriggerListenerWorkflow._spawn_child_run` exactly.
"""
+ from services.temporal._retry_policies import QUICK_ACTIVITY_RETRY
from services.temporal.trigger_listener_workflow import (
+ BOUNDED_STATUS_BROADCASTS_PATCH,
_broadcast_trigger_idle,
_broadcast_trigger_waiting,
_build_run_graph,
@@ -313,12 +357,24 @@ class PollingTriggerWorkflow:
event_id = event.get("id") or workflow.uuid4().hex
child_id = f"{workflow_slug}-{trigger_label}-{event_id}"
- await _broadcast_trigger_idle(
- node_id=trigger_node_id,
- workflow_id=workflow_id,
- event_id=event_id,
- event_type=listener_data.get("event_type", ""),
- )
+ try:
+ unbounded_child_runs = workflow.patched(UNBOUNDED_CHILD_RUNS_PATCH)
+ bounded_broadcasts = workflow.patched(BOUNDED_STATUS_BROADCASTS_PATCH)
+ except RuntimeError: # direct unit invocation outside Temporal runtime
+ unbounded_child_runs = True
+ bounded_broadcasts = True
+ broadcast_retry = QUICK_ACTIVITY_RETRY if bounded_broadcasts else None
+
+ try:
+ await _broadcast_trigger_idle(
+ node_id=trigger_node_id,
+ workflow_id=workflow_id,
+ event_id=event_id,
+ event_type=listener_data.get("event_type", ""),
+ retry_policy=broadcast_retry,
+ )
+ except Exception as exc: # noqa: BLE001 — cosmetic UI signalling
+ workflow.logger.warning(f"Trigger status broadcast failed (non-fatal): {exc}")
if admission_check is not None:
await admission_check()
@@ -329,9 +385,12 @@ class PollingTriggerWorkflow:
"id_reuse_policy": (
WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY
),
- "execution_timeout": timedelta(hours=1),
- "run_timeout": timedelta(hours=1),
}
+ if not unbounded_child_runs:
+ # Replay-only: pre-patch histories recorded 1h caps on the
+ # child-start command. See UNBOUNDED_CHILD_RUNS_PATCH.
+ child_options["execution_timeout"] = timedelta(hours=1)
+ child_options["run_timeout"] = timedelta(hours=1)
if search_attributes is not None:
child_options["search_attributes"] = search_attributes
@@ -352,12 +411,16 @@ class PollingTriggerWorkflow:
workflow.logger.info(f"PollingTriggerWorkflow spawned child run: child_id={child_id} " f"event.id={event.get('id')}")
- await _broadcast_trigger_waiting(
- node_id=trigger_node_id,
- workflow_id=workflow_id,
- event_type=listener_data.get("event_type", ""),
- processed_count=self._processed_count + 1,
- )
+ try:
+ await _broadcast_trigger_waiting(
+ node_id=trigger_node_id,
+ workflow_id=workflow_id,
+ event_type=listener_data.get("event_type", ""),
+ processed_count=self._processed_count + 1,
+ retry_policy=broadcast_retry,
+ )
+ except Exception as exc: # noqa: BLE001 — cosmetic UI signalling
+ workflow.logger.warning(f"Trigger status broadcast failed (non-fatal): {exc}")
__all__ = ["PollingTriggerWorkflow"]
diff --git a/server/services/temporal/search_attributes.py b/server/services/temporal/search_attributes.py
index e1bcc117..75032d94 100644
--- a/server/services/temporal/search_attributes.py
+++ b/server/services/temporal/search_attributes.py
@@ -1,6 +1,6 @@
"""Wave 12 A4: Temporal custom Search Attributes registration.
-Defines the 6 Search Attributes the event framework needs for the
+Defines the Search Attributes the event framework needs for the
Visibility API to route inbound events to running consumer workflows
(see ``services/events/dispatch.py:emit``). Registration is idempotent —
Temporal Server's ``add_search_attributes`` rejects duplicates with an
@@ -42,7 +42,7 @@ class SearchAttributeSpec:
description: str
-# Single source of truth for the 6 event-framework Search Attributes.
+# Single source of truth for the event-framework Search Attributes.
# Anything that names or filters by these attributes (the dispatch
# helper, the Visibility queries in admin handlers, the test suite)
# imports from this list rather than restating the strings.
@@ -103,6 +103,20 @@ EVENT_SEARCH_ATTRIBUTES: Sequence[SearchAttributeSpec] = (
"hour')."
),
),
+ SearchAttributeSpec(
+ name="ControlEventTypes",
+ indexed_type=IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD_LIST,
+ description=(
+ "CloudEvents types a WorkflowControlWorkflow's registered push "
+ "triggers listen for (upserted by the controller as triggers "
+ "register). dispatch.emit reads it to skip signalling "
+ "controllers whose deployment has no matching trigger — "
+ "otherwise every platform event writes ~4 history events into "
+ "EVERY running controller and burns its continue-as-new budget "
+ "on other deployments' traffic. Controllers without the "
+ "attribute (pre-upgrade histories) are treated as match-all."
+ ),
+ ),
)
diff --git a/server/services/temporal/trigger_listener_workflow.py b/server/services/temporal/trigger_listener_workflow.py
index 00d2ee8b..8f04fcf8 100644
--- a/server/services/temporal/trigger_listener_workflow.py
+++ b/server/services/temporal/trigger_listener_workflow.py
@@ -35,6 +35,7 @@ from typing import Any, Dict, List, Optional, Set
from temporalio import workflow
from temporalio.common import (
+ RetryPolicy,
SearchAttributeKey,
SearchAttributePair,
TypedSearchAttributes,
@@ -54,6 +55,53 @@ CHILD_SEARCH_ATTRIBUTES_PATCH = "trigger-child-search-attributes-v1"
COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
"trigger-cooperative-pause-scheduling-v1"
)
+# Child MachinaWorkflow runs used to carry 1h execution/run timeouts.
+# A run may legitimately execute — or stay cooperatively paused — for
+# months, and Temporal's timeout timer keeps ticking through a pause,
+# so any pause longer than the cap silently terminated the run.
+# Liveness is enforced at the activity layer (heartbeats), not by
+# workflow lifetime caps; new executions start children unbounded
+# (Temporal's default). The patch keeps replay compatibility for
+# histories whose recorded child starts carried the old caps.
+UNBOUNDED_CHILD_RUNS_PATCH = "trigger-unbounded-child-runs-v1"
+# The _processed_count threshold above was calibrated assuming ~2-3
+# history events per firing, but the real spawn path costs ~15-25
+# (signal + two broadcast activities + optional graph-load + child
+# start/close + workflow tasks) — the server's ~51,200-event hard
+# termination fires around 2,000-3,500 firings, long before the
+# 16,000-processed checkpoint could. The patched path rolls over on
+# the server's own is_continue_as_new_suggested signal (with the soft
+# cap below as a deterministic backstop), carries still-queued events
+# instead of dropping them, and carries the pause flag instead of
+# blocking the rollover behind a resume that may be months away.
+HISTORY_BOUNDED_CAN_PATCH = "trigger-history-bounded-can-v1"
+_HISTORY_SOFT_CAP = 10_000
+# Continue-as-new argument blobs are capped at 2MiB; carry at most this
+# many still-queued events across a rollover (oldest dropped, logged).
+_MAX_CARRIED_EVENTS = 256
+# Status broadcasts are cosmetic UI signalling. Without an explicit
+# policy Temporal retries a failing activity forever, wedging the
+# serialized spawn loop behind a dead broadcaster. Bounded + swallowed
+# on the patched path (BOUNDED_STATUS_BROADCASTS_PATCH gates the
+# command-visible retry-policy change; the try/except is command-free).
+# Policy: the shared QUICK_ACTIVITY_RETRY constant (cheap side-effect
+# activities fail fast) — never an inline RetryPolicy construction.
+BOUNDED_STATUS_BROADCASTS_PATCH = "trigger-bounded-status-broadcasts-v1"
+
+
+def _history_pressure(soft_cap: int) -> bool:
+ """True when the server suggests continue-as-new or the current
+ history length crossed ``soft_cap``. False outside a real workflow
+ runtime (direct unit invocation), where history cannot grow. Shared
+ by the listener, polling, and controller rollover checks."""
+ try:
+ info = workflow.info()
+ except Exception: # direct unit invocation outside Temporal runtime
+ return False
+ return bool(
+ info.is_continue_as_new_suggested()
+ or info.get_current_history_length() > soft_cap
+ )
def event_workflow_search_attributes(
@@ -151,6 +199,16 @@ class TriggerListenerWorkflow:
use_cooperative_pause_schedule_gate = workflow.patched(
COOPERATIVE_PAUSE_SCHEDULING_PATCH
)
+ use_history_bounded_can = workflow.patched(HISTORY_BOUNDED_CAN_PATCH)
+
+ # Rehydrate rollover-carried state. Command-free, so pre-patch
+ # histories (whose payloads lack the keys) replay identically.
+ self._control_paused = bool(listener_data.get("control_paused"))
+ for carried_event in listener_data.get("pending_events") or []:
+ carried_id = carried_event.get("id")
+ if carried_id and carried_id not in self._seen_event_ids:
+ self._seen_event_ids.add(carried_id)
+ self._matched_events.append(carried_event)
while True:
await workflow.wait_condition(
@@ -186,9 +244,25 @@ class TriggerListenerWorkflow:
workflow.logger.error(f"TriggerListener spawn failed for event.id={event.get('id')}: {exc}")
self._processed_count += 1
- if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:
+ should_rollover = self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW
+ if use_history_bounded_can and not should_rollover:
+ should_rollover = _history_pressure(_HISTORY_SOFT_CAP)
+ if should_rollover:
workflow.logger.info(f"TriggerListener continue_as_new: processed={self._processed_count}")
- if use_cooperative_pause_schedule_gate:
+ if use_history_bounded_can:
+ # Carry queued events (previously dropped at rollover)
+ # and the pause flag (previously the rollover blocked
+ # behind a resume that could be months away, letting
+ # signal traffic overflow the history mid-pause).
+ dropped = max(0, len(self._matched_events) - _MAX_CARRIED_EVENTS)
+ if dropped:
+ workflow.logger.warning(
+ f"TriggerListener rollover dropping {dropped} oldest queued "
+ f"event(s) beyond the {_MAX_CARRIED_EVENTS} carry cap"
+ )
+ listener_data["pending_events"] = self._matched_events[-_MAX_CARRIED_EVENTS:]
+ listener_data["control_paused"] = self._control_paused
+ elif use_cooperative_pause_schedule_gate:
# Do not let continue-as-new reset a pause that landed
# during the final spawn/status broadcast.
await self._wait_until_resumed()
@@ -275,12 +349,26 @@ class TriggerListenerWorkflow:
event_id = event.get("id") or workflow.uuid4().hex
child_id = f"{workflow_slug}-{trigger_label}-{event_id}"
- await _broadcast_trigger_idle(
- node_id=trigger_node_id,
- workflow_id=workflow_id,
- event_id=event_id,
- event_type=event.get("type", ""),
- )
+ from services.temporal._retry_policies import QUICK_ACTIVITY_RETRY
+
+ try:
+ unbounded_child_runs = workflow.patched(UNBOUNDED_CHILD_RUNS_PATCH)
+ bounded_broadcasts = workflow.patched(BOUNDED_STATUS_BROADCASTS_PATCH)
+ except RuntimeError: # direct unit invocation outside Temporal runtime
+ unbounded_child_runs = True
+ bounded_broadcasts = True
+ broadcast_retry = QUICK_ACTIVITY_RETRY if bounded_broadcasts else None
+
+ try:
+ await _broadcast_trigger_idle(
+ node_id=trigger_node_id,
+ workflow_id=workflow_id,
+ event_id=event_id,
+ event_type=event.get("type", ""),
+ retry_policy=broadcast_retry,
+ )
+ except Exception as exc: # noqa: BLE001 — cosmetic UI signalling
+ workflow.logger.warning(f"Trigger status broadcast failed (non-fatal): {exc}")
if admission_check is not None:
await admission_check()
@@ -291,9 +379,12 @@ class TriggerListenerWorkflow:
"id_reuse_policy": (
WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY
),
- "execution_timeout": timedelta(hours=1),
- "run_timeout": timedelta(hours=1),
}
+ if not unbounded_child_runs:
+ # Replay-only: pre-patch histories recorded 1h caps on the
+ # child-start command. See UNBOUNDED_CHILD_RUNS_PATCH.
+ child_options["execution_timeout"] = timedelta(hours=1)
+ child_options["run_timeout"] = timedelta(hours=1)
if search_attributes is not None:
child_options["search_attributes"] = search_attributes
@@ -319,12 +410,16 @@ class TriggerListenerWorkflow:
f"TriggerListener spawned child run: child_id={child_id} " f"event.id={event.get('id')} event.type={event.get('type')}"
)
- await _broadcast_trigger_waiting(
- node_id=trigger_node_id,
- workflow_id=workflow_id,
- event_type=listener_data.get("event_type", ""),
- processed_count=self._processed_count + 1,
- )
+ try:
+ await _broadcast_trigger_waiting(
+ node_id=trigger_node_id,
+ workflow_id=workflow_id,
+ event_type=listener_data.get("event_type", ""),
+ processed_count=self._processed_count + 1,
+ retry_policy=broadcast_retry,
+ )
+ except Exception as exc: # noqa: BLE001 — cosmetic UI signalling
+ workflow.logger.warning(f"Trigger status broadcast failed (non-fatal): {exc}")
# ---------------------------------------------------------------------------
@@ -344,10 +439,20 @@ async def _broadcast_trigger_idle(
workflow_id: Optional[str],
event_id: str,
event_type: str,
+ retry_policy: Optional[RetryPolicy] = None,
) -> None:
"""Broadcast trigger node ``"idle"`` status with a "Graph executing..."
message — matches the legacy collector/processor transition so FE
- shows a firing pulse instead of a stuck "waiting" indicator."""
+ shows a firing pulse instead of a stuck "waiting" indicator.
+
+ ``retry_policy=None`` (Temporal default = unlimited retries) is the
+ replay-compatibility shape for pre-patch histories; new executions
+ pass the bounded policy so a dead broadcaster cannot wedge the
+ serialized spawn loop.
+ """
+ kwargs: Dict[str, Any] = {"start_to_close_timeout": _STATUS_ACTIVITY_TIMEOUT}
+ if retry_policy is not None:
+ kwargs["retry_policy"] = retry_policy
await workflow.execute_activity(
_STATUS_ACTIVITY_NAME,
{
@@ -361,7 +466,7 @@ async def _broadcast_trigger_idle(
},
"workflow_id": workflow_id,
},
- start_to_close_timeout=_STATUS_ACTIVITY_TIMEOUT,
+ **kwargs,
)
@@ -371,10 +476,15 @@ async def _broadcast_trigger_waiting(
workflow_id: Optional[str],
event_type: str,
processed_count: int,
+ retry_policy: Optional[RetryPolicy] = None,
) -> None:
"""Broadcast trigger node back to ``"waiting"`` after the child run
has been spawned (child completes independently per
- ``parent_close_policy=ABANDON``)."""
+ ``parent_close_policy=ABANDON``). Retry-policy contract matches
+ :func:`_broadcast_trigger_idle`."""
+ kwargs: Dict[str, Any] = {"start_to_close_timeout": _STATUS_ACTIVITY_TIMEOUT}
+ if retry_policy is not None:
+ kwargs["retry_policy"] = retry_policy
await workflow.execute_activity(
_STATUS_ACTIVITY_NAME,
{
@@ -388,7 +498,7 @@ async def _broadcast_trigger_waiting(
},
"workflow_id": workflow_id,
},
- start_to_close_timeout=_STATUS_ACTIVITY_TIMEOUT,
+ **kwargs,
)
diff --git a/server/services/temporal/workflow_control_workflow.py b/server/services/temporal/workflow_control_workflow.py
index 34f218dc..9ca480a7 100644
--- a/server/services/temporal/workflow_control_workflow.py
+++ b/server/services/temporal/workflow_control_workflow.py
@@ -2,9 +2,10 @@
import asyncio
from datetime import timedelta
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional, Tuple
from temporalio import workflow
+from temporalio.common import SearchAttributeKey, SearchAttributePair
from temporalio.exceptions import ApplicationError
from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY
@@ -13,6 +14,35 @@ from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY
COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
"workflow-control-cooperative-pause-scheduling-v1"
)
+# The controller is the ONE workflow expected to live for months, and it
+# multiplexes every trigger's signals, poll timers/activities, and child
+# spawns into a single event history. Temporal terminates any workflow
+# around ~51,200 history events, so without continue-as-new a single
+# polling trigger at the default 60s interval killed the whole
+# deployment's control plane in days. The patched path rolls the run
+# over (carrying triggers, control state, queued events, and per-trigger
+# seen-id baselines) whenever the server suggests it or the history
+# crosses the soft cap below. Pre-patch histories replay unchanged.
+CONTINUE_AS_NEW_PATCH = "workflow-control-continue-as-new-v1"
+# Upsert the ControlEventTypes keyword-list Search Attribute so
+# dispatch.emit can skip controllers whose deployment has no matching
+# push trigger (otherwise every platform event lands ~4 history events
+# in every running controller). Patch-gated: upsert is a command.
+CONTROL_EVENT_TYPES_SA_PATCH = "workflow-control-event-types-sa-v1"
+
+# Roll over before the server's hard ceiling. is_continue_as_new_suggested
+# is the primary signal; this cap is the deterministic backstop.
+_HISTORY_SOFT_CAP = 10_000
+# Bounds on state carried across continue-as-new — the CAN argument blob
+# is capped at 2MiB by Temporal, so unbounded carries would trade a
+# history overflow for an argument overflow.
+_MAX_CARRIED_EVENTS = 256
+_MAX_CARRIED_SEEN_IDS = 4_096
+# Defensive floor for user-supplied poll intervals: a few-second interval
+# burns ~100K+ history events/day. Mirrors the plugin-side clamp
+# (PollingTriggerNode.poll_interval_clamp) which the legacy asyncio path
+# applies but workflow payloads historically did not.
+_MIN_POLL_INTERVAL_S = 30
@workflow.defn(name="WorkflowControlWorkflow", sandboxed=False)
@@ -22,6 +52,14 @@ class WorkflowControlWorkflow:
Trigger definitions and inbound events are recorded as signals in this
workflow's own history. Only an actual triggered graph run becomes a child
workflow, so Temporal's workflow list has no per-trigger listener rows.
+
+ Longevity contract: the controller continue-as-news to keep its
+ history bounded. All state a rollover must preserve lives in
+ ``control_data`` — trigger specs (whose ``listener_args["seen_ids"]``
+ the poll loops write back after every cycle), the pending push-event
+ queue, the dedup baseline, and the control state/revision. Callers
+ therefore address the controller by workflow id only, never a pinned
+ run id.
"""
def __init__(self) -> None:
@@ -30,9 +68,13 @@ class WorkflowControlWorkflow:
self._closed = False
self._triggers: dict[str, Dict[str, Any]] = {}
self._events: list[tuple[str, Dict[str, Any]]] = []
- self._seen_event_ids: set[str] = set()
+ # Insertion-ordered so the carry across continue-as-new can keep
+ # the newest entries when trimming to _MAX_CARRIED_SEEN_IDS.
+ self._seen_event_ids: dict[str, None] = {}
self._poll_tasks: dict[str, asyncio.Task] = {}
self._use_cooperative_pause_schedule_gate = False
+ self._can_enabled = False
+ self._can_requested = False
@workflow.signal
async def pause(self) -> None:
@@ -91,6 +133,8 @@ class WorkflowControlWorkflow:
self._triggers[listener_id] = spec
if spec["workflow_type"] == "PollingTriggerWorkflow":
self._poll_tasks[listener_id] = asyncio.create_task(self._poll_trigger(listener_id, spec))
+ else:
+ self._upsert_event_types_attribute()
@workflow.signal
async def on_event(self, event: Dict[str, Any]) -> None:
@@ -105,8 +149,13 @@ class WorkflowControlWorkflow:
continue
dedup_key = f"{listener_id}:{event_id}"
if dedup_key not in self._seen_event_ids:
- self._seen_event_ids.add(dedup_key)
+ self._remember_event_id(dedup_key)
self._events.append((listener_id, event))
+ # A paused controller still receives matching signals; without this
+ # check its history could overflow mid-pause with the run loop
+ # parked. The flag wakes the run loop, which rolls over carrying
+ # the queue (and the paused state) forward.
+ self._maybe_request_rollover()
@workflow.query
def status(self) -> Dict[str, Any]:
@@ -122,23 +171,165 @@ class WorkflowControlWorkflow:
self._use_cooperative_pause_schedule_gate = workflow.patched(
COOPERATIVE_PAUSE_SCHEDULING_PATCH
)
+ self._can_enabled = workflow.patched(CONTINUE_AS_NEW_PATCH)
except Exception: # direct unit invocation outside Temporal runtime
if workflow.in_workflow():
raise
self._use_cooperative_pause_schedule_gate = False
+ self._can_enabled = True
self._state = control_data.get("state", "running")
+ self._seed_carried_state(control_data)
while not self._closed:
await workflow.wait_condition(
- lambda: self._closed or (self._state == "running" and bool(self._events))
+ lambda: (
+ self._closed
+ or self._can_requested
+ or (self._state == "running" and bool(self._events))
+ )
)
if self._closed:
break
+ if self._can_requested:
+ await self._continue_as_new(control_data)
+ # Unreachable in a real run; direct unit invocation returns.
+ break
listener_id, event = self._events.pop(0)
spec = self._triggers.get(listener_id)
- if spec is not None:
+ if spec is None:
+ continue
+ try:
await self._spawn_push_run(event, spec)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc: # noqa: BLE001 — per-event isolation
+ # One bad event (duplicate child id, exhausted broadcast
+ # retries) must not fail the deployment's entire control
+ # plane. Same isolation contract as the listener classes.
+ workflow.logger.error(
+ f"Controller push spawn failed for event.id={event.get('id')}: {exc}"
+ )
+ self._maybe_request_rollover()
return {"state": self._state, "generation": control_data.get("generation")}
+ # ---- continue-as-new machinery ------------------------------------
+
+ def _seed_carried_state(self, control_data: Dict[str, Any]) -> None:
+ """Rehydrate rollover-carried state; no-op for first-run payloads.
+
+ Everything here is command-free, so pre-patch histories (whose
+ payloads lack the carried keys) replay identically.
+ """
+ self._revision = int(control_data.get("revision") or 0)
+ for seen_id in control_data.get("seen_event_ids") or []:
+ self._seen_event_ids[str(seen_id)] = None
+ for pending in control_data.get("pending_events") or []:
+ try:
+ listener_id, event = pending[0], pending[1]
+ except (IndexError, TypeError, KeyError):
+ continue
+ self._events.append((str(listener_id), event))
+ for listener_id, spec in (control_data.get("triggers") or {}).items():
+ listener_id = str(listener_id)
+ if listener_id in self._triggers:
+ continue
+ self._triggers[listener_id] = spec
+ if spec.get("workflow_type") == "PollingTriggerWorkflow":
+ self._poll_tasks[listener_id] = asyncio.create_task(
+ self._poll_trigger(listener_id, spec)
+ )
+ if any(
+ spec.get("workflow_type") != "PollingTriggerWorkflow"
+ for spec in self._triggers.values()
+ ):
+ self._upsert_event_types_attribute()
+
+ def _remember_event_id(self, dedup_key: str) -> None:
+ self._seen_event_ids[dedup_key] = None
+ if len(self._seen_event_ids) > _MAX_CARRIED_SEEN_IDS:
+ oldest = next(iter(self._seen_event_ids))
+ del self._seen_event_ids[oldest]
+
+ def _history_pressure(self) -> bool:
+ from services.temporal.trigger_listener_workflow import _history_pressure
+
+ return _history_pressure(_HISTORY_SOFT_CAP)
+
+ def _maybe_request_rollover(self) -> None:
+ if self._can_enabled and not self._closed and not self._can_requested:
+ if self._history_pressure():
+ self._can_requested = True
+
+ async def _continue_as_new(self, control_data: Dict[str, Any]) -> None:
+ """Roll the run over, carrying everything a controller must keep.
+
+ Poll tasks are cancelled first — their provider ``seen_ids``
+ baselines were written back into each trigger spec's
+ ``listener_args`` after every cycle, so the carried specs restart
+ the loops in the new run without re-emitting old items. Works
+ while paused too: the paused state (and queued events) carry.
+ """
+ for task in self._poll_tasks.values():
+ task.cancel()
+ if self._poll_tasks:
+ await asyncio.gather(*self._poll_tasks.values(), return_exceptions=True)
+ dropped = max(0, len(self._events) - _MAX_CARRIED_EVENTS)
+ if dropped:
+ workflow.logger.warning(
+ f"Controller rollover dropping {dropped} oldest queued event(s) "
+ f"beyond the {_MAX_CARRIED_EVENTS} carry cap"
+ )
+ carried: Dict[str, Any] = {
+ **control_data,
+ "state": self._state,
+ "revision": self._revision,
+ "triggers": self._triggers,
+ "pending_events": [
+ list(item) for item in self._events[-_MAX_CARRIED_EVENTS:]
+ ],
+ "seen_event_ids": list(self._seen_event_ids)[-_MAX_CARRIED_SEEN_IDS:],
+ }
+ workflow.logger.info(
+ f"Controller continue_as_new: triggers={len(self._triggers)} "
+ f"pending={len(carried['pending_events'])} state={self._state}"
+ )
+ workflow.continue_as_new(args=[carried])
+
+ def _upsert_event_types_attribute(self) -> None:
+ """Advertise push event types via the ControlEventTypes attribute.
+
+ Lets ``dispatch.emit`` skip controllers with no matching trigger
+ instead of signalling every running controller with every
+ platform event. Patch-gated (upsert is a command); controllers
+ without the attribute are treated as match-all by dispatch.
+ """
+ try:
+ if not workflow.patched(CONTROL_EVENT_TYPES_SA_PATCH):
+ return
+ except Exception: # direct unit invocation outside Temporal runtime
+ return
+ event_types: set[str] = set()
+ for spec in self._triggers.values():
+ if spec.get("workflow_type") == "PollingTriggerWorkflow":
+ continue
+ for event_type in spec.get("event_types") or [spec.get("event_type")]:
+ if event_type:
+ event_types.add(str(event_type))
+ if not event_types:
+ return
+ try:
+ workflow.upsert_search_attributes(
+ [
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword_list("ControlEventTypes"),
+ sorted(event_types),
+ )
+ ]
+ )
+ except Exception as exc: # noqa: BLE001 — attribute is an optimisation
+ workflow.logger.warning(f"ControlEventTypes upsert failed (non-fatal): {exc}")
+
+ # ---- spawning -------------------------------------------------------
+
async def _spawn_push_run(self, event: Dict[str, Any], spec: Dict[str, Any]) -> None:
from services.temporal.trigger_listener_workflow import (
TriggerListenerWorkflow,
@@ -184,6 +375,11 @@ class WorkflowControlWorkflow:
activity_name = f"poll.{node_type}.v{listener_data.get('version', 1)}"
params = listener_data.get("filter_params", {}) or {}
poll_interval = int(params.get("poll_interval") or _DEFAULT_POLL_INTERVAL_S)
+ if self._can_enabled:
+ # A few-second user-supplied interval overflows history in
+ # hours. Timer durations are recorded commands, so the clamp
+ # rides the same patch as continue-as-new.
+ poll_interval = max(_MIN_POLL_INTERVAL_S, poll_interval)
activity_timeout_s = max(30, poll_interval * _ACTIVITY_TIMEOUT_MULT)
seen_ids: set[str] = set(listener_data.get("seen_ids") or [])
baseline = not seen_ids
@@ -212,39 +408,62 @@ class WorkflowControlWorkflow:
start_to_close_timeout=timedelta(seconds=activity_timeout_s),
retry_policy=DEFAULT_ACTIVITY_RETRY,
)
+ except asyncio.CancelledError:
+ raise
except Exception as exc: # noqa: BLE001
workflow.logger.error(f"Controlled polling trigger failed: {exc}")
+ self._maybe_request_rollover()
continue
seen_ids = set(result.get("seen_ids") or [])
+ # Write the provider baseline back into the carried spec so a
+ # continue-as-new (which cancels this task) restarts the loop
+ # exactly where it left off instead of re-emitting old items.
+ listener_data["seen_ids"] = list(seen_ids)
for event in result.get("events") or []:
event_id = str(event.get("id") or "")
dedup_key = f"{listener_id}:{event_id}"
if not event_id or dedup_key in self._seen_event_ids:
continue
- self._seen_event_ids.add(dedup_key)
+ self._remember_event_id(dedup_key)
if self._state != "running":
await workflow.wait_condition(lambda: self._closed or self._state == "running")
if self._closed:
return
- await runner._spawn_child_run(
- event,
- listener_data,
- admission_check=(
- self._wait_until_running
- if self._use_cooperative_pause_schedule_gate
- else None
- ),
- search_attributes=(
- event_workflow_search_attributes(
- listener_data.get("workflow_id")
- )
- if self._use_cooperative_pause_schedule_gate
- else None
- ),
- )
+ try:
+ await runner._spawn_child_run(
+ event,
+ listener_data,
+ admission_check=(
+ self._wait_until_running
+ if self._use_cooperative_pause_schedule_gate
+ else None
+ ),
+ search_attributes=(
+ event_workflow_search_attributes(
+ listener_data.get("workflow_id")
+ )
+ if self._use_cooperative_pause_schedule_gate
+ else None
+ ),
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception as spawn_exc: # noqa: BLE001 — per-event isolation
+ # A failed spawn must not silently kill this poll task
+ # (the trigger would stop firing forever while the
+ # controller still reported Running).
+ workflow.logger.error(
+ f"Controlled polling spawn failed for event.id={event_id}: {spawn_exc}"
+ )
+ # Poll cycles burn history even with zero events — request
+ # rollover from here too, so a quiet mailbox cannot ride the
+ # controller into the server's hard termination ceiling.
+ self._maybe_request_rollover()
__all__ = [
+ "CONTINUE_AS_NEW_PATCH",
+ "CONTROL_EVENT_TYPES_SA_PATCH",
"COOPERATIVE_PAUSE_SCHEDULING_PATCH",
"WorkflowControlWorkflow",
]
diff --git a/server/tests/temporal/test_durability_hardening.py b/server/tests/temporal/test_durability_hardening.py
new file mode 100644
index 00000000..d5351361
--- /dev/null
+++ b/server/tests/temporal/test_durability_hardening.py
@@ -0,0 +1,749 @@
+"""Months-long durability hardening: contract tests.
+
+The product guarantee: a deployed workflow (running OR paused) survives
+backend restarts, is never auto-terminated, and keeps executing for
+months. These tests lock the mechanisms that deliver it:
+
+- No lifetime caps on new child workflow starts (replay-patched).
+- History-pressure continue-as-new on every long-lived loop (Temporal
+ hard-terminates around ~51,200 history events).
+- The controller carries its full state (triggers, queued events, dedup
+ baseline, control state) across rollovers, so it is addressed by
+ workflow id only — never a pinned run id.
+- dispatch.emit skips controllers whose deployment has no matching push
+ trigger (cross-deployment signal amplification burned every
+ controller's history budget on other deployments' traffic).
+- The startup terminate sweep's active-control guard shares the
+ canonical ACTIVE_STATES set (the old copy omitted "resetting").
+- Boot-time reconcile converges rows a crash left transitional and
+ re-arms the process-local half of running/paused generations.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+from datetime import timedelta
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from services.temporal.polling_trigger_workflow import PollingTriggerWorkflow
+from services.temporal.trigger_listener_workflow import TriggerListenerWorkflow
+from services.temporal.workflow_control_workflow import WorkflowControlWorkflow
+
+SERVER_DIR = Path(__file__).resolve().parents[2]
+
+
+# ---------------------------------------------------------------------------
+# Replay-patch guarding — every new behaviour change that alters recorded
+# commands must ride a workflow.patched marker.
+# ---------------------------------------------------------------------------
+
+
+def test_durability_changes_are_replay_patch_guarded():
+ from nodes.scheduler.cron_scheduler import _workflow as cron_module
+ from services.temporal import agent_workflow as agent_module
+ from services.temporal import polling_trigger_workflow as polling_module
+ from services.temporal import trigger_listener_workflow as trigger_module
+ from services.temporal import workflow as machina_module
+ from services.temporal import workflow_control_workflow as control_module
+ from services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflow
+ from services.temporal.workflow import MachinaWorkflow
+ from nodes.scheduler.cron_scheduler._workflow import CronTriggerWorkflow
+
+ expected = {
+ machina_module.UNBOUNDED_LIFETIMES_PATCH: inspect.getsource(MachinaWorkflow.run),
+ agent_module.UNBOUNDED_LIFETIMES_PATCH: inspect.getsource(AgentWorkflow.run),
+ trigger_module.UNBOUNDED_CHILD_RUNS_PATCH: inspect.getsource(
+ TriggerListenerWorkflow._spawn_child_run
+ ),
+ polling_module.UNBOUNDED_CHILD_RUNS_PATCH: inspect.getsource(
+ PollingTriggerWorkflow._spawn_child_run
+ ),
+ cron_module.UNBOUNDED_CHILD_RUNS_PATCH: inspect.getsource(CronTriggerWorkflow.run),
+ trigger_module.HISTORY_BOUNDED_CAN_PATCH: inspect.getsource(
+ TriggerListenerWorkflow.run
+ ),
+ polling_module.HISTORY_BOUNDED_CAN_PATCH: inspect.getsource(
+ PollingTriggerWorkflow.run
+ ),
+ control_module.CONTINUE_AS_NEW_PATCH: inspect.getsource(
+ WorkflowControlWorkflow.run
+ ),
+ trigger_module.BOUNDED_STATUS_BROADCASTS_PATCH: inspect.getsource(
+ TriggerListenerWorkflow._spawn_child_run
+ ),
+ }
+
+ assert set(expected) == {
+ "machina-unbounded-lifetimes-v1",
+ "agent-unbounded-lifetimes-v1",
+ "trigger-unbounded-child-runs-v1",
+ "polling-unbounded-child-runs-v1",
+ "cron-unbounded-child-runs-v1",
+ "trigger-history-bounded-can-v1",
+ "polling-history-bounded-can-v1",
+ "workflow-control-continue-as-new-v1",
+ "trigger-bounded-status-broadcasts-v1",
+ }
+ # DelegatedTaskWorkflow shares agent_module's marker.
+ assert "UNBOUNDED_LIFETIMES_PATCH" in inspect.getsource(DelegatedTaskWorkflow.run)
+ for patch_id, source in expected.items():
+ assert "workflow.patched(" in source
+ assert patch_id.endswith("-v1")
+
+
+# ---------------------------------------------------------------------------
+# Unbounded child runs — new executions must not carry lifetime caps.
+# ---------------------------------------------------------------------------
+
+
+def _listener_data() -> dict:
+ return {
+ "workflow_id": "wf-1",
+ "trigger_node_id": "trigger-1",
+ "node_type": "webhookTrigger",
+ "event_type": "com.opencompany.webhook.received",
+ "filter_params": {},
+ "nodes": [{"id": "trigger-1", "type": "webhookTrigger", "data": {}}],
+ "edges": [],
+ "session_id": "default",
+ "tenant_id": None,
+ }
+
+
[email protected]("patched_on", [True, False])
+async def test_listener_child_lifetime_caps_are_patch_gated(monkeypatch, patched_on):
+ from services.temporal import trigger_listener_workflow as trigger_module
+
+ temporal_workflow = trigger_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ flags = {
+ trigger_module.UNBOUNDED_CHILD_RUNS_PATCH: patched_on,
+ trigger_module.BOUNDED_STATUS_BROADCASTS_PATCH: patched_on,
+ }
+ monkeypatch.setattr(
+ temporal_workflow, "patched", lambda pid: flags.get(pid, False)
+ )
+ monkeypatch.setattr(temporal_workflow, "execute_activity", AsyncMock())
+
+ captured: dict = {}
+
+ async def start_child(name, args=None, **kwargs):
+ captured.update(kwargs)
+ captured["name"] = name
+
+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)
+
+ listener = TriggerListenerWorkflow()
+ await listener._spawn_child_run({"id": "evt-1", "data": {}}, _listener_data())
+
+ assert captured["name"] == "MachinaWorkflow"
+ if patched_on:
+ assert "execution_timeout" not in captured
+ assert "run_timeout" not in captured
+ else:
+ assert captured["execution_timeout"] == timedelta(hours=1)
+ assert captured["run_timeout"] == timedelta(hours=1)
+
+
+async def test_listener_broadcast_failure_does_not_kill_the_spawn(monkeypatch):
+ """Status broadcasts are cosmetic — a dead broadcaster must neither
+ wedge (unlimited retries) nor fail the spawn."""
+ from services.temporal import trigger_listener_workflow as trigger_module
+
+ temporal_workflow = trigger_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _pid: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "execute_activity",
+ AsyncMock(side_effect=RuntimeError("broadcaster down")),
+ )
+ started = {}
+
+ async def start_child(name, args=None, **kwargs):
+ started["id"] = kwargs.get("id")
+
+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)
+
+ listener = TriggerListenerWorkflow()
+ await listener._spawn_child_run({"id": "evt-1", "data": {}}, _listener_data())
+ assert started["id"] # the child run still started
+
+
+# ---------------------------------------------------------------------------
+# History-pressure continue-as-new.
+# ---------------------------------------------------------------------------
+
+
+class _CanFired(Exception):
+ pass
+
+
+def _pressure_info() -> SimpleNamespace:
+ return SimpleNamespace(
+ is_continue_as_new_suggested=lambda: True,
+ get_current_history_length=lambda: 0,
+ )
+
+
+async def test_polling_rolls_over_under_history_pressure_with_zero_events(monkeypatch):
+ """A quiet mailbox burns ~11 history events per cycle with the
+ processed counter frozen at 0 — the rollover must key on history
+ pressure, not on emitted events."""
+ from services.temporal import polling_trigger_workflow as polling_module
+
+ temporal_workflow = polling_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _pid: True)
+ monkeypatch.setattr(temporal_workflow, "info", _pressure_info)
+
+ sleeps: list = []
+
+ async def record_sleep(duration):
+ sleeps.append(duration)
+
+ monkeypatch.setattr(temporal_workflow, "sleep", record_sleep)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "execute_activity",
+ AsyncMock(return_value={"seen_ids": ["prior", "new"], "events": []}),
+ )
+ captured: dict = {}
+
+ def fake_can(listener_data):
+ captured["carried"] = listener_data
+ raise _CanFired
+
+ monkeypatch.setattr(temporal_workflow, "continue_as_new", fake_can)
+
+ listener_data = _listener_data()
+ listener_data["node_type"] = "googleGmailReceive"
+ listener_data["seen_ids"] = ["prior"] # non-baseline start -> sleeps
+ listener_data["filter_params"] = {"poll_interval": 5} # absurdly hot
+
+ instance = PollingTriggerWorkflow()
+ with pytest.raises(_CanFired):
+ await instance.run(listener_data)
+
+ assert instance._processed_count == 0
+ carried = captured["carried"]
+ assert set(carried["seen_ids"]) == {"prior", "new"}
+ assert carried["control_paused"] is False
+ # The 5s user interval was clamped to the defensive floor.
+ assert sleeps == [timedelta(seconds=polling_module._MIN_POLL_INTERVAL_S)]
+
+
+async def _predicate_wait(predicate):
+ while not predicate():
+ await asyncio.sleep(0)
+
+
+async def test_listener_rollover_carries_queued_events_and_pause_flag(monkeypatch):
+ from services.temporal import trigger_listener_workflow as trigger_module
+
+ temporal_workflow = trigger_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _pid: True)
+ monkeypatch.setattr(temporal_workflow, "info", _pressure_info)
+ monkeypatch.setattr(temporal_workflow, "wait_condition", _predicate_wait)
+
+ captured: dict = {}
+
+ def fake_can(args=None):
+ captured["carried"] = args[0]
+ raise _CanFired
+
+ monkeypatch.setattr(temporal_workflow, "continue_as_new", fake_can)
+
+ instance = TriggerListenerWorkflow()
+ monkeypatch.setattr(instance, "_spawn_child_run", AsyncMock())
+
+ listener_data = _listener_data()
+ # Carried queue from a previous rollover: two events pending.
+ listener_data["pending_events"] = [
+ {"id": "evt-1", "data": {}},
+ {"id": "evt-2", "data": {}},
+ ]
+
+ with pytest.raises(_CanFired):
+ await instance.run(listener_data)
+
+ carried = captured["carried"]
+ # evt-1 processed; the still-queued evt-2 carries instead of dropping.
+ assert [e["id"] for e in carried["pending_events"]] == ["evt-2"]
+ assert carried["control_paused"] is False
+ # Seeded events joined the dedup set so producer retries stay deduped.
+ assert "evt-1" in instance._seen_event_ids
+
+
+# ---------------------------------------------------------------------------
+# Controller rollover state machine.
+# ---------------------------------------------------------------------------
+
+
+def _push_spec(listener_id: str = "l-push") -> dict:
+ return {
+ "listener_id": listener_id,
+ "workflow_type": "TriggerListenerWorkflow",
+ "event_type": "t.x",
+ "event_types": ["t.x"],
+ "trigger_node_id": "n-push",
+ "listener_args": _listener_data(),
+ }
+
+
+def _poll_spec(listener_id: str = "l-poll") -> dict:
+ data = _listener_data()
+ data["node_type"] = "googleGmailReceive"
+ return {
+ "listener_id": listener_id,
+ "workflow_type": "PollingTriggerWorkflow",
+ "event_type": "t.poll",
+ "event_types": ["t.poll"],
+ "trigger_node_id": "n-poll",
+ "listener_args": data,
+ }
+
+
+async def test_controller_on_event_requests_rollover_under_pressure():
+ controller = WorkflowControlWorkflow()
+ controller._can_enabled = True
+ controller._triggers["l-push"] = _push_spec()
+ controller._history_pressure = lambda: True # type: ignore[method-assign]
+
+ await controller.on_event({"id": "evt-1", "type": "t.x"})
+
+ assert controller._events # queued
+ assert controller._can_requested is True
+
+
+def test_controller_seen_event_ids_are_bounded():
+ from services.temporal import workflow_control_workflow as control_module
+
+ controller = WorkflowControlWorkflow()
+ cap = control_module._MAX_CARRIED_SEEN_IDS
+ for index in range(cap + 10):
+ controller._remember_event_id(f"key-{index}")
+ assert len(controller._seen_event_ids) == cap
+ assert "key-0" not in controller._seen_event_ids
+ assert f"key-{cap + 9}" in controller._seen_event_ids
+
+
+async def test_controller_rollover_carries_triggers_events_and_state(monkeypatch):
+ from services.temporal import workflow_control_workflow as control_module
+
+ temporal_workflow = control_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ captured: dict = {}
+
+ def fake_can(args=None):
+ captured["carried"] = args[0]
+ raise _CanFired
+
+ monkeypatch.setattr(temporal_workflow, "continue_as_new", fake_can)
+
+ controller = WorkflowControlWorkflow()
+ controller._state = "paused" # rollover must work mid-pause
+ controller._revision = 7
+ controller._triggers["l-push"] = _push_spec()
+ controller._events.append(("l-push", {"id": "evt-9", "type": "t.x"}))
+ controller._remember_event_id("l-push:evt-9")
+ # A live poll task the rollover must cancel cleanly.
+ blocker = asyncio.Event()
+
+ async def parked_poll():
+ try:
+ await blocker.wait()
+ except asyncio.CancelledError:
+ raise
+
+ controller._poll_tasks["l-poll"] = asyncio.create_task(parked_poll())
+
+ with pytest.raises(_CanFired):
+ await controller._continue_as_new({"generation": 3, "workflow_id": "wf-1"})
+
+ carried = captured["carried"]
+ assert carried["state"] == "paused"
+ assert carried["revision"] == 7
+ assert carried["generation"] == 3
+ assert "l-push" in carried["triggers"]
+ assert [event["id"] for _lid, event in carried["pending_events"]] == ["evt-9"]
+ assert "l-push:evt-9" in carried["seen_event_ids"]
+ assert controller._poll_tasks["l-poll"].cancelled()
+
+
+async def test_controller_seeds_carried_state_and_restarts_poll_loops(monkeypatch):
+ controller = WorkflowControlWorkflow()
+ started: list = []
+
+ async def fake_poll(listener_id, spec):
+ started.append(listener_id)
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(controller, "_poll_trigger", fake_poll)
+
+ controller._seed_carried_state(
+ {
+ "revision": 4,
+ "seen_event_ids": ["l-push:evt-1"],
+ "pending_events": [["l-push", {"id": "evt-2", "type": "t.x"}]],
+ "triggers": {"l-push": _push_spec(), "l-poll": _poll_spec()},
+ }
+ )
+ await asyncio.sleep(0) # let the created task start
+
+ assert controller._revision == 4
+ assert "l-push:evt-1" in controller._seen_event_ids
+ assert controller._events == [("l-push", {"id": "evt-2", "type": "t.x"})]
+ assert set(controller._triggers) == {"l-push", "l-poll"}
+ assert started == ["l-poll"] # only polling triggers get loops
+ controller._poll_tasks["l-poll"].cancel()
+
+
+async def test_controller_poll_loop_writes_seen_ids_back_into_the_spec(monkeypatch):
+ """The provider baseline must live in the carried spec so a rollover
+ restarts the poll loop exactly where it left off."""
+ from services.temporal import workflow_control_workflow as control_module
+
+ temporal_workflow = control_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "wait_condition", _predicate_wait)
+ monkeypatch.setattr(temporal_workflow, "sleep", AsyncMock())
+
+ controller = WorkflowControlWorkflow()
+ controller._can_enabled = False
+ spec = _poll_spec()
+ controller._triggers[spec["listener_id"]] = spec
+
+ async def poll_activity(*_args, **_kwargs):
+ controller._closed = True # one cycle, then exit
+ return {"seen_ids": ["provider-1"], "events": []}
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", poll_activity)
+
+ await controller._poll_trigger(spec["listener_id"], spec)
+ assert spec["listener_args"]["seen_ids"] == ["provider-1"]
+
+
+async def test_controller_push_spawn_failure_does_not_kill_the_run_loop(monkeypatch):
+ from services.temporal import workflow_control_workflow as control_module
+
+ temporal_workflow = control_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "wait_condition", _predicate_wait)
+ monkeypatch.setattr(temporal_workflow, "in_workflow", lambda: False)
+
+ controller = WorkflowControlWorkflow()
+ calls = {"n": 0}
+
+ async def failing_spawn(_event, _spec):
+ calls["n"] += 1
+ if calls["n"] == 1:
+ raise RuntimeError("duplicate child id")
+ controller._closed = True
+
+ monkeypatch.setattr(controller, "_spawn_push_run", failing_spawn)
+ controller._triggers["l-push"] = _push_spec()
+ controller._events.append(("l-push", {"id": "evt-1", "type": "t.x"}))
+ controller._events.append(("l-push", {"id": "evt-2", "type": "t.x"}))
+
+ result = await controller.run({"generation": 1, "state": "running"})
+ # First spawn failed, second still ran; the controller survived both.
+ assert calls["n"] == 2
+ assert result["generation"] == 1
+
+
+# ---------------------------------------------------------------------------
+# dispatch.emit controller narrowing.
+# ---------------------------------------------------------------------------
+
+
+def test_dispatch_controller_helpers_read_both_attribute_shapes():
+ from services.events.dispatch import _controller_event_types, _is_controller
+
+ controller = SimpleNamespace(workflow_type="WorkflowControlWorkflow")
+ listener = SimpleNamespace(workflow_type="TriggerListenerWorkflow")
+ assert _is_controller(controller) is True
+ assert _is_controller(listener) is False
+
+ typed = SimpleNamespace(
+ typed_search_attributes=[
+ SimpleNamespace(
+ key=SimpleNamespace(name="ControlEventTypes"),
+ value=["a.b", "c.d"],
+ )
+ ],
+ search_attributes=None,
+ )
+ assert _controller_event_types(typed) == {"a.b", "c.d"}
+
+ plain = SimpleNamespace(
+ typed_search_attributes=None,
+ search_attributes={"ControlEventTypes": ["x.y"]},
+ )
+ assert _controller_event_types(plain) == {"x.y"}
+
+ absent = SimpleNamespace(typed_search_attributes=None, search_attributes=None)
+ assert _controller_event_types(absent) is None # pre-upgrade: match-all
+
+
+def test_control_event_types_search_attribute_is_registered():
+ from services.temporal.search_attributes import EVENT_SEARCH_ATTRIBUTES
+
+ names = {spec.name for spec in EVENT_SEARCH_ATTRIBUTES}
+ assert "ControlEventTypes" in names
+
+
+# ---------------------------------------------------------------------------
+# Terminate-sweep guard + controller addressing.
+# ---------------------------------------------------------------------------
+
+
+def test_active_states_are_shared_and_include_resetting():
+ from models.database import WORKFLOW_CONTROL_ACTIVE_STATES
+ from services.deployment.control import ACTIVE_STATES
+
+ assert ACTIVE_STATES is WORKFLOW_CONTROL_ACTIVE_STATES
+ assert "resetting" in ACTIVE_STATES
+ # core.database is conftest-stubbed, so verify the real source text
+ # references the shared constant instead of a hand-copied set.
+ database_source = (SERVER_DIR / "core" / "database.py").read_text(encoding="utf-8")
+ assert "WORKFLOW_CONTROL_ACTIVE_STATES" in database_source
+ assert '"starting", "running", "pausing", "paused", "resuming"}' not in database_source
+
+
+def test_controller_handles_are_never_run_id_pinned():
+ """Continue-as-new mints new run ids under the same workflow id; a
+ pinned handle would target a closed run after the first rollover."""
+ from services.deployment import handlers, manager
+
+ handle_source = inspect.getsource(handlers._controller_handle)
+ assert "get_workflow_handle(control.controller_workflow_id)" in handle_source
+
+ manager_source = inspect.getsource(manager)
+ assert 'run_id=control.controller_run_id' not in manager_source
+
+
+def test_terminate_flag_defaults_false_in_both_env_files():
+ for env_name in (".env.template", ".env"):
+ text = (SERVER_DIR.parent / env_name).read_text(encoding="utf-8")
+ assert "TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false" in text
+ assert "TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=true" not in text
+
+
+# ---------------------------------------------------------------------------
+# Boot-time reconcile.
+# ---------------------------------------------------------------------------
+
+
+def _boot_control(status: str, *, graph_nodes=None):
+ return SimpleNamespace(
+ id="workflow-control:wf:1",
+ workflow_id="wf",
+ generation=1,
+ execution_id="execution-1",
+ root_execution_id="execution-1",
+ data_scope_id="execution-1",
+ controller_workflow_id="workflow-control-wf-g1",
+ controller_run_id="controller-run-1",
+ status=status,
+ revision=3,
+ graph_snapshot={
+ "nodes": graph_nodes
+ if graph_nodes is not None
+ else [{"id": "t-1", "type": "webhookTrigger", "data": {}}],
+ "edges": [],
+ },
+ created_at=None,
+ updated_at=None,
+ terminal_reason=None,
+ )
+
+
+async def test_interrupted_start_with_registered_triggers_converges_to_running(
+ monkeypatch,
+):
+ from services.deployment import handlers
+
+ control = _boot_control("starting")
+ running = _boot_control("running")
+ service = SimpleNamespace(
+ transition=AsyncMock(return_value=running),
+ fail=AsyncMock(),
+ database=SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=control)
+ ),
+ )
+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))
+
+ result = await handlers._converge_interrupted_start(
+ service, control, {"state": "running", "triggers": {"l-1": "n-1"}}
+ )
+
+ assert result.status == "running"
+ assert service.transition.await_args.kwargs["status"] == "running"
+ service.fail.assert_not_awaited()
+
+
+async def test_interrupted_start_with_empty_controller_fails_and_closes_it(
+ monkeypatch,
+):
+ from services.deployment import handlers
+
+ control = _boot_control("starting")
+ failed = _boot_control("failed")
+ service = SimpleNamespace(
+ transition=AsyncMock(),
+ fail=AsyncMock(return_value=failed),
+ database=SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=control)
+ ),
+ )
+ reset_signal = AsyncMock()
+ monkeypatch.setattr(handlers, "_signal_controller", reset_signal)
+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))
+
+ result = await handlers._converge_interrupted_start(
+ service, control, {"state": "running", "triggers": {}}
+ )
+
+ assert result.status == "failed"
+ assert service.fail.await_args.args[1] == "interrupted_start"
+ reset_signal.assert_awaited_once_with(control, "reset")
+ service.transition.assert_not_awaited()
+
+
+async def test_interrupted_start_of_triggerless_graph_converges_to_running(
+ monkeypatch,
+):
+ from services.deployment import handlers
+
+ control = _boot_control(
+ "starting",
+ graph_nodes=[{"id": "a-1", "type": "aiAgent", "data": {}}],
+ )
+ running = _boot_control("running")
+ service = SimpleNamespace(
+ transition=AsyncMock(return_value=running),
+ fail=AsyncMock(),
+ database=SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=control)
+ ),
+ )
+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))
+
+ result = await handlers._converge_interrupted_start(
+ service, control, {"state": "running", "triggers": {}}
+ )
+ assert result.status == "running"
+ service.fail.assert_not_awaited()
+
+
+async def test_unreachable_controller_leaves_starting_row_untouched():
+ from services.deployment import handlers
+
+ control = _boot_control("starting")
+ service = SimpleNamespace(transition=AsyncMock(), fail=AsyncMock())
+ result = await handlers._converge_interrupted_start(service, control, None)
+ assert result is control
+ service.transition.assert_not_awaited()
+ service.fail.assert_not_awaited()
+
+
+async def test_rearm_restores_paused_posture_from_snapshot(monkeypatch):
+ from core.container import container
+ from services.deployment import handlers
+
+ control = _boot_control("paused")
+ workflow_service = MagicMock()
+ workflow_service.is_workflow_deployed.return_value = False
+ workflow_service.pause_deployment = MagicMock()
+ workflow_service.update_trigger_pause_status = AsyncMock(return_value=1)
+ monkeypatch.setattr(
+ container, "workflow_service", MagicMock(return_value=workflow_service)
+ )
+ deploy = AsyncMock(return_value={"success": True})
+ setup = AsyncMock(return_value={"success": True})
+ cron_pause = AsyncMock(return_value=0)
+ monkeypatch.setattr(handlers, "handle_deploy_workflow", deploy)
+ monkeypatch.setattr(handlers, "_await_deployment_setup", setup)
+ monkeypatch.setattr(handlers, "_set_cron_pause", cron_pause)
+
+ await handlers._rearm_generation(control)
+
+ deploy_data = deploy.await_args.args[0]
+ assert deploy_data["workflow_id"] == "wf"
+ assert deploy_data["nodes"] == control.graph_snapshot["nodes"]
+ # Generation-scoped persistence, same contract as handle_start_workflow.
+ assert deploy_data["session_id"] == "execution-1"
+ workflow_service.pause_deployment.assert_called_once_with("wf")
+ workflow_service.update_trigger_pause_status.assert_awaited_once_with(
+ "wf", paused=True
+ )
+ cron_pause.assert_awaited_once_with("wf", paused=True, strict=False)
+
+
+async def test_rearm_skips_already_deployed_workflows(monkeypatch):
+ from core.container import container
+ from services.deployment import handlers
+
+ control = _boot_control("running")
+ workflow_service = MagicMock()
+ workflow_service.is_workflow_deployed.return_value = True
+ monkeypatch.setattr(
+ container, "workflow_service", MagicMock(return_value=workflow_service)
+ )
+ deploy = AsyncMock()
+ monkeypatch.setattr(handlers, "handle_deploy_workflow", deploy)
+
+ await handlers._rearm_generation(control)
+ deploy.assert_not_awaited()
+
+
+async def test_boot_reconcile_processes_each_active_row_with_isolation(monkeypatch):
+ from core.container import container
+ from services.deployment import handlers
+
+ healthy = _boot_control("running")
+ broken = _boot_control("running")
+ broken.workflow_id = "wf-broken"
+ database = SimpleNamespace(
+ list_active_workflow_controls=AsyncMock(return_value=[broken, healthy]),
+ )
+ monkeypatch.setattr(container, "database", MagicMock(return_value=database))
+
+ async def reconcile(_service, control):
+ if control.workflow_id == "wf-broken":
+ raise RuntimeError("temporal hiccup")
+ return control, {"state": "running"}
+
+ rearm = AsyncMock()
+ monkeypatch.setattr(handlers, "_reconcile_control", reconcile)
+ monkeypatch.setattr(handlers, "_rearm_generation", rearm)
+
+ processed = await handlers.reconcile_active_controls_on_boot()
+
+ # The broken row was isolated; the healthy one was re-armed.
+ assert processed == 1
+ rearm.assert_awaited_once_with(healthy)
diff --git a/server/tests/test_deployment_canary_listener.py b/server/tests/test_deployment_canary_listener.py
index a6a94a5e..2b9161d0 100644
--- a/server/tests/test_deployment_canary_listener.py
+++ b/server/tests/test_deployment_canary_listener.py
@@ -305,8 +305,11 @@ class TestStartCanaryListener:
assert listener_id == "wf-controlled-webhookTrigger"
client.start_workflow.assert_not_awaited()
+ # Addressed by workflow id only — the controller continue-as-news
+ # to bound its history, so a run_id-pinned handle would target a
+ # closed run after the first rollover.
client.get_workflow_handle.assert_called_once_with(
- "workflow-control-wf-controlled-g1", run_id="controller-run-1",
+ "workflow-control-wf-controlled-g1",
)
signal_name, spec = controller_handle.signal.await_args.args
assert signal_name == "register_trigger"