Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(control): continue-as-new and boot reconcile keep deployments alive for months
trohitg34fa9afd1b1f
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.pyindex 5f753305..eecf8f36 100644--- a/server/services/deployment/handlers.py+++ b/server/services/deployment/handlers.py@@ -452,15 +452,22 @@ async def _start_controller(control) -> Optional[str]:def _controller_handle(control):+ """Handle addressed by workflow id only — never pinned to a run id.++ The controller keeps its event history bounded via continue-as-new,+ which mints a new run id under the same workflow id. A handle pinned+ to ``control.controller_run_id`` (the FIRST run) would target a+ closed run after the first rollover and every pause/resume/update/+ query would fail. The workflow id already carries the generation+ (``workflow-control-<wf>-g<N>``), so unpinned addressing cannot+ reach a different generation's controller.+ """from core.container import containerwrapper = container.temporal_client()if wrapper is None or wrapper.client is None or not control.controller_workflow_id:return None- return wrapper.client.get_workflow_handle(- control.controller_workflow_id,- run_id=control.controller_run_id,- )+ return wrapper.client.get_workflow_handle(control.controller_workflow_id)async def _signal_controller(control, signal_name: str) -> None:@@ -968,6 +975,179 @@ async def _reconcile_control(service: WorkflowControlService, control):return control, controller_status+# ---------------------------------------------------------------------------+# Boot-time reconcile — called once from services.temporal.lifecycle after+# the workers start. Request-time reconciliation stays lazy; this pass closes+# the unattended-server window where durable intent (running/paused rows) and+# runtime behaviour could diverge indefinitely after a backend restart.+# ---------------------------------------------------------------------------+++async def reconcile_active_controls_on_boot() -> int:+ """Converge every active durable control after a backend restart.++ Three responsibilities per active row:++ 1. Run the same lazy :func:`_reconcile_control` used by status reads+ (finishes interrupted pause/resume transitions, fails rows whose+ controller vanished).+ 2. Converge ``starting`` rows — the one state the lazy path never+ touches while its controller exists, so a crash mid-start used to+ wedge the workflow behind a permanent ``workflow_start_pending``.+ Boot is the only moment guaranteed free of a concurrent in-process+ start, which is why this lives here and not in the lazy path.+ 3. Re-arm the process-local half of running/paused generations from+ the persisted graph snapshot: the controller and its Temporal+ children survive a restart on their own, but DeploymentManager+ state (runtime counts, admission, pause fan-in) and in-process+ collectors for non-canary trigger types die with the process and+ previously stayed dead while the row reported ``running`` forever.++ Returns the number of rows processed; per-row failures are logged and+ skipped so one broken generation cannot block the rest.+ """+ from core.container import container++ database = container.database()+ service = WorkflowControlService(database)+ controls = await database.list_active_workflow_controls()+ processed = 0+ for control in controls:+ try:+ control, controller_status = await _reconcile_control(service, control)+ if control.status == "starting":+ control = await _converge_interrupted_start(+ service, control, controller_status+ )+ if control.status in {"running", "paused"}:+ await _rearm_generation(control)+ processed += 1+ except Exception as exc: # noqa: BLE001 — per-row isolation+ logger.warning(+ "Boot reconcile failed for workflow control",+ workflow_id=control.workflow_id,+ status=control.status,+ error=str(exc),+ )+ return processed+++async def _converge_interrupted_start(+ service: WorkflowControlService,+ control,+ controller_status: Optional[Dict[str, Any]],+):+ """Decide what a crash left behind for a row stuck in ``starting``.++ A vanished controller was already failed by ``_reconcile_control``.+ With the controller alive: triggers registered (or a triggerless+ graph) means the deploy effectively completed — commit ``running``.+ A live-but-empty controller for a graph that declares triggers means+ the crash landed before registration — fail the generation (Reset ++ Start rebuilds cleanly) and close the orphan controller so dispatch+ stops signalling it.+ """+ if controller_status is None:+ # Controller unreachable (not missing) — leave transitional; the+ # lazy path retries when Temporal comes back.+ return control++ from constants import WORKFLOW_TRIGGER_TYPES++ triggers_registered = len(controller_status.get("triggers") or {})+ graph_nodes = (control.graph_snapshot or {}).get("nodes") or []+ graph_has_triggers = any(+ node.get("type") in WORKFLOW_TRIGGER_TYPES for node in graph_nodes+ )+ if triggers_registered or not graph_has_triggers:+ try:+ control = await service.transition(+ control,+ expected_revision=control.revision,+ from_statuses={"starting"},+ status="running",+ )+ await _broadcast_control(control)+ except ValueError:+ # Concurrent writer won the CAS; report what the DB has.+ latest = await service.database.get_latest_workflow_control(control.workflow_id)+ if latest is not None and latest.generation == control.generation:+ control = latest+ return control++ try:+ await _signal_controller(control, "reset")+ except Exception as exc: # noqa: BLE001 — best-effort close+ logger.warning(+ "Failed to close orphan controller for interrupted start",+ workflow_id=control.workflow_id,+ error=str(exc),+ )+ try:+ failed = await service.fail(control, "interrupted_start")+ except ValueError:+ return control+ await _broadcast_control(failed)+ return failed+++async def _rearm_generation(control) -> None:+ """Re-establish process-local deployment state for a durable generation.++ Idempotent by construction: canary triggers re-register with the+ controller keyed by listener id (no-op when already known), legacy+ listener starts use ``id_conflict_policy=USE_EXISTING``, and cron+ schedule creation preserves server-owned paused state on redeploy.+ In-process collectors for non-canary trigger types are re-armed+ fresh (they died with the previous process).+ """+ from core.container import container++ workflow_service = container.workflow_service()+ if workflow_service.is_workflow_deployed(control.workflow_id):+ return+ snapshot = control.graph_snapshot or {}+ nodes = snapshot.get("nodes") or []+ edges = snapshot.get("edges") or []+ if not nodes:+ logger.warning(+ "Cannot re-arm generation without a graph snapshot",+ workflow_id=control.workflow_id,+ generation=control.generation,+ )+ return+ deploy_data = {+ "workflow_id": control.workflow_id,+ "nodes": nodes,+ "edges": edges,+ # Runtime persistence is generation-scoped (same contract as+ # handle_start_workflow's deploy call).+ "session_id": control.data_scope_id or control.execution_id,+ "execution_id": control.execution_id,+ "root_execution_id": control.root_execution_id,+ }+ deployed = await handle_deploy_workflow(deploy_data, None)+ if not deployed.get("success"):+ raise RuntimeError(str(deployed.get("error", "deployment_failed")))+ setup = await _await_deployment_setup(control.workflow_id)+ if not setup.get("success"):+ raise RuntimeError(str(setup.get("error", "deployment_setup_failed")))+ if control.status == "paused":+ # Restore the paused posture on the freshly re-armed local half.+ workflow_service.pause_deployment(control.workflow_id)+ await workflow_service.update_trigger_pause_status(+ control.workflow_id,+ paused=True,+ )+ await _set_cron_pause(control.workflow_id, paused=True, strict=False)+ logger.info(+ "Re-armed durable generation after restart",+ workflow_id=control.workflow_id,+ generation=control.generation,+ status=control.status,+ )++async def _await_deployment_setup(workflow_id: str) -> Dict[str, Any]:"""Wait for trigger/listener setup started by the legacy deploy handler."""task = _deployment_tasks.get(workflow_id)diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.pyindex a2eada8b..864d23bf 100644--- a/server/services/deployment/manager.py+++ b/server/services/deployment/manager.py@@ -823,8 +823,11 @@ class DeploymentManager:# The controller owns listeners as child workflows. This keeps# trigger activity in the deployment's Temporal execution tree# instead of presenting each listener as an unrelated root trace.+ # Addressed by workflow id only (never run_id-pinned): the+ # controller continue-as-news to bound its history, and a+ # pinned handle would target a closed run after rollover.await wrapper.client.get_workflow_handle(- control.controller_workflow_id, run_id=control.controller_run_id,+ control.controller_workflow_id,).signal("register_trigger", {"listener_id": listener_id,"workflow_type": workflow_type_name,diff --git a/server/services/events/dispatch.py b/server/services/events/dispatch.pyindex d7815a56..b3edcda3 100644--- a/server/services/events/dispatch.py+++ b/server/services/events/dispatch.py@@ -138,9 +138,29 @@ async def _signal_running_consumers(event: WorkflowEvent) -> None:logger.warning(f"emit: Visibility query failed (query={query!r}): {exc}")return- if not consumers:+ # Every signal is immutable history in the receiving workflow —+ # controllers used to receive EVERY platform event (matching or not)+ # and filter in-workflow, so one busy deployment burned down every+ # other controller's continue-as-new budget. Controllers advertise+ # their push event types via the ControlEventTypes Search Attribute+ # (upserted as triggers register); skip the ones with no match.+ # Controllers without the attribute (pre-upgrade histories) keep the+ # legacy match-all behaviour.+ matched_types = {str(t) for t in event_types}+ targets = []+ skipped = 0+ for wf in consumers:+ if _is_controller(wf):+ declared = _controller_event_types(wf)+ if declared is not None and not (declared & matched_types):+ skipped += 1+ continue+ targets.append(wf)++ if not targets:logger.debug(- f"emit: 0 consumers for event.type={event.type!r} " f"(event.id={event.id})",+ f"emit: 0 consumers for event.type={event.type!r} "+ f"(event.id={event.id}, controllers_skipped={skipped})",)return@@ -148,17 +168,60 @@ async def _signal_running_consumers(event: WorkflowEvent) -> None:# block other consumers (return_exceptions=True surfaces failures# as exceptions in the result list).signal_results = await asyncio.gather(- *[_signal_one(client, wf.id, event) for wf in consumers],+ *[_signal_one(client, wf.id, event) for wf in targets],return_exceptions=True,)delivered = sum(1 for r in signal_results if not isinstance(r, Exception))failed = len(signal_results) - deliveredlogger.info(- f"emit: signalled {delivered}/{len(consumers)} consumers for " f"event.type={event.type!r} (failed={failed}, event.id={event.id})",+ f"emit: signalled {delivered}/{len(targets)} consumers for "+ f"event.type={event.type!r} (failed={failed}, "+ f"controllers_skipped={skipped}, event.id={event.id})",)+def _is_controller(wf: object) -> bool:+ """True when a Visibility-listed execution is a WorkflowControlWorkflow."""+ wf_type = getattr(wf, "workflow_type", None)+ wf_type = getattr(wf_type, "name", None) or wf_type+ return str(wf_type) == "WorkflowControlWorkflow"+++def _controller_event_types(wf: object) -> Optional[set]:+ """Read the ControlEventTypes keyword-list Search Attribute off a+ Visibility-listed execution.++ ``None`` means the attribute is absent (pre-upgrade controller or a+ controller whose upsert failed) — callers treat that as match-all so+ narrowing can never silently disconnect a live deployment. The SDK's+ listed-execution shape varies across versions (typed pairs vs plain+ dict), so both forms are read defensively.+ """+ try:+ typed = getattr(wf, "typed_search_attributes", None)+ if typed is not None:+ for pair in typed:+ key = getattr(getattr(pair, "key", None), "name", None)+ if key != "ControlEventTypes":+ continue+ value = getattr(pair, "value", None)+ if value is None:+ return None+ values = value if isinstance(value, (list, tuple)) else [value]+ return {str(v) for v in values}+ raw = getattr(wf, "search_attributes", None)+ if isinstance(raw, dict) and "ControlEventTypes" in raw:+ value = raw.get("ControlEventTypes")+ if value is None:+ return None+ values = value if isinstance(value, (list, tuple)) else [value]+ return {str(v) for v in values}+ except Exception: # noqa: BLE001 — unreadable attribute == match-all+ return None+ return None++async def _signal_one(client, workflow_id: str, event: WorkflowEvent) -> None:"""Send the ``on_event`` signal to one workflow.diff --git a/server/services/temporal/polling_trigger_workflow.py b/server/services/temporal/polling_trigger_workflow.pyindex 2fa18be2..7b2c9acd 100644--- a/server/services/temporal/polling_trigger_workflow.py+++ b/server/services/temporal/polling_trigger_workflow.py@@ -60,6 +60,27 @@ _ACTIVITY_TIMEOUT_MULT = 4COOPERATIVE_PAUSE_SCHEDULING_PATCH = ("polling-trigger-cooperative-pause-scheduling-v1")+# Same contract as trigger_listener_workflow.UNBOUNDED_CHILD_RUNS_PATCH:+# child runs may execute or stay paused for months, so new executions+# start them without lifetime caps; the patch keeps replay+# compatibility for histories recorded with the old 1h caps.+UNBOUNDED_CHILD_RUNS_PATCH = "polling-unbounded-child-runs-v1"+# _processed_count above increments only per EMITTED event, but every+# poll cycle burns ~8-11 history events (sleep timer + activity ++# workflow tasks) regardless — a quiet mailbox at the 60s default+# reached the server's ~51,200-event hard termination in ~3 days with+# the counter frozen at 0 and continue-as-new never firing. The patched+# path checks history pressure every cycle (server suggestion + the+# soft cap below), clamps user-supplied poll intervals to a sane floor,+# and carries the pause flag across the rollover instead of blocking it.+HISTORY_BOUNDED_CAN_PATCH = "polling-history-bounded-can-v1"+_HISTORY_SOFT_CAP = 10_000+# Defensive floor for user-supplied poll intervals: a few-second+# interval burns ~100K+ history events/day. Mirrors the plugin-side+# clamp (PollingTriggerNode.poll_interval_clamp) that the legacy+# asyncio path applies but workflow payloads historically did not.+# Timer durations are recorded commands, so the clamp rides the patch.+_MIN_POLL_INTERVAL_S = 30@workflow.defn(name="PollingTriggerWorkflow", sandboxed=False)@@ -121,13 +142,22 @@ class PollingTriggerWorkflow:use_cooperative_pause_schedule_gate = workflow.patched(COOPERATIVE_PAUSE_SCHEDULING_PATCH)+ use_history_bounded_can = workflow.patched(HISTORY_BOUNDED_CAN_PATCH)except Exception: # direct unit invocation outside Temporal runtimeif workflow.in_workflow():raiseuse_cooperative_pause_schedule_gate = False+ use_history_bounded_can = True++ # Rehydrate rollover-carried pause state. Command-free, so+ # pre-patch histories (whose payloads lack the key) replay+ # identically.+ self._control_paused = bool(listener_data.get("control_paused"))params = listener_data.get("filter_params", {}) or {}poll_interval = int(params.get("poll_interval") or _DEFAULT_POLL_INTERVAL_S)+ if use_history_bounded_can:+ poll_interval = max(_MIN_POLL_INTERVAL_S, poll_interval)activity_timeout_s = max(30, poll_interval * _ACTIVITY_TIMEOUT_MULT)# Carry seen_ids across continueAsNew. First-start payload has@@ -223,9 +253,21 @@ class PollingTriggerWorkflow:workflow.logger.error(f"PollingTriggerWorkflow spawn failed for event.id={event_id}: {spawn_exc}")self._processed_count += 1- if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:+ should_rollover = self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW+ if use_history_bounded_can and not should_rollover:+ # Poll cycles burn history even with zero emitted events —+ # check pressure every cycle, not only per processed event.+ from services.temporal.trigger_listener_workflow import _history_pressure++ should_rollover = _history_pressure(_HISTORY_SOFT_CAP)+ if should_rollover:workflow.logger.info(f"PollingTriggerWorkflow continue_as_new: processed={self._processed_count}")- if use_cooperative_pause_schedule_gate:+ if use_history_bounded_can:+ # Carry the pause flag instead of blocking the rollover+ # behind a resume that may be months away (signal+ # traffic would overflow the history mid-pause).+ listener_data["control_paused"] = self._control_paused+ elif use_cooperative_pause_schedule_gate:# A pause may arrive during the final waiting-status# broadcast. Continue only after Resume so the next run# cannot reset a still-active pause flag.@@ -252,7 +294,9 @@ class PollingTriggerWorkflow:nodes) stay single-source. Mirrors:meth:`TriggerListenerWorkflow._spawn_child_run` exactly."""+ from services.temporal._retry_policies import QUICK_ACTIVITY_RETRYfrom services.temporal.trigger_listener_workflow import (+ BOUNDED_STATUS_BROADCASTS_PATCH,_broadcast_trigger_idle,_broadcast_trigger_waiting,_build_run_graph,@@ -313,12 +357,24 @@ class PollingTriggerWorkflow:event_id = event.get("id") or workflow.uuid4().hexchild_id = f"{workflow_slug}-{trigger_label}-{event_id}"- await _broadcast_trigger_idle(- node_id=trigger_node_id,- workflow_id=workflow_id,- event_id=event_id,- event_type=listener_data.get("event_type", ""),- )+ try:+ unbounded_child_runs = workflow.patched(UNBOUNDED_CHILD_RUNS_PATCH)+ bounded_broadcasts = workflow.patched(BOUNDED_STATUS_BROADCASTS_PATCH)+ except RuntimeError: # direct unit invocation outside Temporal runtime+ unbounded_child_runs = True+ bounded_broadcasts = True+ broadcast_retry = QUICK_ACTIVITY_RETRY if bounded_broadcasts else None++ try:+ await _broadcast_trigger_idle(+ node_id=trigger_node_id,+ workflow_id=workflow_id,+ event_id=event_id,+ event_type=listener_data.get("event_type", ""),+ retry_policy=broadcast_retry,+ )+ except Exception as exc: # noqa: BLE001 — cosmetic UI signalling+ workflow.logger.warning(f"Trigger status broadcast failed (non-fatal): {exc}")if admission_check is not None:await admission_check()@@ -329,9 +385,12 @@ class PollingTriggerWorkflow:"id_reuse_policy": (WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY),- "execution_timeout": timedelta(hours=1),- "run_timeout": timedelta(hours=1),}+ if not unbounded_child_runs:+ # Replay-only: pre-patch histories recorded 1h caps on the+ # child-start command. See UNBOUNDED_CHILD_RUNS_PATCH.+ child_options["execution_timeout"] = timedelta(hours=1)+ child_options["run_timeout"] = timedelta(hours=1)if search_attributes is not None:child_options["search_attributes"] = search_attributes@@ -352,12 +411,16 @@ class PollingTriggerWorkflow:workflow.logger.info(f"PollingTriggerWorkflow spawned child run: child_id={child_id} " f"event.id={event.get('id')}")- await _broadcast_trigger_waiting(- node_id=trigger_node_id,- workflow_id=workflow_id,- event_type=listener_data.get("event_type", ""),- processed_count=self._processed_count + 1,- )+ try:+ await _broadcast_trigger_waiting(+ node_id=trigger_node_id,+ workflow_id=workflow_id,+ event_type=listener_data.get("event_type", ""),+ processed_count=self._processed_count + 1,+ retry_policy=broadcast_retry,+ )+ except Exception as exc: # noqa: BLE001 — cosmetic UI signalling+ workflow.logger.warning(f"Trigger status broadcast failed (non-fatal): {exc}")__all__ = ["PollingTriggerWorkflow"]diff --git a/server/services/temporal/search_attributes.py b/server/services/temporal/search_attributes.pyindex e1bcc117..75032d94 100644--- a/server/services/temporal/search_attributes.py+++ b/server/services/temporal/search_attributes.py@@ -1,6 +1,6 @@"""Wave 12 A4: Temporal custom Search Attributes registration.-Defines the 6 Search Attributes the event framework needs for the+Defines the Search Attributes the event framework needs for theVisibility API to route inbound events to running consumer workflows(see ``services/events/dispatch.py:emit``). Registration is idempotent —Temporal Server's ``add_search_attributes`` rejects duplicates with an@@ -42,7 +42,7 @@ class SearchAttributeSpec:description: str-# Single source of truth for the 6 event-framework Search Attributes.+# Single source of truth for the event-framework Search Attributes.# Anything that names or filters by these attributes (the dispatch# helper, the Visibility queries in admin handlers, the test suite)# imports from this list rather than restating the strings.@@ -103,6 +103,20 @@ EVENT_SEARCH_ATTRIBUTES: Sequence[SearchAttributeSpec] = ("hour')."),),+ SearchAttributeSpec(+ name="ControlEventTypes",+ indexed_type=IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD_LIST,+ description=(+ "CloudEvents types a WorkflowControlWorkflow's registered push "+ "triggers listen for (upserted by the controller as triggers "+ "register). dispatch.emit reads it to skip signalling "+ "controllers whose deployment has no matching trigger — "+ "otherwise every platform event writes ~4 history events into "+ "EVERY running controller and burns its continue-as-new budget "+ "on other deployments' traffic. Controllers without the "+ "attribute (pre-upgrade histories) are treated as match-all."+ ),+ ),)diff --git a/server/services/temporal/trigger_listener_workflow.py b/server/services/temporal/trigger_listener_workflow.pyindex 00d2ee8b..8f04fcf8 100644--- a/server/services/temporal/trigger_listener_workflow.py+++ b/server/services/temporal/trigger_listener_workflow.py@@ -35,6 +35,7 @@ from typing import Any, Dict, List, Optional, Setfrom temporalio import workflowfrom temporalio.common import (+ RetryPolicy,SearchAttributeKey,SearchAttributePair,TypedSearchAttributes,@@ -54,6 +55,53 @@ CHILD_SEARCH_ATTRIBUTES_PATCH = "trigger-child-search-attributes-v1"COOPERATIVE_PAUSE_SCHEDULING_PATCH = ("trigger-cooperative-pause-scheduling-v1")+# Child MachinaWorkflow runs used to carry 1h execution/run timeouts.+# A run may legitimately execute — or stay cooperatively paused — for+# months, and Temporal's timeout timer keeps ticking through a pause,+# so any pause longer than the cap silently terminated the run.+# Liveness is enforced at the activity layer (heartbeats), not by+# workflow lifetime caps; new executions start children unbounded+# (Temporal's default). The patch keeps replay compatibility for+# histories whose recorded child starts carried the old caps.+UNBOUNDED_CHILD_RUNS_PATCH = "trigger-unbounded-child-runs-v1"+# The _processed_count threshold above was calibrated assuming ~2-3+# history events per firing, but the real spawn path costs ~15-25+# (signal + two broadcast activities + optional graph-load + child+# start/close + workflow tasks) — the server's ~51,200-event hard+# termination fires around 2,000-3,500 firings, long before the+# 16,000-processed checkpoint could. The patched path rolls over on+# the server's own is_continue_as_new_suggested signal (with the soft+# cap below as a deterministic backstop), carries still-queued events+# instead of dropping them, and carries the pause flag instead of+# blocking the rollover behind a resume that may be months away.+HISTORY_BOUNDED_CAN_PATCH = "trigger-history-bounded-can-v1"+_HISTORY_SOFT_CAP = 10_000+# Continue-as-new argument blobs are capped at 2MiB; carry at most this+# many still-queued events across a rollover (oldest dropped, logged).+_MAX_CARRIED_EVENTS = 256+# Status broadcasts are cosmetic UI signalling. Without an explicit+# policy Temporal retries a failing activity forever, wedging the+# serialized spawn loop behind a dead broadcaster. Bounded + swallowed+# on the patched path (BOUNDED_STATUS_BROADCASTS_PATCH gates the+# command-visible retry-policy change; the try/except is command-free).+# Policy: the shared QUICK_ACTIVITY_RETRY constant (cheap side-effect+# activities fail fast) — never an inline RetryPolicy construction.+BOUNDED_STATUS_BROADCASTS_PATCH = "trigger-bounded-status-broadcasts-v1"+++def _history_pressure(soft_cap: int) -> bool:+ """True when the server suggests continue-as-new or the current+ history length crossed ``soft_cap``. False outside a real workflow+ runtime (direct unit invocation), where history cannot grow. Shared+ by the listener, polling, and controller rollover checks."""+ try:+ info = workflow.info()+ except Exception: # direct unit invocation outside Temporal runtime+ return False+ return bool(+ info.is_continue_as_new_suggested()+ or info.get_current_history_length() > soft_cap+ )def event_workflow_search_attributes(@@ -151,6 +199,16 @@ class TriggerListenerWorkflow:use_cooperative_pause_schedule_gate = workflow.patched(COOPERATIVE_PAUSE_SCHEDULING_PATCH)+ use_history_bounded_can = workflow.patched(HISTORY_BOUNDED_CAN_PATCH)++ # Rehydrate rollover-carried state. Command-free, so pre-patch+ # histories (whose payloads lack the keys) replay identically.+ self._control_paused = bool(listener_data.get("control_paused"))+ for carried_event in listener_data.get("pending_events") or []:+ carried_id = carried_event.get("id")+ if carried_id and carried_id not in self._seen_event_ids:+ self._seen_event_ids.add(carried_id)+ self._matched_events.append(carried_event)while True:await workflow.wait_condition(@@ -186,9 +244,25 @@ class TriggerListenerWorkflow:workflow.logger.error(f"TriggerListener spawn failed for event.id={event.get('id')}: {exc}")self._processed_count += 1- if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:+ should_rollover = self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW+ if use_history_bounded_can and not should_rollover:+ should_rollover = _history_pressure(_HISTORY_SOFT_CAP)+ if should_rollover:workflow.logger.info(f"TriggerListener continue_as_new: processed={self._processed_count}")- if use_cooperative_pause_schedule_gate:+ if use_history_bounded_can:+ # Carry queued events (previously dropped at rollover)+ # and the pause flag (previously the rollover blocked+ # behind a resume that could be months away, letting+ # signal traffic overflow the history mid-pause).+ dropped = max(0, len(self._matched_events) - _MAX_CARRIED_EVENTS)+ if dropped:+ workflow.logger.warning(+ f"TriggerListener rollover dropping {dropped} oldest queued "+ f"event(s) beyond the {_MAX_CARRIED_EVENTS} carry cap"+ )+ listener_data["pending_events"] = self._matched_events[-_MAX_CARRIED_EVENTS:]+ listener_data["control_paused"] = self._control_paused+ elif use_cooperative_pause_schedule_gate:# Do not let continue-as-new reset a pause that landed# during the final spawn/status broadcast.await self._wait_until_resumed()@@ -275,12 +349,26 @@ class TriggerListenerWorkflow:event_id = event.get("id") or workflow.uuid4().hexchild_id = f"{workflow_slug}-{trigger_label}-{event_id}"- await _broadcast_trigger_idle(- node_id=trigger_node_id,- workflow_id=workflow_id,- event_id=event_id,- event_type=event.get("type", ""),- )+ from services.temporal._retry_policies import QUICK_ACTIVITY_RETRY++ try:+ unbounded_child_runs = workflow.patched(UNBOUNDED_CHILD_RUNS_PATCH)+ bounded_broadcasts = workflow.patched(BOUNDED_STATUS_BROADCASTS_PATCH)+ except RuntimeError: # direct unit invocation outside Temporal runtime+ unbounded_child_runs = True+ bounded_broadcasts = True+ broadcast_retry = QUICK_ACTIVITY_RETRY if bounded_broadcasts else None++ try:+ await _broadcast_trigger_idle(+ node_id=trigger_node_id,+ workflow_id=workflow_id,+ event_id=event_id,+ event_type=event.get("type", ""),+ retry_policy=broadcast_retry,+ )+ except Exception as exc: # noqa: BLE001 — cosmetic UI signalling+ workflow.logger.warning(f"Trigger status broadcast failed (non-fatal): {exc}")if admission_check is not None:await admission_check()@@ -291,9 +379,12 @@ class TriggerListenerWorkflow:"id_reuse_policy": (WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY),- "execution_timeout": timedelta(hours=1),- "run_timeout": timedelta(hours=1),}+ if not unbounded_child_runs:+ # Replay-only: pre-patch histories recorded 1h caps on the+ # child-start command. See UNBOUNDED_CHILD_RUNS_PATCH.+ child_options["execution_timeout"] = timedelta(hours=1)+ child_options["run_timeout"] = timedelta(hours=1)if search_attributes is not None:child_options["search_attributes"] = search_attributes@@ -319,12 +410,16 @@ class TriggerListenerWorkflow:f"TriggerListener spawned child run: child_id={child_id} " f"event.id={event.get('id')} event.type={event.get('type')}")- await _broadcast_trigger_waiting(- node_id=trigger_node_id,- workflow_id=workflow_id,- event_type=listener_data.get("event_type", ""),- processed_count=self._processed_count + 1,- )+ try:+ await _broadcast_trigger_waiting(+ node_id=trigger_node_id,+ workflow_id=workflow_id,+ event_type=listener_data.get("event_type", ""),+ processed_count=self._processed_count + 1,+ retry_policy=broadcast_retry,+ )+ except Exception as exc: # noqa: BLE001 — cosmetic UI signalling+ workflow.logger.warning(f"Trigger status broadcast failed (non-fatal): {exc}")# ---------------------------------------------------------------------------@@ -344,10 +439,20 @@ async def _broadcast_trigger_idle(workflow_id: Optional[str],event_id: str,event_type: str,+ retry_policy: Optional[RetryPolicy] = None,) -> None:"""Broadcast trigger node ``"idle"`` status with a "Graph executing..."message — matches the legacy collector/processor transition so FE- shows a firing pulse instead of a stuck "waiting" indicator."""+ shows a firing pulse instead of a stuck "waiting" indicator.++ ``retry_policy=None`` (Temporal default = unlimited retries) is the+ replay-compatibility shape for pre-patch histories; new executions+ pass the bounded policy so a dead broadcaster cannot wedge the+ serialized spawn loop.+ """+ kwargs: Dict[str, Any] = {"start_to_close_timeout": _STATUS_ACTIVITY_TIMEOUT}+ if retry_policy is not None:+ kwargs["retry_policy"] = retry_policyawait workflow.execute_activity(_STATUS_ACTIVITY_NAME,{@@ -361,7 +466,7 @@ async def _broadcast_trigger_idle(},"workflow_id": workflow_id,},- start_to_close_timeout=_STATUS_ACTIVITY_TIMEOUT,+ **kwargs,)@@ -371,10 +476,15 @@ async def _broadcast_trigger_waiting(workflow_id: Optional[str],event_type: str,processed_count: int,+ retry_policy: Optional[RetryPolicy] = None,) -> None:"""Broadcast trigger node back to ``"waiting"`` after the child runhas been spawned (child completes independently per- ``parent_close_policy=ABANDON``)."""+ ``parent_close_policy=ABANDON``). Retry-policy contract matches+ :func:`_broadcast_trigger_idle`."""+ kwargs: Dict[str, Any] = {"start_to_close_timeout": _STATUS_ACTIVITY_TIMEOUT}+ if retry_policy is not None:+ kwargs["retry_policy"] = retry_policyawait workflow.execute_activity(_STATUS_ACTIVITY_NAME,{@@ -388,7 +498,7 @@ async def _broadcast_trigger_waiting(},"workflow_id": workflow_id,},- start_to_close_timeout=_STATUS_ACTIVITY_TIMEOUT,+ **kwargs,)diff --git a/server/services/temporal/workflow_control_workflow.py b/server/services/temporal/workflow_control_workflow.pyindex 34f218dc..9ca480a7 100644--- a/server/services/temporal/workflow_control_workflow.py+++ b/server/services/temporal/workflow_control_workflow.py@@ -2,9 +2,10 @@import asynciofrom datetime import timedelta-from typing import Any, Dict+from typing import Any, Dict, List, Optional, Tuplefrom temporalio import workflow+from temporalio.common import SearchAttributeKey, SearchAttributePairfrom temporalio.exceptions import ApplicationErrorfrom services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY@@ -13,6 +14,35 @@ from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRYCOOPERATIVE_PAUSE_SCHEDULING_PATCH = ("workflow-control-cooperative-pause-scheduling-v1")+# The controller is the ONE workflow expected to live for months, and it+# multiplexes every trigger's signals, poll timers/activities, and child+# spawns into a single event history. Temporal terminates any workflow+# around ~51,200 history events, so without continue-as-new a single+# polling trigger at the default 60s interval killed the whole+# deployment's control plane in days. The patched path rolls the run+# over (carrying triggers, control state, queued events, and per-trigger+# seen-id baselines) whenever the server suggests it or the history+# crosses the soft cap below. Pre-patch histories replay unchanged.+CONTINUE_AS_NEW_PATCH = "workflow-control-continue-as-new-v1"+# Upsert the ControlEventTypes keyword-list Search Attribute so+# dispatch.emit can skip controllers whose deployment has no matching+# push trigger (otherwise every platform event lands ~4 history events+# in every running controller). Patch-gated: upsert is a command.+CONTROL_EVENT_TYPES_SA_PATCH = "workflow-control-event-types-sa-v1"++# Roll over before the server's hard ceiling. is_continue_as_new_suggested+# is the primary signal; this cap is the deterministic backstop.+_HISTORY_SOFT_CAP = 10_000+# Bounds on state carried across continue-as-new — the CAN argument blob+# is capped at 2MiB by Temporal, so unbounded carries would trade a+# history overflow for an argument overflow.+_MAX_CARRIED_EVENTS = 256+_MAX_CARRIED_SEEN_IDS = 4_096+# Defensive floor for user-supplied poll intervals: a few-second interval+# burns ~100K+ history events/day. Mirrors the plugin-side clamp+# (PollingTriggerNode.poll_interval_clamp) which the legacy asyncio path+# applies but workflow payloads historically did not.+_MIN_POLL_INTERVAL_S = 30@workflow.defn(name="WorkflowControlWorkflow", sandboxed=False)@@ -22,6 +52,14 @@ class WorkflowControlWorkflow:Trigger definitions and inbound events are recorded as signals in thisworkflow's own history. Only an actual triggered graph run becomes a childworkflow, so Temporal's workflow list has no per-trigger listener rows.++ Longevity contract: the controller continue-as-news to keep its+ history bounded. All state a rollover must preserve lives in+ ``control_data`` — trigger specs (whose ``listener_args["seen_ids"]``+ the poll loops write back after every cycle), the pending push-event+ queue, the dedup baseline, and the control state/revision. Callers+ therefore address the controller by workflow id only, never a pinned+ run id."""def __init__(self) -> None:@@ -30,9 +68,13 @@ class WorkflowControlWorkflow:self._closed = Falseself._triggers: dict[str, Dict[str, Any]] = {}self._events: list[tuple[str, Dict[str, Any]]] = []- self._seen_event_ids: set[str] = set()+ # Insertion-ordered so the carry across continue-as-new can keep+ # the newest entries when trimming to _MAX_CARRIED_SEEN_IDS.+ self._seen_event_ids: dict[str, None] = {}self._poll_tasks: dict[str, asyncio.Task] = {}self._use_cooperative_pause_schedule_gate = False+ self._can_enabled = False+ self._can_requested = False@workflow.signalasync def pause(self) -> None:@@ -91,6 +133,8 @@ class WorkflowControlWorkflow:self._triggers[listener_id] = specif spec["workflow_type"] == "PollingTriggerWorkflow":self._poll_tasks[listener_id] = asyncio.create_task(self._poll_trigger(listener_id, spec))+ else:+ self._upsert_event_types_attribute()@workflow.signalasync def on_event(self, event: Dict[str, Any]) -> None:@@ -105,8 +149,13 @@ class WorkflowControlWorkflow:continuededup_key = f"{listener_id}:{event_id}"if dedup_key not in self._seen_event_ids:- self._seen_event_ids.add(dedup_key)+ self._remember_event_id(dedup_key)self._events.append((listener_id, event))+ # A paused controller still receives matching signals; without this+ # check its history could overflow mid-pause with the run loop+ # parked. The flag wakes the run loop, which rolls over carrying+ # the queue (and the paused state) forward.+ self._maybe_request_rollover()@workflow.querydef status(self) -> Dict[str, Any]:@@ -122,23 +171,165 @@ class WorkflowControlWorkflow:self._use_cooperative_pause_schedule_gate = workflow.patched(COOPERATIVE_PAUSE_SCHEDULING_PATCH)+ self._can_enabled = workflow.patched(CONTINUE_AS_NEW_PATCH)except Exception: # direct unit invocation outside Temporal runtimeif workflow.in_workflow():raiseself._use_cooperative_pause_schedule_gate = False+ self._can_enabled = Trueself._state = control_data.get("state", "running")+ self._seed_carried_state(control_data)while not self._closed:await workflow.wait_condition(- lambda: self._closed or (self._state == "running" and bool(self._events))+ lambda: (+ self._closed+ or self._can_requested+ or (self._state == "running" and bool(self._events))+ ))if self._closed:break+ if self._can_requested:+ await self._continue_as_new(control_data)+ # Unreachable in a real run; direct unit invocation returns.+ breaklistener_id, event = self._events.pop(0)spec = self._triggers.get(listener_id)- if spec is not None:+ if spec is None:+ continue+ try:await self._spawn_push_run(event, spec)+ except asyncio.CancelledError:+ raise+ except Exception as exc: # noqa: BLE001 — per-event isolation+ # One bad event (duplicate child id, exhausted broadcast+ # retries) must not fail the deployment's entire control+ # plane. Same isolation contract as the listener classes.+ workflow.logger.error(+ f"Controller push spawn failed for event.id={event.get('id')}: {exc}"+ )+ self._maybe_request_rollover()return {"state": self._state, "generation": control_data.get("generation")}+ # ---- continue-as-new machinery ------------------------------------++ def _seed_carried_state(self, control_data: Dict[str, Any]) -> None:+ """Rehydrate rollover-carried state; no-op for first-run payloads.++ Everything here is command-free, so pre-patch histories (whose+ payloads lack the carried keys) replay identically.+ """+ self._revision = int(control_data.get("revision") or 0)+ for seen_id in control_data.get("seen_event_ids") or []:+ self._seen_event_ids[str(seen_id)] = None+ for pending in control_data.get("pending_events") or []:+ try:+ listener_id, event = pending[0], pending[1]+ except (IndexError, TypeError, KeyError):+ continue+ self._events.append((str(listener_id), event))+ for listener_id, spec in (control_data.get("triggers") or {}).items():+ listener_id = str(listener_id)+ if listener_id in self._triggers:+ continue+ self._triggers[listener_id] = spec+ if spec.get("workflow_type") == "PollingTriggerWorkflow":+ self._poll_tasks[listener_id] = asyncio.create_task(+ self._poll_trigger(listener_id, spec)+ )+ if any(+ spec.get("workflow_type") != "PollingTriggerWorkflow"+ for spec in self._triggers.values()+ ):+ self._upsert_event_types_attribute()++ def _remember_event_id(self, dedup_key: str) -> None:+ self._seen_event_ids[dedup_key] = None+ if len(self._seen_event_ids) > _MAX_CARRIED_SEEN_IDS:+ oldest = next(iter(self._seen_event_ids))+ del self._seen_event_ids[oldest]++ def _history_pressure(self) -> bool:+ from services.temporal.trigger_listener_workflow import _history_pressure++ return _history_pressure(_HISTORY_SOFT_CAP)++ def _maybe_request_rollover(self) -> None:+ if self._can_enabled and not self._closed and not self._can_requested:+ if self._history_pressure():+ self._can_requested = True++ async def _continue_as_new(self, control_data: Dict[str, Any]) -> None:+ """Roll the run over, carrying everything a controller must keep.++ Poll tasks are cancelled first — their provider ``seen_ids``+ baselines were written back into each trigger spec's+ ``listener_args`` after every cycle, so the carried specs restart+ the loops in the new run without re-emitting old items. Works+ while paused too: the paused state (and queued events) carry.+ """+ for task in self._poll_tasks.values():+ task.cancel()+ if self._poll_tasks:+ await asyncio.gather(*self._poll_tasks.values(), return_exceptions=True)+ dropped = max(0, len(self._events) - _MAX_CARRIED_EVENTS)+ if dropped:+ workflow.logger.warning(+ f"Controller rollover dropping {dropped} oldest queued event(s) "+ f"beyond the {_MAX_CARRIED_EVENTS} carry cap"+ )+ carried: Dict[str, Any] = {+ **control_data,+ "state": self._state,+ "revision": self._revision,+ "triggers": self._triggers,+ "pending_events": [+ list(item) for item in self._events[-_MAX_CARRIED_EVENTS:]+ ],+ "seen_event_ids": list(self._seen_event_ids)[-_MAX_CARRIED_SEEN_IDS:],+ }+ workflow.logger.info(+ f"Controller continue_as_new: triggers={len(self._triggers)} "+ f"pending={len(carried['pending_events'])} state={self._state}"+ )+ workflow.continue_as_new(args=[carried])++ def _upsert_event_types_attribute(self) -> None:+ """Advertise push event types via the ControlEventTypes attribute.++ Lets ``dispatch.emit`` skip controllers with no matching trigger+ instead of signalling every running controller with every+ platform event. Patch-gated (upsert is a command); controllers+ without the attribute are treated as match-all by dispatch.+ """+ try:+ if not workflow.patched(CONTROL_EVENT_TYPES_SA_PATCH):+ return+ except Exception: # direct unit invocation outside Temporal runtime+ return+ event_types: set[str] = set()+ for spec in self._triggers.values():+ if spec.get("workflow_type") == "PollingTriggerWorkflow":+ continue+ for event_type in spec.get("event_types") or [spec.get("event_type")]:+ if event_type:+ event_types.add(str(event_type))+ if not event_types:+ return+ try:+ workflow.upsert_search_attributes(+ [+ SearchAttributePair(+ SearchAttributeKey.for_keyword_list("ControlEventTypes"),+ sorted(event_types),+ )+ ]+ )+ except Exception as exc: # noqa: BLE001 — attribute is an optimisation+ workflow.logger.warning(f"ControlEventTypes upsert failed (non-fatal): {exc}")++ # ---- spawning -------------------------------------------------------+async def _spawn_push_run(self, event: Dict[str, Any], spec: Dict[str, Any]) -> None:from services.temporal.trigger_listener_workflow import (TriggerListenerWorkflow,@@ -184,6 +375,11 @@ class WorkflowControlWorkflow:activity_name = f"poll.{node_type}.v{listener_data.get('version', 1)}"params = listener_data.get("filter_params", {}) or {}poll_interval = int(params.get("poll_interval") or _DEFAULT_POLL_INTERVAL_S)+ if self._can_enabled:+ # A few-second user-supplied interval overflows history in+ # hours. Timer durations are recorded commands, so the clamp+ # rides the same patch as continue-as-new.+ poll_interval = max(_MIN_POLL_INTERVAL_S, poll_interval)activity_timeout_s = max(30, poll_interval * _ACTIVITY_TIMEOUT_MULT)seen_ids: set[str] = set(listener_data.get("seen_ids") or [])baseline = not seen_ids@@ -212,39 +408,62 @@ class WorkflowControlWorkflow:start_to_close_timeout=timedelta(seconds=activity_timeout_s),retry_policy=DEFAULT_ACTIVITY_RETRY,)+ except asyncio.CancelledError:+ raiseexcept Exception as exc: # noqa: BLE001workflow.logger.error(f"Controlled polling trigger failed: {exc}")+ self._maybe_request_rollover()continueseen_ids = set(result.get("seen_ids") or [])+ # Write the provider baseline back into the carried spec so a+ # continue-as-new (which cancels this task) restarts the loop+ # exactly where it left off instead of re-emitting old items.+ listener_data["seen_ids"] = list(seen_ids)for event in result.get("events") or []:event_id = str(event.get("id") or "")dedup_key = f"{listener_id}:{event_id}"if not event_id or dedup_key in self._seen_event_ids:continue- self._seen_event_ids.add(dedup_key)+ self._remember_event_id(dedup_key)if self._state != "running":await workflow.wait_condition(lambda: self._closed or self._state == "running")if self._closed:return- await runner._spawn_child_run(- event,- listener_data,- admission_check=(- self._wait_until_running- if self._use_cooperative_pause_schedule_gate- else None- ),- search_attributes=(- event_workflow_search_attributes(- listener_data.get("workflow_id")- )- if self._use_cooperative_pause_schedule_gate- else None- ),- )+ try:+ await runner._spawn_child_run(+ event,+ listener_data,+ admission_check=(+ self._wait_until_running+ if self._use_cooperative_pause_schedule_gate+ else None+ ),+ search_attributes=(+ event_workflow_search_attributes(+ listener_data.get("workflow_id")+ )+ if self._use_cooperative_pause_schedule_gate+ else None+ ),+ )+ except asyncio.CancelledError:+ raise+ except Exception as spawn_exc: # noqa: BLE001 — per-event isolation+ # A failed spawn must not silently kill this poll task+ # (the trigger would stop firing forever while the+ # controller still reported Running).+ workflow.logger.error(+ f"Controlled polling spawn failed for event.id={event_id}: {spawn_exc}"+ )+ # Poll cycles burn history even with zero events — request+ # rollover from here too, so a quiet mailbox cannot ride the+ # controller into the server's hard termination ceiling.+ self._maybe_request_rollover()__all__ = [+ "CONTINUE_AS_NEW_PATCH",+ "CONTROL_EVENT_TYPES_SA_PATCH","COOPERATIVE_PAUSE_SCHEDULING_PATCH","WorkflowControlWorkflow",]diff --git a/server/tests/temporal/test_durability_hardening.py b/server/tests/temporal/test_durability_hardening.pynew file mode 100644index 00000000..d5351361--- /dev/null+++ b/server/tests/temporal/test_durability_hardening.py@@ -0,0 +1,749 @@+"""Months-long durability hardening: contract tests.++The product guarantee: a deployed workflow (running OR paused) survives+backend restarts, is never auto-terminated, and keeps executing for+months. These tests lock the mechanisms that deliver it:++- No lifetime caps on new child workflow starts (replay-patched).+- History-pressure continue-as-new on every long-lived loop (Temporal+ hard-terminates around ~51,200 history events).+- The controller carries its full state (triggers, queued events, dedup+ baseline, control state) across rollovers, so it is addressed by+ workflow id only — never a pinned run id.+- dispatch.emit skips controllers whose deployment has no matching push+ trigger (cross-deployment signal amplification burned every+ controller's history budget on other deployments' traffic).+- The startup terminate sweep's active-control guard shares the+ canonical ACTIVE_STATES set (the old copy omitted "resetting").+- Boot-time reconcile converges rows a crash left transitional and+ re-arms the process-local half of running/paused generations.+"""++from __future__ import annotations++import asyncio+import inspect+from datetime import timedelta+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest++from services.temporal.polling_trigger_workflow import PollingTriggerWorkflow+from services.temporal.trigger_listener_workflow import TriggerListenerWorkflow+from services.temporal.workflow_control_workflow import WorkflowControlWorkflow++SERVER_DIR = Path(__file__).resolve().parents[2]+++# ---------------------------------------------------------------------------+# Replay-patch guarding — every new behaviour change that alters recorded+# commands must ride a workflow.patched marker.+# ---------------------------------------------------------------------------+++def test_durability_changes_are_replay_patch_guarded():+ from nodes.scheduler.cron_scheduler import _workflow as cron_module+ from services.temporal import agent_workflow as agent_module+ from services.temporal import polling_trigger_workflow as polling_module+ from services.temporal import trigger_listener_workflow as trigger_module+ from services.temporal import workflow as machina_module+ from services.temporal import workflow_control_workflow as control_module+ from services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflow+ from services.temporal.workflow import MachinaWorkflow+ from nodes.scheduler.cron_scheduler._workflow import CronTriggerWorkflow++ expected = {+ machina_module.UNBOUNDED_LIFETIMES_PATCH: inspect.getsource(MachinaWorkflow.run),+ agent_module.UNBOUNDED_LIFETIMES_PATCH: inspect.getsource(AgentWorkflow.run),+ trigger_module.UNBOUNDED_CHILD_RUNS_PATCH: inspect.getsource(+ TriggerListenerWorkflow._spawn_child_run+ ),+ polling_module.UNBOUNDED_CHILD_RUNS_PATCH: inspect.getsource(+ PollingTriggerWorkflow._spawn_child_run+ ),+ cron_module.UNBOUNDED_CHILD_RUNS_PATCH: inspect.getsource(CronTriggerWorkflow.run),+ trigger_module.HISTORY_BOUNDED_CAN_PATCH: inspect.getsource(+ TriggerListenerWorkflow.run+ ),+ polling_module.HISTORY_BOUNDED_CAN_PATCH: inspect.getsource(+ PollingTriggerWorkflow.run+ ),+ control_module.CONTINUE_AS_NEW_PATCH: inspect.getsource(+ WorkflowControlWorkflow.run+ ),+ trigger_module.BOUNDED_STATUS_BROADCASTS_PATCH: inspect.getsource(+ TriggerListenerWorkflow._spawn_child_run+ ),+ }++ assert set(expected) == {+ "machina-unbounded-lifetimes-v1",+ "agent-unbounded-lifetimes-v1",+ "trigger-unbounded-child-runs-v1",+ "polling-unbounded-child-runs-v1",+ "cron-unbounded-child-runs-v1",+ "trigger-history-bounded-can-v1",+ "polling-history-bounded-can-v1",+ "workflow-control-continue-as-new-v1",+ "trigger-bounded-status-broadcasts-v1",+ }+ # DelegatedTaskWorkflow shares agent_module's marker.+ assert "UNBOUNDED_LIFETIMES_PATCH" in inspect.getsource(DelegatedTaskWorkflow.run)+ for patch_id, source in expected.items():+ assert "workflow.patched(" in source+ assert patch_id.endswith("-v1")+++# ---------------------------------------------------------------------------+# Unbounded child runs — new executions must not carry lifetime caps.+# ---------------------------------------------------------------------------+++def _listener_data() -> dict:+ return {+ "workflow_id": "wf-1",+ "trigger_node_id": "trigger-1",+ "node_type": "webhookTrigger",+ "event_type": "com.opencompany.webhook.received",+ "filter_params": {},+ "nodes": [{"id": "trigger-1", "type": "webhookTrigger", "data": {}}],+ "edges": [],+ "session_id": "default",+ "tenant_id": None,+ }++[email protected]("patched_on", [True, False])+async def test_listener_child_lifetime_caps_are_patch_gated(monkeypatch, patched_on):+ from services.temporal import trigger_listener_workflow as trigger_module++ temporal_workflow = trigger_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ flags = {+ trigger_module.UNBOUNDED_CHILD_RUNS_PATCH: patched_on,+ trigger_module.BOUNDED_STATUS_BROADCASTS_PATCH: patched_on,+ }+ monkeypatch.setattr(+ temporal_workflow, "patched", lambda pid: flags.get(pid, False)+ )+ monkeypatch.setattr(temporal_workflow, "execute_activity", AsyncMock())++ captured: dict = {}++ async def start_child(name, args=None, **kwargs):+ captured.update(kwargs)+ captured["name"] = name++ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)++ listener = TriggerListenerWorkflow()+ await listener._spawn_child_run({"id": "evt-1", "data": {}}, _listener_data())++ assert captured["name"] == "MachinaWorkflow"+ if patched_on:+ assert "execution_timeout" not in captured+ assert "run_timeout" not in captured+ else:+ assert captured["execution_timeout"] == timedelta(hours=1)+ assert captured["run_timeout"] == timedelta(hours=1)+++async def test_listener_broadcast_failure_does_not_kill_the_spawn(monkeypatch):+ """Status broadcasts are cosmetic — a dead broadcaster must neither+ wedge (unlimited retries) nor fail the spawn."""+ from services.temporal import trigger_listener_workflow as trigger_module++ temporal_workflow = trigger_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _pid: True)+ monkeypatch.setattr(+ temporal_workflow,+ "execute_activity",+ AsyncMock(side_effect=RuntimeError("broadcaster down")),+ )+ started = {}++ async def start_child(name, args=None, **kwargs):+ started["id"] = kwargs.get("id")++ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)++ listener = TriggerListenerWorkflow()+ await listener._spawn_child_run({"id": "evt-1", "data": {}}, _listener_data())+ assert started["id"] # the child run still started+++# ---------------------------------------------------------------------------+# History-pressure continue-as-new.+# ---------------------------------------------------------------------------+++class _CanFired(Exception):+ pass+++def _pressure_info() -> SimpleNamespace:+ return SimpleNamespace(+ is_continue_as_new_suggested=lambda: True,+ get_current_history_length=lambda: 0,+ )+++async def test_polling_rolls_over_under_history_pressure_with_zero_events(monkeypatch):+ """A quiet mailbox burns ~11 history events per cycle with the+ processed counter frozen at 0 — the rollover must key on history+ pressure, not on emitted events."""+ from services.temporal import polling_trigger_workflow as polling_module++ temporal_workflow = polling_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _pid: True)+ monkeypatch.setattr(temporal_workflow, "info", _pressure_info)++ sleeps: list = []++ async def record_sleep(duration):+ sleeps.append(duration)++ monkeypatch.setattr(temporal_workflow, "sleep", record_sleep)+ monkeypatch.setattr(+ temporal_workflow,+ "execute_activity",+ AsyncMock(return_value={"seen_ids": ["prior", "new"], "events": []}),+ )+ captured: dict = {}++ def fake_can(listener_data):+ captured["carried"] = listener_data+ raise _CanFired++ monkeypatch.setattr(temporal_workflow, "continue_as_new", fake_can)++ listener_data = _listener_data()+ listener_data["node_type"] = "googleGmailReceive"+ listener_data["seen_ids"] = ["prior"] # non-baseline start -> sleeps+ listener_data["filter_params"] = {"poll_interval": 5} # absurdly hot++ instance = PollingTriggerWorkflow()+ with pytest.raises(_CanFired):+ await instance.run(listener_data)++ assert instance._processed_count == 0+ carried = captured["carried"]+ assert set(carried["seen_ids"]) == {"prior", "new"}+ assert carried["control_paused"] is False+ # The 5s user interval was clamped to the defensive floor.+ assert sleeps == [timedelta(seconds=polling_module._MIN_POLL_INTERVAL_S)]+++async def _predicate_wait(predicate):+ while not predicate():+ await asyncio.sleep(0)+++async def test_listener_rollover_carries_queued_events_and_pause_flag(monkeypatch):+ from services.temporal import trigger_listener_workflow as trigger_module++ temporal_workflow = trigger_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _pid: True)+ monkeypatch.setattr(temporal_workflow, "info", _pressure_info)+ monkeypatch.setattr(temporal_workflow, "wait_condition", _predicate_wait)++ captured: dict = {}++ def fake_can(args=None):+ captured["carried"] = args[0]+ raise _CanFired++ monkeypatch.setattr(temporal_workflow, "continue_as_new", fake_can)++ instance = TriggerListenerWorkflow()+ monkeypatch.setattr(instance, "_spawn_child_run", AsyncMock())++ listener_data = _listener_data()+ # Carried queue from a previous rollover: two events pending.+ listener_data["pending_events"] = [+ {"id": "evt-1", "data": {}},+ {"id": "evt-2", "data": {}},+ ]++ with pytest.raises(_CanFired):+ await instance.run(listener_data)++ carried = captured["carried"]+ # evt-1 processed; the still-queued evt-2 carries instead of dropping.+ assert [e["id"] for e in carried["pending_events"]] == ["evt-2"]+ assert carried["control_paused"] is False+ # Seeded events joined the dedup set so producer retries stay deduped.+ assert "evt-1" in instance._seen_event_ids+++# ---------------------------------------------------------------------------+# Controller rollover state machine.+# ---------------------------------------------------------------------------+++def _push_spec(listener_id: str = "l-push") -> dict:+ return {+ "listener_id": listener_id,+ "workflow_type": "TriggerListenerWorkflow",+ "event_type": "t.x",+ "event_types": ["t.x"],+ "trigger_node_id": "n-push",+ "listener_args": _listener_data(),+ }+++def _poll_spec(listener_id: str = "l-poll") -> dict:+ data = _listener_data()+ data["node_type"] = "googleGmailReceive"+ return {+ "listener_id": listener_id,+ "workflow_type": "PollingTriggerWorkflow",+ "event_type": "t.poll",+ "event_types": ["t.poll"],+ "trigger_node_id": "n-poll",+ "listener_args": data,+ }+++async def test_controller_on_event_requests_rollover_under_pressure():+ controller = WorkflowControlWorkflow()+ controller._can_enabled = True+ controller._triggers["l-push"] = _push_spec()+ controller._history_pressure = lambda: True # type: ignore[method-assign]++ await controller.on_event({"id": "evt-1", "type": "t.x"})++ assert controller._events # queued+ assert controller._can_requested is True+++def test_controller_seen_event_ids_are_bounded():+ from services.temporal import workflow_control_workflow as control_module++ controller = WorkflowControlWorkflow()+ cap = control_module._MAX_CARRIED_SEEN_IDS+ for index in range(cap + 10):+ controller._remember_event_id(f"key-{index}")+ assert len(controller._seen_event_ids) == cap+ assert "key-0" not in controller._seen_event_ids+ assert f"key-{cap + 9}" in controller._seen_event_ids+++async def test_controller_rollover_carries_triggers_events_and_state(monkeypatch):+ from services.temporal import workflow_control_workflow as control_module++ temporal_workflow = control_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ captured: dict = {}++ def fake_can(args=None):+ captured["carried"] = args[0]+ raise _CanFired++ monkeypatch.setattr(temporal_workflow, "continue_as_new", fake_can)++ controller = WorkflowControlWorkflow()+ controller._state = "paused" # rollover must work mid-pause+ controller._revision = 7+ controller._triggers["l-push"] = _push_spec()+ controller._events.append(("l-push", {"id": "evt-9", "type": "t.x"}))+ controller._remember_event_id("l-push:evt-9")+ # A live poll task the rollover must cancel cleanly.+ blocker = asyncio.Event()++ async def parked_poll():+ try:+ await blocker.wait()+ except asyncio.CancelledError:+ raise++ controller._poll_tasks["l-poll"] = asyncio.create_task(parked_poll())++ with pytest.raises(_CanFired):+ await controller._continue_as_new({"generation": 3, "workflow_id": "wf-1"})++ carried = captured["carried"]+ assert carried["state"] == "paused"+ assert carried["revision"] == 7+ assert carried["generation"] == 3+ assert "l-push" in carried["triggers"]+ assert [event["id"] for _lid, event in carried["pending_events"]] == ["evt-9"]+ assert "l-push:evt-9" in carried["seen_event_ids"]+ assert controller._poll_tasks["l-poll"].cancelled()+++async def test_controller_seeds_carried_state_and_restarts_poll_loops(monkeypatch):+ controller = WorkflowControlWorkflow()+ started: list = []++ async def fake_poll(listener_id, spec):+ started.append(listener_id)+ await asyncio.Event().wait()++ monkeypatch.setattr(controller, "_poll_trigger", fake_poll)++ controller._seed_carried_state(+ {+ "revision": 4,+ "seen_event_ids": ["l-push:evt-1"],+ "pending_events": [["l-push", {"id": "evt-2", "type": "t.x"}]],+ "triggers": {"l-push": _push_spec(), "l-poll": _poll_spec()},+ }+ )+ await asyncio.sleep(0) # let the created task start++ assert controller._revision == 4+ assert "l-push:evt-1" in controller._seen_event_ids+ assert controller._events == [("l-push", {"id": "evt-2", "type": "t.x"})]+ assert set(controller._triggers) == {"l-push", "l-poll"}+ assert started == ["l-poll"] # only polling triggers get loops+ controller._poll_tasks["l-poll"].cancel()+++async def test_controller_poll_loop_writes_seen_ids_back_into_the_spec(monkeypatch):+ """The provider baseline must live in the carried spec so a rollover+ restarts the poll loop exactly where it left off."""+ from services.temporal import workflow_control_workflow as control_module++ temporal_workflow = control_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "wait_condition", _predicate_wait)+ monkeypatch.setattr(temporal_workflow, "sleep", AsyncMock())++ controller = WorkflowControlWorkflow()+ controller._can_enabled = False+ spec = _poll_spec()+ controller._triggers[spec["listener_id"]] = spec++ async def poll_activity(*_args, **_kwargs):+ controller._closed = True # one cycle, then exit+ return {"seen_ids": ["provider-1"], "events": []}++ monkeypatch.setattr(temporal_workflow, "execute_activity", poll_activity)++ await controller._poll_trigger(spec["listener_id"], spec)+ assert spec["listener_args"]["seen_ids"] == ["provider-1"]+++async def test_controller_push_spawn_failure_does_not_kill_the_run_loop(monkeypatch):+ from services.temporal import workflow_control_workflow as control_module++ temporal_workflow = control_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "wait_condition", _predicate_wait)+ monkeypatch.setattr(temporal_workflow, "in_workflow", lambda: False)++ controller = WorkflowControlWorkflow()+ calls = {"n": 0}++ async def failing_spawn(_event, _spec):+ calls["n"] += 1+ if calls["n"] == 1:+ raise RuntimeError("duplicate child id")+ controller._closed = True++ monkeypatch.setattr(controller, "_spawn_push_run", failing_spawn)+ controller._triggers["l-push"] = _push_spec()+ controller._events.append(("l-push", {"id": "evt-1", "type": "t.x"}))+ controller._events.append(("l-push", {"id": "evt-2", "type": "t.x"}))++ result = await controller.run({"generation": 1, "state": "running"})+ # First spawn failed, second still ran; the controller survived both.+ assert calls["n"] == 2+ assert result["generation"] == 1+++# ---------------------------------------------------------------------------+# dispatch.emit controller narrowing.+# ---------------------------------------------------------------------------+++def test_dispatch_controller_helpers_read_both_attribute_shapes():+ from services.events.dispatch import _controller_event_types, _is_controller++ controller = SimpleNamespace(workflow_type="WorkflowControlWorkflow")+ listener = SimpleNamespace(workflow_type="TriggerListenerWorkflow")+ assert _is_controller(controller) is True+ assert _is_controller(listener) is False++ typed = SimpleNamespace(+ typed_search_attributes=[+ SimpleNamespace(+ key=SimpleNamespace(name="ControlEventTypes"),+ value=["a.b", "c.d"],+ )+ ],+ search_attributes=None,+ )+ assert _controller_event_types(typed) == {"a.b", "c.d"}++ plain = SimpleNamespace(+ typed_search_attributes=None,+ search_attributes={"ControlEventTypes": ["x.y"]},+ )+ assert _controller_event_types(plain) == {"x.y"}++ absent = SimpleNamespace(typed_search_attributes=None, search_attributes=None)+ assert _controller_event_types(absent) is None # pre-upgrade: match-all+++def test_control_event_types_search_attribute_is_registered():+ from services.temporal.search_attributes import EVENT_SEARCH_ATTRIBUTES++ names = {spec.name for spec in EVENT_SEARCH_ATTRIBUTES}+ assert "ControlEventTypes" in names+++# ---------------------------------------------------------------------------+# Terminate-sweep guard + controller addressing.+# ---------------------------------------------------------------------------+++def test_active_states_are_shared_and_include_resetting():+ from models.database import WORKFLOW_CONTROL_ACTIVE_STATES+ from services.deployment.control import ACTIVE_STATES++ assert ACTIVE_STATES is WORKFLOW_CONTROL_ACTIVE_STATES+ assert "resetting" in ACTIVE_STATES+ # core.database is conftest-stubbed, so verify the real source text+ # references the shared constant instead of a hand-copied set.+ database_source = (SERVER_DIR / "core" / "database.py").read_text(encoding="utf-8")+ assert "WORKFLOW_CONTROL_ACTIVE_STATES" in database_source+ assert '"starting", "running", "pausing", "paused", "resuming"}' not in database_source+++def test_controller_handles_are_never_run_id_pinned():+ """Continue-as-new mints new run ids under the same workflow id; a+ pinned handle would target a closed run after the first rollover."""+ from services.deployment import handlers, manager++ handle_source = inspect.getsource(handlers._controller_handle)+ assert "get_workflow_handle(control.controller_workflow_id)" in handle_source++ manager_source = inspect.getsource(manager)+ assert 'run_id=control.controller_run_id' not in manager_source+++def test_terminate_flag_defaults_false_in_both_env_files():+ for env_name in (".env.template", ".env"):+ text = (SERVER_DIR.parent / env_name).read_text(encoding="utf-8")+ assert "TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false" in text+ assert "TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=true" not in text+++# ---------------------------------------------------------------------------+# Boot-time reconcile.+# ---------------------------------------------------------------------------+++def _boot_control(status: str, *, graph_nodes=None):+ return SimpleNamespace(+ id="workflow-control:wf:1",+ workflow_id="wf",+ generation=1,+ execution_id="execution-1",+ root_execution_id="execution-1",+ data_scope_id="execution-1",+ controller_workflow_id="workflow-control-wf-g1",+ controller_run_id="controller-run-1",+ status=status,+ revision=3,+ graph_snapshot={+ "nodes": graph_nodes+ if graph_nodes is not None+ else [{"id": "t-1", "type": "webhookTrigger", "data": {}}],+ "edges": [],+ },+ created_at=None,+ updated_at=None,+ terminal_reason=None,+ )+++async def test_interrupted_start_with_registered_triggers_converges_to_running(+ monkeypatch,+):+ from services.deployment import handlers++ control = _boot_control("starting")+ running = _boot_control("running")+ service = SimpleNamespace(+ transition=AsyncMock(return_value=running),+ fail=AsyncMock(),+ database=SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=control)+ ),+ )+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))++ result = await handlers._converge_interrupted_start(+ service, control, {"state": "running", "triggers": {"l-1": "n-1"}}+ )++ assert result.status == "running"+ assert service.transition.await_args.kwargs["status"] == "running"+ service.fail.assert_not_awaited()+++async def test_interrupted_start_with_empty_controller_fails_and_closes_it(+ monkeypatch,+):+ from services.deployment import handlers++ control = _boot_control("starting")+ failed = _boot_control("failed")+ service = SimpleNamespace(+ transition=AsyncMock(),+ fail=AsyncMock(return_value=failed),+ database=SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=control)+ ),+ )+ reset_signal = AsyncMock()+ monkeypatch.setattr(handlers, "_signal_controller", reset_signal)+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))++ result = await handlers._converge_interrupted_start(+ service, control, {"state": "running", "triggers": {}}+ )++ assert result.status == "failed"+ assert service.fail.await_args.args[1] == "interrupted_start"+ reset_signal.assert_awaited_once_with(control, "reset")+ service.transition.assert_not_awaited()+++async def test_interrupted_start_of_triggerless_graph_converges_to_running(+ monkeypatch,+):+ from services.deployment import handlers++ control = _boot_control(+ "starting",+ graph_nodes=[{"id": "a-1", "type": "aiAgent", "data": {}}],+ )+ running = _boot_control("running")+ service = SimpleNamespace(+ transition=AsyncMock(return_value=running),+ fail=AsyncMock(),+ database=SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=control)+ ),+ )+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))++ result = await handlers._converge_interrupted_start(+ service, control, {"state": "running", "triggers": {}}+ )+ assert result.status == "running"+ service.fail.assert_not_awaited()+++async def test_unreachable_controller_leaves_starting_row_untouched():+ from services.deployment import handlers++ control = _boot_control("starting")+ service = SimpleNamespace(transition=AsyncMock(), fail=AsyncMock())+ result = await handlers._converge_interrupted_start(service, control, None)+ assert result is control+ service.transition.assert_not_awaited()+ service.fail.assert_not_awaited()+++async def test_rearm_restores_paused_posture_from_snapshot(monkeypatch):+ from core.container import container+ from services.deployment import handlers++ control = _boot_control("paused")+ workflow_service = MagicMock()+ workflow_service.is_workflow_deployed.return_value = False+ workflow_service.pause_deployment = MagicMock()+ workflow_service.update_trigger_pause_status = AsyncMock(return_value=1)+ monkeypatch.setattr(+ container, "workflow_service", MagicMock(return_value=workflow_service)+ )+ deploy = AsyncMock(return_value={"success": True})+ setup = AsyncMock(return_value={"success": True})+ cron_pause = AsyncMock(return_value=0)+ monkeypatch.setattr(handlers, "handle_deploy_workflow", deploy)+ monkeypatch.setattr(handlers, "_await_deployment_setup", setup)+ monkeypatch.setattr(handlers, "_set_cron_pause", cron_pause)++ await handlers._rearm_generation(control)++ deploy_data = deploy.await_args.args[0]+ assert deploy_data["workflow_id"] == "wf"+ assert deploy_data["nodes"] == control.graph_snapshot["nodes"]+ # Generation-scoped persistence, same contract as handle_start_workflow.+ assert deploy_data["session_id"] == "execution-1"+ workflow_service.pause_deployment.assert_called_once_with("wf")+ workflow_service.update_trigger_pause_status.assert_awaited_once_with(+ "wf", paused=True+ )+ cron_pause.assert_awaited_once_with("wf", paused=True, strict=False)+++async def test_rearm_skips_already_deployed_workflows(monkeypatch):+ from core.container import container+ from services.deployment import handlers++ control = _boot_control("running")+ workflow_service = MagicMock()+ workflow_service.is_workflow_deployed.return_value = True+ monkeypatch.setattr(+ container, "workflow_service", MagicMock(return_value=workflow_service)+ )+ deploy = AsyncMock()+ monkeypatch.setattr(handlers, "handle_deploy_workflow", deploy)++ await handlers._rearm_generation(control)+ deploy.assert_not_awaited()+++async def test_boot_reconcile_processes_each_active_row_with_isolation(monkeypatch):+ from core.container import container+ from services.deployment import handlers++ healthy = _boot_control("running")+ broken = _boot_control("running")+ broken.workflow_id = "wf-broken"+ database = SimpleNamespace(+ list_active_workflow_controls=AsyncMock(return_value=[broken, healthy]),+ )+ monkeypatch.setattr(container, "database", MagicMock(return_value=database))++ async def reconcile(_service, control):+ if control.workflow_id == "wf-broken":+ raise RuntimeError("temporal hiccup")+ return control, {"state": "running"}++ rearm = AsyncMock()+ monkeypatch.setattr(handlers, "_reconcile_control", reconcile)+ monkeypatch.setattr(handlers, "_rearm_generation", rearm)++ processed = await handlers.reconcile_active_controls_on_boot()++ # The broken row was isolated; the healthy one was re-armed.+ assert processed == 1+ rearm.assert_awaited_once_with(healthy)diff --git a/server/tests/test_deployment_canary_listener.py b/server/tests/test_deployment_canary_listener.pyindex a6a94a5e..2b9161d0 100644--- a/server/tests/test_deployment_canary_listener.py+++ b/server/tests/test_deployment_canary_listener.py@@ -305,8 +305,11 @@ class TestStartCanaryListener:assert listener_id == "wf-controlled-webhookTrigger"client.start_workflow.assert_not_awaited()+ # Addressed by workflow id only — the controller continue-as-news+ # to bound its history, so a run_id-pinned handle would target a+ # closed run after the first rollover.client.get_workflow_handle.assert_called_once_with(- "workflow-control-wf-controlled-g1", run_id="controller-run-1",+ "workflow-control-wf-controlled-g1",)signal_name, spec = controller_handle.signal.await_args.argsassert signal_name == "register_trigger"