Skip to content

Self-improving AI Employees for everyone

feat(temporal): cron Temporal-Schedule canary on cronScheduler (Wave 12 C3)

trohitg9314aff986c1
diff --git a/docs-internal/event_framework.md b/docs-internal/event_framework.md
index b6e9921c..b0d9763b 100644
--- a/docs-internal/event_framework.md
+++ b/docs-internal/event_framework.md
@@ -17,6 +17,7 @@ phase plan lives in `~/.claude/plans/properly-fix-the-tech-dreamy-tarjan.md`.
| C1 canary (webhookTrigger) — `TriggerListenerWorkflow` + Visibility-API registry | ✅ shipped 2026-05-14 (25 tests) |
| C1 rollout (chat / task / telegram / whatsapp) — plugin-self-registered via `canary_registry` | ✅ shipped 2026-05-14 |
| C2 canary (googleGmailReceive) — `PollingTriggerWorkflow` + `as_poll_activity` per-cycle activity | ✅ shipped 2026-05-15 (10 tests) |
+| C3 canary (cronScheduler) — Temporal Schedule + plugin-owned `CronTriggerWorkflow` via `SimplePlugin` | ✅ shipped 2026-05-15 (17 tests) |
| C2 — Polling triggers as long-lived workflows | ⏳ pending |
| C3 — APScheduler → Temporal Schedules | ⏳ pending |
| C4 — Close cross-plugin `_service` reaches (4 sites) | ⏳ pending |
diff --git a/server/nodes/scheduler/cron_scheduler/__init__.py b/server/nodes/scheduler/cron_scheduler/__init__.py
index 120239d7..1a3bc9a8 100644
--- a/server/nodes/scheduler/cron_scheduler/__init__.py
+++ b/server/nodes/scheduler/cron_scheduler/__init__.py
@@ -3,6 +3,21 @@
Cron-expression-based scheduling trigger. Deployment-mode lifecycle
(starting/stopping the cron job) is owned by ``deployment/triggers.py``;
the run-button path executes this plugin once for testing.
+
+Self-registration on import (Wave 12 C3 — Temporal Schedules canary):
+ - ``CronTriggerWorkflow`` (the per-firing workflow shipped in
+ ``._workflow``) is wrapped in :class:`temporalio.plugin.SimplePlugin`
+ and registered with
+ :func:`services.temporal.plugin_registry.register_temporal_plugin`.
+ The Temporal worker passes the registered list to
+ ``Worker(plugins=[...])`` and the SDK's plugin chain merges this
+ plugin's workflow class into the effective worker configuration —
+ framework worker stays plugin-agnostic.
+ - ``cronScheduler`` opts into the canary set via
+ :func:`services.deployment.canary_registry.register_canary_trigger_type`.
+ ``DeploymentManager._setup_cron_trigger`` checks the registry +
+ feature flag; when both are on, the deploy path creates a Temporal
+ ``Schedule`` instead of the in-process APScheduler job.
"""
from __future__ import annotations
@@ -13,12 +28,22 @@ from datetime import datetime, timedelta
from typing import Any, Dict, Literal, Optional
from pydantic import BaseModel, ConfigDict, Field
+from temporalio.plugin import SimplePlugin
from core.logging import get_logger
+from services.deployment.canary_registry import register_canary_trigger_type
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
+from services.temporal.plugin_registry import register_temporal_plugin
+
+from ._workflow import CronTriggerWorkflow
logger = get_logger(__name__)
+register_temporal_plugin(
+ SimplePlugin(name="cron-scheduler", workflows=[CronTriggerWorkflow])
+)
+register_canary_trigger_type("cronScheduler")
+
def _calculate_wait_seconds(p: Dict[str, Any]) -> int:
frequency = p.get("frequency", "minutes")
diff --git a/server/nodes/scheduler/cron_scheduler/_workflow.py b/server/nodes/scheduler/cron_scheduler/_workflow.py
new file mode 100644
index 00000000..4c0b7e21
--- /dev/null
+++ b/server/nodes/scheduler/cron_scheduler/_workflow.py
@@ -0,0 +1,156 @@
+"""Wave 12 C3: per-firing Temporal workflow for the cron trigger.
+
+Plugin-owned per RFC §6.1 — the cron-specific workflow lives in the
+cron_scheduler plugin folder, not at the framework level. The plugin's
+``__init__.py`` publishes the class via
+:func:`services.temporal.workflow_registry.register_temporal_workflow`;
+the Temporal worker collects it on startup.
+
+How it fits with the Temporal Schedule
+--------------------------------------
+
+Wave 12 C3 replaces APScheduler-driven cron with a Temporal Schedule
+(``client.create_schedule``). The Schedule's action is
+``ScheduleActionStartWorkflow`` targeting THIS class. Each firing
+(per the cron expression) starts one :class:`CronTriggerWorkflow`
+run; the run spawns a child :class:`MachinaWorkflow` with the cron
+trigger node pre-executed, then exits. ``parent_close_policy=ABANDON``
+keeps the spawned MachinaWorkflow alive after this workflow returns.
+
+Why we need a separate workflow (vs the Schedule starting MachinaWorkflow
+directly): Schedule action args are **frozen at create time**. Per-tick
+data (firing timestamp) must be computed inside a workflow. This thin
+shim does exactly that and nothing more.
+
+Refs:
+ - https://docs.temporal.io/develop/python/schedules
+ - https://docs.temporal.io/encyclopedia/scheduled-execution
+"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any, Dict
+
+from temporalio import workflow
+from temporalio.common import WorkflowIDReusePolicy
+from temporalio.workflow import ParentClosePolicy
+
+
[email protected](name="CronTriggerWorkflow", sandboxed=False)
+class CronTriggerWorkflow:
+ """One-shot per-firing workflow that spawns a child MachinaWorkflow.
+
+ Determinism note: the only mutable state is the firing-time
+ timestamp from ``workflow.now()`` (deterministic per run);
+ everything else is computed from the static action args.
+ """
+
+ @workflow.run
+ async def run(self, listener_data: Dict[str, Any]) -> Dict[str, Any]:
+ """Spawn one MachinaWorkflow child per cron firing.
+
+ ``listener_data`` shape (deployment-supplied, frozen at
+ schedule creation)::
+
+ {
+ "workflow_id": str, # MachinaOs deployment workflow_id
+ "trigger_node_id": str, # cron node id
+ "node_type": "cronScheduler",
+ "cron_expression": str, # raw crontab string
+ "frequency": str, # human-readable bucket
+ "timezone": str, # IANA tz name
+ "schedule": str, # human-readable description
+ "filter_params": Dict, # plugin params
+ "nodes": List[Dict], # full deployment graph snapshot
+ "edges": List[Dict],
+ "session_id": str,
+ "tenant_id": Optional[str],
+ }
+
+ Returns ``{spawned_child_id, timestamp}`` for the schedule's
+ per-firing history visibility.
+ """
+ trigger_output = _build_trigger_output(listener_data)
+
+ # Reuse the listener filter-graph helper so cron / push / poll
+ # canary paths share identical n8n stop-at-trigger / config-node /
+ # toolkit / agent-tool semantics — single source of truth.
+ from services.temporal.trigger_listener_workflow import _build_run_graph
+
+ trigger_node_id = listener_data["trigger_node_id"]
+ nodes = listener_data["nodes"]
+ edges = listener_data["edges"]
+ session_id = listener_data.get("session_id", "default")
+ deployment_workflow_id = listener_data.get("workflow_id")
+ tenant_id = listener_data.get("tenant_id")
+
+ filtered_nodes, filtered_edges = _build_run_graph(
+ trigger_node_id=trigger_node_id,
+ trigger_output=trigger_output,
+ nodes=nodes,
+ edges=edges,
+ )
+
+ # Schedule fires multiple times; the firing-time component in
+ # the child workflow id makes each tick unique.
+ # WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY guards
+ # against a duplicate at the same instant (Temporal retry of
+ # the schedule action) double-spawning a successful run.
+ firing_iso = trigger_output["timestamp"]
+ child_id = f"cron-{deployment_workflow_id}-{trigger_node_id}-{firing_iso}"
+
+ await workflow.start_child_workflow(
+ "MachinaWorkflow",
+ args=[{
+ "nodes": filtered_nodes,
+ "edges": filtered_edges,
+ "session_id": session_id,
+ "workflow_id": deployment_workflow_id,
+ "tenant_id": tenant_id,
+ }],
+ id=child_id,
+ parent_close_policy=ParentClosePolicy.ABANDON,
+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
+ execution_timeout=timedelta(hours=1),
+ run_timeout=timedelta(hours=1),
+ )
+
+ workflow.logger.info(
+ f"CronTriggerWorkflow spawned child run: child_id={child_id} "
+ f"timestamp={firing_iso}"
+ )
+
+ return {
+ "spawned_child_id": child_id,
+ "timestamp": firing_iso,
+ }
+
+
+def _build_trigger_output(listener_data: Dict[str, Any]) -> Dict[str, Any]:
+ """Construct the cron trigger's output payload for one firing.
+
+ Shape matches the pre-Wave-12 APScheduler tick callback in
+ ``DeploymentManager._setup_cron_trigger.on_tick`` so downstream
+ nodes that read ``{{cronTrigger.timestamp}}`` etc. keep working.
+
+ **Iteration counter trade-off**: APScheduler kept an in-memory
+ ``self._cron_iterations[node_id]`` that incremented per firing.
+ Temporal Schedules don't expose a built-in firing counter and
+ persisting one would require either a long-lived workflow (we'd
+ lose the one-shot simplicity) or DB writes on the hot path.
+ The canary intentionally sets ``iteration`` to ``None`` —
+ downstream nodes that need monotonic iteration can switch to the
+ firing ``timestamp`` (deterministic per run, sortable).
+ """
+ return {
+ "timestamp": workflow.now().isoformat(),
+ "iteration": None,
+ "frequency": listener_data.get("frequency"),
+ "timezone": listener_data.get("timezone"),
+ "schedule": listener_data.get("schedule"),
+ "cron_expression": listener_data.get("cron_expression"),
+ }
+
+
+__all__ = ["CronTriggerWorkflow"]
diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.py
index da43454f..a81b9b52 100644
--- a/server/services/deployment/manager.py
+++ b/server/services/deployment/manager.py
@@ -277,6 +277,12 @@ class DeploymentManager:
# deployment. Visibility-query-based; no local handle dict.
canary_count = await self._cancel_canary_listeners(workflow_id)
+ # Wave 12 C3 canary: delete Temporal Schedules created by this
+ # deployment's cron triggers. Different Temporal resource type
+ # from listener workflows so it needs its own sweep
+ # (client.list_schedules vs client.list_workflows).
+ cron_schedule_count = await self._cancel_canary_cron_schedules(workflow_id)
+
# Clear cron iteration counters for this workflow's cron nodes
for node_id in cron_node_ids:
self._cron_iterations.pop(node_id, None)
@@ -297,6 +303,7 @@ class DeploymentManager:
"crons_cancelled": cron_count,
"waiters_cancelled": waiter_count,
"canary_listeners_cancelled": canary_count,
+ "canary_cron_schedules_cancelled": cron_schedule_count,
"cancelled_listener_node_ids": listener_nodes
}
@@ -356,20 +363,51 @@ class DeploymentManager:
# =========================================================================
async def _setup_cron_trigger(self, node: Dict, workflow_id: str) -> TriggerInfo:
- """Setup cron trigger for a node."""
+ """Setup cron trigger for a node.
+
+ Two paths:
+
+ 1. **Wave 12 C3 canary**: when ``Settings.event_framework_enabled``
+ is on AND ``cronScheduler`` is in the canary registry, create
+ a Temporal :class:`Schedule` whose action is the plugin's
+ :class:`CronTriggerWorkflow`. Survives FastAPI process
+ restart via the Temporal Schedule service.
+ 2. **Legacy**: APScheduler ``register_cron_job`` runs an
+ in-process tick callback. Dies on process restart — kept as
+ the default while the canary stabilises.
+ """
node_id = node['id']
+ node_type = node.get('type', 'cronScheduler')
params = await self.database.get_node_parameters(node_id) or {}
cron_expr = TriggerManager.build_cron_expression(params)
timezone = params.get('timezone', 'UTC')
frequency = params.get('frequency', 'minutes')
+ schedule_desc = self._get_schedule_description(params)
- # Initialize iteration counter for this cron node
+ # Path 1: Wave 12 C3 canary — Temporal Schedule.
+ if await self._canary_listener_enabled_for(node_type):
+ schedule_id = await self._start_canary_cron_schedule(
+ node, workflow_id, params,
+ cron_expr=cron_expr,
+ timezone=timezone,
+ frequency=frequency,
+ schedule_desc=schedule_desc,
+ )
+ if schedule_id is not None:
+ await self._broadcaster.update_node_status(node_id, "waiting", {
+ "message": f"Waiting for schedule: {cron_expr} (Temporal-durable)",
+ "cron_expression": cron_expr,
+ "timezone": timezone,
+ "schedule_id": schedule_id,
+ }, workflow_id=workflow_id)
+ return TriggerInfo(node_id, "cron", job_id=schedule_id)
+ # Fall through to legacy path if Temporal unavailable
+ # (already logged inside _start_canary_cron_schedule).
+
+ # Initialize iteration counter for this cron node (legacy path).
self._cron_iterations[node_id] = 0
- # Build schedule description for output
- schedule_desc = self._get_schedule_description(params)
-
def on_tick():
if self._main_loop and self._main_loop.is_running():
# Increment iteration counter
@@ -719,6 +757,99 @@ class DeploymentManager:
)
return listener_id
+ async def _cancel_canary_cron_schedules(self, workflow_id: str) -> int:
+ """Wave 12 C3: delete every Temporal Schedule created for this
+ deployment's cron triggers.
+
+ Uses ``services.temporal.schedules.delete_cron_schedules_for_deployment``
+ which queries ``client.list_schedules`` with the ``EventWorkflowId``
+ Search Attribute filter. Same no-local-dict contract as
+ :meth:`_cancel_canary_listeners` but against the Schedule
+ resource type (Temporal Schedules and Workflows have separate
+ Visibility lists).
+ """
+ from core.container import container
+ from services.temporal.schedules import delete_cron_schedules_for_deployment
+
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ return 0
+
+ return await delete_cron_schedules_for_deployment(
+ wrapper.client, workflow_id,
+ )
+
+ async def _start_canary_cron_schedule(
+ self,
+ node: Dict,
+ workflow_id: str,
+ params: Dict,
+ *,
+ cron_expr: str,
+ timezone: str,
+ frequency: str,
+ schedule_desc: str,
+ ) -> Optional[str]:
+ """Wave 12 C3 canary: create a Temporal Schedule for a cron trigger.
+
+ Schedule action targets the plugin-owned ``CronTriggerWorkflow``
+ (registered as a :class:`temporalio.plugin.SimplePlugin` from
+ ``nodes/scheduler/cron_scheduler/__init__.py``). Each firing
+ spawns a child :class:`MachinaWorkflow`.
+
+ Returns the Schedule id on success, ``None`` when Temporal isn't
+ reachable so the caller falls back to APScheduler.
+ """
+ from core.container import container
+ from services.temporal.schedules import create_cron_schedule
+
+ node_id = node["id"]
+
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ logger.warning(
+ "Canary cron Schedule requested but Temporal not connected; "
+ "falling back to APScheduler",
+ node_id=node_id, workflow_id=workflow_id,
+ )
+ return None
+
+ state = self._deployments.get(workflow_id)
+ if state is None:
+ raise RuntimeError(f"No deployment state for workflow {workflow_id}")
+
+ listener_data: Dict[str, Any] = {
+ "workflow_id": workflow_id,
+ "trigger_node_id": node_id,
+ "node_type": node.get("type", "cronScheduler"),
+ "cron_expression": cron_expr,
+ "frequency": frequency,
+ "timezone": timezone,
+ "schedule": schedule_desc,
+ "filter_params": params,
+ "nodes": state.nodes,
+ "edges": state.edges,
+ "session_id": state.session_id,
+ }
+
+ schedule_id = await create_cron_schedule(
+ wrapper.client,
+ deployment_workflow_id=workflow_id,
+ node_id=node_id,
+ cron_expression=cron_expr,
+ timezone=timezone,
+ listener_data=listener_data,
+ )
+
+ logger.info(
+ "Canary cron Schedule created",
+ schedule_id=schedule_id,
+ workflow_id=workflow_id,
+ node_id=node_id,
+ cron_expression=cron_expr,
+ )
+ return schedule_id
+
async def _cancel_canary_listeners(self, workflow_id: str) -> int:
"""Cancel all canary listeners for this deployment.
diff --git a/server/services/temporal/plugin_registry.py b/server/services/temporal/plugin_registry.py
new file mode 100644
index 00000000..80f74fd0
--- /dev/null
+++ b/server/services/temporal/plugin_registry.py
@@ -0,0 +1,84 @@
+"""Plugin registry for Temporal SDK ``Worker(plugins=[...])``.
+
+Plugins that ship workflow classes / activities / interceptors build a
+:class:`temporalio.plugin.SimplePlugin` instance and register it from
+their ``__init__.py``. The Temporal worker passes the snapshot to
+``Worker(plugins=[...])`` once at startup; the Temporal SDK's
+chain-of-responsibility ``configure_worker()`` hook merges each plugin's
+contributions into the worker's effective workflow / activity list
+internally.
+
+Same mechanism as the Temporal SDK's own contrib plugins
+(``temporalio.contrib.openai_agents`` ships ``OpenAIAgentsPlugin`` for
+exactly this purpose). Documented at
+https://python.temporal.io/temporalio.plugin.SimplePlugin.html and
+https://docs.temporal.io/develop/plugins-guide .
+
+Why a registry rather than direct framework imports
+---------------------------------------------------
+
+The framework worker stays plugin-agnostic — no
+``from nodes.scheduler... import CronTriggerWorkflow`` at the framework
+level. Plugins push their ``SimplePlugin`` instances in; the worker
+reads the list and hands it to Temporal. Same direction-of-dependency
+contract the eight other Wave-11.I self-registration registries enforce
+(canary_trigger_type / poll_coroutine_factory / filter_builder /
+ws_handler / router / oauth_callback_path / service_refresh /
+output_schema / shutdown_hook / service_factory / social_send_handler).
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from temporalio.plugin import SimplePlugin
+
+from services.plugin.registry import IdempotentRegistry
+
+
+# Keyed by plugin name so duplicates raise at import time (e.g. two
+# plugins both declaring ``name="cron-scheduler"`` would surface a
+# namespace clash). SimplePlugin instances are content-comparable via
+# the IdempotentRegistry's qualname check (reload safe).
+_REGISTRY: IdempotentRegistry[str, SimplePlugin] = IdempotentRegistry(
+ "temporal_plugin"
+)
+
+
+def register_temporal_plugin(plugin: SimplePlugin) -> None:
+ """Publish a Temporal ``SimplePlugin`` for the worker.
+
+ Idempotent on re-import. Raises ``ValueError`` on a namespace
+ conflict (same ``plugin.name`` registered by a different caller).
+
+ Args:
+ plugin: A :class:`temporalio.plugin.SimplePlugin` instance.
+ The ``name`` field is used as the registry key; ``workflows``,
+ ``activities``, ``interceptors`` etc. surface in the
+ worker's effective configuration via Temporal's plugin chain.
+ """
+ # ``SimplePlugin.name`` is a method (per temporalio.plugin API) — call
+ # it to get the actual identifier string the registry keys on.
+ _REGISTRY.register(plugin.name(), plugin)
+
+
+def temporal_plugins() -> List[SimplePlugin]:
+ """Return every registered plugin (snapshot, in registration order).
+
+ The Temporal worker reads this once at startup::
+
+ worker = Worker(
+ client,
+ task_queue=...,
+ plugins=temporal_plugins(),
+ workflows=[MachinaWorkflow, ...],
+ activities=[...],
+ )
+ """
+ return list(_REGISTRY.values())
+
+
+__all__ = [
+ "register_temporal_plugin",
+ "temporal_plugins",
+]
diff --git a/server/services/temporal/schedules.py b/server/services/temporal/schedules.py
new file mode 100644
index 00000000..4c326c00
--- /dev/null
+++ b/server/services/temporal/schedules.py
@@ -0,0 +1,214 @@
+"""Wave 12 C3: Temporal Schedule helpers for cron triggers.
+
+Thin framework-level wrappers around the Temporal Schedule API. Plugin
+ownership stays in the plugin folder — this module's only job is the
+**mechanics** of creating + cancelling Schedules. The plugin's workflow
+class (``CronTriggerWorkflow``) and its specific action args are
+plugin-owned.
+
+Same shape as the Visibility-based cancel sweep in
+:meth:`services.deployment.manager.DeploymentManager._cancel_canary_listeners`
+— deterministic id, no in-memory tracking, server-side state is the
+registry. ``Schedule`` resources live in their OWN Visibility list
+(``client.list_schedules``), distinct from workflow Visibility.
+
+Refs:
+ - https://docs.temporal.io/develop/python/schedules
+ - https://python.temporal.io/temporalio.client.Schedule.html
+ - https://docs.temporal.io/encyclopedia/scheduled-execution
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional
+
+from temporalio.client import (
+ Client,
+ Schedule,
+ ScheduleActionStartWorkflow,
+ ScheduleAlreadyRunningError,
+ ScheduleIntervalSpec, # noqa: F401 — re-exported for any caller that needs it
+ ScheduleOverlapPolicy,
+ SchedulePolicy,
+ ScheduleSpec,
+)
+from temporalio.common import (
+ SearchAttributeKey,
+ SearchAttributePair,
+ TypedSearchAttributes,
+)
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+# Schedule ID prefix used by both create + cancel-sweep. Mirrors the
+# ``trigger-listener-{wf}-{node}`` deterministic-id shape that the C1
+# canary uses for its listener workflows — same locality + uniqueness
+# properties.
+def cron_schedule_id(deployment_workflow_id: str, node_id: str) -> str:
+ """Deterministic Schedule ID for a (deployment, cron-node) pair.
+
+ Mapped to the business entity so re-deploying the same MachinaOs
+ workflow targets the same Schedule. Pairs with the create call's
+ ``"already exists" → no-op`` semantics (per
+ :exc:`temporalio.client.ScheduleAlreadyRunningError`).
+ """
+ return f"cron-schedule-{deployment_workflow_id}-{node_id}"
+
+
+# Child workflow ID prefix Temporal stamps onto each firing. The
+# ``{{.ScheduledStartTime}}`` placeholder is substituted by Temporal
+# server-side, so each tick produces a unique workflow_id without us
+# needing per-tick code (per
+# https://docs.temporal.io/develop/python/schedules#use-a-pre-generated-workflow-id).
+def cron_action_workflow_id(deployment_workflow_id: str, node_id: str) -> str:
+ return f"cron-fire-{deployment_workflow_id}-{node_id}-{{{{.ScheduledStartTime}}}}"
+
+
+async def create_cron_schedule(
+ client: Client,
+ *,
+ deployment_workflow_id: str,
+ node_id: str,
+ cron_expression: str,
+ timezone: str,
+ listener_data: Dict[str, Any],
+ task_queue: str = "machina-tasks",
+ overlap_policy: ScheduleOverlapPolicy = ScheduleOverlapPolicy.SKIP,
+) -> str:
+ """Create-or-reuse a Temporal Schedule for a cron trigger.
+
+ Returns the Schedule's id. Idempotent: a re-deploy with the same
+ ``(deployment_workflow_id, node_id)`` pair reuses the existing
+ Schedule (Temporal raises
+ :exc:`ScheduleAlreadyRunningError` which we swallow as a no-op).
+
+ Args:
+ client: Connected Temporal client.
+ deployment_workflow_id: MachinaOs deployment workflow_id.
+ node_id: Cron-trigger node id.
+ cron_expression: Crontab string (5 or 6 field).
+ timezone: IANA tz name (e.g. ``"America/New_York"``).
+ listener_data: Frozen action args for the workflow run
+ (deployment graph snapshot + cron metadata).
+ task_queue: Worker task queue that hosts ``CronTriggerWorkflow``.
+ overlap_policy: How concurrent firings interact. ``SKIP`` (default)
+ drops a firing if the prior run is still going; mirrors the
+ pre-Wave-12 APScheduler behaviour for slow workflows.
+ """
+ schedule_id = cron_schedule_id(deployment_workflow_id, node_id)
+ action_workflow_id = cron_action_workflow_id(deployment_workflow_id, node_id)
+
+ schedule = Schedule(
+ action=ScheduleActionStartWorkflow(
+ "CronTriggerWorkflow",
+ args=[listener_data],
+ id=action_workflow_id,
+ task_queue=task_queue,
+ ),
+ spec=ScheduleSpec(
+ cron_expressions=[cron_expression],
+ time_zone_name=timezone or "UTC",
+ ),
+ policy=SchedulePolicy(overlap=overlap_policy),
+ )
+
+ # Search Attributes mirror the listener-canary contract so the
+ # cancel sweep can find the Schedule via the same EventWorkflowId
+ # filter. EventTriggerKind="cron" lets ops dashboards filter by
+ # canary kind independently of the underlying primitive. Note:
+ # ``search_attributes`` is a ``client.create_schedule`` kwarg, not
+ # a field on the ``Schedule`` dataclass itself (per the Temporal
+ # SDK API).
+ schedule_search_attributes = TypedSearchAttributes([
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("EventWorkflowId"),
+ deployment_workflow_id,
+ ),
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("TriggerNodeId"),
+ node_id,
+ ),
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("EventTriggerKind"),
+ "cron",
+ ),
+ ])
+
+ try:
+ await client.create_schedule(
+ schedule_id,
+ schedule,
+ search_attributes=schedule_search_attributes,
+ )
+ logger.info(
+ "Created Temporal cron Schedule",
+ schedule_id=schedule_id,
+ cron_expression=cron_expression,
+ timezone=timezone,
+ )
+ except ScheduleAlreadyRunningError:
+ # Idempotent re-deploy. The pre-existing Schedule keeps its
+ # state; the caller's create-or-reuse contract is satisfied.
+ logger.info(
+ "Temporal cron Schedule already exists (re-deploy idempotency)",
+ schedule_id=schedule_id,
+ )
+
+ return schedule_id
+
+
+async def delete_cron_schedules_for_deployment(
+ client: Client,
+ deployment_workflow_id: str,
+) -> int:
+ """Delete every cron Schedule for ``deployment_workflow_id``.
+
+ Visibility-equivalent sweep: ``client.list_schedules(query=...)``
+ is the registry, no local handle dict. Per-Schedule failures don't
+ block the sweep (mirror of
+ :meth:`DeploymentManager._cancel_canary_listeners` semantics).
+
+ Returns count of Schedules deleted.
+ """
+ query = (
+ f"EventWorkflowId='{deployment_workflow_id}' "
+ f"AND EventTriggerKind='cron'"
+ )
+
+ deleted = 0
+ try:
+ async for desc in client.list_schedules(query=query):
+ sched_id = desc.id
+ try:
+ handle = client.get_schedule_handle(sched_id)
+ await handle.delete()
+ deleted += 1
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ f"Failed to delete cron Schedule {sched_id!r}: {exc}",
+ deployment_workflow_id=deployment_workflow_id,
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ f"Visibility query for cron Schedules failed: {exc} "
+ f"(query={query!r})",
+ deployment_workflow_id=deployment_workflow_id,
+ )
+
+ if deleted:
+ logger.info(
+ "Cron Schedules deleted",
+ deployment_workflow_id=deployment_workflow_id,
+ count=deleted,
+ )
+ return deleted
+
+
+__all__ = [
+ "cron_schedule_id",
+ "create_cron_schedule",
+ "delete_cron_schedules_for_deployment",
+]
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index 9c8641ac..8e8fd784 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -29,6 +29,7 @@ from core.logging import get_logger
from .workflow import MachinaWorkflow
from .trigger_listener_workflow import TriggerListenerWorkflow
from .polling_trigger_workflow import PollingTriggerWorkflow
+from .plugin_registry import temporal_plugins
from .activities import (
NodeExecutionActivities,
create_shared_session,
@@ -143,9 +144,18 @@ class TemporalWorkerManager:
per_type = collect_plugin_activities() # no queue filter; all plugins
agent_activities = collect_agent_activities()
polling_activities = collect_polling_activities()
+ # Plugin-owned workflow classes (e.g. cron's
+ # CronTriggerWorkflow) self-register a SimplePlugin via
+ # services.temporal.plugin_registry.register_temporal_plugin
+ # from their plugin __init__.py. The Temporal SDK's plugin
+ # chain merges each registered plugin's workflows / activities
+ # / interceptors into the effective worker configuration —
+ # the framework worker stays plugin-agnostic.
+ plugin_list = temporal_plugins()
self._worker = Worker(
self.client,
task_queue=self.task_queue,
+ plugins=plugin_list,
workflows=[
MachinaWorkflow,
AgentWorkflow,
diff --git a/server/tests/test_cron_canary.py b/server/tests/test_cron_canary.py
new file mode 100644
index 00000000..f5efc61f
--- /dev/null
+++ b/server/tests/test_cron_canary.py
@@ -0,0 +1,440 @@
+"""Wave 12 C3: tests for the cron Temporal-Schedule canary.
+
+Three layers:
+
+1. **plugin_registry contract** — Temporal SimplePlugin registration is
+ idempotent, namespace-keyed, and produces a stable snapshot for the
+ Temporal worker's ``plugins=[]`` argument.
+
+2. **schedules helper** — deterministic ``cron_schedule_id`` derivation;
+ :func:`create_cron_schedule` calls ``client.create_schedule`` with
+ the SimplePlugin-targeting action + Search Attributes;
+ :func:`delete_cron_schedules_for_deployment` queries via Visibility
+ and graceful-deletes each.
+
+3. **DeploymentManager integration** — _start_canary_cron_schedule
+ builds the listener_data payload and calls into schedules.py;
+ _cancel_canary_cron_schedules dispatches into the delete sweep.
+
+4. **Plugin self-registration smoke** — importing
+ ``nodes.scheduler.cron_scheduler`` populates the plugin_registry
+ with the cron SimplePlugin AND the canary_registry with
+ ``cronScheduler``.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from typing import Any, Dict, List
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+
+if "machina" not in sys.modules:
+ _machina = types.ModuleType("machina")
+ _machina.__path__ = []
+ sys.modules["machina"] = _machina
+ _machina_tcp = types.ModuleType("machina.tcp")
+ _machina_tcp.probe_tcp_port = MagicMock(return_value=False)
+ sys.modules["machina.tcp"] = _machina_tcp
+
+
+def fresh_plugin_registry(monkeypatch):
+ """Reset plugin_registry's backing store for test isolation."""
+ from services.temporal import plugin_registry as pr
+
+ fresh = type(pr._REGISTRY)(pr._REGISTRY._name) # type: ignore[attr-defined]
+ monkeypatch.setattr(pr, "_REGISTRY", fresh)
+ return pr
+
+
+# ---------------------------------------------------------------------------
+# plugin_registry contract
+# ---------------------------------------------------------------------------
+
+
+class TestPluginRegistryContract:
+ """SimplePlugin registration shape."""
+
+ def test_empty_registry_yields_empty_list(self, fresh_plugin_registry):
+ assert fresh_plugin_registry.temporal_plugins() == []
+
+ def test_register_then_snapshot(self, fresh_plugin_registry):
+ from temporalio.plugin import SimplePlugin
+
+ plugin = SimplePlugin(name="test-plugin")
+ fresh_plugin_registry.register_temporal_plugin(plugin)
+
+ snap = fresh_plugin_registry.temporal_plugins()
+ assert snap == [plugin]
+
+ def test_idempotent_same_plugin_instance(self, fresh_plugin_registry):
+ from temporalio.plugin import SimplePlugin
+
+ plugin = SimplePlugin(name="test-plugin")
+ fresh_plugin_registry.register_temporal_plugin(plugin)
+ fresh_plugin_registry.register_temporal_plugin(plugin)
+ assert len(fresh_plugin_registry.temporal_plugins()) == 1
+
+ def test_conflicting_name_raises(self, fresh_plugin_registry):
+ from temporalio.plugin import SimplePlugin
+
+ a = SimplePlugin(name="dup-name")
+ b = SimplePlugin(name="dup-name")
+ fresh_plugin_registry.register_temporal_plugin(a)
+ with pytest.raises(ValueError, match="already registered"):
+ fresh_plugin_registry.register_temporal_plugin(b)
+
+ def test_distinct_names_coexist(self, fresh_plugin_registry):
+ from temporalio.plugin import SimplePlugin
+
+ p1 = SimplePlugin(name="plugin-1")
+ p2 = SimplePlugin(name="plugin-2")
+ fresh_plugin_registry.register_temporal_plugin(p1)
+ fresh_plugin_registry.register_temporal_plugin(p2)
+ assert {p.name() for p in fresh_plugin_registry.temporal_plugins()} == {
+ "plugin-1", "plugin-2",
+ }
+
+
+# ---------------------------------------------------------------------------
+# schedules helper
+# ---------------------------------------------------------------------------
+
+
+class TestScheduleIdDerivation:
+ def test_deterministic_id(self):
+ from services.temporal.schedules import cron_schedule_id
+
+ a = cron_schedule_id("wf-1", "cron-1")
+ b = cron_schedule_id("wf-1", "cron-1")
+ assert a == b == "cron-schedule-wf-1-cron-1"
+
+ def test_different_node_different_id(self):
+ from services.temporal.schedules import cron_schedule_id
+
+ assert cron_schedule_id("wf-1", "cron-a") != cron_schedule_id(
+ "wf-1", "cron-b",
+ )
+
+
+class TestCreateCronSchedule:
+ @pytest.mark.asyncio
+ async def test_create_schedule_payload_shape(self):
+ from services.temporal.schedules import create_cron_schedule
+
+ client = MagicMock()
+ client.create_schedule = AsyncMock()
+
+ listener_data = {
+ "workflow_id": "wf-1",
+ "trigger_node_id": "cron-1",
+ "node_type": "cronScheduler",
+ "cron_expression": "*/5 * * * *",
+ }
+
+ schedule_id = await create_cron_schedule(
+ client,
+ deployment_workflow_id="wf-1",
+ node_id="cron-1",
+ cron_expression="*/5 * * * *",
+ timezone="America/New_York",
+ listener_data=listener_data,
+ )
+
+ assert schedule_id == "cron-schedule-wf-1-cron-1"
+ assert client.create_schedule.await_count == 1
+
+ passed_id, passed_schedule = client.create_schedule.call_args.args
+ assert passed_id == schedule_id
+ assert passed_schedule.spec.cron_expressions == ["*/5 * * * *"]
+ assert passed_schedule.spec.time_zone_name == "America/New_York"
+
+ @pytest.mark.asyncio
+ async def test_create_schedule_idempotent_on_already_running(self):
+ """Re-deploy: Temporal raises ScheduleAlreadyRunningError; helper
+ swallows it so the deploy path keeps a deterministic id without
+ the caller needing to special-case retries."""
+ from temporalio.client import ScheduleAlreadyRunningError
+
+ from services.temporal.schedules import create_cron_schedule
+
+ client = MagicMock()
+ client.create_schedule = AsyncMock(side_effect=ScheduleAlreadyRunningError())
+
+ schedule_id = await create_cron_schedule(
+ client,
+ deployment_workflow_id="wf-1",
+ node_id="cron-1",
+ cron_expression="0 * * * *",
+ timezone="UTC",
+ listener_data={},
+ )
+
+ assert schedule_id == "cron-schedule-wf-1-cron-1"
+
+
+class TestDeleteCronSchedulesForDeployment:
+ @pytest.mark.asyncio
+ async def test_query_filters_by_workflow_id(self):
+ from services.temporal.schedules import (
+ delete_cron_schedules_for_deployment,
+ )
+
+ recorded_queries: List[str] = []
+ deleted_ids: List[str] = []
+
+ async def fake_list(query):
+ recorded_queries.append(query)
+ for sched_id in ["cron-schedule-wf-1-a", "cron-schedule-wf-1-b"]:
+ yield MagicMock(id=sched_id)
+
+ def fake_get_handle(sid):
+ handle = MagicMock()
+
+ async def fake_delete():
+ deleted_ids.append(sid)
+
+ handle.delete = fake_delete
+ return handle
+
+ client = MagicMock()
+ client.list_schedules = fake_list
+ client.get_schedule_handle = fake_get_handle
+
+ count = await delete_cron_schedules_for_deployment(client, "wf-1")
+
+ assert count == 2
+ assert sorted(deleted_ids) == [
+ "cron-schedule-wf-1-a", "cron-schedule-wf-1-b",
+ ]
+ # Filter on both deployment workflow_id AND the cron kind tag.
+ assert len(recorded_queries) == 1
+ q = recorded_queries[0]
+ assert "EventWorkflowId='wf-1'" in q
+ assert "EventTriggerKind='cron'" in q
+
+ @pytest.mark.asyncio
+ async def test_per_schedule_failure_does_not_block_sweep(self):
+ from services.temporal.schedules import (
+ delete_cron_schedules_for_deployment,
+ )
+
+ async def fake_list(query):
+ for sid in ["cron-schedule-wf-1-good", "cron-schedule-wf-1-bad"]:
+ yield MagicMock(id=sid)
+
+ def fake_get_handle(sid):
+ handle = MagicMock()
+ if "bad" in sid:
+ async def boom():
+ raise RuntimeError("simulated delete failure")
+ handle.delete = boom
+ else:
+ async def ok():
+ return None
+ handle.delete = ok
+ return handle
+
+ client = MagicMock()
+ client.list_schedules = fake_list
+ client.get_schedule_handle = fake_get_handle
+
+ count = await delete_cron_schedules_for_deployment(client, "wf-1")
+ # Only the good one deleted; the bad one's failure was logged
+ # + skipped.
+ assert count == 1
+
+
+# ---------------------------------------------------------------------------
+# DeploymentManager cron canary integration
+# ---------------------------------------------------------------------------
+
+
+def _node(node_id: str, node_type: str) -> Dict[str, Any]:
+ return {"id": node_id, "type": node_type, "data": {}}
+
+
+def _build_manager(workflow_id: str, nodes=None, edges=None, session_id="sess"):
+ from services.deployment.manager import DeploymentManager
+ from services.deployment.state import DeploymentState
+
+ database = MagicMock()
+ database.get_node_parameters = AsyncMock(return_value={})
+ broadcaster = MagicMock()
+ broadcaster.update_node_status = AsyncMock()
+
+ mgr = DeploymentManager(
+ database=database,
+ execute_workflow_fn=AsyncMock(),
+ store_output_fn=AsyncMock(),
+ broadcaster=broadcaster,
+ )
+ mgr._deployments[workflow_id] = DeploymentState(
+ deployment_id=f"deploy_{workflow_id}",
+ workflow_id=workflow_id,
+ is_running=True,
+ nodes=nodes or [],
+ edges=edges or [],
+ session_id=session_id,
+ )
+ return mgr
+
+
+class TestDeploymentCronCanaryRouting:
+ @pytest.mark.asyncio
+ async def test_start_canary_cron_schedule_calls_helper(self, monkeypatch):
+ """The deployment helper threads the listener_data payload into
+ create_cron_schedule with the right shape."""
+ mgr = _build_manager(
+ "wf-1",
+ nodes=[_node("cron-1", "cronScheduler")],
+ edges=[],
+ )
+
+ recorded: List[Dict[str, Any]] = []
+
+ async def fake_create(client, *, deployment_workflow_id, node_id,
+ cron_expression, timezone, listener_data, **kw):
+ recorded.append({
+ "deployment_workflow_id": deployment_workflow_id,
+ "node_id": node_id,
+ "cron_expression": cron_expression,
+ "timezone": timezone,
+ "listener_data": listener_data,
+ })
+ return f"cron-schedule-{deployment_workflow_id}-{node_id}"
+
+ from services.temporal import schedules
+ monkeypatch.setattr(schedules, "create_cron_schedule", fake_create)
+
+ wrapper = MagicMock()
+ wrapper.client = MagicMock()
+
+ from core import container as container_mod
+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)
+
+ result = await mgr._start_canary_cron_schedule(
+ _node("cron-1", "cronScheduler"),
+ "wf-1",
+ params={"cron": "*"},
+ cron_expr="*/5 * * * *",
+ timezone="UTC",
+ frequency="minutes",
+ schedule_desc="Every 5 minutes",
+ )
+
+ assert result == "cron-schedule-wf-1-cron-1"
+ assert len(recorded) == 1
+ call = recorded[0]
+ assert call["cron_expression"] == "*/5 * * * *"
+ assert call["timezone"] == "UTC"
+ # listener_data carries the graph snapshot + cron metadata.
+ ld = call["listener_data"]
+ assert ld["workflow_id"] == "wf-1"
+ assert ld["trigger_node_id"] == "cron-1"
+ assert ld["cron_expression"] == "*/5 * * * *"
+ assert ld["schedule"] == "Every 5 minutes"
+
+ @pytest.mark.asyncio
+ async def test_start_returns_none_when_temporal_not_connected(self, monkeypatch):
+ mgr = _build_manager(
+ "wf-1",
+ nodes=[_node("cron-1", "cronScheduler")],
+ edges=[],
+ )
+
+ wrapper = MagicMock()
+ wrapper.client = None
+
+ from core import container as container_mod
+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)
+
+ result = await mgr._start_canary_cron_schedule(
+ _node("cron-1", "cronScheduler"),
+ "wf-1",
+ params={},
+ cron_expr="0 * * * *",
+ timezone="UTC",
+ frequency="hours",
+ schedule_desc="Every hour",
+ )
+
+ assert result is None
+
+ @pytest.mark.asyncio
+ async def test_cancel_dispatches_into_delete_sweep(self, monkeypatch):
+ mgr = _build_manager("wf-1")
+
+ delete_calls: List[str] = []
+
+ async def fake_delete(client, deployment_workflow_id):
+ delete_calls.append(deployment_workflow_id)
+ return 3
+
+ from services.temporal import schedules
+ monkeypatch.setattr(
+ schedules,
+ "delete_cron_schedules_for_deployment",
+ fake_delete,
+ )
+
+ wrapper = MagicMock()
+ wrapper.client = MagicMock()
+
+ from core import container as container_mod
+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)
+
+ count = await mgr._cancel_canary_cron_schedules("wf-1")
+ assert count == 3
+ assert delete_calls == ["wf-1"]
+
+ @pytest.mark.asyncio
+ async def test_cancel_returns_zero_when_temporal_disconnected(self, monkeypatch):
+ mgr = _build_manager("wf-1")
+
+ wrapper = MagicMock()
+ wrapper.client = None
+
+ from core import container as container_mod
+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)
+
+ assert await mgr._cancel_canary_cron_schedules("wf-1") == 0
+
+
+# ---------------------------------------------------------------------------
+# Plugin self-registration smoke
+# ---------------------------------------------------------------------------
+
+
+class TestCronPluginSelfRegisters:
+ def test_plugin_import_registers_simple_plugin(self):
+ try:
+ __import__("nodes.scheduler.cron_scheduler")
+ except ImportError as exc: # pragma: no cover
+ pytest.xfail(f"cron_scheduler not importable: {exc}")
+
+ from services.temporal.plugin_registry import temporal_plugins
+
+ names = {p.name() for p in temporal_plugins()}
+ assert "cron-scheduler" in names, (
+ "Importing nodes.scheduler.cron_scheduler should register a "
+ "SimplePlugin named 'cron-scheduler'. Check the __init__.py "
+ "bottom section."
+ )
+
+ def test_plugin_import_registers_canary(self):
+ try:
+ __import__("nodes.scheduler.cron_scheduler")
+ except ImportError as exc: # pragma: no cover
+ pytest.xfail(f"cron_scheduler not importable: {exc}")
+
+ from services.deployment import canary_registry
+
+ assert canary_registry.is_canary_trigger_type("cronScheduler"), (
+ "Importing nodes.scheduler.cron_scheduler should call "
+ "register_canary_trigger_type('cronScheduler')."
+ )