Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(temporal): cron Temporal-Schedule canary on cronScheduler (Wave 12 C3)
trohitg9314aff986c1
diff --git a/docs-internal/event_framework.md b/docs-internal/event_framework.mdindex b6e9921c..b0d9763b 100644--- a/docs-internal/event_framework.md+++ b/docs-internal/event_framework.md@@ -17,6 +17,7 @@ phase plan lives in `~/.claude/plans/properly-fix-the-tech-dreamy-tarjan.md`.| C1 canary (webhookTrigger) — `TriggerListenerWorkflow` + Visibility-API registry | ✅ shipped 2026-05-14 (25 tests) || C1 rollout (chat / task / telegram / whatsapp) — plugin-self-registered via `canary_registry` | ✅ shipped 2026-05-14 || C2 canary (googleGmailReceive) — `PollingTriggerWorkflow` + `as_poll_activity` per-cycle activity | ✅ shipped 2026-05-15 (10 tests) |+| C3 canary (cronScheduler) — Temporal Schedule + plugin-owned `CronTriggerWorkflow` via `SimplePlugin` | ✅ shipped 2026-05-15 (17 tests) || C2 — Polling triggers as long-lived workflows | ⏳ pending || C3 — APScheduler → Temporal Schedules | ⏳ pending || C4 — Close cross-plugin `_service` reaches (4 sites) | ⏳ pending |diff --git a/server/nodes/scheduler/cron_scheduler/__init__.py b/server/nodes/scheduler/cron_scheduler/__init__.pyindex 120239d7..1a3bc9a8 100644--- a/server/nodes/scheduler/cron_scheduler/__init__.py+++ b/server/nodes/scheduler/cron_scheduler/__init__.py@@ -3,6 +3,21 @@Cron-expression-based scheduling trigger. Deployment-mode lifecycle(starting/stopping the cron job) is owned by ``deployment/triggers.py``;the run-button path executes this plugin once for testing.++Self-registration on import (Wave 12 C3 — Temporal Schedules canary):+ - ``CronTriggerWorkflow`` (the per-firing workflow shipped in+ ``._workflow``) is wrapped in :class:`temporalio.plugin.SimplePlugin`+ and registered with+ :func:`services.temporal.plugin_registry.register_temporal_plugin`.+ The Temporal worker passes the registered list to+ ``Worker(plugins=[...])`` and the SDK's plugin chain merges this+ plugin's workflow class into the effective worker configuration —+ framework worker stays plugin-agnostic.+ - ``cronScheduler`` opts into the canary set via+ :func:`services.deployment.canary_registry.register_canary_trigger_type`.+ ``DeploymentManager._setup_cron_trigger`` checks the registry ++ feature flag; when both are on, the deploy path creates a Temporal+ ``Schedule`` instead of the in-process APScheduler job."""from __future__ import annotations@@ -13,12 +28,22 @@ from datetime import datetime, timedeltafrom typing import Any, Dict, Literal, Optionalfrom pydantic import BaseModel, ConfigDict, Field+from temporalio.plugin import SimplePluginfrom core.logging import get_logger+from services.deployment.canary_registry import register_canary_trigger_typefrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue+from services.temporal.plugin_registry import register_temporal_plugin++from ._workflow import CronTriggerWorkflowlogger = get_logger(__name__)+register_temporal_plugin(+ SimplePlugin(name="cron-scheduler", workflows=[CronTriggerWorkflow])+)+register_canary_trigger_type("cronScheduler")+def _calculate_wait_seconds(p: Dict[str, Any]) -> int:frequency = p.get("frequency", "minutes")diff --git a/server/nodes/scheduler/cron_scheduler/_workflow.py b/server/nodes/scheduler/cron_scheduler/_workflow.pynew file mode 100644index 00000000..4c0b7e21--- /dev/null+++ b/server/nodes/scheduler/cron_scheduler/_workflow.py@@ -0,0 +1,156 @@+"""Wave 12 C3: per-firing Temporal workflow for the cron trigger.++Plugin-owned per RFC §6.1 — the cron-specific workflow lives in the+cron_scheduler plugin folder, not at the framework level. The plugin's+``__init__.py`` publishes the class via+:func:`services.temporal.workflow_registry.register_temporal_workflow`;+the Temporal worker collects it on startup.++How it fits with the Temporal Schedule+--------------------------------------++Wave 12 C3 replaces APScheduler-driven cron with a Temporal Schedule+(``client.create_schedule``). The Schedule's action is+``ScheduleActionStartWorkflow`` targeting THIS class. Each firing+(per the cron expression) starts one :class:`CronTriggerWorkflow`+run; the run spawns a child :class:`MachinaWorkflow` with the cron+trigger node pre-executed, then exits. ``parent_close_policy=ABANDON``+keeps the spawned MachinaWorkflow alive after this workflow returns.++Why we need a separate workflow (vs the Schedule starting MachinaWorkflow+directly): Schedule action args are **frozen at create time**. Per-tick+data (firing timestamp) must be computed inside a workflow. This thin+shim does exactly that and nothing more.++Refs:+ - https://docs.temporal.io/develop/python/schedules+ - https://docs.temporal.io/encyclopedia/scheduled-execution+"""++from __future__ import annotations++from datetime import timedelta+from typing import Any, Dict++from temporalio import workflow+from temporalio.common import WorkflowIDReusePolicy+from temporalio.workflow import ParentClosePolicy++[email protected](name="CronTriggerWorkflow", sandboxed=False)+class CronTriggerWorkflow:+ """One-shot per-firing workflow that spawns a child MachinaWorkflow.++ Determinism note: the only mutable state is the firing-time+ timestamp from ``workflow.now()`` (deterministic per run);+ everything else is computed from the static action args.+ """++ @workflow.run+ async def run(self, listener_data: Dict[str, Any]) -> Dict[str, Any]:+ """Spawn one MachinaWorkflow child per cron firing.++ ``listener_data`` shape (deployment-supplied, frozen at+ schedule creation)::++ {+ "workflow_id": str, # MachinaOs deployment workflow_id+ "trigger_node_id": str, # cron node id+ "node_type": "cronScheduler",+ "cron_expression": str, # raw crontab string+ "frequency": str, # human-readable bucket+ "timezone": str, # IANA tz name+ "schedule": str, # human-readable description+ "filter_params": Dict, # plugin params+ "nodes": List[Dict], # full deployment graph snapshot+ "edges": List[Dict],+ "session_id": str,+ "tenant_id": Optional[str],+ }++ Returns ``{spawned_child_id, timestamp}`` for the schedule's+ per-firing history visibility.+ """+ trigger_output = _build_trigger_output(listener_data)++ # Reuse the listener filter-graph helper so cron / push / poll+ # canary paths share identical n8n stop-at-trigger / config-node /+ # toolkit / agent-tool semantics — single source of truth.+ from services.temporal.trigger_listener_workflow import _build_run_graph++ trigger_node_id = listener_data["trigger_node_id"]+ nodes = listener_data["nodes"]+ edges = listener_data["edges"]+ session_id = listener_data.get("session_id", "default")+ deployment_workflow_id = listener_data.get("workflow_id")+ tenant_id = listener_data.get("tenant_id")++ filtered_nodes, filtered_edges = _build_run_graph(+ trigger_node_id=trigger_node_id,+ trigger_output=trigger_output,+ nodes=nodes,+ edges=edges,+ )++ # Schedule fires multiple times; the firing-time component in+ # the child workflow id makes each tick unique.+ # WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY guards+ # against a duplicate at the same instant (Temporal retry of+ # the schedule action) double-spawning a successful run.+ firing_iso = trigger_output["timestamp"]+ child_id = f"cron-{deployment_workflow_id}-{trigger_node_id}-{firing_iso}"++ await workflow.start_child_workflow(+ "MachinaWorkflow",+ args=[{+ "nodes": filtered_nodes,+ "edges": filtered_edges,+ "session_id": session_id,+ "workflow_id": deployment_workflow_id,+ "tenant_id": tenant_id,+ }],+ id=child_id,+ parent_close_policy=ParentClosePolicy.ABANDON,+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,+ execution_timeout=timedelta(hours=1),+ run_timeout=timedelta(hours=1),+ )++ workflow.logger.info(+ f"CronTriggerWorkflow spawned child run: child_id={child_id} "+ f"timestamp={firing_iso}"+ )++ return {+ "spawned_child_id": child_id,+ "timestamp": firing_iso,+ }+++def _build_trigger_output(listener_data: Dict[str, Any]) -> Dict[str, Any]:+ """Construct the cron trigger's output payload for one firing.++ Shape matches the pre-Wave-12 APScheduler tick callback in+ ``DeploymentManager._setup_cron_trigger.on_tick`` so downstream+ nodes that read ``{{cronTrigger.timestamp}}`` etc. keep working.++ **Iteration counter trade-off**: APScheduler kept an in-memory+ ``self._cron_iterations[node_id]`` that incremented per firing.+ Temporal Schedules don't expose a built-in firing counter and+ persisting one would require either a long-lived workflow (we'd+ lose the one-shot simplicity) or DB writes on the hot path.+ The canary intentionally sets ``iteration`` to ``None`` —+ downstream nodes that need monotonic iteration can switch to the+ firing ``timestamp`` (deterministic per run, sortable).+ """+ return {+ "timestamp": workflow.now().isoformat(),+ "iteration": None,+ "frequency": listener_data.get("frequency"),+ "timezone": listener_data.get("timezone"),+ "schedule": listener_data.get("schedule"),+ "cron_expression": listener_data.get("cron_expression"),+ }+++__all__ = ["CronTriggerWorkflow"]diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.pyindex da43454f..a81b9b52 100644--- a/server/services/deployment/manager.py+++ b/server/services/deployment/manager.py@@ -277,6 +277,12 @@ class DeploymentManager:# deployment. Visibility-query-based; no local handle dict.canary_count = await self._cancel_canary_listeners(workflow_id)+ # Wave 12 C3 canary: delete Temporal Schedules created by this+ # deployment's cron triggers. Different Temporal resource type+ # from listener workflows so it needs its own sweep+ # (client.list_schedules vs client.list_workflows).+ cron_schedule_count = await self._cancel_canary_cron_schedules(workflow_id)+# Clear cron iteration counters for this workflow's cron nodesfor node_id in cron_node_ids:self._cron_iterations.pop(node_id, None)@@ -297,6 +303,7 @@ class DeploymentManager:"crons_cancelled": cron_count,"waiters_cancelled": waiter_count,"canary_listeners_cancelled": canary_count,+ "canary_cron_schedules_cancelled": cron_schedule_count,"cancelled_listener_node_ids": listener_nodes}@@ -356,20 +363,51 @@ class DeploymentManager:# =========================================================================async def _setup_cron_trigger(self, node: Dict, workflow_id: str) -> TriggerInfo:- """Setup cron trigger for a node."""+ """Setup cron trigger for a node.++ Two paths:++ 1. **Wave 12 C3 canary**: when ``Settings.event_framework_enabled``+ is on AND ``cronScheduler`` is in the canary registry, create+ a Temporal :class:`Schedule` whose action is the plugin's+ :class:`CronTriggerWorkflow`. Survives FastAPI process+ restart via the Temporal Schedule service.+ 2. **Legacy**: APScheduler ``register_cron_job`` runs an+ in-process tick callback. Dies on process restart — kept as+ the default while the canary stabilises.+ """node_id = node['id']+ node_type = node.get('type', 'cronScheduler')params = await self.database.get_node_parameters(node_id) or {}cron_expr = TriggerManager.build_cron_expression(params)timezone = params.get('timezone', 'UTC')frequency = params.get('frequency', 'minutes')+ schedule_desc = self._get_schedule_description(params)- # Initialize iteration counter for this cron node+ # Path 1: Wave 12 C3 canary — Temporal Schedule.+ if await self._canary_listener_enabled_for(node_type):+ schedule_id = await self._start_canary_cron_schedule(+ node, workflow_id, params,+ cron_expr=cron_expr,+ timezone=timezone,+ frequency=frequency,+ schedule_desc=schedule_desc,+ )+ if schedule_id is not None:+ await self._broadcaster.update_node_status(node_id, "waiting", {+ "message": f"Waiting for schedule: {cron_expr} (Temporal-durable)",+ "cron_expression": cron_expr,+ "timezone": timezone,+ "schedule_id": schedule_id,+ }, workflow_id=workflow_id)+ return TriggerInfo(node_id, "cron", job_id=schedule_id)+ # Fall through to legacy path if Temporal unavailable+ # (already logged inside _start_canary_cron_schedule).++ # Initialize iteration counter for this cron node (legacy path).self._cron_iterations[node_id] = 0- # Build schedule description for output- schedule_desc = self._get_schedule_description(params)-def on_tick():if self._main_loop and self._main_loop.is_running():# Increment iteration counter@@ -719,6 +757,99 @@ class DeploymentManager:)return listener_id+ async def _cancel_canary_cron_schedules(self, workflow_id: str) -> int:+ """Wave 12 C3: delete every Temporal Schedule created for this+ deployment's cron triggers.++ Uses ``services.temporal.schedules.delete_cron_schedules_for_deployment``+ which queries ``client.list_schedules`` with the ``EventWorkflowId``+ Search Attribute filter. Same no-local-dict contract as+ :meth:`_cancel_canary_listeners` but against the Schedule+ resource type (Temporal Schedules and Workflows have separate+ Visibility lists).+ """+ from core.container import container+ from services.temporal.schedules import delete_cron_schedules_for_deployment++ wrapper = container.temporal_client()+ if wrapper is None or wrapper.client is None:+ return 0++ return await delete_cron_schedules_for_deployment(+ wrapper.client, workflow_id,+ )++ async def _start_canary_cron_schedule(+ self,+ node: Dict,+ workflow_id: str,+ params: Dict,+ *,+ cron_expr: str,+ timezone: str,+ frequency: str,+ schedule_desc: str,+ ) -> Optional[str]:+ """Wave 12 C3 canary: create a Temporal Schedule for a cron trigger.++ Schedule action targets the plugin-owned ``CronTriggerWorkflow``+ (registered as a :class:`temporalio.plugin.SimplePlugin` from+ ``nodes/scheduler/cron_scheduler/__init__.py``). Each firing+ spawns a child :class:`MachinaWorkflow`.++ Returns the Schedule id on success, ``None`` when Temporal isn't+ reachable so the caller falls back to APScheduler.+ """+ from core.container import container+ from services.temporal.schedules import create_cron_schedule++ node_id = node["id"]++ wrapper = container.temporal_client()+ if wrapper is None or wrapper.client is None:+ logger.warning(+ "Canary cron Schedule requested but Temporal not connected; "+ "falling back to APScheduler",+ node_id=node_id, workflow_id=workflow_id,+ )+ return None++ state = self._deployments.get(workflow_id)+ if state is None:+ raise RuntimeError(f"No deployment state for workflow {workflow_id}")++ listener_data: Dict[str, Any] = {+ "workflow_id": workflow_id,+ "trigger_node_id": node_id,+ "node_type": node.get("type", "cronScheduler"),+ "cron_expression": cron_expr,+ "frequency": frequency,+ "timezone": timezone,+ "schedule": schedule_desc,+ "filter_params": params,+ "nodes": state.nodes,+ "edges": state.edges,+ "session_id": state.session_id,+ }++ schedule_id = await create_cron_schedule(+ wrapper.client,+ deployment_workflow_id=workflow_id,+ node_id=node_id,+ cron_expression=cron_expr,+ timezone=timezone,+ listener_data=listener_data,+ )++ logger.info(+ "Canary cron Schedule created",+ schedule_id=schedule_id,+ workflow_id=workflow_id,+ node_id=node_id,+ cron_expression=cron_expr,+ )+ return schedule_id+async def _cancel_canary_listeners(self, workflow_id: str) -> int:"""Cancel all canary listeners for this deployment.diff --git a/server/services/temporal/plugin_registry.py b/server/services/temporal/plugin_registry.pynew file mode 100644index 00000000..80f74fd0--- /dev/null+++ b/server/services/temporal/plugin_registry.py@@ -0,0 +1,84 @@+"""Plugin registry for Temporal SDK ``Worker(plugins=[...])``.++Plugins that ship workflow classes / activities / interceptors build a+:class:`temporalio.plugin.SimplePlugin` instance and register it from+their ``__init__.py``. The Temporal worker passes the snapshot to+``Worker(plugins=[...])`` once at startup; the Temporal SDK's+chain-of-responsibility ``configure_worker()`` hook merges each plugin's+contributions into the worker's effective workflow / activity list+internally.++Same mechanism as the Temporal SDK's own contrib plugins+(``temporalio.contrib.openai_agents`` ships ``OpenAIAgentsPlugin`` for+exactly this purpose). Documented at+https://python.temporal.io/temporalio.plugin.SimplePlugin.html and+https://docs.temporal.io/develop/plugins-guide .++Why a registry rather than direct framework imports+---------------------------------------------------++The framework worker stays plugin-agnostic — no+``from nodes.scheduler... import CronTriggerWorkflow`` at the framework+level. Plugins push their ``SimplePlugin`` instances in; the worker+reads the list and hands it to Temporal. Same direction-of-dependency+contract the eight other Wave-11.I self-registration registries enforce+(canary_trigger_type / poll_coroutine_factory / filter_builder /+ws_handler / router / oauth_callback_path / service_refresh /+output_schema / shutdown_hook / service_factory / social_send_handler).+"""++from __future__ import annotations++from typing import List++from temporalio.plugin import SimplePlugin++from services.plugin.registry import IdempotentRegistry+++# Keyed by plugin name so duplicates raise at import time (e.g. two+# plugins both declaring ``name="cron-scheduler"`` would surface a+# namespace clash). SimplePlugin instances are content-comparable via+# the IdempotentRegistry's qualname check (reload safe).+_REGISTRY: IdempotentRegistry[str, SimplePlugin] = IdempotentRegistry(+ "temporal_plugin"+)+++def register_temporal_plugin(plugin: SimplePlugin) -> None:+ """Publish a Temporal ``SimplePlugin`` for the worker.++ Idempotent on re-import. Raises ``ValueError`` on a namespace+ conflict (same ``plugin.name`` registered by a different caller).++ Args:+ plugin: A :class:`temporalio.plugin.SimplePlugin` instance.+ The ``name`` field is used as the registry key; ``workflows``,+ ``activities``, ``interceptors`` etc. surface in the+ worker's effective configuration via Temporal's plugin chain.+ """+ # ``SimplePlugin.name`` is a method (per temporalio.plugin API) — call+ # it to get the actual identifier string the registry keys on.+ _REGISTRY.register(plugin.name(), plugin)+++def temporal_plugins() -> List[SimplePlugin]:+ """Return every registered plugin (snapshot, in registration order).++ The Temporal worker reads this once at startup::++ worker = Worker(+ client,+ task_queue=...,+ plugins=temporal_plugins(),+ workflows=[MachinaWorkflow, ...],+ activities=[...],+ )+ """+ return list(_REGISTRY.values())+++__all__ = [+ "register_temporal_plugin",+ "temporal_plugins",+]diff --git a/server/services/temporal/schedules.py b/server/services/temporal/schedules.pynew file mode 100644index 00000000..4c326c00--- /dev/null+++ b/server/services/temporal/schedules.py@@ -0,0 +1,214 @@+"""Wave 12 C3: Temporal Schedule helpers for cron triggers.++Thin framework-level wrappers around the Temporal Schedule API. Plugin+ownership stays in the plugin folder — this module's only job is the+**mechanics** of creating + cancelling Schedules. The plugin's workflow+class (``CronTriggerWorkflow``) and its specific action args are+plugin-owned.++Same shape as the Visibility-based cancel sweep in+:meth:`services.deployment.manager.DeploymentManager._cancel_canary_listeners`+— deterministic id, no in-memory tracking, server-side state is the+registry. ``Schedule`` resources live in their OWN Visibility list+(``client.list_schedules``), distinct from workflow Visibility.++Refs:+ - https://docs.temporal.io/develop/python/schedules+ - https://python.temporal.io/temporalio.client.Schedule.html+ - https://docs.temporal.io/encyclopedia/scheduled-execution+"""++from __future__ import annotations++from typing import Any, Dict, Iterable, List, Optional++from temporalio.client import (+ Client,+ Schedule,+ ScheduleActionStartWorkflow,+ ScheduleAlreadyRunningError,+ ScheduleIntervalSpec, # noqa: F401 — re-exported for any caller that needs it+ ScheduleOverlapPolicy,+ SchedulePolicy,+ ScheduleSpec,+)+from temporalio.common import (+ SearchAttributeKey,+ SearchAttributePair,+ TypedSearchAttributes,+)++from core.logging import get_logger++logger = get_logger(__name__)+++# Schedule ID prefix used by both create + cancel-sweep. Mirrors the+# ``trigger-listener-{wf}-{node}`` deterministic-id shape that the C1+# canary uses for its listener workflows — same locality + uniqueness+# properties.+def cron_schedule_id(deployment_workflow_id: str, node_id: str) -> str:+ """Deterministic Schedule ID for a (deployment, cron-node) pair.++ Mapped to the business entity so re-deploying the same MachinaOs+ workflow targets the same Schedule. Pairs with the create call's+ ``"already exists" → no-op`` semantics (per+ :exc:`temporalio.client.ScheduleAlreadyRunningError`).+ """+ return f"cron-schedule-{deployment_workflow_id}-{node_id}"+++# Child workflow ID prefix Temporal stamps onto each firing. The+# ``{{.ScheduledStartTime}}`` placeholder is substituted by Temporal+# server-side, so each tick produces a unique workflow_id without us+# needing per-tick code (per+# https://docs.temporal.io/develop/python/schedules#use-a-pre-generated-workflow-id).+def cron_action_workflow_id(deployment_workflow_id: str, node_id: str) -> str:+ return f"cron-fire-{deployment_workflow_id}-{node_id}-{{{{.ScheduledStartTime}}}}"+++async def create_cron_schedule(+ client: Client,+ *,+ deployment_workflow_id: str,+ node_id: str,+ cron_expression: str,+ timezone: str,+ listener_data: Dict[str, Any],+ task_queue: str = "machina-tasks",+ overlap_policy: ScheduleOverlapPolicy = ScheduleOverlapPolicy.SKIP,+) -> str:+ """Create-or-reuse a Temporal Schedule for a cron trigger.++ Returns the Schedule's id. Idempotent: a re-deploy with the same+ ``(deployment_workflow_id, node_id)`` pair reuses the existing+ Schedule (Temporal raises+ :exc:`ScheduleAlreadyRunningError` which we swallow as a no-op).++ Args:+ client: Connected Temporal client.+ deployment_workflow_id: MachinaOs deployment workflow_id.+ node_id: Cron-trigger node id.+ cron_expression: Crontab string (5 or 6 field).+ timezone: IANA tz name (e.g. ``"America/New_York"``).+ listener_data: Frozen action args for the workflow run+ (deployment graph snapshot + cron metadata).+ task_queue: Worker task queue that hosts ``CronTriggerWorkflow``.+ overlap_policy: How concurrent firings interact. ``SKIP`` (default)+ drops a firing if the prior run is still going; mirrors the+ pre-Wave-12 APScheduler behaviour for slow workflows.+ """+ schedule_id = cron_schedule_id(deployment_workflow_id, node_id)+ action_workflow_id = cron_action_workflow_id(deployment_workflow_id, node_id)++ schedule = Schedule(+ action=ScheduleActionStartWorkflow(+ "CronTriggerWorkflow",+ args=[listener_data],+ id=action_workflow_id,+ task_queue=task_queue,+ ),+ spec=ScheduleSpec(+ cron_expressions=[cron_expression],+ time_zone_name=timezone or "UTC",+ ),+ policy=SchedulePolicy(overlap=overlap_policy),+ )++ # Search Attributes mirror the listener-canary contract so the+ # cancel sweep can find the Schedule via the same EventWorkflowId+ # filter. EventTriggerKind="cron" lets ops dashboards filter by+ # canary kind independently of the underlying primitive. Note:+ # ``search_attributes`` is a ``client.create_schedule`` kwarg, not+ # a field on the ``Schedule`` dataclass itself (per the Temporal+ # SDK API).+ schedule_search_attributes = TypedSearchAttributes([+ SearchAttributePair(+ SearchAttributeKey.for_keyword("EventWorkflowId"),+ deployment_workflow_id,+ ),+ SearchAttributePair(+ SearchAttributeKey.for_keyword("TriggerNodeId"),+ node_id,+ ),+ SearchAttributePair(+ SearchAttributeKey.for_keyword("EventTriggerKind"),+ "cron",+ ),+ ])++ try:+ await client.create_schedule(+ schedule_id,+ schedule,+ search_attributes=schedule_search_attributes,+ )+ logger.info(+ "Created Temporal cron Schedule",+ schedule_id=schedule_id,+ cron_expression=cron_expression,+ timezone=timezone,+ )+ except ScheduleAlreadyRunningError:+ # Idempotent re-deploy. The pre-existing Schedule keeps its+ # state; the caller's create-or-reuse contract is satisfied.+ logger.info(+ "Temporal cron Schedule already exists (re-deploy idempotency)",+ schedule_id=schedule_id,+ )++ return schedule_id+++async def delete_cron_schedules_for_deployment(+ client: Client,+ deployment_workflow_id: str,+) -> int:+ """Delete every cron Schedule for ``deployment_workflow_id``.++ Visibility-equivalent sweep: ``client.list_schedules(query=...)``+ is the registry, no local handle dict. Per-Schedule failures don't+ block the sweep (mirror of+ :meth:`DeploymentManager._cancel_canary_listeners` semantics).++ Returns count of Schedules deleted.+ """+ query = (+ f"EventWorkflowId='{deployment_workflow_id}' "+ f"AND EventTriggerKind='cron'"+ )++ deleted = 0+ try:+ async for desc in client.list_schedules(query=query):+ sched_id = desc.id+ try:+ handle = client.get_schedule_handle(sched_id)+ await handle.delete()+ deleted += 1+ except Exception as exc: # noqa: BLE001+ logger.warning(+ f"Failed to delete cron Schedule {sched_id!r}: {exc}",+ deployment_workflow_id=deployment_workflow_id,+ )+ except Exception as exc: # noqa: BLE001+ logger.warning(+ f"Visibility query for cron Schedules failed: {exc} "+ f"(query={query!r})",+ deployment_workflow_id=deployment_workflow_id,+ )++ if deleted:+ logger.info(+ "Cron Schedules deleted",+ deployment_workflow_id=deployment_workflow_id,+ count=deleted,+ )+ return deleted+++__all__ = [+ "cron_schedule_id",+ "create_cron_schedule",+ "delete_cron_schedules_for_deployment",+]diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex 9c8641ac..8e8fd784 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -29,6 +29,7 @@ from core.logging import get_loggerfrom .workflow import MachinaWorkflowfrom .trigger_listener_workflow import TriggerListenerWorkflowfrom .polling_trigger_workflow import PollingTriggerWorkflow+from .plugin_registry import temporal_pluginsfrom .activities import (NodeExecutionActivities,create_shared_session,@@ -143,9 +144,18 @@ class TemporalWorkerManager:per_type = collect_plugin_activities() # no queue filter; all pluginsagent_activities = collect_agent_activities()polling_activities = collect_polling_activities()+ # Plugin-owned workflow classes (e.g. cron's+ # CronTriggerWorkflow) self-register a SimplePlugin via+ # services.temporal.plugin_registry.register_temporal_plugin+ # from their plugin __init__.py. The Temporal SDK's plugin+ # chain merges each registered plugin's workflows / activities+ # / interceptors into the effective worker configuration —+ # the framework worker stays plugin-agnostic.+ plugin_list = temporal_plugins()self._worker = Worker(self.client,task_queue=self.task_queue,+ plugins=plugin_list,workflows=[MachinaWorkflow,AgentWorkflow,diff --git a/server/tests/test_cron_canary.py b/server/tests/test_cron_canary.pynew file mode 100644index 00000000..f5efc61f--- /dev/null+++ b/server/tests/test_cron_canary.py@@ -0,0 +1,440 @@+"""Wave 12 C3: tests for the cron Temporal-Schedule canary.++Three layers:++1. **plugin_registry contract** — Temporal SimplePlugin registration is+ idempotent, namespace-keyed, and produces a stable snapshot for the+ Temporal worker's ``plugins=[]`` argument.++2. **schedules helper** — deterministic ``cron_schedule_id`` derivation;+ :func:`create_cron_schedule` calls ``client.create_schedule`` with+ the SimplePlugin-targeting action + Search Attributes;+ :func:`delete_cron_schedules_for_deployment` queries via Visibility+ and graceful-deletes each.++3. **DeploymentManager integration** — _start_canary_cron_schedule+ builds the listener_data payload and calls into schedules.py;+ _cancel_canary_cron_schedules dispatches into the delete sweep.++4. **Plugin self-registration smoke** — importing+ ``nodes.scheduler.cron_scheduler`` populates the plugin_registry+ with the cron SimplePlugin AND the canary_registry with+ ``cronScheduler``.+"""++from __future__ import annotations++import sys+import types+from typing import Any, Dict, List+from unittest.mock import AsyncMock, MagicMock++import pytest+++if "machina" not in sys.modules:+ _machina = types.ModuleType("machina")+ _machina.__path__ = []+ sys.modules["machina"] = _machina+ _machina_tcp = types.ModuleType("machina.tcp")+ _machina_tcp.probe_tcp_port = MagicMock(return_value=False)+ sys.modules["machina.tcp"] = _machina_tcp+++def fresh_plugin_registry(monkeypatch):+ """Reset plugin_registry's backing store for test isolation."""+ from services.temporal import plugin_registry as pr++ fresh = type(pr._REGISTRY)(pr._REGISTRY._name) # type: ignore[attr-defined]+ monkeypatch.setattr(pr, "_REGISTRY", fresh)+ return pr+++# ---------------------------------------------------------------------------+# plugin_registry contract+# ---------------------------------------------------------------------------+++class TestPluginRegistryContract:+ """SimplePlugin registration shape."""++ def test_empty_registry_yields_empty_list(self, fresh_plugin_registry):+ assert fresh_plugin_registry.temporal_plugins() == []++ def test_register_then_snapshot(self, fresh_plugin_registry):+ from temporalio.plugin import SimplePlugin++ plugin = SimplePlugin(name="test-plugin")+ fresh_plugin_registry.register_temporal_plugin(plugin)++ snap = fresh_plugin_registry.temporal_plugins()+ assert snap == [plugin]++ def test_idempotent_same_plugin_instance(self, fresh_plugin_registry):+ from temporalio.plugin import SimplePlugin++ plugin = SimplePlugin(name="test-plugin")+ fresh_plugin_registry.register_temporal_plugin(plugin)+ fresh_plugin_registry.register_temporal_plugin(plugin)+ assert len(fresh_plugin_registry.temporal_plugins()) == 1++ def test_conflicting_name_raises(self, fresh_plugin_registry):+ from temporalio.plugin import SimplePlugin++ a = SimplePlugin(name="dup-name")+ b = SimplePlugin(name="dup-name")+ fresh_plugin_registry.register_temporal_plugin(a)+ with pytest.raises(ValueError, match="already registered"):+ fresh_plugin_registry.register_temporal_plugin(b)++ def test_distinct_names_coexist(self, fresh_plugin_registry):+ from temporalio.plugin import SimplePlugin++ p1 = SimplePlugin(name="plugin-1")+ p2 = SimplePlugin(name="plugin-2")+ fresh_plugin_registry.register_temporal_plugin(p1)+ fresh_plugin_registry.register_temporal_plugin(p2)+ assert {p.name() for p in fresh_plugin_registry.temporal_plugins()} == {+ "plugin-1", "plugin-2",+ }+++# ---------------------------------------------------------------------------+# schedules helper+# ---------------------------------------------------------------------------+++class TestScheduleIdDerivation:+ def test_deterministic_id(self):+ from services.temporal.schedules import cron_schedule_id++ a = cron_schedule_id("wf-1", "cron-1")+ b = cron_schedule_id("wf-1", "cron-1")+ assert a == b == "cron-schedule-wf-1-cron-1"++ def test_different_node_different_id(self):+ from services.temporal.schedules import cron_schedule_id++ assert cron_schedule_id("wf-1", "cron-a") != cron_schedule_id(+ "wf-1", "cron-b",+ )+++class TestCreateCronSchedule:+ @pytest.mark.asyncio+ async def test_create_schedule_payload_shape(self):+ from services.temporal.schedules import create_cron_schedule++ client = MagicMock()+ client.create_schedule = AsyncMock()++ listener_data = {+ "workflow_id": "wf-1",+ "trigger_node_id": "cron-1",+ "node_type": "cronScheduler",+ "cron_expression": "*/5 * * * *",+ }++ schedule_id = await create_cron_schedule(+ client,+ deployment_workflow_id="wf-1",+ node_id="cron-1",+ cron_expression="*/5 * * * *",+ timezone="America/New_York",+ listener_data=listener_data,+ )++ assert schedule_id == "cron-schedule-wf-1-cron-1"+ assert client.create_schedule.await_count == 1++ passed_id, passed_schedule = client.create_schedule.call_args.args+ assert passed_id == schedule_id+ assert passed_schedule.spec.cron_expressions == ["*/5 * * * *"]+ assert passed_schedule.spec.time_zone_name == "America/New_York"++ @pytest.mark.asyncio+ async def test_create_schedule_idempotent_on_already_running(self):+ """Re-deploy: Temporal raises ScheduleAlreadyRunningError; helper+ swallows it so the deploy path keeps a deterministic id without+ the caller needing to special-case retries."""+ from temporalio.client import ScheduleAlreadyRunningError++ from services.temporal.schedules import create_cron_schedule++ client = MagicMock()+ client.create_schedule = AsyncMock(side_effect=ScheduleAlreadyRunningError())++ schedule_id = await create_cron_schedule(+ client,+ deployment_workflow_id="wf-1",+ node_id="cron-1",+ cron_expression="0 * * * *",+ timezone="UTC",+ listener_data={},+ )++ assert schedule_id == "cron-schedule-wf-1-cron-1"+++class TestDeleteCronSchedulesForDeployment:+ @pytest.mark.asyncio+ async def test_query_filters_by_workflow_id(self):+ from services.temporal.schedules import (+ delete_cron_schedules_for_deployment,+ )++ recorded_queries: List[str] = []+ deleted_ids: List[str] = []++ async def fake_list(query):+ recorded_queries.append(query)+ for sched_id in ["cron-schedule-wf-1-a", "cron-schedule-wf-1-b"]:+ yield MagicMock(id=sched_id)++ def fake_get_handle(sid):+ handle = MagicMock()++ async def fake_delete():+ deleted_ids.append(sid)++ handle.delete = fake_delete+ return handle++ client = MagicMock()+ client.list_schedules = fake_list+ client.get_schedule_handle = fake_get_handle++ count = await delete_cron_schedules_for_deployment(client, "wf-1")++ assert count == 2+ assert sorted(deleted_ids) == [+ "cron-schedule-wf-1-a", "cron-schedule-wf-1-b",+ ]+ # Filter on both deployment workflow_id AND the cron kind tag.+ assert len(recorded_queries) == 1+ q = recorded_queries[0]+ assert "EventWorkflowId='wf-1'" in q+ assert "EventTriggerKind='cron'" in q++ @pytest.mark.asyncio+ async def test_per_schedule_failure_does_not_block_sweep(self):+ from services.temporal.schedules import (+ delete_cron_schedules_for_deployment,+ )++ async def fake_list(query):+ for sid in ["cron-schedule-wf-1-good", "cron-schedule-wf-1-bad"]:+ yield MagicMock(id=sid)++ def fake_get_handle(sid):+ handle = MagicMock()+ if "bad" in sid:+ async def boom():+ raise RuntimeError("simulated delete failure")+ handle.delete = boom+ else:+ async def ok():+ return None+ handle.delete = ok+ return handle++ client = MagicMock()+ client.list_schedules = fake_list+ client.get_schedule_handle = fake_get_handle++ count = await delete_cron_schedules_for_deployment(client, "wf-1")+ # Only the good one deleted; the bad one's failure was logged+ # + skipped.+ assert count == 1+++# ---------------------------------------------------------------------------+# DeploymentManager cron canary integration+# ---------------------------------------------------------------------------+++def _node(node_id: str, node_type: str) -> Dict[str, Any]:+ return {"id": node_id, "type": node_type, "data": {}}+++def _build_manager(workflow_id: str, nodes=None, edges=None, session_id="sess"):+ from services.deployment.manager import DeploymentManager+ from services.deployment.state import DeploymentState++ database = MagicMock()+ database.get_node_parameters = AsyncMock(return_value={})+ broadcaster = MagicMock()+ broadcaster.update_node_status = AsyncMock()++ mgr = DeploymentManager(+ database=database,+ execute_workflow_fn=AsyncMock(),+ store_output_fn=AsyncMock(),+ broadcaster=broadcaster,+ )+ mgr._deployments[workflow_id] = DeploymentState(+ deployment_id=f"deploy_{workflow_id}",+ workflow_id=workflow_id,+ is_running=True,+ nodes=nodes or [],+ edges=edges or [],+ session_id=session_id,+ )+ return mgr+++class TestDeploymentCronCanaryRouting:+ @pytest.mark.asyncio+ async def test_start_canary_cron_schedule_calls_helper(self, monkeypatch):+ """The deployment helper threads the listener_data payload into+ create_cron_schedule with the right shape."""+ mgr = _build_manager(+ "wf-1",+ nodes=[_node("cron-1", "cronScheduler")],+ edges=[],+ )++ recorded: List[Dict[str, Any]] = []++ async def fake_create(client, *, deployment_workflow_id, node_id,+ cron_expression, timezone, listener_data, **kw):+ recorded.append({+ "deployment_workflow_id": deployment_workflow_id,+ "node_id": node_id,+ "cron_expression": cron_expression,+ "timezone": timezone,+ "listener_data": listener_data,+ })+ return f"cron-schedule-{deployment_workflow_id}-{node_id}"++ from services.temporal import schedules+ monkeypatch.setattr(schedules, "create_cron_schedule", fake_create)++ wrapper = MagicMock()+ wrapper.client = MagicMock()++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ result = await mgr._start_canary_cron_schedule(+ _node("cron-1", "cronScheduler"),+ "wf-1",+ params={"cron": "*"},+ cron_expr="*/5 * * * *",+ timezone="UTC",+ frequency="minutes",+ schedule_desc="Every 5 minutes",+ )++ assert result == "cron-schedule-wf-1-cron-1"+ assert len(recorded) == 1+ call = recorded[0]+ assert call["cron_expression"] == "*/5 * * * *"+ assert call["timezone"] == "UTC"+ # listener_data carries the graph snapshot + cron metadata.+ ld = call["listener_data"]+ assert ld["workflow_id"] == "wf-1"+ assert ld["trigger_node_id"] == "cron-1"+ assert ld["cron_expression"] == "*/5 * * * *"+ assert ld["schedule"] == "Every 5 minutes"++ @pytest.mark.asyncio+ async def test_start_returns_none_when_temporal_not_connected(self, monkeypatch):+ mgr = _build_manager(+ "wf-1",+ nodes=[_node("cron-1", "cronScheduler")],+ edges=[],+ )++ wrapper = MagicMock()+ wrapper.client = None++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ result = await mgr._start_canary_cron_schedule(+ _node("cron-1", "cronScheduler"),+ "wf-1",+ params={},+ cron_expr="0 * * * *",+ timezone="UTC",+ frequency="hours",+ schedule_desc="Every hour",+ )++ assert result is None++ @pytest.mark.asyncio+ async def test_cancel_dispatches_into_delete_sweep(self, monkeypatch):+ mgr = _build_manager("wf-1")++ delete_calls: List[str] = []++ async def fake_delete(client, deployment_workflow_id):+ delete_calls.append(deployment_workflow_id)+ return 3++ from services.temporal import schedules+ monkeypatch.setattr(+ schedules,+ "delete_cron_schedules_for_deployment",+ fake_delete,+ )++ wrapper = MagicMock()+ wrapper.client = MagicMock()++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ count = await mgr._cancel_canary_cron_schedules("wf-1")+ assert count == 3+ assert delete_calls == ["wf-1"]++ @pytest.mark.asyncio+ async def test_cancel_returns_zero_when_temporal_disconnected(self, monkeypatch):+ mgr = _build_manager("wf-1")++ wrapper = MagicMock()+ wrapper.client = None++ from core import container as container_mod+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)++ assert await mgr._cancel_canary_cron_schedules("wf-1") == 0+++# ---------------------------------------------------------------------------+# Plugin self-registration smoke+# ---------------------------------------------------------------------------+++class TestCronPluginSelfRegisters:+ def test_plugin_import_registers_simple_plugin(self):+ try:+ __import__("nodes.scheduler.cron_scheduler")+ except ImportError as exc: # pragma: no cover+ pytest.xfail(f"cron_scheduler not importable: {exc}")++ from services.temporal.plugin_registry import temporal_plugins++ names = {p.name() for p in temporal_plugins()}+ assert "cron-scheduler" in names, (+ "Importing nodes.scheduler.cron_scheduler should register a "+ "SimplePlugin named 'cron-scheduler'. Check the __init__.py "+ "bottom section."+ )++ def test_plugin_import_registers_canary(self):+ try:+ __import__("nodes.scheduler.cron_scheduler")+ except ImportError as exc: # pragma: no cover+ pytest.xfail(f"cron_scheduler not importable: {exc}")++ from services.deployment import canary_registry++ assert canary_registry.is_canary_trigger_type("cronScheduler"), (+ "Importing nodes.scheduler.cron_scheduler should call "+ "register_canary_trigger_type('cronScheduler')."+ )