Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(temporal): per-type activity dispatch behind settings flag (F4.A)
trohitg8261b058cfae
diff --git a/server/core/config.py b/server/core/config.pyindex f5b25389..e2ae7417 100644--- a/server/core/config.py+++ b/server/core/config.py@@ -52,6 +52,12 @@ class Settings(BaseSettings):temporal_server_address: str = Field(default="localhost:7233", env="TEMPORAL_SERVER_ADDRESS")temporal_namespace: str = Field(default="default", env="TEMPORAL_NAMESPACE")temporal_task_queue: str = Field(default="machina-tasks", env="TEMPORAL_TASK_QUEUE")+ # F4.A: per-type activity dispatch. When True, MachinaWorkflow.run() schedules+ # `node.{type}.v{version}` per plugin (with task_queue=cls.task_queue) instead+ # of the single `execute_node_activity` legacy name. Workers register per-type+ # activities alongside the legacy one. Default off so existing deployments+ # behave identically; flip via TEMPORAL_PER_TYPE_DISPATCH=true.+ temporal_per_type_dispatch: bool = Field(default=False, env="TEMPORAL_PER_TYPE_DISPATCH")# API Keys (all optional, injected at runtime)google_maps_api_key: Optional[str] = Field(default=None, env="GOOGLE_MAPS_API_KEY")diff --git a/server/services/plugin/base.py b/server/services/plugin/base.pyindex 754f27bd..a6063412 100644--- a/server/services/plugin/base.py+++ b/server/services/plugin/base.py@@ -446,9 +446,17 @@ class BaseNode:@classmethoddef as_activity(cls):"""Wrap this node as a ``@activity.defn`` callable for Temporal- worker registration (11.F). Uses a stable activity name:+ worker registration (F4.A). Stable activity name:``node.{type}.v{version}``.+ Accepts the same ``context`` dict shape as the legacy+ ``execute_node_activity`` so the orchestrator can swap by name+ without reshaping the payload. Delegates to+ ``workflow_service.execute_node(...)`` — same execution pipeline+ the WebSocket path uses — so status broadcasts, parameter+ fetching, NodeContext build, and error handling all match. The+ Temporal worker shares the FastAPI process, so direct DI works.+Returns the decorated async function; the worker collects theseinto ``activities=[...]``."""@@ -457,17 +465,107 @@ class BaseNode:activity_name = f"node.{cls.type}.v{cls.version}"@activity.defn(name=activity_name)- async def _node_activity(payload: Dict[str, Any]) -> Dict[str, Any]:- # Temporal lives in another process — rebuild NodeContext.- context_dict = payload.get("context", {})- instance = cls()- ctx = NodeContext.from_legacy(- node_id=payload["node_id"],- node_type=cls.type,- context=context_dict,- connection_factory=_make_connection_factory(cls, context_dict),+ async def _node_activity(context: Dict[str, Any]) -> Dict[str, Any]:+ from datetime import datetime+ from core.container import container++ node_id = context["node_id"]+ workflow_id = context.get("workflow_id")+ broadcaster = container.status_broadcaster()++ # Pre-executed trigger nodes — return cached output without dispatching.+ if context.get("pre_executed"):+ activity.logger.debug(f"Node {node_id} pre-executed; passthrough")+ result = {+ "success": True,+ "node_id": node_id,+ "node_type": cls.type,+ "result": context.get("trigger_output", {}),+ "pre_executed": True,+ "timestamp": datetime.now().isoformat(),+ }+ await broadcaster.update_node_status(+ node_id, "success", result, workflow_id=workflow_id,+ )+ return result++ # Disabled nodes — skip.+ node_data = context.get("node_data", {})+ if node_data.get("disabled"):+ activity.logger.debug(f"Node {node_id} disabled; skipping")+ result = {+ "success": True,+ "node_id": node_id,+ "node_type": cls.type,+ "skipped": True,+ "reason": "disabled",+ "timestamp": datetime.now().isoformat(),+ }+ await broadcaster.update_node_status(+ node_id, "skipped", {"disabled": True}, workflow_id=workflow_id,+ )+ return result++ # Broadcast executing — UI cyan-glow.+ await broadcaster.update_node_status(+ node_id, "executing", {"node_type": cls.type}, workflow_id=workflow_id,)- return await instance.execute(payload["node_id"], payload["parameters"], ctx)++ try:+ # Heartbeat the long-running side of the pipeline.+ activity.heartbeat(f"Executing {cls.type}: {node_id}")++ # Delegate to the same pipeline the WS handler uses. Parameters+ # are read from DB inside execute_node (handler-specific). The+ # legacy execute_node_activity does this via a WS roundtrip;+ # per-type activities skip the loopback because the worker+ # shares the FastAPI process.+ workflow_service = container.workflow_service()+ result = await workflow_service.execute_node(+ node_id=node_id,+ node_type=cls.type,+ parameters=node_data,+ nodes=context.get("nodes", []),+ edges=context.get("edges", []),+ session_id=context.get("session_id", "default"),+ workflow_id=workflow_id,+ outputs=context.get("inputs", {}),+ )++ result["node_id"] = node_id+ result["node_type"] = cls.type+ result["timestamp"] = datetime.now().isoformat()++ if result.get("success"):+ activity.logger.info(f"Node {node_id} succeeded")+ await broadcaster.update_node_status(+ node_id, "success", result.get("result", {}),+ workflow_id=workflow_id,+ )+ await broadcaster.update_node_output(+ node_id, result.get("result", {}), workflow_id=workflow_id,+ )+ else:+ activity.logger.warning(+ f"Node {node_id} failed: {result.get('error')}"+ )+ await broadcaster.update_node_status(+ node_id, "error",+ {"error": result.get("error")},+ workflow_id=workflow_id,+ )++ activity.heartbeat(f"Node {node_id} completed")+ return result++ except Exception as e:+ error_msg = f"{type(e).__name__}: {e}"+ activity.logger.error(f"Node {node_id} crashed: {error_msg}")+ await broadcaster.update_node_status(+ node_id, "error", {"error": error_msg},+ workflow_id=workflow_id,+ )+ raisereturn _node_activitydiff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex ae72cf29..e9e57d23 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -101,22 +101,38 @@ class TemporalWorkerManager:# Create activity instance with shared sessionself._activities = NodeExecutionActivities(self._session)- # MachinaWorkflow.run() schedules every node via the single- # ``"execute_node_activity"`` name (see services/temporal/workflow.py).- # The Wave 11.F per-plugin activities (``node.{type}.v{version}``)- # added a ~1.6s registration cost at every worker startup but were- # never scheduled by the orchestrator — pure dead code on the hot- # path. Removed. If per-node task-queue dispatch is revived, gate- # behind a Settings flag and wire it into the orchestrator first.+ # F4.A: register per-type activities alongside the legacy+ # `execute_node_activity` dispatcher. The orchestrator at+ # workflow.py:_resolve_activity picks one of the two paths per node+ # based on the temporal_per_type_dispatch flag; both must be served+ # by the worker. Per-type activities register WITHOUT a task_queue+ # filter (cls.task_queue is the *declared* preference; the single+ # worker actually polls self.task_queue regardless). When+ # TemporalWorkerPool is wired (future enhancement), per-queue+ # filtering will move there. Registration cost: ~1.6s startup;+ # zero runtime cost when the flag is off (orchestrator routes to+ # execute_node_activity, per-type entries sit idle).+ from services.temporal.plugin_activities import collect_plugin_activities++ per_type = collect_plugin_activities() # no queue filter; all pluginsself._worker = Worker(self.client,task_queue=self.task_queue,workflows=[MachinaWorkflow],- activities=[self._activities.execute_node_activity],+ activities=[+ self._activities.execute_node_activity,+ *per_type,+ ],# Allow concurrent activity execution for parallel branchesmax_concurrent_activities=self.pool_size,max_concurrent_workflow_tasks=10,)+ logger.info(+ "Registered Temporal activities",+ legacy=1,+ per_type=len(per_type),+ task_queue=self.task_queue,+ )span.set_attribute("task_queue", self.task_queue)span.set_attribute("pool_size", self.pool_size)diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex 34d774e1..9dcddfeb 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -172,18 +172,28 @@ class MachinaWorkflow:"trigger_output": node.get("_trigger_output"),}- # Schedule activity by name (for class-based activities)- # The activity is registered as NodeExecutionActivities.execute_node_activity- handle = workflow.start_activity(- "execute_node_activity",+ # F4.A: per-type dispatch when the plugin class is registered+ # AND the settings flag is on. Frozen-dict lookup is deterministic+ # so it's safe inside the workflow (Temporal Workflow Definition+ # determinism contract). Falls back to the legacy single activity+ # name for any node type without a class registration.+ activity_name, activity_queue = self._resolve_activity(node_type)+ start_kwargs: Dict[str, Any] = dict(args=[context],start_to_close_timeout=timedelta(minutes=10),heartbeat_timeout=timedelta(minutes=2),retry_policy=retry_policy,)+ if activity_queue is not None:+ start_kwargs["task_queue"] = activity_queue++ handle = workflow.start_activity(activity_name, **start_kwargs)running[node_id] = handle- workflow.logger.info(f"Scheduled activity for node: {node_id}")+ workflow.logger.info(+ f"Scheduled activity for node: {node_id} "+ f"(activity={activity_name}, queue={activity_queue or 'default'})"+ )# Exit if nothing running and nothing readyif not running:@@ -238,6 +248,42 @@ class MachinaWorkflow:inputs[dep_id] = outputs[dep_id].get("result", {})return inputs+ def _resolve_activity(self, node_type: str) -> tuple[str, str | None]:+ """Resolve (activity_name, task_queue) for a node type.++ F4.A: when ``settings.temporal_per_type_dispatch`` is on AND the+ plugin class is registered, returns+ ``("node.{type}.v{version}", None)`` so the activity is scheduled+ by per-type name but stays on the workflow's default task queue.+ The default queue is what ``TemporalWorkerManager`` polls today;+ per-queue routing (``cls.task_queue``) becomes meaningful only+ once ``TemporalWorkerPool`` is wired with one worker per queue+ — until then, returning ``cls.task_queue`` would schedule the+ activity to a queue no worker polls and the workflow would hang.++ Falls back to ``("execute_node_activity", None)`` when:+ - the flag is off (preserves pre-F4.A behavior exactly), OR+ - the node type isn't registered as a BaseNode subclass+ (covers legacy types still on the metadata-only path).++ Determinism: lookups go through frozen module-level dicts+ (``_NODE_CLASS_REGISTRY``, ``Settings``) — no I/O. Safe inside+ ``MachinaWorkflow.run`` per the workflow-definition contract.+ Imports are inside the method to keep the workflow module's+ top-level import set minimal and to avoid import-cycle drift.+ """+ from core.config import Settings+ from services.node_registry import get_node_class++ if not Settings().temporal_per_type_dispatch:+ return "execute_node_activity", None++ cls = get_node_class(node_type)+ if cls is None:+ return "execute_node_activity", None++ return f"node.{cls.type}.v{cls.version}", None+async def _wait_any_complete(self, running: Dict[str, Any]) -> tuple:"""Wait for any activity to complete, return (node_id, result).diff --git a/server/tests/temporal/__init__.py b/server/tests/temporal/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/server/tests/temporal/test_dispatch.py b/server/tests/temporal/test_dispatch.pynew file mode 100644index 00000000..df389cda--- /dev/null+++ b/server/tests/temporal/test_dispatch.py@@ -0,0 +1,133 @@+"""F4.A: per-type activity dispatch resolver invariants.++Locks the orchestrator's `_resolve_activity` contract:+ - flag OFF (default): every node routes to legacy `execute_node_activity`+ - flag ON + registered plugin: routes to `node.{type}.v{version}`+ - flag ON + unknown type: falls back to legacy+ - task_queue is always None (per-queue routing waits for TemporalWorkerPool)++If these invariants drift the Temporal worker will either silently lose+per-type activity wiring (regression to single-dispatcher) or schedule+activities on queues no worker polls (workflow hangs). Same coverage+pattern as test_credential_broadcasts / test_status_broadcasts —+introspect the class, exercise the method, assert the shape.+"""++from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import patch++import pytest++import nodes # noqa: F401 -- triggers plugin discovery+from services.temporal.workflow import MachinaWorkflow+++def workflow_instance() -> MachinaWorkflow:+ return MachinaWorkflow()+++def _set_flag(value: bool):+ """Patch the ``Settings`` import used by ``_resolve_activity``.++ ``tests/conftest.py`` stubs ``core.config.Settings`` as a MagicMock,+ so we can't go through the real env var. Instead, swap the lazy+ import inside the resolver for a SimpleNamespace returning the+ desired flag value. Same effective contract — the resolver only+ reads ``Settings().temporal_per_type_dispatch``.+ """++ def fake_settings_factory():+ return SimpleNamespace(temporal_per_type_dispatch=value)++ return patch("core.config.Settings", side_effect=lambda: fake_settings_factory())+++class TestResolveActivityFlagOff:+ """Default behavior: legacy dispatcher for every node, no surprises."""++ def test_known_plugin_routes_to_legacy(self, workflow_instance):+ with _set_flag(False):+ name, queue = workflow_instance._resolve_activity("pythonExecutor")+ assert name == "execute_node_activity"+ assert queue is None++ def test_unknown_type_routes_to_legacy(self, workflow_instance):+ with _set_flag(False):+ name, queue = workflow_instance._resolve_activity("nonExistentType")+ assert name == "execute_node_activity"+ assert queue is None++ def test_agent_routes_to_legacy(self, workflow_instance):+ with _set_flag(False):+ name, queue = workflow_instance._resolve_activity("aiAgent")+ assert name == "execute_node_activity"+ assert queue is None+++class TestResolveActivityFlagOn:+ """Flag on: per-type for registered plugins, legacy fallback otherwise."""++ def test_known_plugin_routes_per_type(self, workflow_instance):+ with _set_flag(True):+ name, queue = workflow_instance._resolve_activity("pythonExecutor")+ assert name == "node.pythonExecutor.v1"+ # F4.A intentionally returns None for queue — per-queue routing+ # depends on TemporalWorkerPool which isn't wired yet.+ assert queue is None++ def test_agent_routes_per_type(self, workflow_instance):+ with _set_flag(True):+ name, queue = workflow_instance._resolve_activity("aiAgent")+ assert name == "node.aiAgent.v1"+ assert queue is None++ def test_unknown_type_falls_back_to_legacy(self, workflow_instance):+ with _set_flag(True):+ name, queue = workflow_instance._resolve_activity("nonExistentType")+ assert name == "execute_node_activity"+ assert queue is None++ def test_specialized_agent_uses_class_version(self, workflow_instance):+ """All 117 plugins default to version=1; the activity name must+ include the version. If a plugin ever overrides cls.version, this+ test catches the schema break-point."""+ with _set_flag(True):+ name, _ = workflow_instance._resolve_activity("coding_agent")+ assert name.startswith("node.coding_agent.v")+ # Extract version; should match the class's declared version.+ from services.node_registry import get_node_class+ cls = get_node_class("coding_agent")+ assert name == f"node.coding_agent.v{cls.version}"+++class TestPerTypeActivityCollection:+ """`collect_plugin_activities()` must return one callable per registered+ plugin class — TemporalWorkerManager's per-type registration depends on+ this returning the full set, not a queue-filtered subset (F4.A puts+ all per-type entries on the default queue until TemporalWorkerPool+ lands)."""++ def test_collect_returns_one_per_plugin(self):+ from services.node_registry import registered_node_classes+ from services.temporal.plugin_activities import collect_plugin_activities++ activities = collect_plugin_activities()+ # Per-type activity count must equal registered plugin count.+ assert len(activities) == len(registered_node_classes())++ def test_per_type_activity_has_temporal_metadata(self):+ """Every per-type activity must carry the @activity.defn decorator+ with a `node.{type}.v{version}` name — otherwise the worker can't+ register it."""+ from services.temporal.plugin_activities import collect_plugin_activities++ activities = collect_plugin_activities()+ for a in activities:+ # temporalio attaches metadata as `__temporal_activity_definition`.+ defn = getattr(a, "__temporal_activity_definition", None)+ assert defn is not None, f"activity {a} missing Temporal defn"+ assert defn.name.startswith("node."), defn.name+ assert ".v" in defn.name, defn.name