Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
refactor(temporal): shared _retry_policies + NodeUserError non-retryable on activity callsites (Wave 12 D1)
trohitg751ab94514ff
diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.pynew file mode 100644index 00000000..001d1fb9--- /dev/null+++ b/server/services/temporal/_retry_policies.py@@ -0,0 +1,85 @@+"""Wave 12 D1: shared Temporal RetryPolicy constants.++Replaces the duplicated inline ``RetryPolicy(1s, 30s, 3 attempts)``+declarations that lived in ``services/temporal/workflow.py:246`` and+``services/temporal/agent_workflow.py:76`` (identical knobs, hand-typed+twice). Adds the missing ``non_retryable_error_types`` declaration so+``NodeUserError`` (user-correctable failures: missing required field,+unknown enum value, bad regex) fails fast instead of burning all three+retry attempts.++Why these specific error types are non-retryable+------------------------------------------------++- ``NodeUserError``: every plugin raises this for user-correctable+ failures (see ``services/plugin/__init__.py``). Retrying just spends+ retry budget — the input won't be different next time. The plugin+ ``cls.retry_policy`` default already lists it as non-retryable+ (Wave 12 A2, ``services/plugin/scaling.py``), but the WORKFLOW-side+ ``workflow.execute_activity(..., retry_policy=...)`` override+ silently lost that contract when the workflow author hand-typed a+ fresh ``RetryPolicy(...)``. This module re-imposes it at the+ workflow callsite.++- ``InvalidEvent``: when ``services/events/dispatch.py:emit`` raises+ on a malformed envelope (missing ``id``, bad ``source``), the+ activity's input is structurally broken and retries can't fix it.++Keep this list short. Adding error types here is a runtime contract+change — every workflow that uses these policies inherits the+non-retryability. Use ``cls.retry_policy.non_retryable_error_types``+on a specific plugin class when only ONE plugin's failure should be+non-retryable.++Refs+----+- https://docs.temporal.io/encyclopedia/retry-policies+- https://python.temporal.io/temporalio.common.RetryPolicy.html+"""++from __future__ import annotations++from datetime import timedelta++from temporalio.common import RetryPolicy+++# Error types that surface user mistakes / structural payload bugs —+# retrying makes no difference. Plugin classes that raise their own+# domain-specific non-retryable errors should override+# ``cls.retry_policy`` rather than expand this list.+NON_RETRYABLE_ERROR_TYPES: tuple[str, ...] = (+ "NodeUserError",+ "InvalidEvent",+)+++# Default activity policy for the orchestrator (MachinaWorkflow,+# AgentWorkflow, PollingTriggerWorkflow). 3 attempts with 1s→30s+# exponential backoff covers transient API blips (DNS, rate limits,+# brief upstream outages) without holding the whole graph hostage to a+# persistent failure.+DEFAULT_ACTIVITY_RETRY: RetryPolicy = RetryPolicy(+ initial_interval=timedelta(seconds=1),+ maximum_interval=timedelta(seconds=30),+ maximum_attempts=3,+ non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),+)+++# Tighter policy for activities that are themselves cheap operations+# (e.g. ``emit_event_activity``, ``broadcast_progress``). Failing fast+# here surfaces wiring bugs in tests + ops without retry-budget noise.+QUICK_ACTIVITY_RETRY: RetryPolicy = RetryPolicy(+ initial_interval=timedelta(milliseconds=250),+ maximum_interval=timedelta(seconds=5),+ maximum_attempts=2,+ non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),+)+++__all__ = [+ "NON_RETRYABLE_ERROR_TYPES",+ "DEFAULT_ACTIVITY_RETRY",+ "QUICK_ACTIVITY_RETRY",+]diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.pyindex d3c8810b..36c4b27b 100644--- a/server/services/temporal/agent_workflow.py+++ b/server/services/temporal/agent_workflow.py@@ -50,7 +50,9 @@ from datetime import timedeltafrom typing import Any, Dict, List, Optionalfrom temporalio import workflow-from temporalio.common import RetryPolicy+from temporalio.common import RetryPolicy # kept for type hints++from ._retry_policies import DEFAULT_ACTIVITY_RETRY# Activity timeouts. LLM step can stream for several minutes on@@ -72,12 +74,11 @@ TOOL_HEARTBEAT_TIMEOUT = timedelta(minutes=2)DEFAULT_MAX_ITERATIONS = 50# Retry policy for the agent's own activities (LLM step, persist,-# compact). Tool activities use their plugin's policy.-AGENT_ACTIVITY_RETRY = RetryPolicy(- initial_interval=timedelta(seconds=1),- maximum_interval=timedelta(seconds=30),- maximum_attempts=3,-)+# compact). Tool activities use their plugin's policy. Wave 12 D1:+# delegates to the shared constant so the policy's+# non_retryable_error_types include ``NodeUserError`` — user-correctable+# failures inside the LLM step fail fast instead of burning 3 retries.+AGENT_ACTIVITY_RETRY: RetryPolicy = DEFAULT_ACTIVITY_RETRY@workflow.defn(sandboxed=False, name="AgentWorkflow")diff --git a/server/services/temporal/polling_trigger_workflow.py b/server/services/temporal/polling_trigger_workflow.pyindex 519f92fc..f4fbe5c8 100644--- a/server/services/temporal/polling_trigger_workflow.py+++ b/server/services/temporal/polling_trigger_workflow.py@@ -40,6 +40,8 @@ from temporalio import workflowfrom temporalio.common import WorkflowIDReusePolicyfrom temporalio.workflow import ParentClosePolicy+from ._retry_policies import DEFAULT_ACTIVITY_RETRY+# Same continueAsNew threshold as TriggerListenerWorkflow — keeps Event# History under Temporal's soft ceiling. The actual count depends on@@ -136,10 +138,16 @@ class PollingTriggerWorkflow:}try:+ # Wave 12 D1: explicit RetryPolicy with+ # non_retryable_error_types=("NodeUserError", ...) —+ # poll-cycle failures from user-correctable causes+ # (bad filter expression, missing credential) fail fast+ # instead of burning 3 retries per cycle.result = await workflow.execute_activity(activity_name,cycle_payload,start_to_close_timeout=timedelta(seconds=activity_timeout_s),+ retry_policy=DEFAULT_ACTIVITY_RETRY,)except Exception as exc: # noqa: BLE001# Activity exhausted its RetryPolicy. Log + continue —diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex b14c5562..05780d58 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -15,7 +15,8 @@ from datetime import timedeltafrom typing import Any, Dict, List, Optional, Setfrom temporalio import workflow-from temporalio.common import RetryPolicy++from ._retry_policies import DEFAULT_ACTIVITY_RETRY# Config handles - nodes connecting via these are config nodes (not executed)# AI Agent handles: input-memory, input-tools, input-model, input-task, input-teammates@@ -242,12 +243,11 @@ class MachinaWorkflow:workflow.logger.info(f"Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")- # 5. Retry policy for node activities- retry_policy = RetryPolicy(- initial_interval=timedelta(seconds=1),- maximum_interval=timedelta(seconds=30),- maximum_attempts=3,- )+ # 5. Retry policy for node activities. Wave 12 D1: imported+ # shared constant declares ``non_retryable_error_types`` so+ # NodeUserError fails fast instead of burning 3 retries on+ # user-correctable failures.+ retry_policy = DEFAULT_ACTIVITY_RETRY# 6. Continuous scheduling looploop_count = 0diff --git a/server/tests/test_retry_policies.py b/server/tests/test_retry_policies.pynew file mode 100644index 00000000..0c08f67b--- /dev/null+++ b/server/tests/test_retry_policies.py@@ -0,0 +1,143 @@+"""Wave 12 D1: shared activity RetryPolicy constants — contract ++no-duplication invariants.++Locks three layers:++1. **Policy contents**: ``DEFAULT_ACTIVITY_RETRY`` declares+ ``non_retryable_error_types`` including ``"NodeUserError"``. Without+ this declaration the workflow-side retry policy silently overrides+ the plugin's class-level non-retryable contract.++2. **No re-typing of the policy in workflow code**: regression catch+ for someone re-introducing+ ``RetryPolicy(initial_interval=..., maximum_interval=..., maximum_attempts=...)``+ inline in workflow files. The shared constant exists so the+ non-retryable list is set in exactly one place.++3. **All three Temporal workflows that execute activities import the+ shared constant**: MachinaWorkflow, AgentWorkflow,+ PollingTriggerWorkflow. (TriggerListenerWorkflow + CronTriggerWorkflow+ don't execute activities — they only spawn child workflows — so+ they don't need a retry policy.)+"""++from __future__ import annotations++import inspect+import re+import sys+import types+from unittest.mock import MagicMock++import pytest+++if "machina" not in sys.modules:+ _machina = types.ModuleType("machina")+ _machina.__path__ = []+ sys.modules["machina"] = _machina+ _machina_tcp = types.ModuleType("machina.tcp")+ _machina_tcp.probe_tcp_port = MagicMock(return_value=False)+ sys.modules["machina.tcp"] = _machina_tcp+++class TestRetryPolicyConstants:+ def test_default_activity_retry_has_node_user_error_non_retryable(self):+ from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY++ non_retryable = DEFAULT_ACTIVITY_RETRY.non_retryable_error_types or ()+ assert "NodeUserError" in non_retryable, (+ "DEFAULT_ACTIVITY_RETRY must list NodeUserError as "+ "non-retryable. Without it, plugins that raise NodeUserError "+ "for user-correctable failures (missing required field, bad "+ "regex, unknown enum value) get retried 3 times instead of "+ "failing fast back to the operator."+ )++ def test_default_activity_retry_max_attempts(self):+ from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY++ # 3 attempts = original + 2 retries. Matches the pre-Wave-12+ # behaviour of the inline-typed RetryPolicy this module replaces.+ assert DEFAULT_ACTIVITY_RETRY.maximum_attempts == 3++ def test_quick_activity_retry_is_fast(self):+ from services.temporal._retry_policies import QUICK_ACTIVITY_RETRY++ # Fast-fail policy for cheap activities (emit_event,+ # broadcast_progress). Fewer attempts + shorter intervals than+ # the default so wiring bugs surface in tests instead of+ # retry-buffering for 90+ seconds.+ assert QUICK_ACTIVITY_RETRY.maximum_attempts <= 3+ non_retryable = QUICK_ACTIVITY_RETRY.non_retryable_error_types or ()+ assert "NodeUserError" in non_retryable+++class TestNoDuplicatedInlineRetryPolicy:+ """Workflow files must import the shared constant — not re-type+ ``RetryPolicy(initial_interval=..., maximum_interval=..., maximum_attempts=N)``+ inline. The shared constant is the single source of truth for the+ non-retryable contract.+ """++ _WORKFLOW_FILES = (+ "services/temporal/workflow.py",+ "services/temporal/agent_workflow.py",+ "services/temporal/polling_trigger_workflow.py",+ "services/temporal/trigger_listener_workflow.py",+ "nodes/scheduler/cron_scheduler/_workflow.py",+ )++ # Matches RetryPolicy(...) with at least one of the three knobs as+ # a kwarg — catches inline construction without flagging+ # ``RetryPolicy`` mentioned in docstrings or imports.+ _INLINE_PATTERN = re.compile(+ r"RetryPolicy\s*\(\s*(initial_interval|maximum_interval|maximum_attempts)\s*=",+ re.MULTILINE,+ )++ def _read(self, rel_path: str) -> str:+ import pathlib++ path = pathlib.Path(__file__).resolve().parent.parent / rel_path+ if not path.exists():+ pytest.skip(f"{rel_path} not present in this checkout")+ return path.read_text(encoding="utf-8")++ @pytest.mark.parametrize("rel_path", _WORKFLOW_FILES)+ def test_no_inline_retry_policy_construction(self, rel_path):+ src = self._read(rel_path)+ match = self._INLINE_PATTERN.search(src)+ assert match is None, (+ f"{rel_path} contains an inline RetryPolicy(...) "+ f"construction at offset {match.start()}: "+ f"{src[match.start():match.end()].strip()}\n"+ "Use services.temporal._retry_policies.DEFAULT_ACTIVITY_RETRY "+ "(or QUICK_ACTIVITY_RETRY for cheap activities) so the "+ "non_retryable_error_types contract stays in one place."+ )+++class TestWorkflowsImportSharedConstant:+ """The three workflow files that execute activities must import the+ shared constant. Source-introspection only — no Temporal runtime."""++ @pytest.mark.parametrize("module_path,expected_constant", [+ ("services.temporal.workflow", "DEFAULT_ACTIVITY_RETRY"),+ ("services.temporal.agent_workflow", "DEFAULT_ACTIVITY_RETRY"),+ ("services.temporal.polling_trigger_workflow", "DEFAULT_ACTIVITY_RETRY"),+ ])+ def test_imports_shared_constant(self, module_path, expected_constant):+ try:+ mod = __import__(module_path, fromlist=[expected_constant])+ except ImportError as exc: # pragma: no cover+ pytest.xfail(f"{module_path} not importable in test env: {exc}")++ src = inspect.getsource(mod)+ # Either an explicit import OR a `from ._retry_policies import ...`+ # line containing the constant name.+ assert expected_constant in src, (+ f"{module_path} must reference "+ f"services.temporal._retry_policies.{expected_constant} "+ "instead of re-typing the policy inline."+ )