Skip to content

Self-improving AI Employees for everyone

refactor(temporal): shared _retry_policies + NodeUserError non-retryable on activity callsites (Wave 12 D1)

trohitg751ab94514ff
diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.py
new file mode 100644
index 00000000..001d1fb9
--- /dev/null
+++ b/server/services/temporal/_retry_policies.py
@@ -0,0 +1,85 @@
+"""Wave 12 D1: shared Temporal RetryPolicy constants.
+
+Replaces the duplicated inline ``RetryPolicy(1s, 30s, 3 attempts)``
+declarations that lived in ``services/temporal/workflow.py:246`` and
+``services/temporal/agent_workflow.py:76`` (identical knobs, hand-typed
+twice). Adds the missing ``non_retryable_error_types`` declaration so
+``NodeUserError`` (user-correctable failures: missing required field,
+unknown enum value, bad regex) fails fast instead of burning all three
+retry attempts.
+
+Why these specific error types are non-retryable
+------------------------------------------------
+
+- ``NodeUserError``: every plugin raises this for user-correctable
+ failures (see ``services/plugin/__init__.py``). Retrying just spends
+ retry budget — the input won't be different next time. The plugin
+ ``cls.retry_policy`` default already lists it as non-retryable
+ (Wave 12 A2, ``services/plugin/scaling.py``), but the WORKFLOW-side
+ ``workflow.execute_activity(..., retry_policy=...)`` override
+ silently lost that contract when the workflow author hand-typed a
+ fresh ``RetryPolicy(...)``. This module re-imposes it at the
+ workflow callsite.
+
+- ``InvalidEvent``: when ``services/events/dispatch.py:emit`` raises
+ on a malformed envelope (missing ``id``, bad ``source``), the
+ activity's input is structurally broken and retries can't fix it.
+
+Keep this list short. Adding error types here is a runtime contract
+change — every workflow that uses these policies inherits the
+non-retryability. Use ``cls.retry_policy.non_retryable_error_types``
+on a specific plugin class when only ONE plugin's failure should be
+non-retryable.
+
+Refs
+----
+- https://docs.temporal.io/encyclopedia/retry-policies
+- https://python.temporal.io/temporalio.common.RetryPolicy.html
+"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+
+from temporalio.common import RetryPolicy
+
+
+# Error types that surface user mistakes / structural payload bugs —
+# retrying makes no difference. Plugin classes that raise their own
+# domain-specific non-retryable errors should override
+# ``cls.retry_policy`` rather than expand this list.
+NON_RETRYABLE_ERROR_TYPES: tuple[str, ...] = (
+ "NodeUserError",
+ "InvalidEvent",
+)
+
+
+# Default activity policy for the orchestrator (MachinaWorkflow,
+# AgentWorkflow, PollingTriggerWorkflow). 3 attempts with 1s→30s
+# exponential backoff covers transient API blips (DNS, rate limits,
+# brief upstream outages) without holding the whole graph hostage to a
+# persistent failure.
+DEFAULT_ACTIVITY_RETRY: RetryPolicy = RetryPolicy(
+ initial_interval=timedelta(seconds=1),
+ maximum_interval=timedelta(seconds=30),
+ maximum_attempts=3,
+ non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),
+)
+
+
+# Tighter policy for activities that are themselves cheap operations
+# (e.g. ``emit_event_activity``, ``broadcast_progress``). Failing fast
+# here surfaces wiring bugs in tests + ops without retry-budget noise.
+QUICK_ACTIVITY_RETRY: RetryPolicy = RetryPolicy(
+ initial_interval=timedelta(milliseconds=250),
+ maximum_interval=timedelta(seconds=5),
+ maximum_attempts=2,
+ non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),
+)
+
+
+__all__ = [
+ "NON_RETRYABLE_ERROR_TYPES",
+ "DEFAULT_ACTIVITY_RETRY",
+ "QUICK_ACTIVITY_RETRY",
+]
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index d3c8810b..36c4b27b 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -50,7 +50,9 @@ from datetime import timedelta
from typing import Any, Dict, List, Optional
from temporalio import workflow
-from temporalio.common import RetryPolicy
+from temporalio.common import RetryPolicy # kept for type hints
+
+from ._retry_policies import DEFAULT_ACTIVITY_RETRY
# Activity timeouts. LLM step can stream for several minutes on
@@ -72,12 +74,11 @@ TOOL_HEARTBEAT_TIMEOUT = timedelta(minutes=2)
DEFAULT_MAX_ITERATIONS = 50
# Retry policy for the agent's own activities (LLM step, persist,
-# compact). Tool activities use their plugin's policy.
-AGENT_ACTIVITY_RETRY = RetryPolicy(
- initial_interval=timedelta(seconds=1),
- maximum_interval=timedelta(seconds=30),
- maximum_attempts=3,
-)
+# compact). Tool activities use their plugin's policy. Wave 12 D1:
+# delegates to the shared constant so the policy's
+# non_retryable_error_types include ``NodeUserError`` — user-correctable
+# failures inside the LLM step fail fast instead of burning 3 retries.
+AGENT_ACTIVITY_RETRY: RetryPolicy = DEFAULT_ACTIVITY_RETRY
@workflow.defn(sandboxed=False, name="AgentWorkflow")
diff --git a/server/services/temporal/polling_trigger_workflow.py b/server/services/temporal/polling_trigger_workflow.py
index 519f92fc..f4fbe5c8 100644
--- a/server/services/temporal/polling_trigger_workflow.py
+++ b/server/services/temporal/polling_trigger_workflow.py
@@ -40,6 +40,8 @@ from temporalio import workflow
from temporalio.common import WorkflowIDReusePolicy
from temporalio.workflow import ParentClosePolicy
+from ._retry_policies import DEFAULT_ACTIVITY_RETRY
+
# Same continueAsNew threshold as TriggerListenerWorkflow — keeps Event
# History under Temporal's soft ceiling. The actual count depends on
@@ -136,10 +138,16 @@ class PollingTriggerWorkflow:
}
try:
+ # Wave 12 D1: explicit RetryPolicy with
+ # non_retryable_error_types=("NodeUserError", ...) —
+ # poll-cycle failures from user-correctable causes
+ # (bad filter expression, missing credential) fail fast
+ # instead of burning 3 retries per cycle.
result = await workflow.execute_activity(
activity_name,
cycle_payload,
start_to_close_timeout=timedelta(seconds=activity_timeout_s),
+ retry_policy=DEFAULT_ACTIVITY_RETRY,
)
except Exception as exc: # noqa: BLE001
# Activity exhausted its RetryPolicy. Log + continue —
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index b14c5562..05780d58 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -15,7 +15,8 @@ from datetime import timedelta
from typing import Any, Dict, List, Optional, Set
from temporalio import workflow
-from temporalio.common import RetryPolicy
+
+from ._retry_policies import DEFAULT_ACTIVITY_RETRY
# Config handles - nodes connecting via these are config nodes (not executed)
# AI Agent handles: input-memory, input-tools, input-model, input-task, input-teammates
@@ -242,12 +243,11 @@ class MachinaWorkflow:
workflow.logger.info(f"Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")
- # 5. Retry policy for node activities
- retry_policy = RetryPolicy(
- initial_interval=timedelta(seconds=1),
- maximum_interval=timedelta(seconds=30),
- maximum_attempts=3,
- )
+ # 5. Retry policy for node activities. Wave 12 D1: imported
+ # shared constant declares ``non_retryable_error_types`` so
+ # NodeUserError fails fast instead of burning 3 retries on
+ # user-correctable failures.
+ retry_policy = DEFAULT_ACTIVITY_RETRY
# 6. Continuous scheduling loop
loop_count = 0
diff --git a/server/tests/test_retry_policies.py b/server/tests/test_retry_policies.py
new file mode 100644
index 00000000..0c08f67b
--- /dev/null
+++ b/server/tests/test_retry_policies.py
@@ -0,0 +1,143 @@
+"""Wave 12 D1: shared activity RetryPolicy constants — contract +
+no-duplication invariants.
+
+Locks three layers:
+
+1. **Policy contents**: ``DEFAULT_ACTIVITY_RETRY`` declares
+ ``non_retryable_error_types`` including ``"NodeUserError"``. Without
+ this declaration the workflow-side retry policy silently overrides
+ the plugin's class-level non-retryable contract.
+
+2. **No re-typing of the policy in workflow code**: regression catch
+ for someone re-introducing
+ ``RetryPolicy(initial_interval=..., maximum_interval=..., maximum_attempts=...)``
+ inline in workflow files. The shared constant exists so the
+ non-retryable list is set in exactly one place.
+
+3. **All three Temporal workflows that execute activities import the
+ shared constant**: MachinaWorkflow, AgentWorkflow,
+ PollingTriggerWorkflow. (TriggerListenerWorkflow + CronTriggerWorkflow
+ don't execute activities — they only spawn child workflows — so
+ they don't need a retry policy.)
+"""
+
+from __future__ import annotations
+
+import inspect
+import re
+import sys
+import types
+from unittest.mock import MagicMock
+
+import pytest
+
+
+if "machina" not in sys.modules:
+ _machina = types.ModuleType("machina")
+ _machina.__path__ = []
+ sys.modules["machina"] = _machina
+ _machina_tcp = types.ModuleType("machina.tcp")
+ _machina_tcp.probe_tcp_port = MagicMock(return_value=False)
+ sys.modules["machina.tcp"] = _machina_tcp
+
+
+class TestRetryPolicyConstants:
+ def test_default_activity_retry_has_node_user_error_non_retryable(self):
+ from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY
+
+ non_retryable = DEFAULT_ACTIVITY_RETRY.non_retryable_error_types or ()
+ assert "NodeUserError" in non_retryable, (
+ "DEFAULT_ACTIVITY_RETRY must list NodeUserError as "
+ "non-retryable. Without it, plugins that raise NodeUserError "
+ "for user-correctable failures (missing required field, bad "
+ "regex, unknown enum value) get retried 3 times instead of "
+ "failing fast back to the operator."
+ )
+
+ def test_default_activity_retry_max_attempts(self):
+ from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY
+
+ # 3 attempts = original + 2 retries. Matches the pre-Wave-12
+ # behaviour of the inline-typed RetryPolicy this module replaces.
+ assert DEFAULT_ACTIVITY_RETRY.maximum_attempts == 3
+
+ def test_quick_activity_retry_is_fast(self):
+ from services.temporal._retry_policies import QUICK_ACTIVITY_RETRY
+
+ # Fast-fail policy for cheap activities (emit_event,
+ # broadcast_progress). Fewer attempts + shorter intervals than
+ # the default so wiring bugs surface in tests instead of
+ # retry-buffering for 90+ seconds.
+ assert QUICK_ACTIVITY_RETRY.maximum_attempts <= 3
+ non_retryable = QUICK_ACTIVITY_RETRY.non_retryable_error_types or ()
+ assert "NodeUserError" in non_retryable
+
+
+class TestNoDuplicatedInlineRetryPolicy:
+ """Workflow files must import the shared constant — not re-type
+ ``RetryPolicy(initial_interval=..., maximum_interval=..., maximum_attempts=N)``
+ inline. The shared constant is the single source of truth for the
+ non-retryable contract.
+ """
+
+ _WORKFLOW_FILES = (
+ "services/temporal/workflow.py",
+ "services/temporal/agent_workflow.py",
+ "services/temporal/polling_trigger_workflow.py",
+ "services/temporal/trigger_listener_workflow.py",
+ "nodes/scheduler/cron_scheduler/_workflow.py",
+ )
+
+ # Matches RetryPolicy(...) with at least one of the three knobs as
+ # a kwarg — catches inline construction without flagging
+ # ``RetryPolicy`` mentioned in docstrings or imports.
+ _INLINE_PATTERN = re.compile(
+ r"RetryPolicy\s*\(\s*(initial_interval|maximum_interval|maximum_attempts)\s*=",
+ re.MULTILINE,
+ )
+
+ def _read(self, rel_path: str) -> str:
+ import pathlib
+
+ path = pathlib.Path(__file__).resolve().parent.parent / rel_path
+ if not path.exists():
+ pytest.skip(f"{rel_path} not present in this checkout")
+ return path.read_text(encoding="utf-8")
+
+ @pytest.mark.parametrize("rel_path", _WORKFLOW_FILES)
+ def test_no_inline_retry_policy_construction(self, rel_path):
+ src = self._read(rel_path)
+ match = self._INLINE_PATTERN.search(src)
+ assert match is None, (
+ f"{rel_path} contains an inline RetryPolicy(...) "
+ f"construction at offset {match.start()}: "
+ f"{src[match.start():match.end()].strip()}\n"
+ "Use services.temporal._retry_policies.DEFAULT_ACTIVITY_RETRY "
+ "(or QUICK_ACTIVITY_RETRY for cheap activities) so the "
+ "non_retryable_error_types contract stays in one place."
+ )
+
+
+class TestWorkflowsImportSharedConstant:
+ """The three workflow files that execute activities must import the
+ shared constant. Source-introspection only — no Temporal runtime."""
+
+ @pytest.mark.parametrize("module_path,expected_constant", [
+ ("services.temporal.workflow", "DEFAULT_ACTIVITY_RETRY"),
+ ("services.temporal.agent_workflow", "DEFAULT_ACTIVITY_RETRY"),
+ ("services.temporal.polling_trigger_workflow", "DEFAULT_ACTIVITY_RETRY"),
+ ])
+ def test_imports_shared_constant(self, module_path, expected_constant):
+ try:
+ mod = __import__(module_path, fromlist=[expected_constant])
+ except ImportError as exc: # pragma: no cover
+ pytest.xfail(f"{module_path} not importable in test env: {exc}")
+
+ src = inspect.getsource(mod)
+ # Either an explicit import OR a `from ._retry_policies import ...`
+ # line containing the constant name.
+ assert expected_constant in src, (
+ f"{module_path} must reference "
+ f"services.temporal._retry_policies.{expected_constant} "
+ "instead of re-typing the policy inline."
+ )