Skip to content

Self-improving AI Employees for everyone

fix(temporal): restore the single-source retry-policy invariant

trohitgfc0908faffb6
diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.py
index 8799f57b..00d0a47d 100644
--- a/server/services/temporal/_retry_policies.py
+++ b/server/services/temporal/_retry_policies.py
@@ -92,9 +92,26 @@ LLM_STEP_RETRY: RetryPolicy = RetryPolicy(
)
+# Terminal-cleanup policy for delegated-task teardown (persisting a
+# cancellation, releasing a subagent permit). These run while the workflow
+# is already unwinding, so giving up early leaks a permit or leaves a task
+# row stuck in a non-terminal state — a durable, user-visible defect that a
+# later retry genuinely can fix. Hence a longer budget than the 3-attempt
+# default. Deliberately NOT carrying ``non_retryable_error_types``: the
+# cleanup activities are idempotent by construction and a NodeUserError here
+# means the cleanup itself is broken, which should still be retried while
+# the worker is alive.
+DELEGATION_CLEANUP_RETRY: RetryPolicy = RetryPolicy(
+ initial_interval=timedelta(seconds=1),
+ maximum_interval=timedelta(seconds=30),
+ maximum_attempts=10,
+)
+
+
__all__ = [
"NON_RETRYABLE_ERROR_TYPES",
"DEFAULT_ACTIVITY_RETRY",
"QUICK_ACTIVITY_RETRY",
"LLM_STEP_RETRY",
+ "DELEGATION_CLEANUP_RETRY",
]
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index c8ed3744..3f0a4738 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -56,7 +56,11 @@ from temporalio.workflow import ActivityCancellationType, ParentClosePolicy
from services.node_registry import get_node_class
-from ._retry_policies import DEFAULT_ACTIVITY_RETRY, LLM_STEP_RETRY
+from ._retry_policies import (
+ DEFAULT_ACTIVITY_RETRY,
+ DELEGATION_CLEANUP_RETRY,
+ LLM_STEP_RETRY,
+)
from .workflow import AGENT_WORKFLOW_TYPES
@@ -105,11 +109,6 @@ def _default_max_iterations() -> int:
# non_retryable_error_types include ``NodeUserError`` — user-correctable
# failures inside the LLM step fail fast instead of burning 3 retries.
AGENT_ACTIVITY_RETRY: RetryPolicy = DEFAULT_ACTIVITY_RETRY
-DELEGATION_CLEANUP_RETRY: RetryPolicy = RetryPolicy(
- initial_interval=timedelta(seconds=1),
- maximum_interval=timedelta(seconds=30),
- maximum_attempts=10,
-)
# Temporal patch marker for per-call command identity and duplicate-name
# validation. Existing histories must retain their recorded activity/child
diff --git a/server/tests/temporal/test_cooperative_pause.py b/server/tests/temporal/test_cooperative_pause.py
index b7e3c5eb..8ab7dd08 100644
--- a/server/tests/temporal/test_cooperative_pause.py
+++ b/server/tests/temporal/test_cooperative_pause.py
@@ -840,13 +840,21 @@ async def test_agent_acquire_cancellation_type_is_replay_patch_guarded(
@pytest.mark.parametrize(
"failing_cleanup",
- [None, "agent.finish_delegation.v1", "agent.release_subagent_permit.v1"],
+ [None, "agent.cancel_delegation.v1", "agent.release_subagent_permit.v1"],
)
@pytest.mark.asyncio
async def test_delegated_cancellation_persists_terminal_releases_and_reraises(
monkeypatch,
failing_cleanup,
):
+ """Cancellation persists a terminal state and releases the permit.
+
+ The terminal transition goes through the dedicated
+ ``agent.cancel_delegation.v1`` activity, not ``finish_delegation`` — the
+ normal finish path applies the retry/requeue policy, which must not run
+ for a cancelled task. Cleanup failures are logged and swallowed so the
+ CancelledError still propagates.
+ """
from services.temporal import agent_workflow as agent_module
temporal_workflow = agent_module.workflow
@@ -881,7 +889,7 @@ async def test_delegated_cancellation_persists_terminal_releases_and_reraises(
if name == failing_cleanup:
raise RuntimeError(f"{name} failed")
if name in {
- "agent.finish_delegation.v1",
+ "agent.cancel_delegation.v1",
"agent.release_subagent_permit.v1",
}:
return {}
@@ -915,10 +923,13 @@ async def test_delegated_cancellation_persists_terminal_releases_and_reraises(
terminal = next(
payload
for name, payload in calls
- if name == "agent.finish_delegation.v1"
+ if name == "agent.cancel_delegation.v1"
)
- assert terminal["success"] is False
- assert terminal["error"] == "Delegated task workflow cancelled"
+ assert terminal["reason"] == "Delegated task workflow cancelled"
+ assert terminal["terminal_event_id"] == "task-1:terminal"
+ # The normal finish path applies retry/requeue policy and must never run
+ # for a cancellation.
+ assert not any(name == "agent.finish_delegation.v1" for name, _ in calls)
assert any(
name == "agent.release_subagent_permit.v1"
for name, _payload in calls