Skip to content

Self-improving AI Employees for everyone

fix(temporal): retry transient LLM failures instead of killing the run; compaction failure is terminal

trohitg74b5a55fdf02
diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.py
index 31d34760..882799d4 100644
--- a/server/services/temporal/_retry_policies.py
+++ b/server/services/temporal/_retry_policies.py
@@ -78,16 +78,23 @@ QUICK_ACTIVITY_RETRY: RetryPolicy = RetryPolicy(
)
-# Wave 17.2: one-shot policy for the LLM step. An LLM API call is NOT
-# idempotent from a cost perspective — every automatic re-attempt after
-# a worker crash / laptop sleep bills the full prompt again (and can
-# double-fire side effects when the first attempt actually completed
-# server-side but the worker died before recording the result). The
-# workflow layer (AgentWorkflow.run) owns the retry decision instead:
-# it catches the ActivityError with the message history intact and
-# surfaces the failure to the canvas rather than silently re-billing.
+# LLM-step policy: retry transient provider failures indefinitely with
+# exponential backoff. A clean provider rejection (429 rate limit, 5xx,
+# network refusal) bills nothing and MUST be retried — the previous
+# one-shot policy (Wave 17.2) killed the whole run on the first 429 and
+# then tripped the deployment circuit breaker. Terminal outcomes still
+# fail fast: ``_as_temporal_llm_error`` marks non-retryable categories
+# (invalid_request, authentication, ...) with ``non_retryable=True``,
+# which overrides this policy, and provider ``retry_after`` hints ride
+# ``ApplicationError.next_retry_delay``. Known trade-off, accepted: an
+# AMBIGUOUS loss (worker crash / timeout where the call may have
+# completed server-side) now also retries and can re-bill a prompt —
+# resilience of long-running deployments wins over billing paranoia.
LLM_STEP_RETRY: RetryPolicy = RetryPolicy(
- maximum_attempts=1,
+ initial_interval=timedelta(seconds=5),
+ backoff_coefficient=2.0,
+ maximum_interval=timedelta(minutes=5),
+ maximum_attempts=0, # unlimited — liveness comes from heartbeats
non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),
)
diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.py
index 374f12be..04e6cba9 100644
--- a/server/services/temporal/agent_activities.py
+++ b/server/services/temporal/agent_activities.py
@@ -37,6 +37,7 @@ from __future__ import annotations
import asyncio
from dataclasses import asdict
+from datetime import timedelta
import hashlib
import json
import logging
@@ -159,11 +160,23 @@ def _as_temporal_llm_error(error: Any):
"retry_after": getattr(error, "retry_after", None),
"retry_after_raw": getattr(error, "retry_after_raw", None),
}
+ # Honor the provider's own pacing: a 429 with Retry-After should wait
+ # exactly that long before the next attempt instead of the policy's
+ # generic backoff.
+ retry_after = details["retry_after"]
+ next_retry_delay = (
+ timedelta(seconds=float(retry_after))
+ if details["retryable"]
+ and isinstance(retry_after, (int, float))
+ and retry_after > 0
+ else None
+ )
return ApplicationError(
safe_message,
details,
type=f"LLMError.{category}",
non_retryable=not details["retryable"],
+ next_retry_delay=next_retry_delay,
)
@@ -2058,17 +2071,14 @@ async def compact_context(payload: Dict[str, Any]) -> Dict[str, Any]:
activity.heartbeat("Compacting agent context")
svc = get_compaction_service()
if svc is None:
- # Singleton wired by the FastAPI lifespan (main.py). A worker
- # bootstrap race must not kill the run — the workflow checks
- # ``success`` and keeps its existing messages on False.
- return {
- "success": False,
- "error": (
- "CompactionService not initialized (worker bootstrap race)"
- ),
- "summary": "",
- "usage": {},
- }
+ # Singleton wired by the FastAPI lifespan (main.py). Retryable:
+ # a worker bootstrap race resolves itself within the retry
+ # policy's backoff window.
+ raise ApplicationError(
+ "CompactionService not initialized (worker bootstrap race)",
+ type="CompactionFailed",
+ non_retryable=False,
+ )
lines: List[str] = []
for wire in payload.get("messages") or []:
@@ -2094,7 +2104,7 @@ async def compact_context(payload: Dict[str, Any]) -> Dict[str, Any]:
)
lines.append(line)
- return await svc.compact_context(
+ result = await svc.compact_context(
session_id=str(payload.get("session_id") or "default"),
node_id=payload["node_id"],
memory_content="\n\n".join(lines),
@@ -2105,5 +2115,16 @@ async def compact_context(payload: Dict[str, Any]) -> Dict[str, Any]:
# request inside the activity or SDK.
explicit_max_retries=0,
)
+ if not result.get("success") or not result.get("summary"):
+ # Compaction is the run's pressure-relief valve. Raising (retryable)
+ # gives transient summarizer failures the activity policy's retry
+ # budget; if it still cannot compact, the workflow fails the run
+ # loudly rather than letting the transcript grow unbounded.
+ raise ApplicationError(
+ f"Compaction failed: {result.get('error') or 'empty summary'}",
+ type="CompactionFailed",
+ non_retryable=False,
+ )
+ return result
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index 44aab368..b4390ffb 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -851,12 +851,12 @@ class AgentWorkflow:
"include_finish_reason": True,
}
- # Wave 17.2: one-shot retry. The LLM call is not idempotent —
- # a worker crash mid-call must not silently re-bill the full
- # prompt (3x under the shared policy). The workflow owns the
- # failure instead: message history is intact here, so a
- # future enhancement can re-ask with context; today we
- # surface the error to the canvas and stop the loop.
+ # Transient provider failures (429 rate limit, 5xx, network)
+ # retry inside the activity under LLM_STEP_RETRY (unlimited,
+ # exponential backoff, provider retry_after honored via
+ # next_retry_delay). Only non-retryable classifications
+ # (invalid_request, authentication, ...) reach this except
+ # block, and those are genuinely terminal for the run.
try:
step_result = await workflow.execute_activity(
"agent.execute_llm_step",
@@ -893,8 +893,8 @@ class AgentWorkflow:
"Retry the run or check server logs."
)
workflow.logger.error(
- f"AgentWorkflow LLM step failed (iteration {iteration + 1}, "
- f"no auto-retry): {raw_detail}"
+ f"AgentWorkflow LLM step failed terminally "
+ f"(iteration {iteration + 1}): {raw_detail}"
)
# A failed model step is terminal for this run. Clear any
# turn-scoped skill badges and publish an error status before
@@ -2090,16 +2090,49 @@ class AgentWorkflow:
"model": payload["model"],
}
await self._wait_until_resumed()
- compact_result = await workflow.execute_activity(
- "agent.compact_context",
- args=[compact_payload],
- activity_id=f"compact-context-{iteration + 1}",
- start_to_close_timeout=COMPACT_MEMORY_TIMEOUT,
- retry_policy=AGENT_ACTIVITY_RETRY,
- )
+ try:
+ compact_result = await workflow.execute_activity(
+ "agent.compact_context",
+ args=[compact_payload],
+ activity_id=f"compact-context-{iteration + 1}",
+ start_to_close_timeout=COMPACT_MEMORY_TIMEOUT,
+ retry_policy=AGENT_ACTIVITY_RETRY,
+ )
+ except Exception as compact_error:
+ # Compaction is the run's pressure-relief valve. If it
+ # fails even after the activity policy's retries, the
+ # transcript can only grow until the provider rejects
+ # it — fail the run loudly NOW, at the moment the
+ # cause is clear, instead of later with a confusing
+ # context-overflow error.
+ cause = getattr(compact_error, "cause", None)
+ compact_detail = str(
+ getattr(cause, "message", "") or cause or compact_error
+ )
+ workflow.logger.error(
+ f"AgentWorkflow compaction failed terminally "
+ f"(iteration {iteration + 1}): {compact_detail}"
+ )
+ await self._emit_phase(
+ agent_node_id,
+ agent_workflow_id,
+ iteration,
+ max_iterations,
+ phase="failed",
+ status="error",
+ )
+ return {
+ "success": False,
+ "error": f"Compaction failed: {compact_detail}",
+ "error_type": "CompactionError",
+ "result": {
+ "iterations": iteration + 1,
+ "usage": usage_total,
+ },
+ }
# The summarizer is another billed model call. Include it in
- # execution-wide usage whether persistence after the call
- # succeeds or fails, but never in the active-context counter.
+ # execution-wide usage, but never in the active-context
+ # counter.
for key, value in (
compact_result.get("usage") or {}
).items():
@@ -2107,38 +2140,28 @@ class AgentWorkflow:
usage_total[key] = (
usage_total.get(key, 0) + value
)
- # Compaction is best-effort. When the service errors or
- # was not initialized (worker bootstrap race), keep the
- # existing messages and let the loop continue — masking
- # the failure would surface as a confused LLM, not a
- # workflow crash.
- if not compact_result.get("success"):
- workflow.logger.warning(
- "AgentWorkflow compaction failed (%s); continuing " "with un-compacted history",
- compact_result.get("error", "no error reported"),
- )
- else:
- summary = compact_result.get("summary", "")
- if summary:
- # Replace the running messages with the summary
- # plus the last user prompt — same pattern
- # ``CompactionService`` uses today in services/ai.py.
- compacted_content = f"## Compacted summary:\n{summary}"
- messages = [
- _native_message(
- role="system",
- content=system,
- ),
- _native_message(
- role="system",
- content=compacted_content,
- ),
- _native_message(
- role="user",
- content=user_prompt,
- ),
- ]
- context_usage_total = {}
+ # The activity raises on any failure, so a result here
+ # always carries a non-empty summary.
+ summary = compact_result.get("summary", "")
+ # Replace the running messages with the summary plus the
+ # last user prompt — same pattern ``CompactionService``
+ # uses today in services/ai.py.
+ compacted_content = f"## Compacted summary:\n{summary}"
+ messages = [
+ _native_message(
+ role="system",
+ content=system,
+ ),
+ _native_message(
+ role="system",
+ content=compacted_content,
+ ),
+ _native_message(
+ role="user",
+ content=user_prompt,
+ ),
+ ]
+ context_usage_total = {}
# ---- Continue-as-new -------------------------------------
# Only at a clean turn boundary, and never while a delegation
diff --git a/server/tests/temporal/test_agent_llm_contract.py b/server/tests/temporal/test_agent_llm_contract.py
index 04bf41e1..d082788b 100644
--- a/server/tests/temporal/test_agent_llm_contract.py
+++ b/server/tests/temporal/test_agent_llm_contract.py
@@ -141,7 +141,68 @@ class TestWorkflowLoopContract:
]
assert all("version" not in m for m in llm_payload["messages"])
assert options["heartbeat_timeout"] == timedelta(minutes=1)
- assert options["retry_policy"].maximum_attempts == 1
+ # Transient provider failures (429/5xx/network) retry indefinitely
+ # with backoff; terminal categories fail fast via the
+ # ApplicationError non_retryable marker, not this policy.
+ policy = options["retry_policy"]
+ assert policy.maximum_attempts == 0
+ assert policy.initial_interval == timedelta(seconds=5)
+ assert policy.maximum_interval == timedelta(minutes=5)
+
+ @pytest.mark.asyncio
+ async def test_compaction_failure_is_terminal_for_the_run(
+ self,
+ monkeypatch,
+ patched_workflow,
+ ):
+ """The pressure-relief valve failing must fail the run loudly.
+
+ Continuing with an ever-growing uncompacted transcript only defers
+ the failure to a confusing provider context-overflow error later.
+ """
+ from services.temporal.agent_workflow import AgentWorkflow
+
+ prepared = _payload()
+ prepared["compaction_threshold"] = 5
+
+ async def fake_execute_activity(name, *, args, **_kwargs):
+ if name == "agent.prepare_payload":
+ return prepared
+ if name == "agent.broadcast_progress":
+ return {"emitted": True}
+ if name == "agent.execute_llm_step":
+ return {
+ "kind": "tool_calls",
+ "calls": [
+ {"id": "missing-1", "name": "not_connected", "args": {}}
+ ],
+ "usage": {
+ "input_tokens": 7,
+ "output_tokens": 3,
+ "total_tokens": 10,
+ },
+ }
+ if name == "agent.compact_context":
+ raise RuntimeError("Compaction failed: summarizer down")
+ if name == "agent.store_output":
+ return {"stored": True}
+ if name == "agent.skill.clear":
+ return {"cleared": True}
+ raise AssertionError(f"Unexpected activity {name}")
+
+ monkeypatch.setattr(
+ patched_workflow,
+ "execute_activity",
+ fake_execute_activity,
+ )
+
+ result = await AgentWorkflow().run(
+ {"node_id": "agent-1", "execution_id": "root-run-1"}
+ )
+
+ assert result["success"] is False
+ assert result["error_type"] == "CompactionError"
+ assert "Compaction failed" in result["error"]
@pytest.mark.asyncio
async def test_compaction_summarizes_live_messages_without_memory(
@@ -300,6 +361,8 @@ class TestLlmStepActivity:
assert "raw body" not in str(temporal_error)
assert temporal_error.type == "LLMError.rate_limit"
assert temporal_error.non_retryable is False
+ # The provider's Retry-After hint paces the next Temporal attempt.
+ assert temporal_error.next_retry_delay == timedelta(seconds=1.5)
assert temporal_error.details == (
{
"provider": "openai",