Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(temporal): retry transient LLM failures instead of killing the run; compaction failure is terminal
trohitg74b5a55fdf02
diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.pyindex 31d34760..882799d4 100644--- a/server/services/temporal/_retry_policies.py+++ b/server/services/temporal/_retry_policies.py@@ -78,16 +78,23 @@ QUICK_ACTIVITY_RETRY: RetryPolicy = RetryPolicy()-# Wave 17.2: one-shot policy for the LLM step. An LLM API call is NOT-# idempotent from a cost perspective — every automatic re-attempt after-# a worker crash / laptop sleep bills the full prompt again (and can-# double-fire side effects when the first attempt actually completed-# server-side but the worker died before recording the result). The-# workflow layer (AgentWorkflow.run) owns the retry decision instead:-# it catches the ActivityError with the message history intact and-# surfaces the failure to the canvas rather than silently re-billing.+# LLM-step policy: retry transient provider failures indefinitely with+# exponential backoff. A clean provider rejection (429 rate limit, 5xx,+# network refusal) bills nothing and MUST be retried — the previous+# one-shot policy (Wave 17.2) killed the whole run on the first 429 and+# then tripped the deployment circuit breaker. Terminal outcomes still+# fail fast: ``_as_temporal_llm_error`` marks non-retryable categories+# (invalid_request, authentication, ...) with ``non_retryable=True``,+# which overrides this policy, and provider ``retry_after`` hints ride+# ``ApplicationError.next_retry_delay``. Known trade-off, accepted: an+# AMBIGUOUS loss (worker crash / timeout where the call may have+# completed server-side) now also retries and can re-bill a prompt —+# resilience of long-running deployments wins over billing paranoia.LLM_STEP_RETRY: RetryPolicy = RetryPolicy(- maximum_attempts=1,+ initial_interval=timedelta(seconds=5),+ backoff_coefficient=2.0,+ maximum_interval=timedelta(minutes=5),+ maximum_attempts=0, # unlimited — liveness comes from heartbeatsnon_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),)diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.pyindex 374f12be..04e6cba9 100644--- a/server/services/temporal/agent_activities.py+++ b/server/services/temporal/agent_activities.py@@ -37,6 +37,7 @@ from __future__ import annotationsimport asynciofrom dataclasses import asdict+from datetime import timedeltaimport hashlibimport jsonimport logging@@ -159,11 +160,23 @@ def _as_temporal_llm_error(error: Any):"retry_after": getattr(error, "retry_after", None),"retry_after_raw": getattr(error, "retry_after_raw", None),}+ # Honor the provider's own pacing: a 429 with Retry-After should wait+ # exactly that long before the next attempt instead of the policy's+ # generic backoff.+ retry_after = details["retry_after"]+ next_retry_delay = (+ timedelta(seconds=float(retry_after))+ if details["retryable"]+ and isinstance(retry_after, (int, float))+ and retry_after > 0+ else None+ )return ApplicationError(safe_message,details,type=f"LLMError.{category}",non_retryable=not details["retryable"],+ next_retry_delay=next_retry_delay,)@@ -2058,17 +2071,14 @@ async def compact_context(payload: Dict[str, Any]) -> Dict[str, Any]:activity.heartbeat("Compacting agent context")svc = get_compaction_service()if svc is None:- # Singleton wired by the FastAPI lifespan (main.py). A worker- # bootstrap race must not kill the run — the workflow checks- # ``success`` and keeps its existing messages on False.- return {- "success": False,- "error": (- "CompactionService not initialized (worker bootstrap race)"- ),- "summary": "",- "usage": {},- }+ # Singleton wired by the FastAPI lifespan (main.py). Retryable:+ # a worker bootstrap race resolves itself within the retry+ # policy's backoff window.+ raise ApplicationError(+ "CompactionService not initialized (worker bootstrap race)",+ type="CompactionFailed",+ non_retryable=False,+ )lines: List[str] = []for wire in payload.get("messages") or []:@@ -2094,7 +2104,7 @@ async def compact_context(payload: Dict[str, Any]) -> Dict[str, Any]:)lines.append(line)- return await svc.compact_context(+ result = await svc.compact_context(session_id=str(payload.get("session_id") or "default"),node_id=payload["node_id"],memory_content="\n\n".join(lines),@@ -2105,5 +2115,16 @@ async def compact_context(payload: Dict[str, Any]) -> Dict[str, Any]:# request inside the activity or SDK.explicit_max_retries=0,)+ if not result.get("success") or not result.get("summary"):+ # Compaction is the run's pressure-relief valve. Raising (retryable)+ # gives transient summarizer failures the activity policy's retry+ # budget; if it still cannot compact, the workflow fails the run+ # loudly rather than letting the transcript grow unbounded.+ raise ApplicationError(+ f"Compaction failed: {result.get('error') or 'empty summary'}",+ type="CompactionFailed",+ non_retryable=False,+ )+ return resultdiff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.pyindex 44aab368..b4390ffb 100644--- a/server/services/temporal/agent_workflow.py+++ b/server/services/temporal/agent_workflow.py@@ -851,12 +851,12 @@ class AgentWorkflow:"include_finish_reason": True,}- # Wave 17.2: one-shot retry. The LLM call is not idempotent —- # a worker crash mid-call must not silently re-bill the full- # prompt (3x under the shared policy). The workflow owns the- # failure instead: message history is intact here, so a- # future enhancement can re-ask with context; today we- # surface the error to the canvas and stop the loop.+ # Transient provider failures (429 rate limit, 5xx, network)+ # retry inside the activity under LLM_STEP_RETRY (unlimited,+ # exponential backoff, provider retry_after honored via+ # next_retry_delay). Only non-retryable classifications+ # (invalid_request, authentication, ...) reach this except+ # block, and those are genuinely terminal for the run.try:step_result = await workflow.execute_activity("agent.execute_llm_step",@@ -893,8 +893,8 @@ class AgentWorkflow:"Retry the run or check server logs.")workflow.logger.error(- f"AgentWorkflow LLM step failed (iteration {iteration + 1}, "- f"no auto-retry): {raw_detail}"+ f"AgentWorkflow LLM step failed terminally "+ f"(iteration {iteration + 1}): {raw_detail}")# A failed model step is terminal for this run. Clear any# turn-scoped skill badges and publish an error status before@@ -2090,16 +2090,49 @@ class AgentWorkflow:"model": payload["model"],}await self._wait_until_resumed()- compact_result = await workflow.execute_activity(- "agent.compact_context",- args=[compact_payload],- activity_id=f"compact-context-{iteration + 1}",- start_to_close_timeout=COMPACT_MEMORY_TIMEOUT,- retry_policy=AGENT_ACTIVITY_RETRY,- )+ try:+ compact_result = await workflow.execute_activity(+ "agent.compact_context",+ args=[compact_payload],+ activity_id=f"compact-context-{iteration + 1}",+ start_to_close_timeout=COMPACT_MEMORY_TIMEOUT,+ retry_policy=AGENT_ACTIVITY_RETRY,+ )+ except Exception as compact_error:+ # Compaction is the run's pressure-relief valve. If it+ # fails even after the activity policy's retries, the+ # transcript can only grow until the provider rejects+ # it — fail the run loudly NOW, at the moment the+ # cause is clear, instead of later with a confusing+ # context-overflow error.+ cause = getattr(compact_error, "cause", None)+ compact_detail = str(+ getattr(cause, "message", "") or cause or compact_error+ )+ workflow.logger.error(+ f"AgentWorkflow compaction failed terminally "+ f"(iteration {iteration + 1}): {compact_detail}"+ )+ await self._emit_phase(+ agent_node_id,+ agent_workflow_id,+ iteration,+ max_iterations,+ phase="failed",+ status="error",+ )+ return {+ "success": False,+ "error": f"Compaction failed: {compact_detail}",+ "error_type": "CompactionError",+ "result": {+ "iterations": iteration + 1,+ "usage": usage_total,+ },+ }# The summarizer is another billed model call. Include it in- # execution-wide usage whether persistence after the call- # succeeds or fails, but never in the active-context counter.+ # execution-wide usage, but never in the active-context+ # counter.for key, value in (compact_result.get("usage") or {}).items():@@ -2107,38 +2140,28 @@ class AgentWorkflow:usage_total[key] = (usage_total.get(key, 0) + value)- # Compaction is best-effort. When the service errors or- # was not initialized (worker bootstrap race), keep the- # existing messages and let the loop continue — masking- # the failure would surface as a confused LLM, not a- # workflow crash.- if not compact_result.get("success"):- workflow.logger.warning(- "AgentWorkflow compaction failed (%s); continuing " "with un-compacted history",- compact_result.get("error", "no error reported"),- )- else:- summary = compact_result.get("summary", "")- if summary:- # Replace the running messages with the summary- # plus the last user prompt — same pattern- # ``CompactionService`` uses today in services/ai.py.- compacted_content = f"## Compacted summary:\n{summary}"- messages = [- _native_message(- role="system",- content=system,- ),- _native_message(- role="system",- content=compacted_content,- ),- _native_message(- role="user",- content=user_prompt,- ),- ]- context_usage_total = {}+ # The activity raises on any failure, so a result here+ # always carries a non-empty summary.+ summary = compact_result.get("summary", "")+ # Replace the running messages with the summary plus the+ # last user prompt — same pattern ``CompactionService``+ # uses today in services/ai.py.+ compacted_content = f"## Compacted summary:\n{summary}"+ messages = [+ _native_message(+ role="system",+ content=system,+ ),+ _native_message(+ role="system",+ content=compacted_content,+ ),+ _native_message(+ role="user",+ content=user_prompt,+ ),+ ]+ context_usage_total = {}# ---- Continue-as-new -------------------------------------# Only at a clean turn boundary, and never while a delegationdiff --git a/server/tests/temporal/test_agent_llm_contract.py b/server/tests/temporal/test_agent_llm_contract.pyindex 04bf41e1..d082788b 100644--- a/server/tests/temporal/test_agent_llm_contract.py+++ b/server/tests/temporal/test_agent_llm_contract.py@@ -141,7 +141,68 @@ class TestWorkflowLoopContract:]assert all("version" not in m for m in llm_payload["messages"])assert options["heartbeat_timeout"] == timedelta(minutes=1)- assert options["retry_policy"].maximum_attempts == 1+ # Transient provider failures (429/5xx/network) retry indefinitely+ # with backoff; terminal categories fail fast via the+ # ApplicationError non_retryable marker, not this policy.+ policy = options["retry_policy"]+ assert policy.maximum_attempts == 0+ assert policy.initial_interval == timedelta(seconds=5)+ assert policy.maximum_interval == timedelta(minutes=5)++ @pytest.mark.asyncio+ async def test_compaction_failure_is_terminal_for_the_run(+ self,+ monkeypatch,+ patched_workflow,+ ):+ """The pressure-relief valve failing must fail the run loudly.++ Continuing with an ever-growing uncompacted transcript only defers+ the failure to a confusing provider context-overflow error later.+ """+ from services.temporal.agent_workflow import AgentWorkflow++ prepared = _payload()+ prepared["compaction_threshold"] = 5++ async def fake_execute_activity(name, *, args, **_kwargs):+ if name == "agent.prepare_payload":+ return prepared+ if name == "agent.broadcast_progress":+ return {"emitted": True}+ if name == "agent.execute_llm_step":+ return {+ "kind": "tool_calls",+ "calls": [+ {"id": "missing-1", "name": "not_connected", "args": {}}+ ],+ "usage": {+ "input_tokens": 7,+ "output_tokens": 3,+ "total_tokens": 10,+ },+ }+ if name == "agent.compact_context":+ raise RuntimeError("Compaction failed: summarizer down")+ if name == "agent.store_output":+ return {"stored": True}+ if name == "agent.skill.clear":+ return {"cleared": True}+ raise AssertionError(f"Unexpected activity {name}")++ monkeypatch.setattr(+ patched_workflow,+ "execute_activity",+ fake_execute_activity,+ )++ result = await AgentWorkflow().run(+ {"node_id": "agent-1", "execution_id": "root-run-1"}+ )++ assert result["success"] is False+ assert result["error_type"] == "CompactionError"+ assert "Compaction failed" in result["error"]@pytest.mark.asyncioasync def test_compaction_summarizes_live_messages_without_memory(@@ -300,6 +361,8 @@ class TestLlmStepActivity:assert "raw body" not in str(temporal_error)assert temporal_error.type == "LLMError.rate_limit"assert temporal_error.non_retryable is False+ # The provider's Retry-After hint paces the next Temporal attempt.+ assert temporal_error.next_retry_delay == timedelta(seconds=1.5)assert temporal_error.details == ({"provider": "openai",