Skip to content

Self-improving AI Employees for everyone

fix(temporal): oversized results and LLM-step bugs fail once (1b.26)

trohitg444c70a583bd
diff --git a/docs-internal/DESIGN.md b/docs-internal/DESIGN.md
index a707fe8c..c9989054 100644
--- a/docs-internal/DESIGN.md
+++ b/docs-internal/DESIGN.md
@@ -145,10 +145,11 @@ newly connecting client, aggregated into the workflow result, copied into
`usable_as_tool` — serialized verbatim into an LLM message.
Temporal caps a blob at **2,097,152 B** for activity results *and inputs*, and
-warns at 524,288 B. `_PayloadSizeError` is not in `NON_RETRYABLE_ERROR_TYPES`,
-so exceeding the cap costs **three attempts** — re-invoking (and re-billing)
-whatever produced the payload each time — before surfacing a generic failure
-that names nothing useful.
+warns at 524,288 B. Exceeding the cap fails the activity as `PayloadsTooLarge`
+(the SDK core's failure type since temporalio 1.31). It is in
+`NON_RETRYABLE_ERROR_TYPES` (1b.26), so whatever produced the payload runs
+once; before, an oversized result cost **three attempts**, re-invoking (and
+re-billing) the producer each time.
So: **large data lives on disk in the workflow workspace, and the node returns
a reference.** `services/media/` provides that for binary media (`AudioRef`,
diff --git a/docs-internal/media_transport.md b/docs-internal/media_transport.md
index 27124008..ee9374eb 100644
--- a/docs-internal/media_transport.md
+++ b/docs-internal/media_transport.md
@@ -23,7 +23,7 @@ in this module exists to make the correct thing the easy thing.
|---|---|---|
| Temporal blob **error** (activity result *and input*, workflow result) | **2,097,152 B** | server default; no custom DataConverter anywhere |
| Temporal blob **warning** | 524,288 B → `PayloadSizeWarning`, unfiltered | |
-| Retries burned before failing | **3** — `_PayloadSizeError` is not in `NON_RETRYABLE_ERROR_TYPES` | [`_retry_policies.py`](../server/services/temporal/_retry_policies.py) |
+| Retries burned before failing | **none** since 1b.26: the SDK core fails an oversized result as `PayloadsTooLarge`, which is in `NON_RETRYABLE_ERROR_TYPES` (before, **3** attempts, each re-billing the producer) | [`_retry_policies.py`](../server/services/temporal/_retry_policies.py) |
| `node_outputs.data` | no cap, written **3×** per store | [`activities.py`](../server/services/temporal/activities.py) |
| WS broadcast | no size guard; payload retained in `_status` **forever** and replayed to every new client | [`status_broadcaster.py`](../server/services/status_broadcaster.py) |
diff --git a/server/services/plugin/scaling.py b/server/services/plugin/scaling.py
index ac57ff33..68df7ea5 100644
--- a/server/services/plugin/scaling.py
+++ b/server/services/plugin/scaling.py
@@ -68,6 +68,9 @@ class RetryPolicy:
# NodeUserError fail fast; the framework surfaces the message
# to the user / agent loop without traceback noise.
"NodeUserError",
+ # A result over the server's payload limit is as big on the next
+ # attempt, which would run (and bill) the call again.
+ "PayloadsTooLarge",
)
)
diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.py
index ddca3c9f..a137d493 100644
--- a/server/services/temporal/_retry_policies.py
+++ b/server/services/temporal/_retry_policies.py
@@ -52,6 +52,9 @@ from temporalio.common import RetryPolicy
NON_RETRYABLE_ERROR_TYPES: tuple[str, ...] = (
"NodeUserError",
"InvalidEvent",
+ # A result over the server's payload limit (the SDK core's failure type)
+ # is as big on the next attempt, which would pay for it again.
+ "PayloadsTooLarge",
)
diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.py
index f7cd314b..baca9ebc 100644
--- a/server/services/temporal/agent_activities.py
+++ b/server/services/temporal/agent_activities.py
@@ -492,31 +492,40 @@ async def _execute_native_llm_step(payload: Dict[str, Any]) -> Dict[str, Any]:
messages_from_wire,
)
- messages = filter_empty_messages(
- messages_from_wire(payload.get("messages") or [])
- )
- _ensure_llm_contents(messages)
-
- tool_defs: List[ToolDef] = []
- for value in payload.get("tools") or []:
- definition = value.get("definition", value)
- tool_defs.append(
- ToolDef(
- name=str(definition.get("name") or ""),
- description=str(definition.get("description") or ""),
- parameters=dict(
- definition.get("parameters")
- or {"type": "object", "properties": {}}
- ),
- )
+ # A step that cannot be built fails once, before the provider is called:
+ # a retry would build it the same way (1b.26).
+ try:
+ messages = filter_empty_messages(
+ messages_from_wire(payload.get("messages") or [])
)
+ _ensure_llm_contents(messages)
+
+ tool_defs: List[ToolDef] = []
+ for value in payload.get("tools") or []:
+ definition = value.get("definition", value)
+ tool_defs.append(
+ ToolDef(
+ name=str(definition.get("name") or ""),
+ description=str(definition.get("description") or ""),
+ parameters=dict(
+ definition.get("parameters")
+ or {"type": "object", "properties": {}}
+ ),
+ )
+ )
- thinking_data = payload.get("thinking_config")
- thinking = (
- ThinkingConfig(**thinking_data)
- if isinstance(thinking_data, dict)
- else None
- )
+ thinking_data = payload.get("thinking_config")
+ thinking = (
+ ThinkingConfig(**thinking_data)
+ if isinstance(thinking_data, dict)
+ else None
+ )
+ except (KeyError, TypeError, ValueError, AttributeError) as exc:
+ raise ApplicationError(
+ f"The model request could not be built: {type(exc).__name__}: {exc}",
+ type="LLMStepFailed",
+ non_retryable=True,
+ ) from exc
# A chat run the owner stopped ends at this step, before paying for it.
chat_run_id = payload.get("chat_run_id")
if isinstance(chat_run_id, str) and chat_run_id and await _chat_run_stopping(chat_run_id):
@@ -580,72 +589,83 @@ async def _execute_native_llm_step(payload: Dict[str, Any]) -> Dict[str, Any]:
emitter.discard()
raise
- if response is _STOPPED:
- # What the owner saw so far is the answer: the reply node posts it,
- # saved as stopped, and the run finishes ``stopped``.
- partial = shown_text(emitter.text).rstrip() if emitter is not None else ""
+ # The provider answered and billed: a bug from here on fails once, since
+ # a retry would pay for the same answer again (1b.26).
+ try:
+ if response is _STOPPED:
+ # What the owner saw so far is the answer: the reply node posts it,
+ # saved as stopped, and the run finishes ``stopped``.
+ partial = shown_text(emitter.text).rstrip() if emitter is not None else ""
+ if emitter is not None:
+ emitter.end(final=True)
+ return await _stopped_llm_step(payload, messages, partial=partial)
if emitter is not None:
- emitter.end(final=True)
- return await _stopped_llm_step(payload, messages, partial=partial)
- if emitter is not None:
- # Text beside tool calls is narration; without them it is the reply.
- emitter.end(final=not response.tool_calls)
+ # Text beside tool calls is narration; without them it is the reply.
+ emitter.end(final=not response.tool_calls)
- assistant = response.assistant_message or Message(
- role="assistant",
- content=response.content or "",
- tool_calls=list(response.tool_calls or []),
- )
- assistant_wire = message_to_wire(assistant)
- usage = asdict(response.usage)
-
- # Persist the conversation exactly as it happened: the message list this
- # activity just sent to the provider plus the reply it got back.
- # ``messages`` above IS the argument handed to ``ChatUnifier.chat`` —
- # nothing is rebuilt, inferred, or re-typed, so the stored conversation
- # cannot drift from the request.
- observe_stage("persistence")
- await _save_conversation(
- payload,
- sent=[dict(message_to_wire(message)) for message in messages],
- assistant_wire=dict(assistant_wire),
- )
+ assistant = response.assistant_message or Message(
+ role="assistant",
+ content=response.content or "",
+ tool_calls=list(response.tool_calls or []),
+ )
+ assistant_wire = message_to_wire(assistant)
+ usage = asdict(response.usage)
+
+ # Persist the conversation exactly as it happened: the message list this
+ # activity just sent to the provider plus the reply it got back.
+ # ``messages`` above IS the argument handed to ``ChatUnifier.chat`` —
+ # nothing is rebuilt, inferred, or re-typed, so the stored conversation
+ # cannot drift from the request.
+ observe_stage("persistence")
+ await _save_conversation(
+ payload,
+ sent=[dict(message_to_wire(message)) for message in messages],
+ assistant_wire=dict(assistant_wire),
+ )
- if response.tool_calls:
- calls = []
- for tool_call in response.tool_calls:
- call = {
- "id": tool_call.id,
- "name": tool_call.name,
- "args": tool_call.args,
+ if response.tool_calls:
+ calls = []
+ for tool_call in response.tool_calls:
+ call = {
+ "id": tool_call.id,
+ "name": tool_call.name,
+ "args": tool_call.args,
+ }
+ raw_arguments = getattr(tool_call, "raw_arguments", None)
+ parse_error = getattr(tool_call, "parse_error", None)
+ if raw_arguments is not None:
+ call["raw_arguments"] = raw_arguments
+ if parse_error:
+ call["parse_error"] = parse_error
+ calls.append(call)
+ result = {
+ "kind": "tool_calls",
+ "assistant_message": assistant_wire,
+ "calls": calls,
+ "usage": usage,
}
- raw_arguments = getattr(tool_call, "raw_arguments", None)
- parse_error = getattr(tool_call, "parse_error", None)
- if raw_arguments is not None:
- call["raw_arguments"] = raw_arguments
- if parse_error:
- call["parse_error"] = parse_error
- calls.append(call)
+ if payload.get("include_finish_reason"):
+ result["finish_reason"] = response.finish_reason
+ return result
+
result = {
- "kind": "tool_calls",
+ "kind": "final",
"assistant_message": assistant_wire,
- "calls": calls,
+ "content": response.content or "",
+ "thinking": response.thinking,
"usage": usage,
}
if payload.get("include_finish_reason"):
result["finish_reason"] = response.finish_reason
return result
-
- result = {
- "kind": "final",
- "assistant_message": assistant_wire,
- "content": response.content or "",
- "thinking": response.thinking,
- "usage": usage,
- }
- if payload.get("include_finish_reason"):
- result["finish_reason"] = response.finish_reason
- return result
+ except ApplicationError:
+ raise
+ except Exception as exc:
+ raise ApplicationError(
+ f"The model's answer could not be handled: {type(exc).__name__}: {exc}",
+ type="LLMStepInternalError",
+ non_retryable=True,
+ ) from exc
async def _save_conversation(
diff --git a/server/tests/temporal/test_llm_retry_status.py b/server/tests/temporal/test_llm_retry_status.py
index b689dee2..c73ef7c6 100644
--- a/server/tests/temporal/test_llm_retry_status.py
+++ b/server/tests/temporal/test_llm_retry_status.py
@@ -3,6 +3,7 @@
import asyncio
from dataclasses import replace
from datetime import timedelta
+from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
@@ -211,3 +212,35 @@ async def test_activity_cancellation_withdraws_partial_and_cancels_provider(step
assert events[-1]["fields"] == {"name": "opencompany.segment_discarded", "value": {"message_id": "run-1.7.1"}}
assert not any(event["suffix"] == "text.ended" for event in events)
run.assert_awaited_once()
+
+
+def test_a_result_too_big_for_temporal_is_not_retried():
+ # 1b.26: the next attempt would be as big, and would pay for it again.
+ from services.plugin.scaling import RetryPolicy
+ from services.temporal._retry_policies import LLM_STEP_RETRY, NON_RETRYABLE_ERROR_TYPES
+
+ assert "PayloadsTooLarge" in NON_RETRYABLE_ERROR_TYPES
+ assert "PayloadsTooLarge" in LLM_STEP_RETRY.non_retryable_error_types
+ assert "PayloadsTooLarge" in RetryPolicy().non_retryable_error_types
+
+
+async def test_a_step_that_cannot_be_built_fails_once_before_the_provider(step):
+ env, run, _broadcaster, payload = step
+ with pytest.raises(ApplicationError) as raised:
+ await env.run(execute_llm_step, {**payload, "thinking_config": {"not_a_field": True}})
+ assert raised.value.type == "LLMStepFailed" and raised.value.non_retryable
+ run.assert_not_awaited()
+
+
+async def test_a_bug_after_the_answer_fails_once(step):
+ # The provider answered and billed: a retry would pay for it again.
+ env, run, _broadcaster, payload = step
+ run.return_value = SimpleNamespace(
+ tool_calls=None, assistant_message=None, content="done", thinking=None, usage="not a usage", finish_reason=None
+ )
+ with pytest.raises(ApplicationError) as raised:
+ await env.run(execute_llm_step, payload)
+ assert raised.value.type == "LLMStepInternalError" and raised.value.non_retryable
+ run.assert_awaited_once()