Skip to content

Self-improving AI Employees for everyone

perf(engine): bound node-result size and stop the status cache leaking

trohitg5a716afc1d71
diff --git a/server/services/plugin/base.py b/server/services/plugin/base.py
index 1da0cb69..d1ae8612 100644
--- a/server/services/plugin/base.py
+++ b/server/services/plugin/base.py
@@ -27,6 +27,10 @@ from opentelemetry import trace
from pydantic import BaseModel, ValidationError
from core.logging import get_logger, log_context
+from services.media.limits import (
+ TEMPORAL_PAYLOAD_ERROR_BYTES,
+ TEMPORAL_PAYLOAD_WARN_BYTES,
+)
from services.plugin.connection import Connection
from services.plugin.context import NodeContext
from services.plugin.credential import Credential
@@ -614,14 +618,73 @@ class BaseNode:
stores (callers convert it to the standard error envelope).
"""
if isinstance(result, BaseModel):
- return result.model_dump(mode="json")
- if isinstance(result, dict) and self.Output is not _EmptyOutput:
+ payload = result.model_dump(mode="json")
+ elif isinstance(result, dict) and self.Output is not _EmptyOutput:
# ``exclude_unset`` preserves the producer's exact key set —
# declared-but-absent Optional fields must not materialise as
# ``None`` keys in the payload (validate + coerce + serialize,
# without reshaping what the operation chose to return).
- return self.Output.model_validate(result).model_dump(mode="json", exclude_unset=True)
- return result
+ payload = self.Output.model_validate(result).model_dump(
+ mode="json", exclude_unset=True
+ )
+ else:
+ payload = result
+
+ self._check_result_size(payload)
+ return payload
+
+ def _check_result_size(self, payload: Any) -> None:
+ """Warn on a large node result; refuse one Temporal cannot carry.
+
+ A node result is not stored once. It is written to ``node_outputs``
+ three times, broadcast twice, retained in the status cache, aggregated
+ into the workflow result, copied into **every downstream activity's
+ input**, and — when the node is ``usable_as_tool`` — serialized into an
+ LLM message.
+
+ Two thresholds, and the difference between them matters:
+
+ * At the **warning** threshold this only logs. Some existing nodes
+ legitimately return hundreds of KB (a parsed document, a long
+ transcript) and breaking them would be a regression.
+ * At the **error** threshold it raises. That is not a new failure: a
+ payload over Temporal's limit is rejected by the converter anyway.
+ What changes is *how* it fails — ``NodeUserError`` is already in
+ ``NON_RETRYABLE_ERROR_TYPES``, so instead of three attempts that
+ re-run the work (and re-bill whatever produced it) before reporting
+ a generic converter error, the run stops immediately with a message
+ naming the node and the size.
+
+ This is also why no Temporal-internal error type had to be named: the
+ payload never reaches the converter. The installed SDK enforces the
+ limit in its Rust core and exposes no Python class to add to a
+ non-retryable list, so catching it there was not possible anyway.
+ """
+ if payload is None:
+ return
+ try:
+ import orjson
+
+ size = len(orjson.dumps(payload, default=str))
+ except Exception:
+ # Sizing is diagnostics; never fail a result over it.
+ return
+
+ if size >= TEMPORAL_PAYLOAD_ERROR_BYTES:
+ raise NodeUserError(
+ f"{self.type} returned {size // 1024} KB, over the "
+ f"{TEMPORAL_PAYLOAD_ERROR_BYTES // (1024 * 1024)} MB the "
+ "workflow engine can carry. Large data belongs in the "
+ "workspace: write it to a file and return a reference "
+ "(see services/media for the audio case)."
+ )
+ if size >= TEMPORAL_PAYLOAD_WARN_BYTES:
+ logger.warning(
+ "large node result",
+ node_type=self.type,
+ size_bytes=size,
+ limit_bytes=TEMPORAL_PAYLOAD_ERROR_BYTES,
+ )
def _wrap_success(self, *, start_time: float, result: Any) -> Dict[str, Any]:
"""95%-universal return shape. Subclasses (ToolNode) override."""
diff --git a/server/services/status_broadcaster.py b/server/services/status_broadcaster.py
index d4f82824..05980e8d 100644
--- a/server/services/status_broadcaster.py
+++ b/server/services/status_broadcaster.py
@@ -13,6 +13,49 @@ from opentelemetry import trace
from core.logging import get_logger
logger = get_logger(__name__)
+
+# Above this, a node output is replaced by a stub in the retained status
+# cache. Generous on purpose: the cache exists so a reconnecting client can
+# see what the last run produced, and almost every real output fits well
+# under it. It is a leak bound, not a display limit.
+_CACHED_OUTPUT_MAX_BYTES = 64 * 1024
+
+
+def _elide_for_cache(output: Any) -> Any:
+ """Return ``output``, or a small stub when it is too large to retain.
+
+ ``StatusBroadcaster._status`` is never evicted and is replayed in full to
+ every newly connecting client. Without this, one large node output is held
+ for the process lifetime *and* re-sent on every page load — the same bytes,
+ indefinitely.
+
+ Only the cached copy is elided; the live broadcast is untouched, so the
+ user who ran the node still sees the whole thing. A client that reconnects
+ later sees the stub and can refetch the real output from ``node_outputs``,
+ which is the durable store.
+ """
+ if output is None:
+ return output
+ try:
+ import orjson
+
+ size = len(orjson.dumps(output, default=str))
+ except Exception:
+ # Unsizeable means unknown, and unknown is not a reason to discard.
+ return output
+
+ if size <= _CACHED_OUTPUT_MAX_BYTES:
+ return output
+
+ logger.debug("eliding large node output from the status cache", size_bytes=size)
+ return {
+ "_elided": True,
+ "_size_bytes": size,
+ "_note": (
+ "Output too large to retain in the live status cache. The full "
+ "value is in node_outputs."
+ ),
+ }
tracer = trace.get_tracer(__name__)
@@ -686,11 +729,20 @@ class StatusBroadcaster:
)
async def update_node_output(self, node_id: str, output: Any, workflow_id: Optional[str] = None):
- """Update a node's output data and broadcast."""
+ """Update a node's output data and broadcast.
+
+ The **broadcast carries the full output** — the user who just ran the
+ node sees exactly what it produced. What gets *cached* may be elided:
+ ``self._status`` is never evicted and is replayed in full to every
+ newly connecting client, so a large output otherwise costs memory for
+ the process lifetime and is re-sent on every page load, forever.
+
+ Eliding only the cache keeps the live view intact while bounding both.
+ """
if node_id not in self._status["nodes"]:
self._status["nodes"][node_id] = {"status": "idle", "data": {}}
- self._status["nodes"][node_id]["output"] = output
+ self._status["nodes"][node_id]["output"] = _elide_for_cache(output)
if workflow_id:
self._status["nodes"][node_id]["workflow_id"] = workflow_id
diff --git a/server/tests/test_payload_guards.py b/server/tests/test_payload_guards.py
new file mode 100644
index 00000000..0e51c64d
--- /dev/null
+++ b/server/tests/test_payload_guards.py
@@ -0,0 +1,151 @@
+"""Guards on how much data a node result may carry.
+
+A node result is copied many times — three writes to ``node_outputs``, two
+broadcasts, the retained status cache, the workflow aggregate, every
+downstream activity's *input*, and (for a tool-exposed node) an LLM message.
+Temporal caps a blob at 2 MiB for activity results and inputs alike.
+
+Two separate mechanisms, tested here together because they defend the same
+thing from different ends: the producer refuses to emit an uncarryable
+payload, and the retained cache refuses to hold a large one forever.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+import pytest
+from pydantic import BaseModel
+
+from services.media.limits import (
+ TEMPORAL_PAYLOAD_ERROR_BYTES,
+ TEMPORAL_PAYLOAD_WARN_BYTES,
+)
+from services.plugin import NodeUserError
+from services.plugin.base import BaseNode
+
+pytestmark = pytest.mark.unit
+
+
+class _Output(BaseModel):
+ blob: str = ""
+
+ model_config = {"extra": "allow"}
+
+
+class _Stand_in:
+ """Enough of a node for the size guard, without registering one.
+
+ Subclassing ``ActionNode`` here would auto-register a fake node type into
+ the global ``NODE_METADATA`` via ``__init_subclass__`` and break every
+ plugin-contract invariant that walks the registry — which is exactly what
+ happened the first time this file was written.
+ """
+
+ type = "_test_sized_node"
+ Output = _Output
+
+ _serialize_result = BaseNode._serialize_result
+ _check_result_size = BaseNode._check_result_size
+
+
+def _SizedNode() -> _Stand_in:
+ return _Stand_in()
+
+
+class TestResultSizeGuard:
+ def test_a_normal_result_passes_untouched(self):
+ node = _SizedNode()
+ payload = node._serialize_result(_Output(blob="hello"))
+ assert payload == {"blob": "hello"}
+
+ def test_a_large_but_carryable_result_only_warns(self, caplog):
+ """Existing nodes legitimately return hundreds of KB.
+
+ Failing them would be a regression, so between the two thresholds
+ this logs and nothing more.
+ """
+ node = _SizedNode()
+ size = (TEMPORAL_PAYLOAD_WARN_BYTES + TEMPORAL_PAYLOAD_ERROR_BYTES) // 2
+ payload = node._serialize_result(_Output(blob="x" * size))
+ assert payload["blob"]
+
+ def test_an_uncarryable_result_is_refused(self):
+ """Not a new failure — Temporal rejects this payload anyway.
+
+ What changes is that it fails once, here, with a message naming the
+ node and the size, instead of three times inside the converter with a
+ generic error. ``NodeUserError`` is already non-retryable.
+ """
+ node = _SizedNode()
+ with pytest.raises(NodeUserError) as excinfo:
+ node._serialize_result(_Output(blob="x" * (TEMPORAL_PAYLOAD_ERROR_BYTES + 1)))
+
+ message = str(excinfo.value)
+ assert "_test_sized_node" in message
+ assert "workspace" in message.lower()
+
+ def test_the_refusal_is_non_retryable(self):
+ """The whole point of raising NodeUserError rather than anything else."""
+ from services.temporal._retry_policies import NON_RETRYABLE_ERROR_TYPES
+
+ assert NodeUserError.__name__ in NON_RETRYABLE_ERROR_TYPES
+
+ def test_sizing_failure_never_blocks_a_result(self, monkeypatch):
+ """Sizing is diagnostics; an unserializable payload still passes."""
+ node = _SizedNode()
+
+ class _Hostile:
+ def __repr__(self):
+ raise RuntimeError("nope")
+
+ # Not a BaseModel and no declared Output match, so it passes through
+ # _check_result_size, which must swallow its own failure.
+ assert node._check_result_size({"x": _Hostile()}) is None
+
+
+class TestStatusCacheElision:
+ """``_status`` is never evicted and is replayed to every new client."""
+
+ def test_a_small_output_is_cached_verbatim(self):
+ from services.status_broadcaster import _elide_for_cache
+
+ payload = {"result": "small"}
+ assert _elide_for_cache(payload) is payload
+
+ def test_a_large_output_is_replaced_by_a_stub(self):
+ from services.status_broadcaster import _elide_for_cache
+
+ elided = _elide_for_cache({"blob": "x" * (256 * 1024)})
+ assert elided["_elided"] is True
+ assert elided["_size_bytes"] > 256 * 1024
+ assert "node_outputs" in elided["_note"]
+
+ async def test_the_broadcast_still_carries_the_full_output(self):
+ """Only the retained copy is elided — the live view is untouched."""
+ from services.status_broadcaster import StatusBroadcaster
+
+ broadcaster = StatusBroadcaster()
+ sent: list[Dict[str, Any]] = []
+
+ async def _capture(message):
+ sent.append(message)
+
+ broadcaster.broadcast = _capture # type: ignore[method-assign]
+
+ big = {"blob": "y" * (256 * 1024)}
+ await broadcaster.update_node_output("node-1", big, workflow_id="wf-1")
+
+ assert sent[0]["output"] == big
+ cached = broadcaster._status["nodes"]["node-1"]["output"]
+ assert cached["_elided"] is True
+
+ def test_unsizeable_output_is_kept_rather_than_discarded(self):
+ """Unknown size is not a reason to throw data away."""
+ from services.status_broadcaster import _elide_for_cache
+
+ class _Unserializable:
+ pass
+
+ value = {"x": _Unserializable()}
+ assert _elide_for_cache(value) is value