Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
perf(engine): bound node-result size and stop the status cache leaking
trohitg5a716afc1d71
diff --git a/server/services/plugin/base.py b/server/services/plugin/base.pyindex 1da0cb69..d1ae8612 100644--- a/server/services/plugin/base.py+++ b/server/services/plugin/base.py@@ -27,6 +27,10 @@ from opentelemetry import tracefrom pydantic import BaseModel, ValidationErrorfrom core.logging import get_logger, log_context+from services.media.limits import (+ TEMPORAL_PAYLOAD_ERROR_BYTES,+ TEMPORAL_PAYLOAD_WARN_BYTES,+)from services.plugin.connection import Connectionfrom services.plugin.context import NodeContextfrom services.plugin.credential import Credential@@ -614,14 +618,73 @@ class BaseNode:stores (callers convert it to the standard error envelope)."""if isinstance(result, BaseModel):- return result.model_dump(mode="json")- if isinstance(result, dict) and self.Output is not _EmptyOutput:+ payload = result.model_dump(mode="json")+ elif isinstance(result, dict) and self.Output is not _EmptyOutput:# ``exclude_unset`` preserves the producer's exact key set —# declared-but-absent Optional fields must not materialise as# ``None`` keys in the payload (validate + coerce + serialize,# without reshaping what the operation chose to return).- return self.Output.model_validate(result).model_dump(mode="json", exclude_unset=True)- return result+ payload = self.Output.model_validate(result).model_dump(+ mode="json", exclude_unset=True+ )+ else:+ payload = result++ self._check_result_size(payload)+ return payload++ def _check_result_size(self, payload: Any) -> None:+ """Warn on a large node result; refuse one Temporal cannot carry.++ A node result is not stored once. It is written to ``node_outputs``+ three times, broadcast twice, retained in the status cache, aggregated+ into the workflow result, copied into **every downstream activity's+ input**, and — when the node is ``usable_as_tool`` — serialized into an+ LLM message.++ Two thresholds, and the difference between them matters:++ * At the **warning** threshold this only logs. Some existing nodes+ legitimately return hundreds of KB (a parsed document, a long+ transcript) and breaking them would be a regression.+ * At the **error** threshold it raises. That is not a new failure: a+ payload over Temporal's limit is rejected by the converter anyway.+ What changes is *how* it fails — ``NodeUserError`` is already in+ ``NON_RETRYABLE_ERROR_TYPES``, so instead of three attempts that+ re-run the work (and re-bill whatever produced it) before reporting+ a generic converter error, the run stops immediately with a message+ naming the node and the size.++ This is also why no Temporal-internal error type had to be named: the+ payload never reaches the converter. The installed SDK enforces the+ limit in its Rust core and exposes no Python class to add to a+ non-retryable list, so catching it there was not possible anyway.+ """+ if payload is None:+ return+ try:+ import orjson++ size = len(orjson.dumps(payload, default=str))+ except Exception:+ # Sizing is diagnostics; never fail a result over it.+ return++ if size >= TEMPORAL_PAYLOAD_ERROR_BYTES:+ raise NodeUserError(+ f"{self.type} returned {size // 1024} KB, over the "+ f"{TEMPORAL_PAYLOAD_ERROR_BYTES // (1024 * 1024)} MB the "+ "workflow engine can carry. Large data belongs in the "+ "workspace: write it to a file and return a reference "+ "(see services/media for the audio case)."+ )+ if size >= TEMPORAL_PAYLOAD_WARN_BYTES:+ logger.warning(+ "large node result",+ node_type=self.type,+ size_bytes=size,+ limit_bytes=TEMPORAL_PAYLOAD_ERROR_BYTES,+ )def _wrap_success(self, *, start_time: float, result: Any) -> Dict[str, Any]:"""95%-universal return shape. Subclasses (ToolNode) override."""diff --git a/server/services/status_broadcaster.py b/server/services/status_broadcaster.pyindex d4f82824..05980e8d 100644--- a/server/services/status_broadcaster.py+++ b/server/services/status_broadcaster.py@@ -13,6 +13,49 @@ from opentelemetry import tracefrom core.logging import get_loggerlogger = get_logger(__name__)++# Above this, a node output is replaced by a stub in the retained status+# cache. Generous on purpose: the cache exists so a reconnecting client can+# see what the last run produced, and almost every real output fits well+# under it. It is a leak bound, not a display limit.+_CACHED_OUTPUT_MAX_BYTES = 64 * 1024+++def _elide_for_cache(output: Any) -> Any:+ """Return ``output``, or a small stub when it is too large to retain.++ ``StatusBroadcaster._status`` is never evicted and is replayed in full to+ every newly connecting client. Without this, one large node output is held+ for the process lifetime *and* re-sent on every page load — the same bytes,+ indefinitely.++ Only the cached copy is elided; the live broadcast is untouched, so the+ user who ran the node still sees the whole thing. A client that reconnects+ later sees the stub and can refetch the real output from ``node_outputs``,+ which is the durable store.+ """+ if output is None:+ return output+ try:+ import orjson++ size = len(orjson.dumps(output, default=str))+ except Exception:+ # Unsizeable means unknown, and unknown is not a reason to discard.+ return output++ if size <= _CACHED_OUTPUT_MAX_BYTES:+ return output++ logger.debug("eliding large node output from the status cache", size_bytes=size)+ return {+ "_elided": True,+ "_size_bytes": size,+ "_note": (+ "Output too large to retain in the live status cache. The full "+ "value is in node_outputs."+ ),+ }tracer = trace.get_tracer(__name__)@@ -686,11 +729,20 @@ class StatusBroadcaster:)async def update_node_output(self, node_id: str, output: Any, workflow_id: Optional[str] = None):- """Update a node's output data and broadcast."""+ """Update a node's output data and broadcast.++ The **broadcast carries the full output** — the user who just ran the+ node sees exactly what it produced. What gets *cached* may be elided:+ ``self._status`` is never evicted and is replayed in full to every+ newly connecting client, so a large output otherwise costs memory for+ the process lifetime and is re-sent on every page load, forever.++ Eliding only the cache keeps the live view intact while bounding both.+ """if node_id not in self._status["nodes"]:self._status["nodes"][node_id] = {"status": "idle", "data": {}}- self._status["nodes"][node_id]["output"] = output+ self._status["nodes"][node_id]["output"] = _elide_for_cache(output)if workflow_id:self._status["nodes"][node_id]["workflow_id"] = workflow_iddiff --git a/server/tests/test_payload_guards.py b/server/tests/test_payload_guards.pynew file mode 100644index 00000000..0e51c64d--- /dev/null+++ b/server/tests/test_payload_guards.py@@ -0,0 +1,151 @@+"""Guards on how much data a node result may carry.++A node result is copied many times — three writes to ``node_outputs``, two+broadcasts, the retained status cache, the workflow aggregate, every+downstream activity's *input*, and (for a tool-exposed node) an LLM message.+Temporal caps a blob at 2 MiB for activity results and inputs alike.++Two separate mechanisms, tested here together because they defend the same+thing from different ends: the producer refuses to emit an uncarryable+payload, and the retained cache refuses to hold a large one forever.+"""++from __future__ import annotations++from typing import Any, Dict++import pytest+from pydantic import BaseModel++from services.media.limits import (+ TEMPORAL_PAYLOAD_ERROR_BYTES,+ TEMPORAL_PAYLOAD_WARN_BYTES,+)+from services.plugin import NodeUserError+from services.plugin.base import BaseNode++pytestmark = pytest.mark.unit+++class _Output(BaseModel):+ blob: str = ""++ model_config = {"extra": "allow"}+++class _Stand_in:+ """Enough of a node for the size guard, without registering one.++ Subclassing ``ActionNode`` here would auto-register a fake node type into+ the global ``NODE_METADATA`` via ``__init_subclass__`` and break every+ plugin-contract invariant that walks the registry — which is exactly what+ happened the first time this file was written.+ """++ type = "_test_sized_node"+ Output = _Output++ _serialize_result = BaseNode._serialize_result+ _check_result_size = BaseNode._check_result_size+++def _SizedNode() -> _Stand_in:+ return _Stand_in()+++class TestResultSizeGuard:+ def test_a_normal_result_passes_untouched(self):+ node = _SizedNode()+ payload = node._serialize_result(_Output(blob="hello"))+ assert payload == {"blob": "hello"}++ def test_a_large_but_carryable_result_only_warns(self, caplog):+ """Existing nodes legitimately return hundreds of KB.++ Failing them would be a regression, so between the two thresholds+ this logs and nothing more.+ """+ node = _SizedNode()+ size = (TEMPORAL_PAYLOAD_WARN_BYTES + TEMPORAL_PAYLOAD_ERROR_BYTES) // 2+ payload = node._serialize_result(_Output(blob="x" * size))+ assert payload["blob"]++ def test_an_uncarryable_result_is_refused(self):+ """Not a new failure — Temporal rejects this payload anyway.++ What changes is that it fails once, here, with a message naming the+ node and the size, instead of three times inside the converter with a+ generic error. ``NodeUserError`` is already non-retryable.+ """+ node = _SizedNode()+ with pytest.raises(NodeUserError) as excinfo:+ node._serialize_result(_Output(blob="x" * (TEMPORAL_PAYLOAD_ERROR_BYTES + 1)))++ message = str(excinfo.value)+ assert "_test_sized_node" in message+ assert "workspace" in message.lower()++ def test_the_refusal_is_non_retryable(self):+ """The whole point of raising NodeUserError rather than anything else."""+ from services.temporal._retry_policies import NON_RETRYABLE_ERROR_TYPES++ assert NodeUserError.__name__ in NON_RETRYABLE_ERROR_TYPES++ def test_sizing_failure_never_blocks_a_result(self, monkeypatch):+ """Sizing is diagnostics; an unserializable payload still passes."""+ node = _SizedNode()++ class _Hostile:+ def __repr__(self):+ raise RuntimeError("nope")++ # Not a BaseModel and no declared Output match, so it passes through+ # _check_result_size, which must swallow its own failure.+ assert node._check_result_size({"x": _Hostile()}) is None+++class TestStatusCacheElision:+ """``_status`` is never evicted and is replayed to every new client."""++ def test_a_small_output_is_cached_verbatim(self):+ from services.status_broadcaster import _elide_for_cache++ payload = {"result": "small"}+ assert _elide_for_cache(payload) is payload++ def test_a_large_output_is_replaced_by_a_stub(self):+ from services.status_broadcaster import _elide_for_cache++ elided = _elide_for_cache({"blob": "x" * (256 * 1024)})+ assert elided["_elided"] is True+ assert elided["_size_bytes"] > 256 * 1024+ assert "node_outputs" in elided["_note"]++ async def test_the_broadcast_still_carries_the_full_output(self):+ """Only the retained copy is elided — the live view is untouched."""+ from services.status_broadcaster import StatusBroadcaster++ broadcaster = StatusBroadcaster()+ sent: list[Dict[str, Any]] = []++ async def _capture(message):+ sent.append(message)++ broadcaster.broadcast = _capture # type: ignore[method-assign]++ big = {"blob": "y" * (256 * 1024)}+ await broadcaster.update_node_output("node-1", big, workflow_id="wf-1")++ assert sent[0]["output"] == big+ cached = broadcaster._status["nodes"]["node-1"]["output"]+ assert cached["_elided"] is True++ def test_unsizeable_output_is_kept_rather_than_discarded(self):+ """Unknown size is not a reason to throw data away."""+ from services.status_broadcaster import _elide_for_cache++ class _Unserializable:+ pass++ value = {"x": _Unserializable()}+ assert _elide_for_cache(value) is value