Skip to content

Self-improving AI Employees for everyone

fix(polling): a poll cycle stays under the payload limit and its deadline (1b.18)

trohitg2f62393ede88
diff --git a/docs-internal/event_framework.md b/docs-internal/event_framework.md
index 21bb5e21..6644ec18 100644
--- a/docs-internal/event_framework.md
+++ b/docs-internal/event_framework.md
@@ -510,6 +510,8 @@ Both polling paths grew `seen: Set[str]` unboundedly:
At Gmail's ~100 msgs/day / 60s poll, this hit ~36K entries / ~1.4MB just for IDs in a year. Fix: rebase `seen = current` at end of every cycle in both paths ([polling.py](../server/services/plugin/polling.py)). Items the provider no longer reports drop out. Bounded by the provider's natural window size.
+Since 1b.18 the activity's `seen_ids` is `current` minus the new ids a cycle defers: it walks new ids oldest first (numeric ids as numbers), stops admitting once its events would pass 512,000 bytes of JSON (`POLL_RESULT_MAX_BYTES`; the first event is always admitted) or once detail fetching has used two thirds of the attempt's deadline, and leaves the rest new for the next cycle; `post_emit` runs only for emitted events. A single event over the 2 MiB payload limit still fails its cycle.
+
Semantic note: if a filtered item disappears then reappears (e.g. user marks an email unread again under `is:unread`), it re-emits. That's the correct semantic for visibility-filtered providers; the old unbounded set suppressed legitimate re-emits.
### 5. Template resolution against trigger output (canary path)
diff --git a/docs-internal/node-logic-flows/microsoft/msMailReceive.md b/docs-internal/node-logic-flows/microsoft/msMailReceive.md
index 4cd5eb2a..9d3ba30c 100644
--- a/docs-internal/node-logic-flows/microsoft/msMailReceive.md
+++ b/docs-internal/node-logic-flows/microsoft/msMailReceive.md
@@ -110,7 +110,10 @@ flowchart TD
"not connected" `PermissionError` annotated with `provider` (see Auth),
which is raised as `NodeUserError` and not retried; a `fetch_detail` failure skips
that id with a warning; `post_emit` failures are swallowed; the result is
- `{"events": [...], "seen_ids": list(current)}`.
+ `{"events": [...], "seen_ids": [...]}`: the events of at most 512,000 bytes
+ (the first always included, fetched within two thirds of the attempt's
+ deadline), and the current ids minus the new ones the cycle deferred, which
+ the next cycle takes.
- **Legacy coroutine** (`_build_poll_coroutine`): setup failure ends the
poller; baseline failure logs a warning and treats everything as new on the
next cycle; per-cycle exceptions are logged and retried next interval.
diff --git a/server/services/plugin/polling.py b/server/services/plugin/polling.py
index 7a566238..145cfd3e 100644
--- a/server/services/plugin/polling.py
+++ b/server/services/plugin/polling.py
@@ -46,7 +46,9 @@ Out of scope (commit-stage)
from __future__ import annotations
import asyncio
-from typing import Any, Callable, ClassVar, Dict, Set, Tuple
+import json
+import time
+from typing import Any, Callable, ClassVar, Dict, Optional, Set, Tuple
from core.logging import get_logger
from services.plugin.base import NodeUserError
@@ -54,6 +56,30 @@ from services.plugin.trigger import TriggerNode
logger = get_logger(__name__)
+#: One poll cycle's events add up to at most this many bytes of JSON (1b.18):
+#: the bound controller_queue.event_batches puts on a queue page, under the
+#: SDK's 524,288-byte payload warning. The first event is always admitted, so a
+#: single event over the payload limit still fails its cycle (RFC-0006 12.3).
+POLL_RESULT_MAX_BYTES = 512_000
+
+
+def _oldest_first(msg_id: Any) -> tuple:
+ """Order ids numeric-as-number (IMAP uids), the rest as text after them."""
+ text = str(msg_id)
+ return (0, int(text)) if text.isdecimal() else (1, text)
+
+
+def _detail_budget_s() -> Optional[float]:
+ """Two thirds of this attempt's deadline for fetching details (1b.18): a
+ burst of new messages (one provider call each) defers its tail to the next
+ cycle instead of timing every attempt out. None outside an activity."""
+ from temporalio import activity
+
+ if not activity.in_activity():
+ return None
+ deadline = activity.info().start_to_close_timeout
+ return deadline.total_seconds() * 2 / 3 if deadline else None
+
class PollingTriggerNode(TriggerNode, abstract=True):
"""Polling trigger with a unified loop body.
@@ -190,8 +216,13 @@ class PollingTriggerNode(TriggerNode, abstract=True):
Activity result (out)::
{
- "events": List[Dict], # NEW event payloads this cycle
- "seen_ids": List[str], # union of prior seen + current
+ "events": List[Dict], # new event payloads, oldest first, at most
+ # POLL_RESULT_MAX_BYTES of JSON (the first
+ # always included) and fetched within two
+ # thirds of the attempt's deadline
+ "seen_ids": List[str], # the provider's current ids minus the new
+ # ids this cycle did not reach (they stay
+ # new for the next cycle)
}
Determinism: every operation runs inside the activity (network
@@ -208,6 +239,7 @@ class PollingTriggerNode(TriggerNode, abstract=True):
@activity.defn(name=activity_name)
async def _poll_cycle_activity(payload: Dict[str, Any]) -> Dict[str, Any]:
+ started = time.monotonic()
node_id = payload.get("node_id", "")
params = payload.get("params", {}) or {}
prior_seen: Set[str] = set(payload.get("seen_ids") or [])
@@ -246,7 +278,14 @@ class PollingTriggerNode(TriggerNode, abstract=True):
new_ids = current - prior_seen
events: list[Dict[str, Any]] = []
- for msg_id in new_ids:
+ attempted: Set[str] = set()
+ size = 0
+ budget = _detail_budget_s()
+ # Oldest first, so a cycle that stops early leaves the newest
+ # ids for the next one, and a retried cycle picks the same ids.
+ for msg_id in sorted(new_ids, key=_oldest_first):
+ if budget is not None and attempted and time.monotonic() - started > budget:
+ break # out of time: the rest stays new for the next cycle
try:
detail = await instance.fetch_detail(service, msg_id, params)
except Exception as exc: # noqa: BLE001
@@ -257,16 +296,23 @@ class PollingTriggerNode(TriggerNode, abstract=True):
msg_id=msg_id,
error=str(exc),
)
+ attempted.add(msg_id) # a failed detail stays seen, as before
continue
# The workflow needs a stable event.id for cross-cycle
# dedup. Fall back to the provider id when fetch_detail
# doesn't supply one.
if "id" not in detail:
detail["id"] = msg_id
+ # The Temporal payload encoding; the first event always fits.
+ item_bytes = len(json.dumps(detail, separators=(",", ":")))
+ if events and size + item_bytes > POLL_RESULT_MAX_BYTES:
+ break # this id and the ones after it stay new for the next cycle
+ attempted.add(msg_id)
events.append(detail)
- # post_emit side effect (e.g. mark-as-read). Failures
- # MUST NOT block the next emit or kill the cycle —
- # mirror the legacy coroutine semantics.
+ size += item_bytes
+ # post_emit side effect (e.g. mark-as-read), only for an
+ # event in the result. Failures MUST NOT block the next
+ # emit or kill the cycle — mirror the legacy coroutine.
try:
await instance.post_emit(service, msg_id, params)
except Exception:
@@ -280,8 +326,18 @@ class PollingTriggerNode(TriggerNode, abstract=True):
# ever got evicted). At Gmail's ~100 msgs/day cadence the
# set hit ~36K entries in a year — ~1.4MB just for IDs,
# serialised through Temporal payload on every cycle. Now
- # bounded by the provider's natural window size.
- return {"events": events, "seen_ids": list(current)}
+ # bounded by the provider's natural window size, minus the
+ # ids this cycle deferred, which stay new for the next one.
+ deferred = new_ids - attempted
+ if deferred:
+ logger.info(
+ "Polling activity deferred ids to the next cycle",
+ node_id=node_id,
+ node_type=cls.type,
+ deferred=len(deferred),
+ result_bytes=size,
+ )
+ return {"events": events, "seen_ids": list(current - deferred)}
return _poll_cycle_activity
diff --git a/server/tests/test_polling_trigger_workflow.py b/server/tests/test_polling_trigger_workflow.py
index 76bd28e3..4ee47d58 100644
--- a/server/tests/test_polling_trigger_workflow.py
+++ b/server/tests/test_polling_trigger_workflow.py
@@ -92,11 +92,14 @@ def _make_fake_polling_class(
type = cls_type
display_name = "Test"
# abstract=True suppresses registration
+ services: List[_FakeService] = [] # one per cycle, for the assertions
async def setup_service(self, params):
if setup_error is not None:
raise setup_error
- return _FakeService()
+ service = _FakeService()
+ type(self).services.append(service)
+ return service
async def fetch_ids(self, service, params):
if fetch_ids_error is not None:
@@ -301,6 +304,91 @@ class TestAsPollActivity:
assert {e["id"] for e in result["events"]} == {"x", "y"}
+class TestPollResultByteCap:
+ """1b.18: one cycle's events stay under the payload limit; what does not
+ fit (in bytes, or in the attempt's time) stays new for the next cycle."""
+
+ @staticmethod
+ def _cycle(cls, seen):
+ inner = TestAsPollActivity._underlying(None, cls.as_poll_activity())
+ return inner({"node_id": "n1", "params": {}, "seen_ids": list(seen), "baseline_only": False})
+
+ @staticmethod
+ def _big(msg_id):
+ return {"id": msg_id, "body": "x" * 200_000}
+
+ @pytest.mark.asyncio
+ async def test_the_cap_defers_the_rest_and_the_next_cycle_takes_it(self):
+ import json
+
+ from services.plugin.polling import POLL_RESULT_MAX_BYTES
+
+ cls = _make_fake_polling_class("test.cap", current_ids={"1", "2", "3", "4", "5"}, detail_payload_fn=self._big)
+ seen: List[str] = []
+ emitted = []
+ for _ in range(4):
+ result = await self._cycle(cls, seen)
+ emitted.append([event["id"] for event in result["events"]])
+ assert len(json.dumps(result["events"], separators=(",", ":"))) <= POLL_RESULT_MAX_BYTES + len(result["events"]) + 1
+ seen = result["seen_ids"]
+ assert emitted == [["1", "2"], ["3", "4"], ["5"], []]
+ # post_emit (mark-as-read) ran only for the emitted events.
+ assert cls.services[0].detail_calls == ["1", "2", "3"]
+ assert cls.services[0].post_emit_calls == ["1", "2"]
+
+ @pytest.mark.asyncio
+ async def test_the_first_item_is_always_admitted(self):
+ def payload(msg_id):
+ return {"id": msg_id, "body": "x" * (600_000 if msg_id == "1" else 10)}
+
+ cls = _make_fake_polling_class("test.first", current_ids={"1", "2"}, detail_payload_fn=payload)
+ first = await self._cycle(cls, [])
+ assert [event["id"] for event in first["events"]] == ["1"]
+ assert set(first["seen_ids"]) == {"1"}
+ second = await self._cycle(cls, first["seen_ids"])
+ assert [event["id"] for event in second["events"]] == ["2"]
+
+ @pytest.mark.asyncio
+ async def test_numeric_ids_are_walked_as_numbers_oldest_first(self):
+ cls = _make_fake_polling_class("test.order", current_ids={"10", "9", "2", "abc"})
+ result = await self._cycle(cls, [])
+ assert cls.services[0].detail_calls == ["2", "9", "10", "abc"]
+ assert [event["id"] for event in result["events"]] == ["2", "9", "10", "abc"]
+
+ @pytest.mark.asyncio
+ async def test_a_failed_detail_stays_attempted(self):
+ def payload(msg_id):
+ if msg_id == "2":
+ raise RuntimeError("detail failed")
+ return {"id": msg_id}
+
+ cls = _make_fake_polling_class("test.failed", current_ids={"1", "2", "3"}, detail_payload_fn=payload)
+ result = await self._cycle(cls, [])
+ assert [event["id"] for event in result["events"]] == ["1", "3"]
+ assert set(result["seen_ids"]) == {"1", "2", "3"} # not retried, as before
+
+ @pytest.mark.asyncio
+ async def test_a_cycle_out_of_time_defers_the_rest(self, monkeypatch):
+ # Two thirds of a 3 s deadline: 2 s. A burst of slow detail calls
+ # (IMAP) must not time every attempt out.
+ from dataclasses import replace
+ from datetime import timedelta
+ from types import SimpleNamespace
+
+ from temporalio.testing import ActivityEnvironment
+
+ from services.plugin import polling
+
+ clock = iter([0.0, 0.5, 3.0])
+ monkeypatch.setattr(polling, "time", SimpleNamespace(monotonic=lambda: next(clock)))
+ cls = _make_fake_polling_class("test.budget", current_ids={"1", "2", "3"})
+ env = ActivityEnvironment()
+ env.info = replace(env.info, start_to_close_timeout=timedelta(seconds=3))
+ result = await env.run(cls.as_poll_activity(), {"node_id": "n1", "params": {}, "seen_ids": [], "baseline_only": False})
+ assert [event["id"] for event in result["events"]] == ["1", "2"]
+ assert set(result["seen_ids"]) == {"1", "2"}
+
+
# ---------------------------------------------------------------------------
# C2d — PollingTriggerWorkflow body
# ---------------------------------------------------------------------------