Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): keep the crisp messages run budget across attempts (#119843)

Daniel RC4ab23f4024bc
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/crisp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/crisp.py
index a4473b2c18d9..85b35e74cc47 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/crisp.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/crisp.py
@@ -52,6 +52,7 @@ ONE_MILLISECOND = timedelta(milliseconds=1)
class CrispResumeConfig:
paginator_state: dict[str, Any]
started_at: str | None = None
+ conversations_loaded: int = 0
class CrispPathPaginator(PageNumberPaginator):
@@ -249,6 +250,8 @@ def windowed_messages_resource(
manager: ResumableSourceManager[CrispResumeConfig],
save_state: Callable[[dict[str, Any] | None], None],
initial_state: dict[str, Any] | None,
+ loaded_before: int,
+ with_messages: set[str],
) -> Resource:
fanout = settings.fanout
assert fanout is not None
@@ -279,8 +282,6 @@ def windowed_messages_resource(
rows.extend(page if isinstance(page, list) else [page])
return rows
- with_messages: set[str] = set()
-
def count_conversation(row: dict[str, Any]) -> dict[str, Any]:
with_messages.add(str(row.get("session_id")))
return row
@@ -293,6 +294,33 @@ def windowed_messages_resource(
*fanout.child_response_actions,
*(child_endpoint.get("response_actions") or []),
]
+
+ finished_before = set((initial_state or {}).get("completed") or [])
+ child_path = child_endpoint["path"]
+ assert child_path is not None
+ cutoff_ms = started_at.timestamp() * 1000
+
+ def unread(row: dict[str, Any]) -> bool:
+ updated_at = row.get("updated_at")
+ if isinstance(updated_at, int | float) and updated_at > cutoff_ms:
+ # parent_path_map defers this conversation, so the fan-out reads nothing for it.
+ return False
+ return child_path.format(session_id=quote(str(row.get("session_id")), safe="")) not in finished_before
+
+ # An earlier attempt can stop inside a window. If this attempt stopped before it finished that
+ # window, the watermark of the run would skip the window's unread conversations. So a resumed
+ # attempt may stop only after it reaches a window with a conversation that it has to read.
+ caught_up = not finished_before
+
+ def done() -> bool:
+ return caught_up and loaded_before + len(with_messages) >= CONVERSATIONS_PER_RUN
+
+ def windows() -> Iterator[list[dict[str, Any]]]:
+ nonlocal caught_up
+ for rows in conversation_windows(list_window, start, started_at, done, manager.safe_point):
+ caught_up = caught_up or any(unread(row) for row in rows)
+ yield rows
+
rest_config: RESTAPIConfig = {
"client": client_config(config, api_version),
"resources": [
@@ -301,13 +329,7 @@ def windowed_messages_resource(
"table_name": parent.name,
"write_disposition": "replace",
"endpoint": endpoint_config(parent, config.website_id),
- "data_iterator": lambda: conversation_windows(
- list_window,
- start,
- started_at,
- lambda: len(with_messages) >= CONVERSATIONS_PER_RUN,
- manager.safe_point,
- ),
+ "data_iterator": windows,
"data_map": parent_path_map(fanout.resolve_param, fanout.include_from_parent, started_at),
},
{
@@ -373,10 +395,17 @@ def crisp_source(
else datetime.now(UTC)
)
+ loaded_before = resume.conversations_loaded if resume is not None else 0
+ with_messages: set[str] = set()
+
def save_state(state: dict[str, Any] | None) -> None:
if state is not None:
manager.save_state(
- CrispResumeConfig(paginator_state=state, started_at=started_at.isoformat() if defer_changed else None)
+ CrispResumeConfig(
+ paginator_state=state,
+ started_at=started_at.isoformat() if defer_changed else None,
+ conversations_loaded=loaded_before + len(with_messages) if defer_changed else 0,
+ )
)
initial_state = resume.paginator_state if resume is not None else None
@@ -390,6 +419,8 @@ def crisp_source(
manager,
save_state,
initial_state,
+ loaded_before,
+ with_messages,
)
elif settings.fanout is not None:
parent = ENDPOINTS[settings.fanout.parent_name]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp.py
index 05ec1367c6ec..1731c886f860 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp.py
@@ -145,6 +145,63 @@ def test_incremental_messages_read_small_windows_oldest_first_and_stop_after_the
assert result.response is not None and result.response.sort_mode == "desc"
+ ("finished", "loaded", "expected"),
+ [
+ (["c0", "c1"], 2, ["c2", "c3"]),
+ (["c0", "c1", "c2"], 3, ["c3"]),
+ ],
+)
+def test_resumed_incremental_messages_keep_the_run_budget_of_earlier_attempts(
+ config: CrispSourceConfig,
+ monkeypatch: pytest.MonkeyPatch,
+ finished: list[str],
+ loaded: int,
+ expected: list[str],
+) -> None:
+ monkeypatch.setattr(crisp_module, "WINDOW_PAGE_SIZE", 2)
+ monkeypatch.setattr(crisp_module, "CONVERSATIONS_PER_RUN", 3)
+ hour_ms = 3_600_000
+ first = int(datetime(2026, 1, 1, tzinfo=UTC).timestamp() * 1000)
+ conversations: list[dict[str, Any]] = [
+ {"session_id": "c0", "updated_at": first},
+ {"session_id": "c1", "updated_at": first + hour_ms},
+ {"session_id": "c2", "updated_at": first + 2 * hour_ms},
+ {"session_id": "c3", "updated_at": first + 2 * hour_ms},
+ {"session_id": "c4", "updated_at": first + 3 * hour_ms},
+ ]
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path.endswith("/messages"):
+ if request.param("timestamp_before") is not None:
+ return page([])
+ return page([{"fingerprint": 1, "timestamp": 1000}])
+ low = datetime.fromisoformat(request.param("filter_date_start") or "").timestamp() * 1000
+ high = datetime.fromisoformat(request.param("filter_date_end") or "").timestamp() * 1000
+ matching = [row for row in conversations if low <= row["updated_at"] <= high]
+ number = int(request.path.rsplit("/", 1)[1])
+ return page(matching[(number - 1) * 2 : number * 2])
+
+ root = f"website/{config.website_id}/conversation"
+ result = SourceDriver(CrispSource(), config).run(
+ "messages",
+ respond,
+ incremental_field="conversation_updated_at",
+ resume_state=CrispResumeConfig(
+ paginator_state={
+ "completed": [f"{root}/{session_id}/messages" for session_id in finished],
+ "current": None,
+ "child_state": None,
+ },
+ started_at=datetime(2026, 2, 1, tzinfo=UTC).isoformat(),
+ conversations_loaded=loaded,
+ ),
+ )
+ assert result.raised is None
+ assert [row["session_id"] for row in result.rows] == expected
+ assert not any("/c4/" in path for path in result.paths)
+
+
def test_incremental_messages_page_through_a_window_that_no_split_can_shrink(config: CrispSourceConfig) -> None:
updated_at = int(datetime(2026, 1, 1, tzinfo=UTC).timestamp() * 1000)
conversations: list[dict[str, Any]] = [