:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): keep the crisp messages run budget across attempts (#119843)
Daniel RC4ab23f4024bc
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/crisp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/crisp.pyindex a4473b2c18d9..85b35e74cc47 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/crisp.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/crisp.py@@ -52,6 +52,7 @@ ONE_MILLISECOND = timedelta(milliseconds=1)class CrispResumeConfig:paginator_state: dict[str, Any]started_at: str | None = None+ conversations_loaded: int = 0class CrispPathPaginator(PageNumberPaginator):@@ -249,6 +250,8 @@ def windowed_messages_resource(manager: ResumableSourceManager[CrispResumeConfig],save_state: Callable[[dict[str, Any] | None], None],initial_state: dict[str, Any] | None,+ loaded_before: int,+ with_messages: set[str],) -> Resource:fanout = settings.fanoutassert fanout is not None@@ -279,8 +282,6 @@ def windowed_messages_resource(rows.extend(page if isinstance(page, list) else [page])return rows- with_messages: set[str] = set()-def count_conversation(row: dict[str, Any]) -> dict[str, Any]:with_messages.add(str(row.get("session_id")))return row@@ -293,6 +294,33 @@ def windowed_messages_resource(*fanout.child_response_actions,*(child_endpoint.get("response_actions") or []),]++ finished_before = set((initial_state or {}).get("completed") or [])+ child_path = child_endpoint["path"]+ assert child_path is not None+ cutoff_ms = started_at.timestamp() * 1000++ def unread(row: dict[str, Any]) -> bool:+ updated_at = row.get("updated_at")+ if isinstance(updated_at, int | float) and updated_at > cutoff_ms:+ # parent_path_map defers this conversation, so the fan-out reads nothing for it.+ return False+ return child_path.format(session_id=quote(str(row.get("session_id")), safe="")) not in finished_before++ # An earlier attempt can stop inside a window. If this attempt stopped before it finished that+ # window, the watermark of the run would skip the window's unread conversations. So a resumed+ # attempt may stop only after it reaches a window with a conversation that it has to read.+ caught_up = not finished_before++ def done() -> bool:+ return caught_up and loaded_before + len(with_messages) >= CONVERSATIONS_PER_RUN++ def windows() -> Iterator[list[dict[str, Any]]]:+ nonlocal caught_up+ for rows in conversation_windows(list_window, start, started_at, done, manager.safe_point):+ caught_up = caught_up or any(unread(row) for row in rows)+ yield rows+rest_config: RESTAPIConfig = {"client": client_config(config, api_version),"resources": [@@ -301,13 +329,7 @@ def windowed_messages_resource("table_name": parent.name,"write_disposition": "replace","endpoint": endpoint_config(parent, config.website_id),- "data_iterator": lambda: conversation_windows(- list_window,- start,- started_at,- lambda: len(with_messages) >= CONVERSATIONS_PER_RUN,- manager.safe_point,- ),+ "data_iterator": windows,"data_map": parent_path_map(fanout.resolve_param, fanout.include_from_parent, started_at),},{@@ -373,10 +395,17 @@ def crisp_source(else datetime.now(UTC))+ loaded_before = resume.conversations_loaded if resume is not None else 0+ with_messages: set[str] = set()+def save_state(state: dict[str, Any] | None) -> None:if state is not None:manager.save_state(- CrispResumeConfig(paginator_state=state, started_at=started_at.isoformat() if defer_changed else None)+ CrispResumeConfig(+ paginator_state=state,+ started_at=started_at.isoformat() if defer_changed else None,+ conversations_loaded=loaded_before + len(with_messages) if defer_changed else 0,+ ))initial_state = resume.paginator_state if resume is not None else None@@ -390,6 +419,8 @@ def crisp_source(manager,save_state,initial_state,+ loaded_before,+ with_messages,)elif settings.fanout is not None:parent = ENDPOINTS[settings.fanout.parent_name]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp.pyindex 05ec1367c6ec..1731c886f860 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crisp/tests/test_crisp.py@@ -145,6 +145,63 @@ def test_incremental_messages_read_small_windows_oldest_first_and_stop_after_theassert result.response is not None and result.response.sort_mode == "desc"+ ("finished", "loaded", "expected"),+ [+ (["c0", "c1"], 2, ["c2", "c3"]),+ (["c0", "c1", "c2"], 3, ["c3"]),+ ],+)+def test_resumed_incremental_messages_keep_the_run_budget_of_earlier_attempts(+ config: CrispSourceConfig,+ monkeypatch: pytest.MonkeyPatch,+ finished: list[str],+ loaded: int,+ expected: list[str],+) -> None:+ monkeypatch.setattr(crisp_module, "WINDOW_PAGE_SIZE", 2)+ monkeypatch.setattr(crisp_module, "CONVERSATIONS_PER_RUN", 3)+ hour_ms = 3_600_000+ first = int(datetime(2026, 1, 1, tzinfo=UTC).timestamp() * 1000)+ conversations: list[dict[str, Any]] = [+ {"session_id": "c0", "updated_at": first},+ {"session_id": "c1", "updated_at": first + hour_ms},+ {"session_id": "c2", "updated_at": first + 2 * hour_ms},+ {"session_id": "c3", "updated_at": first + 2 * hour_ms},+ {"session_id": "c4", "updated_at": first + 3 * hour_ms},+ ]++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path.endswith("/messages"):+ if request.param("timestamp_before") is not None:+ return page([])+ return page([{"fingerprint": 1, "timestamp": 1000}])+ low = datetime.fromisoformat(request.param("filter_date_start") or "").timestamp() * 1000+ high = datetime.fromisoformat(request.param("filter_date_end") or "").timestamp() * 1000+ matching = [row for row in conversations if low <= row["updated_at"] <= high]+ number = int(request.path.rsplit("/", 1)[1])+ return page(matching[(number - 1) * 2 : number * 2])++ root = f"website/{config.website_id}/conversation"+ result = SourceDriver(CrispSource(), config).run(+ "messages",+ respond,+ incremental_field="conversation_updated_at",+ resume_state=CrispResumeConfig(+ paginator_state={+ "completed": [f"{root}/{session_id}/messages" for session_id in finished],+ "current": None,+ "child_state": None,+ },+ started_at=datetime(2026, 2, 1, tzinfo=UTC).isoformat(),+ conversations_loaded=loaded,+ ),+ )+ assert result.raised is None+ assert [row["session_id"] for row in result.rows] == expected+ assert not any("/c4/" in path for path in result.paths)++def test_incremental_messages_page_through_a_window_that_no_split_can_shrink(config: CrispSourceConfig) -> None:updated_at = int(datetime(2026, 1, 1, tzinfo=UTC).timestamp() * 1000)conversations: list[dict[str, Any]] = [