:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): keep buffered rows when a resumable import hands off (#120597)
Daniel RC2ef2157be430
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.pyindex bb80a140d73e..e377b02b2759 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py@@ -58,3 +58,13 @@ class PipelineSafePointHandler:def source_items_are_framework_output(items: Any) -> bool:"""Whether the pipeline iterates the REST framework's own generator, with no source wrapper around it."""return isinstance(items, Resource)+++def framework_cursor_covers_buffered_rows(items: Any) -> bool:+ """Whether the REST framework saved, before the page handed on last, a state that fetches the page after it.++ Then the cursor the pipeline confirmed covers every row it still buffers. A wrapped resource applies+ the state only when the wrapper asks for the next page, a resource built without a resume hook never+ saves one, and the last page of a listing has no next page.+ """+ return isinstance(items, Resource) and items.last_page_advanced_cursordiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex 6be1ab90b5dc..0b28e1bb8a58 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -61,6 +61,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.p)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.safe_point import (PipelineSafePointHandler,+ framework_cursor_covers_buffered_rows,source_items_are_framework_output,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (@@ -943,12 +944,18 @@ class PipelineV3(Generic[ResumableData]):await self._commit_resume_state()if should_check_shutdown(self._schema, self._resource, self._reset_pipeline, source_is_resumable):- if self._handoff_checkpoint is not None and self._shutdown_monitor.is_worker_shutdown():- # The next attempt continues after the last staged batch, so staging the- # buffered rows now keeps it from extracting them again. A source with its- # own cursor does not get this: its cursor can be behind the buffered rows,- # and staging them would load them a second time after the resume.- await stage_remaining_rows()+ if self._shutdown_monitor.is_worker_shutdown() and (+ self._handoff_checkpoint is not None+ or (source_is_resumable and framework_cursor_covers_buffered_rows(items))+ ):+ # Only when the confirmed cursor already covers the buffered rows; otherwise+ # the resume would load them twice. The cursor is not committed when the+ # staging fails, so the next attempt reads the rows again; a chunk staged+ # before the failure still loads. The hand-off happens either way.+ try:+ await stage_remaining_rows()+ except Exception:+ await self._logger.aexception("Failed to stage the rows buffered before the hand-off")self._shutdown_monitor.raise_if_is_worker_shutdown()awaiting_source = Trueexcept SourcePreemptedError:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex 0c855e24726d..8cac34d88cc9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -1862,19 +1862,38 @@ class TestResumeCursorCommit:assert cast(AsyncMock, pipeline._process_batch).await_count == 2assert [json.loads(call.args[1])["id"] for call in redis.set.call_args_list] == expected_committed- @pytest.mark.parametrize("wrapped", [False, True], ids=["rest_resource", "wrapped_rest_resource"])+ @pytest.mark.parametrize(+ "chunk_size,page_count,wrapped,expected_written,expected_committed",+ [+ pytest.param(1, 3, False, [1], [{"page": 1}], id="rest_resource"),+ pytest.param(1, 3, True, [1], [], id="wrapped_rest_resource"),+ # The first page is still buffered at the hand-off. The framework staged its next-page state+ # before the yield, so the page is written and the cursor commits; a wrapper applies the+ # state only when it asks for the next page, so its rows stay unwritten and are read again.+ pytest.param(100, 3, False, [1], [{"page": 1}], id="rest_resource_with_buffered_rows"),+ pytest.param(100, 3, True, [], [], id="wrapped_rest_resource_with_buffered_rows"),+ # The last page has no next-page state, so the cursor still names it: writing it would make+ # the next attempt load it twice wherever the loader appends.+ pytest.param(100, 1, False, [], [], id="rest_resource_last_page_buffered"),+ ],+ )@pytest.mark.asyncioasync def test_a_rest_source_handed_off_after_its_first_batch_resumes_without_losing_rows(- self, wrapped: bool+ self,+ chunk_size: int,+ page_count: int,+ wrapped: bool,+ expected_written: list[int],+ expected_committed: list[dict],) -> None:redis = MagicMock()inputs = cast(SourceInputs, SimpleNamespace(team_id=1, job_id="job-1", logger=MagicMock()))manager = ResumableSourceManager[_PageCursor](inputs, _PageCursor)- pages = {"/items": [[{"id": 1}], [{"id": 2}], [{"id": 3}]]}+ pages = {"/items": [[{"id": row_id}] for row_id in range(1, page_count + 1)]}def rest_items(state: dict | None):resource = rest_api_resource(- paged_config(pages, ["items"]),+ paged_config(pages, ["items"], total_pages=page_count - 1),1,"job-1",None,@@ -1884,7 +1903,8 @@ class TestResumeCursorCommit:return (lambda: (page for page in resource)) if wrapped else (lambda: resource)pipeline = _runnable_pipeline(cast(ResumableSourceManager[_Cursor], manager), rest_items(None))- pipeline._batcher = Batcher(MagicMock(), chunk_size=1, primary_keys=["id"])+ pipeline._batcher = Batcher(MagicMock(), chunk_size=chunk_size, primary_keys=["id"])+ cast(MagicMock, pipeline._shutdown_monitor).is_worker_shutdown.return_value = Truecast(MagicMock, pipeline._shutdown_monitor).raise_if_is_worker_shutdown.side_effect = WorkerShuttingDownError("id", "type", "queue", 1, "workflow", "workflow_type")@@ -1899,12 +1919,9 @@ class TestResumeCursorCommit:committed = [json.loads(call.args[1]) for call in redis.set.call_args_list]resumed = [row["id"] for page in rest_items(committed[-1] if committed else None)() for row in page]- assert written == [1]- if wrapped:- assert set(written + resumed) == {1, 2, 3}- else:- assert committed == [{"page": 1}]- assert written + resumed == [1, 2, 3]+ assert (written, committed) == (expected_written, expected_committed)+ # A wrapped resource re-reads its last written page; the merge on the primary key absorbs it.+ assert sorted(set(written + resumed)) == list(range(1, page_count + 1))@pytest.mark.asyncioasync def test_cursor_not_persisted_for_rows_the_batcher_still_holds(self) -> None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.pyindex 55af97783645..aaff0c82139e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py@@ -446,13 +446,11 @@ def clerk_source(def items() -> Iterable[Any] | AsyncIterable[Any]:if webhook_enabled:return webhook_source_manager.get_items(table_transformer=_webhook_table_transformer)- return polled_items()-- def polled_items() -> Iterator[Any]:if gated_feature is None:- yield from resource- return- yield from _skip_when_feature_disabled(iter(resource), endpoint, gated_feature, logger)+ # The resource itself, so the framework stages a page's resume state before the page is+ # yielded and a hand-off keeps the rows the pipeline still buffers.+ return resource+ return _skip_when_feature_disabled(iter(resource), endpoint, gated_feature, logger)if endpoint_config.partition_key is None:return SourceResponse(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/resource.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/resource.pyindex f33f239118a6..2cb9012d7345 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/resource.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/resource.py@@ -21,6 +21,8 @@ class PageCheckpoints:def __init__(self, resume_hook: ResumeHook) -> None:self._resume_hook = resume_hookself._pending: list[Optional[dict[str, Any]]] = []+ # The last page of a listing comes with no next-page state, so the cursor stays on that page.+ self.last_page_advanced_cursor = Falsedef defer(self, state: Optional[dict[str, Any]]) -> None:self._pending.append(state)@@ -30,6 +32,7 @@ class PageCheckpoints:def apply(self, *, page_is_handed_on: bool) -> None:pending, self._pending = self._pending, []+ self.last_page_advanced_cursor = any(state is not None for state in pending)if not pending:returnif page_is_handed_on:@@ -81,6 +84,16 @@ class Resource:self._data_from = data_fromself._page_checkpoints = page_checkpoints+ @property+ def last_page_advanced_cursor(self) -> bool:+ """Whether the page handed on last came with a saved state that fetches the page after it.++ True only while the pipeline iterates this resource itself: a wrapper applies the state when it+ asks for the next page. False for the last page of a listing, whose state is None, and for a+ resource built without a resume hook.+ """+ return self._page_checkpoints is not None and self._page_checkpoints.last_page_advanced_cursor+@propertydef column_hints(self) -> Optional[dict[str, Any]]:"""Return a mapping of column name to ``data_type`` extracted from thediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.pyindex af02b04e0154..ffdb6b77b345 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py@@ -111,6 +111,16 @@ def test_a_page_reaches_the_pipeline_with_its_resume_state_only_when_nothing_wraassert events == expected+def test_the_resource_says_whether_the_page_handed_on_advanced_the_cursor() -> None:+ resource = _resource(paged_config(ITEMS, ["items"], total_pages=1), "items", lambda state: None)++ with activate_safe_point(lambda: None, covers_framework_checkpoints=True):+ advanced = [resource.last_page_advanced_cursor for _page in resource]++ # The last page comes with no next-page state, so a cursor committed then still names it.+ assert advanced == [True, False]++def test_a_safe_point_inside_the_hook_waits_until_the_page_is_handed_on() -> None:events: list[Any] = []