Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): keep buffered rows when a resumable import hands off (#120597)

Daniel RC2ef2157be430
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py
index bb80a140d73e..e377b02b2759 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/safe_point.py
@@ -58,3 +58,13 @@ class PipelineSafePointHandler:
def source_items_are_framework_output(items: Any) -> bool:
"""Whether the pipeline iterates the REST framework's own generator, with no source wrapper around it."""
return isinstance(items, Resource)
+
+
+def framework_cursor_covers_buffered_rows(items: Any) -> bool:
+ """Whether the REST framework saved, before the page handed on last, a state that fetches the page after it.
+
+ Then the cursor the pipeline confirmed covers every row it still buffers. A wrapped resource applies
+ the state only when the wrapper asks for the next page, a resource built without a resume hook never
+ saves one, and the last page of a listing has no next page.
+ """
+ return isinstance(items, Resource) and items.last_page_advanced_cursor
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index 6be1ab90b5dc..0b28e1bb8a58 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -61,6 +61,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.p
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.safe_point import (
PipelineSafePointHandler,
+ framework_cursor_covers_buffered_rows,
source_items_are_framework_output,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import (
@@ -943,12 +944,18 @@ class PipelineV3(Generic[ResumableData]):
await self._commit_resume_state()
if should_check_shutdown(self._schema, self._resource, self._reset_pipeline, source_is_resumable):
- if self._handoff_checkpoint is not None and self._shutdown_monitor.is_worker_shutdown():
- # The next attempt continues after the last staged batch, so staging the
- # buffered rows now keeps it from extracting them again. A source with its
- # own cursor does not get this: its cursor can be behind the buffered rows,
- # and staging them would load them a second time after the resume.
- await stage_remaining_rows()
+ if self._shutdown_monitor.is_worker_shutdown() and (
+ self._handoff_checkpoint is not None
+ or (source_is_resumable and framework_cursor_covers_buffered_rows(items))
+ ):
+ # Only when the confirmed cursor already covers the buffered rows; otherwise
+ # the resume would load them twice. The cursor is not committed when the
+ # staging fails, so the next attempt reads the rows again; a chunk staged
+ # before the failure still loads. The hand-off happens either way.
+ try:
+ await stage_remaining_rows()
+ except Exception:
+ await self._logger.aexception("Failed to stage the rows buffered before the hand-off")
self._shutdown_monitor.raise_if_is_worker_shutdown()
awaiting_source = True
except SourcePreemptedError:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index 0c855e24726d..8cac34d88cc9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -1862,19 +1862,38 @@ class TestResumeCursorCommit:
assert cast(AsyncMock, pipeline._process_batch).await_count == 2
assert [json.loads(call.args[1])["id"] for call in redis.set.call_args_list] == expected_committed
- @pytest.mark.parametrize("wrapped", [False, True], ids=["rest_resource", "wrapped_rest_resource"])
+ @pytest.mark.parametrize(
+ "chunk_size,page_count,wrapped,expected_written,expected_committed",
+ [
+ pytest.param(1, 3, False, [1], [{"page": 1}], id="rest_resource"),
+ pytest.param(1, 3, True, [1], [], id="wrapped_rest_resource"),
+ # The first page is still buffered at the hand-off. The framework staged its next-page state
+ # before the yield, so the page is written and the cursor commits; a wrapper applies the
+ # state only when it asks for the next page, so its rows stay unwritten and are read again.
+ pytest.param(100, 3, False, [1], [{"page": 1}], id="rest_resource_with_buffered_rows"),
+ pytest.param(100, 3, True, [], [], id="wrapped_rest_resource_with_buffered_rows"),
+ # The last page has no next-page state, so the cursor still names it: writing it would make
+ # the next attempt load it twice wherever the loader appends.
+ pytest.param(100, 1, False, [], [], id="rest_resource_last_page_buffered"),
+ ],
+ )
@pytest.mark.asyncio
async def test_a_rest_source_handed_off_after_its_first_batch_resumes_without_losing_rows(
- self, wrapped: bool
+ self,
+ chunk_size: int,
+ page_count: int,
+ wrapped: bool,
+ expected_written: list[int],
+ expected_committed: list[dict],
) -> None:
redis = MagicMock()
inputs = cast(SourceInputs, SimpleNamespace(team_id=1, job_id="job-1", logger=MagicMock()))
manager = ResumableSourceManager[_PageCursor](inputs, _PageCursor)
- pages = {"/items": [[{"id": 1}], [{"id": 2}], [{"id": 3}]]}
+ pages = {"/items": [[{"id": row_id}] for row_id in range(1, page_count + 1)]}
def rest_items(state: dict | None):
resource = rest_api_resource(
- paged_config(pages, ["items"]),
+ paged_config(pages, ["items"], total_pages=page_count - 1),
1,
"job-1",
None,
@@ -1884,7 +1903,8 @@ class TestResumeCursorCommit:
return (lambda: (page for page in resource)) if wrapped else (lambda: resource)
pipeline = _runnable_pipeline(cast(ResumableSourceManager[_Cursor], manager), rest_items(None))
- pipeline._batcher = Batcher(MagicMock(), chunk_size=1, primary_keys=["id"])
+ pipeline._batcher = Batcher(MagicMock(), chunk_size=chunk_size, primary_keys=["id"])
+ cast(MagicMock, pipeline._shutdown_monitor).is_worker_shutdown.return_value = True
cast(MagicMock, pipeline._shutdown_monitor).raise_if_is_worker_shutdown.side_effect = WorkerShuttingDownError(
"id", "type", "queue", 1, "workflow", "workflow_type"
)
@@ -1899,12 +1919,9 @@ class TestResumeCursorCommit:
committed = [json.loads(call.args[1]) for call in redis.set.call_args_list]
resumed = [row["id"] for page in rest_items(committed[-1] if committed else None)() for row in page]
- assert written == [1]
- if wrapped:
- assert set(written + resumed) == {1, 2, 3}
- else:
- assert committed == [{"page": 1}]
- assert written + resumed == [1, 2, 3]
+ assert (written, committed) == (expected_written, expected_committed)
+ # A wrapped resource re-reads its last written page; the merge on the primary key absorbs it.
+ assert sorted(set(written + resumed)) == list(range(1, page_count + 1))
@pytest.mark.asyncio
async def test_cursor_not_persisted_for_rows_the_batcher_still_holds(self) -> None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
index 55af97783645..aaff0c82139e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
@@ -446,13 +446,11 @@ def clerk_source(
def items() -> Iterable[Any] | AsyncIterable[Any]:
if webhook_enabled:
return webhook_source_manager.get_items(table_transformer=_webhook_table_transformer)
- return polled_items()
-
- def polled_items() -> Iterator[Any]:
if gated_feature is None:
- yield from resource
- return
- yield from _skip_when_feature_disabled(iter(resource), endpoint, gated_feature, logger)
+ # The resource itself, so the framework stages a page's resume state before the page is
+ # yielded and a hand-off keeps the rows the pipeline still buffers.
+ return resource
+ return _skip_when_feature_disabled(iter(resource), endpoint, gated_feature, logger)
if endpoint_config.partition_key is None:
return SourceResponse(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/resource.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/resource.py
index f33f239118a6..2cb9012d7345 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/resource.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/resource.py
@@ -21,6 +21,8 @@ class PageCheckpoints:
def __init__(self, resume_hook: ResumeHook) -> None:
self._resume_hook = resume_hook
self._pending: list[Optional[dict[str, Any]]] = []
+ # The last page of a listing comes with no next-page state, so the cursor stays on that page.
+ self.last_page_advanced_cursor = False
def defer(self, state: Optional[dict[str, Any]]) -> None:
self._pending.append(state)
@@ -30,6 +32,7 @@ class PageCheckpoints:
def apply(self, *, page_is_handed_on: bool) -> None:
pending, self._pending = self._pending, []
+ self.last_page_advanced_cursor = any(state is not None for state in pending)
if not pending:
return
if page_is_handed_on:
@@ -81,6 +84,16 @@ class Resource:
self._data_from = data_from
self._page_checkpoints = page_checkpoints
+ @property
+ def last_page_advanced_cursor(self) -> bool:
+ """Whether the page handed on last came with a saved state that fetches the page after it.
+
+ True only while the pipeline iterates this resource itself: a wrapper applies the state when it
+ asks for the next page. False for the last page of a listing, whose state is None, and for a
+ resource built without a resume hook.
+ """
+ return self._page_checkpoints is not None and self._page_checkpoints.last_page_advanced_cursor
+
@property
def column_hints(self) -> Optional[dict[str, Any]]:
"""Return a mapping of column name to ``data_type`` extracted from the
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py
index af02b04e0154..ffdb6b77b345 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py
@@ -111,6 +111,16 @@ def test_a_page_reaches_the_pipeline_with_its_resume_state_only_when_nothing_wra
assert events == expected
+def test_the_resource_says_whether_the_page_handed_on_advanced_the_cursor() -> None:
+ resource = _resource(paged_config(ITEMS, ["items"], total_pages=1), "items", lambda state: None)
+
+ with activate_safe_point(lambda: None, covers_framework_checkpoints=True):
+ advanced = [resource.last_page_advanced_cursor for _page in resource]
+
+ # The last page comes with no next-page state, so a cursor committed then still names it.
+ assert advanced == [True, False]
+
+
def test_a_safe_point_inside_the_hook_waits_until_the_page_is_handed_on() -> None:
events: list[Any] = []