Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(stripe): stop resumed balance-transaction sweeps duplicating rows (#125264)

Daniel Snitkovskiy0199d6ea1aa4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_warehouse_parent.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_warehouse_parent.py
index 397da27aa9e3..83cf9ec758ad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_warehouse_parent.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_warehouse_parent.py
@@ -1,3 +1,4 @@
+import shutil
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
@@ -20,6 +21,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
WarehouseParentTableNotFoundError,
iter_parent_pages_from_warehouse,
iter_parent_pages_with_positions,
+ parent_table_id,
+ pin_parent_table_version,
resolve_parent_table_ref,
)
@@ -596,6 +599,68 @@ def test_a_pinned_version_keeps_its_positions_through_a_compaction(tmp_path: Pat
assert [row["id"] for page in resumed for row in page.rows] == before[6:]
+def _table_with_a_later_commit(tmp_path: Path) -> tuple[str, int, ParentTableRef]:
+ uri = _write_parent_table(tmp_path)
+ checkpointed = deltalake.DeltaTable(uri).version()
+ deltalake.write_deltalake(
+ uri, pa.table({"id": ["1"], "last_seen": ["2026-04-01"], "title": ["z"]}), mode="overwrite"
+ )
+ return uri, checkpointed, ParentTableRef(uri=uri, version=deltalake.DeltaTable(uri).version())
+
+
+def test_pin_parent_table_version_reads_the_checkpointed_snapshot(tmp_path: Path) -> None:
+ uri, checkpointed, newest = _table_with_a_later_commit(tmp_path)
+
+ with patch.object(warehouse_parent, "delta_storage_options", return_value={}):
+ pinned = pin_parent_table_version(newest, checkpointed)
+
+ assert pinned == ParentTableRef(uri=uri, version=checkpointed)
+ assert sorted(
+ row["id"] for page in _patched_reader(uri, version=checkpointed, columns=["id"], page_size=10) for row in page
+ ) == ["1", "2", "3"]
+
+
+def test_pin_parent_table_version_keeps_the_current_pin(tmp_path: Path) -> None:
+ _uri, _checkpointed, newest = _table_with_a_later_commit(tmp_path)
+
+ assert pin_parent_table_version(newest, newest.version) is newest
+
+
+ "rollback,sweep_budget,version_offset",
+ [
+ (timedelta(0), timedelta(0), 0),
+ (warehouse_parent.MAX_SNAPSHOT_ROLLBACK, warehouse_parent.MAX_SNAPSHOT_ROLLBACK, 0),
+ (warehouse_parent.MAX_SNAPSHOT_ROLLBACK, warehouse_parent.RESUMED_SWEEP_BUDGET, 5),
+ ],
+ ids=["outside_the_vacuum_window", "no_retention_left_for_the_sweep", "newer_than_the_table"],
+)
+def test_pin_parent_table_version_refuses_a_version_it_cannot_vouch_for(
+ rollback: timedelta, sweep_budget: timedelta, version_offset: int, tmp_path: Path
+) -> None:
+ _uri, checkpointed, newest = _table_with_a_later_commit(tmp_path)
+
+ with (
+ patch.object(warehouse_parent, "delta_storage_options", return_value={}),
+ patch.object(warehouse_parent, "MAX_SNAPSHOT_ROLLBACK", rollback),
+ patch.object(warehouse_parent, "RESUMED_SWEEP_BUDGET", sweep_budget),
+ ):
+ assert pin_parent_table_version(newest, checkpointed + version_offset) is None
+
+
+def test_parent_table_id_survives_writes_but_not_a_rebuild(tmp_path: Path) -> None:
+ uri, checkpointed, newest = _table_with_a_later_commit(tmp_path)
+
+ with patch.object(warehouse_parent, "delta_storage_options", return_value={}):
+ original = parent_table_id(ParentTableRef(uri=uri, version=checkpointed))
+ assert original is not None
+ assert parent_table_id(newest) == original
+
+ shutil.rmtree(uri)
+ rebuilt = ParentTableRef(uri=_write_parent_table(tmp_path), version=0)
+ assert parent_table_id(rebuilt) != original
+
+
def test_scan_position_matches_only_its_own_table_version_and_filter() -> None:
# The guard a caller uses before trusting stored state: offsets count emitted rows, so a
# different table, version or moved filter floor makes the same offset a different row.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/warehouse_parent.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/warehouse_parent.py
index 6994dc7839e7..05f2fd5027ca 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/warehouse_parent.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/warehouse_parent.py
@@ -34,6 +34,11 @@ MAX_PARENT_PAGE_SIZE = 5000
# version can only be tombstoned after that version, so any pin inside the window is safe.
MAX_SNAPSHOT_ROLLBACK = VACUUM_RETENTION - dt.timedelta(hours=4)
+# How long a resumed fan-out may still read the snapshot it re-pins. The rest of its sweep runs
+# after the re-pin, so the snapshot needs this much retention left on top of the margin above.
+# The longest clean fan-out sweep in production takes about 6 hours.
+RESUMED_SWEEP_BUDGET = dt.timedelta(hours=8)
+
logger = structlog.get_logger(__name__)
@@ -329,6 +334,61 @@ def try_resolve_parent_table(
return None
+def pin_parent_table_version(table: ParentTableRef, version: int) -> ParentTableRef | None:
+ """`table` pinned back to an earlier `version`, or None when that version may no longer be readable.
+
+ A resumable fan-out checkpoints positions against the version its first attempt pinned, and a
+ parent that commits often (a webhook-synced one commits every drain) has moved on by the time a
+ later attempt resolves it. Reading that attempt from the checkpointed version keeps the position
+ meaningful instead of forcing the sweep back to its first row.
+
+ The caller has to check `parent_table_id` first: a rebuilt table reuses version numbers. A file
+ live at `version` can only be tombstoned by a later commit, and vacuum deletes it only once the
+ tombstone ages past its retention. So the version stays readable while the first commit
+ after it is inside `MAX_SNAPSHOT_ROLLBACK`, the bound `_snapshot_pin_as_of` puts on a rollback.
+ The rest of the sweep reads it after this check, so the version also needs
+ `RESUMED_SWEEP_BUDGET` of that window left, or vacuum can delete its files mid-sweep.
+
+ Returns None rather than raising on any failure, so the caller's start-over branch covers an
+ unreadable log too.
+ """
+ if version == table.version:
+ return table
+ if version > table.version:
+ # Versions restart at 0 when a table is rebuilt, so a newer number than the current pin
+ # names a version of a table that no longer exists.
+ return None
+ try:
+ delta_table = deltalake.DeltaTable(table.uri, storage_options=delta_storage_options())
+ newer_commits = delta_table.history(limit=delta_table.version() - version)
+ first_after = next((entry for entry in newer_commits if entry.get("version") == version + 1), None)
+ if first_after is None:
+ return None
+ tombstoned_from = dt.datetime.fromtimestamp(first_after["timestamp"] / 1000, tz=dt.UTC)
+ except Exception:
+ logger.warning("data_imports.fanout_parent_repin_failed", uri=table.uri, version=version, exc_info=True)
+ return None
+ if dt.datetime.now(dt.UTC) - tombstoned_from > MAX_SNAPSHOT_ROLLBACK - RESUMED_SWEEP_BUDGET:
+ return None
+ return ParentTableRef(uri=table.uri, version=version)
+
+
+def parent_table_id(table: ParentTableRef) -> str | None:
+ """The Delta table id of `table`, or None when it cannot be read.
+
+ Writes, schema changes and compaction keep the id, while a reset purges the table and rebuilds
+ it at the same URI under a new one. A caller that stores a version for a later attempt compares
+ this id too, because a rebuilt table reuses version numbers from 0.
+ """
+ try:
+ delta_table = deltalake.DeltaTable(table.uri, version=table.version, storage_options=delta_storage_options())
+ # deltalake annotates the id as int, but the Delta log stores it as a uuid string.
+ return str(delta_table.metadata().id)
+ except Exception:
+ logger.warning("data_imports.fanout_parent_table_id_failed", uri=table.uri, exc_info=True)
+ return None
+
+
def parent_snapshot_covers_through(team_id: int, source_id: str, parent_name: str) -> dt.datetime | None:
"""How far the parent's data is guaranteed complete, or None if it has never completed a sync.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
index 8a3892ba147d..da29d89b8b7f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
@@ -569,6 +569,9 @@ class StripeResumeConfig:
warehouse_row_offset: int | None = None
warehouse_table_uri: str | None = None
warehouse_version: int | None = None
+ # The parent's Delta table id. A reset purges and rebuilds the table at the same URI with
+ # versions from 0, so the URI and version alone can name a different table.
+ warehouse_table_id: str | None = None
def _scrub_client_secrets(obj: Any) -> Any:
@@ -611,8 +614,10 @@ class _WarehouseParentRows:
schema_name: str,
parent_name: str,
start_position: Optional["ScanPosition"],
+ table_id: Optional[str] = None,
) -> None:
self._table = table
+ self.table_id = table_id
self._columns = columns
self._parent_name = parent_name
self._page_size = page_size
@@ -664,7 +669,9 @@ def _flush_staging_state(
yield table
-def _resume_state(position: Optional["ScanPosition"], parent_id: Optional[str]) -> StripeResumeConfig:
+def _resume_state(
+ position: Optional["ScanPosition"], parent_id: Optional[str], parent_pages: Optional[_WarehouseParentRows] = None
+) -> StripeResumeConfig:
"""Resume state in whichever coordinate system this sweep is walking."""
if position is not None:
return StripeResumeConfig(
@@ -672,6 +679,7 @@ def _resume_state(position: Optional["ScanPosition"], parent_id: Optional[str])
warehouse_row_offset=position.row_offset,
warehouse_table_uri=position.table_uri,
warehouse_version=position.version,
+ warehouse_table_id=parent_pages.table_id if parent_pages else None,
)
return StripeResumeConfig(starting_after=parent_id)
@@ -699,9 +707,9 @@ def _nested_checkpoint(
# Without an id there is no cursor into the parent's own pages, so the sweep can only move
# past it. The one resource in this shape returns a single row per parent, which is
# therefore finished by the time that row is written.
- return _resume_state(parent_pages.position_after_current if parent_pages else None, chunk_parent)
+ return _resume_state(parent_pages.position_after_current if parent_pages else None, chunk_parent, parent_pages)
return dataclasses.replace(
- _resume_state(parent_pages.position_at_current if parent_pages else None, last_finished_parent),
+ _resume_state(parent_pages.position_at_current if parent_pages else None, last_finished_parent, parent_pages),
nested_parent_id=chunk_parent,
nested_starting_after=py_table.column("id")[-1].as_py(),
)
@@ -734,7 +742,8 @@ def _warehouse_start_position(
Discards state saved against another table, version or coordinate system — including
`starting_after` cursors from a run that took the API path, which name a place in Stripe's
ordering and mean nothing here. A discarded position restarts the sweep rather than
- resuming somewhere arbitrary.
+ resuming somewhere arbitrary. `_reconcile_warehouse_resume` drops such state before the
+ pipeline reads it, so that restart overwrites the table instead of appending to it.
"""
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.warehouse_parent import ( # noqa: PLC0415
ScanPosition,
@@ -978,6 +987,7 @@ def get_rows(
should_use_incremental_field: bool = False,
warehouse_parent: Optional["ParentTableRef"] = None,
include_zero_balance_customers: bool = False,
+ warehouse_table_id: Optional[str] = None,
):
new_client = _client_factory(api_key, account_id, api_version)
client = new_client(None)
@@ -1043,6 +1053,7 @@ def get_rows(
schema_name=endpoint,
parent_name=converted.schema,
start_position=warehouse_start,
+ table_id=warehouse_table_id,
)
parent_rows = parent_pages
else:
@@ -1097,7 +1108,7 @@ def get_rows(
or time.monotonic() - last_checkpoint_at >= NESTED_SWEEP_CHECKPOINT_SECONDS
)
if checkpoint_due and last_finished_parent is not None:
- position = _resume_state(last_finished_position, last_finished_parent)
+ position = _resume_state(last_finished_position, last_finished_parent, parent_pages)
if batcher.should_yield(include_incomplete_chunk=True):
yield from _flush_staging_state(batcher, resumable_source_manager, _fixed_state(position))
else:
@@ -1215,7 +1226,7 @@ def get_rows(
last_cur = py_table.column(resource.nested_parent_param)[-1].as_py()
else:
last_cur = py_table.column("id")[-1].as_py()
- return _resume_state(parent_pages.position_after_current if parent_pages else None, last_cur)
+ return _resume_state(parent_pages.position_after_current if parent_pages else None, last_cur, parent_pages)
yield from _flush_staging_state(batcher, resumable_source_manager, trailing_state)
@@ -1496,6 +1507,76 @@ def _resolve_warehouse_parent(
)
+def _reconcile_warehouse_resume(
+ endpoint: str,
+ warehouse_parent: Optional["ParentTableRef"],
+ resumable_source_manager: ResumableSourceManager[StripeResumeConfig],
+ logger: FilteringBoundLogger,
+) -> tuple[Optional["ParentTableRef"], Optional[str]]:
+ """The parent table this attempt sweeps and its Delta table id, after making the stored resume
+ state agree with them.
+
+ The pipeline appends every batch of an attempt that finds resume state to what the earlier
+ attempts wrote, and only an attempt that finds none overwrites the table. So a sweep that
+ restarts from its first parent while the state is still there writes every row from before the
+ interruption a second time, once per restart.
+
+ The usual cause is the parent moving on: a webhook-synced Customer table commits every drain,
+ so the next attempt resolves a newer version than its checkpoint names. That attempt is pinned
+ back to the checkpointed version while it is still readable, so the position applies again.
+ Any other state the sweep would discard — another or a rebuilt table, a version past the vacuum
+ window, a cursor from the other parent path — is dropped here, at build time, before the
+ pipeline checks for it, so the restart runs as a fresh attempt that overwrites the table.
+ """
+ if endpoint not in WAREHOUSE_PARENT_FANOUT:
+ return warehouse_parent, None
+
+ table_id: Optional[str] = None
+ if warehouse_parent is not None:
+ # noqa reason: keeps deltalake/pyarrow off the import path of every Stripe import.
+ from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.warehouse_parent import ( # noqa: PLC0415
+ parent_table_id,
+ pin_parent_table_version,
+ )
+
+ table_id = parent_table_id(warehouse_parent)
+
+ if not resumable_source_manager.can_resume():
+ return warehouse_parent, table_id
+ resume_config = resumable_source_manager.load_state()
+ if resume_config is None:
+ return warehouse_parent, table_id
+
+ if warehouse_parent is None:
+ if resume_config.warehouse_fragment_index is None:
+ return None, None
+ elif (
+ table_id is not None
+ and resume_config.warehouse_table_id == table_id
+ and resume_config.warehouse_table_uri == warehouse_parent.uri
+ and resume_config.warehouse_version is not None
+ ):
+ pinned = pin_parent_table_version(warehouse_parent, resume_config.warehouse_version)
+ if pinned is not None and _warehouse_start_position(resume_config, pinned) is not None:
+ logger.info(
+ "Stripe: resuming the sweep on the parent version it checkpointed against",
+ endpoint=endpoint,
+ pinned_version=pinned.version,
+ latest_version=warehouse_parent.version,
+ fragment_index=resume_config.warehouse_fragment_index,
+ row_offset=resume_config.warehouse_row_offset,
+ )
+ return pinned, table_id
+
+ logger.info(
+ "Stripe: dropping resume state the sweep cannot continue from, so this attempt rewrites the table",
+ endpoint=endpoint,
+ parent_source="warehouse" if warehouse_parent is not None else "api",
+ )
+ resumable_source_manager.clear_state()
+ return warehouse_parent, table_id
+
+
def stripe_source(
api_key: str,
account_id: Optional[str],
@@ -1529,6 +1610,11 @@ def stripe_source(
webhook_only = endpoint in WEBHOOK_ONLY_ENDPOINTS
webhook_enabled = async_to_sync(webhook_source_manager.webhook_enabled)(webhook_only=webhook_only)
warehouse_parent = _resolve_warehouse_parent(endpoint, team_id, source_id, use_warehouse_parent)
+ warehouse_table_id: Optional[str] = None
+ if not webhook_enabled:
+ warehouse_parent, warehouse_table_id = _reconcile_warehouse_resume(
+ endpoint, warehouse_parent, resumable_source_manager, logger
+ )
def items():
if webhook_enabled:
@@ -1569,6 +1655,7 @@ def stripe_source(
api_version=api_version,
warehouse_parent=warehouse_parent,
include_zero_balance_customers=include_zero_balance_customers,
+ warehouse_table_id=warehouse_table_id,
)
# A few Stripe objects carry no timestamp at all, so there is nothing to partition on — the
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
index 1c02ed6d952e..2c97b8b6b1d0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
@@ -1,7 +1,9 @@
+import shutil
import functools
import threading
import contextvars
from collections.abc import Callable
+from datetime import timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Optional, cast
@@ -2732,3 +2734,221 @@ class TestStripeNestedSweepResume:
assert "starting_after" not in nested_params[0]
assert [row["id"] for row in rows] == ["txn_1"]
+
+
+class _PipelineAroundTheSweep:
+ """Stands in for the pipeline and the resumable source manager around the Stripe source.
+
+ It holds the two behaviours the duplicate rows came from: a cursor commits only after the
+ table it covers is written, and an attempt that finds resume state appends to what earlier
+ attempts wrote, while an attempt that finds none overwrites the table.
+ """
+
+ def __init__(self) -> None:
+ self.table: list[dict] = []
+ self.committed: Optional[stripe_module.StripeResumeConfig] = None
+ self._staged: Optional[stripe_module.StripeResumeConfig] = None
+
+ def can_resume(self) -> bool:
+ return self.committed is not None
+
+ def load_state(self) -> Optional[stripe_module.StripeResumeConfig]:
+ return self.committed
+
+ def save_state(self, state: stripe_module.StripeResumeConfig) -> None:
+ self._staged = state
+
+ def clear_state(self) -> None:
+ self.committed = None
+ self._staged = None
+
+ def safe_point(self) -> None:
+ pass
+
+ def run_attempt(self, resolved: ParentTableRef, nested_method, interrupt_after_tables: Optional[int] = None):
+ resource = StripeNestedResource(
+ method=nested_method,
+ nested_parent_param="customer",
+ parent_id="id",
+ parent=StripeResource(method=MagicMock(side_effect=AssertionError("the listing must not be paged"))),
+ parent_name=CUSTOMER_RESOURCE_NAME,
+ )
+ webhooks = MagicMock(spec=WebhookSourceManager)
+ webhooks.webhook_enabled = mock.AsyncMock(return_value=False)
+ with (
+ patch.object(stripe_module, "StripeClient"),
+ patch.object(stripe_module, "STRIPE_CHUNK_SIZE", 1),
+ patch.object(
+ stripe_module, "_build_resources", return_value={CUSTOMER_BALANCE_TRANSACTION_RESOURCE_NAME: resource}
+ ),
+ patch.object(stripe_module, "_resolve_warehouse_parent", return_value=resolved),
+ ):
+ response = stripe_module.stripe_source(
+ api_key="sk_test_123",
+ account_id=None,
+ endpoint=CUSTOMER_BALANCE_TRANSACTION_RESOURCE_NAME,
+ db_incremental_field_last_value=None,
+ db_incremental_field_earliest_value=None,
+ logger=MagicMock(),
+ resumable_source_manager=cast(Any, self),
+ webhook_source_manager=webhooks,
+ api_version=STRIPE_API_VERSION_ACACIA,
+ team_id=1,
+ source_id="source-1",
+ use_warehouse_parent=True,
+ )
+ # The pipeline asks only after the source is built, which is the source's last chance
+ # to drop resume state it will not honour.
+ if not self.can_resume():
+ self.table = []
+ items = response.items()
+ for written, table in enumerate(items, start=1):
+ self.table.extend(table.to_pylist())
+ if self._staged is not None:
+ self.committed, self._staged = self._staged, None
+ if interrupt_after_tables is not None and written >= interrupt_after_tables:
+ items.close()
+ return
+
+
+def _paging_nested_method():
+ """A nested list method that honours `starting_after`, as Stripe does, and records each call.
+
+ A resumed sweep re-enters the parent it was paginating with a cursor into that parent's own
+ pages, so a fake that ignored the cursor would duplicate a row the real API would not return.
+ """
+ calls: list[tuple[str, Optional[str]]] = []
+
+ def _nested(customer, params):
+ cursor = params.get("starting_after")
+ calls.append((customer, cursor))
+ return _list_object([] if cursor == f"cbt_{customer}" else [{"id": f"cbt_{customer}"}])
+
+ return _nested, calls
+
+
+class TestStripeWarehouseResumeAcrossParentVersions:
+ """A resumed `CustomerBalanceTransaction` run whose parent table committed since the last attempt.
+
+ The Customer schema is webhook-synced, so its Delta table gains a version every drain, and
+ each attempt of a long sweep pins whichever version is newest when it starts.
+ """
+
+ _CUSTOMERS = [{"id": "cus_1", "balance": -1}, {"id": "cus_2", "balance": -1}, {"id": "cus_3", "balance": -1}]
+
+ def _interrupted_run(self, tmp_path) -> tuple[_PipelineAroundTheSweep, str, ParentTableRef]:
+ uri = _write_customer_parent_table(tmp_path, self._CUSTOMERS)
+ pinned = ParentTableRef(uri=uri, version=deltalake.DeltaTable(uri).version())
+ pipeline = _PipelineAroundTheSweep()
+ nested_method, _ = _paging_nested_method()
+ pipeline.run_attempt(pinned, nested_method, interrupt_after_tables=1)
+ assert [row["id"] for row in pipeline.table] == ["cbt_cus_1"]
+ deltalake.write_deltalake(
+ uri,
+ pa.table({"id": [c["id"] for c in self._CUSTOMERS], "balance": [-1, -2, -1]}),
+ mode="overwrite",
+ )
+ return pipeline, uri, ParentTableRef(uri=uri, version=deltalake.DeltaTable(uri).version())
+
+ def test_the_resumed_attempt_continues_on_the_version_it_checkpointed_against(self, tmp_path):
+ pipeline, _uri, newest = self._interrupted_run(tmp_path)
+ nested_method, calls = _paging_nested_method()
+
+ pipeline.run_attempt(newest, nested_method)
+
+ assert [row["id"] for row in pipeline.table] == ["cbt_cus_1", "cbt_cus_2", "cbt_cus_3"]
+ assert calls == [("cus_1", "cbt_cus_1"), ("cus_2", None), ("cus_3", None)]
+
+ def test_a_checkpoint_too_old_to_read_restarts_as_a_fresh_attempt(self, tmp_path):
+ pipeline, _uri, newest = self._interrupted_run(tmp_path)
+ nested_method, calls = _paging_nested_method()
+
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.warehouse_parent.MAX_SNAPSHOT_ROLLBACK",
+ timedelta(0),
+ ):
+ pipeline.run_attempt(newest, nested_method)
+
+ assert [row["id"] for row in pipeline.table] == ["cbt_cus_1", "cbt_cus_2", "cbt_cus_3"]
+ assert calls == [("cus_1", None), ("cus_2", None), ("cus_3", None)]
+
+ def test_a_rebuilt_parent_table_restarts_as_a_fresh_attempt(self, tmp_path):
+ pipeline, uri, _newest = self._interrupted_run(tmp_path)
+ shutil.rmtree(uri)
+ _write_customer_parent_table(tmp_path, self._CUSTOMERS)
+ deltalake.write_deltalake(uri, pa.table({"id": ["cus_4"], "balance": [-1]}), mode="append")
+ nested_method, calls = _paging_nested_method()
+
+ pipeline.run_attempt(ParentTableRef(uri=uri, version=deltalake.DeltaTable(uri).version()), nested_method)
+
+ assert sorted(row["id"] for row in pipeline.table) == ["cbt_cus_1", "cbt_cus_2", "cbt_cus_3", "cbt_cus_4"]
+ assert all(cursor is None for _customer, cursor in calls)
+
+ @pytest.mark.parametrize(
+ "parent_path,state",
+ [
+ ("warehouse", stripe_module.StripeResumeConfig(starting_after="cus_1")),
+ (
+ "warehouse",
+ stripe_module.StripeResumeConfig(
+ warehouse_fragment_index=0,
+ warehouse_row_offset=1,
+ warehouse_table_uri="s3://elsewhere",
+ warehouse_version=0,
+ ),
+ ),
+ (
+ "api",
+ stripe_module.StripeResumeConfig(
+ warehouse_fragment_index=0,
+ warehouse_row_offset=1,
+ warehouse_table_uri="s3://elsewhere",
+ warehouse_version=0,
+ ),
+ ),
+ ],
+ ids=["api_cursor_on_the_warehouse_path", "another_table", "warehouse_position_on_the_api_path"],
+ )
+ def test_resume_state_the_sweep_cannot_honour_is_dropped_before_the_pipeline_reads_it(
+ self, parent_path, state, tmp_path
+ ):
+ uri = _write_customer_parent_table(tmp_path, self._CUSTOMERS)
+ resolved = ParentTableRef(uri=uri, version=deltalake.DeltaTable(uri).version())
+ manager = MagicMock()
+ manager.can_resume.return_value = True
+ manager.load_state.return_value = state
+
+ reconciled, _table_id = stripe_module._reconcile_warehouse_resume(
+ CUSTOMER_BALANCE_TRANSACTION_RESOURCE_NAME,
+ resolved if parent_path == "warehouse" else None,
+ manager,
+ MagicMock(),
+ )
+
+ manager.clear_state.assert_called_once()
+ assert reconciled == (resolved if parent_path == "warehouse" else None)
+
+ @pytest.mark.parametrize(
+ "endpoint,state",
+ [
+ (CUSTOMER_BALANCE_TRANSACTION_RESOURCE_NAME, stripe_module.StripeResumeConfig(starting_after="cus_1")),
+ (
+ CUSTOMER_PAYMENT_METHOD_RESOURCE_NAME,
+ stripe_module.StripeResumeConfig(
+ warehouse_fragment_index=0,
+ warehouse_row_offset=1,
+ warehouse_table_uri="s3://x",
+ warehouse_version=0,
+ ),
+ ),
+ ],
+ ids=["api_cursor_on_the_api_path", "unconverted_endpoint"],
+ )
+ def test_resume_state_the_sweep_honours_is_kept(self, endpoint, state):
+ manager = MagicMock()
+ manager.can_resume.return_value = True
+ manager.load_state.return_value = state
+
+ stripe_module._reconcile_warehouse_resume(endpoint, None, manager, MagicMock())
+
+ manager.clear_state.assert_not_called()