:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(stripe): read an incremental backlog oldest first so each sync completes (#119897)
Daniel RCb81331962b0a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.pyindex 40698369d467..bff96c28ed9a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py@@ -42,7 +42,8 @@ class _ListWithAllLines(Generic[_ParentT, _LineT]):session to every thread, `requests.Session` is not documented as thread-safe, and the retryingclient keeps per-request state. The page client comes from the same factory, so a 429 oneither leg slows the whole pool. Objects are yielded in list order because the- `starting_after` resume cursor is the last yielded id.+ `starting_after` resume cursor is the last yielded id. Under `ending_before` the list reads+ oldest first, as Stripe's own auto-pagination does."""object_name: ClassVar[str]@@ -82,15 +83,18 @@ class _ListWithAllLines(Generic[_ParentT, _LineT]):pool = ThreadPoolExecutor(max_workers=self._concurrency + 1, thread_name_prefix=f"stripe-{self.object_name.replace(' ', '-')}-lines")+ # `ending_before` lists oldest first, the same way Stripe's own auto-pagination reads it.+ oldest_first = "ending_before" in self.params and "starting_after" not in self.paramstry:while not page.is_empty:- next_page = submit_with_context(pool, page.next_page)+ next_page = submit_with_context(pool, page.previous_page if oldest_first else page.next_page)+ objects = list(reversed(page.data)) if oldest_first else page.dataline_futures = [submit_with_context(pool, self._fetch_lines, obj.id) if obj.lines.has_more and obj.id else None- for obj in page.data+ for obj in objects]- for obj, future in zip(page.data, line_futures):+ for obj, future in zip(objects, line_futures):if obj.lines.has_more and future is None:self.logger.warning(f"Stripe {self.object_name} {obj.id} has no id")continuediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.pyindex 795bf6b2b5c5..5444befe5868 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py@@ -2,8 +2,10 @@ import osimport reimport timeimport inspect+import itertoolsimport dataclasses-from collections.abc import Callable, Iterable, Iterator, Mapping+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping+from datetime import UTC, datetimefrom typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast, get_args, get_type_hintsimport orjson@@ -141,6 +143,11 @@ NESTED_SWEEP_CHECKPOINT_PARENTS = 5000# take long enough that a worker shutdown waits on the sweep instead of handing it off.NESTED_SWEEP_CHECKPOINT_SECONDS = 60.0+# An incremental read that is far behind stops after this long, and the run completes. Import+# workers restart often, and a run that a restart interrupts loses its rows, so a longer read+# finishes less often. A read that is not behind ends long before this.+INCREMENTAL_RUN_SECONDS = 15 * 60+_JSON_WHITESPACE = frozenset(b" \t\n\r\f\v")_OPEN_BRACE = ord("{")_CLOSE_BRACE = ord("}")@@ -376,7 +383,7 @@ def _call_stripe(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:raise-def _iter_cleaning_errors(objects: Iterable[Any]) -> Iterator[Any]:+def _iter_cleaning_errors(objects: Iterable[Any]) -> Generator[Any]:"""Iterate a Stripe listing and clean errors raised while it pages, as `_call_stripe` does.A listing wrapper such as `InvoiceListWithAllLines` makes its first request only when it is@@ -748,19 +755,11 @@ def _warehouse_start_position(return position if position.matches(table, None) else None-def _batch_and_yield(- objects: Any,- batcher: Batcher,- incremental_field_name: Optional[str] = None,- stop_at_or_before: Optional[int] = None,-):- """Batch raw Stripe objects into pa.Tables and yield them, stopping early once we reach an object- we already have (`stop_at_or_before`). Yielding in small batches lets the pipeline write and- persist the incremental watermark frequently, so progress survives a heartbeat timeout or redeploy."""+def _batch_and_yield(objects: Any, batcher: Batcher):+ """Batch raw Stripe objects into pa.Tables and yield them. Yielding in small batches lets the+ pipeline write and persist the incremental watermark frequently, so progress survives a+ heartbeat timeout or redeploy."""for obj in objects:- if stop_at_or_before is not None and incremental_field_name is not None:- if obj[incremental_field_name] <= stop_at_or_before:- breakbatcher.batch(_scrub_client_secrets(obj))while batcher.should_yield():yield batcher.get_table()@@ -1222,6 +1221,9 @@ def get_rows(return+ # The run budget covers both listings, so the clock starts before the first one.+ incremental_started_at = time.monotonic()+# check for any objects less than the minimum object we already haveif db_incremental_field_earliest_value is not None:logger.debug(@@ -1242,21 +1244,76 @@ def get_rows(if db_incremental_field_last_value is not None:logger.debug(f"Stripe: iterating latest objects from resource: created[gt] = {db_incremental_field_last_value}")- stripe_objects = _call_stripe(- resource.method,- params={- **default_params,- **resource.params,- f"created[gt]": db_incremental_field_last_value,- },+ watermark = _coerce_incremental_cursor(db_incremental_field_last_value)+ if watermark is None:+ stripe_objects = _call_stripe(+ resource.method,+ params={**default_params, **resource.params, f"created[gt]": db_incremental_field_last_value},+ )+ yield from _batch_and_yield(_iter_cleaning_errors(stripe_objects.auto_paging_iter()), batcher)+ return++ latest_objects = _latest_objects_oldest_first(+ resource,+ {**default_params, **resource.params},+ watermark,+ incremental_field_name,+ logger,+ started_at=incremental_started_at,)- yield from _batch_and_yield(- _iter_cleaning_errors(stripe_objects.auto_paging_iter()),- batcher,- incremental_field_name=incremental_field_name,- stop_at_or_before=_coerce_incremental_cursor(db_incremental_field_last_value),+ yield from _batch_and_yield(latest_objects, batcher)+++def _latest_objects_oldest_first(+ resource: StripeResource,+ params: dict[str, Any],+ watermark: int,+ incremental_field_name: str,+ logger: FilteringBoundLogger,+ started_at: float,+) -> Iterator[Any]:+ """Yield the objects created after `watermark`, oldest first when they fill more than a page.++ Stripe lists newest first, and the watermark moves only when a run completes, so a newest-first+ read has no safe place to stop. `ending_before` the newest object at or before the watermark+ lists the newer objects oldest first. The newest value read is then always a safe watermark, so+ after `INCREMENTAL_RUN_SECONDS` the read stops between two values and the run completes.+ """++ def listing(extra_params: dict[str, Any]) -> Generator[Any]:+ return _iter_cleaning_errors(+ _call_stripe(resource.method, params={**params, **extra_params}).auto_paging_iter())+ page_size = int(params.get("limit", DEFAULT_LIMIT))+ newest_first = listing({"created[gt]": watermark})+ first_page = list(itertools.islice(newest_first, page_size + 1))+ anchor = next(listing({"created[lte]": watermark, "limit": 1}), None) if len(first_page) > page_size else None+ if anchor is None:+ # Almost every sync reads one page. Without an object at or before the watermark, for example+ # after Stripe dropped its old events, nothing marks where an oldest-first read starts.+ objects: Iterator[Any] = itertools.chain(first_page, newest_first)+ else:+ newest_first.close()+ objects = listing({"ending_before": anchor["id"]})++ last_value: int | None = None+ for obj in objects:+ value: int = obj[incremental_field_name]+ # Some endpoints filter on `created` and track another field, so the watermark is checked again.+ if value <= watermark:+ continue+ # The read stops only between two values, because the next sync skips the objects at the last value.+ out_of_time = anchor is not None and time.monotonic() - started_at >= INCREMENTAL_RUN_SECONDS+ if out_of_time and last_value is not None and value != last_value:+ logger.warning(+ f"Stripe: this sync read objects created up to {datetime.fromtimestamp(last_value, tz=UTC).isoformat()}. "+ "The next sync continues from there."+ )+ return+ last_value = value+ yield obj+# How far an invoice is through its lifecycle. An uncollectible invoice can still be paid or voided._INVOICE_STATUS_STAGE = {"draft": 1, "open": 2, "uncollectible": 3, "paid": 4, "void": 4}@@ -1524,7 +1581,7 @@ def stripe_source(name=endpoint,column_hints=column_hints,webhook_only=webhook_only,- # Stripe data is returned in descending timestamp order+ # Stripe lists newest first. A backlog read lists oldest first, which is safe because desc mode stages the max value.sort_mode="desc",partition_count=1 if partitioned else None, # this enables partitioningpartition_size=1 if partitioned else None, # this enables partitioningdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.pyindex 5aca520c2683..dd91b7f8390c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resParentTableRef,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ DriveResult,RecordedRequest,ScriptedResponse,SourceDriver,@@ -66,6 +67,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.conHISTORY_EVENT_TYPE_COLUMN,HISTORY_PREVIOUS_ATTRIBUTES_COLUMN,HISTORY_SNAPSHOT_EVENT_TYPE,+ INVOICE_ITEM_RESOURCE_NAME,INVOICE_PAYMENT_RESOURCE_NAME,INVOICE_RESOURCE_NAME,PAYMENT_INTENT_RESOURCE_NAME,@@ -235,6 +237,107 @@ class TestStripeGetRowsIncrementalCursor:assert len(tables) > 1+NOW = 1_800_000_000+++def _stripe_list_api(objects: list[dict[str, Any]], field: str) -> Callable[[RecordedRequest], ScriptedResponse]:+ # Stripe filters on `created` even for an object that only shows another timestamp, such as an invoice item's `date`.+ def respond(request: RecordedRequest) -> ScriptedResponse:+ newer_than, up_to = request.param("created[gt]"), request.param("created[lte]")+ rows = [+ obj+ for obj in sorted(objects, key=lambda obj: (-obj[field], obj["id"]))+ if (newer_than is None or obj[field] > int(newer_than)) and (up_to is None or obj[field] <= int(up_to))+ ]+ ids = [obj["id"] for obj in rows]+ limit = int(request.param("limit") or DEFAULT_LIMIT)+ starting_after, ending_before = request.param("starting_after"), request.param("ending_before")+ if ending_before is not None:+ newer = rows[: ids.index(ending_before)]+ page, has_more = newer[-limit:], len(newer) > limit+ else:+ older = rows[ids.index(starting_after) + 1 :] if starting_after is not None else rows+ page, has_more = older[:limit], len(older) > limit+ return ScriptedResponse(json={"object": "list", "url": "/v1/list", "data": page, "has_more": has_more})++ return respond+++class TestStripeLatestObjectsOldestFirst:+ def _sync(self, resource: str, objects: list[dict[str, Any]], field: str, watermark: int) -> DriveResult:+ result = SourceDriver(+ StripeSource(),+ StripeSourceConfig(auth_method=StripeAuthMethodConfig(selection="api_key", stripe_secret_key="sk_test")),+ ).run(+ resource,+ _stripe_list_api(objects, field),+ incremental_field=field,+ db_incremental_field_last_value=watermark,+ api_version=STRIPE_API_VERSION_ACACIA,+ )+ assert result.raised is None+ return result++ @pytest.mark.parametrize(+ "resource, object_type, field, run_seconds, several_runs",+ [+ (CHARGE_RESOURCE_NAME, "charge", "created", 15 * 60, False),+ (CHARGE_RESOURCE_NAME, "charge", "created", 0, True),+ (INVOICE_ITEM_RESOURCE_NAME, "invoiceitem", "date", 0, True),+ (INVOICE_RESOURCE_NAME, "invoice", "created", 0, True),+ ],+ ids=["within_budget", "budget_spent", "invoice_item_by_date", "invoice_with_lines"],+ )+ def test_a_backlog_loads_each_object_once_across_runs(+ self, resource: str, object_type: str, field: str, run_seconds: int, several_runs: bool+ ) -> None:+ watermark = NOW - 40 * 86_400+ # Older objects are loaded already. Pairs share a timestamp, so a run must not stop between them.+ timestamps = [watermark - 86_400, watermark] + [NOW - 600 * (i // 2) for i in range(250)]+ objects: list[dict[str, Any]] = [+ {+ "id": f"obj_{i:03}",+ "object": object_type,+ field: timestamp,+ "lines": {"object": "list", "data": [], "has_more": False, "url": "/v1/lines"},+ }+ for i, timestamp in enumerate(timestamps)+ ]+ loaded: list[str] = []+ runs = 0+ with patch.object(stripe_module, "INCREMENTAL_RUN_SECONDS", run_seconds):+ while rows := [+ row for table in self._sync(resource, objects, field, watermark).items for row in table.to_pylist()+ ]:+ runs += 1+ loaded.extend(row["id"] for row in rows)+ # The pipeline stages the newest value a desc source read as the run's watermark.+ watermark = max(row[field] for row in rows)++ assert sorted(loaded) == sorted(obj["id"] for obj in objects[2:])+ assert (runs > 1) is several_runs++ def test_a_backlog_with_nothing_at_or_before_the_watermark_loads_in_one_run(self) -> None:+ charges: list[dict[str, Any]] = [+ {"id": f"ch_{i:03}", "object": "charge", "created": NOW - 60 * i} for i in range(150)+ ]++ with patch.object(stripe_module, "INCREMENTAL_RUN_SECONDS", 0):+ result = self._sync(CHARGE_RESOURCE_NAME, charges, "created", NOW - 86_400)++ assert sorted(row["id"] for table in result.items for row in table.to_pylist()) == sorted(+ charge["id"] for charge in charges+ )++ def test_a_read_within_one_page_keeps_one_request(self) -> None:+ charges = [{"id": f"ch_{day}", "object": "charge", "created": NOW - day * 86_400} for day in (1, 300)]++ result = self._sync(CHARGE_RESOURCE_NAME, charges, "created", NOW - 3 * 365 * 86_400)++ assert len(result.requests) == 1+ assert [row["id"] for table in result.items for row in table.to_pylist()] == ["ch_1", "ch_300"]++class TestStripeFlatResourceResumeCursor:def test_restarts_listing_when_resume_cursor_no_longer_exists(self):# A cursor saved by a previous run can outlive the object it names (e.g. the