Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(stripe): read an incremental backlog oldest first so each sync completes (#119897)

Daniel RCb81331962b0a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
index 40698369d467..bff96c28ed9a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
@@ -42,7 +42,8 @@ class _ListWithAllLines(Generic[_ParentT, _LineT]):
session to every thread, `requests.Session` is not documented as thread-safe, and the retrying
client keeps per-request state. The page client comes from the same factory, so a 429 on
either leg slows the whole pool. Objects are yielded in list order because the
- `starting_after` resume cursor is the last yielded id.
+ `starting_after` resume cursor is the last yielded id. Under `ending_before` the list reads
+ oldest first, as Stripe's own auto-pagination does.
"""
object_name: ClassVar[str]
@@ -82,15 +83,18 @@ class _ListWithAllLines(Generic[_ParentT, _LineT]):
pool = ThreadPoolExecutor(
max_workers=self._concurrency + 1, thread_name_prefix=f"stripe-{self.object_name.replace(' ', '-')}-lines"
)
+ # `ending_before` lists oldest first, the same way Stripe's own auto-pagination reads it.
+ oldest_first = "ending_before" in self.params and "starting_after" not in self.params
try:
while not page.is_empty:
- next_page = submit_with_context(pool, page.next_page)
+ next_page = submit_with_context(pool, page.previous_page if oldest_first else page.next_page)
+ objects = list(reversed(page.data)) if oldest_first else page.data
line_futures = [
submit_with_context(pool, self._fetch_lines, obj.id) if obj.lines.has_more and obj.id else None
- for obj in page.data
+ for obj in objects
]
- for obj, future in zip(page.data, line_futures):
+ for obj, future in zip(objects, line_futures):
if obj.lines.has_more and future is None:
self.logger.warning(f"Stripe {self.object_name} {obj.id} has no id")
continue
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
index 795bf6b2b5c5..5444befe5868 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
@@ -2,8 +2,10 @@ import os
import re
import time
import inspect
+import itertools
import dataclasses
-from collections.abc import Callable, Iterable, Iterator, Mapping
+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping
+from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast, get_args, get_type_hints
import orjson
@@ -141,6 +143,11 @@ NESTED_SWEEP_CHECKPOINT_PARENTS = 5000
# take long enough that a worker shutdown waits on the sweep instead of handing it off.
NESTED_SWEEP_CHECKPOINT_SECONDS = 60.0
+# An incremental read that is far behind stops after this long, and the run completes. Import
+# workers restart often, and a run that a restart interrupts loses its rows, so a longer read
+# finishes less often. A read that is not behind ends long before this.
+INCREMENTAL_RUN_SECONDS = 15 * 60
+
_JSON_WHITESPACE = frozenset(b" \t\n\r\f\v")
_OPEN_BRACE = ord("{")
_CLOSE_BRACE = ord("}")
@@ -376,7 +383,7 @@ def _call_stripe(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
raise
-def _iter_cleaning_errors(objects: Iterable[Any]) -> Iterator[Any]:
+def _iter_cleaning_errors(objects: Iterable[Any]) -> Generator[Any]:
"""Iterate a Stripe listing and clean errors raised while it pages, as `_call_stripe` does.
A listing wrapper such as `InvoiceListWithAllLines` makes its first request only when it is
@@ -748,19 +755,11 @@ def _warehouse_start_position(
return position if position.matches(table, None) else None
-def _batch_and_yield(
- objects: Any,
- batcher: Batcher,
- incremental_field_name: Optional[str] = None,
- stop_at_or_before: Optional[int] = None,
-):
- """Batch raw Stripe objects into pa.Tables and yield them, stopping early once we reach an object
- we already have (`stop_at_or_before`). Yielding in small batches lets the pipeline write and
- persist the incremental watermark frequently, so progress survives a heartbeat timeout or redeploy."""
+def _batch_and_yield(objects: Any, batcher: Batcher):
+ """Batch raw Stripe objects into pa.Tables and yield them. Yielding in small batches lets the
+ pipeline write and persist the incremental watermark frequently, so progress survives a
+ heartbeat timeout or redeploy."""
for obj in objects:
- if stop_at_or_before is not None and incremental_field_name is not None:
- if obj[incremental_field_name] <= stop_at_or_before:
- break
batcher.batch(_scrub_client_secrets(obj))
while batcher.should_yield():
yield batcher.get_table()
@@ -1222,6 +1221,9 @@ def get_rows(
return
+ # The run budget covers both listings, so the clock starts before the first one.
+ incremental_started_at = time.monotonic()
+
# check for any objects less than the minimum object we already have
if db_incremental_field_earliest_value is not None:
logger.debug(
@@ -1242,21 +1244,76 @@ def get_rows(
if db_incremental_field_last_value is not None:
logger.debug(f"Stripe: iterating latest objects from resource: created[gt] = {db_incremental_field_last_value}")
- stripe_objects = _call_stripe(
- resource.method,
- params={
- **default_params,
- **resource.params,
- f"created[gt]": db_incremental_field_last_value,
- },
+ watermark = _coerce_incremental_cursor(db_incremental_field_last_value)
+ if watermark is None:
+ stripe_objects = _call_stripe(
+ resource.method,
+ params={**default_params, **resource.params, f"created[gt]": db_incremental_field_last_value},
+ )
+ yield from _batch_and_yield(_iter_cleaning_errors(stripe_objects.auto_paging_iter()), batcher)
+ return
+
+ latest_objects = _latest_objects_oldest_first(
+ resource,
+ {**default_params, **resource.params},
+ watermark,
+ incremental_field_name,
+ logger,
+ started_at=incremental_started_at,
)
- yield from _batch_and_yield(
- _iter_cleaning_errors(stripe_objects.auto_paging_iter()),
- batcher,
- incremental_field_name=incremental_field_name,
- stop_at_or_before=_coerce_incremental_cursor(db_incremental_field_last_value),
+ yield from _batch_and_yield(latest_objects, batcher)
+
+
+def _latest_objects_oldest_first(
+ resource: StripeResource,
+ params: dict[str, Any],
+ watermark: int,
+ incremental_field_name: str,
+ logger: FilteringBoundLogger,
+ started_at: float,
+) -> Iterator[Any]:
+ """Yield the objects created after `watermark`, oldest first when they fill more than a page.
+
+ Stripe lists newest first, and the watermark moves only when a run completes, so a newest-first
+ read has no safe place to stop. `ending_before` the newest object at or before the watermark
+ lists the newer objects oldest first. The newest value read is then always a safe watermark, so
+ after `INCREMENTAL_RUN_SECONDS` the read stops between two values and the run completes.
+ """
+
+ def listing(extra_params: dict[str, Any]) -> Generator[Any]:
+ return _iter_cleaning_errors(
+ _call_stripe(resource.method, params={**params, **extra_params}).auto_paging_iter()
)
+ page_size = int(params.get("limit", DEFAULT_LIMIT))
+ newest_first = listing({"created[gt]": watermark})
+ first_page = list(itertools.islice(newest_first, page_size + 1))
+ anchor = next(listing({"created[lte]": watermark, "limit": 1}), None) if len(first_page) > page_size else None
+ if anchor is None:
+ # Almost every sync reads one page. Without an object at or before the watermark, for example
+ # after Stripe dropped its old events, nothing marks where an oldest-first read starts.
+ objects: Iterator[Any] = itertools.chain(first_page, newest_first)
+ else:
+ newest_first.close()
+ objects = listing({"ending_before": anchor["id"]})
+
+ last_value: int | None = None
+ for obj in objects:
+ value: int = obj[incremental_field_name]
+ # Some endpoints filter on `created` and track another field, so the watermark is checked again.
+ if value <= watermark:
+ continue
+ # The read stops only between two values, because the next sync skips the objects at the last value.
+ out_of_time = anchor is not None and time.monotonic() - started_at >= INCREMENTAL_RUN_SECONDS
+ if out_of_time and last_value is not None and value != last_value:
+ logger.warning(
+ f"Stripe: this sync read objects created up to {datetime.fromtimestamp(last_value, tz=UTC).isoformat()}. "
+ "The next sync continues from there."
+ )
+ return
+ last_value = value
+ yield obj
+
# How far an invoice is through its lifecycle. An uncollectible invoice can still be paid or voided.
_INVOICE_STATUS_STAGE = {"draft": 1, "open": 2, "uncollectible": 3, "paid": 4, "void": 4}
@@ -1524,7 +1581,7 @@ def stripe_source(
name=endpoint,
column_hints=column_hints,
webhook_only=webhook_only,
- # Stripe data is returned in descending timestamp order
+ # Stripe lists newest first. A backlog read lists oldest first, which is safe because desc mode stages the max value.
sort_mode="desc",
partition_count=1 if partitioned else None, # this enables partitioning
partition_size=1 if partitioned else None, # this enables partitioning
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
index 5aca520c2683..dd91b7f8390c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
ParentTableRef,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ DriveResult,
RecordedRequest,
ScriptedResponse,
SourceDriver,
@@ -66,6 +67,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con
HISTORY_EVENT_TYPE_COLUMN,
HISTORY_PREVIOUS_ATTRIBUTES_COLUMN,
HISTORY_SNAPSHOT_EVENT_TYPE,
+ INVOICE_ITEM_RESOURCE_NAME,
INVOICE_PAYMENT_RESOURCE_NAME,
INVOICE_RESOURCE_NAME,
PAYMENT_INTENT_RESOURCE_NAME,
@@ -235,6 +237,107 @@ class TestStripeGetRowsIncrementalCursor:
assert len(tables) > 1
+NOW = 1_800_000_000
+
+
+def _stripe_list_api(objects: list[dict[str, Any]], field: str) -> Callable[[RecordedRequest], ScriptedResponse]:
+ # Stripe filters on `created` even for an object that only shows another timestamp, such as an invoice item's `date`.
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ newer_than, up_to = request.param("created[gt]"), request.param("created[lte]")
+ rows = [
+ obj
+ for obj in sorted(objects, key=lambda obj: (-obj[field], obj["id"]))
+ if (newer_than is None or obj[field] > int(newer_than)) and (up_to is None or obj[field] <= int(up_to))
+ ]
+ ids = [obj["id"] for obj in rows]
+ limit = int(request.param("limit") or DEFAULT_LIMIT)
+ starting_after, ending_before = request.param("starting_after"), request.param("ending_before")
+ if ending_before is not None:
+ newer = rows[: ids.index(ending_before)]
+ page, has_more = newer[-limit:], len(newer) > limit
+ else:
+ older = rows[ids.index(starting_after) + 1 :] if starting_after is not None else rows
+ page, has_more = older[:limit], len(older) > limit
+ return ScriptedResponse(json={"object": "list", "url": "/v1/list", "data": page, "has_more": has_more})
+
+ return respond
+
+
+class TestStripeLatestObjectsOldestFirst:
+ def _sync(self, resource: str, objects: list[dict[str, Any]], field: str, watermark: int) -> DriveResult:
+ result = SourceDriver(
+ StripeSource(),
+ StripeSourceConfig(auth_method=StripeAuthMethodConfig(selection="api_key", stripe_secret_key="sk_test")),
+ ).run(
+ resource,
+ _stripe_list_api(objects, field),
+ incremental_field=field,
+ db_incremental_field_last_value=watermark,
+ api_version=STRIPE_API_VERSION_ACACIA,
+ )
+ assert result.raised is None
+ return result
+
+ @pytest.mark.parametrize(
+ "resource, object_type, field, run_seconds, several_runs",
+ [
+ (CHARGE_RESOURCE_NAME, "charge", "created", 15 * 60, False),
+ (CHARGE_RESOURCE_NAME, "charge", "created", 0, True),
+ (INVOICE_ITEM_RESOURCE_NAME, "invoiceitem", "date", 0, True),
+ (INVOICE_RESOURCE_NAME, "invoice", "created", 0, True),
+ ],
+ ids=["within_budget", "budget_spent", "invoice_item_by_date", "invoice_with_lines"],
+ )
+ def test_a_backlog_loads_each_object_once_across_runs(
+ self, resource: str, object_type: str, field: str, run_seconds: int, several_runs: bool
+ ) -> None:
+ watermark = NOW - 40 * 86_400
+ # Older objects are loaded already. Pairs share a timestamp, so a run must not stop between them.
+ timestamps = [watermark - 86_400, watermark] + [NOW - 600 * (i // 2) for i in range(250)]
+ objects: list[dict[str, Any]] = [
+ {
+ "id": f"obj_{i:03}",
+ "object": object_type,
+ field: timestamp,
+ "lines": {"object": "list", "data": [], "has_more": False, "url": "/v1/lines"},
+ }
+ for i, timestamp in enumerate(timestamps)
+ ]
+ loaded: list[str] = []
+ runs = 0
+ with patch.object(stripe_module, "INCREMENTAL_RUN_SECONDS", run_seconds):
+ while rows := [
+ row for table in self._sync(resource, objects, field, watermark).items for row in table.to_pylist()
+ ]:
+ runs += 1
+ loaded.extend(row["id"] for row in rows)
+ # The pipeline stages the newest value a desc source read as the run's watermark.
+ watermark = max(row[field] for row in rows)
+
+ assert sorted(loaded) == sorted(obj["id"] for obj in objects[2:])
+ assert (runs > 1) is several_runs
+
+ def test_a_backlog_with_nothing_at_or_before_the_watermark_loads_in_one_run(self) -> None:
+ charges: list[dict[str, Any]] = [
+ {"id": f"ch_{i:03}", "object": "charge", "created": NOW - 60 * i} for i in range(150)
+ ]
+
+ with patch.object(stripe_module, "INCREMENTAL_RUN_SECONDS", 0):
+ result = self._sync(CHARGE_RESOURCE_NAME, charges, "created", NOW - 86_400)
+
+ assert sorted(row["id"] for table in result.items for row in table.to_pylist()) == sorted(
+ charge["id"] for charge in charges
+ )
+
+ def test_a_read_within_one_page_keeps_one_request(self) -> None:
+ charges = [{"id": f"ch_{day}", "object": "charge", "created": NOW - day * 86_400} for day in (1, 300)]
+
+ result = self._sync(CHARGE_RESOURCE_NAME, charges, "created", NOW - 3 * 365 * 86_400)
+
+ assert len(result.requests) == 1
+ assert [row["id"] for table in result.items for row in table.to_pylist()] == ["ch_1", "ch_300"]
+
+
class TestStripeFlatResourceResumeCursor:
def test_restarts_listing_when_resume_cursor_no_longer_exists(self):
# A cursor saved by a previous run can outlive the object it names (e.g. the