Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(stripe): re-fetch expanded objects for webhook rows (#118548)

Daniel RCea24a2be8f2d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
index 61a1ce9767ff..40698369d467 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
@@ -7,6 +7,8 @@ import stripe as stripe_lib
from stripe import CreditNote, CreditNoteLineItem, Invoice, InvoiceLineItem, ListObject, StripeClient
from structlog.types import FilteringBoundLogger
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.request_pacer import (
RequestPacer,
submit_with_context,
@@ -27,6 +29,8 @@ _LineT = TypeVar("_LineT", InvoiceLineItem, CreditNoteLineItem)
# Receives the Retry-After seconds from a 429, or None when Stripe sent none.
RateLimitCallback = Callable[[Optional[float]], None]
ClientFactory = Callable[[RateLimitCallback], StripeClient]
+# Gets the pool's rate-limit callback and its pacer turn, which the client takes before every request.
+PacedClientFactory = Callable[[RateLimitCallback, Callable[[], None]], StripeClient]
class _ListWithAllLines(Generic[_ParentT, _LineT]):
@@ -154,3 +158,76 @@ class CreditNoteListWithAllLines(_ListWithAllLines[CreditNote, CreditNoteLineIte
def _attach_lines(self, obj: CreditNote, lines: list[CreditNoteLineItem]) -> None:
obj.lines.data = lines
+
+
+@frozen
+class WebhookRefetch:
+ """How to complete one object type whose webhook payload can leave out data the list sync includes."""
+
+ is_incomplete: Callable[[dict[str, Any]], bool]
+ # Gets the event payload, so a step can keep the event's fields and add only what is missing.
+ complete: Callable[[StripeClient, dict[str, Any]], dict[str, Any]]
+
+
+class WebhookObjectRefetcher:
+ """Completes incomplete webhook objects with reads from the API.
+
+ Stripe never expands an event payload, so an event object can be missing data the list sync
+ fetches: an invoice carries only its first 10 lines, a tiered price has no tiers. Upserting that
+ object would overwrite the complete row. Only incomplete objects cost API calls. The reads use a
+ pool with per-thread clients, as the invoice line fan-out does, and objects keep their order. The
+ client takes a pacer turn before every request, so line pages and retries count against the rate.
+ """
+
+ def __init__(
+ self,
+ refetch: WebhookRefetch,
+ logger: FilteringBoundLogger,
+ client_factory_provider: Callable[[], PacedClientFactory],
+ concurrency: int = LINE_FETCH_CONCURRENCY,
+ requests_per_second: float = LINE_REQUESTS_PER_SECOND,
+ ) -> None:
+ self._refetch = refetch
+ self._logger = logger
+ self._client_factory_provider = client_factory_provider
+ self._concurrency = concurrency
+ self._pacer = RequestPacer(requests_per_second)
+
+ def complete(self, objects: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ if not any(self._refetch.is_incomplete(obj) for obj in objects):
+ return objects
+
+ # Resolved once per batch, so a long webhook drain picks up a refreshed OAuth token.
+ client_factory = self._client_factory_provider()
+ thread_clients = threading.local()
+ pool = ThreadPoolExecutor(max_workers=self._concurrency, thread_name_prefix="stripe-webhook-refetch")
+ try:
+ futures = [
+ submit_with_context(pool, self._complete_one, obj, client_factory, thread_clients)
+ if self._refetch.is_incomplete(obj)
+ else None
+ for obj in objects
+ ]
+ completed = [obj if future is None else future.result() for obj, future in zip(objects, futures)]
+ finally:
+ pool.shutdown(wait=False, cancel_futures=True)
+
+ self._logger.debug(f"Stripe: completed {sum(f is not None for f in futures)} incomplete webhook objects")
+ return completed
+
+ def _complete_one(
+ self, obj: dict[str, Any], client_factory: PacedClientFactory, thread_clients: threading.local
+ ) -> dict[str, Any]:
+ client: Optional[StripeClient] = getattr(thread_clients, "client", None)
+ if client is None:
+ client = client_factory(self._pacer.throttled, self._pacer.wait_turn)
+ thread_clients.client = client
+ try:
+ return self._refetch.complete(client, obj)
+ except stripe_lib.InvalidRequestError as e:
+ # A rejection of this one object fails the same way on every retry, and a raise would
+ # fail the whole batch and replay it on every sync. Auth, rate-limit and server errors
+ # are not InvalidRequestError, so they still fail the batch.
+ log = self._logger.debug if getattr(e, "code", None) == "resource_missing" else self._logger.warning
+ log(f"Stripe: kept the webhook payload of {obj.get('object')} {obj['id']}: {e.user_message}")
+ return obj
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/source.py
index 142eebbb47e3..f940b9a80a9b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/source.py
@@ -1,5 +1,7 @@
from typing import TYPE_CHECKING, Optional, cast
+from django.db import close_old_connections, connection
+
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
@@ -483,6 +485,15 @@ If automatic creation failed with a permissions error, the fix depends on how yo
raise ValueError("Stripe access token not found")
return integration.access_token
+ def _get_api_key_closing_connection(self, config: StripeSourceConfig, team_id: int) -> str:
+ # Runs in a long-lived executor thread outside Django's request cycle, so close the
+ # connection an OAuth lookup opens instead of leaving it idle on that thread.
+ close_old_connections()
+ try:
+ return self._get_api_key(config, team_id)
+ finally:
+ connection.close()
+
def get_schemas(
self,
config: StripeSourceConfig,
@@ -720,6 +731,7 @@ If automatic creation failed with a permissions error, the fix depends on how yo
# conversion that declares a parent without threading these silently keeps polling
# the parent API while the fan-out telemetry reports otherwise.
use_warehouse_parent=inputs.fanout_warehouse_reuse,
+ api_key_provider=lambda: self._get_api_key_closing_connection(config, inputs.team_id),
include_zero_balance_customers=bool(
config.balance_history_all_customers and config.balance_history_all_customers.enabled
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
index f8a0e0204d4c..795bf6b2b5c5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
@@ -31,6 +31,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.bas
WebhookDeletionResult,
WebhookSyncResult,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.batching import DEFAULT_BATCH_ROW_LIMIT
from products.warehouse_sources.backend.temporal.data_imports.sources.common.fanout_telemetry import (
log_fanout_parent_rows_consumed,
)
@@ -93,7 +94,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.cus
ClientFactory,
CreditNoteListWithAllLines,
InvoiceListWithAllLines,
+ PacedClientFactory,
RateLimitCallback,
+ WebhookObjectRefetcher,
+ WebhookRefetch,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.settings import (
APPEND_ONLY_INCREMENTAL_FIELDS,
@@ -262,9 +266,19 @@ class _RateLimitRetryingRequestsClient(RequestsClient):
# scope the non-list-body retry to reads, never a single-object write response.
_last_request_method: str = ""
- def __init__(self, *args: Any, on_rate_limited: Optional[RateLimitCallback] = None, **kwargs: Any) -> None:
+ def __init__(
+ self,
+ *args: Any,
+ on_rate_limited: Optional[RateLimitCallback] = None,
+ before_request: Optional[Callable[[], None]] = None,
+ expects_lists: bool = True,
+ **kwargs: Any,
+ ) -> None:
super().__init__(*args, **kwargs)
self._on_rate_limited = on_rate_limited
+ self._before_request = before_request
+ # A client that also retrieves single objects cannot treat a non-list GET body as corrupt.
+ self._expects_lists = expects_lists
def request( # type: ignore[override] # mirrors RequestsClient.request, which already narrows HTTPClient's (str→bytes)
self,
@@ -274,6 +288,8 @@ class _RateLimitRetryingRequestsClient(RequestsClient):
post_data: Any = None,
) -> tuple[bytes, int, Mapping[str, str]]:
self._last_request_method = (method or "").lower()
+ if self._before_request is not None:
+ self._before_request()
return super().request(method, url, headers, post_data)
def _should_retry(
@@ -310,16 +326,25 @@ class _RateLimitRetryingRequestsClient(RequestsClient):
return True
# A complete but non-list body identifies a corrupt list read only for a GET — a write
# (webhook create/update/delete) legitimately returns a single object, so gate on method.
- return self._last_request_method == "get" and _is_non_list_stripe_response(body)
+ return self._expects_lists and self._last_request_method == "get" and _is_non_list_stripe_response(body)
-def _tracked_stripe_http_client(on_rate_limited: Optional[RateLimitCallback] = None) -> RequestsClient:
+def _tracked_stripe_http_client(
+ on_rate_limited: Optional[RateLimitCallback] = None,
+ before_request: Optional[Callable[[], None]] = None,
+ expects_lists: bool = True,
+) -> RequestsClient:
"""Wrap a tracked `requests.Session` in Stripe's `RequestsClient` so every
Stripe SDK call participates in our HTTP logging, metrics, and sample capture.
Uses a subclass that additionally retries 429 rate limits and truncated list responses via the
SDK's built-in backoff, and reports each 429 to `on_rate_limited`."""
- return _RateLimitRetryingRequestsClient(session=make_tracked_session(), on_rate_limited=on_rate_limited)
+ return _RateLimitRetryingRequestsClient(
+ session=make_tracked_session(),
+ on_rate_limited=on_rate_limited,
+ before_request=before_request,
+ expects_lists=expects_lists,
+ )
def _clean_stripe_error_message(msg: str) -> str:
@@ -912,6 +937,36 @@ def _build_resources(
}
+def _client_factory(
+ api_key: str, account_id: Optional[str], api_version: str
+) -> Callable[[Optional[RateLimitCallback]], StripeClient]:
+ def new_client(on_rate_limited: Optional[RateLimitCallback]) -> StripeClient:
+ return _stripe_client(api_key, account_id, api_version, _tracked_stripe_http_client(on_rate_limited))
+
+ return new_client
+
+
+def _refetch_client_factory(api_key: str, account_id: Optional[str], api_version: str) -> PacedClientFactory:
+ def new_client(on_rate_limited: RateLimitCallback, before_request: Callable[[], None]) -> StripeClient:
+ http_client = _tracked_stripe_http_client(on_rate_limited, before_request=before_request, expects_lists=False)
+ return _stripe_client(api_key, account_id, api_version, http_client)
+
+ return new_client
+
+
+def _stripe_client(
+ api_key: str, account_id: Optional[str], api_version: str, http_client: RequestsClient
+) -> StripeClient:
+ return StripeClient(
+ api_key,
+ stripe_account=account_id,
+ stripe_version=api_version,
+ max_network_retries=2,
+ base_addresses=_stripe_base_addresses(),
+ http_client=http_client,
+ )
+
+
def get_rows(
api_key: str,
endpoint: str,
@@ -925,17 +980,8 @@ def get_rows(
warehouse_parent: Optional["ParentTableRef"] = None,
include_zero_balance_customers: bool = False,
):
- def new_client(on_rate_limited: Optional[RateLimitCallback] = None) -> StripeClient:
- return StripeClient(
- api_key,
- stripe_account=account_id,
- stripe_version=api_version,
- max_network_retries=2,
- base_addresses=_stripe_base_addresses(),
- http_client=_tracked_stripe_http_client(on_rate_limited=on_rate_limited),
- )
-
- client = new_client()
+ new_client = _client_factory(api_key, account_id, api_version)
+ client = new_client(None)
default_params = {"limit": DEFAULT_LIMIT}
resources = _build_resources(
client,
@@ -1222,7 +1268,7 @@ def _webhook_tie_break(obj: dict) -> int:
return _INVOICE_STATUS_STAGE.get(obj.get("status") or "", 0)
-def _webhook_table_transformer(table: pa.Table) -> pa.Table:
+def _latest_webhook_objects(table: pa.Table) -> list[dict[str, Any]]:
data_col = table.column("data").to_pylist()
created_col = table.column("created").to_pylist()
@@ -1250,8 +1296,70 @@ def _webhook_table_transformer(table: pa.Table) -> pa.Table:
if existing is None or rank >= existing[0]:
best_by_id[obj_id] = (rank, obj)
- rows = [_scrub_client_secrets(obj) for _, obj in best_by_id.values()]
- return table_from_py_list(rows)
+ return [obj for _, obj in best_by_id.values()]
+
+
+def _webhook_table_transformer(table: pa.Table) -> pa.Table:
+ return table_from_py_list([_scrub_client_secrets(obj) for obj in _latest_webhook_objects(table)])
+
+
+def _refetching_webhook_table_transformer(refetcher: WebhookObjectRefetcher) -> Callable[[pa.Table], pa.Table]:
+ def transform(table: pa.Table) -> pa.Table:
+ objects = refetcher.complete(_latest_webhook_objects(table))
+ return table_from_py_list([_scrub_client_secrets(obj) for obj in objects])
+
+ return transform
+
+
+def _lines_truncated(obj: dict[str, Any]) -> bool:
+ lines = obj.get("lines")
+ return isinstance(lines, Mapping) and bool(lines.get("has_more"))
+
+
+def _tiers_missing(obj: dict[str, Any]) -> bool:
+ return obj.get("billing_scheme") == "tiered" and not obj.get("tiers")
+
+
+def _discounts_unexpanded(obj: dict[str, Any]) -> bool:
+ items = (obj.get("items") or {}).get("data") or []
+ discounts = [*(obj.get("discounts") or []), *(d for item in items for d in (item.get("discounts") or []))]
+ return any(isinstance(discount, str) for discount in discounts)
+
+
+def _with_all_lines(obj: dict[str, Any], lines: ListObject[Any]) -> dict[str, Any]:
+ return {**obj, "lines": {**obj["lines"], "data": list(lines.auto_paging_iter()), "has_more": False, "url": None}}
+
+
+# Each entry adds back what the event payload leaves out. Invoices and credit notes keep the event's
+# fields and only fetch their lines, which is one request per object instead of two.
+WEBHOOK_REFETCH: dict[str, WebhookRefetch] = {
+ INVOICE_RESOURCE_NAME: WebhookRefetch(
+ is_incomplete=_lines_truncated,
+ complete=lambda client, invoice: _with_all_lines(
+ invoice, client.invoices.line_items.list(invoice["id"], params={"limit": DEFAULT_LIMIT})
+ ),
+ ),
+ CREDIT_NOTE_RESOURCE_NAME: WebhookRefetch(
+ is_incomplete=_lines_truncated,
+ complete=lambda client, credit_note: _with_all_lines(
+ credit_note, client.credit_notes.line_items.list(credit_note["id"], params={"limit": DEFAULT_LIMIT})
+ ),
+ ),
+ PRICE_RESOURCE_NAME: WebhookRefetch(
+ is_incomplete=_tiers_missing,
+ complete=lambda client, price: client.prices.retrieve(price["id"], params={"expand": ["tiers"]}),
+ ),
+ PLAN_RESOURCE_NAME: WebhookRefetch(
+ is_incomplete=_tiers_missing,
+ complete=lambda client, plan: client.plans.retrieve(plan["id"], params={"expand": ["tiers"]}),
+ ),
+ SUBSCRIPTION_RESOURCE_NAME: WebhookRefetch(
+ is_incomplete=_discounts_unexpanded,
+ complete=lambda client, subscription: client.subscriptions.retrieve(
+ subscription["id"], params={"expand": ["discounts", "items.data.discounts"]}
+ ),
+ ),
+}
def _webhook_history_table_transformer(table: pa.Table) -> pa.Table:
@@ -1345,6 +1453,7 @@ def stripe_source(
team_id: Optional[int] = None,
source_id: Optional[str] = None,
use_warehouse_parent: bool = False,
+ api_key_provider: Optional[Callable[[], str]] = None,
include_zero_balance_customers: bool = False,
):
# Only the endpoints with a PostHog-managed canonical schema have column hints; the rest let the
@@ -1368,12 +1477,28 @@ def stripe_source(
if webhook_enabled:
# History tables keep every event as its own row; everything else collapses each
# batch to the latest state per object id and upserts it.
- table_transformer = (
- _webhook_history_table_transformer
- if endpoint == CUSTOMER_PAYMENT_METHOD_HISTORY_RESOURCE_NAME
- else _webhook_table_transformer
+ refetch = WEBHOOK_REFETCH.get(endpoint)
+ table_transformer: Callable[[pa.Table], pa.Table]
+ if endpoint == CUSTOMER_PAYMENT_METHOD_HISTORY_RESOURCE_NAME:
+ table_transformer = _webhook_history_table_transformer
+ elif refetch is not None:
+ table_transformer = _refetching_webhook_table_transformer(
+ WebhookObjectRefetcher(
+ refetch,
+ logger,
+ client_factory_provider=lambda: _refetch_client_factory(
+ api_key_provider() if api_key_provider is not None else api_key, account_id, api_version
+ ),
+ )
+ )
+ else:
+ table_transformer = _webhook_table_transformer
+ # Objects grow once their lines or tiers are filled in, after the batch limits were
+ # checked, so a batch that completes objects holds no more rows than a list sync chunk.
+ batch_row_limit = STRIPE_CHUNK_SIZE if refetch is not None else DEFAULT_BATCH_ROW_LIMIT
+ return webhook_source_manager.get_items(
+ table_transformer=table_transformer, batch_row_limit=batch_row_limit
)
- return webhook_source_manager.get_items(table_transformer=table_transformer)
return get_rows(
api_key=api_key,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
index 4a833fe43d0d..f552221ee7bd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
@@ -1,6 +1,7 @@
import functools
import threading
import contextvars
+from collections.abc import Callable
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Optional, cast
@@ -27,9 +28,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.fan
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.warehouse_parent import (
ParentTableRef,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import SourceDriver
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.fake_network import RecordedRequest
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.responders import ScriptedResponse
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
+ ScriptedResponse,
+ SourceDriver,
+ scripted_network,
+ source_inputs,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.stripe import (
StripeAuthMethodConfig,
@@ -67,6 +72,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con
PAYMENT_LINK_RESOURCE_NAME,
PAYMENT_METHOD_HISTORY_MAPPING_KEY,
PLAN_RESOURCE_NAME,
+ PRICE_RESOURCE_NAME,
PROMOTION_CODE_RESOURCE_NAME,
QUOTE_RESOURCE_NAME,
RESOURCE_TO_STRIPE_OBJECT_TYPE,
@@ -87,6 +93,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.custom import (
InvoiceListWithAllLines,
RateLimitCallback,
+ WebhookObjectRefetcher,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.settings import (
ENDPOINTS,
@@ -1641,6 +1648,200 @@ class TestWebhookUpsertCollapse:
assert [row["status"] for row in rows] == [expected_status]
+def _stripe_list(data: list[dict[str, Any]], url: str, has_more: bool = False) -> dict[str, Any]:
+ return {"object": "list", "data": data, "has_more": has_more, "url": url}
+
+
+_FIRST_PAGE = [{"id": f"line_{i}"} for i in range(10)]
+_ALL_LINES = [*_FIRST_PAGE, {"id": "line_10"}]
+
+
+def _error(status: int, error_type: str, code: str) -> ScriptedResponse:
+ return ScriptedResponse(status=status, json={"error": {"type": error_type, "code": code, "message": "Rejected"}})
+
+
+_INCOMPLETE_INVOICE = {
+ "id": "in_1",
+ "object": "invoice",
+ "status": "paid",
+ "lines": _stripe_list(_FIRST_PAGE, "/v1/invoices/in_1/lines", True),
+}
+
+
+class TestWebhookRefetch:
+ def _transform(self, endpoint: str, obj: dict[str, Any]) -> list[dict[str, Any]]:
+ refetcher = WebhookObjectRefetcher(
+ stripe_module.WEBHOOK_REFETCH[endpoint],
+ MagicMock(),
+ client_factory_provider=lambda: stripe_module._refetch_client_factory(
+ "sk_test", None, STRIPE_API_VERSION_ACACIA
+ ),
+ )
+ events = table_from_py_list([_event_row("evt_1", f"{obj['object']}.updated", 1700000100, obj)])
+ return stripe_module._refetching_webhook_table_transformer(refetcher)(events).to_pylist()
+
+ @parameterized.expand(
+ [
+ (
+ "invoice lines past the first page",
+ INVOICE_RESOURCE_NAME,
+ _INCOMPLETE_INVOICE,
+ {"/v1/invoices/in_1/lines": ({"limit": "100"}, _stripe_list(_ALL_LINES, "/v1/invoices/in_1/lines"))},
+ lambda row: (
+ row["status"] == "paid"
+ and [line["id"] for line in orjson.loads(row["lines"])["data"]]
+ == [expected["id"] for expected in _ALL_LINES]
+ ),
+ ),
+ (
+ "credit note lines past the first page",
+ CREDIT_NOTE_RESOURCE_NAME,
+ {
+ "id": "cn_1",
+ "object": "credit_note",
+ "lines": _stripe_list(_FIRST_PAGE, "/v1/credit_notes/cn_1/lines", True),
+ },
+ {
+ "/v1/credit_notes/cn_1/lines": (
+ {"limit": "100"},
+ _stripe_list(_ALL_LINES, "/v1/credit_notes/cn_1/lines"),
+ )
+ },
+ lambda row: len(orjson.loads(row["lines"])["data"]) == len(_ALL_LINES),
+ ),
+ (
+ "tiers of a tiered price",
+ PRICE_RESOURCE_NAME,
+ {"id": "price_1", "object": "price", "billing_scheme": "tiered"},
+ {
+ "/v1/prices/price_1": (
+ {"expand[0]": "tiers"},
+ {
+ "id": "price_1",
+ "object": "price",
+ "billing_scheme": "tiered",
+ "tiers": [{"up_to": 10, "unit_amount": 5}, {"up_to": None, "unit_amount": 3}],
+ },
+ )
+ },
+ lambda row: len(orjson.loads(row["tiers"])) == 2,
+ ),
+ (
+ "expanded subscription discounts",
+ SUBSCRIPTION_RESOURCE_NAME,
+ {"id": "sub_1", "object": "subscription", "discounts": ["di_1"]},
+ {
+ "/v1/subscriptions/sub_1": (
+ {"expand[0]": "discounts", "expand[1]": "items.data.discounts"},
+ {
+ "id": "sub_1",
+ "object": "subscription",
+ "discounts": [{"id": "di_1", "object": "discount", "coupon": {"id": "co_1"}}],
+ },
+ )
+ },
+ lambda row: orjson.loads(row["discounts"])[0]["coupon"]["id"] == "co_1",
+ ),
+ ]
+ )
+ def test_incomplete_object_is_completed(
+ self,
+ _name: str,
+ endpoint: str,
+ event_object: dict[str, Any],
+ answers: dict[str, tuple[dict[str, str], dict[str, Any]]],
+ row_is_complete: Callable[[dict[str, Any]], bool],
+ ) -> None:
+ def script(request: RecordedRequest) -> ScriptedResponse:
+ return ScriptedResponse(json=answers[request.path][1])
+
+ with scripted_network(script) as network:
+ rows = self._transform(endpoint, event_object)
+
+ sent = [
+ (request.path, {name: request.param(name) for name in answers[request.path][0]})
+ for request in network.requests_log
+ ]
+ assert sent == [(path, params) for path, (params, _) in answers.items()]
+ assert len(rows) == 1
+ assert row_is_complete(rows[0])
+
+ @parameterized.expand(
+ [
+ (
+ "complete object",
+ {"lines": _stripe_list(_FIRST_PAGE, "/v1/invoices/in_1/lines")},
+ _error(404, "invalid_request_error", "resource_missing"),
+ 0,
+ ),
+ ("object deleted since the event", {}, _error(404, "invalid_request_error", "resource_missing"), 1),
+ ("object Stripe rejects", {}, _error(400, "invalid_request_error", "parameter_unknown"), 1),
+ ]
+ )
+ def test_event_payload_is_kept(
+ self, _name: str, fields: dict[str, Any], answer: ScriptedResponse, expected_requests: int
+ ) -> None:
+ with scripted_network([answer]) as network:
+ rows = self._transform(INVOICE_RESOURCE_NAME, {**_INCOMPLETE_INVOICE, **fields})
+
+ assert len(network.requests_log) == expected_requests
+ assert [line["id"] for line in orjson.loads(rows[0]["lines"])["data"]] == [
+ expected["id"] for expected in _FIRST_PAGE
+ ]
+
+ @parameterized.expand(
+ [
+ ("revoked key", _error(401, "invalid_request_error", "api_key_expired"), stripe_lib.AuthenticationError),
+ (
+ "missing permission",
+ _error(403, "invalid_request_error", "secret_key_required"),
+ stripe_lib.PermissionError,
+ ),
+ ]
+ )
+ def test_account_wide_error_fails_the_batch(
+ self, _name: str, answer: ScriptedResponse, error: type[Exception]
+ ) -> None:
+ with scripted_network([answer]), pytest.raises(error):
+ self._transform(INVOICE_RESOURCE_NAME, _INCOMPLETE_INVOICE)
+
+ def test_webhook_sync_completes_invoice_lines(self) -> None:
+ source = StripeSource()
+ config = StripeSourceConfig(
+ auth_method=StripeAuthMethodConfig(selection="api_key", stripe_secret_key="sk_test")
+ )
+ webhooks = MagicMock(spec=WebhookSourceManager)
+ webhooks.webhook_enabled = mock.AsyncMock(return_value=True)
+ events = table_from_py_list([_event_row("evt_1", "invoice.paid", 1700000100, _INCOMPLETE_INVOICE)])
+
+ with scripted_network(lambda request: ScriptedResponse(json=_stripe_list(_ALL_LINES, request.path))):
+ inputs = source_inputs(INVOICE_RESOURCE_NAME)
+ with mock.patch.object(source, "get_webhook_source_manager", return_value=webhooks):
+ response = source.source_for_pipeline(config, source.get_resumable_source_manager(inputs), inputs)
+ response.items()
+ transformer = webhooks.get_items.call_args.kwargs["table_transformer"]
+ rows = transformer(events).to_pylist()
+
+ assert len(orjson.loads(rows[0]["lines"])["data"]) == len(_ALL_LINES)
+ assert webhooks.get_items.call_args.kwargs["batch_row_limit"] == stripe_module.STRIPE_CHUNK_SIZE
+
+ def test_client_takes_a_pacer_turn_before_every_line_page(self) -> None:
+ turns: list[str] = []
+ pages = [
+ ScriptedResponse(json=_stripe_list(_FIRST_PAGE, "/v1/invoices/in_1/lines", has_more=True)),
+ ScriptedResponse(json=_stripe_list([{"id": "line_10"}], "/v1/invoices/in_1/lines")),
+ ]
+ client = stripe_module._refetch_client_factory("sk_test", None, STRIPE_API_VERSION_ACACIA)(
+ lambda _retry_after: None, lambda: turns.append("turn")
+ )
+
+ with scripted_network(pages):
+ lines = list(client.invoices.line_items.list("in_1", params={"limit": 10}).auto_paging_iter())
+
+ assert len(lines) == 11
+ assert turns == ["turn", "turn"]
+
+
class TestEndpointCatalogWiring:
def setup_method(self):
self.resources = stripe_module._build_resources(MagicMock(), logger=None)