:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(stripe): re-fetch expanded objects for webhook rows (#118548)
Daniel RCea24a2be8f2d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.pyindex 61a1ce9767ff..40698369d467 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py@@ -7,6 +7,8 @@ import stripe as stripe_libfrom stripe import CreditNote, CreditNoteLineItem, Invoice, InvoiceLineItem, ListObject, StripeClientfrom structlog.types import FilteringBoundLogger+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.request_pacer import (RequestPacer,submit_with_context,@@ -27,6 +29,8 @@ _LineT = TypeVar("_LineT", InvoiceLineItem, CreditNoteLineItem)# Receives the Retry-After seconds from a 429, or None when Stripe sent none.RateLimitCallback = Callable[[Optional[float]], None]ClientFactory = Callable[[RateLimitCallback], StripeClient]+# Gets the pool's rate-limit callback and its pacer turn, which the client takes before every request.+PacedClientFactory = Callable[[RateLimitCallback, Callable[[], None]], StripeClient]class _ListWithAllLines(Generic[_ParentT, _LineT]):@@ -154,3 +158,76 @@ class CreditNoteListWithAllLines(_ListWithAllLines[CreditNote, CreditNoteLineItedef _attach_lines(self, obj: CreditNote, lines: list[CreditNoteLineItem]) -> None:obj.lines.data = lines+++@frozen+class WebhookRefetch:+ """How to complete one object type whose webhook payload can leave out data the list sync includes."""++ is_incomplete: Callable[[dict[str, Any]], bool]+ # Gets the event payload, so a step can keep the event's fields and add only what is missing.+ complete: Callable[[StripeClient, dict[str, Any]], dict[str, Any]]+++class WebhookObjectRefetcher:+ """Completes incomplete webhook objects with reads from the API.++ Stripe never expands an event payload, so an event object can be missing data the list sync+ fetches: an invoice carries only its first 10 lines, a tiered price has no tiers. Upserting that+ object would overwrite the complete row. Only incomplete objects cost API calls. The reads use a+ pool with per-thread clients, as the invoice line fan-out does, and objects keep their order. The+ client takes a pacer turn before every request, so line pages and retries count against the rate.+ """++ def __init__(+ self,+ refetch: WebhookRefetch,+ logger: FilteringBoundLogger,+ client_factory_provider: Callable[[], PacedClientFactory],+ concurrency: int = LINE_FETCH_CONCURRENCY,+ requests_per_second: float = LINE_REQUESTS_PER_SECOND,+ ) -> None:+ self._refetch = refetch+ self._logger = logger+ self._client_factory_provider = client_factory_provider+ self._concurrency = concurrency+ self._pacer = RequestPacer(requests_per_second)++ def complete(self, objects: list[dict[str, Any]]) -> list[dict[str, Any]]:+ if not any(self._refetch.is_incomplete(obj) for obj in objects):+ return objects++ # Resolved once per batch, so a long webhook drain picks up a refreshed OAuth token.+ client_factory = self._client_factory_provider()+ thread_clients = threading.local()+ pool = ThreadPoolExecutor(max_workers=self._concurrency, thread_name_prefix="stripe-webhook-refetch")+ try:+ futures = [+ submit_with_context(pool, self._complete_one, obj, client_factory, thread_clients)+ if self._refetch.is_incomplete(obj)+ else None+ for obj in objects+ ]+ completed = [obj if future is None else future.result() for obj, future in zip(objects, futures)]+ finally:+ pool.shutdown(wait=False, cancel_futures=True)++ self._logger.debug(f"Stripe: completed {sum(f is not None for f in futures)} incomplete webhook objects")+ return completed++ def _complete_one(+ self, obj: dict[str, Any], client_factory: PacedClientFactory, thread_clients: threading.local+ ) -> dict[str, Any]:+ client: Optional[StripeClient] = getattr(thread_clients, "client", None)+ if client is None:+ client = client_factory(self._pacer.throttled, self._pacer.wait_turn)+ thread_clients.client = client+ try:+ return self._refetch.complete(client, obj)+ except stripe_lib.InvalidRequestError as e:+ # A rejection of this one object fails the same way on every retry, and a raise would+ # fail the whole batch and replay it on every sync. Auth, rate-limit and server errors+ # are not InvalidRequestError, so they still fail the batch.+ log = self._logger.debug if getattr(e, "code", None) == "resource_missing" else self._logger.warning+ log(f"Stripe: kept the webhook payload of {obj.get('object')} {obj['id']}: {e.user_message}")+ return objdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/source.pyindex 142eebbb47e3..f940b9a80a9b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/source.py@@ -1,5 +1,7 @@from typing import TYPE_CHECKING, Optional, cast+from django.db import close_old_connections, connection+from products.warehouse_sources.backend.facade.codebase_detection import detectfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager@@ -483,6 +485,15 @@ If automatic creation failed with a permissions error, the fix depends on how yoraise ValueError("Stripe access token not found")return integration.access_token+ def _get_api_key_closing_connection(self, config: StripeSourceConfig, team_id: int) -> str:+ # Runs in a long-lived executor thread outside Django's request cycle, so close the+ # connection an OAuth lookup opens instead of leaving it idle on that thread.+ close_old_connections()+ try:+ return self._get_api_key(config, team_id)+ finally:+ connection.close()+def get_schemas(self,config: StripeSourceConfig,@@ -720,6 +731,7 @@ If automatic creation failed with a permissions error, the fix depends on how yo# conversion that declares a parent without threading these silently keeps polling# the parent API while the fan-out telemetry reports otherwise.use_warehouse_parent=inputs.fanout_warehouse_reuse,+ api_key_provider=lambda: self._get_api_key_closing_connection(config, inputs.team_id),include_zero_balance_customers=bool(config.balance_history_all_customers and config.balance_history_all_customers.enabled),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.pyindex f8a0e0204d4c..795bf6b2b5c5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py@@ -31,6 +31,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.basWebhookDeletionResult,WebhookSyncResult,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.batching import DEFAULT_BATCH_ROW_LIMITfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.fanout_telemetry import (log_fanout_parent_rows_consumed,)@@ -93,7 +94,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.cusClientFactory,CreditNoteListWithAllLines,InvoiceListWithAllLines,+ PacedClientFactory,RateLimitCallback,+ WebhookObjectRefetcher,+ WebhookRefetch,)from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.settings import (APPEND_ONLY_INCREMENTAL_FIELDS,@@ -262,9 +266,19 @@ class _RateLimitRetryingRequestsClient(RequestsClient):# scope the non-list-body retry to reads, never a single-object write response._last_request_method: str = ""- def __init__(self, *args: Any, on_rate_limited: Optional[RateLimitCallback] = None, **kwargs: Any) -> None:+ def __init__(+ self,+ *args: Any,+ on_rate_limited: Optional[RateLimitCallback] = None,+ before_request: Optional[Callable[[], None]] = None,+ expects_lists: bool = True,+ **kwargs: Any,+ ) -> None:super().__init__(*args, **kwargs)self._on_rate_limited = on_rate_limited+ self._before_request = before_request+ # A client that also retrieves single objects cannot treat a non-list GET body as corrupt.+ self._expects_lists = expects_listsdef request( # type: ignore[override] # mirrors RequestsClient.request, which already narrows HTTPClient's (str→bytes)self,@@ -274,6 +288,8 @@ class _RateLimitRetryingRequestsClient(RequestsClient):post_data: Any = None,) -> tuple[bytes, int, Mapping[str, str]]:self._last_request_method = (method or "").lower()+ if self._before_request is not None:+ self._before_request()return super().request(method, url, headers, post_data)def _should_retry(@@ -310,16 +326,25 @@ class _RateLimitRetryingRequestsClient(RequestsClient):return True# A complete but non-list body identifies a corrupt list read only for a GET — a write# (webhook create/update/delete) legitimately returns a single object, so gate on method.- return self._last_request_method == "get" and _is_non_list_stripe_response(body)+ return self._expects_lists and self._last_request_method == "get" and _is_non_list_stripe_response(body)-def _tracked_stripe_http_client(on_rate_limited: Optional[RateLimitCallback] = None) -> RequestsClient:+def _tracked_stripe_http_client(+ on_rate_limited: Optional[RateLimitCallback] = None,+ before_request: Optional[Callable[[], None]] = None,+ expects_lists: bool = True,+) -> RequestsClient:"""Wrap a tracked `requests.Session` in Stripe's `RequestsClient` so everyStripe SDK call participates in our HTTP logging, metrics, and sample capture.Uses a subclass that additionally retries 429 rate limits and truncated list responses via theSDK's built-in backoff, and reports each 429 to `on_rate_limited`."""- return _RateLimitRetryingRequestsClient(session=make_tracked_session(), on_rate_limited=on_rate_limited)+ return _RateLimitRetryingRequestsClient(+ session=make_tracked_session(),+ on_rate_limited=on_rate_limited,+ before_request=before_request,+ expects_lists=expects_lists,+ )def _clean_stripe_error_message(msg: str) -> str:@@ -912,6 +937,36 @@ def _build_resources(}+def _client_factory(+ api_key: str, account_id: Optional[str], api_version: str+) -> Callable[[Optional[RateLimitCallback]], StripeClient]:+ def new_client(on_rate_limited: Optional[RateLimitCallback]) -> StripeClient:+ return _stripe_client(api_key, account_id, api_version, _tracked_stripe_http_client(on_rate_limited))++ return new_client+++def _refetch_client_factory(api_key: str, account_id: Optional[str], api_version: str) -> PacedClientFactory:+ def new_client(on_rate_limited: RateLimitCallback, before_request: Callable[[], None]) -> StripeClient:+ http_client = _tracked_stripe_http_client(on_rate_limited, before_request=before_request, expects_lists=False)+ return _stripe_client(api_key, account_id, api_version, http_client)++ return new_client+++def _stripe_client(+ api_key: str, account_id: Optional[str], api_version: str, http_client: RequestsClient+) -> StripeClient:+ return StripeClient(+ api_key,+ stripe_account=account_id,+ stripe_version=api_version,+ max_network_retries=2,+ base_addresses=_stripe_base_addresses(),+ http_client=http_client,+ )++def get_rows(api_key: str,endpoint: str,@@ -925,17 +980,8 @@ def get_rows(warehouse_parent: Optional["ParentTableRef"] = None,include_zero_balance_customers: bool = False,):- def new_client(on_rate_limited: Optional[RateLimitCallback] = None) -> StripeClient:- return StripeClient(- api_key,- stripe_account=account_id,- stripe_version=api_version,- max_network_retries=2,- base_addresses=_stripe_base_addresses(),- http_client=_tracked_stripe_http_client(on_rate_limited=on_rate_limited),- )-- client = new_client()+ new_client = _client_factory(api_key, account_id, api_version)+ client = new_client(None)default_params = {"limit": DEFAULT_LIMIT}resources = _build_resources(client,@@ -1222,7 +1268,7 @@ def _webhook_tie_break(obj: dict) -> int:return _INVOICE_STATUS_STAGE.get(obj.get("status") or "", 0)-def _webhook_table_transformer(table: pa.Table) -> pa.Table:+def _latest_webhook_objects(table: pa.Table) -> list[dict[str, Any]]:data_col = table.column("data").to_pylist()created_col = table.column("created").to_pylist()@@ -1250,8 +1296,70 @@ def _webhook_table_transformer(table: pa.Table) -> pa.Table:if existing is None or rank >= existing[0]:best_by_id[obj_id] = (rank, obj)- rows = [_scrub_client_secrets(obj) for _, obj in best_by_id.values()]- return table_from_py_list(rows)+ return [obj for _, obj in best_by_id.values()]+++def _webhook_table_transformer(table: pa.Table) -> pa.Table:+ return table_from_py_list([_scrub_client_secrets(obj) for obj in _latest_webhook_objects(table)])+++def _refetching_webhook_table_transformer(refetcher: WebhookObjectRefetcher) -> Callable[[pa.Table], pa.Table]:+ def transform(table: pa.Table) -> pa.Table:+ objects = refetcher.complete(_latest_webhook_objects(table))+ return table_from_py_list([_scrub_client_secrets(obj) for obj in objects])++ return transform+++def _lines_truncated(obj: dict[str, Any]) -> bool:+ lines = obj.get("lines")+ return isinstance(lines, Mapping) and bool(lines.get("has_more"))+++def _tiers_missing(obj: dict[str, Any]) -> bool:+ return obj.get("billing_scheme") == "tiered" and not obj.get("tiers")+++def _discounts_unexpanded(obj: dict[str, Any]) -> bool:+ items = (obj.get("items") or {}).get("data") or []+ discounts = [*(obj.get("discounts") or []), *(d for item in items for d in (item.get("discounts") or []))]+ return any(isinstance(discount, str) for discount in discounts)+++def _with_all_lines(obj: dict[str, Any], lines: ListObject[Any]) -> dict[str, Any]:+ return {**obj, "lines": {**obj["lines"], "data": list(lines.auto_paging_iter()), "has_more": False, "url": None}}+++# Each entry adds back what the event payload leaves out. Invoices and credit notes keep the event's+# fields and only fetch their lines, which is one request per object instead of two.+WEBHOOK_REFETCH: dict[str, WebhookRefetch] = {+ INVOICE_RESOURCE_NAME: WebhookRefetch(+ is_incomplete=_lines_truncated,+ complete=lambda client, invoice: _with_all_lines(+ invoice, client.invoices.line_items.list(invoice["id"], params={"limit": DEFAULT_LIMIT})+ ),+ ),+ CREDIT_NOTE_RESOURCE_NAME: WebhookRefetch(+ is_incomplete=_lines_truncated,+ complete=lambda client, credit_note: _with_all_lines(+ credit_note, client.credit_notes.line_items.list(credit_note["id"], params={"limit": DEFAULT_LIMIT})+ ),+ ),+ PRICE_RESOURCE_NAME: WebhookRefetch(+ is_incomplete=_tiers_missing,+ complete=lambda client, price: client.prices.retrieve(price["id"], params={"expand": ["tiers"]}),+ ),+ PLAN_RESOURCE_NAME: WebhookRefetch(+ is_incomplete=_tiers_missing,+ complete=lambda client, plan: client.plans.retrieve(plan["id"], params={"expand": ["tiers"]}),+ ),+ SUBSCRIPTION_RESOURCE_NAME: WebhookRefetch(+ is_incomplete=_discounts_unexpanded,+ complete=lambda client, subscription: client.subscriptions.retrieve(+ subscription["id"], params={"expand": ["discounts", "items.data.discounts"]}+ ),+ ),+}def _webhook_history_table_transformer(table: pa.Table) -> pa.Table:@@ -1345,6 +1453,7 @@ def stripe_source(team_id: Optional[int] = None,source_id: Optional[str] = None,use_warehouse_parent: bool = False,+ api_key_provider: Optional[Callable[[], str]] = None,include_zero_balance_customers: bool = False,):# Only the endpoints with a PostHog-managed canonical schema have column hints; the rest let the@@ -1368,12 +1477,28 @@ def stripe_source(if webhook_enabled:# History tables keep every event as its own row; everything else collapses each# batch to the latest state per object id and upserts it.- table_transformer = (- _webhook_history_table_transformer- if endpoint == CUSTOMER_PAYMENT_METHOD_HISTORY_RESOURCE_NAME- else _webhook_table_transformer+ refetch = WEBHOOK_REFETCH.get(endpoint)+ table_transformer: Callable[[pa.Table], pa.Table]+ if endpoint == CUSTOMER_PAYMENT_METHOD_HISTORY_RESOURCE_NAME:+ table_transformer = _webhook_history_table_transformer+ elif refetch is not None:+ table_transformer = _refetching_webhook_table_transformer(+ WebhookObjectRefetcher(+ refetch,+ logger,+ client_factory_provider=lambda: _refetch_client_factory(+ api_key_provider() if api_key_provider is not None else api_key, account_id, api_version+ ),+ )+ )+ else:+ table_transformer = _webhook_table_transformer+ # Objects grow once their lines or tiers are filled in, after the batch limits were+ # checked, so a batch that completes objects holds no more rows than a list sync chunk.+ batch_row_limit = STRIPE_CHUNK_SIZE if refetch is not None else DEFAULT_BATCH_ROW_LIMIT+ return webhook_source_manager.get_items(+ table_transformer=table_transformer, batch_row_limit=batch_row_limit)- return webhook_source_manager.get_items(table_transformer=table_transformer)return get_rows(api_key=api_key,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.pyindex 4a833fe43d0d..f552221ee7bd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py@@ -1,6 +1,7 @@import functoolsimport threadingimport contextvars+from collections.abc import Callablefrom pathlib import Pathfrom types import SimpleNamespacefrom typing import Any, Optional, cast@@ -27,9 +28,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.fanfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.warehouse_parent import (ParentTableRef,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import SourceDriver-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.fake_network import RecordedRequest-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.responders import ScriptedResponse+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,+ ScriptedResponse,+ SourceDriver,+ scripted_network,+ source_inputs,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.stripe import (StripeAuthMethodConfig,@@ -67,6 +72,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.conPAYMENT_LINK_RESOURCE_NAME,PAYMENT_METHOD_HISTORY_MAPPING_KEY,PLAN_RESOURCE_NAME,+ PRICE_RESOURCE_NAME,PROMOTION_CODE_RESOURCE_NAME,QUOTE_RESOURCE_NAME,RESOURCE_TO_STRIPE_OBJECT_TYPE,@@ -87,6 +93,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.confrom products.warehouse_sources.backend.temporal.data_imports.sources.stripe.custom import (InvoiceListWithAllLines,RateLimitCallback,+ WebhookObjectRefetcher,)from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.settings import (ENDPOINTS,@@ -1641,6 +1648,200 @@ class TestWebhookUpsertCollapse:assert [row["status"] for row in rows] == [expected_status]+def _stripe_list(data: list[dict[str, Any]], url: str, has_more: bool = False) -> dict[str, Any]:+ return {"object": "list", "data": data, "has_more": has_more, "url": url}+++_FIRST_PAGE = [{"id": f"line_{i}"} for i in range(10)]+_ALL_LINES = [*_FIRST_PAGE, {"id": "line_10"}]+++def _error(status: int, error_type: str, code: str) -> ScriptedResponse:+ return ScriptedResponse(status=status, json={"error": {"type": error_type, "code": code, "message": "Rejected"}})+++_INCOMPLETE_INVOICE = {+ "id": "in_1",+ "object": "invoice",+ "status": "paid",+ "lines": _stripe_list(_FIRST_PAGE, "/v1/invoices/in_1/lines", True),+}+++class TestWebhookRefetch:+ def _transform(self, endpoint: str, obj: dict[str, Any]) -> list[dict[str, Any]]:+ refetcher = WebhookObjectRefetcher(+ stripe_module.WEBHOOK_REFETCH[endpoint],+ MagicMock(),+ client_factory_provider=lambda: stripe_module._refetch_client_factory(+ "sk_test", None, STRIPE_API_VERSION_ACACIA+ ),+ )+ events = table_from_py_list([_event_row("evt_1", f"{obj['object']}.updated", 1700000100, obj)])+ return stripe_module._refetching_webhook_table_transformer(refetcher)(events).to_pylist()++ @parameterized.expand(+ [+ (+ "invoice lines past the first page",+ INVOICE_RESOURCE_NAME,+ _INCOMPLETE_INVOICE,+ {"/v1/invoices/in_1/lines": ({"limit": "100"}, _stripe_list(_ALL_LINES, "/v1/invoices/in_1/lines"))},+ lambda row: (+ row["status"] == "paid"+ and [line["id"] for line in orjson.loads(row["lines"])["data"]]+ == [expected["id"] for expected in _ALL_LINES]+ ),+ ),+ (+ "credit note lines past the first page",+ CREDIT_NOTE_RESOURCE_NAME,+ {+ "id": "cn_1",+ "object": "credit_note",+ "lines": _stripe_list(_FIRST_PAGE, "/v1/credit_notes/cn_1/lines", True),+ },+ {+ "/v1/credit_notes/cn_1/lines": (+ {"limit": "100"},+ _stripe_list(_ALL_LINES, "/v1/credit_notes/cn_1/lines"),+ )+ },+ lambda row: len(orjson.loads(row["lines"])["data"]) == len(_ALL_LINES),+ ),+ (+ "tiers of a tiered price",+ PRICE_RESOURCE_NAME,+ {"id": "price_1", "object": "price", "billing_scheme": "tiered"},+ {+ "/v1/prices/price_1": (+ {"expand[0]": "tiers"},+ {+ "id": "price_1",+ "object": "price",+ "billing_scheme": "tiered",+ "tiers": [{"up_to": 10, "unit_amount": 5}, {"up_to": None, "unit_amount": 3}],+ },+ )+ },+ lambda row: len(orjson.loads(row["tiers"])) == 2,+ ),+ (+ "expanded subscription discounts",+ SUBSCRIPTION_RESOURCE_NAME,+ {"id": "sub_1", "object": "subscription", "discounts": ["di_1"]},+ {+ "/v1/subscriptions/sub_1": (+ {"expand[0]": "discounts", "expand[1]": "items.data.discounts"},+ {+ "id": "sub_1",+ "object": "subscription",+ "discounts": [{"id": "di_1", "object": "discount", "coupon": {"id": "co_1"}}],+ },+ )+ },+ lambda row: orjson.loads(row["discounts"])[0]["coupon"]["id"] == "co_1",+ ),+ ]+ )+ def test_incomplete_object_is_completed(+ self,+ _name: str,+ endpoint: str,+ event_object: dict[str, Any],+ answers: dict[str, tuple[dict[str, str], dict[str, Any]]],+ row_is_complete: Callable[[dict[str, Any]], bool],+ ) -> None:+ def script(request: RecordedRequest) -> ScriptedResponse:+ return ScriptedResponse(json=answers[request.path][1])++ with scripted_network(script) as network:+ rows = self._transform(endpoint, event_object)++ sent = [+ (request.path, {name: request.param(name) for name in answers[request.path][0]})+ for request in network.requests_log+ ]+ assert sent == [(path, params) for path, (params, _) in answers.items()]+ assert len(rows) == 1+ assert row_is_complete(rows[0])++ @parameterized.expand(+ [+ (+ "complete object",+ {"lines": _stripe_list(_FIRST_PAGE, "/v1/invoices/in_1/lines")},+ _error(404, "invalid_request_error", "resource_missing"),+ 0,+ ),+ ("object deleted since the event", {}, _error(404, "invalid_request_error", "resource_missing"), 1),+ ("object Stripe rejects", {}, _error(400, "invalid_request_error", "parameter_unknown"), 1),+ ]+ )+ def test_event_payload_is_kept(+ self, _name: str, fields: dict[str, Any], answer: ScriptedResponse, expected_requests: int+ ) -> None:+ with scripted_network([answer]) as network:+ rows = self._transform(INVOICE_RESOURCE_NAME, {**_INCOMPLETE_INVOICE, **fields})++ assert len(network.requests_log) == expected_requests+ assert [line["id"] for line in orjson.loads(rows[0]["lines"])["data"]] == [+ expected["id"] for expected in _FIRST_PAGE+ ]++ @parameterized.expand(+ [+ ("revoked key", _error(401, "invalid_request_error", "api_key_expired"), stripe_lib.AuthenticationError),+ (+ "missing permission",+ _error(403, "invalid_request_error", "secret_key_required"),+ stripe_lib.PermissionError,+ ),+ ]+ )+ def test_account_wide_error_fails_the_batch(+ self, _name: str, answer: ScriptedResponse, error: type[Exception]+ ) -> None:+ with scripted_network([answer]), pytest.raises(error):+ self._transform(INVOICE_RESOURCE_NAME, _INCOMPLETE_INVOICE)++ def test_webhook_sync_completes_invoice_lines(self) -> None:+ source = StripeSource()+ config = StripeSourceConfig(+ auth_method=StripeAuthMethodConfig(selection="api_key", stripe_secret_key="sk_test")+ )+ webhooks = MagicMock(spec=WebhookSourceManager)+ webhooks.webhook_enabled = mock.AsyncMock(return_value=True)+ events = table_from_py_list([_event_row("evt_1", "invoice.paid", 1700000100, _INCOMPLETE_INVOICE)])++ with scripted_network(lambda request: ScriptedResponse(json=_stripe_list(_ALL_LINES, request.path))):+ inputs = source_inputs(INVOICE_RESOURCE_NAME)+ with mock.patch.object(source, "get_webhook_source_manager", return_value=webhooks):+ response = source.source_for_pipeline(config, source.get_resumable_source_manager(inputs), inputs)+ response.items()+ transformer = webhooks.get_items.call_args.kwargs["table_transformer"]+ rows = transformer(events).to_pylist()++ assert len(orjson.loads(rows[0]["lines"])["data"]) == len(_ALL_LINES)+ assert webhooks.get_items.call_args.kwargs["batch_row_limit"] == stripe_module.STRIPE_CHUNK_SIZE++ def test_client_takes_a_pacer_turn_before_every_line_page(self) -> None:+ turns: list[str] = []+ pages = [+ ScriptedResponse(json=_stripe_list(_FIRST_PAGE, "/v1/invoices/in_1/lines", has_more=True)),+ ScriptedResponse(json=_stripe_list([{"id": "line_10"}], "/v1/invoices/in_1/lines")),+ ]+ client = stripe_module._refetch_client_factory("sk_test", None, STRIPE_API_VERSION_ACACIA)(+ lambda _retry_after: None, lambda: turns.append("turn")+ )++ with scripted_network(pages):+ lines = list(client.invoices.line_items.list("in_1", params={"limit": 10}).auto_paging_iter())++ assert len(lines) == 11+ assert turns == ["turn", "turn"]++class TestEndpointCatalogWiring:def setup_method(self):self.resources = stripe_module._build_resources(MagicMock(), logger=None)