:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(stripe): fetch every credit note line (#118546)
Daniel RCb4bc4bfb615e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.pyindex bbbfd2d765c3..61a1ce9767ff 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py@@ -1,10 +1,10 @@import threadingfrom collections.abc import Callable, Iteratorfrom concurrent.futures import ThreadPoolExecutor-from typing import Optional, ParamSpec, TypeVar+from typing import Any, ClassVar, Generic, Optional, ParamSpec, TypeVarimport stripe as stripe_lib-from stripe import Invoice, InvoiceLineItem, InvoiceService, ListObject, StripeClient+from stripe import CreditNote, CreditNoteLineItem, Invoice, InvoiceLineItem, ListObject, StripeClientfrom structlog.types import FilteringBoundLoggerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.request_pacer import (@@ -21,27 +21,31 @@ LINE_REQUESTS_PER_SECOND = 20.0_P = ParamSpec("_P")_T = TypeVar("_T")+_ParentT = TypeVar("_ParentT", Invoice, CreditNote)+_LineT = TypeVar("_LineT", InvoiceLineItem, CreditNoteLineItem)# Receives the Retry-After seconds from a 429, or None when Stripe sent none.RateLimitCallback = Callable[[Optional[float]], None]ClientFactory = Callable[[RateLimitCallback], StripeClient]-class InvoiceListWithAllLines:- """Invoice listing that expands each invoice's line items.+class _ListWithAllLines(Generic[_ParentT, _LineT]):+ """Listing that expands each object's embedded `lines` list.- The list endpoint embeds at most 10 lines per invoice, so an invoice above that needs one extra+ The list endpoint embeds at most 10 lines per object, so an object above that needs one extra`/lines` call. Those calls run on a small thread pool while the next page downloads. Each workerthread builds its own client from the factory: Stripe's RequestsClient hands a caller-suppliedsession to every thread, `requests.Session` is not documented as thread-safe, and the retryingclient keeps per-request state. The page client comes from the same factory, so a 429 on- either leg slows the whole pool. Invoices are yielded in list order because the+ either leg slows the whole pool. Objects are yielded in list order because the`starting_after` resume cursor is the last yielded id."""+ object_name: ClassVar[str]+def __init__(self,- params: InvoiceService.ListParams,+ params: Any,logger: FilteringBoundLogger,client_factory: ClientFactory,concurrency: int = LINE_FETCH_CONCURRENCY,@@ -54,60 +58,68 @@ class InvoiceListWithAllLines:self._pacer = RequestPacer(requests_per_second)self._thread_clients = threading.local()- def auto_paging_iter(self) -> Iterator[Invoice]:+ def _list_page(self, client: StripeClient) -> ListObject[_ParentT]:+ raise NotImplementedError++ def _list_lines(self, client: StripeClient, object_id: str) -> ListObject[_LineT]:+ raise NotImplementedError++ def _attach_lines(self, obj: _ParentT, lines: list[_LineT]) -> None:+ raise NotImplementedError++ def auto_paging_iter(self) -> Iterator[_ParentT]:page_client = self._client_factory(self._pacer.throttled)- page: ListObject[Invoice] = page_client.invoices.list(params=self.params)+ page = self._list_page(page_client)total_line_calls = 0- invoice_count = 0+ object_count = 0# The next-page fetch is submitted first, so the extra worker picks it up at once and the# line calls fill the others.- pool = ThreadPoolExecutor(max_workers=self._concurrency + 1, thread_name_prefix="stripe-invoice-lines")+ pool = ThreadPoolExecutor(+ max_workers=self._concurrency + 1, thread_name_prefix=f"stripe-{self.object_name.replace(' ', '-')}-lines"+ )try:while not page.is_empty:next_page = submit_with_context(pool, page.next_page)line_futures = [- submit_with_context(pool, self._fetch_lines, invoice.id)- if invoice.lines.has_more and invoice.id- else None- for invoice in page.data+ submit_with_context(pool, self._fetch_lines, obj.id) if obj.lines.has_more and obj.id else None+ for obj in page.data]- for invoice, future in zip(page.data, line_futures):- if invoice.lines.has_more and future is None:- self.logger.warning(f"Invoice {invoice.id} has no id")+ for obj, future in zip(page.data, line_futures):+ if obj.lines.has_more and future is None:+ self.logger.warning(f"Stripe {self.object_name} {obj.id} has no id")continueif future is not None:all_lines = future.result()if all_lines is not None:total_line_calls += len(all_lines) // 100 + 1- invoice.lines.data = all_lines- invoice.lines.has_more = False- invoice.lines.url = None # type: ignore+ self._attach_lines(obj, all_lines)+ obj.lines.has_more = False+ obj.lines.url = None # type: ignore- yield invoice+ yield obj- invoice_count += 1- if invoice_count % 10000 == 0:- self.logger.info(f"Stripe: processed {invoice_count} invoices")+ object_count += 1+ if object_count % 10000 == 0:+ self.logger.info(f"Stripe: processed {object_count} {self.object_name}s")page = next_page.result()finally:# The consumer may close the generator early; never block on requests still in flight.pool.shutdown(wait=False, cancel_futures=True)- self.logger.debug(f"Stripe: made {total_line_calls} calls for invoice line items")+ self.logger.debug(f"Stripe: made {total_line_calls} calls for {self.object_name} line items")- def _fetch_lines(self, invoice_id: str) -> Optional[list[InvoiceLineItem]]:+ def _fetch_lines(self, object_id: str) -> Optional[list[_LineT]]:self._pacer.wait_turn()try:- line_items = self._thread_client().invoices.line_items.list(invoice=invoice_id, params={"limit": 100})- return list(line_items.auto_paging_iter())+ return list(self._list_lines(self._thread_client(), object_id).auto_paging_iter())except stripe_lib.InvalidRequestError as e:if getattr(e, "code", None) != "resource_missing":raise- self.logger.debug(f"Stripe: invoice {invoice_id} no longer exists, yielding with partial lines")+ self.logger.debug(f"Stripe: {self.object_name} {object_id} no longer exists, yielding with partial lines")return Nonedef _thread_client(self) -> StripeClient:@@ -116,3 +128,29 @@ class InvoiceListWithAllLines:client = self._client_factory(self._pacer.throttled)self._thread_clients.client = clientreturn client+++class InvoiceListWithAllLines(_ListWithAllLines[Invoice, InvoiceLineItem]):+ object_name = "invoice"++ def _list_page(self, client: StripeClient) -> ListObject[Invoice]:+ return client.invoices.list(params=self.params)++ def _list_lines(self, client: StripeClient, object_id: str) -> ListObject[InvoiceLineItem]:+ return client.invoices.line_items.list(invoice=object_id, params={"limit": 100})++ def _attach_lines(self, obj: Invoice, lines: list[InvoiceLineItem]) -> None:+ obj.lines.data = lines+++class CreditNoteListWithAllLines(_ListWithAllLines[CreditNote, CreditNoteLineItem]):+ object_name = "credit note"++ def _list_page(self, client: StripeClient) -> ListObject[CreditNote]:+ return client.credit_notes.list(params=self.params)++ def _list_lines(self, client: StripeClient, object_id: str) -> ListObject[CreditNoteLineItem]:+ return client.credit_notes.line_items.list(credit_note=object_id, params={"limit": 100})++ def _attach_lines(self, obj: CreditNote, lines: list[CreditNoteLineItem]) -> None:+ obj.lines.data = linesdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.pyindex ec116cf79c4d..f8a0e0204d4c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py@@ -91,6 +91,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con)from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.custom import (ClientFactory,+ CreditNoteListWithAllLines,InvoiceListWithAllLines,RateLimitCallback,)@@ -350,6 +351,27 @@ def _call_stripe(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:raise+def _iter_cleaning_errors(objects: Iterable[Any]) -> Iterator[Any]:+ """Iterate a Stripe listing and clean errors raised while it pages, as `_call_stripe` does.++ A listing wrapper such as `InvoiceListWithAllLines` makes its first request only when it is+ iterated, so `_call_stripe` around the method call never sees that request's error.+ """+ iterator = iter(objects)+ try:+ while True:+ try:+ obj = _call_stripe(next, iterator)+ except StopIteration:+ return+ yield obj+ finally:+ # Close the inner generator now, so a listing wrapper shuts down its thread pool.+ close = getattr(iterator, "close", None)+ if close is not None:+ close()++def _is_stripe_resource_missing_error(error: stripe_lib.StripeError) -> bool:"""True for Stripe's ``resource_missing`` 404 — e.g. a customer deleted between when welisted it and when we fetched its nested resources. The object is genuinely gone, so the@@ -734,8 +756,8 @@ def _build_resources(checks). Nested resources carry their parent on `.parent`, so callers can derive thenested→parent linkage without restating it elsewhere.- `logger` and `client_factory` are only consumed by InvoiceListWithAllLines; pass None when the- caller doesn't need the wrapped invoice expansion (e.g. validation, which just probes the list+ `logger` and `client_factory` are only consumed by the invoice and credit note line expansion;+ pass None when the caller doesn't need it (e.g. validation, which just probes the listendpoint). The factory gives each of its worker threads a client of its own."""return {@@ -771,7 +793,13 @@ def _build_resources("expand": ["data.discounts", "data.items.data.discounts"],},),- CREDIT_NOTE_RESOURCE_NAME: StripeResource(method=client.credit_notes.list),+ CREDIT_NOTE_RESOURCE_NAME: StripeResource(+ method=(+ (lambda params: CreditNoteListWithAllLines(params, logger, client_factory=client_factory)) # type: ignore+ if logger is not None and client_factory is not None+ else client.credit_notes.list+ )+ ),COUPON_RESOURCE_NAME: StripeResource(method=client.coupons.list),# The one nested resource that reads its parent from the warehouse. Its skip predicate# answers from the parent row itself, so a run is dominated by paging the customer@@ -1106,7 +1134,7 @@ def get_rows(else:def _list_resource(params: dict[str, Any]) -> Any:- return _call_stripe(resource.method, params=params).auto_paging_iter()+ return _iter_cleaning_errors(_call_stripe(resource.method, params=params).auto_paging_iter())list_params = {**default_params, **resource.params, **resume_params}cursor_restarted = False@@ -1162,7 +1190,7 @@ def get_rows(f"created[lt]": db_incremental_field_earliest_value,},)- yield from _batch_and_yield(stripe_objects.auto_paging_iter(), batcher)+ yield from _batch_and_yield(_iter_cleaning_errors(stripe_objects.auto_paging_iter()), batcher)# check for any objects more than the maximum object we already haveif db_incremental_field_last_value is not None:@@ -1177,7 +1205,7 @@ def get_rows(},)yield from _batch_and_yield(- stripe_objects.auto_paging_iter(),+ _iter_cleaning_errors(stripe_objects.auto_paging_iter()),batcher,incremental_field_name=incremental_field_name,stop_at_or_before=_coerce_incremental_cursor(db_incremental_field_last_value),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.pyindex 2f8e71a08ca7..4a833fe43d0d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py@@ -46,6 +46,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.conBILLING_METER_RESOURCE_NAME,CHARGE_RESOURCE_NAME,CHECKOUT_SESSION_RESOURCE_NAME,+ CREDIT_NOTE_RESOURCE_NAME,CUSTOMER_BALANCE_TRANSACTION_RESOURCE_NAME,CUSTOMER_PAYMENT_METHOD_HISTORY_RESOURCE_NAME,CUSTOMER_PAYMENT_METHOD_RESOURCE_NAME,@@ -61,6 +62,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.conHISTORY_PREVIOUS_ATTRIBUTES_COLUMN,HISTORY_SNAPSHOT_EVENT_TYPE,INVOICE_PAYMENT_RESOURCE_NAME,+ INVOICE_RESOURCE_NAME,PAYMENT_INTENT_RESOURCE_NAME,PAYMENT_LINK_RESOURCE_NAME,PAYMENT_METHOD_HISTORY_MAPPING_KEY,@@ -971,6 +973,40 @@ class TestInvoiceListWithAllLines:)+def test_credit_note_rows_carry_lines_past_the_embedded_page():+ embedded = [{"id": f"cnli_{i}", "object": "credit_note_line_item"} for i in range(10)]+ all_lines = [*embedded, {"id": "cnli_10", "object": "credit_note_line_item"}]++ def script(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/v1/credit_notes/cn_1/lines":+ return ScriptedResponse(json={"object": "list", "data": all_lines, "has_more": False, "url": request.path})+ lines = {"object": "list", "data": embedded, "has_more": True, "url": "/v1/credit_notes/cn_1/lines"}+ credit_note = {"id": "cn_1", "object": "credit_note", "created": 1700000000, "lines": lines}+ return ScriptedResponse(json={"object": "list", "data": [credit_note], "has_more": False, "url": request.path})++ config = StripeSourceConfig(auth_method=StripeAuthMethodConfig(selection="api_key", stripe_secret_key="sk_test"))+ result = SourceDriver(StripeSource(), config).run(CREDIT_NOTE_RESOURCE_NAME, script)++ assert result.raised is None+ rows = [row for table in result.rows for row in table.to_pylist()]+ lines_column = orjson.loads(rows[0]["lines"])+ assert [line["id"] for line in lines_column["data"]] == [line["id"] for line in all_lines]+ assert lines_column["has_more"] is False++[email protected]([(INVOICE_RESOURCE_NAME,), (CREDIT_NOTE_RESOURCE_NAME,)])+def test_first_page_error_of_a_line_expanding_listing_is_cleaned(endpoint):+ redacted = "rk_live_" + "*" * 80 + "abcd"+ message = f"The provided key '{redacted}' does not have the required permissions for this endpoint"+ forbidden = ScriptedResponse(status=403, json={"error": {"type": "invalid_request_error", "message": message}})++ config = StripeSourceConfig(auth_method=StripeAuthMethodConfig(selection="api_key", stripe_secret_key="sk_test"))+ result = SourceDriver(StripeSource(), config).run(endpoint, [forbidden])++ assert isinstance(result.raised, stripe_lib.PermissionError)+ assert "rk_live_***abcd" in str(result.raised)++@parameterized.expand([("switch_off", False, ["cus_credit"]), ("switch_on", True, ["cus_credit", "cus_zero"])])def test_balance_history_switch_decides_which_customers_are_swept(_name, enabled, expected_customers):def script(request: RecordedRequest) -> ScriptedResponse:diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.pyindex 683d4b35ddbb..d1c0cdc988e4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py@@ -113,7 +113,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.conREFUND_RESOURCE_NAME as STRIPE_REFUND_RESOURCE_NAME,SUBSCRIPTION_RESOURCE_NAME as STRIPE_SUBSCRIPTION_RESOURCE_NAME,)-from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.custom import InvoiceListWithAllLines+from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.custom import (+ CreditNoteListWithAllLines,+ InvoiceListWithAllLines,+)from products.warehouse_sources.backend.temporal.data_imports.tests.e2e.queue_replay import (PostgresQueueReplay,replay_v3_consumer,@@ -596,6 +599,7 @@ async def _execute_run(mock.patch.object(PostHogRESTClient, "paginate", mock_paginate_pages),mock.patch.object(ListObject, "auto_paging_iter", return_value=iter(mock_data_response)),mock.patch.object(InvoiceListWithAllLines, "auto_paging_iter", return_value=iter(mock_data_response)),+ mock.patch.object(CreditNoteListWithAllLines, "auto_paging_iter", return_value=iter(mock_data_response)),override_settings(BUCKET_URL=f"s3://{BUCKET_NAME}",BUCKET_PATH=BUCKET_NAME,