Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(stripe): fetch every credit note line (#118546)

Daniel RCb4bc4bfb615e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
index bbbfd2d765c3..61a1ce9767ff 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/custom.py
@@ -1,10 +1,10 @@
import threading
from collections.abc import Callable, Iterator
from concurrent.futures import ThreadPoolExecutor
-from typing import Optional, ParamSpec, TypeVar
+from typing import Any, ClassVar, Generic, Optional, ParamSpec, TypeVar
import stripe as stripe_lib
-from stripe import Invoice, InvoiceLineItem, InvoiceService, ListObject, StripeClient
+from stripe import CreditNote, CreditNoteLineItem, Invoice, InvoiceLineItem, ListObject, StripeClient
from structlog.types import FilteringBoundLogger
from products.warehouse_sources.backend.temporal.data_imports.sources.common.request_pacer import (
@@ -21,27 +21,31 @@ LINE_REQUESTS_PER_SECOND = 20.0
_P = ParamSpec("_P")
_T = TypeVar("_T")
+_ParentT = TypeVar("_ParentT", Invoice, CreditNote)
+_LineT = TypeVar("_LineT", InvoiceLineItem, CreditNoteLineItem)
# Receives the Retry-After seconds from a 429, or None when Stripe sent none.
RateLimitCallback = Callable[[Optional[float]], None]
ClientFactory = Callable[[RateLimitCallback], StripeClient]
-class InvoiceListWithAllLines:
- """Invoice listing that expands each invoice's line items.
+class _ListWithAllLines(Generic[_ParentT, _LineT]):
+ """Listing that expands each object's embedded `lines` list.
- The list endpoint embeds at most 10 lines per invoice, so an invoice above that needs one extra
+ The list endpoint embeds at most 10 lines per object, so an object above that needs one extra
`/lines` call. Those calls run on a small thread pool while the next page downloads. Each worker
thread builds its own client from the factory: Stripe's RequestsClient hands a caller-supplied
session to every thread, `requests.Session` is not documented as thread-safe, and the retrying
client keeps per-request state. The page client comes from the same factory, so a 429 on
- either leg slows the whole pool. Invoices are yielded in list order because the
+ either leg slows the whole pool. Objects are yielded in list order because the
`starting_after` resume cursor is the last yielded id.
"""
+ object_name: ClassVar[str]
+
def __init__(
self,
- params: InvoiceService.ListParams,
+ params: Any,
logger: FilteringBoundLogger,
client_factory: ClientFactory,
concurrency: int = LINE_FETCH_CONCURRENCY,
@@ -54,60 +58,68 @@ class InvoiceListWithAllLines:
self._pacer = RequestPacer(requests_per_second)
self._thread_clients = threading.local()
- def auto_paging_iter(self) -> Iterator[Invoice]:
+ def _list_page(self, client: StripeClient) -> ListObject[_ParentT]:
+ raise NotImplementedError
+
+ def _list_lines(self, client: StripeClient, object_id: str) -> ListObject[_LineT]:
+ raise NotImplementedError
+
+ def _attach_lines(self, obj: _ParentT, lines: list[_LineT]) -> None:
+ raise NotImplementedError
+
+ def auto_paging_iter(self) -> Iterator[_ParentT]:
page_client = self._client_factory(self._pacer.throttled)
- page: ListObject[Invoice] = page_client.invoices.list(params=self.params)
+ page = self._list_page(page_client)
total_line_calls = 0
- invoice_count = 0
+ object_count = 0
# The next-page fetch is submitted first, so the extra worker picks it up at once and the
# line calls fill the others.
- pool = ThreadPoolExecutor(max_workers=self._concurrency + 1, thread_name_prefix="stripe-invoice-lines")
+ pool = ThreadPoolExecutor(
+ max_workers=self._concurrency + 1, thread_name_prefix=f"stripe-{self.object_name.replace(' ', '-')}-lines"
+ )
try:
while not page.is_empty:
next_page = submit_with_context(pool, page.next_page)
line_futures = [
- submit_with_context(pool, self._fetch_lines, invoice.id)
- if invoice.lines.has_more and invoice.id
- else None
- for invoice in page.data
+ submit_with_context(pool, self._fetch_lines, obj.id) if obj.lines.has_more and obj.id else None
+ for obj in page.data
]
- for invoice, future in zip(page.data, line_futures):
- if invoice.lines.has_more and future is None:
- self.logger.warning(f"Invoice {invoice.id} has no id")
+ for obj, future in zip(page.data, line_futures):
+ if obj.lines.has_more and future is None:
+ self.logger.warning(f"Stripe {self.object_name} {obj.id} has no id")
continue
if future is not None:
all_lines = future.result()
if all_lines is not None:
total_line_calls += len(all_lines) // 100 + 1
- invoice.lines.data = all_lines
- invoice.lines.has_more = False
- invoice.lines.url = None # type: ignore
+ self._attach_lines(obj, all_lines)
+ obj.lines.has_more = False
+ obj.lines.url = None # type: ignore
- yield invoice
+ yield obj
- invoice_count += 1
- if invoice_count % 10000 == 0:
- self.logger.info(f"Stripe: processed {invoice_count} invoices")
+ object_count += 1
+ if object_count % 10000 == 0:
+ self.logger.info(f"Stripe: processed {object_count} {self.object_name}s")
page = next_page.result()
finally:
# The consumer may close the generator early; never block on requests still in flight.
pool.shutdown(wait=False, cancel_futures=True)
- self.logger.debug(f"Stripe: made {total_line_calls} calls for invoice line items")
+ self.logger.debug(f"Stripe: made {total_line_calls} calls for {self.object_name} line items")
- def _fetch_lines(self, invoice_id: str) -> Optional[list[InvoiceLineItem]]:
+ def _fetch_lines(self, object_id: str) -> Optional[list[_LineT]]:
self._pacer.wait_turn()
try:
- line_items = self._thread_client().invoices.line_items.list(invoice=invoice_id, params={"limit": 100})
- return list(line_items.auto_paging_iter())
+ return list(self._list_lines(self._thread_client(), object_id).auto_paging_iter())
except stripe_lib.InvalidRequestError as e:
if getattr(e, "code", None) != "resource_missing":
raise
- self.logger.debug(f"Stripe: invoice {invoice_id} no longer exists, yielding with partial lines")
+ self.logger.debug(f"Stripe: {self.object_name} {object_id} no longer exists, yielding with partial lines")
return None
def _thread_client(self) -> StripeClient:
@@ -116,3 +128,29 @@ class InvoiceListWithAllLines:
client = self._client_factory(self._pacer.throttled)
self._thread_clients.client = client
return client
+
+
+class InvoiceListWithAllLines(_ListWithAllLines[Invoice, InvoiceLineItem]):
+ object_name = "invoice"
+
+ def _list_page(self, client: StripeClient) -> ListObject[Invoice]:
+ return client.invoices.list(params=self.params)
+
+ def _list_lines(self, client: StripeClient, object_id: str) -> ListObject[InvoiceLineItem]:
+ return client.invoices.line_items.list(invoice=object_id, params={"limit": 100})
+
+ def _attach_lines(self, obj: Invoice, lines: list[InvoiceLineItem]) -> None:
+ obj.lines.data = lines
+
+
+class CreditNoteListWithAllLines(_ListWithAllLines[CreditNote, CreditNoteLineItem]):
+ object_name = "credit note"
+
+ def _list_page(self, client: StripeClient) -> ListObject[CreditNote]:
+ return client.credit_notes.list(params=self.params)
+
+ def _list_lines(self, client: StripeClient, object_id: str) -> ListObject[CreditNoteLineItem]:
+ return client.credit_notes.line_items.list(credit_note=object_id, params={"limit": 100})
+
+ def _attach_lines(self, obj: CreditNote, lines: list[CreditNoteLineItem]) -> None:
+ obj.lines.data = lines
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
index ec116cf79c4d..f8a0e0204d4c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
@@ -91,6 +91,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con
)
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.custom import (
ClientFactory,
+ CreditNoteListWithAllLines,
InvoiceListWithAllLines,
RateLimitCallback,
)
@@ -350,6 +351,27 @@ def _call_stripe(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
raise
+def _iter_cleaning_errors(objects: Iterable[Any]) -> Iterator[Any]:
+ """Iterate a Stripe listing and clean errors raised while it pages, as `_call_stripe` does.
+
+ A listing wrapper such as `InvoiceListWithAllLines` makes its first request only when it is
+ iterated, so `_call_stripe` around the method call never sees that request's error.
+ """
+ iterator = iter(objects)
+ try:
+ while True:
+ try:
+ obj = _call_stripe(next, iterator)
+ except StopIteration:
+ return
+ yield obj
+ finally:
+ # Close the inner generator now, so a listing wrapper shuts down its thread pool.
+ close = getattr(iterator, "close", None)
+ if close is not None:
+ close()
+
+
def _is_stripe_resource_missing_error(error: stripe_lib.StripeError) -> bool:
"""True for Stripe's ``resource_missing`` 404 — e.g. a customer deleted between when we
listed it and when we fetched its nested resources. The object is genuinely gone, so the
@@ -734,8 +756,8 @@ def _build_resources(
checks). Nested resources carry their parent on `.parent`, so callers can derive the
nested→parent linkage without restating it elsewhere.
- `logger` and `client_factory` are only consumed by InvoiceListWithAllLines; pass None when the
- caller doesn't need the wrapped invoice expansion (e.g. validation, which just probes the list
+ `logger` and `client_factory` are only consumed by the invoice and credit note line expansion;
+ pass None when the caller doesn't need it (e.g. validation, which just probes the list
endpoint). The factory gives each of its worker threads a client of its own.
"""
return {
@@ -771,7 +793,13 @@ def _build_resources(
"expand": ["data.discounts", "data.items.data.discounts"],
},
),
- CREDIT_NOTE_RESOURCE_NAME: StripeResource(method=client.credit_notes.list),
+ CREDIT_NOTE_RESOURCE_NAME: StripeResource(
+ method=(
+ (lambda params: CreditNoteListWithAllLines(params, logger, client_factory=client_factory)) # type: ignore
+ if logger is not None and client_factory is not None
+ else client.credit_notes.list
+ )
+ ),
COUPON_RESOURCE_NAME: StripeResource(method=client.coupons.list),
# The one nested resource that reads its parent from the warehouse. Its skip predicate
# answers from the parent row itself, so a run is dominated by paging the customer
@@ -1106,7 +1134,7 @@ def get_rows(
else:
def _list_resource(params: dict[str, Any]) -> Any:
- return _call_stripe(resource.method, params=params).auto_paging_iter()
+ return _iter_cleaning_errors(_call_stripe(resource.method, params=params).auto_paging_iter())
list_params = {**default_params, **resource.params, **resume_params}
cursor_restarted = False
@@ -1162,7 +1190,7 @@ def get_rows(
f"created[lt]": db_incremental_field_earliest_value,
},
)
- yield from _batch_and_yield(stripe_objects.auto_paging_iter(), batcher)
+ yield from _batch_and_yield(_iter_cleaning_errors(stripe_objects.auto_paging_iter()), batcher)
# check for any objects more than the maximum object we already have
if db_incremental_field_last_value is not None:
@@ -1177,7 +1205,7 @@ def get_rows(
},
)
yield from _batch_and_yield(
- stripe_objects.auto_paging_iter(),
+ _iter_cleaning_errors(stripe_objects.auto_paging_iter()),
batcher,
incremental_field_name=incremental_field_name,
stop_at_or_before=_coerce_incremental_cursor(db_incremental_field_last_value),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
index 2f8e71a08ca7..4a833fe43d0d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
@@ -46,6 +46,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con
BILLING_METER_RESOURCE_NAME,
CHARGE_RESOURCE_NAME,
CHECKOUT_SESSION_RESOURCE_NAME,
+ CREDIT_NOTE_RESOURCE_NAME,
CUSTOMER_BALANCE_TRANSACTION_RESOURCE_NAME,
CUSTOMER_PAYMENT_METHOD_HISTORY_RESOURCE_NAME,
CUSTOMER_PAYMENT_METHOD_RESOURCE_NAME,
@@ -61,6 +62,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con
HISTORY_PREVIOUS_ATTRIBUTES_COLUMN,
HISTORY_SNAPSHOT_EVENT_TYPE,
INVOICE_PAYMENT_RESOURCE_NAME,
+ INVOICE_RESOURCE_NAME,
PAYMENT_INTENT_RESOURCE_NAME,
PAYMENT_LINK_RESOURCE_NAME,
PAYMENT_METHOD_HISTORY_MAPPING_KEY,
@@ -971,6 +973,40 @@ class TestInvoiceListWithAllLines:
)
+def test_credit_note_rows_carry_lines_past_the_embedded_page():
+ embedded = [{"id": f"cnli_{i}", "object": "credit_note_line_item"} for i in range(10)]
+ all_lines = [*embedded, {"id": "cnli_10", "object": "credit_note_line_item"}]
+
+ def script(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/v1/credit_notes/cn_1/lines":
+ return ScriptedResponse(json={"object": "list", "data": all_lines, "has_more": False, "url": request.path})
+ lines = {"object": "list", "data": embedded, "has_more": True, "url": "/v1/credit_notes/cn_1/lines"}
+ credit_note = {"id": "cn_1", "object": "credit_note", "created": 1700000000, "lines": lines}
+ return ScriptedResponse(json={"object": "list", "data": [credit_note], "has_more": False, "url": request.path})
+
+ config = StripeSourceConfig(auth_method=StripeAuthMethodConfig(selection="api_key", stripe_secret_key="sk_test"))
+ result = SourceDriver(StripeSource(), config).run(CREDIT_NOTE_RESOURCE_NAME, script)
+
+ assert result.raised is None
+ rows = [row for table in result.rows for row in table.to_pylist()]
+ lines_column = orjson.loads(rows[0]["lines"])
+ assert [line["id"] for line in lines_column["data"]] == [line["id"] for line in all_lines]
+ assert lines_column["has_more"] is False
+
+
[email protected]([(INVOICE_RESOURCE_NAME,), (CREDIT_NOTE_RESOURCE_NAME,)])
+def test_first_page_error_of_a_line_expanding_listing_is_cleaned(endpoint):
+ redacted = "rk_live_" + "*" * 80 + "abcd"
+ message = f"The provided key '{redacted}' does not have the required permissions for this endpoint"
+ forbidden = ScriptedResponse(status=403, json={"error": {"type": "invalid_request_error", "message": message}})
+
+ config = StripeSourceConfig(auth_method=StripeAuthMethodConfig(selection="api_key", stripe_secret_key="sk_test"))
+ result = SourceDriver(StripeSource(), config).run(endpoint, [forbidden])
+
+ assert isinstance(result.raised, stripe_lib.PermissionError)
+ assert "rk_live_***abcd" in str(result.raised)
+
+
@parameterized.expand([("switch_off", False, ["cus_credit"]), ("switch_on", True, ["cus_credit", "cus_zero"])])
def test_balance_history_switch_decides_which_customers_are_swept(_name, enabled, expected_customers):
def script(request: RecordedRequest) -> ScriptedResponse:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
index 683d4b35ddbb..d1c0cdc988e4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
@@ -113,7 +113,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.con
REFUND_RESOURCE_NAME as STRIPE_REFUND_RESOURCE_NAME,
SUBSCRIPTION_RESOURCE_NAME as STRIPE_SUBSCRIPTION_RESOURCE_NAME,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.custom import InvoiceListWithAllLines
+from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.custom import (
+ CreditNoteListWithAllLines,
+ InvoiceListWithAllLines,
+)
from products.warehouse_sources.backend.temporal.data_imports.tests.e2e.queue_replay import (
PostgresQueueReplay,
replay_v3_consumer,
@@ -596,6 +599,7 @@ async def _execute_run(
mock.patch.object(PostHogRESTClient, "paginate", mock_paginate_pages),
mock.patch.object(ListObject, "auto_paging_iter", return_value=iter(mock_data_response)),
mock.patch.object(InvoiceListWithAllLines, "auto_paging_iter", return_value=iter(mock_data_response)),
+ mock.patch.object(CreditNoteListWithAllLines, "auto_paging_iter", return_value=iter(mock_data_response)),
override_settings(
BUCKET_URL=f"s3://{BUCKET_NAME}",
BUCKET_PATH=BUCKET_NAME,