Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add dodo payments credit ledger, grants, wallets and emails tables (#119243)

Tom Owersaafbaf5ba116
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 4224c2d42b70..8c9c86a6c3f1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -3883,12 +3883,12 @@ Diffed against: <https://raw.githubusercontent.com/dodopayments/dodopayments-pyt
- [x] `Subscription usage history (GET /subscriptions/{subscription_id}/usage-history)` — reconcile billed usage, free allowances, and chargeable units across subscription billing periods (high)
- [x] `Payout breakup details (GET /payouts/{payout_id}/breakup/details)` — reconcile settlement line items with imported payments, refunds, and disputes (high)
- [x] `Credit entitlement balances (GET /credit-entitlements/{credit_entitlement_id}/balances)` — measure remaining customer credits and overages beyond the entitlement catalog (high)
-- [ ] `Credit entitlement ledger (GET /credit-entitlements/{credit_entitlement_id}/balances/{customer_id}/ledger)` — track credit consumption and balance changes by customer and entitlement (high)
-- [ ] `Credit entitlement grants (GET /credit-entitlements/{credit_entitlement_id}/balances/{customer_id}/grants)` — analyze granted credits, expiry, and remaining quantities by customer (medium)
-- [ ] `Customer wallets (GET /customers/{customer_id}/wallets)` — measure customer wallet balances and prepaid funds by currency (medium)
+- [x] `Credit entitlement ledger (GET /credit-entitlements/{credit_entitlement_id}/balances/{customer_id}/ledger)` — track credit consumption and balance changes by customer and entitlement (high)
+- [x] `Credit entitlement grants (GET /credit-entitlements/{credit_entitlement_id}/balances/{customer_id}/grants)` — analyze granted credits, expiry, and remaining quantities by customer (medium)
+- [x] `Customer wallets (GET /customers/{customer_id}/wallets)` — measure customer wallet balances and prepaid funds by currency (medium)
- [ ] `Customer wallet ledger entries (GET /customers/{customer_id}/wallets/ledger-entries)` — reconcile wallet credits and debits with customer payment activity (medium)
- [ ] `Entitlement grants (GET /entitlements/{id}/grants)` — associate entitlement definitions with customer access grants and subscription references (medium)
-- [ ] `Customer emails (GET /customers/{customer_id}/emails)` — analyze transactional email delivery activity by customer and message type (medium)
+- [x] `Customer emails (GET /customers/{customer_id}/emails)` — analyze transactional email delivery activity by customer and message type (medium)
Note: Extracted every GET path from the official Python SDK operation index and checked selected resource modules and response types. The advertised documentation spec URL returned HTML, so the SDK provided the usable reference. Existing tables cover top-level billing and catalog collections; missing resources need payment, payout, subscription, customer, or entitlement fan-out. Webhooks, payment-method plumbing, invoice PDFs, and duplicate CSV exports were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/canonical_descriptions.py
index f2289050f842..b81a3800752f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/canonical_descriptions.py
@@ -425,4 +425,84 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"last_transaction_at": "Timestamp of the last credit transaction on the balance.",
},
},
+ "credit_entitlement_ledger": {
+ "description": "Credit transactions on each customer's balance for each credit entitlement.",
+ "docs_url": f"{_DOCS_ROOT}/credit-entitlements/list-customer-ledger",
+ "columns": {
+ "id": "Unique identifier of the ledger entry.",
+ "credit_entitlement_id": "Credit entitlement the entry is for.",
+ "customer_id": "Customer the entry belongs to.",
+ "brand_id": "Brand the entry belongs to.",
+ "business_id": "Business the entry belongs to.",
+ "transaction_type": (
+ "Type of transaction, for example credit_added, credit_deducted, credit_expired, "
+ "overage_charged or manual_adjustment."
+ ),
+ "is_credit": "Whether the entry adds credits (true) or removes them (false).",
+ "amount": "Number of credits in the transaction.",
+ "balance_before": "Credit balance before the transaction.",
+ "balance_after": "Credit balance after the transaction.",
+ "overage_before": "Overage before the transaction.",
+ "overage_after": "Overage after the transaction.",
+ "grant_id": "Credit grant the transaction applies to, if any.",
+ "reference_id": "ID of the object that caused the transaction, if any.",
+ "reference_type": "Type of the object that caused the transaction, if any.",
+ "description": "Description of the transaction.",
+ "metadata": "Metadata attached to the entry.",
+ "created_at": "Timestamp when the entry was created.",
+ },
+ },
+ "credit_entitlement_grants": {
+ "description": "Credit grants given to each customer for each credit entitlement.",
+ "docs_url": f"{_DOCS_ROOT}/credit-entitlements/list-customer-grants",
+ "columns": {
+ "id": "Unique identifier of the grant.",
+ "credit_entitlement_id": "Credit entitlement the grant is for.",
+ "customer_id": "Customer the grant belongs to.",
+ "initial_amount": "Number of credits granted.",
+ "remaining_amount": "Number of granted credits not yet used.",
+ "is_expired": "Whether the grant has expired.",
+ "is_rolled_over": "Whether the grant was rolled over from an earlier grant.",
+ "rollover_count": "Number of times the credits were rolled over.",
+ "parent_grant_id": "Grant this grant was rolled over from, if any.",
+ "source_type": "Origin of the grant: subscription, one_time, addon, api or rollover.",
+ "source_id": "ID of the object that created the grant, if any.",
+ "expires_at": "Timestamp when the grant expires, if it expires.",
+ "metadata": "Metadata attached to the grant.",
+ "created_at": "Timestamp when the grant was created.",
+ "updated_at": "Timestamp when the grant was last modified.",
+ },
+ },
+ "customer_wallets": {
+ "description": "Prepaid wallet balance of each customer, one row per currency.",
+ "docs_url": f"{_DOCS_ROOT}/customers/get-customer-wallets",
+ "columns": {
+ "customer_id": "Customer the wallet belongs to.",
+ "currency": "Currency of the wallet, as an ISO 4217 code.",
+ "balance": "Wallet balance, in the currency's smallest unit.",
+ "created_at": "Timestamp when the wallet was created.",
+ "updated_at": "Timestamp when the wallet was last modified.",
+ },
+ },
+ "customer_emails": {
+ "description": "Emails Dodo Payments sent to each customer, with their delivery status.",
+ "docs_url": f"{_DOCS_ROOT}/customers/list-customer-emails",
+ "columns": {
+ "customer_id": "Customer the email was sent to.",
+ "email_log_id": "Unique identifier of the email.",
+ "category": (
+ "Group the email belongs to: payments, refunds, subscriptions, dunning_recovery, entitlements or auth."
+ ),
+ "email_type": "Kind of email, for example payment_successful.",
+ "status": "Delivery status: sent, delivered, failed, complained or blocked.",
+ "failure_code": "Reason the email did not arrive. Null unless the email failed.",
+ "failure_reason": "Explanation of the failure code. Null unless the email failed.",
+ "from": "Address the email was sent from.",
+ "recipient": "Address the email was sent to.",
+ "subject": "Subject line of the email.",
+ "has_preview": "Whether the email has content to show.",
+ "policies": "Actions allowed on the email.",
+ "created_at": "Timestamp when the email was sent.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/dodopayments.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/dodopayments.py
index 2bc8254adef4..ab99a65cb9d1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/dodopayments.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/dodopayments.py
@@ -117,11 +117,9 @@ def get_resource(
if should_use_incremental_field and config.start_param and incremental_start_value:
params[config.start_param] = incremental_start_value
if config.fanout is not None:
- params[config.fanout.resolve_param] = {
- "type": "resolve",
- "resource": config.fanout.parent_name,
- "field": config.fanout.resolve_field,
- }
+ resolve_fields = {config.fanout.resolve_param: config.fanout.resolve_field, **config.extra_resolve_fields}
+ for param, parent_field in resolve_fields.items():
+ params[param] = {"type": "resolve", "resource": config.fanout.parent_name, "field": parent_field}
endpoint_config: Endpoint = {
"path": config.path,
@@ -225,16 +223,15 @@ def dodopayments_source(
initial_paginator_state=initial_paginator_state,
)
else:
- # An incremental child's cursor is a parent field copied onto its rows, so the watermark
- # filters the parent listing, not the child request.
- rest_config["resources"].insert(
- 0,
- get_resource(
- config.fanout.parent_name,
- should_use_incremental_field and bool(config.incremental_fields),
- incremental_start_value,
- ),
- )
+ # An incremental child without a date filter of its own takes its cursor from a parent
+ # field copied onto its rows, so the watermark filters the parent listing instead.
+ filter_parent = should_use_incremental_field and bool(config.incremental_fields) and config.start_param is None
+ ancestor: Optional[str] = config.fanout.parent_name
+ while ancestor is not None:
+ rest_config["resources"].insert(0, get_resource(ancestor, filter_parent, incremental_start_value))
+ filter_parent = False
+ ancestor_fanout = DODOPAYMENTS_ENDPOINTS[ancestor].fanout
+ ancestor = ancestor_fanout.parent_name if ancestor_fanout is not None else None
resources = rest_api_resources(
rest_config,
team_id,
@@ -266,6 +263,8 @@ def dodopayments_source(
# cannot claim rows arrive oldest-first. "desc" holds the watermark back until the sync
# finishes, which is correct whichever order the API actually returns.
sort_mode="desc",
+ # The framework gives a two-level fan-out no resume hook, so it has no cursor to resume from.
+ supports_resume=config.fanout is None or DODOPAYMENTS_ENDPOINTS[config.fanout.parent_name].fanout is None,
**partition_kwargs,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/settings.py
index 6c1062352180..f204627625c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/settings.py
@@ -56,6 +56,9 @@ class DodoPaymentsEndpointConfig:
restated: bool = False
# Per-parent child endpoints, read once for each row of the parent listing.
fanout: DependentEndpointConfig | None = None
+ # Path params bound from the same parent row beside `fanout.resolve_param`, mapped to the
+ # parent field each one reads.
+ extra_resolve_fields: dict[str, str] = field(default_factory=dict)
DODOPAYMENTS_ENDPOINTS: dict[str, DodoPaymentsEndpointConfig] = {
@@ -202,6 +205,65 @@ DODOPAYMENTS_ENDPOINTS: dict[str, DodoPaymentsEndpointConfig] = {
include_from_parent=[],
),
),
+ # Ledger entries are immutable and take their own `start_date` filter, so incremental runs
+ # filter each ledger request and walk the parents in full. The parents are themselves a
+ # fan-out (entitlement -> customer balance), so this chain carries no resume state.
+ "credit_entitlement_ledger": DodoPaymentsEndpointConfig(
+ path="/credit-entitlements/{credit_entitlement_id}/balances/{customer_id}/ledger",
+ primary_keys=["credit_entitlement_id", "customer_id", "id"],
+ start_param="start_date",
+ incremental_fields=[incremental_field("created_at")],
+ fanout=DependentEndpointConfig(
+ child_response_actions=SKIP_MISSING_PARENT,
+ parent_name="credit_entitlement_balances",
+ resolve_param="customer_id",
+ resolve_field="customer_id",
+ include_from_parent=[],
+ ),
+ extra_resolve_fields={"credit_entitlement_id": "credit_entitlement_id"},
+ ),
+ # `remaining_amount` and `is_expired` change after a grant is created and there is no date
+ # filter, so this is full refresh only.
+ "credit_entitlement_grants": DodoPaymentsEndpointConfig(
+ path="/credit-entitlements/{credit_entitlement_id}/balances/{customer_id}/grants",
+ primary_keys=["credit_entitlement_id", "customer_id", "id"],
+ fanout=DependentEndpointConfig(
+ child_response_actions=SKIP_MISSING_PARENT,
+ parent_name="credit_entitlement_balances",
+ resolve_param="customer_id",
+ resolve_field="customer_id",
+ include_from_parent=[],
+ ),
+ extra_resolve_fields={"credit_entitlement_id": "credit_entitlement_id"},
+ ),
+ # One row per currency. The body is `{"items": [...], "total_balance_usd": ...}` with no
+ # pagination, and each wallet already carries `customer_id`.
+ "customer_wallets": DodoPaymentsEndpointConfig(
+ path="/customers/{customer_id}/wallets",
+ primary_keys=["customer_id", "currency"],
+ paginated=False,
+ fanout=DependentEndpointConfig(
+ child_response_actions=SKIP_MISSING_PARENT,
+ parent_name="customers",
+ resolve_param="customer_id",
+ resolve_field="customer_id",
+ include_from_parent=[],
+ ),
+ ),
+ # A customer gets emails long after the customer is created, so the customer listing cannot
+ # window them, and an email's delivery `status` changes after it is sent. Full refresh only.
+ "customer_emails": DodoPaymentsEndpointConfig(
+ path="/customers/{customer_id}/emails",
+ primary_keys=["customer_id", "email_log_id"],
+ fanout=DependentEndpointConfig(
+ child_response_actions=SKIP_MISSING_PARENT,
+ parent_name="customers",
+ resolve_param="customer_id",
+ resolve_field="customer_id",
+ include_from_parent=["customer_id"],
+ parent_field_renames={"customer_id": "customer_id"},
+ ),
+ ),
}
ENDPOINTS = tuple(DODOPAYMENTS_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/tests/test_dodopayments.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/tests/test_dodopayments.py
index 07385c3dab20..8c07c23a3364 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/tests/test_dodopayments.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dodopayments/tests/test_dodopayments.py
@@ -156,11 +156,14 @@ class TestEndpointCatalog:
# Without a server-side filter an "incremental" sync would still fetch every page. A fan-out
# child is filtered through its parent listing instead.
for name, config in DODOPAYMENTS_ENDPOINTS.items():
- filter_config = DODOPAYMENTS_ENDPOINTS[config.fanout.parent_name] if config.fanout else config
- if config.incremental_fields:
- assert filter_config.start_param is not None, name
- if config.fanout:
- assert config.start_param is None, name
+ if not config.incremental_fields:
+ continue
+ filter_config = (
+ DODOPAYMENTS_ENDPOINTS[config.fanout.parent_name]
+ if config.fanout and config.start_param is None
+ else config
+ )
+ assert filter_config.start_param is not None, name
def test_incremental_cursor_matches_the_partition_key(self):
# The cursor and the partition key must both be creation timestamps that never change.
@@ -215,6 +218,25 @@ class TestFanout:
{"id": "bal_1", "credit_entitlement_id": "cde_1", "customer_id": "cus_1", "balance": "10"},
{"id": "bal_1", "credit_entitlement_id": "cde_1", "customer_id": "cus_1", "balance": "10"},
),
+ (
+ "customer_emails",
+ {"customer_id": "cus_1", "created_at": "2024-05-01T00:00:00Z"},
+ "/customers/cus_1/emails",
+ {"email_log_id": "eml_1", "email_type": "payment_successful", "status": "delivered"},
+ {
+ "email_log_id": "eml_1",
+ "email_type": "payment_successful",
+ "status": "delivered",
+ "customer_id": "cus_1",
+ },
+ ),
+ (
+ "customer_wallets",
+ {"customer_id": "cus_1", "created_at": "2024-05-01T00:00:00Z"},
+ "/customers/cus_1/wallets",
+ {"customer_id": "cus_1", "currency": "USD", "balance": 500},
+ {"customer_id": "cus_1", "currency": "USD", "balance": 500},
+ ),
],
)
def test_child_is_read_per_parent_with_parent_fields(
@@ -264,6 +286,46 @@ class TestFanout:
assert result.requests[0].param("created_at_gte") == expected_parent_filter
assert set(result.requests[1].query) == expected_child_query
+ @pytest.mark.parametrize(
+ "endpoint, should_use_incremental_field, expected_child_filter",
+ [
+ # The ledger filters its own requests, so the parent listings stay unfiltered.
+ ("credit_entitlement_ledger", True, "2024-05-01T10:00:00Z"),
+ ("credit_entitlement_grants", False, None),
+ ],
+ )
+ def test_customer_credit_child_binds_both_ids_from_each_balance(
+ self, endpoint: str, should_use_incremental_field: bool, expected_child_filter: str | None
+ ) -> None:
+ child = endpoint.removeprefix("credit_entitlement_")
+ result = _driver().run(
+ endpoint,
+ [
+ _response([{"id": "cde_1"}]),
+ _response(
+ [
+ {"id": "bal_1", "credit_entitlement_id": "cde_1", "customer_id": "cus_1"},
+ {"id": "bal_2", "credit_entitlement_id": "cde_1", "customer_id": "cus_2"},
+ ]
+ ),
+ _response([{"id": "row_1", "credit_entitlement_id": "cde_1", "customer_id": "cus_1"}]),
+ _response([{"id": "row_2", "credit_entitlement_id": "cde_1", "customer_id": "cus_2"}]),
+ ],
+ should_use_incremental_field=should_use_incremental_field,
+ db_incremental_field_last_value=datetime(2024, 5, 1, 10, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/credit-entitlements",
+ "/credit-entitlements/cde_1/balances",
+ f"/credit-entitlements/cde_1/balances/cus_1/{child}",
+ f"/credit-entitlements/cde_1/balances/cus_2/{child}",
+ ]
+ assert [set(request.query) for request in result.requests[:2]] == [{"page_size", "page_number"}] * 2
+ assert [request.param("start_date") for request in result.requests[2:]] == [expected_child_filter] * 2
+ assert [row["id"] for row in result.rows] == ["row_1", "row_2"]
+
def test_paginated_child_walks_every_page_of_each_parent(self) -> None:
def periods(count: int, prefix: str) -> ScriptedResponse:
return _response([{"start_date": f"{prefix}{i}", "meters": []} for i in range(count)])