:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add paymongo payout transactions, subscriptions and invoices (#124779)
Tom Owers2ab7d127682e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex ce326f0798b6..9fade89394a6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9220,10 +9220,10 @@ Today (5): `links`, `payments`, `payouts`, `refunds`, `webhooks`Diffed against: <https://docs.paymongo.com/llms.txt>-- [ ] `Payout Transactions (GET /v1/payouts/{id}/transactions)` — reconcile payouts against payments, refunds, disputes, and other settlement entries (high)+- [x] `Payout Transactions (GET /v1/payouts/{id}/transactions)` — reconcile payouts against payments, refunds, disputes, and other settlement entries (high)- [ ] `Net Revenue (GET /v2/analytics/net_revenue; GET /v2/analytics/net_revenue_breakdown)` — measure revenue after fees, taxes, disputes, and adjustments (high)-- [ ] `Subscriptions (GET /v1/subscriptions)` — measure recurring revenue, subscription growth, and churn (high)-- [ ] `Invoices (GET /v1/invoices)` — track unpaid, overdue, and paid invoices for receivables analysis (high)+- [x] `Subscriptions (GET /v1/subscriptions)` — measure recurring revenue, subscription growth, and churn (high)+- [x] `Invoices (GET /v1/invoices)` — track unpaid, overdue, and paid invoices for receivables analysis (high)- [ ] `Customers (GET /v1/customers)` — analyze customer cohorts and connect customer records to subscription activity (medium)- [ ] `Subscription Plans (GET /v1/subscriptions/plans)` — compare subscription pricing, billing intervals, and plan performance (medium)- [ ] `Ledger Transactions (GET /v1/ledger_transactions)` — analyze accounting transactions and their debit and credit entries (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.pyindex 41914a348063..9893db7e1951 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py@@ -53,4 +53,43 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"status": "Whether the webhook is enabled or disabled.",},},+ "payout_transactions": {+ "description": "Payments, refunds, disputes, and adjustments settled in each payout.",+ "docs_url": "https://docs.paymongo.com/reference/getpayouttransactions",+ "columns": {+ "id": "Transaction identifier, such as a payment, refund, or dispute ID.",+ "_payouts_id": "Identifier of the payout this transaction belongs to.",+ "type": "Transaction type, such as payment, refund, dispute, or adjustment.",+ "amount": "Transaction amount in the currency's smallest unit.",+ "fee": "Transaction fee, for payments.",+ "net_amount": "Transaction amount after fees.",+ "balance_transaction_id": "Identifier of the associated balance transaction.",+ "created_at": "Time the transaction was created, in UTC.",+ },+ },+ "subscriptions": {+ "description": "Recurring billing subscriptions that link customers to plans.",+ "docs_url": "https://docs.paymongo.com/reference/subscription-resource",+ "columns": {+ "id": "PayMongo subscription identifier.",+ "customer_id": "Identifier of the subscribed customer.",+ "plan": "Plan the customer subscribes to, with amount and billing interval.",+ "status": "Current subscription status.",+ "latest_invoice": "Most recent invoice generated for the subscription.",+ "next_billing_schedule": "Date of the next billing cycle.",+ },+ },+ "invoices": {+ "description": "Sales invoices sent to customers, with amounts due, payment status, and due dates.",+ "docs_url": "https://docs.paymongo.com/reference/list_invoices-2",+ "columns": {+ "id": "PayMongo invoice identifier.",+ "number": "Invoice number shown to the customer.",+ "status": "Invoice status, such as draft, open, paid, overdue, or void.",+ "amount_due": "Amount still due, in the currency's smallest unit.",+ "amount_paid": "Amount already paid, in the currency's smallest unit.",+ "due_at": "Time the invoice payment is due.",+ "created_at": "Time the invoice was created.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.pyindex 2e8443a90397..7d247e4f72a5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py@@ -104,14 +104,26 @@ def get_resource(name: str) -> EndpointResource:}else:endpoint["params"] = {"limit": PAGE_SIZE}- endpoint["paginator"] = PaymongoCursorPaginator()+ endpoint["paginator"] = PaymongoCursorPaginator(cursor_param=settings.cursor_param)+ resource: EndpointResource = {+ "name": name,+ "endpoint": endpoint,+ "data_map": normalize_row,+ "primary_key": list(settings.primary_keys),+ }if name == "refunds":endpoint["params"] = {"payment_id": {"type": "resolve", "resource": "payments", "field": "id"},"data.attributes.limit": PAGE_SIZE,}endpoint["paginator"] = PaymongoCursorPaginator(cursor_param="data.attributes.after")- return {"name": name, "endpoint": endpoint, "data_map": normalize_row, "primary_key": list(settings.primary_keys)}+ elif name == "payout_transactions":+ endpoint["params"] = {+ "payout_id": {"type": "resolve", "resource": "payouts", "field": "id"},+ "limit": PAGE_SIZE,+ }+ resource["include_from_parent"] = ["id"]+ return resourcedef pull_rows(@@ -124,15 +136,16 @@ def pull_rows(def save_state(paginator_state: dict[str, Any] | None) -> None:manager.save_state(PaymongoResumeConfig(paginator_state=paginator_state, finished=paginator_state is None))+ settings = schema_for_resource(ENDPOINTS, inputs.schema_name)auth = PaymongoAuth(username=api_key, password="")with make_tracked_session(redact_values=auth.secret_values(), capture=inputs.schema_name != "webhooks") as session:config: RESTAPIConfig = {- "client": {"base_url": BASE_URL, "auth": auth, "session": session, "request_timeout": 30},+ "client": {"base_url": settings.base_url, "auth": auth, "session": session, "request_timeout": 30},"resources": [get_resource(inputs.schema_name)],}initial_state = state.paginator_state if state else None- if inputs.schema_name == "refunds":- config["resources"].insert(0, get_resource("payments"))+ if settings.parent:+ config["resources"].insert(0, get_resource(settings.parent))resources = rest_api_resources(config,inputs.team_id,@@ -141,7 +154,7 @@ def pull_rows(resume_hook=save_state,initial_paginator_state=initial_state,)- resource = next(resource for resource in resources if resource.name == "refunds")+ resource = next(resource for resource in resources if resource.name == inputs.schema_name)else:resource = rest_api_resource(config,@@ -227,23 +240,35 @@ class PaymongoClient:def __init__(self, api_key: str) -> None:self.auth = PaymongoAuth(username=api_key, password="")- def request(self, method: str, path: str, **kwargs: Any) -> Response:+ def request(self, method: str, path: str, base_url: str = BASE_URL, **kwargs: Any) -> Response:with make_tracked_session(redact_values=self.auth.secret_values(), capture=False) as session:- response = session.request(method, BASE_URL + path, auth=self.auth, timeout=30, **kwargs)+ response = session.request(method, base_url + path, auth=self.auth, timeout=30, **kwargs)response.raise_for_status()return responsedef validate_credentials(self, schema_name: str | None) -> tuple[bool, str | None]:name = schema_name or "payments"endpoint = schema_for_resource(ENDPOINTS, name)- path = "payments" if name == "refunds" else endpoint.path+ probe = ENDPOINTS[endpoint.parent] if endpoint.parent else endpointtry:- response = self.request("GET", path, params={} if endpoint.pagination == "single" else {"limit": 1})- if name == "refunds" and response.json()["data"]:+ response = self.request(+ "GET",+ probe.path,+ base_url=probe.base_url,+ params={} if probe.pagination == "single" else {"limit": 1},+ )+ parents = response.json()["data"] if endpoint.parent else []+ if name == "refunds" and parents:self.request("GET","refunds",- params={"data.attributes.payment_id": response.json()["data"][0]["id"], "data.attributes.limit": 1},+ params={"data.attributes.payment_id": parents[0]["id"], "data.attributes.limit": 1},+ )+ elif name == "payout_transactions" and parents:+ self.request(+ "GET",+ endpoint.path.format(payout_id=quote(parents[0]["id"], safe="")),+ params={"limit": 1},)except HTTPError as error:status = error.response.status_code if error.response is not None else Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.pyindex bebc6bded30a..d0bf8b431bcb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py@@ -5,6 +5,7 @@ from posthog.dataclasses import frozenfrom products.warehouse_sources.backend.types import IncrementalFieldBASE_URL = "https://api.paymongo.com/v1/"+INVOICES_BASE_URL = "https://invoices-api.paymongo.com/v1/"PAGE_SIZE = 10WEBHOOK_EVENTS = ("payment.paid", "payment.failed", "payment.refunded", "payment.refund.updated")@@ -15,14 +16,26 @@ class PaymongoEndpoint:pagination: Literal["after", "payout", "single"] = "after"primary_keys: tuple[str, ...] = ("id",)partition_key: str = "created_at"+ base_url: str = BASE_URL+ cursor_param: str = "after"+ parent: str | None = NoneENDPOINTS = {"payments": PaymongoEndpoint(path="payments"),- "refunds": PaymongoEndpoint(path="refunds?data.attributes.payment_id={payment_id}"),+ "refunds": PaymongoEndpoint(path="refunds?data.attributes.payment_id={payment_id}", parent="payments"),"links": PaymongoEndpoint(path="payment_links", pagination="single"),"payouts": PaymongoEndpoint(path="payouts", pagination="payout"),"webhooks": PaymongoEndpoint(path="webhooks"),+ # PayMongo documents no uniqueness for transaction IDs across payouts, so key them per payout.+ "payout_transactions": PaymongoEndpoint(+ path="payouts/{payout_id}/transactions",+ pagination="payout",+ primary_keys=("_payouts_id", "id"),+ parent="payouts",+ ),+ "subscriptions": PaymongoEndpoint(path="subscriptions"),+ "invoices": PaymongoEndpoint(path="invoices", base_url=INVOICES_BASE_URL, cursor_param="starting_after"),}# Creation-time filters cannot capture later refunds or status changes, and remain unverified.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.pyindex 99fcd8be1ffb..c81735e7f553 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py@@ -77,7 +77,7 @@ class PaymongoSource(ResumableSource[PaymongoSourceConfig, PaymongoResumeConfig]label="PayMongo",iconPath="/static/services/paymongo.png",docsUrl="https://posthog.com/docs/cdp/sources/paymongo",- caption="Import payments, refunds, payment links, payouts, and webhook settings from PayMongo.",+ caption="Import payments, refunds, payment links, payouts and their transactions, subscriptions, invoices, and webhook settings from PayMongo.",fields=cast(list[FieldType],[diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.pyindex b3f8b6f03141..dfccdc59799b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py@@ -94,15 +94,16 @@ def respond(http: MagicMock, bodies: list[dict[str, Any]], statuses: list[int] |@pytest.mark.parametrize(- "name,metadata",+ "name,metadata,url,cursor_param",[- ("payments", {"has_more": True}),- ("webhooks", {"has_more": True}),- ("payouts", {"pagination": {"next_cursor": "pay_1"}}),+ ("payments", {"has_more": True}, "https://api.paymongo.com/v1/payments", "after"),+ ("webhooks", {"has_more": True}, "https://api.paymongo.com/v1/webhooks", "after"),+ ("payouts", {"pagination": {"next_cursor": "pay_1"}}, "https://api.paymongo.com/v1/payouts", "after"),+ ("invoices", {"has_more": True}, "https://invoices-api.paymongo.com/v1/invoices", "starting_after"),],)def test_pagination_auth_normalization_and_checkpoint(- http: MagicMock, manager: MagicMock, name: str, metadata: dict[str, Any]+ http: MagicMock, manager: MagicMock, name: str, metadata: dict[str, Any], url: str, cursor_param: str) -> None:sent = respond(http,@@ -120,8 +121,9 @@ def test_pagination_auth_normalization_and_checkpoint(assert list(pages) == []assert manager.save_state.call_args.args[0].finishedassert sent[0].headers["Authorization"] == "Basic c2tfdGVzdF9mYWtlOg=="+ assert (sent[0].url or "").split("?")[0] == urlassert parse_qs(urlsplit(sent[0].url or "").query) == {"limit": ["10"]}- assert parse_qs(urlsplit(sent[1].url or "").query) == {"limit": ["10"], "after": ["pay_1"]}+ assert parse_qs(urlsplit(sent[1].url or "").query) == {"limit": ["10"], cursor_param: ["pay_1"]}@pytest.mark.parametrize("finished", [False, True])@@ -160,6 +162,35 @@ def test_refund_fanout_keeps_payment_and_child_cursors() -> None:]+def test_payout_transaction_fanout_keys_rows_by_payout() -> None:+ result = driver().run(+ "payout_transactions",+ [+ ScriptedResponse(json={"data": [row("po_one")], "pagination": {"next_cursor": "po_one"}}),+ ScriptedResponse(+ json={"data": [row("pay_one", payout_id="po_one")], "pagination": {"next_cursor": "pay_one"}}+ ),+ ScriptedResponse(json={"data": [row("ref_one", payout_id="po_one")], "pagination": {"next_cursor": None}}),+ ScriptedResponse(json={"data": [row("po_two")], "pagination": {"next_cursor": None}}),+ ScriptedResponse(json={"data": [row("pay_one", payout_id="po_two")], "pagination": {"next_cursor": None}}),+ ],+ )+ assert result.raised is None+ assert [(item["_payouts_id"], item["id"]) for item in result.rows] == [+ ("po_one", "pay_one"),+ ("po_one", "ref_one"),+ ("po_two", "pay_one"),+ ]+ assert result.paths == [+ "/v1/payouts",+ "/v1/payouts/po_one/transactions",+ "/v1/payouts/po_one/transactions",+ "/v1/payouts",+ "/v1/payouts/po_two/transactions",+ ]+ assert result.queries[2] == {"limit": ("10",), "after": ("pay_one",)}++@pytest.mark.parametrize("body,message", [({"data": [row("pay_same")], "has_more": True}, "repeated"), ({"unexpected": []}, "data")])@@ -388,22 +419,38 @@ def test_webhook_info_finds_matching_url_on_later_page(exists: bool) -> None:assert "fake-secret" not in repr(info)[email protected]("status", [200, 403])-def test_refund_permissions_probe_child_endpoint(status: int) -> None:+ "status,schema,parent_path,child_path,child_query",+ [+ (+ status,+ "refunds",+ "/v1/payments",+ "/v1/refunds",+ {"data.attributes.payment_id": ("pay_one",), "data.attributes.limit": ("1",)},+ )+ for status in (200, 403)+ ]+ + [+ (status, "payout_transactions", "/v1/payouts", "/v1/payouts/pay_one/transactions", {"limit": ("1",)})+ for status in (200, 403)+ ],+)+def test_child_permissions_probe_child_endpoint(+ status: int, schema: str, parent_path: str, child_path: str, child_query: dict[str, tuple[str, ...]]+) -> None:with scripted_network([ScriptedResponse(json={"data": [row("pay_one")]}),ScriptedResponse(status=status, json={"data": []}),]) as network:- valid, message = PaymongoClient("fake").validate_credentials("refunds")+ valid, message = PaymongoClient("fake").validate_credentials(schema)assert valid is (status == 200)if status == 200:assert message is Noneelse:assert message is not None and "permissions" in message- assert network.requests_log[1].path == "/v1/refunds"- assert network.requests_log[1].query == {- "data.attributes.payment_id": ("pay_one",),- "data.attributes.limit": ("1",),- }+ assert network.requests_log[0].path == parent_path+ assert network.requests_log[1].path == child_path+ assert network.requests_log[1].query == child_query