Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add paymongo payout transactions, subscriptions and invoices (#124779)

Tom Owers2ab7d127682e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index ce326f0798b6..9fade89394a6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9220,10 +9220,10 @@ Today (5): `links`, `payments`, `payouts`, `refunds`, `webhooks`
Diffed against: <https://docs.paymongo.com/llms.txt>
-- [ ] `Payout Transactions (GET /v1/payouts/{id}/transactions)` — reconcile payouts against payments, refunds, disputes, and other settlement entries (high)
+- [x] `Payout Transactions (GET /v1/payouts/{id}/transactions)` — reconcile payouts against payments, refunds, disputes, and other settlement entries (high)
- [ ] `Net Revenue (GET /v2/analytics/net_revenue; GET /v2/analytics/net_revenue_breakdown)` — measure revenue after fees, taxes, disputes, and adjustments (high)
-- [ ] `Subscriptions (GET /v1/subscriptions)` — measure recurring revenue, subscription growth, and churn (high)
-- [ ] `Invoices (GET /v1/invoices)` — track unpaid, overdue, and paid invoices for receivables analysis (high)
+- [x] `Subscriptions (GET /v1/subscriptions)` — measure recurring revenue, subscription growth, and churn (high)
+- [x] `Invoices (GET /v1/invoices)` — track unpaid, overdue, and paid invoices for receivables analysis (high)
- [ ] `Customers (GET /v1/customers)` — analyze customer cohorts and connect customer records to subscription activity (medium)
- [ ] `Subscription Plans (GET /v1/subscriptions/plans)` — compare subscription pricing, billing intervals, and plan performance (medium)
- [ ] `Ledger Transactions (GET /v1/ledger_transactions)` — analyze accounting transactions and their debit and credit entries (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py
index 41914a348063..9893db7e1951 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py
@@ -53,4 +53,43 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"status": "Whether the webhook is enabled or disabled.",
},
},
+ "payout_transactions": {
+ "description": "Payments, refunds, disputes, and adjustments settled in each payout.",
+ "docs_url": "https://docs.paymongo.com/reference/getpayouttransactions",
+ "columns": {
+ "id": "Transaction identifier, such as a payment, refund, or dispute ID.",
+ "_payouts_id": "Identifier of the payout this transaction belongs to.",
+ "type": "Transaction type, such as payment, refund, dispute, or adjustment.",
+ "amount": "Transaction amount in the currency's smallest unit.",
+ "fee": "Transaction fee, for payments.",
+ "net_amount": "Transaction amount after fees.",
+ "balance_transaction_id": "Identifier of the associated balance transaction.",
+ "created_at": "Time the transaction was created, in UTC.",
+ },
+ },
+ "subscriptions": {
+ "description": "Recurring billing subscriptions that link customers to plans.",
+ "docs_url": "https://docs.paymongo.com/reference/subscription-resource",
+ "columns": {
+ "id": "PayMongo subscription identifier.",
+ "customer_id": "Identifier of the subscribed customer.",
+ "plan": "Plan the customer subscribes to, with amount and billing interval.",
+ "status": "Current subscription status.",
+ "latest_invoice": "Most recent invoice generated for the subscription.",
+ "next_billing_schedule": "Date of the next billing cycle.",
+ },
+ },
+ "invoices": {
+ "description": "Sales invoices sent to customers, with amounts due, payment status, and due dates.",
+ "docs_url": "https://docs.paymongo.com/reference/list_invoices-2",
+ "columns": {
+ "id": "PayMongo invoice identifier.",
+ "number": "Invoice number shown to the customer.",
+ "status": "Invoice status, such as draft, open, paid, overdue, or void.",
+ "amount_due": "Amount still due, in the currency's smallest unit.",
+ "amount_paid": "Amount already paid, in the currency's smallest unit.",
+ "due_at": "Time the invoice payment is due.",
+ "created_at": "Time the invoice was created.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py
index 2e8443a90397..7d247e4f72a5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py
@@ -104,14 +104,26 @@ def get_resource(name: str) -> EndpointResource:
}
else:
endpoint["params"] = {"limit": PAGE_SIZE}
- endpoint["paginator"] = PaymongoCursorPaginator()
+ endpoint["paginator"] = PaymongoCursorPaginator(cursor_param=settings.cursor_param)
+ resource: EndpointResource = {
+ "name": name,
+ "endpoint": endpoint,
+ "data_map": normalize_row,
+ "primary_key": list(settings.primary_keys),
+ }
if name == "refunds":
endpoint["params"] = {
"payment_id": {"type": "resolve", "resource": "payments", "field": "id"},
"data.attributes.limit": PAGE_SIZE,
}
endpoint["paginator"] = PaymongoCursorPaginator(cursor_param="data.attributes.after")
- return {"name": name, "endpoint": endpoint, "data_map": normalize_row, "primary_key": list(settings.primary_keys)}
+ elif name == "payout_transactions":
+ endpoint["params"] = {
+ "payout_id": {"type": "resolve", "resource": "payouts", "field": "id"},
+ "limit": PAGE_SIZE,
+ }
+ resource["include_from_parent"] = ["id"]
+ return resource
def pull_rows(
@@ -124,15 +136,16 @@ def pull_rows(
def save_state(paginator_state: dict[str, Any] | None) -> None:
manager.save_state(PaymongoResumeConfig(paginator_state=paginator_state, finished=paginator_state is None))
+ settings = schema_for_resource(ENDPOINTS, inputs.schema_name)
auth = PaymongoAuth(username=api_key, password="")
with make_tracked_session(redact_values=auth.secret_values(), capture=inputs.schema_name != "webhooks") as session:
config: RESTAPIConfig = {
- "client": {"base_url": BASE_URL, "auth": auth, "session": session, "request_timeout": 30},
+ "client": {"base_url": settings.base_url, "auth": auth, "session": session, "request_timeout": 30},
"resources": [get_resource(inputs.schema_name)],
}
initial_state = state.paginator_state if state else None
- if inputs.schema_name == "refunds":
- config["resources"].insert(0, get_resource("payments"))
+ if settings.parent:
+ config["resources"].insert(0, get_resource(settings.parent))
resources = rest_api_resources(
config,
inputs.team_id,
@@ -141,7 +154,7 @@ def pull_rows(
resume_hook=save_state,
initial_paginator_state=initial_state,
)
- resource = next(resource for resource in resources if resource.name == "refunds")
+ resource = next(resource for resource in resources if resource.name == inputs.schema_name)
else:
resource = rest_api_resource(
config,
@@ -227,23 +240,35 @@ class PaymongoClient:
def __init__(self, api_key: str) -> None:
self.auth = PaymongoAuth(username=api_key, password="")
- def request(self, method: str, path: str, **kwargs: Any) -> Response:
+ def request(self, method: str, path: str, base_url: str = BASE_URL, **kwargs: Any) -> Response:
with make_tracked_session(redact_values=self.auth.secret_values(), capture=False) as session:
- response = session.request(method, BASE_URL + path, auth=self.auth, timeout=30, **kwargs)
+ response = session.request(method, base_url + path, auth=self.auth, timeout=30, **kwargs)
response.raise_for_status()
return response
def validate_credentials(self, schema_name: str | None) -> tuple[bool, str | None]:
name = schema_name or "payments"
endpoint = schema_for_resource(ENDPOINTS, name)
- path = "payments" if name == "refunds" else endpoint.path
+ probe = ENDPOINTS[endpoint.parent] if endpoint.parent else endpoint
try:
- response = self.request("GET", path, params={} if endpoint.pagination == "single" else {"limit": 1})
- if name == "refunds" and response.json()["data"]:
+ response = self.request(
+ "GET",
+ probe.path,
+ base_url=probe.base_url,
+ params={} if probe.pagination == "single" else {"limit": 1},
+ )
+ parents = response.json()["data"] if endpoint.parent else []
+ if name == "refunds" and parents:
self.request(
"GET",
"refunds",
- params={"data.attributes.payment_id": response.json()["data"][0]["id"], "data.attributes.limit": 1},
+ params={"data.attributes.payment_id": parents[0]["id"], "data.attributes.limit": 1},
+ )
+ elif name == "payout_transactions" and parents:
+ self.request(
+ "GET",
+ endpoint.path.format(payout_id=quote(parents[0]["id"], safe="")),
+ params={"limit": 1},
)
except HTTPError as error:
status = error.response.status_code if error.response is not None else None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py
index bebc6bded30a..d0bf8b431bcb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py
@@ -5,6 +5,7 @@ from posthog.dataclasses import frozen
from products.warehouse_sources.backend.types import IncrementalField
BASE_URL = "https://api.paymongo.com/v1/"
+INVOICES_BASE_URL = "https://invoices-api.paymongo.com/v1/"
PAGE_SIZE = 10
WEBHOOK_EVENTS = ("payment.paid", "payment.failed", "payment.refunded", "payment.refund.updated")
@@ -15,14 +16,26 @@ class PaymongoEndpoint:
pagination: Literal["after", "payout", "single"] = "after"
primary_keys: tuple[str, ...] = ("id",)
partition_key: str = "created_at"
+ base_url: str = BASE_URL
+ cursor_param: str = "after"
+ parent: str | None = None
ENDPOINTS = {
"payments": PaymongoEndpoint(path="payments"),
- "refunds": PaymongoEndpoint(path="refunds?data.attributes.payment_id={payment_id}"),
+ "refunds": PaymongoEndpoint(path="refunds?data.attributes.payment_id={payment_id}", parent="payments"),
"links": PaymongoEndpoint(path="payment_links", pagination="single"),
"payouts": PaymongoEndpoint(path="payouts", pagination="payout"),
"webhooks": PaymongoEndpoint(path="webhooks"),
+ # PayMongo documents no uniqueness for transaction IDs across payouts, so key them per payout.
+ "payout_transactions": PaymongoEndpoint(
+ path="payouts/{payout_id}/transactions",
+ pagination="payout",
+ primary_keys=("_payouts_id", "id"),
+ parent="payouts",
+ ),
+ "subscriptions": PaymongoEndpoint(path="subscriptions"),
+ "invoices": PaymongoEndpoint(path="invoices", base_url=INVOICES_BASE_URL, cursor_param="starting_after"),
}
# Creation-time filters cannot capture later refunds or status changes, and remain unverified.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py
index 99fcd8be1ffb..c81735e7f553 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py
@@ -77,7 +77,7 @@ class PaymongoSource(ResumableSource[PaymongoSourceConfig, PaymongoResumeConfig]
label="PayMongo",
iconPath="/static/services/paymongo.png",
docsUrl="https://posthog.com/docs/cdp/sources/paymongo",
- caption="Import payments, refunds, payment links, payouts, and webhook settings from PayMongo.",
+ caption="Import payments, refunds, payment links, payouts and their transactions, subscriptions, invoices, and webhook settings from PayMongo.",
fields=cast(
list[FieldType],
[
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py
index b3f8b6f03141..dfccdc59799b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py
@@ -94,15 +94,16 @@ def respond(http: MagicMock, bodies: list[dict[str, Any]], statuses: list[int] |
@pytest.mark.parametrize(
- "name,metadata",
+ "name,metadata,url,cursor_param",
[
- ("payments", {"has_more": True}),
- ("webhooks", {"has_more": True}),
- ("payouts", {"pagination": {"next_cursor": "pay_1"}}),
+ ("payments", {"has_more": True}, "https://api.paymongo.com/v1/payments", "after"),
+ ("webhooks", {"has_more": True}, "https://api.paymongo.com/v1/webhooks", "after"),
+ ("payouts", {"pagination": {"next_cursor": "pay_1"}}, "https://api.paymongo.com/v1/payouts", "after"),
+ ("invoices", {"has_more": True}, "https://invoices-api.paymongo.com/v1/invoices", "starting_after"),
],
)
def test_pagination_auth_normalization_and_checkpoint(
- http: MagicMock, manager: MagicMock, name: str, metadata: dict[str, Any]
+ http: MagicMock, manager: MagicMock, name: str, metadata: dict[str, Any], url: str, cursor_param: str
) -> None:
sent = respond(
http,
@@ -120,8 +121,9 @@ def test_pagination_auth_normalization_and_checkpoint(
assert list(pages) == []
assert manager.save_state.call_args.args[0].finished
assert sent[0].headers["Authorization"] == "Basic c2tfdGVzdF9mYWtlOg=="
+ assert (sent[0].url or "").split("?")[0] == url
assert parse_qs(urlsplit(sent[0].url or "").query) == {"limit": ["10"]}
- assert parse_qs(urlsplit(sent[1].url or "").query) == {"limit": ["10"], "after": ["pay_1"]}
+ assert parse_qs(urlsplit(sent[1].url or "").query) == {"limit": ["10"], cursor_param: ["pay_1"]}
@pytest.mark.parametrize("finished", [False, True])
@@ -160,6 +162,35 @@ def test_refund_fanout_keeps_payment_and_child_cursors() -> None:
]
+def test_payout_transaction_fanout_keys_rows_by_payout() -> None:
+ result = driver().run(
+ "payout_transactions",
+ [
+ ScriptedResponse(json={"data": [row("po_one")], "pagination": {"next_cursor": "po_one"}}),
+ ScriptedResponse(
+ json={"data": [row("pay_one", payout_id="po_one")], "pagination": {"next_cursor": "pay_one"}}
+ ),
+ ScriptedResponse(json={"data": [row("ref_one", payout_id="po_one")], "pagination": {"next_cursor": None}}),
+ ScriptedResponse(json={"data": [row("po_two")], "pagination": {"next_cursor": None}}),
+ ScriptedResponse(json={"data": [row("pay_one", payout_id="po_two")], "pagination": {"next_cursor": None}}),
+ ],
+ )
+ assert result.raised is None
+ assert [(item["_payouts_id"], item["id"]) for item in result.rows] == [
+ ("po_one", "pay_one"),
+ ("po_one", "ref_one"),
+ ("po_two", "pay_one"),
+ ]
+ assert result.paths == [
+ "/v1/payouts",
+ "/v1/payouts/po_one/transactions",
+ "/v1/payouts/po_one/transactions",
+ "/v1/payouts",
+ "/v1/payouts/po_two/transactions",
+ ]
+ assert result.queries[2] == {"limit": ("10",), "after": ("pay_one",)}
+
+
@pytest.mark.parametrize(
"body,message", [({"data": [row("pay_same")], "has_more": True}, "repeated"), ({"unexpected": []}, "data")]
)
@@ -388,22 +419,38 @@ def test_webhook_info_finds_matching_url_on_later_page(exists: bool) -> None:
assert "fake-secret" not in repr(info)
[email protected]("status", [200, 403])
-def test_refund_permissions_probe_child_endpoint(status: int) -> None:
+ "status,schema,parent_path,child_path,child_query",
+ [
+ (
+ status,
+ "refunds",
+ "/v1/payments",
+ "/v1/refunds",
+ {"data.attributes.payment_id": ("pay_one",), "data.attributes.limit": ("1",)},
+ )
+ for status in (200, 403)
+ ]
+ + [
+ (status, "payout_transactions", "/v1/payouts", "/v1/payouts/pay_one/transactions", {"limit": ("1",)})
+ for status in (200, 403)
+ ],
+)
+def test_child_permissions_probe_child_endpoint(
+ status: int, schema: str, parent_path: str, child_path: str, child_query: dict[str, tuple[str, ...]]
+) -> None:
with scripted_network(
[
ScriptedResponse(json={"data": [row("pay_one")]}),
ScriptedResponse(status=status, json={"data": []}),
]
) as network:
- valid, message = PaymongoClient("fake").validate_credentials("refunds")
+ valid, message = PaymongoClient("fake").validate_credentials(schema)
assert valid is (status == 200)
if status == 200:
assert message is None
else:
assert message is not None and "permissions" in message
- assert network.requests_log[1].path == "/v1/refunds"
- assert network.requests_log[1].query == {
- "data.attributes.payment_id": ("pay_one",),
- "data.attributes.limit": ("1",),
- }
+ assert network.requests_log[0].path == parent_path
+ assert network.requests_log[1].path == child_path
+ assert network.requests_log[1].query == child_query