:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Mono source (#119786)
Tom Owerse6abee809fc7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 945bf237d676..f8629272251a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8185,10 +8185,10 @@ Today (3): `accounts`, `customers`, `transactions`Diffed against: <https://docs.mono.co/llms.txt>-- [ ] `Bank Coverage (GET /v3/institutions)` — join institution IDs on synced accounts to the complete bank coverage and supported product catalogue (high)-- [ ] `Fetch all Payments (GET /v2/payments/transactions)` — analyze payment collection revenue, failures, fees, and refunds separately from linked bank transactions (high)-- [ ] `Get all Mandates (GET /v3/payments/mandates)` — track direct debit authorizations, their states, and customer relationships for recurring collection analysis (high)-- [ ] `Retrieve All Debits (GET /v3/payments/mandates/debits)` — measure successful and failed mandate collections, collected amounts, and debit fees (high)+- [x] `Bank Coverage (GET /v3/institutions)` — join institution IDs on synced accounts to the complete bank coverage and supported product catalogue (high)+- [x] `Fetch all Payments (GET /v2/payments/transactions)` — analyze payment collection revenue, failures, fees, and refunds separately from linked bank transactions (high)+- [x] `Get all Mandates (GET /v3/payments/mandates)` — track direct debit authorizations, their states, and customer relationships for recurring collection analysis (high)+- [x] `Retrieve All Debits (GET /v3/payments/mandates/debits)` — measure successful and failed mandate collections, collected amounts, and debit fees (high)- [ ] `Payout (GET /v2/payments/payouts)` — reconcile payment collections with settlement amounts and payout status (high)- [ ] `Payout Transactions (GET /v2/payments/payout/{payout}/transactions)` — match individual collections to settlements and explain payout totals (high)- [ ] `Disbursements and distributions (GET /v3/payments/disburse/disbursements; GET /v3/payments/disburse/disbursements/{batch_id}/distributions)` — analyze outgoing payment batches and recipient distributions, including payment status and amounts (high)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/canonical_descriptions.pyindex d8ccbdc0ead7..d2690bd67c0a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/canonical_descriptions.py@@ -41,4 +41,80 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"category": "The transaction category assigned by Mono.",},},+ "institutions": {+ "description": "Financial institutions that Mono supports, with their authentication methods and product scopes.",+ "docs_url": "https://docs.mono.co/api/miscellaneous/bank-coverage",+ "columns": {+ "id": "The institution identifier.",+ "institution": "The institution name.",+ "type": "The banking type, such as personal banking, business banking, or mobile banking.",+ "nip_code": "The NIBSS Instant Payment code of the institution.",+ "bank_code": "The bank code of the institution.",+ "country": "The country code of the institution.",+ "auth_methods": "The methods customers can use to connect accounts at the institution.",+ "scope": "The Mono products that the institution supports, such as financial data or direct debit.",+ },+ },+ "payments": {+ "description": "One-time and recurring payments collected through Mono DirectPay and Direct Debit.",+ "docs_url": "https://docs.mono.co/api/directpay/fetch-payments",+ "columns": {+ "id": "The payment identifier.",+ "channel": "The payment channel, such as mandate.",+ "type": "The payment type, such as onetime-debit or variable-debit.",+ "status": "The payment status, such as pending, processing, failed, or successful.",+ "amount": "The payment amount in the currency's smallest unit, such as kobo.",+ "fee": "The fee for the payment in the currency's smallest unit.",+ "fee_bearer": "Who pays the fee, the business or the customer.",+ "currency": "The payment currency.",+ "description": "The payment description.",+ "reference": "The payment reference supplied by the business.",+ "account": "The bank account that paid.",+ "customer": "The customer who paid.",+ "refunded": "Whether the payment was refunded.",+ "created_at": "When the payment was created.",+ "updated_at": "When the payment was last updated.",+ },+ },+ "mandates": {+ "description": "Direct debit mandates that customers authorized for the business.",+ "docs_url": "https://docs.mono.co/api/direct-debit/mandate/get-all-mandates",+ "columns": {+ "id": "The mandate identifier.",+ "status": "The mandate status, such as approved.",+ "reference": "The mandate reference supplied by the business.",+ "amount": "The mandate amount in the currency's smallest unit, such as kobo.",+ "balance": "The amount that remains available to debit under the mandate.",+ "mandate_type": "The mandate type, such as emandate.",+ "debit_type": "Whether the mandate allows fixed or variable debits.",+ "account_name": "The name on the debited bank account.",+ "account_number": "The debited bank account number.",+ "live_mode": "Whether the mandate is in live mode.",+ "approved": "Whether the customer approved the mandate.",+ "ready_to_debit": "Whether the mandate can be debited.",+ "institution": "The bank that holds the debited account.",+ "customer": "The identifier of the customer who owns the mandate.",+ "start_date": "When the mandate starts.",+ "end_date": "When the mandate ends.",+ "date": "When the mandate was created.",+ },+ },+ "mandate_debits": {+ "description": "Debits made against direct debit mandates.",+ "docs_url": "https://docs.mono.co/api/direct-debit/account/fetch-all-debits",+ "columns": {+ "mandate": "The identifier of the debited mandate.",+ "reference": "The debit reference.",+ "amount": "The debit amount in the currency's smallest unit, such as kobo.",+ "fee": "The fee for the debit in the currency's smallest unit.",+ "fee_bearer": "Who pays the fee, the business or the customer.",+ "currency": "The debit currency.",+ "type": "The debit type, such as variable-debit.",+ "status": "The debit status, such as successful, failed, or processing.",+ "narration": "The debit narration.",+ "live_mode": "Whether the debit is in live mode.",+ "refunded": "Whether the debit was refunded.",+ "date": "When the debit was made.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/mono.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/mono.pyindex bb86eaff5101..f4f8accc0a49 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/mono.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/mono.py@@ -21,7 +21,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (ClientConfig,+ Endpoint,EndpointResource,+ PaginatorConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource@@ -30,7 +32,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_from products.warehouse_sources.backend.temporal.data_imports.sources.mono.settings import (AUTH_ERROR,BASE_URL,+ DATE_FILTERS,ENDPOINTS,+ INCREMENTAL_FIELDS,+ PAGE_SIZE,PERMISSION_ERROR,PRIMARY_KEYS,)@@ -115,6 +120,29 @@ def validate_credentials(config: MonoSourceConfig, api_version: str, team_id: inreturn True, None+def incremental_start(+ endpoint: str, should_use_incremental_field: bool, db_incremental_field_last_value: object+) -> date | None:+ if not should_use_incremental_field or db_incremental_field_last_value is None:+ return None+ watermark = parse_datetime_value(db_incremental_field_last_value)+ if watermark is None:+ raise ValueError(f"Mono received an invalid {endpoint} watermark. Reset the {endpoint} table and retry.")+ # A one-day overlap covers the API's date precision and avoids equal start/end dates.+ return watermark.date() - timedelta(days=1)+++def endpoint_paginator(endpoint: str) -> PaginatorConfig | None:+ if endpoint in ("institutions", "mandate_debits"):+ return "single_page"+ if endpoint == "payments":+ return PageNumberPaginator(base_page=1, total_path="meta.paging.pages")+ if endpoint == "mandates":+ # The documented response has no paging metadata, so stop at the first empty page.+ return PageNumberPaginator(base_page=1)+ return None++def mono_source(config: MonoSourceConfig,endpoint: str,@@ -129,39 +157,46 @@ def mono_source(resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Noneparams: dict[str, Any] = {}start = end = None+ start_day = incremental_start(endpoint, should_use_incremental_field, db_incremental_field_last_value)+ if endpoint == "transactions":+ configured_start = parse_start_date(config.start_date)+ start_day = max(configured_start, start_day) if start_day else configured_start+ if resume is not None:+ start, end = resume.start, resume.end+ elif endpoint in DATE_FILTERS and start_day is not None:+ date_format = DATE_FILTERS[endpoint][2]+ start, end = start_day.strftime(date_format), datetime.now(UTC).strftime(date_format)+ if start is not None and end is not None and endpoint in DATE_FILTERS:+ start_param, end_param, _ = DATE_FILTERS[endpoint]+ params = {start_param: start, end_param: end}if endpoint == "transactions":- start_day = parse_start_date(config.start_date)- if should_use_incremental_field and db_incremental_field_last_value is not None:- watermark = parse_datetime_value(db_incremental_field_last_value)- if watermark is None:- raise ValueError(- "Mono received an invalid transaction watermark. Reset the transaction table and retry."- )- # A one-day overlap covers the API's date precision and avoids equal start/end dates.- start_day = max(start_day, watermark.date() - timedelta(days=1))- start = resume.start if resume else start_day.strftime("%d-%m-%Y")- end = resume.end if resume else datetime.now(UTC).strftime("%d-%m-%Y")params = {"account_id": {"type": "resolve", "resource": "accounts", "field": "id"},- "start": start,- "end": end,+ **params,"paginate": "true",- "limit": 100,+ "limit": PAGE_SIZE,}+ elif endpoint in ("payments", "mandates"):+ params["limit"] = PAGE_SIZE++ endpoint_config: Endpoint = {+ "path": path,+ "params": params,+ "data_selector": "data.payments" if endpoint == "payments" else "data",+ "data_selector_required": True,+ }+ paginator = endpoint_paginator(endpoint)+ if paginator is not None:+ endpoint_config["paginator"] = paginatorresource: EndpointResource = {"name": endpoint,"table_name": endpoint,"table_format": "delta","write_disposition": {"disposition": "merge", "strategy": "upsert"}- if endpoint == "transactions" and should_use_incremental_field+ if endpoint in INCREMENTAL_FIELDS and should_use_incremental_fieldelse "replace",- "endpoint": {- "path": path,- "params": params,- "data_selector": "data",- "data_selector_required": True,- },+ "endpoint": endpoint_config,}resources: list[str | EndpointResource] = []if endpoint == "transactions":@@ -194,7 +229,8 @@ def mono_source(name=endpoint,items=lambda: result,primary_keys=PRIMARY_KEYS[endpoint],- # Each account has its own transaction order, so only checkpoint the watermark after all accounts finish.+ # Mono lists newest first, and transactions restart that order for each account,+ # so only checkpoint the watermark after the whole sync finishes.sort_mode="desc",on_complete=resumable_source_manager.clear_state,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/settings.pyindex c104139c78aa..c69dcc8fac67 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/settings.py@@ -1,15 +1,24 @@from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldTypeBASE_URL = "https://api.withmono.com"+# Relative paths follow the pinned API version. Mono serves bank coverage and Direct Debit only on v3.ENDPOINTS = {"customers": "customers","accounts": "accounts","transactions": "accounts/{account_id}/transactions",+ "institutions": f"{BASE_URL}/v3/institutions",+ "payments": "payments/transactions",+ "mandates": f"{BASE_URL}/v3/payments/mandates",+ "mandate_debits": f"{BASE_URL}/v3/payments/mandates/debits",}PRIMARY_KEYS = {"customers": ["id"],"accounts": ["id"],"transactions": ["account_id", "id"],+ "institutions": ["id"],+ "payments": ["id"],+ "mandates": ["id"],+ "mandate_debits": ["mandate", "reference"],}INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {"transactions": [@@ -20,6 +29,29 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {"field_type": IncrementalFieldType.DateTime,}],+ "payments": [+ {+ "field": "created_at",+ "type": IncrementalFieldType.DateTime,+ "label": "created_at",+ "field_type": IncrementalFieldType.DateTime,+ }+ ],+ "mandate_debits": [+ {+ "field": "date",+ "type": IncrementalFieldType.DateTime,+ "label": "date",+ "field_type": IncrementalFieldType.DateTime,+ }+ ],+}+# Date filter query params and their formats for each incremental endpoint.+DATE_FILTERS = {+ "transactions": ("start", "end", "%d-%m-%Y"),+ "payments": ("start", "end", "%d-%m-%Y"),+ "mandate_debits": ("start_date", "end_date", "%Y-%m-%d"),}+PAGE_SIZE = 100AUTH_ERROR = "Mono rejected the secret API key. Copy the secret key from your Mono app and reconnect."PERMISSION_ERROR = "Mono denied access. Check that your Mono app has access to the requested data."diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/source.pyindex b0bad5f5ef2d..367b1d86f47b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/source.py@@ -74,7 +74,9 @@ class MonoSource(ResumableSource[MonoSourceConfig, MonoResumeConfig]):force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only={"transactions"})+ return build_endpoint_schemas(+ ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only={"transactions", "payments", "mandate_debits"}+ )def validate_credentials(self,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/tests/test_mono.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/tests/test_mono.pyindex 3d8355e36f74..ee77f248500c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/tests/test_mono.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/tests/test_mono.py@@ -123,6 +123,130 @@ def test_resumes_child_page_with_original_date_bounds_and_skips_completed_accoun}+def payments_page(rows: list[dict[str, Any]], pages: int) -> ScriptedResponse:+ return ScriptedResponse(+ json={"status": "successful", "data": {"payments": rows}, "meta": {"paging": {"pages": pages, "next": None}}}+ )+++@time_machine.travel("2026-06-15T12:00:00Z", tick=False)+ "endpoint,incremental,watermark,script,expected_path,expected_queries,expected_ids",+ [+ (+ "institutions",+ False,+ None,+ [page([{"id": "institution-1"}, {"id": "institution-2"}])],+ "/v3/institutions",+ [{}],+ ["institution-1", "institution-2"],+ ),+ (+ "mandates",+ False,+ None,+ [page([{"id": "mandate-1"}, {"id": "mandate-2"}]), page([{"id": "mandate-3"}]), page([])],+ "/v3/payments/mandates",+ [{"page": (str(number),), "limit": ("100",)} for number in (1, 2, 3)],+ ["mandate-1", "mandate-2", "mandate-3"],+ ),+ (+ "payments",+ False,+ "2026-06-10T15:00:00Z",+ [payments_page([{"id": "payment-1"}], 2), payments_page([{"id": "payment-2"}], 2)],+ "/v2/payments/transactions",+ [{"page": (str(number),), "limit": ("100",)} for number in (1, 2)],+ ["payment-1", "payment-2"],+ ),+ (+ "payments",+ True,+ None,+ [payments_page([{"id": "payment-1"}], 1)],+ "/v2/payments/transactions",+ [{"page": ("1",), "limit": ("100",)}],+ ["payment-1"],+ ),+ (+ "payments",+ True,+ "2026-06-10T15:00:00Z",+ [payments_page([{"id": "payment-1"}], 1)],+ "/v2/payments/transactions",+ [{"page": ("1",), "limit": ("100",), "start": ("09-06-2026",), "end": ("15-06-2026",)}],+ ["payment-1"],+ ),+ (+ "mandate_debits",+ False,+ None,+ [page([{"mandate": "mandate-1", "reference": "debit-1"}], "https://api.withmono.com/v3/x?page=2")],+ "/v3/payments/mandates/debits",+ [{}],+ ["debit-1"],+ ),+ (+ "mandate_debits",+ True,+ datetime(2026, 6, 10, 15, tzinfo=UTC),+ [page([{"mandate": "mandate-1", "reference": "debit-1"}])],+ "/v3/payments/mandates/debits",+ [{"start_date": ("2026-06-09",), "end_date": ("2026-06-15",)}],+ ["debit-1"],+ ),+ ],+)+def test_payment_and_coverage_tables_request_the_right_version_filters_and_pages(+ endpoint: str,+ incremental: bool,+ watermark: object,+ script: list[ScriptedResponse],+ expected_path: str,+ expected_queries: list[dict[str, tuple[str, ...]]],+ expected_ids: list[str],+) -> None:+ result = driver().run(+ endpoint, script, should_use_incremental_field=incremental, db_incremental_field_last_value=watermark+ )++ assert result.raised is None+ assert [row.get("reference", row.get("id")) for row in result.rows] == expected_ids+ assert result.paths == [expected_path] * len(expected_queries)+ assert result.queries == expected_queries+ assert all(request.headers["mono-sec-key"] == CONFIG.api_key for request in result.requests)+++ "start,end,expected_query",+ [+ (None, None, {"page": ("3",), "limit": ("100",)}),+ (+ "09-06-2026",+ "15-06-2026",+ {"page": ("3",), "limit": ("100",), "start": ("09-06-2026",), "end": ("15-06-2026",)},+ ),+ ],+)+@time_machine.travel("2026-06-16T12:00:00Z", tick=False)+def test_resumed_payments_keep_the_saved_date_window(+ start: str | None, end: str | None, expected_query: dict[str, tuple[str, ...]]+) -> None:+ saved = MonoResumeConfig(paginator_state={"page": 3}, start=start, end=end)+ result = driver().run(+ "payments",+ [payments_page([{"id": "payment-5"}], 3)],+ resume_state=saved,+ should_use_incremental_field=start is not None,+ db_incremental_field_last_value="2026-06-15T00:00:00Z" if start else None,+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["payment-5"]+ assert result.queries == [expected_query]++@pytest.mark.parametrize("next_url", ["?page=1", "?page=0", "?page=no", "?offset=2", "?page=2&page=3"])def test_invalid_next_page_fails_instead_of_truncating_or_looping(next_url: str) -> None:result = driver().run("customers", [page([{"id": "customer-1"}], next_url)])@@ -200,7 +324,7 @@ def test_invalid_watermark_and_unknown_table_fail_before_any_request() -> None:unknown_table = driver().run("missing", [])assert isinstance(invalid_watermark.raised, ValueError)- assert "invalid transaction watermark" in str(invalid_watermark.raised)+ assert "invalid transactions watermark" in str(invalid_watermark.raised)assert isinstance(unknown_table.raised, UnknownResourceError)assert invalid_watermark.requests == []assert unknown_table.requests == []