Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Mono source (#119786)

Tom Owerse6abee809fc7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 945bf237d676..f8629272251a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8185,10 +8185,10 @@ Today (3): `accounts`, `customers`, `transactions`
Diffed against: <https://docs.mono.co/llms.txt>
-- [ ] `Bank Coverage (GET /v3/institutions)` — join institution IDs on synced accounts to the complete bank coverage and supported product catalogue (high)
-- [ ] `Fetch all Payments (GET /v2/payments/transactions)` — analyze payment collection revenue, failures, fees, and refunds separately from linked bank transactions (high)
-- [ ] `Get all Mandates (GET /v3/payments/mandates)` — track direct debit authorizations, their states, and customer relationships for recurring collection analysis (high)
-- [ ] `Retrieve All Debits (GET /v3/payments/mandates/debits)` — measure successful and failed mandate collections, collected amounts, and debit fees (high)
+- [x] `Bank Coverage (GET /v3/institutions)` — join institution IDs on synced accounts to the complete bank coverage and supported product catalogue (high)
+- [x] `Fetch all Payments (GET /v2/payments/transactions)` — analyze payment collection revenue, failures, fees, and refunds separately from linked bank transactions (high)
+- [x] `Get all Mandates (GET /v3/payments/mandates)` — track direct debit authorizations, their states, and customer relationships for recurring collection analysis (high)
+- [x] `Retrieve All Debits (GET /v3/payments/mandates/debits)` — measure successful and failed mandate collections, collected amounts, and debit fees (high)
- [ ] `Payout (GET /v2/payments/payouts)` — reconcile payment collections with settlement amounts and payout status (high)
- [ ] `Payout Transactions (GET /v2/payments/payout/{payout}/transactions)` — match individual collections to settlements and explain payout totals (high)
- [ ] `Disbursements and distributions (GET /v3/payments/disburse/disbursements; GET /v3/payments/disburse/disbursements/{batch_id}/distributions)` — analyze outgoing payment batches and recipient distributions, including payment status and amounts (high)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/canonical_descriptions.py
index d8ccbdc0ead7..d2690bd67c0a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/canonical_descriptions.py
@@ -41,4 +41,80 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"category": "The transaction category assigned by Mono.",
},
},
+ "institutions": {
+ "description": "Financial institutions that Mono supports, with their authentication methods and product scopes.",
+ "docs_url": "https://docs.mono.co/api/miscellaneous/bank-coverage",
+ "columns": {
+ "id": "The institution identifier.",
+ "institution": "The institution name.",
+ "type": "The banking type, such as personal banking, business banking, or mobile banking.",
+ "nip_code": "The NIBSS Instant Payment code of the institution.",
+ "bank_code": "The bank code of the institution.",
+ "country": "The country code of the institution.",
+ "auth_methods": "The methods customers can use to connect accounts at the institution.",
+ "scope": "The Mono products that the institution supports, such as financial data or direct debit.",
+ },
+ },
+ "payments": {
+ "description": "One-time and recurring payments collected through Mono DirectPay and Direct Debit.",
+ "docs_url": "https://docs.mono.co/api/directpay/fetch-payments",
+ "columns": {
+ "id": "The payment identifier.",
+ "channel": "The payment channel, such as mandate.",
+ "type": "The payment type, such as onetime-debit or variable-debit.",
+ "status": "The payment status, such as pending, processing, failed, or successful.",
+ "amount": "The payment amount in the currency's smallest unit, such as kobo.",
+ "fee": "The fee for the payment in the currency's smallest unit.",
+ "fee_bearer": "Who pays the fee, the business or the customer.",
+ "currency": "The payment currency.",
+ "description": "The payment description.",
+ "reference": "The payment reference supplied by the business.",
+ "account": "The bank account that paid.",
+ "customer": "The customer who paid.",
+ "refunded": "Whether the payment was refunded.",
+ "created_at": "When the payment was created.",
+ "updated_at": "When the payment was last updated.",
+ },
+ },
+ "mandates": {
+ "description": "Direct debit mandates that customers authorized for the business.",
+ "docs_url": "https://docs.mono.co/api/direct-debit/mandate/get-all-mandates",
+ "columns": {
+ "id": "The mandate identifier.",
+ "status": "The mandate status, such as approved.",
+ "reference": "The mandate reference supplied by the business.",
+ "amount": "The mandate amount in the currency's smallest unit, such as kobo.",
+ "balance": "The amount that remains available to debit under the mandate.",
+ "mandate_type": "The mandate type, such as emandate.",
+ "debit_type": "Whether the mandate allows fixed or variable debits.",
+ "account_name": "The name on the debited bank account.",
+ "account_number": "The debited bank account number.",
+ "live_mode": "Whether the mandate is in live mode.",
+ "approved": "Whether the customer approved the mandate.",
+ "ready_to_debit": "Whether the mandate can be debited.",
+ "institution": "The bank that holds the debited account.",
+ "customer": "The identifier of the customer who owns the mandate.",
+ "start_date": "When the mandate starts.",
+ "end_date": "When the mandate ends.",
+ "date": "When the mandate was created.",
+ },
+ },
+ "mandate_debits": {
+ "description": "Debits made against direct debit mandates.",
+ "docs_url": "https://docs.mono.co/api/direct-debit/account/fetch-all-debits",
+ "columns": {
+ "mandate": "The identifier of the debited mandate.",
+ "reference": "The debit reference.",
+ "amount": "The debit amount in the currency's smallest unit, such as kobo.",
+ "fee": "The fee for the debit in the currency's smallest unit.",
+ "fee_bearer": "Who pays the fee, the business or the customer.",
+ "currency": "The debit currency.",
+ "type": "The debit type, such as variable-debit.",
+ "status": "The debit status, such as successful, failed, or processing.",
+ "narration": "The debit narration.",
+ "live_mode": "Whether the debit is in live mode.",
+ "refunded": "Whether the debit was refunded.",
+ "date": "When the debit was made.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/mono.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/mono.py
index bb86eaff5101..f4f8accc0a49 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/mono.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/mono.py
@@ -21,7 +21,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
+ Endpoint,
EndpointResource,
+ PaginatorConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
@@ -30,7 +32,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
from products.warehouse_sources.backend.temporal.data_imports.sources.mono.settings import (
AUTH_ERROR,
BASE_URL,
+ DATE_FILTERS,
ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ PAGE_SIZE,
PERMISSION_ERROR,
PRIMARY_KEYS,
)
@@ -115,6 +120,29 @@ def validate_credentials(config: MonoSourceConfig, api_version: str, team_id: in
return True, None
+def incremental_start(
+ endpoint: str, should_use_incremental_field: bool, db_incremental_field_last_value: object
+) -> date | None:
+ if not should_use_incremental_field or db_incremental_field_last_value is None:
+ return None
+ watermark = parse_datetime_value(db_incremental_field_last_value)
+ if watermark is None:
+ raise ValueError(f"Mono received an invalid {endpoint} watermark. Reset the {endpoint} table and retry.")
+ # A one-day overlap covers the API's date precision and avoids equal start/end dates.
+ return watermark.date() - timedelta(days=1)
+
+
+def endpoint_paginator(endpoint: str) -> PaginatorConfig | None:
+ if endpoint in ("institutions", "mandate_debits"):
+ return "single_page"
+ if endpoint == "payments":
+ return PageNumberPaginator(base_page=1, total_path="meta.paging.pages")
+ if endpoint == "mandates":
+ # The documented response has no paging metadata, so stop at the first empty page.
+ return PageNumberPaginator(base_page=1)
+ return None
+
+
def mono_source(
config: MonoSourceConfig,
endpoint: str,
@@ -129,39 +157,46 @@ def mono_source(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
params: dict[str, Any] = {}
start = end = None
+ start_day = incremental_start(endpoint, should_use_incremental_field, db_incremental_field_last_value)
+ if endpoint == "transactions":
+ configured_start = parse_start_date(config.start_date)
+ start_day = max(configured_start, start_day) if start_day else configured_start
+ if resume is not None:
+ start, end = resume.start, resume.end
+ elif endpoint in DATE_FILTERS and start_day is not None:
+ date_format = DATE_FILTERS[endpoint][2]
+ start, end = start_day.strftime(date_format), datetime.now(UTC).strftime(date_format)
+ if start is not None and end is not None and endpoint in DATE_FILTERS:
+ start_param, end_param, _ = DATE_FILTERS[endpoint]
+ params = {start_param: start, end_param: end}
if endpoint == "transactions":
- start_day = parse_start_date(config.start_date)
- if should_use_incremental_field and db_incremental_field_last_value is not None:
- watermark = parse_datetime_value(db_incremental_field_last_value)
- if watermark is None:
- raise ValueError(
- "Mono received an invalid transaction watermark. Reset the transaction table and retry."
- )
- # A one-day overlap covers the API's date precision and avoids equal start/end dates.
- start_day = max(start_day, watermark.date() - timedelta(days=1))
- start = resume.start if resume else start_day.strftime("%d-%m-%Y")
- end = resume.end if resume else datetime.now(UTC).strftime("%d-%m-%Y")
params = {
"account_id": {"type": "resolve", "resource": "accounts", "field": "id"},
- "start": start,
- "end": end,
+ **params,
"paginate": "true",
- "limit": 100,
+ "limit": PAGE_SIZE,
}
+ elif endpoint in ("payments", "mandates"):
+ params["limit"] = PAGE_SIZE
+
+ endpoint_config: Endpoint = {
+ "path": path,
+ "params": params,
+ "data_selector": "data.payments" if endpoint == "payments" else "data",
+ "data_selector_required": True,
+ }
+ paginator = endpoint_paginator(endpoint)
+ if paginator is not None:
+ endpoint_config["paginator"] = paginator
resource: EndpointResource = {
"name": endpoint,
"table_name": endpoint,
"table_format": "delta",
"write_disposition": {"disposition": "merge", "strategy": "upsert"}
- if endpoint == "transactions" and should_use_incremental_field
+ if endpoint in INCREMENTAL_FIELDS and should_use_incremental_field
else "replace",
- "endpoint": {
- "path": path,
- "params": params,
- "data_selector": "data",
- "data_selector_required": True,
- },
+ "endpoint": endpoint_config,
}
resources: list[str | EndpointResource] = []
if endpoint == "transactions":
@@ -194,7 +229,8 @@ def mono_source(
name=endpoint,
items=lambda: result,
primary_keys=PRIMARY_KEYS[endpoint],
- # Each account has its own transaction order, so only checkpoint the watermark after all accounts finish.
+ # Mono lists newest first, and transactions restart that order for each account,
+ # so only checkpoint the watermark after the whole sync finishes.
sort_mode="desc",
on_complete=resumable_source_manager.clear_state,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/settings.py
index c104139c78aa..c69dcc8fac67 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/settings.py
@@ -1,15 +1,24 @@
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
BASE_URL = "https://api.withmono.com"
+# Relative paths follow the pinned API version. Mono serves bank coverage and Direct Debit only on v3.
ENDPOINTS = {
"customers": "customers",
"accounts": "accounts",
"transactions": "accounts/{account_id}/transactions",
+ "institutions": f"{BASE_URL}/v3/institutions",
+ "payments": "payments/transactions",
+ "mandates": f"{BASE_URL}/v3/payments/mandates",
+ "mandate_debits": f"{BASE_URL}/v3/payments/mandates/debits",
}
PRIMARY_KEYS = {
"customers": ["id"],
"accounts": ["id"],
"transactions": ["account_id", "id"],
+ "institutions": ["id"],
+ "payments": ["id"],
+ "mandates": ["id"],
+ "mandate_debits": ["mandate", "reference"],
}
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
"transactions": [
@@ -20,6 +29,29 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
"field_type": IncrementalFieldType.DateTime,
}
],
+ "payments": [
+ {
+ "field": "created_at",
+ "type": IncrementalFieldType.DateTime,
+ "label": "created_at",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+ "mandate_debits": [
+ {
+ "field": "date",
+ "type": IncrementalFieldType.DateTime,
+ "label": "date",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+}
+# Date filter query params and their formats for each incremental endpoint.
+DATE_FILTERS = {
+ "transactions": ("start", "end", "%d-%m-%Y"),
+ "payments": ("start", "end", "%d-%m-%Y"),
+ "mandate_debits": ("start_date", "end_date", "%Y-%m-%d"),
}
+PAGE_SIZE = 100
AUTH_ERROR = "Mono rejected the secret API key. Copy the secret key from your Mono app and reconnect."
PERMISSION_ERROR = "Mono denied access. Check that your Mono app has access to the requested data."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/source.py
index b0bad5f5ef2d..367b1d86f47b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/source.py
@@ -74,7 +74,9 @@ class MonoSource(ResumableSource[MonoSourceConfig, MonoResumeConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only={"transactions"})
+ return build_endpoint_schemas(
+ ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only={"transactions", "payments", "mandate_debits"}
+ )
def validate_credentials(
self,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/tests/test_mono.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/tests/test_mono.py
index 3d8355e36f74..ee77f248500c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mono/tests/test_mono.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mono/tests/test_mono.py
@@ -123,6 +123,130 @@ def test_resumes_child_page_with_original_date_bounds_and_skips_completed_accoun
}
+def payments_page(rows: list[dict[str, Any]], pages: int) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={"status": "successful", "data": {"payments": rows}, "meta": {"paging": {"pages": pages, "next": None}}}
+ )
+
+
+@time_machine.travel("2026-06-15T12:00:00Z", tick=False)
+ "endpoint,incremental,watermark,script,expected_path,expected_queries,expected_ids",
+ [
+ (
+ "institutions",
+ False,
+ None,
+ [page([{"id": "institution-1"}, {"id": "institution-2"}])],
+ "/v3/institutions",
+ [{}],
+ ["institution-1", "institution-2"],
+ ),
+ (
+ "mandates",
+ False,
+ None,
+ [page([{"id": "mandate-1"}, {"id": "mandate-2"}]), page([{"id": "mandate-3"}]), page([])],
+ "/v3/payments/mandates",
+ [{"page": (str(number),), "limit": ("100",)} for number in (1, 2, 3)],
+ ["mandate-1", "mandate-2", "mandate-3"],
+ ),
+ (
+ "payments",
+ False,
+ "2026-06-10T15:00:00Z",
+ [payments_page([{"id": "payment-1"}], 2), payments_page([{"id": "payment-2"}], 2)],
+ "/v2/payments/transactions",
+ [{"page": (str(number),), "limit": ("100",)} for number in (1, 2)],
+ ["payment-1", "payment-2"],
+ ),
+ (
+ "payments",
+ True,
+ None,
+ [payments_page([{"id": "payment-1"}], 1)],
+ "/v2/payments/transactions",
+ [{"page": ("1",), "limit": ("100",)}],
+ ["payment-1"],
+ ),
+ (
+ "payments",
+ True,
+ "2026-06-10T15:00:00Z",
+ [payments_page([{"id": "payment-1"}], 1)],
+ "/v2/payments/transactions",
+ [{"page": ("1",), "limit": ("100",), "start": ("09-06-2026",), "end": ("15-06-2026",)}],
+ ["payment-1"],
+ ),
+ (
+ "mandate_debits",
+ False,
+ None,
+ [page([{"mandate": "mandate-1", "reference": "debit-1"}], "https://api.withmono.com/v3/x?page=2")],
+ "/v3/payments/mandates/debits",
+ [{}],
+ ["debit-1"],
+ ),
+ (
+ "mandate_debits",
+ True,
+ datetime(2026, 6, 10, 15, tzinfo=UTC),
+ [page([{"mandate": "mandate-1", "reference": "debit-1"}])],
+ "/v3/payments/mandates/debits",
+ [{"start_date": ("2026-06-09",), "end_date": ("2026-06-15",)}],
+ ["debit-1"],
+ ),
+ ],
+)
+def test_payment_and_coverage_tables_request_the_right_version_filters_and_pages(
+ endpoint: str,
+ incremental: bool,
+ watermark: object,
+ script: list[ScriptedResponse],
+ expected_path: str,
+ expected_queries: list[dict[str, tuple[str, ...]]],
+ expected_ids: list[str],
+) -> None:
+ result = driver().run(
+ endpoint, script, should_use_incremental_field=incremental, db_incremental_field_last_value=watermark
+ )
+
+ assert result.raised is None
+ assert [row.get("reference", row.get("id")) for row in result.rows] == expected_ids
+ assert result.paths == [expected_path] * len(expected_queries)
+ assert result.queries == expected_queries
+ assert all(request.headers["mono-sec-key"] == CONFIG.api_key for request in result.requests)
+
+
+ "start,end,expected_query",
+ [
+ (None, None, {"page": ("3",), "limit": ("100",)}),
+ (
+ "09-06-2026",
+ "15-06-2026",
+ {"page": ("3",), "limit": ("100",), "start": ("09-06-2026",), "end": ("15-06-2026",)},
+ ),
+ ],
+)
+@time_machine.travel("2026-06-16T12:00:00Z", tick=False)
+def test_resumed_payments_keep_the_saved_date_window(
+ start: str | None, end: str | None, expected_query: dict[str, tuple[str, ...]]
+) -> None:
+ saved = MonoResumeConfig(paginator_state={"page": 3}, start=start, end=end)
+ result = driver().run(
+ "payments",
+ [payments_page([{"id": "payment-5"}], 3)],
+ resume_state=saved,
+ should_use_incremental_field=start is not None,
+ db_incremental_field_last_value="2026-06-15T00:00:00Z" if start else None,
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["payment-5"]
+ assert result.queries == [expected_query]
+
+
@pytest.mark.parametrize("next_url", ["?page=1", "?page=0", "?page=no", "?offset=2", "?page=2&page=3"])
def test_invalid_next_page_fails_instead_of_truncating_or_looping(next_url: str) -> None:
result = driver().run("customers", [page([{"id": "customer-1"}], next_url)])
@@ -200,7 +324,7 @@ def test_invalid_watermark_and_unknown_table_fail_before_any_request() -> None:
unknown_table = driver().run("missing", [])
assert isinstance(invalid_watermark.raised, ValueError)
- assert "invalid transaction watermark" in str(invalid_watermark.raised)
+ assert "invalid transactions watermark" in str(invalid_watermark.raised)
assert isinstance(unknown_table.raised, UnknownResourceError)
assert invalid_watermark.requests == []
assert unknown_table.requests == []