:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add hitpay payouts, invoices and balance transactions (#122037)
Tom Owers4741ee8a95af
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 50c4ee489ce3..e68085aeffea 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -5967,10 +5967,10 @@ Today (5): `Charges`, `Customers`, `PaymentRequests`, `RecurringBilling`, `SubscDiffed against: <https://docs.hitpayapp.com/openapi.json>-- [ ] `Payouts (GET /v1/payouts)` — reconcile collected payments with settlement amounts and bank payout status (high)-- [ ] `Balance Transactions (GET /v1/balances/{balance_provider}/transactions)` — explain balance changes from payments, fees, refunds, and settlements (high)-- [ ] `Invoices (GET /v1/invoices)` — measure invoiced revenue, unpaid balances, and collection status alongside payment requests (high)-- [ ] `Orders (GET /v1/orders)` — analyze order revenue and purchase activity beyond individual payment records (high)+- [x] `Payouts (GET /v1/payouts)` — reconcile collected payments with settlement amounts and bank payout status (high)+- [x] `Balance Transactions (GET /v1/balances/{balance_provider}/transactions)` — explain balance changes from payments, fees, refunds, and settlements (high)+- [x] `Invoices (GET /v1/invoices)` — measure invoiced revenue, unpaid balances, and collection status alongside payment requests (high)+- [ ] `Orders (GET /v1/orders)` — analyze order revenue and purchase activity beyond individual payment records (high); skipped, the spec documents no page or cursor parameter for this endpoint and describes the response as a single order, so a full sync cannot be paged reliably- [ ] `Products (GET /v1/products)` — analyze product sales, prices, inventory, and catalog status (medium)- [ ] `Transfers (GET /v1/transfers)` — track outgoing payments to third parties, including their amounts and completion status (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/canonical_descriptions.pyindex 080d9ce436bb..b1a047911d69 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/canonical_descriptions.py@@ -117,4 +117,58 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "Time the agreement was last updated.",},},+ "Payouts": {+ "description": "A settlement of the business's available balance to its own linked bank account.",+ "docs_url": "https://docs.hitpayapp.com/reference/list-payouts",+ "columns": {+ "id": "Unique identifier for the payout.",+ "status": "Payout status: scheduled, processing, paid, failed, canceled, or pending.",+ "currency": "Three-letter currency code the payout was sent in.",+ "amount": "Payout amount in major units of the payout currency.",+ "source_currency": "Three-letter currency code of the balance the payout was drawn from.",+ "source_amount": "Amount drawn from the source balance, in major units.",+ "bank_name": "Name of the bank the payout was sent to.",+ "bank_account": "Masked number of the receiving bank account.",+ "remark": "Free-text note attached to the payout.",+ "reference": "Reference for the payout.",+ "failure_code": "Failure code, present when the payout failed.",+ "failure_message": "Failure reason, present when the payout failed.",+ "arrival_date": "Date the payout is expected to arrive in, or arrived in, the bank account.",+ "created_at": "Time the payout was created.",+ "updated_at": "Time the payout was last updated.",+ },+ },+ "Invoices": {+ "description": "An invoice the merchant issues to a customer, with its amount, dates, and payment link.",+ "docs_url": "https://docs.hitpayapp.com/reference/get-invoices",+ "columns": {+ "id": "Unique identifier for the invoice.",+ "invoice_number": "Human-readable invoice number.",+ "reference": "Merchant-supplied reference for the invoice.",+ "email": "Email address of the customer the invoice was issued to.",+ "currency": "Three-letter currency code for the invoice amount.",+ "amount": "Invoice amount in major units of the invoice currency.",+ "status": "Invoice status: draft, sent, pending, overdue, or paid.",+ "invoice_date": "Date the invoice was issued.",+ "due_date": "Date payment of the invoice is due.",+ "invoice_link": "Hosted link the customer opens to view and pay the invoice.",+ "created_at": "Time the invoice was created.",+ "updated_at": "Time the invoice was last updated.",+ },+ },+ "BalanceTransactions": {+ "description": "A ledger line on the Cards (by Adyen) balance, such as a credit for a charge or a debit for a payout.",+ "docs_url": "https://docs.hitpayapp.com/reference/get-balance-transactions",+ "columns": {+ "datetime": "Time the transaction was booked on the balance.",+ "description": "Description of the transaction (e.g. the charge or payout it relates to).",+ "charge_id": "Identifier of the related charge, if any.",+ "order_id": "Identifier of the related order, if any.",+ "payout_id": "Identifier of the related payout, if any.",+ "transfer_id": "Identifier of the balance transfer behind the transaction.",+ "debit": "Amount taken from the balance, as a negative number, or null for credits.",+ "credit": "Amount added to the balance, or null for debits.",+ "currency": "Three-letter currency code of the transaction.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/hitpay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/hitpay.pyindex 2846526521db..87156650af60 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/hitpay.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/hitpay.py@@ -60,11 +60,11 @@ def _redact_values(api_key: str, platform_api_key: Optional[str]) -> tuple[str,return tuple(value for value in (api_key, platform_api_key) if value)-def _format_charge_date(value: Any) -> str:- """Format the incremental watermark for Charges' `date_from` filter (YYYY-MM-DD, day granularity).+def _format_date_filter(value: Any) -> str:+ """Format the incremental watermark for an endpoint's date filter (YYYY-MM-DD, day granularity).Truncating to the day rounds the lower bound down, so a resumed sync re-reads the watermark- day rather than skipping charges created later on it. The merge upsert dedupes the overlap.+ day rather than skipping rows created later on it. The merge upsert dedupes the overlap."""normalized = coerce_datetime_to_utc(value)if normalized is None:@@ -77,7 +77,7 @@ def _incremental_param(cursor_path: str) -> dict[str, Any]:"type": "incremental","cursor_path": cursor_path,"initial_value": "1970-01-01",- "convert": _format_charge_date,+ "convert": _format_date_filter,}@@ -95,7 +95,7 @@ def get_resource(endpoint: str, should_use_incremental_field: bool) -> EndpointRparams: dict[str, Any] = {}if is_incremental:- params["date_from"] = _incremental_param("created_at")+ params[config.incremental_param] = _incremental_param(config.incremental_fields[0]["field"])return {"name": endpoint,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/settings.pyindex 44e2af31b246..2e31e0dcf375 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/settings.py@@ -8,12 +8,12 @@ from products.warehouse_sources.backend.types import IncrementalField, IncrementPaginationKind = Literal["cursor", "page", "single"]-def _created_at_incremental_fields() -> list[IncrementalField]:+def _datetime_incremental_fields(field_name: str = "created_at") -> list[IncrementalField]:return [{- "label": "created_at",+ "label": field_name,"type": IncrementalFieldType.DateTime,- "field": "created_at",+ "field": field_name,"field_type": IncrementalFieldType.DateTime,},]@@ -31,6 +31,8 @@ class HitpayEndpointConfig:primary_keys: list[str] = field(default_factory=lambda: ["id"])partition_key: str | None = "created_at"incremental_fields: list[IncrementalField] = field(default_factory=list)+ # Day-granularity (YYYY-MM-DD) lower-bound filter on the incremental field.+ incremental_param: str = "date_from"HITPAY_ENDPOINTS: dict[str, HitpayEndpointConfig] = {@@ -47,7 +49,7 @@ HITPAY_ENDPOINTS: dict[str, HitpayEndpointConfig] = {table_name="charges",path="/v1/charges",pagination="cursor",- incremental_fields=_created_at_incremental_fields(),+ incremental_fields=_datetime_incremental_fields(),),"SubscriptionPlans": HitpayEndpointConfig(name="SubscriptionPlans",@@ -63,6 +65,33 @@ HITPAY_ENDPOINTS: dict[str, HitpayEndpointConfig] = {pagination="page",page_param="page",),+ "Payouts": HitpayEndpointConfig(+ name="Payouts",+ table_name="payouts",+ path="/v1/payouts",+ pagination="cursor",+ # Payouts change status after creation and the API only filters on creation date, so+ # an incremental sync would never refresh older payouts. Full refresh only.+ ),+ "Invoices": HitpayEndpointConfig(+ name="Invoices",+ table_name="invoices",+ path="/v1/invoices",+ pagination="cursor",+ ),+ "BalanceTransactions": HitpayEndpointConfig(+ name="BalanceTransactions",+ table_name="balance_transactions",+ # `adyen` is the only balance provider HitPay documents (Singapore merchants with a+ # Cards by Adyen balance).+ path="/v1/balances/adyen/transactions",+ pagination="cursor",+ # Rows carry no `id`; a transfer can post more than one ledger line, so pair it with the timestamp.+ primary_keys=["transfer_id", "datetime"],+ partition_key="datetime",+ incremental_fields=_datetime_incremental_fields("datetime"),+ incremental_param="created_at_from",+ ),"RecurringBilling": HitpayEndpointConfig(name="RecurringBilling",table_name="recurring_billing",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/source.pyindex ab38d4fe4ac1..8694c0c51ace 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/source.py@@ -111,7 +111,8 @@ class HitpaySource(ResumableSource[HitpaySourceConfig, HitpayResumeConfig]):label="HitPay",caption=("Connect your HitPay account using an API key to sync payment requests, charges, "- "subscription plans, customers, and recurring billing. Find your key in the HitPay "+ "subscription plans, customers, recurring billing, payouts, invoices, and balance "+ "transactions. Find your key in the HitPay ""dashboard under **Settings** > **Payment Gateway**."),keywords=["payments", "paynow", "duitnow", "billing"],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay.pyindex 5260434d63f3..9968d8c82232 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay.py@@ -15,7 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.hitpay import HitpaySourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.hitpay.hitpay import (HitpayResumeConfig,- _format_charge_date,+ _format_date_filter,_paginator_for,base_url_for_environment,validate_credentials,@@ -40,7 +40,7 @@ class TestBaseUrlForEnvironment:assert base_url_for_environment(environment) == expected-class TestFormatChargeDate:+class TestFormatDateFilter:@parameterized.expand([("utc_datetime", datetime(2026, 3, 4, 2, 58, 14, tzinfo=UTC), "2026-03-04"),@@ -50,7 +50,7 @@ class TestFormatChargeDate:])def test_format(self, _name: str, value: Any, expected: str) -> None:- assert _format_charge_date(value) == expected+ assert _format_date_filter(value) == expectedclass TestPaginatorFor:@@ -113,16 +113,31 @@ class TestHitpaySourceCursorEndpoint:assert result.paths == ["/v1/charges", "/v1/charges"]assert result.params("cursor") == [None, "abc"]- def test_incremental_date_from_sent_when_enabled(self) -> None:+ @parameterized.expand(+ [+ ("charges", "Charges", "created_at", "/v1/charges", "date_from"),+ (+ "balance_transactions",+ "BalanceTransactions",+ "datetime",+ "/v1/balances/adyen/transactions",+ "created_at_from",+ ),+ ]+ )+ def test_incremental_date_filter_sent_when_enabled(+ self, _name: str, endpoint: str, incremental_field: str, expected_path: str, expected_param: str+ ) -> None:result = _driver().run(- "Charges",+ endpoint,[_response([{"id": "1"}], extra={"meta": {"next_cursor": None}})],- incremental_field="created_at",+ incremental_field=incremental_field,db_incremental_field_last_value=datetime(2026, 3, 4, tzinfo=UTC),)assert result.raised is None- assert result.params("date_from") == ["2026-03-04"]+ assert result.paths == [expected_path]+ assert result.params(expected_param) == ["2026-03-04"]class TestRecurringBillingFanOut:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay_source.pyindex 11398689f9fc..dd9b90a63df0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay_source.py@@ -7,8 +7,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.hitpay import HitpaySourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.hitpay.source import HitpaySource-_INCREMENTAL_ENDPOINTS = {"Charges"}-_FULL_REFRESH_ENDPOINTS = {"PaymentRequests", "SubscriptionPlans", "Customers", "RecurringBilling"}+_INCREMENTAL_ENDPOINTS = {"Charges", "BalanceTransactions"}+_FULL_REFRESH_ENDPOINTS = {+ "PaymentRequests",+ "SubscriptionPlans",+ "Customers",+ "RecurringBilling",+ "Invoices",+ "Payouts",+}class TestHitpaySource: