Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add hitpay payouts, invoices and balance transactions (#122037)

Tom Owers4741ee8a95af
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 50c4ee489ce3..e68085aeffea 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5967,10 +5967,10 @@ Today (5): `Charges`, `Customers`, `PaymentRequests`, `RecurringBilling`, `Subsc
Diffed against: <https://docs.hitpayapp.com/openapi.json>
-- [ ] `Payouts (GET /v1/payouts)` — reconcile collected payments with settlement amounts and bank payout status (high)
-- [ ] `Balance Transactions (GET /v1/balances/{balance_provider}/transactions)` — explain balance changes from payments, fees, refunds, and settlements (high)
-- [ ] `Invoices (GET /v1/invoices)` — measure invoiced revenue, unpaid balances, and collection status alongside payment requests (high)
-- [ ] `Orders (GET /v1/orders)` — analyze order revenue and purchase activity beyond individual payment records (high)
+- [x] `Payouts (GET /v1/payouts)` — reconcile collected payments with settlement amounts and bank payout status (high)
+- [x] `Balance Transactions (GET /v1/balances/{balance_provider}/transactions)` — explain balance changes from payments, fees, refunds, and settlements (high)
+- [x] `Invoices (GET /v1/invoices)` — measure invoiced revenue, unpaid balances, and collection status alongside payment requests (high)
+- [ ] `Orders (GET /v1/orders)` — analyze order revenue and purchase activity beyond individual payment records (high); skipped, the spec documents no page or cursor parameter for this endpoint and describes the response as a single order, so a full sync cannot be paged reliably
- [ ] `Products (GET /v1/products)` — analyze product sales, prices, inventory, and catalog status (medium)
- [ ] `Transfers (GET /v1/transfers)` — track outgoing payments to third parties, including their amounts and completion status (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/canonical_descriptions.py
index 080d9ce436bb..b1a047911d69 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/canonical_descriptions.py
@@ -117,4 +117,58 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "Time the agreement was last updated.",
},
},
+ "Payouts": {
+ "description": "A settlement of the business's available balance to its own linked bank account.",
+ "docs_url": "https://docs.hitpayapp.com/reference/list-payouts",
+ "columns": {
+ "id": "Unique identifier for the payout.",
+ "status": "Payout status: scheduled, processing, paid, failed, canceled, or pending.",
+ "currency": "Three-letter currency code the payout was sent in.",
+ "amount": "Payout amount in major units of the payout currency.",
+ "source_currency": "Three-letter currency code of the balance the payout was drawn from.",
+ "source_amount": "Amount drawn from the source balance, in major units.",
+ "bank_name": "Name of the bank the payout was sent to.",
+ "bank_account": "Masked number of the receiving bank account.",
+ "remark": "Free-text note attached to the payout.",
+ "reference": "Reference for the payout.",
+ "failure_code": "Failure code, present when the payout failed.",
+ "failure_message": "Failure reason, present when the payout failed.",
+ "arrival_date": "Date the payout is expected to arrive in, or arrived in, the bank account.",
+ "created_at": "Time the payout was created.",
+ "updated_at": "Time the payout was last updated.",
+ },
+ },
+ "Invoices": {
+ "description": "An invoice the merchant issues to a customer, with its amount, dates, and payment link.",
+ "docs_url": "https://docs.hitpayapp.com/reference/get-invoices",
+ "columns": {
+ "id": "Unique identifier for the invoice.",
+ "invoice_number": "Human-readable invoice number.",
+ "reference": "Merchant-supplied reference for the invoice.",
+ "email": "Email address of the customer the invoice was issued to.",
+ "currency": "Three-letter currency code for the invoice amount.",
+ "amount": "Invoice amount in major units of the invoice currency.",
+ "status": "Invoice status: draft, sent, pending, overdue, or paid.",
+ "invoice_date": "Date the invoice was issued.",
+ "due_date": "Date payment of the invoice is due.",
+ "invoice_link": "Hosted link the customer opens to view and pay the invoice.",
+ "created_at": "Time the invoice was created.",
+ "updated_at": "Time the invoice was last updated.",
+ },
+ },
+ "BalanceTransactions": {
+ "description": "A ledger line on the Cards (by Adyen) balance, such as a credit for a charge or a debit for a payout.",
+ "docs_url": "https://docs.hitpayapp.com/reference/get-balance-transactions",
+ "columns": {
+ "datetime": "Time the transaction was booked on the balance.",
+ "description": "Description of the transaction (e.g. the charge or payout it relates to).",
+ "charge_id": "Identifier of the related charge, if any.",
+ "order_id": "Identifier of the related order, if any.",
+ "payout_id": "Identifier of the related payout, if any.",
+ "transfer_id": "Identifier of the balance transfer behind the transaction.",
+ "debit": "Amount taken from the balance, as a negative number, or null for credits.",
+ "credit": "Amount added to the balance, or null for debits.",
+ "currency": "Three-letter currency code of the transaction.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/hitpay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/hitpay.py
index 2846526521db..87156650af60 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/hitpay.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/hitpay.py
@@ -60,11 +60,11 @@ def _redact_values(api_key: str, platform_api_key: Optional[str]) -> tuple[str,
return tuple(value for value in (api_key, platform_api_key) if value)
-def _format_charge_date(value: Any) -> str:
- """Format the incremental watermark for Charges' `date_from` filter (YYYY-MM-DD, day granularity).
+def _format_date_filter(value: Any) -> str:
+ """Format the incremental watermark for an endpoint's date filter (YYYY-MM-DD, day granularity).
Truncating to the day rounds the lower bound down, so a resumed sync re-reads the watermark
- day rather than skipping charges created later on it. The merge upsert dedupes the overlap.
+ day rather than skipping rows created later on it. The merge upsert dedupes the overlap.
"""
normalized = coerce_datetime_to_utc(value)
if normalized is None:
@@ -77,7 +77,7 @@ def _incremental_param(cursor_path: str) -> dict[str, Any]:
"type": "incremental",
"cursor_path": cursor_path,
"initial_value": "1970-01-01",
- "convert": _format_charge_date,
+ "convert": _format_date_filter,
}
@@ -95,7 +95,7 @@ def get_resource(endpoint: str, should_use_incremental_field: bool) -> EndpointR
params: dict[str, Any] = {}
if is_incremental:
- params["date_from"] = _incremental_param("created_at")
+ params[config.incremental_param] = _incremental_param(config.incremental_fields[0]["field"])
return {
"name": endpoint,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/settings.py
index 44e2af31b246..2e31e0dcf375 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/settings.py
@@ -8,12 +8,12 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
PaginationKind = Literal["cursor", "page", "single"]
-def _created_at_incremental_fields() -> list[IncrementalField]:
+def _datetime_incremental_fields(field_name: str = "created_at") -> list[IncrementalField]:
return [
{
- "label": "created_at",
+ "label": field_name,
"type": IncrementalFieldType.DateTime,
- "field": "created_at",
+ "field": field_name,
"field_type": IncrementalFieldType.DateTime,
},
]
@@ -31,6 +31,8 @@ class HitpayEndpointConfig:
primary_keys: list[str] = field(default_factory=lambda: ["id"])
partition_key: str | None = "created_at"
incremental_fields: list[IncrementalField] = field(default_factory=list)
+ # Day-granularity (YYYY-MM-DD) lower-bound filter on the incremental field.
+ incremental_param: str = "date_from"
HITPAY_ENDPOINTS: dict[str, HitpayEndpointConfig] = {
@@ -47,7 +49,7 @@ HITPAY_ENDPOINTS: dict[str, HitpayEndpointConfig] = {
table_name="charges",
path="/v1/charges",
pagination="cursor",
- incremental_fields=_created_at_incremental_fields(),
+ incremental_fields=_datetime_incremental_fields(),
),
"SubscriptionPlans": HitpayEndpointConfig(
name="SubscriptionPlans",
@@ -63,6 +65,33 @@ HITPAY_ENDPOINTS: dict[str, HitpayEndpointConfig] = {
pagination="page",
page_param="page",
),
+ "Payouts": HitpayEndpointConfig(
+ name="Payouts",
+ table_name="payouts",
+ path="/v1/payouts",
+ pagination="cursor",
+ # Payouts change status after creation and the API only filters on creation date, so
+ # an incremental sync would never refresh older payouts. Full refresh only.
+ ),
+ "Invoices": HitpayEndpointConfig(
+ name="Invoices",
+ table_name="invoices",
+ path="/v1/invoices",
+ pagination="cursor",
+ ),
+ "BalanceTransactions": HitpayEndpointConfig(
+ name="BalanceTransactions",
+ table_name="balance_transactions",
+ # `adyen` is the only balance provider HitPay documents (Singapore merchants with a
+ # Cards by Adyen balance).
+ path="/v1/balances/adyen/transactions",
+ pagination="cursor",
+ # Rows carry no `id`; a transfer can post more than one ledger line, so pair it with the timestamp.
+ primary_keys=["transfer_id", "datetime"],
+ partition_key="datetime",
+ incremental_fields=_datetime_incremental_fields("datetime"),
+ incremental_param="created_at_from",
+ ),
"RecurringBilling": HitpayEndpointConfig(
name="RecurringBilling",
table_name="recurring_billing",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/source.py
index ab38d4fe4ac1..8694c0c51ace 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/source.py
@@ -111,7 +111,8 @@ class HitpaySource(ResumableSource[HitpaySourceConfig, HitpayResumeConfig]):
label="HitPay",
caption=(
"Connect your HitPay account using an API key to sync payment requests, charges, "
- "subscription plans, customers, and recurring billing. Find your key in the HitPay "
+ "subscription plans, customers, recurring billing, payouts, invoices, and balance "
+ "transactions. Find your key in the HitPay "
"dashboard under **Settings** > **Payment Gateway**."
),
keywords=["payments", "paynow", "duitnow", "billing"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay.py
index 5260434d63f3..9968d8c82232 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay.py
@@ -15,7 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.hitpay import HitpaySourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.hitpay.hitpay import (
HitpayResumeConfig,
- _format_charge_date,
+ _format_date_filter,
_paginator_for,
base_url_for_environment,
validate_credentials,
@@ -40,7 +40,7 @@ class TestBaseUrlForEnvironment:
assert base_url_for_environment(environment) == expected
-class TestFormatChargeDate:
+class TestFormatDateFilter:
@parameterized.expand(
[
("utc_datetime", datetime(2026, 3, 4, 2, 58, 14, tzinfo=UTC), "2026-03-04"),
@@ -50,7 +50,7 @@ class TestFormatChargeDate:
]
)
def test_format(self, _name: str, value: Any, expected: str) -> None:
- assert _format_charge_date(value) == expected
+ assert _format_date_filter(value) == expected
class TestPaginatorFor:
@@ -113,16 +113,31 @@ class TestHitpaySourceCursorEndpoint:
assert result.paths == ["/v1/charges", "/v1/charges"]
assert result.params("cursor") == [None, "abc"]
- def test_incremental_date_from_sent_when_enabled(self) -> None:
+ @parameterized.expand(
+ [
+ ("charges", "Charges", "created_at", "/v1/charges", "date_from"),
+ (
+ "balance_transactions",
+ "BalanceTransactions",
+ "datetime",
+ "/v1/balances/adyen/transactions",
+ "created_at_from",
+ ),
+ ]
+ )
+ def test_incremental_date_filter_sent_when_enabled(
+ self, _name: str, endpoint: str, incremental_field: str, expected_path: str, expected_param: str
+ ) -> None:
result = _driver().run(
- "Charges",
+ endpoint,
[_response([{"id": "1"}], extra={"meta": {"next_cursor": None}})],
- incremental_field="created_at",
+ incremental_field=incremental_field,
db_incremental_field_last_value=datetime(2026, 3, 4, tzinfo=UTC),
)
assert result.raised is None
- assert result.params("date_from") == ["2026-03-04"]
+ assert result.paths == [expected_path]
+ assert result.params(expected_param) == ["2026-03-04"]
class TestRecurringBillingFanOut:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay_source.py
index 11398689f9fc..dd9b90a63df0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hitpay/tests/test_hitpay_source.py
@@ -7,8 +7,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.hitpay import HitpaySourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.hitpay.source import HitpaySource
-_INCREMENTAL_ENDPOINTS = {"Charges"}
-_FULL_REFRESH_ENDPOINTS = {"PaymentRequests", "SubscriptionPlans", "Customers", "RecurringBilling"}
+_INCREMENTAL_ENDPOINTS = {"Charges", "BalanceTransactions"}
+_FULL_REFRESH_ENDPOINTS = {
+ "PaymentRequests",
+ "SubscriptionPlans",
+ "Customers",
+ "RecurringBilling",
+ "Invoices",
+ "Payouts",
+}
class TestHitpaySource: