Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add plaid recurring, balance, identity, items (#126777)

Tom Owersba218b6f102f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 04de4566a0e9..a3dc4d67c250 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9525,10 +9525,10 @@ Diffed against: <https://raw.githubusercontent.com/plaid/plaid-openapi/master/20
- [x] `/investments/transactions/get` — buys, sells, dividends and fees; the investments analogue of the transactions table we already sync (high)
- [x] `/liabilities/get` — loan and credit-card detail (APR, minimum payment, due date, origination) that never appears in accounts or transactions (high)
- [x] `/institutions/get_by_id (and /institutions/get)` — lookup resolving the institution_id on the item whose accounts we sync - today the bank is only an opaque ID (high)
-- [ ] `/transactions/recurring/get` — Plaid-derived recurring inflow/outflow streams - the headline subscription and income-detection product (high)
-- [ ] `/accounts/balance/get` — live available/current balance refresh; the balances embedded in /accounts/get are cached and can be stale (medium)
-- [ ] `/identity/get` — account owner names, emails, phones and addresses, the standard join key to customer records (medium)
-- [ ] `/item/get` — item metadata - institution, enabled products, consent expiry, last error - needed to explain gaps in synced data (medium)
+- [x] `/transactions/recurring/get` — Plaid-derived recurring inflow/outflow streams - the headline subscription and income-detection product (high)
+- [x] `/accounts/balance/get` — live available/current balance refresh; the balances embedded in /accounts/get are cached and can be stale (medium)
+- [x] `/identity/get` — account owner names, emails, phones and addresses, the standard join key to customer records (medium)
+- [x] `/item/get` — item metadata - institution, enabled products, consent expiry, last error - needed to explain gaps in synced data (medium)
- [ ] `/transfer/list` — money-movement records for Plaid Transfer customers, an analytical object with no equivalent today (medium)
- [ ] `/transfer/event/list` — transfer state-transition history (pending, posted, returned) for settlement and return-rate analysis (medium)
- [ ] `/statements/list` — available bank statement metadata per account for reconciliation coverage checks (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py
index 47c0d3dd5180..ae40b527a216 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py
@@ -112,4 +112,69 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"connection_availability": "Whether Plaid supports new connections to the institution.",
},
},
+ "recurring_transactions": {
+ "description": "A recurring stream of inflow (income) or outflow (expense, subscription) transactions that Plaid detected on an account of the linked Item.",
+ "docs_url": "https://plaid.com/docs/api/products/transactions/#transactionsrecurringget",
+ "columns": {
+ "stream_id": "Unique identifier for the recurring stream.",
+ "account_id": "ID of the account the stream belongs to.",
+ "direction": "Whether the stream is an inflow (money in) or an outflow (money out).",
+ "description": "Description of the transaction stream.",
+ "merchant_name": "Merchant associated with the stream.",
+ "first_date": "Posted date of the earliest transaction in the stream.",
+ "last_date": "Posted date of the latest transaction in the stream.",
+ "predicted_next_date": "Predicted date of the next payment in the stream, if Plaid can predict it.",
+ "frequency": "Frequency of the stream (e.g. WEEKLY, BIWEEKLY, SEMI_MONTHLY, MONTHLY, ANNUALLY).",
+ "transaction_ids": "IDs of the transactions that belong to the stream.",
+ "average_amount": "Average amount of the transactions in the stream.",
+ "last_amount": "Amount of the latest transaction in the stream.",
+ "is_active": "Whether the stream is currently active.",
+ "status": "Status of the stream (e.g. MATURE, EARLY_DETECTION, TOMBSTONED).",
+ "personal_finance_category": "Plaid's personal finance category of the stream.",
+ "is_user_modified": "Whether the stream was modified by the end user.",
+ },
+ },
+ "balances": {
+ "description": "A financial account of the linked Plaid Item with balances that Plaid refreshed in real time from the institution.",
+ "docs_url": "https://plaid.com/docs/api/products/signal/#accountsbalanceget",
+ "columns": {
+ "account_id": "Unique identifier for the account.",
+ "balances": "Real-time balance information for the account (available, current, limit, currency).",
+ "name": "The account's name, as provided by the institution.",
+ "mask": "The last 2-4 digits of the account number.",
+ "type": "Account type (e.g. depository, credit, loan, investment).",
+ "subtype": "Account subtype (e.g. checking, savings, credit card).",
+ },
+ },
+ "identity": {
+ "description": "A financial account of the linked Plaid Item with the identity information the institution holds about its owners.",
+ "docs_url": "https://plaid.com/docs/api/products/identity/#identityget",
+ "columns": {
+ "account_id": "Unique identifier for the account.",
+ "owners": "Account owners, each with names, emails, phone numbers and addresses as provided by the institution.",
+ "name": "The account's name, as provided by the institution.",
+ "mask": "The last 2-4 digits of the account number.",
+ "type": "Account type (e.g. depository, credit, loan, investment).",
+ "subtype": "Account subtype (e.g. checking, savings, credit card).",
+ },
+ },
+ "items": {
+ "description": "The linked Plaid Item: the connection to a financial institution, with its products, consent and error state.",
+ "docs_url": "https://plaid.com/docs/api/items/#itemget",
+ "columns": {
+ "item_id": "Unique identifier for the Item.",
+ "institution_id": "ID of the institution the Item is connected to.",
+ "institution_name": "Name of the institution the Item is connected to.",
+ "products": "Products the Item was initialized with in Link or added after Link.",
+ "billed_products": "Products that Plaid bills for on the Item.",
+ "available_products": "Products the institution supports that the Item can still add.",
+ "consented_products": "Products the end user consented to.",
+ "consent_expiration_time": "Date and time the end user's consent expires, if the institution sets an expiry.",
+ "error": "The error that currently blocks the Item from updating, if any.",
+ "update_type": "Whether the Item updates in the background or needs user interaction to refresh.",
+ "webhook": "URL Plaid sends webhooks for the Item to.",
+ "created_at": "Date and time the Item was created.",
+ "status": "Status of the last successful and failed updates of the Item's transactions and investments data, and of the last webhook.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
index dc3285afed1e..0acd900306c2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
@@ -1,7 +1,7 @@
import time
import dataclasses
from collections.abc import Iterator
-from datetime import UTC, date, datetime
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
import requests
@@ -27,6 +27,9 @@ DEFAULT_START_DATE = "2000-01-01"
# /transactions/get is limited to 30 req/min per Item — space requests out.
REQUEST_INTERVAL_SECONDS = 2.1
REQUEST_TIMEOUT_SECONDS = 120
+# Capital One non-depository accounts and Apple Card have no real-time balance, and Plaid rejects
+# /accounts/balance/get for them without options.min_last_updated_datetime. Other institutions ignore it.
+BALANCE_MAX_STALENESS = timedelta(days=7)
MAX_RETRY_ATTEMPTS = 5
@@ -53,6 +56,8 @@ PRODUCT_GATED_ENDPOINTS = {
"investment_holdings": "investments",
"investment_transactions": "investments",
"liabilities": "liabilities",
+ "recurring_transactions": "transactions",
+ "identity": "identity",
}
@@ -92,8 +97,12 @@ def _format_date(value: Any) -> str:
return str(value)[:10]
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
def _today() -> str:
- return datetime.now(UTC).strftime("%Y-%m-%d")
+ return _now().strftime("%Y-%m-%d")
def _error_code(response: requests.Response) -> str | None:
@@ -172,6 +181,20 @@ def get_rows(
yield items
return
+ if endpoint == "balances":
+ min_last_updated = (_now() - BALANCE_MAX_STALENESS).strftime("%Y-%m-%dT%H:%M:%SZ")
+ data = post("/accounts/balance/get", {"options": {"min_last_updated_datetime": min_last_updated}})
+ items = data.get("accounts", []) or []
+ if items:
+ yield items
+ return
+
+ if endpoint == "items":
+ data = post("/item/get", {})
+ if item := data.get("item"):
+ yield [{**item, "status": data.get("status")}]
+ return
+
if endpoint == "institutions":
institution_id = get_item().get("institution_id")
if not institution_id:
@@ -212,6 +235,24 @@ def get_rows(
yield rows
return
+ if endpoint == "recurring_transactions":
+ data = post("/transactions/recurring/get", {})
+ rows = [
+ {**stream, "direction": direction}
+ for direction, streams_key in (("inflow", "inflow_streams"), ("outflow", "outflow_streams"))
+ for stream in data.get(streams_key) or []
+ ]
+ if rows:
+ yield rows
+ return
+
+ if endpoint == "identity":
+ data = post("/identity/get", {})
+ items = data.get("accounts", []) or []
+ if items:
+ yield items
+ return
+
path, rows_key, total_key = OFFSET_PAGINATED_ENDPOINTS[endpoint]
start_date = (
_format_date(db_incremental_field_last_value)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py
index eaf0791c4230..9a5d7c04c7f1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py
@@ -65,6 +65,31 @@ PLAID_ENDPOINTS: dict[str, PlaidEndpointConfig] = {
name="institutions",
primary_keys=["institution_id"],
),
+ "recurring_transactions": PlaidEndpointConfig(
+ name="recurring_transactions",
+ primary_keys=["stream_id"],
+ # Recurring Transactions is a separately billed add-on to Transactions.
+ should_sync_default=False,
+ description="Syncing this table starts Plaid's Recurring Transactions subscription for the Item if it does not have one yet.",
+ ),
+ "balances": PlaidEndpointConfig(
+ name="balances",
+ primary_keys=["account_id"],
+ # Plaid bills Balance per request, unlike the cached balances on accounts.
+ should_sync_default=False,
+ description="Each sync of this table makes one billed Plaid Balance request.",
+ ),
+ # One row per account, with the account owners' names, emails, phone numbers and addresses.
+ "identity": PlaidEndpointConfig(
+ name="identity",
+ primary_keys=["account_id"],
+ should_sync_default=False,
+ description="Contains the names, emails, phone numbers and addresses of the account owners.",
+ ),
+ "items": PlaidEndpointConfig(
+ name="items",
+ primary_keys=["item_id"],
+ ),
}
ENDPOINTS = tuple(PLAID_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py
index 53d24fc9c977..e3e5140c0d83 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py
@@ -85,7 +85,7 @@ class PlaidSource(ResumableSource[PlaidSourceConfig, PlaidResumeConfig]):
category=DataWarehouseSourceCategory.PAYMENTS___BILLING,
label="Plaid",
keywords=["bank", "banking", "open banking", "financial", "fintech", "transactions"],
- caption="""Connect a Plaid Item to pull its accounts, transactions, investments, liabilities, and institution into the PostHog Data warehouse.
+ caption="""Connect a Plaid Item to pull its accounts, balances, identity, transactions, recurring transactions, investments, liabilities, institution, and Item details into the PostHog Data warehouse.
You can find your client ID and secret in the [Plaid dashboard](https://dashboard.plaid.com/developers/keys). The access token identifies one linked Item (institution connection), obtained when a user completes Plaid Link — add one source per Item.""",
iconPath="/static/services/plaid.png",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py
index 96b3465429d8..345fc0972cec 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py
@@ -79,12 +79,28 @@ class TestValidateCredentials:
class TestGetRowsAccounts:
- def test_accounts_single_fetch(self) -> None:
- result = _driver().run("accounts", [ScriptedResponse(json={"accounts": [{"account_id": "a1"}]})])
+ @pytest.mark.parametrize(
+ "endpoint, preamble, path",
+ [
+ ("accounts", [], "/accounts/get"),
+ ("balances", [], "/accounts/balance/get"),
+ ("identity", [_item(products=["identity"])], "/identity/get"),
+ ],
+ )
+ def test_account_endpoints_single_fetch(self, endpoint: str, preamble: list[ScriptedResponse], path: str) -> None:
+ result = _driver().run(endpoint, [*preamble, ScriptedResponse(json={"accounts": [{"account_id": "a1"}]})])
assert result.raised is None
assert result.items == [[{"account_id": "a1"}]]
- assert result.paths == ["/accounts/get"]
+ assert result.paths == [*(["/item/get"] if preamble else []), path]
+
+ @mock.patch(f"{_MODULE}._now")
+ def test_balances_send_min_last_updated_datetime(self, mock_now: mock.MagicMock) -> None:
+ mock_now.return_value = datetime(2024, 6, 8, 12, 0, 0, tzinfo=UTC)
+ result = _driver().run("balances", [ScriptedResponse(json={"accounts": []})])
+
+ assert result.raised is None
+ assert result.requests[0].json()["options"] == {"min_last_updated_datetime": "2024-06-01T12:00:00Z"}
class TestGetRowsTransactions:
@@ -151,9 +167,12 @@ class TestGetRowsTransactions:
class TestGetRowsProductGatedEndpoints:
- @pytest.mark.parametrize("endpoint", ["investment_holdings", "investment_transactions", "liabilities"])
+ @pytest.mark.parametrize(
+ "endpoint",
+ ["investment_holdings", "investment_transactions", "liabilities", "recurring_transactions", "identity"],
+ )
def test_skips_item_without_product(self, endpoint: str) -> None:
- result = _driver().run(endpoint, [_item(billed_products=["transactions"], products=["transactions"])])
+ result = _driver().run(endpoint, [_item(billed_products=["auth"], products=["auth"])])
assert result.raised is None
assert result.rows == []
@@ -234,6 +253,41 @@ class TestGetRowsProductGatedEndpoints:
{"account_id": "m1", "interest_rate": {"percentage": 3.5}, "liability_type": "mortgage"},
]
+ def test_recurring_streams_flattened_with_direction(self) -> None:
+ result = _driver().run(
+ "recurring_transactions",
+ [
+ _item(products=["transactions"]),
+ ScriptedResponse(
+ json={
+ "inflow_streams": [{"stream_id": "in1", "frequency": "SEMI_MONTHLY"}],
+ "outflow_streams": [{"stream_id": "out1", "frequency": "MONTHLY"}],
+ "updated_datetime": "2024-05-01T00:00:00Z",
+ }
+ ),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"stream_id": "in1", "frequency": "SEMI_MONTHLY", "direction": "inflow"},
+ {"stream_id": "out1", "frequency": "MONTHLY", "direction": "outflow"},
+ ]
+ assert result.paths == ["/item/get", "/transactions/recurring/get"]
+
+
+class TestGetRowsItems:
+ def test_item_row_carries_status(self) -> None:
+ status = {"transactions": {"last_successful_update": "2024-05-01T00:00:00Z"}}
+ result = _driver().run(
+ "items",
+ [ScriptedResponse(json={"item": {"item_id": "item-1", "error": None}, "status": status})],
+ )
+
+ assert result.raised is None
+ assert result.rows == [{"item_id": "item-1", "error": None, "status": status}]
+ assert result.paths == ["/item/get"]
+
class TestGetRowsInstitutions:
def test_looks_up_item_institution(self) -> None: