Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add plaid investments, liabilities and institutions tables (#126278)

Tom Owerse89d0b14f2f1
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 89d7494eae9f..8bb831e8d24f 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -854,7 +854,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/pipeliner/pipeliner.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pipeliner/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/planhat/planhat.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/planhat/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/plausible/plausible.py
diff --git a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
index bff693f0d231..2cdec71af8e6 100644
--- a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
+++ b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
@@ -308,7 +308,6 @@ pendo/pendo.py::_iter_list_endpoint
persona/persona.py::get_rows
pinterest_ads/pinterest_ads.py::_iter_entity_rows
pinterest_ads/pinterest_ads.py::_iter_fanned_out_analytics
-plaid/plaid.py::get_rows
platform_sh/platform_sh.py::get_rows
polar/polar.py::get_rows
postgres/postgres.py::postgres_source.get_rows.offset_chunking
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 0710066e35a8..c7928131b61e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9494,10 +9494,10 @@ Today (2): `accounts`, `transactions`
Diffed against: <https://raw.githubusercontent.com/plaid/plaid-openapi/master/2020-09-14.yml>
-- [ ] `/investments/holdings/get` — positions and cost basis per investment account - accounts alone give no portfolio detail (high)
-- [ ] `/investments/transactions/get` — buys, sells, dividends and fees; the investments analogue of the transactions table we already sync (high)
-- [ ] `/liabilities/get` — loan and credit-card detail (APR, minimum payment, due date, origination) that never appears in accounts or transactions (high)
-- [ ] `/institutions/get_by_id (and /institutions/get)` — lookup resolving the institution_id on the item whose accounts we sync - today the bank is only an opaque ID (high)
+- [x] `/investments/holdings/get` — positions and cost basis per investment account - accounts alone give no portfolio detail (high)
+- [x] `/investments/transactions/get` — buys, sells, dividends and fees; the investments analogue of the transactions table we already sync (high)
+- [x] `/liabilities/get` — loan and credit-card detail (APR, minimum payment, due date, origination) that never appears in accounts or transactions (high)
+- [x] `/institutions/get_by_id (and /institutions/get)` — lookup resolving the institution_id on the item whose accounts we sync - today the bank is only an opaque ID (high)
- [ ] `/transactions/recurring/get` — Plaid-derived recurring inflow/outflow streams - the headline subscription and income-detection product (high)
- [ ] `/accounts/balance/get` — live available/current balance refresh; the balances embedded in /accounts/get are cached and can be stale (medium)
- [ ] `/identity/get` — account owner names, emails, phones and addresses, the standard join key to customer records (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py
index 9d542d15bd14..47c0d3dd5180 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py
@@ -44,4 +44,72 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"personal_finance_category": "Detailed and primary personal finance category for the transaction.",
},
},
+ "investment_transactions": {
+ "description": "An investment transaction (buy, sell, dividend, fee, transfer, etc.) on an investment account belonging to a linked Plaid Item.",
+ "docs_url": "https://plaid.com/docs/api/products/investments/#investmentstransactionsget",
+ "columns": {
+ "investment_transaction_id": "Unique identifier for the investment transaction.",
+ "account_id": "ID of the account the transaction posted against.",
+ "security_id": "ID of the security the transaction relates to.",
+ "date": "Posting date of the transaction, typically the settlement date.",
+ "transaction_datetime": "When the order was initiated, for select institutions.",
+ "name": "The institution's description of the transaction.",
+ "quantity": "Units of the security involved (positive for buys, negative for sells).",
+ "amount": "Complete value of the transaction (positive when cash is debited, negative when credited).",
+ "price": "Price of the security at which the transaction occurred.",
+ "fees": "Combined value of all fees applied to the transaction.",
+ "type": "Transaction type (buy, sell, cancel, cash, fee, transfer).",
+ "subtype": "Detailed transaction subtype (e.g. dividend, interest, contribution).",
+ "iso_currency_code": "ISO-4217 currency code of the transaction.",
+ },
+ },
+ "investment_holdings": {
+ "description": "A position in one security held in an investment account belonging to a linked Plaid Item.",
+ "docs_url": "https://plaid.com/docs/api/products/investments/#investmentsholdingsget",
+ "columns": {
+ "account_id": "ID of the account that holds the position.",
+ "security_id": "ID of the security held.",
+ "quantity": "Total quantity of the security held, as reported by the institution.",
+ "cost_basis": "Total cost basis of the holding (the amount spent to acquire the assets currently held).",
+ "institution_price": "Last price the institution gave for the security.",
+ "institution_price_as_of": "Date at which institution_price was current.",
+ "institution_value": "Value of the holding, as reported by the institution.",
+ "vested_quantity": "Quantity of vested assets held, for equities.",
+ "vested_value": "Value of the vested holdings, as reported by the institution.",
+ "iso_currency_code": "ISO-4217 currency code of the holding.",
+ },
+ },
+ "liabilities": {
+ "description": "Loan or credit card detail (rates, payments, due dates, origination) for a credit or loan account belonging to a linked Plaid Item.",
+ "docs_url": "https://plaid.com/docs/api/products/liabilities/#liabilitiesget",
+ "columns": {
+ "account_id": "ID of the credit or loan account the liability belongs to.",
+ "liability_type": "Kind of liability: credit, mortgage, student, loan, or line_of_credit.",
+ "aprs": "Interest rates that apply to a credit card account.",
+ "is_overdue": "Whether a payment is currently overdue.",
+ "last_payment_amount": "Amount of the last payment.",
+ "last_payment_date": "Date of the last payment.",
+ "last_statement_balance": "Total amount owed as of the last statement.",
+ "minimum_payment_amount": "Minimum payment due for the next billing cycle.",
+ "next_payment_due_date": "Due date of the next payment.",
+ "origination_date": "Date the loan was originally lent.",
+ "origination_principal_amount": "Original principal balance of the loan.",
+ "interest_rate": "Interest rate of a mortgage (percentage and type).",
+ "interest_rate_percentage": "Interest rate of a student loan, as a percentage.",
+ "maturity_date": "Date on which a mortgage is due in full.",
+ },
+ },
+ "institutions": {
+ "description": "The financial institution the linked Plaid Item is connected to.",
+ "docs_url": "https://plaid.com/docs/api/institutions/#institutionsget_by_id",
+ "columns": {
+ "institution_id": "Unique identifier for the institution.",
+ "name": "Official name of the institution.",
+ "products": "Plaid products the institution supports.",
+ "country_codes": "Country codes the institution supports.",
+ "routing_numbers": "Routing numbers known to be associated with the institution.",
+ "oauth": "Whether the institution has an OAuth login flow.",
+ "connection_availability": "Whether Plaid supports new connections to the institution.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
index e3644621850d..dc3285afed1e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
@@ -30,10 +30,40 @@ REQUEST_TIMEOUT_SECONDS = 120
MAX_RETRY_ATTEMPTS = 5
+# Every country Plaid supports; /institutions/get_by_id requires the list and the Item's country is not known.
+INSTITUTION_COUNTRY_CODES = [
+ "US", "GB", "ES", "NL", "FR", "IE", "CA", "DE", "IT", "PL",
+ "DK", "NO", "SE", "EE", "LT", "LV", "PT", "BE", "AT", "FI",
+] # fmt: skip
+# Item errors that mean the Item has no data for the product, so the table is empty, not failed.
+NO_PRODUCT_DATA_ERROR_CODES = frozenset(
+ {"NO_ACCOUNTS", "NO_INVESTMENT_ACCOUNTS", "NO_LIABILITY_ACCOUNTS", "PRODUCTS_NOT_SUPPORTED"}
+)
+# endpoint -> (path, rows key, total count key) for endpoints paginated with options.offset.
+OFFSET_PAGINATED_ENDPOINTS = {
+ "transactions": ("/transactions/get", "transactions", "total_transactions"),
+ "investment_transactions": (
+ "/investments/transactions/get",
+ "investment_transactions",
+ "total_investment_transactions",
+ ),
+}
+# endpoint -> the Plaid product the Item must already have before we call the endpoint.
+PRODUCT_GATED_ENDPOINTS = {
+ "investment_holdings": "investments",
+ "investment_transactions": "investments",
+ "liabilities": "liabilities",
+}
+
+
class PlaidRetryableError(Exception):
pass
+class PlaidNoProductDataError(Exception):
+ pass
+
+
@dataclasses.dataclass
class PlaidResumeConfig:
# /transactions/get paginates with options.offset; static body parts are
@@ -66,6 +96,14 @@ def _today() -> str:
return datetime.now(UTC).strftime("%Y-%m-%d")
+def _error_code(response: requests.Response) -> str | None:
+ try:
+ body = response.json()
+ except ValueError:
+ return None
+ return body.get("error_code") if isinstance(body, dict) else None
+
+
def validate_credentials(environment: str, client_id: str, secret: str, access_token: str) -> bool:
"""Confirm the credential triple is valid with a cheap /item/get probe."""
try:
@@ -100,21 +138,33 @@ def get_rows(
wait=wait_exponential_jitter(initial=5, max=120),
reraise=True,
)
- def post(path: str, body: dict[str, Any]) -> dict[str, Any]:
+ def post(path: str, body: dict[str, Any], with_access_token: bool = True) -> dict[str, Any]:
# Stay under the 30 req/min per-Item budget.
time.sleep(REQUEST_INTERVAL_SECONDS)
url = f"{base_url}{path}"
- response = session.post(url, json={**credentials, **body}, timeout=REQUEST_TIMEOUT_SECONDS)
+ auth = credentials if with_access_token else {"client_id": client_id, "secret": secret}
+ response = session.post(url, json={**auth, **body}, timeout=REQUEST_TIMEOUT_SECONDS)
if response.status_code == 429 or response.status_code >= 500:
raise PlaidRetryableError(f"Plaid API error (retryable): status={response.status_code}, url={url}")
+ if response.status_code == 400 and endpoint in PRODUCT_GATED_ENDPOINTS:
+ error_code = _error_code(response)
+ if error_code in NO_PRODUCT_DATA_ERROR_CODES:
+ raise PlaidNoProductDataError(error_code)
+ # Returned while Plaid still extracts the Item's investment history.
+ if error_code == "PRODUCT_NOT_READY":
+ raise PlaidRetryableError(f"Plaid product not ready yet: url={url}")
+
if not response.ok:
logger.error(f"Plaid API error: status={response.status_code}, body={response.text}, url={url}")
response.raise_for_status()
return response.json()
+ def get_item() -> dict[str, Any]:
+ return post("/item/get", {}).get("item") or {}
+
if endpoint == "accounts":
data = post("/accounts/get", {})
items = data.get("accounts", []) or []
@@ -122,41 +172,84 @@ def get_rows(
yield items
return
- # transactions
- start_date = (
- _format_date(db_incremental_field_last_value)
- if should_use_incremental_field and db_incremental_field_last_value is not None
- else DEFAULT_START_DATE
- )
- end_date = _today()
-
- resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- offset = resume_config.offset if resume_config is not None else 0
- if resume_config is not None:
- logger.debug(f"Plaid: resuming transactions from offset {offset}")
-
- while True:
+ if endpoint == "institutions":
+ institution_id = get_item().get("institution_id")
+ if not institution_id:
+ return
data = post(
- "/transactions/get",
- {
- "start_date": start_date,
- "end_date": end_date,
- "options": {"count": PAGE_SIZE, "offset": offset},
- },
+ "/institutions/get_by_id",
+ {"institution_id": institution_id, "country_codes": INSTITUTION_COUNTRY_CODES},
+ with_access_token=False,
)
- items = data.get("transactions", []) or []
+ if institution := data.get("institution"):
+ yield [institution]
+ return
- if items:
- yield items
+ try:
+ if product := PRODUCT_GATED_ENDPOINTS.get(endpoint):
+ # Calling a product endpoint on an Item without that product adds the product to the Item,
+ # and Plaid bills for it, so skip the table instead.
+ item = get_item()
+ if product not in {*(item.get("billed_products") or []), *(item.get("products") or [])}:
+ logger.info(f"Plaid: Item does not have the {product} product, skipping {endpoint}")
+ return
+
+ if endpoint == "investment_holdings":
+ data = post("/investments/holdings/get", {})
+ items = data.get("holdings", []) or []
+ if items:
+ yield items
+ return
+
+ if endpoint == "liabilities":
+ data = post("/liabilities/get", {})
+ rows = [
+ {**liability, "liability_type": liability_type}
+ for liability_type, liabilities in (data.get("liabilities") or {}).items()
+ for liability in liabilities or []
+ ]
+ if rows:
+ yield rows
+ return
+
+ path, rows_key, total_key = OFFSET_PAGINATED_ENDPOINTS[endpoint]
+ start_date = (
+ _format_date(db_incremental_field_last_value)
+ if should_use_incremental_field and db_incremental_field_last_value is not None
+ else DEFAULT_START_DATE
+ )
+ end_date = _today()
+
+ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ offset = resume_config.offset if resume_config is not None else 0
+ if resume_config is not None:
+ logger.debug(f"Plaid: resuming {endpoint} from offset {offset}")
+
+ while True:
+ data = post(
+ path,
+ {
+ "start_date": start_date,
+ "end_date": end_date,
+ "options": {"count": PAGE_SIZE, "offset": offset},
+ },
+ )
+ items = data.get(rows_key, []) or []
+
+ total = data.get(total_key)
+ offset += len(items)
+ has_more = bool(items) and not (isinstance(total, int) and offset >= total)
+
+ if has_more:
+ resumable_source_manager.save_state(PlaidResumeConfig(offset=offset))
- total = data.get("total_transactions")
- offset += len(items)
- if not items or (isinstance(total, int) and offset >= total):
- break
+ if items:
+ yield items
- # Save state AFTER yielding the page so a crash re-yields the last page
- # (merge dedupes on primary key) rather than skipping it.
- resumable_source_manager.save_state(PlaidResumeConfig(offset=offset))
+ if not has_more:
+ break
+ except PlaidNoProductDataError as error:
+ logger.info(f"Plaid: Item has no data for {endpoint} ({error}), skipping")
def plaid_source(
@@ -185,13 +278,13 @@ def plaid_source(
should_use_incremental_field=should_use_incremental_field,
db_incremental_field_last_value=db_incremental_field_last_value,
),
- primary_keys=[config.primary_key],
+ primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
- partition_mode="datetime" if endpoint == "transactions" else None,
- partition_format="month" if endpoint == "transactions" else None,
- partition_keys=["date"] if endpoint == "transactions" else None,
- # /transactions/get returns newest-first; the pipeline commits desc
- # watermarks only when a run completes.
- sort_mode="desc" if endpoint == "transactions" else "asc",
+ partition_mode="datetime" if config.partition_key else None,
+ partition_format="month" if config.partition_key else None,
+ partition_keys=[config.partition_key] if config.partition_key else None,
+ # /transactions/get and /investments/transactions/get return newest-first;
+ # the pipeline commits desc watermarks only when a run completes.
+ sort_mode="desc" if endpoint in OFFSET_PAGINATED_ENDPOINTS else "asc",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py
index deb958e674b9..eaf0791c4230 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py
@@ -3,11 +3,14 @@ from dataclasses import dataclass, field
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=True)
class PlaidEndpointConfig:
name: str
- primary_key: str
+ primary_keys: list[str]
incremental_fields: list[IncrementalField] = field(default_factory=list)
+ partition_key: str | None = None
+ should_sync_default: bool = True
+ description: str | None = None
# One PostHog source connects one Plaid Item (the access token the user
@@ -17,11 +20,11 @@ class PlaidEndpointConfig:
PLAID_ENDPOINTS: dict[str, PlaidEndpointConfig] = {
"accounts": PlaidEndpointConfig(
name="accounts",
- primary_key="account_id",
+ primary_keys=["account_id"],
),
"transactions": PlaidEndpointConfig(
name="transactions",
- primary_key="transaction_id",
+ primary_keys=["transaction_id"],
incremental_fields=[
{
"label": "date",
@@ -30,11 +33,48 @@ PLAID_ENDPOINTS: dict[str, PlaidEndpointConfig] = {
"field_type": IncrementalFieldType.Date,
},
],
+ partition_key="date",
+ ),
+ "investment_transactions": PlaidEndpointConfig(
+ name="investment_transactions",
+ primary_keys=["investment_transaction_id"],
+ # Investments at Link only adds the Holdings subscription; the first
+ # /investments/transactions/get call adds the separately billed Transactions one.
+ should_sync_default=False,
+ description="Syncing this table starts Plaid's Investments Transactions subscription for the Item if it does not have one yet.",
+ incremental_fields=[
+ {
+ "label": "date",
+ "type": IncrementalFieldType.Date,
+ "field": "date",
+ "field_type": IncrementalFieldType.Date,
+ },
+ ],
+ partition_key="date",
+ ),
+ # A holding has no ID of its own: Plaid returns one position per security per account.
+ "investment_holdings": PlaidEndpointConfig(
+ name="investment_holdings",
+ primary_keys=["account_id", "security_id"],
+ ),
+ "liabilities": PlaidEndpointConfig(
+ name="liabilities",
+ primary_keys=["account_id"],
+ ),
+ "institutions": PlaidEndpointConfig(
+ name="institutions",
+ primary_keys=["institution_id"],
),
}
ENDPOINTS = tuple(PLAID_ENDPOINTS.keys())
+SHOULD_SYNC_DEFAULT: dict[str, bool] = {name: config.should_sync_default for name, config in PLAID_ENDPOINTS.items()}
+
+DESCRIPTIONS: dict[str, str] = {
+ name: config.description for name, config in PLAID_ENDPOINTS.items() if config.description
+}
+
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in PLAID_ENDPOINTS.items() if config.incremental_fields
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py
index c35ae74e6cf2..53d24fc9c977 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py
@@ -28,8 +28,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.plaid.plai
validate_credentials as validate_plaid_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.plaid.settings import (
+ DESCRIPTIONS,
ENDPOINTS,
INCREMENTAL_FIELDS,
+ SHOULD_SYNC_DEFAULT,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -83,7 +85,7 @@ class PlaidSource(ResumableSource[PlaidSourceConfig, PlaidResumeConfig]):
category=DataWarehouseSourceCategory.PAYMENTS___BILLING,
label="Plaid",
keywords=["bank", "banking", "open banking", "financial", "fintech", "transactions"],
- caption="""Connect a Plaid Item to pull its accounts and transactions into the PostHog Data warehouse.
+ caption="""Connect a Plaid Item to pull its accounts, transactions, investments, liabilities, and institution into the PostHog Data warehouse.
You can find your client ID and secret in the [Plaid dashboard](https://dashboard.plaid.com/developers/keys). The access token identifies one linked Item (institution connection), obtained when a user completes Plaid Link — add one source per Item.""",
iconPath="/static/services/plaid.png",
@@ -146,7 +148,13 @@ You can find your client ID and secret in the [Plaid dashboard](https://dashboar
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ descriptions=DESCRIPTIONS,
+ should_sync_default=SHOULD_SYNC_DEFAULT,
+ )
def validate_credentials(
self, config: PlaidSourceConfig, team_id: int, schema_name: Optional[str] = None, api_version: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py
index 36eed2229a65..96b3465429d8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py
@@ -12,6 +12,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.plaid import PlaidSourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.plaid.plaid import (
DEFAULT_START_DATE,
+ INSTITUTION_COUNTRY_CODES,
+ OFFSET_PAGINATED_ENDPOINTS,
PAGE_SIZE,
PlaidResumeConfig,
_base_url,
@@ -31,6 +33,10 @@ def _driver() -> SourceDriver:
)
+def _item(**fields) -> ScriptedResponse:
+ return ScriptedResponse(json={"item": {"item_id": "item-1", **fields}})
+
+
class TestBaseUrl:
def test_invalid_environment_raises(self):
with pytest.raises(ValueError):
@@ -82,27 +88,35 @@ class TestGetRowsAccounts:
class TestGetRowsTransactions:
+ @pytest.mark.parametrize(
+ "endpoint, preamble, path",
+ [
+ ("transactions", [], "/transactions/get"),
+ ("investment_transactions", [_item(billed_products=["investments"])], "/investments/transactions/get"),
+ ],
+ )
@mock.patch(f"{_MODULE}._today")
- def test_paginates_via_offset_until_total(self, mock_today: mock.MagicMock) -> None:
+ def test_paginates_via_offset_until_total(
+ self, mock_today: mock.MagicMock, endpoint: str, preamble: list[ScriptedResponse], path: str
+ ) -> None:
mock_today.return_value = "2024-06-01"
- full_page = [{"transaction_id": str(i), "date": "2024-01-01"} for i in range(PAGE_SIZE)]
+ _, rows_key, total_key = OFFSET_PAGINATED_ENDPOINTS[endpoint]
+ full_page = [{"id": str(i), "date": "2024-01-01"} for i in range(PAGE_SIZE)]
result = _driver().run(
- "transactions",
+ endpoint,
[
- ScriptedResponse(json={"transactions": full_page, "total_transactions": PAGE_SIZE + 1}),
- ScriptedResponse(
- json={
- "transactions": [{"transaction_id": "last", "date": "2024-01-02"}],
- "total_transactions": PAGE_SIZE + 1,
- }
- ),
+ *preamble,
+ ScriptedResponse(json={rows_key: full_page, total_key: PAGE_SIZE + 1}),
+ ScriptedResponse(json={rows_key: [{"id": "last", "date": "2024-01-02"}], total_key: PAGE_SIZE + 1}),
],
)
assert result.raised is None
- assert len(result.items) == 2
+ assert len(result.rows) == PAGE_SIZE + 1
assert result.saved_states == [PlaidResumeConfig(offset=PAGE_SIZE)]
- bodies = [request.json() for request in result.requests]
+ page_requests = result.requests[len(preamble) :]
+ assert [request.path for request in page_requests] == [path, path]
+ bodies = [request.json() for request in page_requests]
assert bodies[0]["options"] == {"count": PAGE_SIZE, "offset": 0}
assert bodies[1]["options"] == {"count": PAGE_SIZE, "offset": PAGE_SIZE}
assert bodies[0]["start_date"] == DEFAULT_START_DATE
@@ -136,18 +150,129 @@ class TestGetRowsTransactions:
assert body["options"]["offset"] == 1500
+class TestGetRowsProductGatedEndpoints:
+ @pytest.mark.parametrize("endpoint", ["investment_holdings", "investment_transactions", "liabilities"])
+ def test_skips_item_without_product(self, endpoint: str) -> None:
+ result = _driver().run(endpoint, [_item(billed_products=["transactions"], products=["transactions"])])
+
+ assert result.raised is None
+ assert result.rows == []
+ assert result.paths == ["/item/get"]
+
+ @pytest.mark.parametrize(
+ "error_code, should_raise",
+ [
+ ("NO_INVESTMENT_ACCOUNTS", False),
+ ("INVALID_PRODUCT", True),
+ ],
+ )
+ def test_product_error_handling(self, error_code: str, should_raise: bool) -> None:
+ result = _driver().run(
+ "investment_holdings",
+ [
+ _item(products=["investments"]),
+ ScriptedResponse(status=400, json={"error_type": "ITEM_ERROR", "error_code": error_code}),
+ ],
+ )
+
+ assert (result.raised is not None) is should_raise
+ assert result.rows == []
+ assert result.paths == ["/item/get", "/investments/holdings/get"]
+
+ def test_holdings_rows(self) -> None:
+ holding = {"account_id": "a1", "security_id": "s1", "quantity": 10, "cost_basis": 100.0}
+ result = _driver().run(
+ "investment_holdings",
+ [
+ _item(billed_products=["investments"]),
+ ScriptedResponse(json={"holdings": [holding], "securities": [{"security_id": "s1"}]}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [holding]
+
+ def test_retries_product_not_ready(self) -> None:
+ result = _driver().run(
+ "investment_transactions",
+ [
+ _item(billed_products=["investments"]),
+ ScriptedResponse(status=400, json={"error_type": "ITEM_ERROR", "error_code": "PRODUCT_NOT_READY"}),
+ ScriptedResponse(
+ json={
+ "investment_transactions": [{"investment_transaction_id": "t1", "date": "2024-01-01"}],
+ "total_investment_transactions": 1,
+ }
+ ),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["investment_transaction_id"] for row in result.rows] == ["t1"]
+ assert result.paths == ["/item/get", "/investments/transactions/get", "/investments/transactions/get"]
+
+ def test_liabilities_flattened_with_type(self) -> None:
+ result = _driver().run(
+ "liabilities",
+ [
+ _item(billed_products=["liabilities"]),
+ ScriptedResponse(
+ json={
+ "liabilities": {
+ "credit": [{"account_id": "c1", "aprs": []}],
+ "mortgage": [{"account_id": "m1", "interest_rate": {"percentage": 3.5}}],
+ "student": None,
+ }
+ }
+ ),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"account_id": "c1", "aprs": [], "liability_type": "credit"},
+ {"account_id": "m1", "interest_rate": {"percentage": 3.5}, "liability_type": "mortgage"},
+ ]
+
+
+class TestGetRowsInstitutions:
+ def test_looks_up_item_institution(self) -> None:
+ institution = {"institution_id": "ins_1", "name": "Example Bank"}
+ result = _driver().run(
+ "institutions",
+ [_item(institution_id="ins_1"), ScriptedResponse(json={"institution": institution})],
+ )
+
+ assert result.raised is None
+ assert result.rows == [institution]
+ assert result.paths == ["/item/get", "/institutions/get_by_id"]
+ assert result.requests[1].json() == {
+ "client_id": "cid",
+ "secret": "sec",
+ "institution_id": "ins_1",
+ "country_codes": INSTITUTION_COUNTRY_CODES,
+ }
+
+ def test_item_without_institution_yields_nothing(self) -> None:
+ result = _driver().run("institutions", [_item(institution_id=None)])
+
+ assert result.raised is None
+ assert result.rows == []
+ assert result.paths == ["/item/get"]
+
+
class TestPlaidSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
config = PLAID_ENDPOINTS[endpoint]
- result = _driver().run(endpoint, [ScriptedResponse(json={endpoint: [], "total_transactions": 0})])
+ result = _driver().run(endpoint, always(ScriptedResponse(json={"item": {}})))
assert result.raised is None
response = result.response
assert response is not None
assert response.name == endpoint
- assert response.primary_keys == [config.primary_key]
- if endpoint == "transactions":
+ assert response.primary_keys == config.primary_keys
+ if endpoint in OFFSET_PAGINATED_ENDPOINTS:
# Newest-first ordering — watermark commits only at run end.
assert response.sort_mode == "desc"
assert response.partition_mode == "datetime"