:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add plaid investments, liabilities and institutions tables (#126278)
Tom Owerse89d0b14f2f1
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 89d7494eae9f..8bb831e8d24f 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -854,7 +854,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/pipeliner/pipeliner.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pipeliner/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/planhat/planhat.py1 products/warehouse_sources/backend/temporal/data_imports/sources/planhat/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/plausible/plausible.pydiff --git a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txtindex bff693f0d231..2cdec71af8e6 100644--- a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt+++ b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt@@ -308,7 +308,6 @@ pendo/pendo.py::_iter_list_endpointpersona/persona.py::get_rowspinterest_ads/pinterest_ads.py::_iter_entity_rowspinterest_ads/pinterest_ads.py::_iter_fanned_out_analytics-plaid/plaid.py::get_rowsplatform_sh/platform_sh.py::get_rowspolar/polar.py::get_rowspostgres/postgres.py::postgres_source.get_rows.offset_chunkingdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 0710066e35a8..c7928131b61e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9494,10 +9494,10 @@ Today (2): `accounts`, `transactions`Diffed against: <https://raw.githubusercontent.com/plaid/plaid-openapi/master/2020-09-14.yml>-- [ ] `/investments/holdings/get` — positions and cost basis per investment account - accounts alone give no portfolio detail (high)-- [ ] `/investments/transactions/get` — buys, sells, dividends and fees; the investments analogue of the transactions table we already sync (high)-- [ ] `/liabilities/get` — loan and credit-card detail (APR, minimum payment, due date, origination) that never appears in accounts or transactions (high)-- [ ] `/institutions/get_by_id (and /institutions/get)` — lookup resolving the institution_id on the item whose accounts we sync - today the bank is only an opaque ID (high)+- [x] `/investments/holdings/get` — positions and cost basis per investment account - accounts alone give no portfolio detail (high)+- [x] `/investments/transactions/get` — buys, sells, dividends and fees; the investments analogue of the transactions table we already sync (high)+- [x] `/liabilities/get` — loan and credit-card detail (APR, minimum payment, due date, origination) that never appears in accounts or transactions (high)+- [x] `/institutions/get_by_id (and /institutions/get)` — lookup resolving the institution_id on the item whose accounts we sync - today the bank is only an opaque ID (high)- [ ] `/transactions/recurring/get` — Plaid-derived recurring inflow/outflow streams - the headline subscription and income-detection product (high)- [ ] `/accounts/balance/get` — live available/current balance refresh; the balances embedded in /accounts/get are cached and can be stale (medium)- [ ] `/identity/get` — account owner names, emails, phones and addresses, the standard join key to customer records (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.pyindex 9d542d15bd14..47c0d3dd5180 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/canonical_descriptions.py@@ -44,4 +44,72 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"personal_finance_category": "Detailed and primary personal finance category for the transaction.",},},+ "investment_transactions": {+ "description": "An investment transaction (buy, sell, dividend, fee, transfer, etc.) on an investment account belonging to a linked Plaid Item.",+ "docs_url": "https://plaid.com/docs/api/products/investments/#investmentstransactionsget",+ "columns": {+ "investment_transaction_id": "Unique identifier for the investment transaction.",+ "account_id": "ID of the account the transaction posted against.",+ "security_id": "ID of the security the transaction relates to.",+ "date": "Posting date of the transaction, typically the settlement date.",+ "transaction_datetime": "When the order was initiated, for select institutions.",+ "name": "The institution's description of the transaction.",+ "quantity": "Units of the security involved (positive for buys, negative for sells).",+ "amount": "Complete value of the transaction (positive when cash is debited, negative when credited).",+ "price": "Price of the security at which the transaction occurred.",+ "fees": "Combined value of all fees applied to the transaction.",+ "type": "Transaction type (buy, sell, cancel, cash, fee, transfer).",+ "subtype": "Detailed transaction subtype (e.g. dividend, interest, contribution).",+ "iso_currency_code": "ISO-4217 currency code of the transaction.",+ },+ },+ "investment_holdings": {+ "description": "A position in one security held in an investment account belonging to a linked Plaid Item.",+ "docs_url": "https://plaid.com/docs/api/products/investments/#investmentsholdingsget",+ "columns": {+ "account_id": "ID of the account that holds the position.",+ "security_id": "ID of the security held.",+ "quantity": "Total quantity of the security held, as reported by the institution.",+ "cost_basis": "Total cost basis of the holding (the amount spent to acquire the assets currently held).",+ "institution_price": "Last price the institution gave for the security.",+ "institution_price_as_of": "Date at which institution_price was current.",+ "institution_value": "Value of the holding, as reported by the institution.",+ "vested_quantity": "Quantity of vested assets held, for equities.",+ "vested_value": "Value of the vested holdings, as reported by the institution.",+ "iso_currency_code": "ISO-4217 currency code of the holding.",+ },+ },+ "liabilities": {+ "description": "Loan or credit card detail (rates, payments, due dates, origination) for a credit or loan account belonging to a linked Plaid Item.",+ "docs_url": "https://plaid.com/docs/api/products/liabilities/#liabilitiesget",+ "columns": {+ "account_id": "ID of the credit or loan account the liability belongs to.",+ "liability_type": "Kind of liability: credit, mortgage, student, loan, or line_of_credit.",+ "aprs": "Interest rates that apply to a credit card account.",+ "is_overdue": "Whether a payment is currently overdue.",+ "last_payment_amount": "Amount of the last payment.",+ "last_payment_date": "Date of the last payment.",+ "last_statement_balance": "Total amount owed as of the last statement.",+ "minimum_payment_amount": "Minimum payment due for the next billing cycle.",+ "next_payment_due_date": "Due date of the next payment.",+ "origination_date": "Date the loan was originally lent.",+ "origination_principal_amount": "Original principal balance of the loan.",+ "interest_rate": "Interest rate of a mortgage (percentage and type).",+ "interest_rate_percentage": "Interest rate of a student loan, as a percentage.",+ "maturity_date": "Date on which a mortgage is due in full.",+ },+ },+ "institutions": {+ "description": "The financial institution the linked Plaid Item is connected to.",+ "docs_url": "https://plaid.com/docs/api/institutions/#institutionsget_by_id",+ "columns": {+ "institution_id": "Unique identifier for the institution.",+ "name": "Official name of the institution.",+ "products": "Plaid products the institution supports.",+ "country_codes": "Country codes the institution supports.",+ "routing_numbers": "Routing numbers known to be associated with the institution.",+ "oauth": "Whether the institution has an OAuth login flow.",+ "connection_availability": "Whether Plaid supports new connections to the institution.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.pyindex e3644621850d..dc3285afed1e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py@@ -30,10 +30,40 @@ REQUEST_TIMEOUT_SECONDS = 120MAX_RETRY_ATTEMPTS = 5+# Every country Plaid supports; /institutions/get_by_id requires the list and the Item's country is not known.+INSTITUTION_COUNTRY_CODES = [+ "US", "GB", "ES", "NL", "FR", "IE", "CA", "DE", "IT", "PL",+ "DK", "NO", "SE", "EE", "LT", "LV", "PT", "BE", "AT", "FI",+] # fmt: skip+# Item errors that mean the Item has no data for the product, so the table is empty, not failed.+NO_PRODUCT_DATA_ERROR_CODES = frozenset(+ {"NO_ACCOUNTS", "NO_INVESTMENT_ACCOUNTS", "NO_LIABILITY_ACCOUNTS", "PRODUCTS_NOT_SUPPORTED"}+)+# endpoint -> (path, rows key, total count key) for endpoints paginated with options.offset.+OFFSET_PAGINATED_ENDPOINTS = {+ "transactions": ("/transactions/get", "transactions", "total_transactions"),+ "investment_transactions": (+ "/investments/transactions/get",+ "investment_transactions",+ "total_investment_transactions",+ ),+}+# endpoint -> the Plaid product the Item must already have before we call the endpoint.+PRODUCT_GATED_ENDPOINTS = {+ "investment_holdings": "investments",+ "investment_transactions": "investments",+ "liabilities": "liabilities",+}++class PlaidRetryableError(Exception):pass+class PlaidNoProductDataError(Exception):+ pass++@dataclasses.dataclassclass PlaidResumeConfig:# /transactions/get paginates with options.offset; static body parts are@@ -66,6 +96,14 @@ def _today() -> str:return datetime.now(UTC).strftime("%Y-%m-%d")+def _error_code(response: requests.Response) -> str | None:+ try:+ body = response.json()+ except ValueError:+ return None+ return body.get("error_code") if isinstance(body, dict) else None++def validate_credentials(environment: str, client_id: str, secret: str, access_token: str) -> bool:"""Confirm the credential triple is valid with a cheap /item/get probe."""try:@@ -100,21 +138,33 @@ def get_rows(wait=wait_exponential_jitter(initial=5, max=120),reraise=True,)- def post(path: str, body: dict[str, Any]) -> dict[str, Any]:+ def post(path: str, body: dict[str, Any], with_access_token: bool = True) -> dict[str, Any]:# Stay under the 30 req/min per-Item budget.time.sleep(REQUEST_INTERVAL_SECONDS)url = f"{base_url}{path}"- response = session.post(url, json={**credentials, **body}, timeout=REQUEST_TIMEOUT_SECONDS)+ auth = credentials if with_access_token else {"client_id": client_id, "secret": secret}+ response = session.post(url, json={**auth, **body}, timeout=REQUEST_TIMEOUT_SECONDS)if response.status_code == 429 or response.status_code >= 500:raise PlaidRetryableError(f"Plaid API error (retryable): status={response.status_code}, url={url}")+ if response.status_code == 400 and endpoint in PRODUCT_GATED_ENDPOINTS:+ error_code = _error_code(response)+ if error_code in NO_PRODUCT_DATA_ERROR_CODES:+ raise PlaidNoProductDataError(error_code)+ # Returned while Plaid still extracts the Item's investment history.+ if error_code == "PRODUCT_NOT_READY":+ raise PlaidRetryableError(f"Plaid product not ready yet: url={url}")+if not response.ok:logger.error(f"Plaid API error: status={response.status_code}, body={response.text}, url={url}")response.raise_for_status()return response.json()+ def get_item() -> dict[str, Any]:+ return post("/item/get", {}).get("item") or {}+if endpoint == "accounts":data = post("/accounts/get", {})items = data.get("accounts", []) or []@@ -122,41 +172,84 @@ def get_rows(yield itemsreturn- # transactions- start_date = (- _format_date(db_incremental_field_last_value)- if should_use_incremental_field and db_incremental_field_last_value is not None- else DEFAULT_START_DATE- )- end_date = _today()-- resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- offset = resume_config.offset if resume_config is not None else 0- if resume_config is not None:- logger.debug(f"Plaid: resuming transactions from offset {offset}")-- while True:+ if endpoint == "institutions":+ institution_id = get_item().get("institution_id")+ if not institution_id:+ returndata = post(- "/transactions/get",- {- "start_date": start_date,- "end_date": end_date,- "options": {"count": PAGE_SIZE, "offset": offset},- },+ "/institutions/get_by_id",+ {"institution_id": institution_id, "country_codes": INSTITUTION_COUNTRY_CODES},+ with_access_token=False,)- items = data.get("transactions", []) or []+ if institution := data.get("institution"):+ yield [institution]+ return- if items:- yield items+ try:+ if product := PRODUCT_GATED_ENDPOINTS.get(endpoint):+ # Calling a product endpoint on an Item without that product adds the product to the Item,+ # and Plaid bills for it, so skip the table instead.+ item = get_item()+ if product not in {*(item.get("billed_products") or []), *(item.get("products") or [])}:+ logger.info(f"Plaid: Item does not have the {product} product, skipping {endpoint}")+ return++ if endpoint == "investment_holdings":+ data = post("/investments/holdings/get", {})+ items = data.get("holdings", []) or []+ if items:+ yield items+ return++ if endpoint == "liabilities":+ data = post("/liabilities/get", {})+ rows = [+ {**liability, "liability_type": liability_type}+ for liability_type, liabilities in (data.get("liabilities") or {}).items()+ for liability in liabilities or []+ ]+ if rows:+ yield rows+ return++ path, rows_key, total_key = OFFSET_PAGINATED_ENDPOINTS[endpoint]+ start_date = (+ _format_date(db_incremental_field_last_value)+ if should_use_incremental_field and db_incremental_field_last_value is not None+ else DEFAULT_START_DATE+ )+ end_date = _today()++ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ offset = resume_config.offset if resume_config is not None else 0+ if resume_config is not None:+ logger.debug(f"Plaid: resuming {endpoint} from offset {offset}")++ while True:+ data = post(+ path,+ {+ "start_date": start_date,+ "end_date": end_date,+ "options": {"count": PAGE_SIZE, "offset": offset},+ },+ )+ items = data.get(rows_key, []) or []++ total = data.get(total_key)+ offset += len(items)+ has_more = bool(items) and not (isinstance(total, int) and offset >= total)++ if has_more:+ resumable_source_manager.save_state(PlaidResumeConfig(offset=offset))- total = data.get("total_transactions")- offset += len(items)- if not items or (isinstance(total, int) and offset >= total):- break+ if items:+ yield items- # Save state AFTER yielding the page so a crash re-yields the last page- # (merge dedupes on primary key) rather than skipping it.- resumable_source_manager.save_state(PlaidResumeConfig(offset=offset))+ if not has_more:+ break+ except PlaidNoProductDataError as error:+ logger.info(f"Plaid: Item has no data for {endpoint} ({error}), skipping")def plaid_source(@@ -185,13 +278,13 @@ def plaid_source(should_use_incremental_field=should_use_incremental_field,db_incremental_field_last_value=db_incremental_field_last_value,),- primary_keys=[config.primary_key],+ primary_keys=config.primary_keys,partition_count=1,partition_size=1,- partition_mode="datetime" if endpoint == "transactions" else None,- partition_format="month" if endpoint == "transactions" else None,- partition_keys=["date"] if endpoint == "transactions" else None,- # /transactions/get returns newest-first; the pipeline commits desc- # watermarks only when a run completes.- sort_mode="desc" if endpoint == "transactions" else "asc",+ partition_mode="datetime" if config.partition_key else None,+ partition_format="month" if config.partition_key else None,+ partition_keys=[config.partition_key] if config.partition_key else None,+ # /transactions/get and /investments/transactions/get return newest-first;+ # the pipeline commits desc watermarks only when a run completes.+ sort_mode="desc" if endpoint in OFFSET_PAGINATED_ENDPOINTS else "asc",)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.pyindex deb958e674b9..eaf0791c4230 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/settings.py@@ -3,11 +3,14 @@ from dataclasses import dataclass, fieldfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=True)class PlaidEndpointConfig:name: str- primary_key: str+ primary_keys: list[str]incremental_fields: list[IncrementalField] = field(default_factory=list)+ partition_key: str | None = None+ should_sync_default: bool = True+ description: str | None = None# One PostHog source connects one Plaid Item (the access token the user@@ -17,11 +20,11 @@ class PlaidEndpointConfig:PLAID_ENDPOINTS: dict[str, PlaidEndpointConfig] = {"accounts": PlaidEndpointConfig(name="accounts",- primary_key="account_id",+ primary_keys=["account_id"],),"transactions": PlaidEndpointConfig(name="transactions",- primary_key="transaction_id",+ primary_keys=["transaction_id"],incremental_fields=[{"label": "date",@@ -30,11 +33,48 @@ PLAID_ENDPOINTS: dict[str, PlaidEndpointConfig] = {"field_type": IncrementalFieldType.Date,},],+ partition_key="date",+ ),+ "investment_transactions": PlaidEndpointConfig(+ name="investment_transactions",+ primary_keys=["investment_transaction_id"],+ # Investments at Link only adds the Holdings subscription; the first+ # /investments/transactions/get call adds the separately billed Transactions one.+ should_sync_default=False,+ description="Syncing this table starts Plaid's Investments Transactions subscription for the Item if it does not have one yet.",+ incremental_fields=[+ {+ "label": "date",+ "type": IncrementalFieldType.Date,+ "field": "date",+ "field_type": IncrementalFieldType.Date,+ },+ ],+ partition_key="date",+ ),+ # A holding has no ID of its own: Plaid returns one position per security per account.+ "investment_holdings": PlaidEndpointConfig(+ name="investment_holdings",+ primary_keys=["account_id", "security_id"],+ ),+ "liabilities": PlaidEndpointConfig(+ name="liabilities",+ primary_keys=["account_id"],+ ),+ "institutions": PlaidEndpointConfig(+ name="institutions",+ primary_keys=["institution_id"],),}ENDPOINTS = tuple(PLAID_ENDPOINTS.keys())+SHOULD_SYNC_DEFAULT: dict[str, bool] = {name: config.should_sync_default for name, config in PLAID_ENDPOINTS.items()}++DESCRIPTIONS: dict[str, str] = {+ name: config.description for name, config in PLAID_ENDPOINTS.items() if config.description+}+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {name: config.incremental_fields for name, config in PLAID_ENDPOINTS.items() if config.incremental_fields}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.pyindex c35ae74e6cf2..53d24fc9c977 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/source.py@@ -28,8 +28,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.plaid.plaivalidate_credentials as validate_plaid_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.plaid.settings import (+ DESCRIPTIONS,ENDPOINTS,INCREMENTAL_FIELDS,+ SHOULD_SYNC_DEFAULT,)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -83,7 +85,7 @@ class PlaidSource(ResumableSource[PlaidSourceConfig, PlaidResumeConfig]):category=DataWarehouseSourceCategory.PAYMENTS___BILLING,label="Plaid",keywords=["bank", "banking", "open banking", "financial", "fintech", "transactions"],- caption="""Connect a Plaid Item to pull its accounts and transactions into the PostHog Data warehouse.+ caption="""Connect a Plaid Item to pull its accounts, transactions, investments, liabilities, and institution into the PostHog Data warehouse.You can find your client ID and secret in the [Plaid dashboard](https://dashboard.plaid.com/developers/keys). The access token identifies one linked Item (institution connection), obtained when a user completes Plaid Link — add one source per Item.""",iconPath="/static/services/plaid.png",@@ -146,7 +148,13 @@ You can find your client ID and secret in the [Plaid dashboard](https://dashboarforce_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)+ return build_endpoint_schemas(+ ENDPOINTS,+ INCREMENTAL_FIELDS,+ names,+ descriptions=DESCRIPTIONS,+ should_sync_default=SHOULD_SYNC_DEFAULT,+ )def validate_credentials(self, config: PlaidSourceConfig, team_id: int, schema_name: Optional[str] = None, api_version: str | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.pyindex 36eed2229a65..96b3465429d8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/tests/test_plaid.py@@ -12,6 +12,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.plaid import PlaidSourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.plaid.plaid import (DEFAULT_START_DATE,+ INSTITUTION_COUNTRY_CODES,+ OFFSET_PAGINATED_ENDPOINTS,PAGE_SIZE,PlaidResumeConfig,_base_url,@@ -31,6 +33,10 @@ def _driver() -> SourceDriver:)+def _item(**fields) -> ScriptedResponse:+ return ScriptedResponse(json={"item": {"item_id": "item-1", **fields}})++class TestBaseUrl:def test_invalid_environment_raises(self):with pytest.raises(ValueError):@@ -82,27 +88,35 @@ class TestGetRowsAccounts:class TestGetRowsTransactions:+ @pytest.mark.parametrize(+ "endpoint, preamble, path",+ [+ ("transactions", [], "/transactions/get"),+ ("investment_transactions", [_item(billed_products=["investments"])], "/investments/transactions/get"),+ ],+ )@mock.patch(f"{_MODULE}._today")- def test_paginates_via_offset_until_total(self, mock_today: mock.MagicMock) -> None:+ def test_paginates_via_offset_until_total(+ self, mock_today: mock.MagicMock, endpoint: str, preamble: list[ScriptedResponse], path: str+ ) -> None:mock_today.return_value = "2024-06-01"- full_page = [{"transaction_id": str(i), "date": "2024-01-01"} for i in range(PAGE_SIZE)]+ _, rows_key, total_key = OFFSET_PAGINATED_ENDPOINTS[endpoint]+ full_page = [{"id": str(i), "date": "2024-01-01"} for i in range(PAGE_SIZE)]result = _driver().run(- "transactions",+ endpoint,[- ScriptedResponse(json={"transactions": full_page, "total_transactions": PAGE_SIZE + 1}),- ScriptedResponse(- json={- "transactions": [{"transaction_id": "last", "date": "2024-01-02"}],- "total_transactions": PAGE_SIZE + 1,- }- ),+ *preamble,+ ScriptedResponse(json={rows_key: full_page, total_key: PAGE_SIZE + 1}),+ ScriptedResponse(json={rows_key: [{"id": "last", "date": "2024-01-02"}], total_key: PAGE_SIZE + 1}),],)assert result.raised is None- assert len(result.items) == 2+ assert len(result.rows) == PAGE_SIZE + 1assert result.saved_states == [PlaidResumeConfig(offset=PAGE_SIZE)]- bodies = [request.json() for request in result.requests]+ page_requests = result.requests[len(preamble) :]+ assert [request.path for request in page_requests] == [path, path]+ bodies = [request.json() for request in page_requests]assert bodies[0]["options"] == {"count": PAGE_SIZE, "offset": 0}assert bodies[1]["options"] == {"count": PAGE_SIZE, "offset": PAGE_SIZE}assert bodies[0]["start_date"] == DEFAULT_START_DATE@@ -136,18 +150,129 @@ class TestGetRowsTransactions:assert body["options"]["offset"] == 1500+class TestGetRowsProductGatedEndpoints:+ @pytest.mark.parametrize("endpoint", ["investment_holdings", "investment_transactions", "liabilities"])+ def test_skips_item_without_product(self, endpoint: str) -> None:+ result = _driver().run(endpoint, [_item(billed_products=["transactions"], products=["transactions"])])++ assert result.raised is None+ assert result.rows == []+ assert result.paths == ["/item/get"]++ @pytest.mark.parametrize(+ "error_code, should_raise",+ [+ ("NO_INVESTMENT_ACCOUNTS", False),+ ("INVALID_PRODUCT", True),+ ],+ )+ def test_product_error_handling(self, error_code: str, should_raise: bool) -> None:+ result = _driver().run(+ "investment_holdings",+ [+ _item(products=["investments"]),+ ScriptedResponse(status=400, json={"error_type": "ITEM_ERROR", "error_code": error_code}),+ ],+ )++ assert (result.raised is not None) is should_raise+ assert result.rows == []+ assert result.paths == ["/item/get", "/investments/holdings/get"]++ def test_holdings_rows(self) -> None:+ holding = {"account_id": "a1", "security_id": "s1", "quantity": 10, "cost_basis": 100.0}+ result = _driver().run(+ "investment_holdings",+ [+ _item(billed_products=["investments"]),+ ScriptedResponse(json={"holdings": [holding], "securities": [{"security_id": "s1"}]}),+ ],+ )++ assert result.raised is None+ assert result.rows == [holding]++ def test_retries_product_not_ready(self) -> None:+ result = _driver().run(+ "investment_transactions",+ [+ _item(billed_products=["investments"]),+ ScriptedResponse(status=400, json={"error_type": "ITEM_ERROR", "error_code": "PRODUCT_NOT_READY"}),+ ScriptedResponse(+ json={+ "investment_transactions": [{"investment_transaction_id": "t1", "date": "2024-01-01"}],+ "total_investment_transactions": 1,+ }+ ),+ ],+ )++ assert result.raised is None+ assert [row["investment_transaction_id"] for row in result.rows] == ["t1"]+ assert result.paths == ["/item/get", "/investments/transactions/get", "/investments/transactions/get"]++ def test_liabilities_flattened_with_type(self) -> None:+ result = _driver().run(+ "liabilities",+ [+ _item(billed_products=["liabilities"]),+ ScriptedResponse(+ json={+ "liabilities": {+ "credit": [{"account_id": "c1", "aprs": []}],+ "mortgage": [{"account_id": "m1", "interest_rate": {"percentage": 3.5}}],+ "student": None,+ }+ }+ ),+ ],+ )++ assert result.raised is None+ assert result.rows == [+ {"account_id": "c1", "aprs": [], "liability_type": "credit"},+ {"account_id": "m1", "interest_rate": {"percentage": 3.5}, "liability_type": "mortgage"},+ ]+++class TestGetRowsInstitutions:+ def test_looks_up_item_institution(self) -> None:+ institution = {"institution_id": "ins_1", "name": "Example Bank"}+ result = _driver().run(+ "institutions",+ [_item(institution_id="ins_1"), ScriptedResponse(json={"institution": institution})],+ )++ assert result.raised is None+ assert result.rows == [institution]+ assert result.paths == ["/item/get", "/institutions/get_by_id"]+ assert result.requests[1].json() == {+ "client_id": "cid",+ "secret": "sec",+ "institution_id": "ins_1",+ "country_codes": INSTITUTION_COUNTRY_CODES,+ }++ def test_item_without_institution_yields_nothing(self) -> None:+ result = _driver().run("institutions", [_item(institution_id=None)])++ assert result.raised is None+ assert result.rows == []+ assert result.paths == ["/item/get"]++class TestPlaidSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = PLAID_ENDPOINTS[endpoint]- result = _driver().run(endpoint, [ScriptedResponse(json={endpoint: [], "total_transactions": 0})])+ result = _driver().run(endpoint, always(ScriptedResponse(json={"item": {}})))assert result.raised is Noneresponse = result.responseassert response is not Noneassert response.name == endpoint- assert response.primary_keys == [config.primary_key]- if endpoint == "transactions":+ assert response.primary_keys == config.primary_keys+ if endpoint in OFFSET_PAGINATED_ENDPOINTS:# Newest-first ordering — watermark commits only at run end.assert response.sort_mode == "desc"assert response.partition_mode == "datetime"