Open-source personal finance manager. Self-hosted, privacy-first.
fix(pluggy): hint per-account institution for banking-group brokerage accounts (#724)
Victor Hugo dos Santosda8218cb7c8a
diff --git a/backend/.env.example b/backend/.env.exampleindex d8eed5c7..674295be 100644--- a/backend/.env.example+++ b/backend/.env.example@@ -10,6 +10,13 @@ REDIS_URL=redis://localhost:6379/0# (e.g. to avoid the dependency on the Brazilian government CSV endpoint).TESOURO_DIRETO_ENABLED=true+# Names the banks behind a Pluggy connection that groups several brands, by+# looking the account's COMPE code up on BrasilAPI (public Central Bank+# registry mirror). Off by default: nothing leaves the server until you opt+# in. When on, only the three-digit code is sent, never your data, and only+# for a connection whose accounts of one type sit at different banks.+BRASILAPI_INSTITUTION_LOOKUP_ENABLED=false+# Number of trusted reverse proxies in front of the backend (the bundled nginx# frontend counts as 1). Rate limiting keys requests by client IP; leaving this# at 0 in a proxied deployment means every client shares one rate-limit bucket.diff --git a/backend/app/core/config.py b/backend/app/core/config.pyindex 2f3b7649..7ea369e9 100644--- a/backend/app/core/config.py+++ b/backend/app/core/config.py@@ -33,6 +33,9 @@ class Settings(BaseSettings):# Set explicitly only when the URL registered in the provider dashboard# differs from where the app is served.pluggy_oauth_redirect_uri: str = ""+ # Gates the BrasilAPI outbound lookup in _lookup_bank_info (pluggy.py).+ # Off by default: no third-party call happens until a deploy opts in.+ brasilapi_institution_lookup_enabled: bool = False# Enable Banking (European PSD2 banks)enable_banking_app_id: str = ""diff --git a/backend/app/providers/pluggy.py b/backend/app/providers/pluggy.pyindex e51cbe1a..dae0bfb0 100644--- a/backend/app/providers/pluggy.py+++ b/backend/app/providers/pluggy.py@@ -67,6 +67,112 @@ def _compe_from_transfer_number(transfer_number) -> Optional[str]:return code.zfill(3) if code.isdigit() else None+# Public, community-maintained mirror of the Bacen (Central Bank) COMPE+# participant registry — free, no API key, no rate limits seen in practice.+# https://brasilapi.com.br/api/banks/v1/{code}+BRASIL_API_BANKS_URL = "https://brasilapi.com.br/api/banks/v1"+_BANK_INFO_CACHE_TTL_SECONDS = 60 * 60 * 24 * 30 # 30 days — COMPE codes are static+++async def _lookup_bank_info(compe: str) -> Optional[dict]:+ """Best-effort ``{"name", "logo_url"}`` for a COMPE code via BrasilAPI.++ Cached in Redis since a bank's registered name/logo effectively never+ change. A lookup failure (network, cache, anything) must never break+ sync — it just means the affected accounts fall back to the connection's+ own institution fields, same as any Pluggy connection without this hint.++ Gated by ``brasilapi_institution_lookup_enabled``: this is the only place+ a COMPE code leaves the server, so deploys must opt in before any call+ to BrasilAPI is made.+ """+ if not get_settings().brasilapi_institution_lookup_enabled:+ return None++ from app.core.redis import get_redis++ cache_key = f"pluggy:bank_info:{compe}"+ redis_client = None+ try:+ redis_client = await get_redis()+ cached = await redis_client.get(cache_key)+ if cached is not None:+ return json.loads(cached) if cached else None+ except Exception:+ redis_client = None++ try:+ async with httpx.AsyncClient(timeout=5.0) as client:+ resp = await client.get(f"{BRASIL_API_BANKS_URL}/{int(compe)}")+ resp.raise_for_status()+ payload = resp.json()+ # `name` (e.g. "BCO XP S.A.") is short, matching the style Pluggy+ # itself uses for account names — `fullName` is the full legal+ # razão social and too long for an institution label.+ name = payload.get("name") or None+ logo_url = payload.get("logo_url") or None+ info = {"name": name, "logo_url": logo_url} if name else None+ except Exception:+ logger.warning(+ "BrasilAPI bank-info lookup failed for COMPE %s", compe, exc_info=True+ )+ return None++ if redis_client is not None:+ try:+ await redis_client.set(+ cache_key, json.dumps(info) if info else "", ex=_BANK_INFO_CACHE_TTL_SECONDS+ )+ except Exception:+ pass+ return info+++async def _annotate_institutions(+ raw_accounts: list[dict], accounts: list[AccountData]+) -> None:+ """Tell apart same-subtype accounts that belong to different legal+ institutions under one Pluggy connection.++ Pluggy is normally one institution per connection, but a banking group+ with a brokerage arm (e.g. XP) reports both the bank's checking account+ and the brokerage's settlement account as BANK/CHECKING_ACCOUNT — same+ `type`, no field distinguishes them except the COMPE code embedded in+ `bankData.transferNumber`. When every account of a given `type` in this+ connection shares one COMPE code (the common case: one checking + one+ savings, same bank), nothing is ambiguous and no account is touched.+ Only when a `type` collides across more than one COMPE code do we+ resolve and set the institution_* hint fields — which `_resolve_institution`+ (connection_service.py) turns into an `Institution` row, same mechanism+ SimpleFIN already uses for its own multi-institution connections+ (issue #345).+ """+ codes_by_external_id: dict[str, str] = {}+ codes_by_type: dict[str, set[str]] = {}+ for raw, acc in zip(raw_accounts, accounts):+ code = _compe_from_transfer_number((raw.get("bankData") or {}).get("transferNumber"))+ if not code:+ continue+ codes_by_external_id[acc.external_id] = code+ codes_by_type.setdefault(acc.type, set()).add(code)++ ambiguous_types = {t for t, codes in codes_by_type.items() if len(codes) > 1}+ if not ambiguous_types:+ return++ for acc in accounts:+ if acc.type not in ambiguous_types:+ continue+ code = codes_by_external_id.get(acc.external_id)+ if not code:+ continue+ info = await _lookup_bank_info(code)+ if info:+ acc.institution_external_id = code+ acc.institution_name = info["name"]+ acc.institution_logo_url = info["logo_url"]++def _resolve_connector_logo(connector: dict, accounts: list[dict]) -> Optional[str]:"""Institution logo URL for a Pluggy connection, or None.@@ -384,6 +490,7 @@ class PluggyProvider(BankProvider):account_list = []for acc in raw_accounts:account_list.append(_build_account_data(acc, self._map_account_type))+ await _annotate_institutions(raw_accounts, account_list)return ConnectionData(external_id=item_id,@@ -428,6 +535,7 @@ class PluggyProvider(BankProvider):accounts = []for acc in data.get("results", []):accounts.append(_build_account_data(acc, self._map_account_type))+ await _annotate_institutions(data.get("results", []), accounts)return accountsasync def get_transactions(diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.pyindex 823a4d41..cd1c065b 100644--- a/backend/app/services/connection_service.py+++ b/backend/app/services/connection_service.py@@ -117,9 +117,11 @@ async def _resolve_institution(Matched by the provider's stable org id when it sends one, so a bankrenamed on the provider side updates its row in place instead of mintinga new one (review on #654); name identity is the fallback for servers- that only send a name. Providers without per-account hints- (Pluggy/Enable — one institution per connection) return None, and- serialization falls back to the connection's own fields.+ that only send a name. Most Pluggy/Enable connections are one+ institution and never send this hint (returns None, serialization falls+ back to the connection's own fields) — the exception is a Pluggy+ connection spanning a banking group's brokerage arm (issue #723), which+ the provider detects and hints the same way SimpleFIN already does."""name = _clean_institution_name(acc_data.institution_name)if not name:@@ -1117,6 +1119,7 @@ async def handle_oauth_callback(connection_id=connection.id,external_id=acc_data.external_id,name=acc_data.name,+ display_name=institution.name if institution else None,masked_number=acc_data.masked_number,type=acc_data.type,balance=acc_data.balance,@@ -2029,6 +2032,10 @@ async def sync_connection(# Backfills existing accounts on next sync (issue #345).if institution is not None:account.institution_id = institution.id+ # Only when the user hasn't named the account themselves —+ # never overwrite a manual display_name.+ if account.display_name is None:+ account.display_name = institution.nameif acc_data.type == "credit_card":# Preserve existing CC metadata when the provider doesn't# expose it. Pluggy's creditData fields (limit, close/due@@ -2061,6 +2068,7 @@ async def sync_connection(connection_id=connection.id,external_id=acc_data.external_id,name=acc_data.name,+ display_name=institution.name if institution else None,masked_number=acc_data.masked_number,type=acc_data.type,balance=acc_data.balance,diff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.pyindex 9939495a..f93b323a 100644--- a/backend/tests/test_connection_service.py+++ b/backend/tests/test_connection_service.py@@ -721,6 +721,43 @@ async def test_handle_oauth_callback_creates_connection(session: AsyncSession, tassert transaction.description_is_rule_managed is False+async def test_handle_oauth_callback_names_disambiguated_account_from_institution(+ session: AsyncSession, test_user, test_workspace,+):+ """A connection spanning a banking group's brokerage arm (issue #723) sends+ a per-account institution hint. The account's display_name adopts it on+ creation, since the raw provider name alone ("XP" for both) can't tell the+ two apart in the account list (issue #724)."""+ mock_provider = AsyncMock()+ mock_provider.handle_oauth_callback = AsyncMock(return_value=ConnectionData(+ external_id="ext-oauth-3",+ institution_name="XP",+ credentials={"token": "xyz"},+ accounts=[+ AccountData(+ external_id="acc-broker", name="XP",+ type="checking", balance=Decimal("100"), currency="BRL",+ institution_external_id="348",+ institution_name="XP Investimentos CCTVM S/A",+ ),+ ],+ ))+ mock_provider.get_transactions = AsyncMock(return_value=[])++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ await handle_oauth_callback(session, test_workspace.id, test_user.id, "auth-code", "pluggy")++ account = (+ await session.execute(select(Account).where(Account.external_id == "acc-broker"))+ ).scalar_one()+ assert account.name == "XP"+ assert account.display_name == "XP Investimentos CCTVM S/A"++@pytest.mark.asyncioasync def test_handle_oauth_callback_deduplicates_initial_transactions(session: AsyncSession, test_user, test_workspace@@ -3816,6 +3853,7 @@ async def test_sync_wires_account_institutions_and_reaps_orphans(assert inst.name == "Chase"refreshed = await session.get(Account, account_id)assert refreshed is not None and refreshed.institution_id == inst.id+ assert refreshed.display_name == "Chase" # backfilled: had no display_name yetassert await session.get(Inst, stray_id) is None # reaped: nothing references itassert await session.get(Inst, kept_id) is not None # wallet keeps its label@@ -3833,6 +3871,35 @@ async def test_sync_wires_account_institutions_and_reaps_orphans(assert {i.name for i in read.institutions} == {"Chase Bank", "Departed Brokerage"}+async def test_sync_keeps_user_defined_account_display_name(+ session: AsyncSession, test_user, test_workspace,+):+ """A user-chosen display_name is never clobbered by an institution hint,+ even when the account is newly resolved as ambiguous (issue #724)."""+ conn = await _make_connection(session, test_user.id, "Sync Bank")+ conn_id = conn.id++ session.add(Account(+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,+ connection_id=conn_id, external_id="acc-1", name="Checking",+ display_name="My Checking", type="checking",+ balance=Decimal("0"), currency="USD",+ ))+ await session.commit()++ with patch("app.services.connection_service.get_provider", return_value=_institution_provider("Chase")), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ await sync_connection(session, conn_id, test_workspace.id, test_user.id)++ refreshed = (+ await session.execute(select(Account).where(Account.external_id == "acc-1"))+ ).scalar_one()+ assert refreshed.display_name == "My Checking"++@pytest.mark.asyncioasync def test_sync_holdings_readopts_orphaned_wallet_after_reconnect(session: AsyncSession, test_user, test_workspace,diff --git a/backend/tests/test_providers_pluggy.py b/backend/tests/test_providers_pluggy.pyindex a69e5963..1bb6840c 100644--- a/backend/tests/test_providers_pluggy.py+++ b/backend/tests/test_providers_pluggy.py@@ -474,3 +474,123 @@ def test_build_account_data_without_subtype_still_maps_bank_to_checking():out = _build_account_data(acc, PluggyProvider._map_account_type)assert out.type == "checking"+++# ----- per-account institution hints for banking-group brokerage accounts+# (issue #723) -----+++def _bank_account(external_id: str, compe: str) -> dict:+ return {+ "id": external_id,+ "name": "XP",+ "type": "BANK",+ "subtype": "CHECKING_ACCOUNT",+ "balance": 0,+ "currencyCode": "BRL",+ "bankData": {"transferNumber": f"{compe}/0001/00437907-0"},+ }+++async def test_annotate_institutions_resolves_colliding_checking_accounts():+ """Two BANK/CHECKING_ACCOUNT rows in the same connection with different+ COMPE codes (bank vs. its brokerage arm, e.g. XP 348/102) are ambiguous —+ both get institution_* hints resolved."""+ from app.providers.pluggy import _annotate_institutions, _build_account_data++ raw = [_bank_account("acc-bank", "348"), _bank_account("acc-broker", "102")]+ accounts = [_build_account_data(a, PluggyProvider._map_account_type) for a in raw]++ async def fake_lookup(compe):+ return {+ "348": {"name": "Banco XP S.A.", "logo_url": "https://x/348.svg"},+ "102": {"name": "XP Investimentos CCTVM S/A", "logo_url": "https://x/102.svg"},+ }[compe]++ with patch("app.providers.pluggy._lookup_bank_info", side_effect=fake_lookup):+ await _annotate_institutions(raw, accounts)++ by_id = {a.external_id: a for a in accounts}+ assert by_id["acc-bank"].institution_external_id == "348"+ assert by_id["acc-bank"].institution_name == "Banco XP S.A."+ assert by_id["acc-bank"].institution_logo_url == "https://x/348.svg"+ assert by_id["acc-broker"].institution_external_id == "102"+ assert by_id["acc-broker"].institution_name == "XP Investimentos CCTVM S/A"+++async def test_annotate_institutions_leaves_non_colliding_accounts_untouched():+ """Checking + savings at the same bank (identical COMPE code) is the+ common case — no ambiguity, no lookup call, no institution_* hint set."""+ from app.providers.pluggy import _annotate_institutions, _build_account_data++ checking = _bank_account("acc-checking", "237")+ savings = dict(checking, id="acc-savings", subtype="SAVINGS_ACCOUNT")+ raw = [checking, savings]+ accounts = [_build_account_data(a, PluggyProvider._map_account_type) for a in raw]++ with patch("app.providers.pluggy._lookup_bank_info") as fake_lookup:+ await _annotate_institutions(raw, accounts)++ fake_lookup.assert_not_called()+ assert all(a.institution_name is None for a in accounts)+ assert all(a.institution_external_id is None for a in accounts)+++async def test_lookup_bank_info_returns_cached_value_without_http_call():+ from app.providers.pluggy import _lookup_bank_info++ fake_redis = MagicMock()+ fake_redis.get = AsyncMock(+ return_value='{"name": "Banco XP S.A.", "logo_url": "https://x/348.svg"}'+ )++ with patch("app.providers.pluggy.get_settings") as mock_settings, \+ patch("app.core.redis.get_redis", new=AsyncMock(return_value=fake_redis)), \+ patch("app.providers.pluggy.httpx.AsyncClient") as fake_client_cls:+ mock_settings.return_value.brasilapi_institution_lookup_enabled = True+ result = await _lookup_bank_info("348")++ assert result == {"name": "Banco XP S.A.", "logo_url": "https://x/348.svg"}+ fake_client_cls.assert_not_called()+++async def test_lookup_bank_info_is_none_and_non_fatal_on_http_failure():+ """A BrasilAPI outage must never break sync — it just means no hint."""+ from app.providers.pluggy import _lookup_bank_info++ fake_redis = MagicMock()+ fake_redis.get = AsyncMock(return_value=None)+ fake_redis.set = AsyncMock()++ fake_client = MagicMock()+ fake_client.get = AsyncMock(side_effect=RuntimeError("boom"))+ fake_client.__aenter__ = AsyncMock(return_value=fake_client)+ fake_client.__aexit__ = AsyncMock(return_value=None)++ with patch("app.providers.pluggy.get_settings") as mock_settings, \+ patch("app.core.redis.get_redis", new=AsyncMock(return_value=fake_redis)), \+ patch("app.providers.pluggy.httpx.AsyncClient", return_value=fake_client):+ mock_settings.return_value.brasilapi_institution_lookup_enabled = True+ result = await _lookup_bank_info("348")++ assert result is None+++async def test_lookup_bank_info_disabled_by_default_skips_http_call():+ """No BrasilAPI call — not even a cache read — until a deploy opts in."""+ from app.providers.pluggy import _lookup_bank_info++ with patch("app.providers.pluggy.get_settings") as mock_settings, \+ patch("app.core.redis.get_redis") as fake_get_redis, \+ patch("app.providers.pluggy.httpx.AsyncClient") as fake_client_cls:+ mock_settings.return_value.brasilapi_institution_lookup_enabled = False+ result = await _lookup_bank_info("348")++ assert result is None+ fake_get_redis.assert_not_called()+ fake_client_cls.assert_not_called()