Open-source personal finance manager. Self-hosted, privacy-first.
fix(simplefin): keep syncing healthy accounts when one needs reauthorization (#1269)
Tássio Noronha7ef283210afc
diff --git a/backend/app/providers/simplefin.py b/backend/app/providers/simplefin.pyindex deb5fe07..9062ad62 100644--- a/backend/app/providers/simplefin.py+++ b/backend/app/providers/simplefin.py@@ -188,6 +188,10 @@ class SimpleFinProvider(BankProvider):def __init__(self) -> None:self.action_required_warnings: list[dict] = []+ # External ids of accounts whose transaction fetch stopped early on+ # an authorization failure. Their result is partial (earlier chunks+ # only), so the sync layer must not treat it as the full live set.+ self.incomplete_account_ids: set[str] = set()@propertydef name(self) -> str:@@ -492,19 +496,77 @@ class SimpleFinProvider(BankProvider):cursor + timedelta(days=SIMPLEFIN_MAX_WINDOW_DAYS),end_exclusive,)- payload = await self._fetch_accounts(- credentials,- start_date=cursor,- end_date=chunk_end,- account_id=account_external_id,- pending=True,- )+ try:+ payload = await self._fetch_accounts(+ credentials,+ start_date=cursor,+ end_date=chunk_end,+ account_id=account_external_id,+ pending=True,+ )+ except ProviderUserActionRequired as exc:+ # The bridge refused the request outright (HTTP 401/403):+ # this account's auth expired. Same wall as the errlist case+ # below, reached the other way. One account's expiry must not+ # abort the whole connection's sync (issue #1077): record the+ # warning so the sync layer flags the connection, keep what+ # we already fetched for this account, and let the other+ # accounts sync normally. The access URL itself is never+ # invalidated, so the user reauthorizes the affected bank at+ # the bridge and the next sync picks this account back up+ # without touching the token.+ self.action_required_warnings.append(+ {"code": exc.code, "msg": str(exc)}+ )+ logger.warning(+ "SimpleFIN get_transactions: bridge refused the request "+ "for account %s (code=%s); skipping it for this sync",+ account_external_id,+ exc.code,+ )+ self.incomplete_account_ids.add(account_external_id)+ return transactionsmatching_accounts = [raw_acc for raw_acc in payload.get("accounts") or []if str(raw_acc.get("id") or "") == account_external_id]+ errlist = payload.get("errlist") or []+ reauth = [+ e for e in errlist+ if (e.get("code") or "").lower() in _REAUTH_ERROR_CODES+ ]+ if reauth:+ # Reauth warnings are always recorded, so the connection+ # stays flagged for reauthorization. When the bridge answered+ # but has no data for this account, its sub-connection's auth+ # expired: soft-skip it instead of aborting the whole+ # connection's sync (issue #1077). The next sync retries+ # automatically once the user reauthorizes the affected bank+ # at the bridge; no new token is required.+ for e in reauth:+ logger.warning(+ "SimpleFIN get_transactions: account %s warning "+ "code=%s msg=%s",+ account_external_id,+ e.get("code"),+ e.get("msg") or e.get("message"),+ )+ self.action_required_warnings.extend(reauth)+ if not matching_accounts:+ logger.warning(+ "SimpleFIN get_transactions: account %s needs "+ "reauthorization (codes=%s); skipping it for this "+ "sync",+ account_external_id,+ [e.get("code") for e in reauth],+ )+ self.incomplete_account_ids.add(account_external_id)+ return transactionswarnings = _surface_errors(- payload.get("errlist") or [],+ [+ e for e in errlist+ if (e.get("code") or "").lower() not in _REAUTH_ERROR_CODES+ ],context="get_transactions",has_usable_data=bool(matching_accounts),)diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.pyindex d91e9938..38520c54 100644--- a/backend/app/services/connection_service.py+++ b/backend/app/services/connection_service.py@@ -1687,6 +1687,7 @@ async def _cleanup_phantom_duplicates(session: AsyncSession,connection_id: uuid.UUID,live_external_ids: Optional[dict[uuid.UUID, set[str]]] = None,+ skip_account_ids: Optional[set[uuid.UUID]] = None,) -> set[uuid.UUID]:"""Delete synced transactions that are phantom duplicates.@@ -1710,13 +1711,21 @@ async def _cleanup_phantom_duplicates(provider's fetch window, so this sync has no evidence about them, and theywere already checked while they were in it.+ ``skip_account_ids`` are accounts whose fetch this sync was cut short+ (e.g. an authorization failure mid-window). Their live set is partial,+ so a row from the missing part would look like a re-keyed twin of a+ returned sibling; they are left alone until a complete fetch.+Returns the ids of the accounts that lost a row, so the caller canreconcile their opening balances against what is left."""accounts_result = await session.execute(select(Account.id).where(Account.connection_id == connection_id))- account_ids = [row[0] for row in accounts_result.all()]+ account_ids = [+ row[0] for row in accounts_result.all()+ if row[0] not in (skip_account_ids or set())+ ]if not account_ids:return set()@@ -2102,6 +2111,7 @@ async def sync_connection(new_tx_ids: list[uuid.UUID] = []merged_count = 0live_txn_external_ids: dict[uuid.UUID, set[str]] = {}+ incomplete_fetch_account_ids: set[uuid.UUID] = set()accounts_data = await provider.get_accounts(credentials)incoming_external_ids = {acc.external_id for acc in accounts_data}incoming_institution_external_ids = frozenset(@@ -2260,6 +2270,9 @@ async def sync_connection(live_txn_external_ids[account.id] = {txn.external_id for txn in transactions_data}+ incomplete = getattr(provider, "incomplete_account_ids", None)+ if isinstance(incomplete, set) and acc_data.external_id in incomplete:+ incomplete_fetch_account_ids.add(account.id)if not import_pending:transactions_data = [t for t in transactions_data if t.status != "pending"]@@ -2540,7 +2553,8 @@ async def sync_connection(# same payment with different ids. Once transfer detection has paired# the real one, the orphan twin gets removed here.touched_account_ids = await _cleanup_phantom_duplicates(- session, connection.id, live_txn_external_ids+ session, connection.id, live_txn_external_ids,+ skip_account_ids=incomplete_fetch_account_ids,)# The opening balances above were reconciled with the phantoms still@@ -2583,7 +2597,13 @@ async def sync_connection(for orphan in orphaned_institutions.scalars().all():await session.delete(orphan)- connection.last_sync_at = datetime.now(timezone.utc)+ # An account whose fetch was cut short (issue #1077) is missing the+ # rest of the window. The next fetch starts from last_sync_at, so+ # moving it forward would leave that gap behind for good once the+ # user reauthorizes; keep it until every account fetched in full.+ # The healthy accounts just re-read a longer window, deduplicated.+ if not incomplete_fetch_account_ids:+ connection.last_sync_at = datetime.now(timezone.utc)action_required_warnings = getattr(provider, "action_required_warnings", None)if isinstance(action_required_warnings, list) and action_required_warnings:logger.warning(diff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.pyindex 1b2e0ec3..b1f628fb 100644--- a/backend/tests/test_connection_service.py+++ b/backend/tests/test_connection_service.py@@ -1,5 +1,5 @@import uuid-from datetime import date, datetime, timezone+from datetime import date, datetime, timedelta, timezonefrom decimal import Decimalfrom unittest.mock import AsyncMock, MagicMock, patch@@ -932,6 +932,110 @@ async def test_sync_connection_partial_user_action_warning_marks_connection_erroassert synced_tx.description == "GROCERY"+async def test_sync_connection_simplefin_partial_expiry_keeps_healthy_account(+ session: AsyncSession, test_user, test_workspace,+):+ """One expired sub-connection must not block the rest (issue #1077).++ The real SimpleFinProvider runs against a fake bridge where one of the+ two accounts needs reauthorization. The healthy account's new+ transaction must still land, and the connection comes back flagged+ (status "error") so the UI offers retry plus bridge reauthorization,+ instead of the sync aborting and rolling the healthy account back.+ """+ import httpx++ from app.providers.simplefin import SimpleFinProvider+ from tests.test_providers_simplefin import _patched_client++ conn = await _make_connection(session, test_user.id, "Bridge Bank")+ conn.provider = "simplefin"+ conn.credentials = {"access_url": "https://u:[email protected]/simplefin"}+ previous_sync = datetime.now(timezone.utc) - timedelta(days=5)+ conn.last_sync_at = previous_sync+ await session.commit()+ await session.refresh(conn)++ now_epoch = int(datetime.now(timezone.utc).timestamp())+ day = 86400+ conn_a = {+ "conn_id": "C-A", "name": "Bank A", "org_id": "A",+ "org_url": "https://bank-a.example",+ }+ conn_b = {+ "conn_id": "C-B", "name": "Bank B", "org_id": "B",+ "org_url": "https://bank-b.example",+ }+ auth_err = {"code": "con.auth", "msg": "Authentication required", "conn_id": "C-B"}++ def handler(request: httpx.Request) -> httpx.Response:+ account = request.url.params.get("account")+ if account == "acc-healthy":+ return httpx.Response(200, json={+ "errlist": [],+ "connections": [conn_a],+ "accounts": [{+ "id": "acc-healthy", "name": "Checking", "currency": "USD",+ "balance": "100.00",+ "transactions": [{+ "id": "tx-healthy-new", "posted": now_epoch - day,+ "transacted_at": now_epoch - day,+ "amount": "-42.10", "description": "STORE PURCHASE",+ }],+ }],+ })+ if account == "acc-expired":+ # The bridge's documented partial-expiry shape for the per-account+ # query: no data, con.auth in the errlist.+ return httpx.Response(200, json={+ "errlist": [auth_err],+ "connections": [conn_b],+ "accounts": [],+ })+ # Whole-connection query: both accounts still listed, B flagged stale.+ return httpx.Response(200, json={+ "errlist": [auth_err],+ "connections": [conn_a, conn_b],+ "accounts": [+ {+ "id": "acc-healthy", "name": "Checking", "currency": "USD",+ "balance": "100.00", "balance-date": now_epoch - day,+ },+ {+ "id": "acc-expired", "name": "Savings", "currency": "USD",+ "balance": "50.00", "balance-date": now_epoch - 10 * day,+ },+ ],+ })++ provider = SimpleFinProvider()+ with patch("app.services.connection_service.get_provider", return_value=provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock), \+ _patched_client(handler):+ result_conn, merged, outcome = await sync_connection(+ session, conn.id, test_workspace.id, test_user.id+ )++ # The sync itself succeeds (no 409, no rollback of the healthy account)...+ assert outcome == "refreshed"+ healthy_tx = (await session.execute(+ select(Transaction).where(Transaction.external_id == "tx-healthy-new")+ )).scalar_one()+ assert healthy_tx.description == "STORE PURCHASE"+ # ...and the connection is flagged for reauthorization instead of the+ # whole link being dead.+ assert result_conn.status == "error"+ assert any(w.get("code") == "con.auth" for w in provider.action_required_warnings)+ # The expired account fetched nothing, so the sync cursor stays put: once+ # the bank is reauthorized, the next sync still covers the missed days.+ kept = result_conn.last_sync_at+ assert kept is not None+ assert kept.replace(tzinfo=timezone.utc) == previous_sync++@pytest.mark.asyncioasync def test_sync_connection_simplefin_rekey_does_not_duplicate_account_or_tx(session: AsyncSession, test_user, test_workspace,@@ -1937,6 +2041,70 @@ async def test_phantom_cleanup_keeps_a_same_day_repeat_the_provider_still_returnassert await synced_ids() == {"pix-to-nubank", "pix-to-mercado-pago"}+async def test_phantom_cleanup_skips_an_account_whose_fetch_was_cut_short(+ session: AsyncSession, test_user, test_workspace+):+ """An authorization failure on a later window chunk leaves a partial+ live set: the paired sibling came back, the unpaired row from the+ failed chunk did not. That is not evidence of a re-keyed twin, so the+ cleanup must leave the account alone for this sync (#1077)."""+ conn = await _make_connection(session, test_user.id, "Partial Bank")++ account = Account(+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,+ connection_id=conn.id, external_id="partial-acc-1", name="Checking",+ type="checking", balance=Decimal("50000"), currency="BRL",+ )+ session.add(account)+ await session.flush()+ account_id = account.id++ payment_day = date.today()+ for external_id, pair in (("returned-paired", uuid.uuid4()), ("in-failed-chunk", None)):+ session.add(Transaction(+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,+ account_id=account_id, external_id=external_id,+ description="Pix enviado Own Name", amount=Decimal("10000.00"),+ date=payment_day, type="debit", status="posted", source="sync",+ currency="BRL", transfer_pair_id=pair,+ created_at=datetime.now(timezone.utc),+ ))+ await session.commit()++ mock_provider = AsyncMock()+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})+ mock_provider.action_required_warnings = [{"code": "credentials_invalid"}]+ mock_provider.incomplete_account_ids = {"partial-acc-1"}+ mock_provider.get_accounts = AsyncMock(return_value=[+ AccountData(+ external_id="partial-acc-1", name="Checking", type="checking",+ balance=Decimal("50000"), currency="BRL",+ ),+ ])+ mock_provider.get_transactions = AsyncMock(return_value=[+ TransactionData(+ external_id="returned-paired", description="Pix enviado Own Name",+ amount=Decimal("10000.00"), date=payment_day, type="debit",+ currency="BRL", status="posted",+ ),+ ])++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)++ rows = (await session.execute(+ select(Transaction.external_id).where(+ Transaction.account_id == account_id,+ Transaction.source == "sync",+ )+ )).scalars().all()+ assert set(rows) == {"returned-paired", "in-failed-chunk"}++@pytest.mark.asyncioasync def test_phantom_cleanup_counts_a_returned_pending_id_with_import_pending_off(session: AsyncSession, test_user, test_workspacediff --git a/backend/tests/test_providers_simplefin.py b/backend/tests/test_providers_simplefin.pyindex 70c37c72..175647de 100644--- a/backend/tests/test_providers_simplefin.py+++ b/backend/tests/test_providers_simplefin.py@@ -369,6 +369,95 @@ async def test_get_transactions_auth_errlist_with_requested_account_is_soft_warnassert any("con.auth" in rec.getMessage() for rec in caplog.records)+async def test_get_transactions_expired_account_soft_skips_and_flags_reauth(caplog):+ """A per-account auth expiry must skip the account, not abort (issue #1077).++ The bridge answers a per-account query for an expired sub-connection+ with no accounts plus a ``con.auth`` errlist entry. The provider+ records the warning for the sync layer and returns what it has (nothing+ for this account) instead of raising, so the rest of the connection+ keeps syncing and the connection stays flagged for reauthorization.+ """+ import logging as stdlogging++ def handler(request: httpx.Request) -> httpx.Response:+ assert request.url.params.get("account") == "acc-expired"+ return httpx.Response(+ 200,+ json={+ "errlist": [+ {"code": "con.auth", "msg": "Authentication required", "conn_id": "C-B"}+ ],+ "accounts": [],+ },+ )++ creds = {"access_url": "https://u:[email protected]/simplefin"}+ provider = SimpleFinProvider()+ with caplog.at_level(stdlogging.WARNING, logger="app.providers.simplefin"), _patched_client(handler):+ txns = await provider.get_transactions(creds, "acc-expired", since=date(2026, 9, 1))++ assert txns == []+ assert [w["code"] for w in provider.action_required_warnings] == ["con.auth"]+ assert provider.incomplete_account_ids == {"acc-expired"}+ assert any("reauthorization" in rec.getMessage() for rec in caplog.records)+++async def test_get_transactions_http_refusal_soft_skips_and_flags_reauth(caplog):+ """An HTTP refusal of the per-account query must skip, not abort (issue #1077)."""+ import logging as stdlogging++ def handler(request: httpx.Request) -> httpx.Response:+ assert request.url.params.get("account") == "acc-expired"+ return httpx.Response(403, text="authentication required")++ creds = {"access_url": "https://u:[email protected]/simplefin"}+ provider = SimpleFinProvider()+ with caplog.at_level(stdlogging.WARNING, logger="app.providers.simplefin"), _patched_client(handler):+ txns = await provider.get_transactions(creds, "acc-expired", since=date(2026, 9, 1))++ assert txns == []+ assert [w["code"] for w in provider.action_required_warnings] == ["credentials_invalid"]+ assert provider.incomplete_account_ids == {"acc-expired"}+ assert any("refused" in rec.getMessage() for rec in caplog.records)+++async def test_get_transactions_refusal_after_first_chunk_marks_fetch_incomplete():+ """A refusal on a later window chunk keeps the earlier chunk's rows but+ marks the account's fetch as incomplete, so the sync layer does not+ treat the partial result as everything the bridge still has."""+ calls: list[str] = []++ def handler(request: httpx.Request) -> httpx.Response:+ calls.append(request.url.params.get("start-date"))+ if len(calls) == 1:+ return httpx.Response(200, json={+ "errlist": [],+ "accounts": [{+ "id": "acc-1", "currency": "USD", "balance": "10.00",+ "transactions": [{+ "id": "tx-early", "posted": 1767225600,+ "amount": "-5.00", "description": "EARLY ROW",+ }],+ }],+ })+ return httpx.Response(403, text="authentication required")++ creds = {"access_url": "https://u:[email protected]/simplefin"}+ provider = SimpleFinProvider()+ with _patched_client(handler):+ txns = await provider.get_transactions(+ creds, "acc-1", since=date.today() - timedelta(days=150)+ )++ assert len(calls) == 2+ assert [t.external_id for t in txns] == ["tx-early"]+ assert provider.incomplete_account_ids == {"acc-1"}++# ----- transactions -----------------------------------------------------------