Open-source personal finance manager. Self-hosted, privacy-first.
fix(enable-banking): prevent duplicate transactions from pagination loops (#761)
jfrcombeaff06a00e0
diff --git a/backend/app/providers/enable_banking.py b/backend/app/providers/enable_banking.pyindex c0a73898..6b3cd7bc 100644--- a/backend/app/providers/enable_banking.py+++ b/backend/app/providers/enable_banking.py@@ -547,6 +547,8 @@ class EnableBankingProvider(BankProvider):date_to = date.today().isoformat()transactions: list[TransactionData] = []continuation_key: Optional[str] = None+ seen_continuation_keys: set[str] = set()+ seen_transaction_ids: set[str] = set()for _ in range(TRANSACTION_PAGE_LIMIT):params: dict[str, Any] = {"date_from": date_from, "date_to": date_to}if continuation_key:@@ -560,11 +562,25 @@ class EnableBankingProvider(BankProvider):parsed = self._build_transaction(account_external_id, raw_txn, status, payee_source)- if parsed:+ # A broken pagination cursor can make Enable Banking return a+ # page we have already consumed. Keep the result idempotent+ # even before the repeated cursor is detected below.+ if parsed and parsed.external_id not in seen_transaction_ids:transactions.append(parsed)- continuation_key = page.get("continuation_key") or None- if not continuation_key:+ seen_transaction_ids.add(parsed.external_id)++ next_continuation_key = page.get("continuation_key") or None+ if not next_continuation_key:+ break+ if next_continuation_key in seen_continuation_keys:+ logger.warning(+ "Enable Banking pagination loop detected for account %s "+ "(repeated continuation key); stopping pagination",+ account_external_id,+ )break+ seen_continuation_keys.add(next_continuation_key)+ continuation_key = next_continuation_keyreturn transactions@staticmethoddiff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.pyindex aeda752b..41814c71 100644--- a/backend/app/services/connection_service.py+++ b/backend/app/services/connection_service.py@@ -1086,7 +1086,16 @@ async def handle_oauth_callback(transactions_data = await provider.get_transactions(connection_data.credentials, acc_data.external_id, None)+ seen_external_ids: set[str] = set()for txn_data in transactions_data:+ # Providers should return each external transaction once, but an+ # upstream pagination bug can repeat a page. This account was just+ # created, so batch-local tracking avoids both duplicate inserts+ # and an unnecessary SELECT for every imported transaction.+ if txn_data.external_id in seen_external_ids:+ continue+ seen_external_ids.add(txn_data.external_id)+# Pending↔posted twin (and the credit-card installment variant).# When the same logical operation comes back under a new external# id with a different status, fingerprint match prevents thediff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.pyindex 5c37955a..08ad79ff 100644--- a/backend/tests/test_connection_service.py+++ b/backend/tests/test_connection_service.py@@ -660,6 +660,41 @@ async def test_handle_oauth_callback_creates_connection(session: AsyncSession, tassert transaction.description_is_rule_managed is False+async def test_handle_oauth_callback_deduplicates_initial_transactions(+ session: AsyncSession, test_user, test_workspace+):+ """Repeated provider pages cannot create duplicate external ids on setup."""+ mock_provider = AsyncMock()+ mock_provider.handle_oauth_callback = AsyncMock(return_value=ConnectionData(+ external_id="ext-oauth-duplicate",+ institution_name="Test Bank",+ credentials={"token": "abc"},+ accounts=[AccountData(+ external_id="acc-duplicate", name="Checking",+ type="checking", balance=Decimal("1000"), currency="BRL",+ )],+ ))+ duplicate = TransactionData(+ external_id="tx-duplicate", description="UBER", amount=Decimal("25"),+ date=date.today(), type="debit", currency="BRL",+ )+ mock_provider.get_transactions = AsyncMock(return_value=[duplicate, duplicate])++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ await handle_oauth_callback(+ session, test_workspace.id, test_user.id, "auth-code", "enable_banking"+ )++ rows = (await session.execute(+ select(Transaction).where(Transaction.external_id == "tx-duplicate")+ )).scalars().all()+ assert len(rows) == 1++@pytest.mark.asyncioasync def test_handle_oauth_callback_with_payee(session: AsyncSession, test_user, test_workspace):mock_provider = AsyncMock()diff --git a/backend/tests/test_providers_enable_banking.py b/backend/tests/test_providers_enable_banking.pyindex 25a8207d..3c2f40bd 100644--- a/backend/tests/test_providers_enable_banking.py+++ b/backend/tests/test_providers_enable_banking.py@@ -380,6 +380,44 @@ async def test_get_transactions_parses_nested_and_flat_shapes(eb_keys):assert {t.status for t in flat} == {"posted", "pending"}+async def test_get_transactions_stops_on_repeated_continuation_key(eb_keys, caplog):+ """A provider cursor loop must not duplicate a page up to the safety cap."""+ provider = EnableBankingProvider()+ requests: list[httpx.Request] = []+ page = {+ "transactions": [+ {+ "entry_reference": "looped-tx-1",+ "status": "BOOK",+ "transaction_amount": {"amount": "10.00", "currency": "EUR"},+ "credit_debit_indicator": "DBIT",+ "booking_date": "2026-05-10",+ }+ ],+ "continuation_key": "cursor-a",+ }++ def handler(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ return httpx.Response(200, json=page)++ credentials = {+ "session_id": "sess-x",+ "valid_until": "2099-01-01T00:00:00Z",+ }+ with _patch_client(provider, handler), caplog.at_level("WARNING"):+ transactions = await provider.get_transactions(+ credentials, "acc-1", date(2026, 5, 1)+ )++ assert len(requests) == 2+ assert requests[0].url.params.get("continuation_key") is None+ assert requests[1].url.params["continuation_key"] == "cursor-a"+ assert [transaction.external_id for transaction in transactions] == ["looped-tx-1"]+ assert "pagination loop detected" in caplog.text++@pytest.mark.asyncioasync def test_refresh_credentials_expired_raises(eb_keys):provider = EnableBankingProvider()