Open-source personal finance manager. Self-hosted, privacy-first.
fix: tolerate duplicate (account_id, external_id) rows in sync and import (#456)
Tássio Noronha7b5931912cae
diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.pyindex 4e436cf8..8cf6ac84 100644--- a/backend/app/services/connection_service.py+++ b/backend/app/services/connection_service.py@@ -1282,12 +1282,21 @@ async def sync_connection(for txn_data in transactions_data:existing = await session.execute(- select(Transaction).where(+ select(Transaction)+ .where(Transaction.account_id == account.id,Transaction.external_id == txn_data.external_id,)+ .order_by(Transaction.created_at))- existing_tx = existing.scalar_one_or_none()+ # `.first()` rather than `.scalar_one_or_none()`: a prior sync+ # race (two overlapping passes both select-then-insert the same+ # external_id before either commits) can leave two rows sharing+ # (account_id, external_id). scalar_one_or_none() would raise+ # MultipleResultsFound and abort the whole connection's sync;+ # we instead reconcile onto the oldest matching row and skip+ # re-inserting, so a stray duplicate is harmless and never grows.+ existing_tx = existing.scalars().first()if existing_tx:# User-flagged rows are frozen: skip status/bill drift so# a re-sync can't revive a transaction the user hid.diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.pyindex f26144c5..f8e6bac2 100644--- a/backend/app/services/import_service.py+++ b/backend/app/services/import_service.py@@ -630,7 +630,12 @@ async def import_transactions(Transaction.description == txn_data.description,))- if existing.scalar_one_or_none():+ # `.first()` rather than `.scalar_one_or_none()`: the dedup key can+ # legitimately match more than one row (e.g. a prior sync/import race+ # left a duplicate, or a bank reuses one FITID across statements),+ # and we only need to know whether *any* match exists. Requiring+ # exactly one would raise MultipleResultsFound and abort the import.+ if existing.scalars().first() is not None:skipped += 1continuediff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex 215c322a..3991258d 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -42,7 +42,10 @@ dev = ["pytest-xdist>=3.6.1","httpx>=0.26.0","aiosqlite>=0.19.0",- "ruff>=0.1.0",+ # Upper-bounded so CI lint stays deterministic: ruff 0.16 reformats the+ # historical alembic/ migration import blocks. Bump deliberately (with the+ # accompanying reformat) rather than letting CI float to a newer release.+ "ruff>=0.1.0,<0.16",][tool.setuptools.packages.find]diff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.pyindex dae6b201..24350731 100644--- a/backend/tests/test_connection_service.py+++ b/backend/tests/test_connection_service.py@@ -706,6 +706,72 @@ async def test_sync_connection_new_transactions(session: AsyncSession, test_userassert merged == 0+async def test_sync_connection_tolerates_duplicate_transaction_rows(+ session: AsyncSession, test_user, test_workspace+):+ """A pre-existing (account_id, external_id) duplicate — the state an earlier+ concurrent-sync race leaves behind — must not abort the sync. Regression for+ the MultipleResultsFound crash at the transaction dedup lookup.+ """+ conn = await _make_connection(session, test_user.id, "Dup Bank")++ account = Account(+ id=uuid.uuid4(), user_id=test_user.id, connection_id=conn.id,+ external_id="dup-acc-1", name="Checking", type="checking",+ balance=Decimal("500"), currency="BRL",+ )+ session.add(account)+ await session.flush()+ account_id = account.id # capture before sync commits/expires the ORM object++ # Two rows sharing (account_id, external_id): exactly what a sync race+ # leaves behind, and what scalar_one_or_none() used to choke on.+ for _ in range(2):+ session.add(Transaction(+ id=uuid.uuid4(), user_id=test_user.id, account_id=account_id,+ external_id="dup-tx-1", description="SPOTIFY", amount=Decimal("23.90"),+ date=date.today(), type="debit", status="pending", source="sync",+ created_at=datetime.now(timezone.utc),+ ))+ await session.commit()++ mock_provider = AsyncMock()+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})+ mock_provider.get_accounts = AsyncMock(return_value=[+ AccountData(+ external_id="dup-acc-1", name="Checking",+ type="checking", balance=Decimal("500"), currency="BRL",+ ),+ ])+ mock_provider.get_transactions = AsyncMock(return_value=[+ TransactionData(+ external_id="dup-tx-1", description="SPOTIFY",+ amount=Decimal("23.90"), date=date.today(), type="debit",+ currency="BRL", status="posted",+ ),+ ])++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service._cleanup_phantom_duplicates", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ result_conn, _ = await sync_connection(session, conn.id, test_workspace.id, test_user.id)++ assert result_conn.status == "active"+ # Sync reconciled onto an existing row instead of inserting a third, and did+ # not raise. The incoming "posted" status is applied to one of the twins.+ rows = (await session.execute(+ select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.external_id == "dup-tx-1",+ )+ )).scalars().all()+ assert len(rows) == 2+ assert any(r.status == "posted" for r in rows)++@pytest.mark.asyncioasync def test_sync_connection_not_found(session: AsyncSession, test_user, test_workspace):with pytest.raises(ValueError, match="not found"):diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.pyindex ba3afe7c..abfa38a1 100644--- a/backend/tests/test_import_service.py+++ b/backend/tests/test_import_service.py@@ -2157,4 +2157,47 @@ class TestForceUncategorized:tx = (await session.execute(select(Transaction).where(Transaction.import_id == import_log_id))).scalar_one()- assert tx.category_id == cat.id\ No newline at end of file+ assert tx.category_id == cat.id+++@patch("app.services.fx_rate_service._provider")+async def test_import_tolerates_duplicate_external_id_rows(+ mock_provider, session: AsyncSession, test_user: User, test_workspace, test_account: Account,+):+ """Two existing rows sharing (account_id, external_id, date) must not crash+ the importer's duplicate check. Regression for the MultipleResultsFound+ crash: the incoming charge is skipped as a duplicate, no exception raised.+ """+ from app.models.transaction import Transaction+ from app.schemas.transaction import TransactionImport+ from sqlalchemy import select++ d = date(2026, 1, 15)+ for _ in range(2):+ session.add(Transaction(+ id=uuid.uuid4(), user_id=test_user.id, account_id=test_account.id,+ external_id="FITID-DUP", description="SPOTIFY", amount=Decimal("23.90"),+ date=d, type="debit", source="ofx",+ ))+ await session.commit()++ txns = [TransactionImport(+ description="SPOTIFY", amount=Decimal("23.90"), date=d,+ type="debit", external_id="FITID-DUP",+ )]++ imported, skipped, _, _ = await import_transactions(+ session, test_workspace.id, test_user.id, test_account.id, txns, "ofx",+ detected_format="ofx",+ )++ assert imported == 0+ assert skipped == 1+ remaining = (await session.execute(+ select(Transaction).where(+ Transaction.account_id == test_account.id,+ Transaction.external_id == "FITID-DUP",+ )+ )).scalars().all()+ assert len(remaining) == 2\ No newline at end of file