Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix: tolerate duplicate (account_id, external_id) rows in sync and import (#456)

Tássio Noronha7b5931912cae
diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.py
index 4e436cf8..8cf6ac84 100644
--- a/backend/app/services/connection_service.py
+++ b/backend/app/services/connection_service.py
@@ -1282,12 +1282,21 @@ async def sync_connection(
for txn_data in transactions_data:
existing = await session.execute(
- select(Transaction).where(
+ select(Transaction)
+ .where(
Transaction.account_id == account.id,
Transaction.external_id == txn_data.external_id,
)
+ .order_by(Transaction.created_at)
)
- existing_tx = existing.scalar_one_or_none()
+ # `.first()` rather than `.scalar_one_or_none()`: a prior sync
+ # race (two overlapping passes both select-then-insert the same
+ # external_id before either commits) can leave two rows sharing
+ # (account_id, external_id). scalar_one_or_none() would raise
+ # MultipleResultsFound and abort the whole connection's sync;
+ # we instead reconcile onto the oldest matching row and skip
+ # re-inserting, so a stray duplicate is harmless and never grows.
+ existing_tx = existing.scalars().first()
if existing_tx:
# User-flagged rows are frozen: skip status/bill drift so
# a re-sync can't revive a transaction the user hid.
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.py
index f26144c5..f8e6bac2 100644
--- a/backend/app/services/import_service.py
+++ b/backend/app/services/import_service.py
@@ -630,7 +630,12 @@ async def import_transactions(
Transaction.description == txn_data.description,
)
)
- if existing.scalar_one_or_none():
+ # `.first()` rather than `.scalar_one_or_none()`: the dedup key can
+ # legitimately match more than one row (e.g. a prior sync/import race
+ # left a duplicate, or a bank reuses one FITID across statements),
+ # and we only need to know whether *any* match exists. Requiring
+ # exactly one would raise MultipleResultsFound and abort the import.
+ if existing.scalars().first() is not None:
skipped += 1
continue
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 215c322a..3991258d 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -42,7 +42,10 @@ dev = [
"pytest-xdist>=3.6.1",
"httpx>=0.26.0",
"aiosqlite>=0.19.0",
- "ruff>=0.1.0",
+ # Upper-bounded so CI lint stays deterministic: ruff 0.16 reformats the
+ # historical alembic/ migration import blocks. Bump deliberately (with the
+ # accompanying reformat) rather than letting CI float to a newer release.
+ "ruff>=0.1.0,<0.16",
]
[tool.setuptools.packages.find]
diff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.py
index dae6b201..24350731 100644
--- a/backend/tests/test_connection_service.py
+++ b/backend/tests/test_connection_service.py
@@ -706,6 +706,72 @@ async def test_sync_connection_new_transactions(session: AsyncSession, test_user
assert merged == 0
+async def test_sync_connection_tolerates_duplicate_transaction_rows(
+ session: AsyncSession, test_user, test_workspace
+):
+ """A pre-existing (account_id, external_id) duplicate — the state an earlier
+ concurrent-sync race leaves behind — must not abort the sync. Regression for
+ the MultipleResultsFound crash at the transaction dedup lookup.
+ """
+ conn = await _make_connection(session, test_user.id, "Dup Bank")
+
+ account = Account(
+ id=uuid.uuid4(), user_id=test_user.id, connection_id=conn.id,
+ external_id="dup-acc-1", name="Checking", type="checking",
+ balance=Decimal("500"), currency="BRL",
+ )
+ session.add(account)
+ await session.flush()
+ account_id = account.id # capture before sync commits/expires the ORM object
+
+ # Two rows sharing (account_id, external_id): exactly what a sync race
+ # leaves behind, and what scalar_one_or_none() used to choke on.
+ for _ in range(2):
+ session.add(Transaction(
+ id=uuid.uuid4(), user_id=test_user.id, account_id=account_id,
+ external_id="dup-tx-1", description="SPOTIFY", amount=Decimal("23.90"),
+ date=date.today(), type="debit", status="pending", source="sync",
+ created_at=datetime.now(timezone.utc),
+ ))
+ await session.commit()
+
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="dup-acc-1", name="Checking",
+ type="checking", balance=Decimal("500"), currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="dup-tx-1", description="SPOTIFY",
+ amount=Decimal("23.90"), date=date.today(), type="debit",
+ currency="BRL", status="posted",
+ ),
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service._cleanup_phantom_duplicates", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ result_conn, _ = await sync_connection(session, conn.id, test_workspace.id, test_user.id)
+
+ assert result_conn.status == "active"
+ # Sync reconciled onto an existing row instead of inserting a third, and did
+ # not raise. The incoming "posted" status is applied to one of the twins.
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.external_id == "dup-tx-1",
+ )
+ )).scalars().all()
+ assert len(rows) == 2
+ assert any(r.status == "posted" for r in rows)
+
+
@pytest.mark.asyncio
async def test_sync_connection_not_found(session: AsyncSession, test_user, test_workspace):
with pytest.raises(ValueError, match="not found"):
diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.py
index ba3afe7c..abfa38a1 100644
--- a/backend/tests/test_import_service.py
+++ b/backend/tests/test_import_service.py
@@ -2157,4 +2157,47 @@ class TestForceUncategorized:
tx = (await session.execute(
select(Transaction).where(Transaction.import_id == import_log_id)
)).scalar_one()
- assert tx.category_id == cat.id
\ No newline at end of file
+ assert tx.category_id == cat.id
+
+
+@patch("app.services.fx_rate_service._provider")
+async def test_import_tolerates_duplicate_external_id_rows(
+ mock_provider, session: AsyncSession, test_user: User, test_workspace, test_account: Account,
+):
+ """Two existing rows sharing (account_id, external_id, date) must not crash
+ the importer's duplicate check. Regression for the MultipleResultsFound
+ crash: the incoming charge is skipped as a duplicate, no exception raised.
+ """
+ from app.models.transaction import Transaction
+ from app.schemas.transaction import TransactionImport
+ from sqlalchemy import select
+
+ d = date(2026, 1, 15)
+ for _ in range(2):
+ session.add(Transaction(
+ id=uuid.uuid4(), user_id=test_user.id, account_id=test_account.id,
+ external_id="FITID-DUP", description="SPOTIFY", amount=Decimal("23.90"),
+ date=d, type="debit", source="ofx",
+ ))
+ await session.commit()
+
+ txns = [TransactionImport(
+ description="SPOTIFY", amount=Decimal("23.90"), date=d,
+ type="debit", external_id="FITID-DUP",
+ )]
+
+ imported, skipped, _, _ = await import_transactions(
+ session, test_workspace.id, test_user.id, test_account.id, txns, "ofx",
+ detected_format="ofx",
+ )
+
+ assert imported == 0
+ assert skipped == 1
+ remaining = (await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == test_account.id,
+ Transaction.external_id == "FITID-DUP",
+ )
+ )).scalars().all()
+ assert len(remaining) == 2
\ No newline at end of file