Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix: look payees up by exact name before folding case (#678) (#683)

Tássio Noronhac17e4232f587
diff --git a/backend/app/services/payee_service.py b/backend/app/services/payee_service.py
index 2316623e..f48959c1 100644
--- a/backend/app/services/payee_service.py
+++ b/backend/app/services/payee_service.py
@@ -3,7 +3,7 @@ from datetime import date
from decimal import Decimal
from typing import Optional, cast
-from sqlalchemy import CursorResult, case, select, func, update, delete
+from sqlalchemy import CursorResult, case, literal, select, func, update, delete
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
@@ -66,6 +66,23 @@ async def get_payee(session: AsyncSession, payee_id: uuid.UUID, workspace_id: uu
return result.scalar_one_or_none()
+
+def _same_name_as(name: str):
+ """The predicate the uq_payees_workspace_id_lower_name index enforces,
+ with both sides folded by the database.
+
+ Folding the incoming name in Python instead looks equivalent and is not:
+ `lower()` follows the database's locale, and a cluster created with the
+ C locale (a common default in Kubernetes Postgres charts, unlike the
+ en_US.utf8 our compose file gets) leaves every non-ASCII capital alone
+ where Python folds it. "MÜLLER GmbH" then never matched itself: the
+ lookup missed, the insert hit the index, the retry missed again, and a
+ whole bank sync went down over one counterparty (#678). Sending the name
+ through the same `lower(trim())` the index uses makes lookup and
+ constraint agree by construction, whatever the locale.
+ """
+ return func.lower(func.trim(Payee.name)) == func.lower(func.trim(literal(name)))
+
async def get_or_create_payee(
session: AsyncSession,
user_id: uuid.UUID,
@@ -95,7 +112,7 @@ async def get_or_create_payee(
# lookup hits the same row the unique constraint would reject.
lookup = select(Payee).where(
Payee.workspace_id == workspace_id,
- func.lower(func.trim(Payee.name)) == name.lower(),
+ _same_name_as(name),
)
result = await session.execute(lookup)
payee = result.scalar_one_or_none()
@@ -196,7 +213,7 @@ async def create_payee(
existing = await session.execute(
select(Payee).where(
Payee.workspace_id == workspace_id,
- func.lower(func.trim(Payee.name)) == name.lower(),
+ _same_name_as(name),
)
)
if existing.scalar_one_or_none():
@@ -242,7 +259,7 @@ async def update_payee(
existing = await session.execute(
select(Payee).where(
Payee.workspace_id == workspace_id,
- func.lower(func.trim(Payee.name)) == name.lower(),
+ _same_name_as(name),
Payee.id != payee_id,
)
)
diff --git a/backend/tests/test_payee_lookup_collation.py b/backend/tests/test_payee_lookup_collation.py
new file mode 100644
index 00000000..406f9804
--- /dev/null
+++ b/backend/tests/test_payee_lookup_collation.py
@@ -0,0 +1,224 @@
+"""A payee whose name carries a non-ASCII capital must be found, not reinserted.
+
+Regression cover for #678. `get_or_create_payee` looked a counterparty up
+with `lower(payees.name) = :name_lowered_in_python`, which silently assumes
+Postgres and Python fold case the same way. They do not: `lower()` follows
+the database's locale, so a cluster created with the C locale (a common
+default in Kubernetes Postgres charts, unlike the `en_US.utf8` our compose
+file gets) leaves every non-ASCII capital alone while Python folds it.
+
+A name like "MÜLLER GmbH" therefore never matched itself. The first
+transaction naming that counterparty inserted it, the second one missed the
+lookup and inserted it again, and the unique constraint took down the whole
+bank sync: no accounts, no transactions, connection marked `error`.
+
+SQLite's `lower()` is ASCII-only for exactly the same reason, so the test
+database reproduces the affected deployment without needing one.
+
+The rule these tests pin: two transactions naming one counterparty create
+one payee, whatever alphabet the bank writes it in.
+"""
+import uuid
+from datetime import date, datetime, timezone
+from decimal import Decimal
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.bank_connection import BankConnection
+from app.models.payee import Payee
+from app.providers.base import AccountData, ConnectionData, TransactionData
+from app.services.connection_service import handle_oauth_callback, sync_connection
+from app.services.payee_service import get_or_create_payee
+
+# The counterparty as the bank writes it. Any non-ASCII capital does it:
+# an umlaut here, but a Cyrillic or accented name failed identically.
+COUNTERPARTY = "MÜLLER GmbH"
+
+
+def _txn(external_id: str) -> TransactionData:
+ return TransactionData(
+ external_id=external_id,
+ description="SEPA TRANSFER",
+ amount=Decimal("50"),
+ date=date.today(),
+ type="debit",
+ currency="EUR",
+ payee=COUNTERPARTY,
+ )
+
+
+def _accounts() -> list[AccountData]:
+ """Two accounts at one bank, the shape the report failed on: the
+ counterparty is shared, so the second account is where it collided."""
+ return [
+ AccountData(
+ external_id="acc-1", name="Current", type="checking",
+ balance=Decimal("100"), currency="EUR",
+ ),
+ AccountData(
+ external_id="acc-2", name="Savings", type="savings",
+ balance=Decimal("200"), currency="EUR",
+ ),
+ ]
+
+
+async def _transactions_for(credentials, external_id, since=None, payee_source="auto"):
+ return [_txn(f"{external_id}-t1"), _txn(f"{external_id}-t2")]
+
+
+async def _payees(session: AsyncSession, workspace_id: uuid.UUID) -> list[Payee]:
+ rows = await session.execute(
+ select(Payee).where(Payee.workspace_id == workspace_id)
+ )
+ return list(rows.scalars().all())
+
+
+# ---------------------------------------------------------------------------
+# the lookup itself
+# ---------------------------------------------------------------------------
+async def test_an_accented_name_finds_its_own_row(
+ session: AsyncSession, test_user, test_workspace
+):
+ """The whole bug in two calls: the second one used to insert a duplicate."""
+ first = await get_or_create_payee(
+ session, test_user.id, COUNTERPARTY, workspace_id=test_workspace.id
+ )
+ second = await get_or_create_payee(
+ session, test_user.id, COUNTERPARTY, workspace_id=test_workspace.id
+ )
+ await session.commit()
+
+ assert first.id == second.id
+ assert len(await _payees(session, test_workspace.id)) == 1
+
+
+async def test_case_folding_still_reuses_an_ascii_name(
+ session: AsyncSession, test_user, test_workspace
+):
+ """Exact-match-first must not cost us the case folding that was there:
+ a bank writing "Amazon" today and "AMAZON" tomorrow still means one
+ counterparty."""
+ first = await get_or_create_payee(
+ session, test_user.id, "Amazon", workspace_id=test_workspace.id
+ )
+ second = await get_or_create_payee(
+ session, test_user.id, "AMAZON", workspace_id=test_workspace.id
+ )
+ await session.commit()
+
+ assert first.id == second.id
+ assert len(await _payees(session, test_workspace.id)) == 1
+
+
+async def test_a_row_that_appears_between_lookup_and_insert_is_adopted(
+ session: AsyncSession, test_user, test_workspace
+):
+ """The other way this constraint gets hit: two syncs of one connection
+ overlapping, each having looked before the other inserted. Simulated by
+ blinding the pre-checks, which is what a C-locale database did to them.
+ The insert must lose gracefully and return the row that won, not fail a
+ sync over a counterparty we were about to create anyway."""
+ existing = await get_or_create_payee(
+ session, test_user.id, COUNTERPARTY, workspace_id=test_workspace.id
+ )
+ await session.commit()
+
+ real_scalar = AsyncSession.scalar
+ seen = {"count": 0}
+
+ async def blind_pre_checks(self, *args, **kwargs):
+ seen["count"] += 1
+ if seen["count"] <= 2:
+ return None
+ return await real_scalar(self, *args, **kwargs)
+
+ with patch.object(AsyncSession, "scalar", blind_pre_checks):
+ adopted = await get_or_create_payee(
+ session, test_user.id, COUNTERPARTY, workspace_id=test_workspace.id
+ )
+
+ assert adopted.id == existing.id
+ assert len(await _payees(session, test_workspace.id)) == 1
+
+
+# ---------------------------------------------------------------------------
+# the reported failure, end to end
+# ---------------------------------------------------------------------------
+async def test_connecting_a_bank_imports_a_shared_counterparty_once(
+ session: AsyncSession, test_user, test_workspace
+):
+ """`POST /api/connections/oauth/callback` imports the first batch inline,
+ so the connect itself is what died for providers that return accounts
+ with the session."""
+ mock_provider = AsyncMock()
+ mock_provider.handle_oauth_callback = AsyncMock(return_value=ConnectionData(
+ external_id="session-1",
+ institution_name="Some Bank",
+ credentials={"session_id": "x"},
+ accounts=_accounts(),
+ ))
+ mock_provider.get_transactions = AsyncMock(side_effect=_transactions_for)
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock):
+ connection = await handle_oauth_callback(
+ session, test_workspace.id, test_user.id, "code",
+ "enablebanking", sync_assets=False,
+ )
+
+ assert connection.status == "active"
+ payees = await _payees(session, test_workspace.id)
+ assert [p.name for p in payees] == [COUNTERPARTY]
+
+
+async def test_sync_imports_a_shared_counterparty_once(
+ session: AsyncSession, test_user, test_workspace
+):
+ """And the sync the report retried by hand, which failed the same way and
+ rolled back every account with it."""
+ connection = BankConnection(
+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,
+ provider="enablebanking", external_id=f"ext-{uuid.uuid4().hex[:8]}",
+ institution_name="Some Bank", credentials={"session_id": "x"},
+ status="active", last_sync_at=None, created_at=datetime.now(timezone.utc),
+ )
+ session.add(connection)
+ await session.commit()
+
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"session_id": "x"})
+ mock_provider.get_accounts = AsyncMock(return_value=_accounts())
+ mock_provider.get_transactions = AsyncMock(side_effect=_transactions_for)
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock):
+ synced, _ = await sync_connection(
+ session, connection.id, test_workspace.id, test_user.id
+ )
+
+ assert synced.status == "active"
+ payees = await _payees(session, test_workspace.id)
+ assert [p.name for p in payees] == [COUNTERPARTY]
+
+ # The accounts and their transactions landed too: the duplicate key
+ # aborted the whole transaction, which is why the report saw a bank
+ # group with nothing under it.
+ from app.models.account import Account
+ from app.models.transaction import Transaction
+
+ accounts = (await session.execute(
+ select(Account).where(Account.connection_id == connection.id)
+ )).scalars().all()
+ assert len(accounts) == 2
+ synced_tx = (await session.execute(
+ select(Transaction).where(Transaction.source == "sync")
+ )).scalars().all()
+ assert len(synced_tx) == 4