Open-source personal finance manager. Self-hosted, privacy-first.
fix(sync): keep a same-day repeat the provider still returns out of phantom cleanup (#1122)
Michael409e136ff6ac
diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.pyindex cd1c065b..e7d49d65 100644--- a/backend/app/services/connection_service.py+++ b/backend/app/services/connection_service.py@@ -1581,6 +1581,7 @@ async def _find_synced_duplicate(async def _cleanup_phantom_duplicates(session: AsyncSession,connection_id: uuid.UUID,+ live_external_ids: Optional[dict[uuid.UUID, set[str]]] = None,) -> set[uuid.UUID]:"""Delete synced transactions that are phantom duplicates.@@ -1595,6 +1596,15 @@ async def _cleanup_phantom_duplicates(us distinguish the duplicate from a legitimate same-day repeat (e.g. tworeal Uber rides for the same fare).+ ``live_external_ids`` maps an account id to the transaction ids the+ provider returned for it in this sync. For an account in that map, a+ sibling with a different id marks the tx as a phantom only when exactly+ one of the two ids is still returned, which is what a re-keyed twin looks+ like. Both ids returned means two real payments (e.g. two same-day Pix of+ the same amount, #1038). Neither returned means the rows have left the+ provider's fetch window, so this sync has no evidence about them, and they+ were already checked while they were in it.+Returns the ids of the accounts that lost a row, so the caller canreconcile their opening balances against what is left."""@@ -1616,6 +1626,7 @@ async def _cleanup_phantom_duplicates(touched: set[uuid.UUID] = set()for tx in unmatched:+ live = (live_external_ids or {}).get(tx.account_id)date_lo = tx.date - timedelta(days=1)date_hi = tx.date + timedelta(days=1)sibling_result = await session.execute(@@ -1631,6 +1642,12 @@ async def _cleanup_phantom_duplicates())for sibling in sibling_result.scalars():+ if (+ live is not None+ and tx.external_id != sibling.external_id+ and (tx.external_id in live) == (sibling.external_id in live)+ ):+ continueif token_overlap(sibling.original_description or sibling.description,tx.original_description or tx.description,@@ -1968,6 +1985,7 @@ async def sync_connection(user_currency = user.primary_currency if user else get_settings().default_currencynew_tx_ids: list[uuid.UUID] = []merged_count = 0+ live_txn_external_ids: dict[uuid.UUID, set[str]] = {}accounts_data = await provider.get_accounts(credentials)incoming_external_ids = {acc.external_id for acc in accounts_data}institution_cache: dict[str, Institution] = {}@@ -2109,6 +2127,11 @@ async def sync_connection(credentials, acc_data.external_id, since, payee_source=payee_source)+ # Every id the provider returned, pending ones included, so the+ # phantom cleanup still sees a pending row the filter below skips.+ live_txn_external_ids[account.id] = {+ txn.external_id for txn in transactions_data+ }if not import_pending:transactions_data = [t for t in transactions_data if t.status != "pending"]@@ -2384,7 +2407,9 @@ async def sync_connection(# Clean up phantom duplicates: providers occasionally double-report the# same payment with different ids. Once transfer detection has paired# the real one, the orphan twin gets removed here.- touched_account_ids = await _cleanup_phantom_duplicates(session, connection.id)+ touched_account_ids = await _cleanup_phantom_duplicates(+ session, connection.id, live_txn_external_ids+ )# The opening balances above were reconciled with the phantoms still# counted. Reconcile the open accounts that lost one again, so thediff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.pyindex f93b323a..cd02a233 100644--- a/backend/tests/test_connection_service.py+++ b/backend/tests/test_connection_service.py@@ -1531,6 +1531,152 @@ async def test_phantom_cleanup_leaves_a_closed_accounts_opening_balance_alone(assert [row.external_id for row in rows] == ["paired"]+async def test_phantom_cleanup_keeps_a_same_day_repeat_the_provider_still_returns(+ session: AsyncSession, test_user, test_workspace+):+ """Two real same-day transfers of the same amount, only one of them paired.+ The provider still returns both ids, so the unpaired one is not a phantom+ and the cleanup must keep it (#1038). A later sync whose fetch window no+ longer reaches them must keep it too.+ """+ conn = await _make_connection(session, test_user.id, "Repeat Bank")++ account = Account(+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,+ connection_id=conn.id, external_id="repeat-acc-1", name="Checking",+ type="checking", balance=Decimal("50000"), currency="BRL",+ )+ session.add(account)+ await session.flush()+ account_id = account.id++ payment_day = date.today()+ for external_id, pair in (+ ("pix-to-nubank", uuid.uuid4()),+ ("pix-to-mercado-pago", None),+ ):+ session.add(Transaction(+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,+ account_id=account_id, external_id=external_id,+ description="Pix enviado Own Name", amount=Decimal("10000.00"),+ date=payment_day, type="debit", status="posted", source="sync",+ currency="BRL", transfer_pair_id=pair,+ created_at=datetime.now(timezone.utc),+ ))+ await session.commit()++ mock_provider = AsyncMock()+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})+ mock_provider.get_accounts = AsyncMock(return_value=[+ AccountData(+ external_id="repeat-acc-1", name="Checking", type="checking",+ balance=Decimal("50000"), currency="BRL",+ ),+ ])+ mock_provider.get_transactions = AsyncMock(return_value=[+ TransactionData(+ external_id=external_id, description="Pix enviado Own Name",+ amount=Decimal("10000.00"), date=payment_day, type="debit",+ currency="BRL", status="posted",+ )+ for external_id in ("pix-to-nubank", "pix-to-mercado-pago")+ ])++ async def synced_ids() -> set[str | None]:+ rows = (await session.execute(+ select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.source == "sync",+ )+ )).scalars().all()+ return {row.external_id for row in rows}++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)+ assert await synced_ids() == {"pix-to-nubank", "pix-to-mercado-pago"}++ # Two weeks on, the incremental fetch no longer returns either id.+ mock_provider.get_transactions = AsyncMock(return_value=[])+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)+ assert await synced_ids() == {"pix-to-nubank", "pix-to-mercado-pago"}+++async def test_phantom_cleanup_counts_a_returned_pending_id_with_import_pending_off(+ session: AsyncSession, test_user, test_workspace+):+ """With import_pending off the sync skips pending rows, but the provider+ still returned them. A pending row stored before the setting changed, whose+ id the provider still returns next to a paired same-day sibling, is a real+ repeat and the cleanup must keep it.+ """+ conn = await _make_connection(+ session, test_user.id, "Pending Bank", settings={"import_pending": False}+ )++ account = Account(+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,+ connection_id=conn.id, external_id="pending-acc-1", name="Checking",+ type="checking", balance=Decimal("50000"), currency="BRL",+ )+ session.add(account)+ await session.flush()+ account_id = account.id++ payment_day = date.today()+ for external_id, status, pair in (+ ("pix-to-nubank", "posted", uuid.uuid4()),+ ("pix-to-mercado-pago", "pending", None),+ ):+ session.add(Transaction(+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,+ account_id=account_id, external_id=external_id,+ description="Pix enviado Own Name", amount=Decimal("10000.00"),+ date=payment_day, type="debit", status=status, source="sync",+ currency="BRL", transfer_pair_id=pair,+ created_at=datetime.now(timezone.utc),+ ))+ await session.commit()++ mock_provider = AsyncMock()+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})+ mock_provider.get_accounts = AsyncMock(return_value=[+ AccountData(+ external_id="pending-acc-1", name="Checking", type="checking",+ balance=Decimal("50000"), currency="BRL",+ ),+ ])+ mock_provider.get_transactions = AsyncMock(return_value=[+ TransactionData(+ external_id=external_id, description="Pix enviado Own Name",+ amount=Decimal("10000.00"), date=payment_day, type="debit",+ currency="BRL", status=status,+ )+ for external_id, status in (+ ("pix-to-nubank", "posted"),+ ("pix-to-mercado-pago", "pending"),+ )+ ])++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)++ rows = (await session.execute(+ select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.source == "sync",+ )+ )).scalars().all()+ assert {row.external_id for row in rows} == {"pix-to-nubank", "pix-to-mercado-pago"}++@pytest.mark.asyncioasync def test_sync_connection_tolerates_duplicate_transaction_rows(session: AsyncSession, test_user, test_workspace