Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix(sync): keep a same-day repeat the provider still returns out of phantom cleanup (#1122)

Michael409e136ff6ac
diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.py
index cd1c065b..e7d49d65 100644
--- a/backend/app/services/connection_service.py
+++ b/backend/app/services/connection_service.py
@@ -1581,6 +1581,7 @@ async def _find_synced_duplicate(
async def _cleanup_phantom_duplicates(
session: AsyncSession,
connection_id: uuid.UUID,
+ live_external_ids: Optional[dict[uuid.UUID, set[str]]] = None,
) -> set[uuid.UUID]:
"""Delete synced transactions that are phantom duplicates.
@@ -1595,6 +1596,15 @@ async def _cleanup_phantom_duplicates(
us distinguish the duplicate from a legitimate same-day repeat (e.g. two
real Uber rides for the same fare).
+ ``live_external_ids`` maps an account id to the transaction ids the
+ provider returned for it in this sync. For an account in that map, a
+ sibling with a different id marks the tx as a phantom only when exactly
+ one of the two ids is still returned, which is what a re-keyed twin looks
+ like. Both ids returned means two real payments (e.g. two same-day Pix of
+ the same amount, #1038). Neither returned means the rows have left the
+ provider's fetch window, so this sync has no evidence about them, and they
+ were already checked while they were in it.
+
Returns the ids of the accounts that lost a row, so the caller can
reconcile their opening balances against what is left.
"""
@@ -1616,6 +1626,7 @@ async def _cleanup_phantom_duplicates(
touched: set[uuid.UUID] = set()
for tx in unmatched:
+ live = (live_external_ids or {}).get(tx.account_id)
date_lo = tx.date - timedelta(days=1)
date_hi = tx.date + timedelta(days=1)
sibling_result = await session.execute(
@@ -1631,6 +1642,12 @@ async def _cleanup_phantom_duplicates(
)
)
for sibling in sibling_result.scalars():
+ if (
+ live is not None
+ and tx.external_id != sibling.external_id
+ and (tx.external_id in live) == (sibling.external_id in live)
+ ):
+ continue
if token_overlap(
sibling.original_description or sibling.description,
tx.original_description or tx.description,
@@ -1968,6 +1985,7 @@ async def sync_connection(
user_currency = user.primary_currency if user else get_settings().default_currency
new_tx_ids: list[uuid.UUID] = []
merged_count = 0
+ live_txn_external_ids: dict[uuid.UUID, set[str]] = {}
accounts_data = await provider.get_accounts(credentials)
incoming_external_ids = {acc.external_id for acc in accounts_data}
institution_cache: dict[str, Institution] = {}
@@ -2109,6 +2127,11 @@ async def sync_connection(
credentials, acc_data.external_id, since, payee_source=payee_source
)
+ # Every id the provider returned, pending ones included, so the
+ # phantom cleanup still sees a pending row the filter below skips.
+ live_txn_external_ids[account.id] = {
+ txn.external_id for txn in transactions_data
+ }
if not import_pending:
transactions_data = [t for t in transactions_data if t.status != "pending"]
@@ -2384,7 +2407,9 @@ async def sync_connection(
# Clean up phantom duplicates: providers occasionally double-report the
# same payment with different ids. Once transfer detection has paired
# the real one, the orphan twin gets removed here.
- touched_account_ids = await _cleanup_phantom_duplicates(session, connection.id)
+ touched_account_ids = await _cleanup_phantom_duplicates(
+ session, connection.id, live_txn_external_ids
+ )
# The opening balances above were reconciled with the phantoms still
# counted. Reconcile the open accounts that lost one again, so the
diff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.py
index f93b323a..cd02a233 100644
--- a/backend/tests/test_connection_service.py
+++ b/backend/tests/test_connection_service.py
@@ -1531,6 +1531,152 @@ async def test_phantom_cleanup_leaves_a_closed_accounts_opening_balance_alone(
assert [row.external_id for row in rows] == ["paired"]
+async def test_phantom_cleanup_keeps_a_same_day_repeat_the_provider_still_returns(
+ session: AsyncSession, test_user, test_workspace
+):
+ """Two real same-day transfers of the same amount, only one of them paired.
+ The provider still returns both ids, so the unpaired one is not a phantom
+ and the cleanup must keep it (#1038). A later sync whose fetch window no
+ longer reaches them must keep it too.
+ """
+ conn = await _make_connection(session, test_user.id, "Repeat Bank")
+
+ account = Account(
+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,
+ connection_id=conn.id, external_id="repeat-acc-1", name="Checking",
+ type="checking", balance=Decimal("50000"), currency="BRL",
+ )
+ session.add(account)
+ await session.flush()
+ account_id = account.id
+
+ payment_day = date.today()
+ for external_id, pair in (
+ ("pix-to-nubank", uuid.uuid4()),
+ ("pix-to-mercado-pago", None),
+ ):
+ session.add(Transaction(
+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,
+ account_id=account_id, external_id=external_id,
+ description="Pix enviado Own Name", amount=Decimal("10000.00"),
+ date=payment_day, type="debit", status="posted", source="sync",
+ currency="BRL", transfer_pair_id=pair,
+ created_at=datetime.now(timezone.utc),
+ ))
+ await session.commit()
+
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="repeat-acc-1", name="Checking", type="checking",
+ balance=Decimal("50000"), currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id=external_id, description="Pix enviado Own Name",
+ amount=Decimal("10000.00"), date=payment_day, type="debit",
+ currency="BRL", status="posted",
+ )
+ for external_id in ("pix-to-nubank", "pix-to-mercado-pago")
+ ])
+
+ async def synced_ids() -> set[str | None]:
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ return {row.external_id for row in rows}
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)
+ assert await synced_ids() == {"pix-to-nubank", "pix-to-mercado-pago"}
+
+ # Two weeks on, the incremental fetch no longer returns either id.
+ mock_provider.get_transactions = AsyncMock(return_value=[])
+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)
+ assert await synced_ids() == {"pix-to-nubank", "pix-to-mercado-pago"}
+
+
+async def test_phantom_cleanup_counts_a_returned_pending_id_with_import_pending_off(
+ session: AsyncSession, test_user, test_workspace
+):
+ """With import_pending off the sync skips pending rows, but the provider
+ still returned them. A pending row stored before the setting changed, whose
+ id the provider still returns next to a paired same-day sibling, is a real
+ repeat and the cleanup must keep it.
+ """
+ conn = await _make_connection(
+ session, test_user.id, "Pending Bank", settings={"import_pending": False}
+ )
+
+ account = Account(
+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,
+ connection_id=conn.id, external_id="pending-acc-1", name="Checking",
+ type="checking", balance=Decimal("50000"), currency="BRL",
+ )
+ session.add(account)
+ await session.flush()
+ account_id = account.id
+
+ payment_day = date.today()
+ for external_id, status, pair in (
+ ("pix-to-nubank", "posted", uuid.uuid4()),
+ ("pix-to-mercado-pago", "pending", None),
+ ):
+ session.add(Transaction(
+ id=uuid.uuid4(), user_id=test_user.id, workspace_id=test_workspace.id,
+ account_id=account_id, external_id=external_id,
+ description="Pix enviado Own Name", amount=Decimal("10000.00"),
+ date=payment_day, type="debit", status=status, source="sync",
+ currency="BRL", transfer_pair_id=pair,
+ created_at=datetime.now(timezone.utc),
+ ))
+ await session.commit()
+
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="pending-acc-1", name="Checking", type="checking",
+ balance=Decimal("50000"), currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id=external_id, description="Pix enviado Own Name",
+ amount=Decimal("10000.00"), date=payment_day, type="debit",
+ currency="BRL", status=status,
+ )
+ for external_id, status in (
+ ("pix-to-nubank", "posted"),
+ ("pix-to-mercado-pago", "pending"),
+ )
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ assert {row.external_id for row in rows} == {"pix-to-nubank", "pix-to-mercado-pago"}
+
+
@pytest.mark.asyncio
async def test_sync_connection_tolerates_duplicate_transaction_rows(
session: AsyncSession, test_user, test_workspace