Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix: dedupe synced transactions across re-emitted external ids (#101) (#183)

Tássio Noronha93f0e0fdff07
diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.py
index 09f43099..3eb54987 100644
--- a/backend/app/services/connection_service.py
+++ b/backend/app/services/connection_service.py
@@ -439,6 +439,28 @@ async def handle_oauth_callback(
connection_data.credentials, acc_data.external_id, None
)
for txn_data in transactions_data:
+ # Pending↔posted twin (and the credit-card installment variant).
+ # When the same logical operation comes back under a new external
+ # id with a different status, fingerprint match prevents the
+ # second copy from landing.
+ synced_dup = await _find_synced_duplicate(session, account.id, txn_data)
+ if synced_dup:
+ if synced_dup.status == "pending" and txn_data.status == "posted":
+ synced_dup.status = "posted"
+ synced_dup.external_id = txn_data.external_id
+ synced_dup.raw_data = txn_data.raw_data
+ if (
+ txn_data.bill_external_id
+ and synced_dup.effective_bill_date is None
+ ):
+ bill = bills_by_external_id.get(txn_data.bill_external_id)
+ if bill is not None and synced_dup.bill_id != bill.id:
+ synced_dup.bill_id = bill.id
+ apply_effective_date(
+ synced_dup, account, bill_due_date=bill.due_date
+ )
+ continue
+
category_id = await _match_pluggy_category(
session, user_id, txn_data.pluggy_category, enabled=use_provider_cats
)
@@ -565,6 +587,81 @@ async def _fuzzy_match_manual(
return None
+async def _find_synced_duplicate(
+ session: AsyncSession,
+ account_id: uuid.UUID,
+ txn_data,
+) -> Optional[Transaction]:
+ """Find an existing synced row that the incoming `txn_data` is a twin of.
+
+ The `(account_id, external_id)` lookup only catches the case where a
+ provider keeps the same id while a row's `status` flips pending→posted.
+ It misses two patterns where the same logical operation comes back with
+ two different external ids:
+
+ 1. The provider re-emits the operation with a new id when its state
+ changes — e.g. a scheduled/pending row replaced by a posted row.
+ Same account/date/amount/type with statuses differing.
+ 2. A credit-card installment that lands on the current bill but is also
+ still scheduled against the next bill. Two different external ids
+ and two different bills, but the same installment fingerprint
+ `(purchase_date, number, total, amount, type)`.
+
+ Returns the existing Transaction the caller should reuse; the caller
+ decides whether to upgrade its status (pending→posted + swap external_id)
+ or skip the incoming insert. Synthetic bill-charge rows
+ (`bill_charge:*`) are excluded — they have their own idempotency keys.
+ """
+ # Path 1: installment fingerprint. Highly specific, so we don't require a
+ # description match on top.
+ if (
+ txn_data.installment_purchase_date is not None
+ and txn_data.installment_number is not None
+ and txn_data.total_installments is not None
+ ):
+ result = await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.source == "sync",
+ Transaction.installment_purchase_date == txn_data.installment_purchase_date,
+ Transaction.installment_number == txn_data.installment_number,
+ Transaction.total_installments == txn_data.total_installments,
+ Transaction.amount == txn_data.amount,
+ Transaction.type == txn_data.type,
+ Transaction.external_id != txn_data.external_id,
+ )
+ )
+ for candidate in result.scalars():
+ if candidate.external_id and candidate.external_id.startswith("bill_charge:"):
+ continue
+ return candidate
+
+ # Path 2: pending↔posted twin on the same account/date/amount/type. The
+ # status differential is the load-bearing signal — without it we'd risk
+ # collapsing two genuinely separate transactions that happen to share a
+ # day and amount. A light description-similarity check guards against
+ # the residual false positive of two different merchants charging the
+ # same amount the same day where one is pending and one is posted.
+ result = await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.source == "sync",
+ Transaction.date == txn_data.date,
+ Transaction.amount == txn_data.amount,
+ Transaction.type == txn_data.type,
+ Transaction.status != txn_data.status,
+ Transaction.external_id != txn_data.external_id,
+ )
+ )
+ for candidate in result.scalars():
+ if candidate.external_id and candidate.external_id.startswith("bill_charge:"):
+ continue
+ if _description_similarity(candidate.description, txn_data.description) >= 0.7:
+ return candidate
+
+ return None
+
+
async def _cleanup_phantom_duplicates(
session: AsyncSession,
connection_id: uuid.UUID,
@@ -1008,6 +1105,33 @@ async def sync_connection(
merged_count += 1
continue
+ # Pass 3: pending↔posted twin (and the credit-card
+ # installment variant). When the same logical operation
+ # comes back under a new external id with a different
+ # status, fingerprint match collapses it instead of letting
+ # both rows land.
+ synced_dup = await _find_synced_duplicate(
+ session, account.id, txn_data
+ )
+ if synced_dup:
+ if synced_dup.status == "pending" and txn_data.status == "posted":
+ # Posted truth wins: swap in the new id so subsequent
+ # syncs match by external_id and update raw_data.
+ synced_dup.status = "posted"
+ synced_dup.external_id = txn_data.external_id
+ synced_dup.raw_data = txn_data.raw_data
+ if (
+ txn_data.bill_external_id
+ and synced_dup.effective_bill_date is None
+ ):
+ bill = bills_by_external_id.get(txn_data.bill_external_id)
+ if bill is not None and synced_dup.bill_id != bill.id:
+ synced_dup.bill_id = bill.id
+ apply_effective_date(
+ synced_dup, account, bill_due_date=bill.due_date
+ )
+ continue
+
category_id = await _match_pluggy_category(
session, user_id, txn_data.pluggy_category, enabled=use_provider_cats
)
diff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.py
index 48d5d512..a8a555c4 100644
--- a/backend/tests/test_connection_service.py
+++ b/backend/tests/test_connection_service.py
@@ -1521,3 +1521,399 @@ async def test_sync_updates_existing_bill_idempotently(
assert len(rows) == 1, "second sync must not duplicate the row"
assert rows[0].due_date == date(2026, 4, 16)
assert rows[0].total_amount == Decimal("125.50")
+
+
+# ---------------------------------------------------------------------------
+# Synced-transaction duplicate prevention
+# ---------------------------------------------------------------------------
+
+
+async def test_sync_dedupes_pending_posted_twin_in_same_fetch(
+ session: AsyncSession, test_user,
+):
+ """A provider emits the same operation twice in a single fetch — once
+ pending (the scheduled row) and once posted (the executed row) — under
+ two different external_ids. Only the posted row must land."""
+ conn = await _make_connection(session, test_user.id, "Twin Bank")
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="twin-acc-1", name="Conta Corrente",
+ type="checking", balance=Decimal("1000"), currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="provider-id-pending",
+ description="INVESTIMENTO/OPERACAOB3* - DOCTO: 8162",
+ amount=Decimal("943.23"), date=date(2026, 4, 20),
+ type="debit", currency="BRL", status="pending",
+ ),
+ TransactionData(
+ external_id="provider-id-posted",
+ description="INVESTIMENTO/OPERACAOB3* - DOCTO: 1270397",
+ amount=Decimal("943.23"), date=date(2026, 4, 20),
+ type="debit", currency="BRL", status="posted",
+ ),
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.user_id == test_user.id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ assert len(rows) == 1, "pending+posted twin must collapse to a single row"
+ assert rows[0].status == "posted"
+ assert rows[0].external_id == "provider-id-posted"
+
+
+async def test_sync_dedupes_pending_posted_twin_with_identical_descriptions(
+ session: AsyncSession, test_user,
+):
+ """Same case as above but the descriptions are byte-identical — the
+ status differential alone is enough to collapse them."""
+ conn = await _make_connection(session, test_user.id, "Identical Desc Bank")
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="id-acc-1", name="Conta",
+ type="checking", balance=Decimal("0"), currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="provider-pending",
+ description="PIX AGENDADO BENEFICIARIO XYZ",
+ amount=Decimal("250.00"), date=date(2026, 4, 22),
+ type="debit", currency="BRL", status="pending",
+ ),
+ TransactionData(
+ external_id="provider-posted",
+ description="PIX AGENDADO BENEFICIARIO XYZ",
+ amount=Decimal("250.00"), date=date(2026, 4, 22),
+ type="debit", currency="BRL", status="posted",
+ ),
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.user_id == test_user.id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ assert len(rows) == 1
+ assert rows[0].status == "posted"
+
+
+async def test_sync_keeps_unrelated_pending_and_posted_with_different_descriptions(
+ session: AsyncSession, test_user,
+):
+ """Two unrelated transactions that happen to share a date and amount —
+ one pending, one posted, completely different merchants — must NOT be
+ collapsed. The description-similarity guard protects against this
+ false positive."""
+ conn = await _make_connection(session, test_user.id, "Unrelated Bank")
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="unr-acc-1", name="Conta",
+ type="checking", balance=Decimal("0"), currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="unrelated-pending",
+ description="STARBUCKS COFFEE",
+ amount=Decimal("25.00"), date=date(2026, 4, 22),
+ type="debit", currency="BRL", status="pending",
+ ),
+ TransactionData(
+ external_id="unrelated-posted",
+ description="UBER TRIP",
+ amount=Decimal("25.00"), date=date(2026, 4, 22),
+ type="debit", currency="BRL", status="posted",
+ ),
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.user_id == test_user.id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ assert len(rows) == 2, "unrelated transactions must not be collapsed"
+
+
+async def test_sync_upgrades_pending_to_posted_when_twin_arrives(
+ session: AsyncSession, test_user,
+):
+ """When the pending row was synced first and the posted twin arrives on
+ the next sync with a different external_id, the existing row must be
+ upgraded in place — not duplicated."""
+ conn = await _make_connection(session, test_user.id, "Twin Upgrade Bank")
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="up-acc-1", name="Conta",
+ type="checking", balance=Decimal("0"), currency="BRL",
+ ),
+ ])
+ pending = TransactionData(
+ external_id="provider-pending",
+ description="PIX AGENDADO - DOCTO: 11111",
+ amount=Decimal("100.00"), date=date(2026, 4, 20),
+ type="debit", currency="BRL", status="pending",
+ )
+ mock_provider.get_transactions = AsyncMock(return_value=[pending])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_user.id)
+
+ # Second sync: posted twin arrives with a new id and identifier; the
+ # pending row is also still in the feed (providers don't always drop the
+ # scheduled row immediately).
+ posted = TransactionData(
+ external_id="provider-posted",
+ description="PIX AGENDADO - DOCTO: 22222",
+ amount=Decimal("100.00"), date=date(2026, 4, 20),
+ type="debit", currency="BRL", status="posted",
+ )
+ mock_provider.get_transactions = AsyncMock(return_value=[pending, posted])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.user_id == test_user.id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ assert len(rows) == 1, "pending+posted twins must collapse to one row"
+ # Posted truth wins: status flipped and external_id swapped to the new one
+ # so subsequent syncs match by id.
+ assert rows[0].status == "posted"
+ assert rows[0].external_id == "provider-posted"
+
+
+async def test_sync_dedupes_advanced_installment_payment(
+ session: AsyncSession, test_user,
+):
+ """A credit-card installment paid in advance shows up as posted on the
+ current bill *and* pending on the next bill. Two different external
+ ids, two different bill ids, but same installment fingerprint
+ (purchase_date / number / total). Only the posted row must land,
+ linked to the current bill."""
+ from app.models.credit_card_bill import CreditCardBill
+
+ conn = await _make_connection(session, test_user.id, "Inst Bank")
+
+ bill_current = BillData(
+ external_id="bill-current",
+ due_date=date(2026, 5, 10),
+ total_amount=Decimal("241.50"),
+ currency="BRL",
+ )
+ bill_next = BillData(
+ external_id="bill-next",
+ due_date=date(2026, 6, 10),
+ total_amount=Decimal("241.50"),
+ currency="BRL",
+ )
+
+ posted_current = TransactionData(
+ external_id="provider-inst-posted",
+ description="HTM*INAA CONSULTOR 06/12",
+ amount=Decimal("241.50"), date=date(2026, 4, 28),
+ type="debit", currency="BRL", status="posted",
+ installment_number=6, total_installments=12,
+ installment_total_amount=Decimal("2898.00"),
+ installment_purchase_date=date(2025, 11, 28),
+ bill_external_id="bill-current",
+ )
+ pending_next = TransactionData(
+ external_id="provider-inst-pending",
+ description="HTM*INAA CONSULTOR 06/12",
+ amount=Decimal("241.50"), date=date(2026, 5, 9),
+ type="debit", currency="BRL", status="pending",
+ installment_number=6, total_installments=12,
+ installment_total_amount=Decimal("2898.00"),
+ installment_purchase_date=date(2025, 11, 28),
+ bill_external_id="bill-next",
+ )
+
+ mock_provider = _cc_provider_mock(
+ bills=[bill_current, bill_next],
+ transactions=[posted_current, pending_next],
+ )
+
+ p1, p2, p3 = _patch_sync_helpers()
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ p1, p2, p3:
+ await sync_connection(session, conn.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.user_id == test_user.id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ assert len(rows) == 1, (
+ "advanced installment must not double-count: POSTED on current bill "
+ "and PENDING on next bill are the same logical charge"
+ )
+ survivor = rows[0]
+ assert survivor.status == "posted"
+ assert survivor.installment_number == 6
+ assert survivor.total_installments == 12
+
+ bill_current_row = (await session.execute(
+ select(CreditCardBill).where(CreditCardBill.external_id == "bill-current")
+ )).scalar_one()
+ assert survivor.bill_id == bill_current_row.id, (
+ "survivor must stay linked to the bill that actually paid the installment"
+ )
+
+
+async def test_sync_dedupes_advanced_installment_when_pending_lands_first(
+ session: AsyncSession, test_user,
+):
+ """Same as the previous test but the pending next-bill row arrives
+ before the posted current-bill row in the fetch list. Order must not
+ matter — posted still wins."""
+ conn = await _make_connection(session, test_user.id, "Inst Order Bank")
+
+ bill_current = BillData(
+ external_id="bill-current-2",
+ due_date=date(2026, 5, 10),
+ total_amount=Decimal("100"), currency="BRL",
+ )
+ bill_next = BillData(
+ external_id="bill-next-2",
+ due_date=date(2026, 6, 10),
+ total_amount=Decimal("100"), currency="BRL",
+ )
+
+ pending = TransactionData(
+ external_id="provider-pend-first",
+ description="LIVRARIA SARAIVA 03/06",
+ amount=Decimal("50.00"), date=date(2026, 5, 5),
+ type="debit", currency="BRL", status="pending",
+ installment_number=3, total_installments=6,
+ installment_total_amount=Decimal("300.00"),
+ installment_purchase_date=date(2026, 3, 5),
+ bill_external_id="bill-next-2",
+ )
+ posted = TransactionData(
+ external_id="provider-post-second",
+ description="LIVRARIA SARAIVA 03/06",
+ amount=Decimal("50.00"), date=date(2026, 4, 28),
+ type="debit", currency="BRL", status="posted",
+ installment_number=3, total_installments=6,
+ installment_total_amount=Decimal("300.00"),
+ installment_purchase_date=date(2026, 3, 5),
+ bill_external_id="bill-current-2",
+ )
+
+ mock_provider = _cc_provider_mock(
+ bills=[bill_current, bill_next],
+ transactions=[pending, posted], # pending first
+ )
+
+ p1, p2, p3 = _patch_sync_helpers()
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ p1, p2, p3:
+ await sync_connection(session, conn.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.user_id == test_user.id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ assert len(rows) == 1
+ assert rows[0].status == "posted"
+ assert rows[0].external_id == "provider-post-second"
+
+
+async def test_sync_keeps_genuine_same_day_repeats(
+ session: AsyncSession, test_user,
+):
+ """Two genuine same-day same-amount transactions with byte-identical
+ descriptions and identical statuses must NOT be collapsed — those are
+ real repeats (e.g. two identical fares charged on the same day), not
+ provider-side duplicates. Guards against false positives in the new
+ dedup."""
+ conn = await _make_connection(session, test_user.id, "Repeat Bank")
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "t"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="rep-acc-1", name="Conta",
+ type="checking", balance=Decimal("0"), currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="uber-1",
+ description="UBER TRIP",
+ amount=Decimal("25.00"), date=date(2026, 4, 20),
+ type="debit", currency="BRL", status="posted",
+ ),
+ TransactionData(
+ external_id="uber-2",
+ description="UBER TRIP",
+ amount=Decimal("25.00"), date=date(2026, 4, 20),
+ type="debit", currency="BRL", status="posted",
+ ),
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.user_id == test_user.id,
+ Transaction.source == "sync",
+ )
+ )).scalars().all()
+ assert len(rows) == 2, "identical-description same-day repeats must be kept"