Open-source personal finance manager. Self-hosted, privacy-first.
fix: harden connection sync correctness (#374)
Albert Dolkunf4550a77651d
diff --git a/backend/app/providers/simplefin.py b/backend/app/providers/simplefin.pyindex 0bb51efb..21e66002 100644--- a/backend/app/providers/simplefin.py+++ b/backend/app/providers/simplefin.py@@ -47,7 +47,7 @@ logger = logging.getLogger(__name__)# initial sync (since=None) we walk backwards in chunks of this size; for# follow-up syncs the sync layer's typical 30-90 day window fits in one call.SIMPLEFIN_MAX_WINDOW_DAYS = 90-SIMPLEFIN_DEFAULT_HISTORY_DAYS = 90+SIMPLEFIN_DEFAULT_HISTORY_DAYS = 45 # routine overlap, not the per-request capSIMPLEFIN_INITIAL_HISTORY_DAYS = 365 # ~1 year backfill on first connectSIMPLEFIN_HTTP_TIMEOUT = 60.0@@ -152,18 +152,21 @@ def _ticker(value: Any) -> Optional[str]:return value.strip().upper()[:32] or None-def _surface_errors(errlist: list[dict], context: str) -> None:- """Raise a typed exception for auth errors; log everything else.+def _surface_errors(+ errlist: list[dict], context: str, *, has_usable_data: bool = False+) -> list[dict]:+ """Raise for auth errors only when the response has no usable data.- SimpleFIN's spec says: *Always show those errors to your end users.* For- the codes the user can act on (re-auth), we raise so the API layer can- flip the connection status. For the rest we log and continue — the- response usually still has usable data alongside the warnings.+ SimpleFIN's spec says: *Always show those errors to your end users.* The+ Bridge can return top-level ``con.auth`` warnings for one institution while+ still returning fresh account data for other institutions under the same+ Access URL. In that partial-success case, log the warning and keep the sync+ moving so one stale sub-connection does not block every healthy account."""if not errlist:- return+ return []reauth = [e for e in errlist if (e.get("code") or "").lower() in _REAUTH_ERROR_CODES]- if reauth:+ if reauth and not has_usable_data:first = reauth[0]raise ProviderUserActionRequired(first.get("msg") or first.get("message") or "Reauthorize at SimpleFIN Bridge",@@ -177,11 +180,15 @@ def _surface_errors(errlist: list[dict], context: str) -> None:entry.get("code"),entry.get("msg") or entry.get("message"),)+ return reauthclass SimpleFinProvider(BankProvider):"""SimpleFIN Bridge connector."""+ def __init__(self) -> None:+ self.action_required_warnings: list[dict] = []+@propertydef name(self) -> str:return "simplefin"@@ -439,7 +446,12 @@ class SimpleFinProvider(BankProvider):async def get_accounts(self, credentials: dict) -> list[AccountData]:payload = await self._fetch_accounts(credentials, pending=False)- _surface_errors(payload.get("errlist") or [], context="get_accounts")+ warnings = _surface_errors(+ payload.get("errlist") or [],+ context="get_accounts",+ has_usable_data=bool(payload.get("accounts") or []),+ )+ self.action_required_warnings.extend(warnings)_, accounts = self._parse_accounts(payload)return accounts@@ -454,30 +466,47 @@ class SimpleFinProvider(BankProvider):if since is None:start_date = end_date - timedelta(days=SIMPLEFIN_INITIAL_HISTORY_DAYS)else:- start_date = since+ # The Bridge can expose bank transactions weeks after their posted+ # date. Always re-read one full provider window so delayed rows are+ # eventually recovered instead of aging out of the sync cursor.+ start_date = min(+ since,+ end_date - timedelta(days=SIMPLEFIN_DEFAULT_HISTORY_DAYS - 1),+ )# Walk the request window in 90-day chunks (SimpleFIN's per-call cap).transactions: list[TransactionData] = []seen_ids: set[str] = set()cursor = start_date- while cursor <= end_date:- chunk_end = min(cursor + timedelta(days=SIMPLEFIN_MAX_WINDOW_DAYS), end_date)+ end_exclusive = end_date + timedelta(days=1)+ while cursor < end_exclusive:+ chunk_end = min(+ cursor + timedelta(days=SIMPLEFIN_MAX_WINDOW_DAYS),+ end_exclusive,+ )payload = await self._fetch_accounts(credentials,start_date=cursor,- end_date=chunk_end + timedelta(days=1),+ end_date=chunk_end,account_id=account_external_id,pending=True,)- _surface_errors(payload.get("errlist") or [], context="get_transactions")- for raw_acc in payload.get("accounts") or []:- if str(raw_acc.get("id") or "") != account_external_id:- continue+ matching_accounts = [+ raw_acc for raw_acc in payload.get("accounts") or []+ if str(raw_acc.get("id") or "") == account_external_id+ ]+ warnings = _surface_errors(+ payload.get("errlist") or [],+ context="get_transactions",+ has_usable_data=bool(matching_accounts),+ )+ self.action_required_warnings.extend(warnings)+ for raw_acc in matching_accounts:for raw_txn in raw_acc.get("transactions") or []:parsed = self._build_transaction(raw_txn, payee_source)if parsed and parsed.external_id not in seen_ids:seen_ids.add(parsed.external_id)transactions.append(parsed)- cursor = chunk_end + timedelta(days=1)+ cursor = chunk_endreturn transactions@staticmethod@@ -524,7 +553,12 @@ class SimpleFinProvider(BankProvider):async def get_holdings(self, credentials: dict) -> list[HoldingData]:payload = await self._fetch_accounts(credentials, pending=False)- _surface_errors(payload.get("errlist") or [], context="get_holdings")+ warnings = _surface_errors(+ payload.get("errlist") or [],+ context="get_holdings",+ has_usable_data=bool(payload.get("accounts") or []),+ )+ self.action_required_warnings.extend(warnings)holdings: list[HoldingData] = []for raw_acc in payload.get("accounts") or []:acc_currency = raw_acc.get("currency") or "USD"diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.pyindex 9263e635..efd0c6f4 100644--- a/backend/app/services/connection_service.py+++ b/backend/app/services/connection_service.py@@ -52,9 +52,12 @@ from app.services.rule_service import apply_rules_to_transaction, preview_rules_from app.services.transfer_detection_service import detect_transfer_pairsfrom app.services.fx_rate_service import stamp_primary_amountfrom app.services.payee_service import get_or_create_payee+from app.services.transaction_match_service import find_unique_transaction_matchlogger = logging.getLogger(__name__)+LOCAL_IMPORT_SOURCES = {"import", "csv", "ofx", "qif", "camt"}+settings = get_settings()_PROVIDER_SELL_DATE_METADATA_KEY = "_securo_provider_sell_date"@@ -895,13 +898,20 @@ async def get_connections(session: AsyncSession, workspace_id: uuid.UUID) -> lisasync def get_connection(- session: AsyncSession, connection_id: uuid.UUID, workspace_id: uuid.UUID+ session: AsyncSession,+ connection_id: uuid.UUID,+ workspace_id: uuid.UUID,+ *,+ for_update: bool = False,) -> Optional[BankConnection]:- result = await session.execute(+ statement = (select(BankConnection).where(BankConnection.id == connection_id, BankConnection.workspace_id == workspace_id).options(selectinload(BankConnection.accounts)))+ if for_update:+ statement = statement.with_for_update()+ result = await session.execute(statement)return result.scalar_one_or_none()@@ -1133,6 +1143,7 @@ async def handle_oauth_callback(transactions_data = await provider.get_transactions(connection_data.credentials, acc_data.external_id, None)+ incoming_external_ids = {txn.external_id for txn in transactions_data}seen_external_ids: set[str] = set()for txn_data in transactions_data:# Providers should return each external transaction once, but an@@ -1147,14 +1158,19 @@ async def handle_oauth_callback(# When the same logical operation comes back under a new external# id with a different status, fingerprint match prevents the# second copy from landing.- synced_dup = await _find_synced_duplicate(session, account.id, txn_data)+ synced_dup = await _find_synced_duplicate(+ session, account.id, txn_data, incoming_external_ids+ )if synced_dup:if synced_dup.original_description is None:synced_dup.original_description = txn_data.descriptionif synced_dup.status == "pending" and txn_data.status == "posted":synced_dup.status = "posted"- synced_dup.external_id = txn_data.external_id- synced_dup.raw_data = txn_data.raw_data+ _merge_sync_metadata(+ synced_dup,+ txn_data,+ replace_external_id=synced_dup.source == "sync",+ )if (txn_data.bill_external_idand synced_dup.effective_bill_date is None@@ -1165,6 +1181,12 @@ async def handle_oauth_callback(apply_effective_date(synced_dup, account, bill_due_date=bill.due_date)+ elif synced_dup.external_id != txn_data.external_id:+ _merge_sync_metadata(+ synced_dup,+ txn_data,+ replace_external_id=synced_dup.source == "sync",+ )continuecategory_id = await _find_installment_category(@@ -1270,6 +1292,80 @@ async def handle_oauth_callback(return connection+def _normalized_account_name(name: str | None) -> str:+ """Normalize provider account names for fallback matching.++ Some SimpleFIN bridges re-key account ids, but the display name / masked+ suffix stays stable (e.g. "High Yield Savings Account (9402)").+ """+ return " ".join((name or "").casefold().split())+++async def _find_existing_connected_account(+ session: AsyncSession,+ connection: BankConnection,+ acc_data: AccountData,+ institution: Optional[Institution],+ incoming_external_ids: set[str],+) -> Optional[Account]:+ """Find an existing account for incoming provider account data.++ Primary identity is provider external_id. For SimpleFIN only, fall back to+ stable account name + currency because some bridges emit fresh UUID-like+ account ids on each pull; when that happens, blindly keying by id creates a+ new app account on every sync.+ """+ result = await session.execute(+ select(Account).where(+ Account.connection_id == connection.id,+ Account.external_id == acc_data.external_id,+ )+ )+ account = result.scalar_one_or_none()+ if account or connection.provider != "simplefin":+ return account++ normalized_name = _normalized_account_name(acc_data.name)+ if not normalized_name:+ return None++ candidates_result = await session.execute(+ select(Account).where(+ Account.connection_id == connection.id,+ Account.currency == acc_data.currency,+ )+ )+ candidates = [+ candidate+ for candidate in candidates_result.scalars().all()+ if candidate.external_id not in incoming_external_ids+ and _normalized_account_name(candidate.name) == normalized_name+ ]+ if institution is not None:+ matched_institution = [+ candidate for candidate in candidates+ if candidate.institution_id == institution.id+ ]+ if matched_institution:+ candidates = matched_institution+ else:+ candidates = [+ candidate for candidate in candidates+ if candidate.institution_id is None+ ]+ else:+ candidates = [+ candidate for candidate in candidates+ if candidate.institution_id is None+ ]+ if len(candidates) != 1:+ return None++ account = candidates[0]+ account.external_id = acc_data.external_id+ return account++async def _fuzzy_match_manual(session: AsyncSession,account_id: uuid.UUID,@@ -1310,12 +1406,41 @@ async def _fuzzy_match_manual(return None+def _merge_sync_metadata(+ transaction: Transaction,+ txn_data,+ *,+ replace_external_id: bool = False,+) -> None:+ if transaction.source == "sync" and transaction.date.year <= 1970:+ transaction.date = txn_data.date+ if transaction.effective_date.year <= 1970:+ transaction.effective_date = txn_data.date+ if transaction.source in LOCAL_IMPORT_SOURCES:+ transaction.import_id = None+ replace_external_id = True+ if txn_data.external_id and (replace_external_id or not transaction.external_id):+ transaction.external_id = txn_data.external_id+ if txn_data.raw_data:+ if not transaction.raw_data:+ transaction.raw_data = txn_data.raw_data+ elif isinstance(transaction.raw_data, dict) and isinstance(txn_data.raw_data, dict):+ merged = {**transaction.raw_data, **txn_data.raw_data}+ if merged != transaction.raw_data:+ transaction.raw_data = merged+ if transaction.original_description is None:+ transaction.original_description = txn_data.description+ if txn_data.payee and not transaction.payee:+ transaction.payee = txn_data.payee++async def _find_synced_duplicate(session: AsyncSession,account_id: uuid.UUID,txn_data,+ incoming_external_ids: set[str],) -> Optional[Transaction]:- """Find an existing synced row that the incoming `txn_data` is a twin of.+ """Find an existing row that the incoming `txn_data` is a twin of.The `(account_id, external_id)` lookup only catches the case where aprovider keeps the same id while a row's `status` flips pending→posted.@@ -1331,8 +1456,8 @@ async def _find_synced_duplicate(`(purchase_date, number, total, amount, type)`.Returns the existing Transaction the caller should reuse; the caller- decides whether to upgrade its status (pending→posted + swap external_id)- or skip the incoming insert. Synthetic bill-charge rows+ decides whether to upgrade its status (pending→posted + swap external_id),+ enrich an imported row, or skip the incoming insert. Synthetic bill-charge rows(`bill_charge:*`) are excluded — they have their own idempotency keys."""# Path 1: installment fingerprint. Highly specific, so we don't require a@@ -1385,7 +1510,68 @@ async def _find_synced_duplicate() >= 0.7:return candidate- return None+ # Path 3: exact posted/transacted timestamp fingerprint. Some SimpleFIN+ # bridges re-key already-posted rows on later pulls, so status does not+ # differ. The raw bank timestamps plus same account/date/amount/type and a+ # near-identical description are specific enough to collapse the re-keyed+ # row while avoiding broad same-day/same-amount merchant dedupe.+ raw = txn_data.raw_data if isinstance(txn_data.raw_data, dict) else {}+ posted = raw.get("posted")+ transacted_at = raw.get("transacted_at")+ if posted is not None or transacted_at is not None:+ result = await session.execute(+ select(Transaction).where(+ Transaction.account_id == account_id,+ or_(+ Transaction.source == "sync",+ (+ Transaction.source.in_(LOCAL_IMPORT_SOURCES)+ & Transaction.raw_data.is_not(None)+ ),+ ),+ Transaction.date >= txn_data.date - timedelta(days=3),+ Transaction.date <= txn_data.date + timedelta(days=3),+ Transaction.amount == txn_data.amount,+ Transaction.type == txn_data.type,+ Transaction.status == txn_data.status,+ Transaction.external_id != txn_data.external_id,+ )+ )+ for candidate in result.scalars():+ if candidate.external_id and candidate.external_id.startswith("bill_charge:"):+ continue+ if candidate.external_id in incoming_external_ids:+ continue+ candidate_raw = candidate.raw_data if isinstance(candidate.raw_data, dict) else {}+ candidate_descriptions = (+ candidate_raw.get("description"), candidate.payee, candidate.description,+ )+ incoming_descriptions = (+ raw.get("description"), txn_data.payee, txn_data.description,+ )+ description_matches = any(+ left and right and token_overlap(left, right) >= 0.9+ for left in candidate_descriptions+ for right in incoming_descriptions+ )+ if (+ candidate_raw.get("posted") == posted+ and candidate_raw.get("transacted_at") == transacted_at+ and description_matches+ ):+ return candidate++ # Path 5: local import history from before the account was connected.+ # Posting lag and statement exports can shift the date or shorten the+ # merchant description, so accept only one exact normalized merchant/payee.+ return await find_unique_transaction_match(+ session,+ account_id,+ txn_data,+ LOCAL_IMPORT_SOURCES,+ unclaimed_only=True,+ exclude_external_ids=incoming_external_ids,+ )async def _cleanup_phantom_duplicates(@@ -1689,7 +1875,9 @@ async def sync_connection(requesting_user_id: uuid.UUID,trigger_provider_refresh: bool = False,) -> tuple[BankConnection, int]:- connection = await get_connection(session, connection_id, workspace_id)+ connection = await get_connection(+ session, connection_id, workspace_id, for_update=True+ )if not connection:raise ValueError("Connection not found")if not connection.credentials:@@ -1750,14 +1938,19 @@ async def sync_connection(# read. Providers that don't expose an on-demand refresh return# "skipped" via the default implementation and we proceed normally.if trigger_provider_refresh:+ connection.settings = {+ **conn_settings,+ "last_provider_refresh_at": datetime.now(timezone.utc).isoformat(),+ }outcome = await provider.trigger_refresh(credentials)if outcome == "needs_user_action":# Surfacing reconnect immediately is better than silently# reading stale data the user knows is stale.connection.status = "error"await session.commit()- raise RuntimeError(- "Provider needs the user to reconnect before fetching fresh data"+ raise ProviderUserActionRequired(+ "Provider needs the user to reconnect before fetching fresh data",+ code="credentials_invalid",)# "refreshed", "skipped", or "failed" all fall through to a read.# On "failed" we read whatever cached copy the provider has —@@ -1769,15 +1962,19 @@ async def sync_connection(new_tx_ids: list[uuid.UUID] = []merged_count = 0accounts_data = await provider.get_accounts(credentials)+ incoming_external_ids = {acc.external_id for acc in accounts_data}institution_cache: dict[str, Institution] = {}for acc_data in accounts_data:- result = await session.execute(- select(Account).where(- Account.connection_id == connection.id,- Account.external_id == acc_data.external_id,- )+ institution = await _resolve_institution(+ session, connection.id, institution_cache, acc_data+ )+ account = await _find_existing_connected_account(+ session,+ connection,+ acc_data,+ institution,+ incoming_external_ids,)- account = result.scalar_one_or_none()if account is not None:account.user_id = user_id@@ -1796,10 +1993,6 @@ async def sync_connection(.values(user_id=user_id, workspace_id=connection.workspace_id))- institution = await _resolve_institution(- session, connection.id, institution_cache, acc_data- )-# Honor user intent: a closed connected account stays closed and is# not touched by sync. The row is left alone (no balance/name# rewrite, no new transactions) but the connection link is kept so@@ -1860,6 +2053,7 @@ async def sync_connection(is_cc = acc_data.type == "credit_card"account = Account(user_id=user_id,+ workspace_id=workspace_id,connection_id=connection.id,external_id=acc_data.external_id,name=acc_data.name,@@ -1906,6 +2100,7 @@ async def sync_connection(if not import_pending:transactions_data = [t for t in transactions_data if t.status != "pending"]+ incoming_external_ids = {txn.external_id for txn in transactions_data}for txn_data in transactions_data:existing = await session.execute(select(Transaction)@@ -1928,8 +2123,7 @@ async def sync_connection(# a re-sync can't revive a transaction the user hid.if existing_tx.is_ignored:continue- if existing_tx.original_description is None:- existing_tx.original_description = txn_data.description+ _merge_sync_metadata(existing_tx, txn_data)if existing_tx.status == "pending" and txn_data.status == "posted":existing_tx.status = "posted"# Self-heal bill linkage: a tx that pre-dates the bills@@ -1959,13 +2153,8 @@ async def sync_connection(if fuzzy_match:if fuzzy_match.is_ignored:continue- fuzzy_match.external_id = txn_data.external_id+ _merge_sync_metadata(fuzzy_match, txn_data)fuzzy_match.source = "sync"- fuzzy_match.raw_data = txn_data.raw_data- if fuzzy_match.original_description is None:- fuzzy_match.original_description = txn_data.description- if not fuzzy_match.payee and txn_data.payee:- fuzzy_match.payee = txn_data.payeemerged_count += 1continue@@ -1975,7 +2164,7 @@ async def sync_connection(# status, fingerprint match collapses it instead of letting# both rows land.synced_dup = await _find_synced_duplicate(- session, account.id, txn_data+ session, account.id, txn_data, incoming_external_ids)if synced_dup:if synced_dup.original_description is None:@@ -1984,8 +2173,11 @@ async def sync_connection(# Posted truth wins: swap in the new id so subsequent# syncs match by external_id and update raw_data.synced_dup.status = "posted"- synced_dup.external_id = txn_data.external_id- synced_dup.raw_data = txn_data.raw_data+ _merge_sync_metadata(+ synced_dup,+ txn_data,+ replace_external_id=synced_dup.source == "sync",+ )if (txn_data.bill_external_idand synced_dup.effective_bill_date is None@@ -1996,6 +2188,15 @@ async def sync_connection(apply_effective_date(synced_dup, account, bill_due_date=bill.due_date)+ elif synced_dup.external_id != txn_data.external_id:+ # Same logical posted row re-keyed by a provider such+ # as SimpleFIN. Keep the user's row and move its+ # idempotency key forward instead of inserting a twin.+ _merge_sync_metadata(+ synced_dup,+ txn_data,+ replace_external_id=synced_dup.source == "sync",+ )continueincoming_currency = (@@ -2199,7 +2400,17 @@ async def sync_connection(await session.delete(orphan)connection.last_sync_at = datetime.now(timezone.utc)- connection.status = "active"+ action_required_warnings = getattr(provider, "action_required_warnings", None)+ if isinstance(action_required_warnings, list) and action_required_warnings:+ logger.warning(+ "Provider %s synced with %d user-action warning(s) for connection %s",+ connection.provider,+ len(action_required_warnings),+ connection.id,+ )+ connection.status = "error"+ else:+ connection.status = "active"await session.commit()await session.refresh(connection)return connection, merged_countdiff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.pyindex 438ca6d1..45071117 100644--- a/backend/app/services/import_service.py+++ b/backend/app/services/import_service.py@@ -15,6 +15,7 @@ from sqlalchemy.ext.asyncio import AsyncSessionfrom app.core.config import get_settingsfrom app.models.account import Account+from app.models.bank_connection import BankConnectionfrom app.models.category import Categoryfrom app.models.rule import Rulefrom app.models.transaction import Transaction@@ -26,6 +27,7 @@ from app.services.rule_engine import apply_rule_actions, evaluate_conditions, mefrom app.services.rule_service import apply_rules_to_transaction, preview_rules_for_transactionfrom app.services.fx_rate_service import stamp_primary_amountfrom app.services.payee_service import get_or_create_payee+from app.services.transaction_match_service import find_unique_transaction_match# Descriptions used by some Brazilian banks (e.g. Banco do Brasil) for@@ -743,10 +745,19 @@ async def import_transactions(await session.flush() # Get the import_log.id# Look up account currency for fallback- account_result = await session.execute(- select(Account).where(Account.id == account_id)- )+ account_result = await session.execute(select(Account).where(Account.id == account_id))account = account_result.scalar_one_or_none()+ if account and account.connection_id:+ await session.execute(+ select(BankConnection.id)+ .where(BankConnection.id == account.connection_id)+ .with_for_update()+ )+ if account:+ account_result = await session.execute(+ select(Account).where(Account.id == account_id).with_for_update()+ )+ account = account_result.scalar_one()account_currency = account.currency if account else get_settings().default_currency# Build category name → id map scoped to the workspace, on the same terms@@ -767,6 +778,7 @@ async def import_transactions(imported = 0landed: list[Transaction] = []skipped = 0+ matched_existing_ids: set[uuid.UUID] = set()effective_format = (detected_format or source or "").lower()should_detect_duplicates = detect_duplicates if effective_format == "csv" else True@@ -780,30 +792,45 @@ async def import_transactions(# Formats without unique IDs fall back to transaction fields; compare# both descriptions because rules may have changed the displayed one.if txn_data.external_id:- existing = await session.execute(- select(Transaction).where(- Transaction.account_id == account_id,- Transaction.external_id == txn_data.external_id,- Transaction.date == txn_data.date,- )+ existing_statement = select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.external_id == txn_data.external_id,+ Transaction.date == txn_data.date,)else:- existing = await session.execute(- select(Transaction).where(- Transaction.account_id == account_id,- Transaction.date == txn_data.date,- Transaction.amount == txn_data.amount,- Transaction.type == txn_data.type,- or_(- Transaction.description == txn_data.description,- Transaction.original_description == txn_data.description,- ),- )+ existing_statement = select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.date == txn_data.date,+ Transaction.amount == txn_data.amount,+ Transaction.type == txn_data.type,+ or_(+ Transaction.description == txn_data.description,+ Transaction.original_description == txn_data.description,+ ),+ )+ # `.first()` rather than `.scalar_one_or_none()`: the dedup key can+ # legitimately match more than one row (e.g. a prior sync/import race+ # left a duplicate, or a bank reuses one FITID across statements),+ # and we only need to know whether *any* match exists. Requiring+ # exactly one would raise MultipleResultsFound and abort the import.+ if matched_existing_ids and not txn_data.external_id:+ existing_statement = existing_statement.where(+ Transaction.id.not_in(matched_existing_ids)+ )+ existing = await session.execute(+ existing_statement.order_by(Transaction.created_at, Transaction.id)+ )+ duplicate = existing.scalars().first()+ if not duplicate:+ duplicate = await find_unique_transaction_match(+ session,+ account_id,+ txn_data,+ {"sync"},+ exclude_ids=matched_existing_ids,)- # `.first()` is intentional: duplicate keys can legitimately match- # multiple rows after an import/sync race or reused bank identifier,- # and duplicate detection only needs to establish that any row exists.- if existing.scalars().first() is not None:+ if duplicate:+ matched_existing_ids.add(duplicate.id)skipped += 1continue@@ -913,6 +940,8 @@ async def import_transactions(session.add(incoming)await session.flush()+ if should_detect_duplicates and not txn_data.external_id:+ matched_existing_ids.add(incoming.id)if recurring_link is not None:recurring_match_service.advance_past(recurring_link, txn_data.date)diff --git a/backend/app/services/transaction_match_service.py b/backend/app/services/transaction_match_service.pynew file mode 100644index 00000000..2e62fe69--- /dev/null+++ b/backend/app/services/transaction_match_service.py@@ -0,0 +1,102 @@+import re+import uuid+from collections.abc import Collection+from datetime import timedelta+from typing import Optional++from sqlalchemy import or_, select+from sqlalchemy.ext.asyncio import AsyncSession++from app.models.transaction import Transaction+++def _normalize_description(value: str | None) -> str:+ return " ".join(re.sub(r"[^\w]+", " ", (value or "").casefold()).split())+++def descriptions_match(a: str | None, b: str | None) -> bool:+ """Match only normalized merchant text; never infer from shared fragments."""+ normalized = [_normalize_description(value) for value in (a, b)]+ return bool(normalized[0]) and normalized[0] == normalized[1]+++async def find_unique_transaction_match(+ session: AsyncSession,+ account_id: uuid.UUID,+ txn_data,+ sources: Collection[str],+ *,+ date_tolerance_days: int = 3,+ unclaimed_only: bool = False,+ exclude_ids: Collection[uuid.UUID] = (),+ exclude_external_ids: Collection[str] = (),+) -> Optional[Transaction]:+ """Return one unambiguous imported/provider counterpart, never a guess."""+ date_lo = txn_data.date - timedelta(days=date_tolerance_days)+ date_hi = txn_data.date + timedelta(days=date_tolerance_days)+ statement = select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.source.in_(sources),+ Transaction.amount == txn_data.amount,+ Transaction.type == txn_data.type,+ Transaction.date >= date_lo,+ Transaction.date <= date_hi,+ )+ if unclaimed_only:+ statement = statement.where(Transaction.raw_data.is_(None))+ if exclude_ids:+ statement = statement.where(Transaction.id.not_in(exclude_ids))+ if exclude_external_ids:+ statement = statement.where(+ or_(+ Transaction.external_id.is_(None),+ Transaction.external_id.not_in(exclude_external_ids),+ )+ )+ result = await session.execute(statement)++ incoming_descriptions = (+ txn_data.description,+ getattr(txn_data, "payee", None),+ getattr(txn_data, "payee_raw", None),+ )+ matches = [+ candidate+ for candidate in result.scalars().all()+ if any(+ descriptions_match(candidate_description, incoming_description)+ for candidate_description in (+ candidate.description,+ candidate.original_description,+ candidate.payee,+ )+ for incoming_description in incoming_descriptions+ )+ ]+ if len(matches) <= 1:+ return matches[0] if matches else None++ incoming_exact = {+ normalized+ for value in incoming_descriptions+ if (normalized := _normalize_description(value))+ }+ exact_matches = [+ candidate+ for candidate in matches+ if incoming_exact+ & {+ normalized+ for value in (+ candidate.description,+ candidate.original_description,+ candidate.payee,+ )+ if (normalized := _normalize_description(value))+ }+ ]+ if exact_matches:+ # Indistinguishable exact candidates form a multiset. Consume one+ # deterministically; the caller's exclusions prevent reusing it.+ return min(exact_matches, key=lambda candidate: str(candidate.id))+ return Nonediff --git a/backend/app/tasks/sync_tasks.py b/backend/app/tasks/sync_tasks.pyindex f562e872..96a4ca8e 100644--- a/backend/app/tasks/sync_tasks.py+++ b/backend/app/tasks/sync_tasks.py@@ -15,6 +15,20 @@ from app.services import connection_servicelogger = logging.getLogger(__name__)STALE_THRESHOLD = timedelta(hours=4)+PROVIDER_REFRESH_THRESHOLD = timedelta(hours=20)+++def _should_trigger_provider_refresh(settings: dict | None, now: datetime) -> bool:+ value = (settings or {}).get("last_provider_refresh_at")+ if not isinstance(value, str):+ return True+ try:+ last_refresh = datetime.fromisoformat(value)+ except ValueError:+ return True+ if last_refresh.tzinfo is None:+ return True+ return now - last_refresh >= PROVIDER_REFRESH_THRESHOLDdef _make_session_maker():@@ -33,7 +47,10 @@ async def _sync_all() -> int:async with session_maker() as session:result = await session.execute(select(- BankConnection.id, BankConnection.user_id, BankConnection.last_sync_at+ BankConnection.id,+ BankConnection.user_id,+ BankConnection.last_sync_at,+ BankConnection.settings,).where(BankConnection.status.in_(["active", "error"]),(BankConnection.last_sync_at < cutoff)@@ -48,10 +65,17 @@ async def _sync_all() -> int:cutoff.isoformat(),)- for conn_id, user_id, last_sync in connections:+ for conn_id, user_id, last_sync, settings in connections:try:logger.info("Syncing connection %s (last_sync=%s)", conn_id, last_sync)- await _sync_one(session_maker, conn_id, user_id)+ await _sync_one(+ session_maker,+ conn_id,+ user_id,+ trigger_provider_refresh=_should_trigger_provider_refresh(+ settings, datetime.now(timezone.utc)+ ),+ )synced += 1except ProviderNotConfiguredError as exc:# Actionable one-liner instead of a buried traceback: this@@ -65,7 +89,13 @@ async def _sync_all() -> int:await engine.dispose()-async def _sync_one(session_maker, connection_id: uuid.UUID, user_id: uuid.UUID) -> None:+async def _sync_one(+ session_maker,+ connection_id: uuid.UUID,+ user_id: uuid.UUID,+ *,+ trigger_provider_refresh: bool = False,+) -> None:"""Sync a single connection. Error status is set by sync_connection itself."""async with session_maker() as session:workspace_id = await session.scalar(@@ -75,7 +105,11 @@ async def _sync_one(session_maker, connection_id: uuid.UUID, user_id: uuid.UUID)logger.warning("Connection %s has no workspace; skipping sync", connection_id)returnawait connection_service.sync_connection(- session, connection_id, workspace_id, user_id+ session,+ connection_id,+ workspace_id,+ user_id,+ trigger_provider_refresh=trigger_provider_refresh,)diff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.pyindex c3bcc097..31e82215 100644--- a/backend/tests/test_connection_service.py+++ b/backend/tests/test_connection_service.py@@ -1,16 +1,19 @@import uuidfrom datetime import date, datetime, timezonefrom decimal import Decimal-from unittest.mock import AsyncMock, patch+from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom sqlalchemy import select+from sqlalchemy.dialects import postgresqlfrom sqlalchemy.ext.asyncio import AsyncSessionfrom app.models.account import Accountfrom app.models.asset import Assetfrom app.models.bank_connection import BankConnectionfrom app.models.category import Category+from app.models.import_log import ImportLog+from app.models.institution import Institutionfrom app.models.transaction import Transactionfrom app.schemas.rule import RuleAction, RuleCondition, RuleCreatefrom app.providers.base import (@@ -24,6 +27,8 @@ from app.providers.base import ()from app.services.text_similarity import token_overlapfrom app.services.connection_service import (+ _find_existing_connected_account,+ _merge_sync_metadata,_match_pluggy_category,create_connect_token,delete_connection,@@ -90,6 +95,62 @@ def test_description_similarity_no_overlap():assert token_overlap("abc", "xyz") == 0.0+def test_merge_sync_metadata_repairs_invalid_epoch_date():+ tx = Transaction(+ id=uuid.uuid4(),+ user_id=uuid.uuid4(),+ account_id=uuid.uuid4(),+ description="Pending",+ amount=Decimal("5"),+ date=date(1970, 1, 1),+ effective_date=date(1970, 1, 1),+ type="debit",+ source="sync",+ status="pending",+ )+ txn_data = TransactionData(+ external_id="provider-id",+ description="Pending",+ amount=Decimal("5"),+ date=date(2026, 6, 27),+ type="debit",+ )++ _merge_sync_metadata(tx, txn_data)++ assert tx.date == date(2026, 6, 27)+ assert tx.effective_date == date(2026, 6, 27)+++def test_merge_sync_metadata_rekey_preserves_old_raw_fields():+ tx = Transaction(+ id=uuid.uuid4(),+ user_id=uuid.uuid4(),+ account_id=uuid.uuid4(),+ external_id="old-id",+ description="Coffee",+ amount=Decimal("5"),+ date=date(2026, 6, 27),+ type="debit",+ source="sync",+ status="posted",+ raw_data={"posted": 1, "old_only": True},+ )+ txn_data = TransactionData(+ external_id="new-id",+ description="Coffee",+ amount=Decimal("5"),+ date=date(2026, 6, 27),+ type="debit",+ raw_data={"posted": 2, "new_only": True},+ )++ _merge_sync_metadata(tx, txn_data, replace_external_id=True)++ assert tx.external_id == "new-id"+ assert tx.raw_data == {"posted": 2, "old_only": True, "new_only": True}++def test_description_similarity_none():assert token_overlap(None, "hello") == 0.0assert token_overlap("hello", None) == 0.0@@ -764,6 +825,472 @@ async def test_sync_connection_new_transactions(session: AsyncSession, test_userassert transaction.original_description == "GROCERY"+async def test_sync_connection_partial_user_action_warning_marks_connection_error(+ session: AsyncSession, test_user, test_workspace,+):+ conn = await _make_connection(session, test_user.id, "Partial SimpleFIN")+ mock_provider = AsyncMock()+ mock_provider.action_required_warnings = [+ {"code": "credentials_invalid", "message": "Apple Card auth required"}+ ]+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "refreshed"})+ mock_provider.get_accounts = AsyncMock(return_value=[+ AccountData(+ external_id="partial-acc-1", name="Checking",+ type="checking", balance=Decimal("2000"), currency="BRL",+ ),+ ])+ mock_provider.get_transactions = AsyncMock(return_value=[+ TransactionData(+ external_id="partial-tx-1", description="GROCERY",+ amount=Decimal("80"), date=date.today(), type="debit", currency="BRL",+ ),+ ])++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ result_conn, merged = await sync_connection(session, conn.id, test_workspace.id, test_user.id)++ assert result_conn.status == "error"+ assert merged == 0+ synced_tx = (await session.execute(+ select(Transaction).where(Transaction.external_id == "partial-tx-1")+ )).scalar_one()+ assert synced_tx.description == "GROCERY"+++async def test_sync_connection_simplefin_rekey_does_not_duplicate_account_or_tx(+ session: AsyncSession, test_user, test_workspace,+):+ conn = await _make_connection(session, test_user.id, "SimpleFIN Bank")+ conn.provider = "simplefin"++ account = Account(+ id=uuid.uuid4(),+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ connection_id=conn.id,+ external_id="old-account-id",+ name="High Yield Savings Account (9402)",+ display_name="Albert · Amex HYSA",+ type="savings",+ balance=Decimal("100.00"),+ currency="USD",+ )+ session.add(account)+ existing = Transaction(+ id=uuid.uuid4(),+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ account_id=account.id,+ external_id="old-tx-id",+ description="ROBINHOOD FUNDS",+ amount=Decimal("3086.00"),+ date=date(2026, 6, 16),+ type="credit",+ currency="USD",+ source="sync",+ status="posted",+ raw_data={"posted": 1781618248, "transacted_at": 1781611200},+ created_at=datetime.now(timezone.utc),+ )+ session.add(existing)+ await session.commit()++ mock_provider = AsyncMock()+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "refreshed"})+ mock_provider.get_accounts = AsyncMock(return_value=[+ AccountData(+ external_id="new-account-id",+ name="High Yield Savings Account (9402)",+ type="checking",+ balance=Decimal("100.00"),+ currency="USD",+ ),+ ])+ mock_provider.get_transactions = AsyncMock(return_value=[+ TransactionData(+ external_id="new-tx-id",+ description="ROBINHOOD FUNDS",+ amount=Decimal("3086.00"),+ date=date(2026, 6, 16),+ type="credit",+ currency="USD",+ status="posted",+ payee="Robinhood",+ raw_data={"posted": 1781618248, "transacted_at": 1781611200},+ ),+ ])++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ result_conn, merged = await sync_connection(session, conn.id, test_workspace.id, test_user.id)++ assert result_conn.status == "active"+ assert merged == 0++ accounts = (await session.execute(+ select(Account).where(Account.connection_id == conn.id)+ )).scalars().all()+ assert len(accounts) == 1+ assert accounts[0].id == account.id+ assert accounts[0].external_id == "new-account-id"+ assert accounts[0].display_name == "Albert · Amex HYSA"+ assert accounts[0].type == "savings"++ txs = (await session.execute(+ select(Transaction).where(Transaction.account_id == account.id)+ )).scalars().all()+ sync_txs = [tx for tx in txs if tx.source == "sync"]+ assert len(sync_txs) == 1+ assert sync_txs[0].id == existing.id+ assert sync_txs[0].external_id == "new-tx-id"+++async def test_simplefin_rekey_respects_institution_and_closed_account(+ session: AsyncSession, test_user, test_workspace,+):+ conn = await _make_connection(session, test_user.id, "SimpleFIN")+ conn.provider = "simplefin"+ bank_a = Institution(+ connection_id=conn.id, external_id="bank-a", name="Bank A"+ )+ bank_b = Institution(+ connection_id=conn.id, external_id="bank-b", name="Bank B"+ )+ session.add_all([bank_a, bank_b])+ await session.flush()+ closed_bank_a = Account(+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ connection_id=conn.id,+ institution_id=bank_a.id,+ external_id="old-bank-a",+ name="Checking",+ type="checking",+ balance=Decimal("10"),+ currency="USD",+ is_closed=True,+ )+ customized_bank_b = Account(+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ connection_id=conn.id,+ institution_id=bank_b.id,+ external_id="old-bank-b",+ name="Checking",+ display_name="My Bank B",+ type="checking",+ balance=Decimal("20"),+ currency="USD",+ )+ session.add_all([closed_bank_a, customized_bank_b])+ await session.flush()++ matched = await _find_existing_connected_account(+ session,+ conn,+ AccountData(+ external_id="new-bank-a",+ name="Checking",+ type="checking",+ balance=Decimal("10"),+ currency="USD",+ institution_external_id="bank-a",+ institution_name="Bank A",+ ),+ bank_a,+ {"new-bank-a"},+ )++ assert matched is closed_bank_a+ assert closed_bank_a.external_id == "new-bank-a"+ assert customized_bank_b.external_id == "old-bank-b"+++async def test_simplefin_rekey_reserves_ids_from_later_accounts(+ session: AsyncSession, test_user, test_workspace,+):+ conn = await _make_connection(session, test_user.id, "SimpleFIN")+ conn.provider = "simplefin"+ later_account = Account(+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ connection_id=conn.id,+ external_id="later-account-id",+ name="Checking",+ type="checking",+ balance=Decimal("10"),+ currency="USD",+ )+ session.add(later_account)+ await session.flush()++ matched = await _find_existing_connected_account(+ session,+ conn,+ AccountData(+ external_id="new-account-id",+ name="Checking",+ type="checking",+ balance=Decimal("10"),+ currency="USD",+ ),+ None,+ {"new-account-id", "later-account-id"},+ )++ assert matched is None+ assert later_account.external_id == "later-account-id"+++async def test_simplefin_rekey_refuses_same_institution_ambiguity(+ session: AsyncSession, test_user, test_workspace,+):+ conn = await _make_connection(session, test_user.id, "SimpleFIN")+ conn.provider = "simplefin"+ institution = Institution(+ connection_id=conn.id, external_id="bank-a", name="Bank A"+ )+ session.add(institution)+ await session.flush()+ accounts = [+ Account(+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ connection_id=conn.id,+ institution_id=institution.id,+ external_id=f"old-{index}",+ name="Checking",+ type="checking",+ balance=Decimal("10"),+ currency="USD",+ )+ for index in range(2)+ ]+ session.add_all(accounts)+ await session.flush()++ matched = await _find_existing_connected_account(+ session,+ conn,+ AccountData(+ external_id="rotated-id",+ name="Checking",+ type="checking",+ balance=Decimal("10"),+ currency="USD",+ institution_external_id="bank-a",+ institution_name="Bank A",+ ),+ institution,+ {"rotated-id"},+ )++ assert matched is None+ assert {account.external_id for account in accounts} == {"old-0", "old-1"}+++async def test_sync_upserts_matching_csv_import_without_overwriting_user_fields(+ session: AsyncSession, test_user, test_workspace,+):+ conn = await _make_connection(session, test_user.id, "Imported History Bank")+ account_id = uuid.uuid4()+ category_id = uuid.uuid4()+ import_log_id = uuid.uuid4()+ imported_id = uuid.uuid4()+ second_imported_id = uuid.uuid4()+ category = Category(+ id=category_id,+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ name="Already Categorized",+ icon="tag",+ color="#000000",+ is_system=False,+ )+ account = Account(+ id=account_id,+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ connection_id=conn.id,+ external_id="import-acc-1",+ name="Checking",+ type="checking",+ balance=Decimal("-42.00"),+ currency="BRL",+ )+ import_log = ImportLog(+ id=import_log_id,+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ account_id=account_id,+ filename="history.csv",+ format="csv",+ transaction_count=2,+ total_debit=Decimal("42.00"),+ )+ imported = Transaction(+ id=imported_id,+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ account_id=account_id,+ external_id="csv-row-1",+ description="Spotify",+ amount=Decimal("42.00"),+ date=date(2026, 6, 21),+ type="debit",+ currency="BRL",+ source="import",+ import_id=import_log_id,+ category_id=category_id,+ )+ second_imported = Transaction(+ id=second_imported_id,+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ account_id=account_id,+ external_id="csv-row-2",+ description="Spotify",+ amount=Decimal("42.00"),+ date=date(2026, 6, 21),+ type="debit",+ currency="BRL",+ source="import",+ import_id=import_log_id,+ category_id=category_id,+ )+ session.add_all([category, account, import_log, imported, second_imported])+ await session.commit()++ mock_provider = AsyncMock()+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "refreshed"})+ mock_provider.get_accounts = AsyncMock(return_value=[+ AccountData(+ external_id="import-acc-1",+ name="Checking",+ type="checking",+ balance=Decimal("-42.00"),+ currency="BRL",+ ),+ ])+ mock_provider.get_transactions = AsyncMock(return_value=[+ TransactionData(+ external_id="provider-tx-1",+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ amount=Decimal("42.00"),+ date=date(2026, 6, 20),+ type="debit",+ currency="BRL",+ payee="Spotify",+ raw_data={+ "posted": 1781913600,+ "description": "SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ },+ ),+ TransactionData(+ external_id="provider-tx-2",+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ amount=Decimal("42.00"),+ date=date(2026, 6, 20),+ type="debit",+ currency="BRL",+ payee="Spotify",+ raw_data={+ "posted": 1781913600,+ "description": "SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ },+ ),+ TransactionData(+ external_id="provider-tx-3",+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ amount=Decimal("42.00"),+ date=date(2026, 6, 20),+ type="debit",+ currency="BRL",+ payee="Spotify",+ raw_data={+ "posted": 1781913600,+ "description": "SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ },+ ),+ ])++ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock) as apply_rules:+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)++ rows = (await session.execute(+ select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.source != "opening_balance",+ )+ )).scalars().all()+ assert len(rows) == 3+ claimed_rows = [+ row for row in rows if row.id in {imported_id, second_imported_id}+ ]+ assert len(claimed_rows) == 2+ assert {row.external_id for row in claimed_rows} == {+ "provider-tx-1", "provider-tx-2",+ }+ assert all(row.source == "import" for row in claimed_rows)+ assert all(row.import_id is None for row in claimed_rows)+ assert all(row.category_id == category_id for row in claimed_rows)+ assert all(row.description == "Spotify" for row in claimed_rows)+ assert all(row.date == date(2026, 6, 21) for row in claimed_rows)+ assert all(row.payee == "Spotify" for row in claimed_rows)+ assert {row.external_id for row in rows} == {+ "provider-tx-1", "provider-tx-2", "provider-tx-3",+ }+ assert apply_rules.await_count == 1++ mock_provider.get_transactions = AsyncMock(return_value=[+ TransactionData(+ external_id="provider-tx-4",+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ amount=Decimal("42.00"),+ date=date(2026, 6, 20),+ type="debit",+ currency="BRL",+ payee="Spotify",+ raw_data={+ "posted": 1781913600,+ "description": "SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ },+ ),+ *mock_provider.get_transactions.return_value[1:],+ ])+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)++ rekeyed_rows = (await session.execute(+ select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.source != "opening_balance",+ )+ )).scalars().all()+ assert len(rekeyed_rows) == 3+ assert {row.external_id for row in rekeyed_rows} == {+ "provider-tx-4", "provider-tx-2", "provider-tx-3",+ }++@pytest.mark.asyncioasync def test_sync_connection_tolerates_duplicate_transaction_rows(session: AsyncSession, test_user, test_workspace@@ -830,10 +1357,34 @@ async def test_sync_connection_tolerates_duplicate_transaction_rows(assert any(r.status == "posted" for r in rows)+@pytest.mark.asyncioasync def test_sync_connection_not_found(session: AsyncSession, test_user, test_workspace):- with pytest.raises(ValueError, match="not found"):- await sync_connection(session, uuid.uuid4(), test_workspace.id, test_user.id)+ connection_id = uuid.uuid4()+ with patch(+ "app.services.connection_service.get_connection", wraps=get_connection+ ) as locked_get_connection:+ with pytest.raises(ValueError, match="not found"):+ await sync_connection(session, connection_id, test_workspace.id, test_user.id)+ locked_get_connection.assert_awaited_once_with(+ session, connection_id, test_workspace.id, for_update=True+ )+++async def test_get_connection_can_lock_row_for_update():+ session = AsyncMock(spec=AsyncSession)+ result = MagicMock()+ result.scalar_one_or_none.return_value = None+ session.execute.return_value = result++ await get_connection(+ session, uuid.uuid4(), uuid.uuid4(), for_update=True+ )++ statement = session.execute.await_args.args[0]+ sql = str(statement.compile(dialect=postgresql.dialect()))+ assert sql.rstrip().endswith("FOR UPDATE")@pytest.mark.asyncio@@ -2751,12 +3302,14 @@ async def test_sync_keeps_genuine_same_day_repeats(description="UBER TRIP",amount=Decimal("25.00"), date=date(2026, 4, 20),type="debit", currency="BRL", status="posted",+ raw_data={"posted": 1776643200, "transacted_at": 1776643200},),TransactionData(external_id="uber-2",description="UBER TRIP",amount=Decimal("25.00"), date=date(2026, 4, 20),type="debit", currency="BRL", status="posted",+ raw_data={"posted": 1776643200, "transacted_at": 1776643200},),])@@ -2772,7 +3325,7 @@ async def test_sync_keeps_genuine_same_day_repeats(Transaction.source == "sync",))).scalars().all()- assert len(rows) == 2, "identical-description same-day repeats must be kept"+ assert {row.external_id for row in rows} == {"uber-1", "uber-2"}# ---------------------------------------------------------------------------diff --git a/backend/tests/test_connection_service_coverage.py b/backend/tests/test_connection_service_coverage.pyindex 94854a8c..55414276 100644--- a/backend/tests/test_connection_service_coverage.py+++ b/backend/tests/test_connection_service_coverage.py@@ -487,16 +487,19 @@ async def test_sync_trigger_refresh_needs_user_action(session: AsyncSession, tesmock_provider.trigger_refresh = AsyncMock(return_value="needs_user_action")with patch("app.services.connection_service.get_provider", return_value=mock_provider):- with pytest.raises(RuntimeError, match="reconnect"):+ with pytest.raises(ProviderUserActionRequired, match="reconnect") as exc:await sync_connection(session, conn.id, test_workspace.id, test_user.id,trigger_provider_refresh=True,)+ assert exc.value.code == "credentials_invalid"# status marked error and committedrefreshed = (await session.execute(select(BankConnection).where(BankConnection.id == conn.id))).scalar_one()assert refreshed.status == "error"+ assert refreshed.settings is not None+ assert "last_provider_refresh_at" in refreshed.settings@pytest.mark.asyncio@@ -519,6 +522,8 @@ async def test_sync_trigger_refresh_refreshed_then_reads(session: AsyncSession,trigger_provider_refresh=True,)assert result.status == "active"+ assert result.settings is not None+ assert "last_provider_refresh_at" in result.settingsmock_provider.trigger_refresh.assert_awaited_once()diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.pyindex c5349b1d..772eb5ac 100644--- a/backend/tests/test_import_service.py+++ b/backend/tests/test_import_service.py@@ -2047,13 +2047,130 @@ class TestOfxInstallmentDedup:)await import_transactions(session, test_workspace.id, test_user.id, test_account.id, [txn], "ofx")imported, skipped, _, _ = await import_transactions(- session, test_workspace.id, test_user.id, test_account.id, [txn], "ofx",+ session,+ test_workspace.id,+ test_user.id,+ test_account.id,+ [txn, txn.model_copy()],+ "ofx",)assert imported == 0- assert skipped == 1+ assert skipped == 2class TestCsvDuplicateDetectionToggle:+ @pytest.mark.asyncio+ async def test_csv_identical_new_rows_remain_distinct(+ self, session: AsyncSession, test_user: User, test_workspace, test_account: Account,+ ):+ from app.schemas.transaction import TransactionImport++ row = TransactionImport(+ description="Coffee Shop",+ amount=Decimal("8.50"),+ date=date(2026, 6, 15),+ type="debit",+ )+ imported, skipped, _, _ = await import_transactions(+ session,+ test_workspace.id,+ test_user.id,+ test_account.id,+ [row, row.model_copy()],+ "csv",+ )++ assert (imported, skipped) == (2, 0)++ @pytest.mark.asyncio+ async def test_csv_exact_match_consumes_one_synced_row_once(+ self, session: AsyncSession, test_user: User, test_workspace, test_account: Account,+ ):+ from app.models.transaction import Transaction+ from app.schemas.transaction import TransactionImport++ session.add(Transaction(+ id=uuid.uuid4(),+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ account_id=test_account.id,+ external_id="provider-coffee",+ description="Coffee Shop",+ amount=Decimal("8.50"),+ date=date(2026, 6, 15),+ type="debit",+ source="sync",+ status="posted",+ ))+ await session.commit()++ row = TransactionImport(+ description="Coffee Shop",+ amount=Decimal("8.50"),+ date=date(2026, 6, 15),+ type="debit",+ )+ imported, skipped, _, _ = await import_transactions(+ session,+ test_workspace.id,+ test_user.id,+ test_account.id,+ [row, row.model_copy()],+ "csv",+ )++ assert (imported, skipped) == (1, 1)++ @pytest.mark.asyncio+ async def test_csv_payee_match_consumes_one_synced_row_once(+ self, session: AsyncSession, test_user: User, test_workspace, test_account: Account,+ ):+ from app.models.transaction import Transaction+ from app.schemas.transaction import TransactionImport++ synced = Transaction(+ id=uuid.uuid4(),+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ account_id=test_account.id,+ external_id="provider-spotify",+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",+ payee="Spotify",+ amount=Decimal("12.99"),+ date=date(2026, 6, 15),+ type="debit",+ source="sync",+ status="posted",+ )+ session.add(synced)+ await session.commit()++ imported, skipped, _, _ = await import_transactions(+ session,+ test_workspace.id,+ test_user.id,+ test_account.id,+ [+ TransactionImport(+ description="Spotify",+ payee_raw="Spotify",+ amount=Decimal("12.99"),+ date=date(2026, 6, 16),+ type="debit",+ ),+ TransactionImport(+ description="Spotify",+ payee_raw="Spotify",+ amount=Decimal("12.99"),+ date=date(2026, 6, 16),+ type="debit",+ ),+ ],+ "csv",+ )++ assert (imported, skipped) == (1, 1)+@pytest.mark.asyncioasync def test_csv_detect_duplicates_false_allows_duplicates(self, session: AsyncSession, test_user: User, test_workspace, test_account: Account,@@ -2513,6 +2630,48 @@ async def test_import_tolerates_duplicate_external_id_rows(assert len(remaining) == 2+async def test_import_external_id_reconciles_matching_synced_transaction(+ session: AsyncSession, test_user: User, test_workspace, test_account: Account,+):+ from app.models.transaction import Transaction+ from app.schemas.transaction import TransactionImport++ session.add(Transaction(+ id=uuid.uuid4(),+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ account_id=test_account.id,+ external_id="provider-id",+ description="Music subscription",+ original_description="SPOTIFY",+ amount=Decimal("23.90"),+ date=date(2026, 1, 15),+ type="debit",+ source="sync",+ ))+ await session.commit()++ imported, skipped, _, _ = await import_transactions(+ session,+ test_workspace.id,+ test_user.id,+ test_account.id,+ [TransactionImport(+ external_id="ofx-fitid",+ description="SPOTIFY",+ amount=Decimal("23.90"),+ date=date(2026, 1, 15),+ type="debit",+ )],+ "ofx",+ detected_format="ofx",+ )++ assert imported == 0+ assert skipped == 1++def test_normalize_amount_swiss_single_quotes():from app.services.import_service import normalize_amountassert normalize_amount("1'050.45") == "1050.45"diff --git a/backend/tests/test_providers_simplefin.py b/backend/tests/test_providers_simplefin.pyindex 6dd00b41..ffd2f3bc 100644--- a/backend/tests/test_providers_simplefin.py+++ b/backend/tests/test_providers_simplefin.py@@ -182,6 +182,38 @@ async def test_auth_errlist_raises_user_action_required():assert exc.value.code == "credentials_invalid"+async def test_auth_errlist_with_usable_accounts_is_soft_warning(caplog):+ """A stale SimpleFIN sub-connection must not block healthy accounts."""+ import logging as stdlogging++ def handler(request: httpx.Request) -> httpx.Response:+ return httpx.Response(+ 200,+ json={+ "errlist": [+ {"code": "con.auth", "msg": "Apple Card auth required", "conn_id": "APPLE"}+ ],+ "accounts": [+ {+ "id": "acc-healthy",+ "name": "Healthy Checking",+ "currency": "USD",+ "balance": "42.00",+ }+ ],+ },+ )++ creds = {"access_url": "https://u:[email protected]/simplefin"}+ provider = SimpleFinProvider()+ with caplog.at_level(stdlogging.WARNING, logger="app.providers.simplefin"), _patched_client(handler):+ accounts = await provider.get_accounts(creds)++ assert [acc.external_id for acc in accounts] == ["acc-healthy"]+ assert any("con.auth" in rec.getMessage() for rec in caplog.records)++@pytest.mark.asyncioasync def test_act_failed_is_soft_warning(caplog):"""``act.failed`` is transient — keep going, just log."""@@ -242,6 +274,45 @@ async def test_missing_access_url_raises_session_expired():await SimpleFinProvider().get_accounts({})+async def test_get_transactions_auth_errlist_with_requested_account_is_soft_warning(caplog):+ """A top-level auth warning must not block requested-account transactions."""+ import logging as stdlogging++ def handler(request: httpx.Request) -> httpx.Response:+ assert request.url.params.get("account") == "acc-healthy"+ return httpx.Response(+ 200,+ json={+ "errlist": [+ {"code": "con.auth", "msg": "Apple Card auth required", "conn_id": "APPLE"}+ ],+ "accounts": [+ {+ "id": "acc-healthy",+ "transactions": [+ {+ "id": "tx-healthy",+ "amount": "-12.34",+ "posted": 1672531200,+ "description": "Coffee",+ }+ ],+ }+ ],+ },+ )++ creds = {"access_url": "https://u:[email protected]/simplefin"}+ with caplog.at_level(stdlogging.WARNING, logger="app.providers.simplefin"), _patched_client(handler):+ txns = await SimpleFinProvider().get_transactions(+ creds, "acc-healthy", since=date(2023, 1, 1)+ )++ assert [txn.external_id for txn in txns] == ["tx-healthy"]+ assert any("con.auth" in rec.getMessage() for rec in caplog.records)++# ----- transactions -----------------------------------------------------------@@ -363,6 +434,29 @@ async def test_get_transactions_chunks_long_windows():assert len(calls) >= 3 # 200 days / 90-day window+async def test_get_transactions_followup_rewinds_one_valid_full_window():+ calls: list[tuple[int, int]] = []++ def handler(request: httpx.Request) -> httpx.Response:+ calls.append((+ int(request.url.params["start-date"]),+ int(request.url.params["end-date"]),+ ))+ return httpx.Response(+ 200, json={"accounts": [{"id": "acc-1", "transactions": []}]}+ )++ creds = {"access_url": "https://u:[email protected]/simplefin"}+ with _patched_client(handler):+ await SimpleFinProvider().get_transactions(+ creds, "acc-1", since=date.today() - timedelta(days=14)+ )++ assert len(calls) == 1+ assert calls[0][1] - calls[0][0] == 45 * 24 * 60 * 60++# ----- holdings ---------------------------------------------------------------diff --git a/backend/tests/test_sync_tasks.py b/backend/tests/test_sync_tasks.pynew file mode 100644index 00000000..2964582d--- /dev/null+++ b/backend/tests/test_sync_tasks.py@@ -0,0 +1,61 @@+import uuid+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock, patch++import pytest++from app.models.bank_connection import BankConnection+from app.tasks.sync_tasks import _should_trigger_provider_refresh, _sync_one+++def test_should_trigger_provider_refresh_daily():+ now = datetime(2026, 6, 29, tzinfo=timezone.utc)++ assert _should_trigger_provider_refresh(None, now)+ assert _should_trigger_provider_refresh({"last_provider_refresh_at": "invalid"}, now)+ assert _should_trigger_provider_refresh(+ {"last_provider_refresh_at": (now - timedelta(hours=20)).isoformat()}, now+ )+ assert not _should_trigger_provider_refresh(+ {"last_provider_refresh_at": (now - timedelta(hours=4)).isoformat()}, now+ )+++async def test_sync_one_forwards_provider_refresh(session, test_user, test_workspace):+ conn = BankConnection(+ id=uuid.uuid4(),+ user_id=test_user.id,+ workspace_id=test_workspace.id,+ provider="test",+ external_id="provider-conn",+ institution_name="Test Bank",+ credentials={"token": "fake"},+ status="active",+ last_sync_at=datetime.now(timezone.utc),+ created_at=datetime.now(timezone.utc),+ )+ session.add(conn)+ await session.commit()++ def session_maker():+ return session++ with patch(+ "app.tasks.sync_tasks.connection_service.sync_connection",+ new_callable=AsyncMock,+ ) as sync_connection:+ await _sync_one(+ session_maker,+ conn.id,+ test_user.id,+ trigger_provider_refresh=True,+ )++ sync_connection.assert_awaited_once_with(+ session,+ conn.id,+ test_workspace.id,+ test_user.id,+ trigger_provider_refresh=True,+ )