Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix: harden connection sync correctness (#374)

Albert Dolkunf4550a77651d
diff --git a/backend/app/providers/simplefin.py b/backend/app/providers/simplefin.py
index 0bb51efb..21e66002 100644
--- a/backend/app/providers/simplefin.py
+++ b/backend/app/providers/simplefin.py
@@ -47,7 +47,7 @@ logger = logging.getLogger(__name__)
# initial sync (since=None) we walk backwards in chunks of this size; for
# follow-up syncs the sync layer's typical 30-90 day window fits in one call.
SIMPLEFIN_MAX_WINDOW_DAYS = 90
-SIMPLEFIN_DEFAULT_HISTORY_DAYS = 90
+SIMPLEFIN_DEFAULT_HISTORY_DAYS = 45 # routine overlap, not the per-request cap
SIMPLEFIN_INITIAL_HISTORY_DAYS = 365 # ~1 year backfill on first connect
SIMPLEFIN_HTTP_TIMEOUT = 60.0
@@ -152,18 +152,21 @@ def _ticker(value: Any) -> Optional[str]:
return value.strip().upper()[:32] or None
-def _surface_errors(errlist: list[dict], context: str) -> None:
- """Raise a typed exception for auth errors; log everything else.
+def _surface_errors(
+ errlist: list[dict], context: str, *, has_usable_data: bool = False
+) -> list[dict]:
+ """Raise for auth errors only when the response has no usable data.
- SimpleFIN's spec says: *Always show those errors to your end users.* For
- the codes the user can act on (re-auth), we raise so the API layer can
- flip the connection status. For the rest we log and continue — the
- response usually still has usable data alongside the warnings.
+ SimpleFIN's spec says: *Always show those errors to your end users.* The
+ Bridge can return top-level ``con.auth`` warnings for one institution while
+ still returning fresh account data for other institutions under the same
+ Access URL. In that partial-success case, log the warning and keep the sync
+ moving so one stale sub-connection does not block every healthy account.
"""
if not errlist:
- return
+ return []
reauth = [e for e in errlist if (e.get("code") or "").lower() in _REAUTH_ERROR_CODES]
- if reauth:
+ if reauth and not has_usable_data:
first = reauth[0]
raise ProviderUserActionRequired(
first.get("msg") or first.get("message") or "Reauthorize at SimpleFIN Bridge",
@@ -177,11 +180,15 @@ def _surface_errors(errlist: list[dict], context: str) -> None:
entry.get("code"),
entry.get("msg") or entry.get("message"),
)
+ return reauth
class SimpleFinProvider(BankProvider):
"""SimpleFIN Bridge connector."""
+ def __init__(self) -> None:
+ self.action_required_warnings: list[dict] = []
+
@property
def name(self) -> str:
return "simplefin"
@@ -439,7 +446,12 @@ class SimpleFinProvider(BankProvider):
async def get_accounts(self, credentials: dict) -> list[AccountData]:
payload = await self._fetch_accounts(credentials, pending=False)
- _surface_errors(payload.get("errlist") or [], context="get_accounts")
+ warnings = _surface_errors(
+ payload.get("errlist") or [],
+ context="get_accounts",
+ has_usable_data=bool(payload.get("accounts") or []),
+ )
+ self.action_required_warnings.extend(warnings)
_, accounts = self._parse_accounts(payload)
return accounts
@@ -454,30 +466,47 @@ class SimpleFinProvider(BankProvider):
if since is None:
start_date = end_date - timedelta(days=SIMPLEFIN_INITIAL_HISTORY_DAYS)
else:
- start_date = since
+ # The Bridge can expose bank transactions weeks after their posted
+ # date. Always re-read one full provider window so delayed rows are
+ # eventually recovered instead of aging out of the sync cursor.
+ start_date = min(
+ since,
+ end_date - timedelta(days=SIMPLEFIN_DEFAULT_HISTORY_DAYS - 1),
+ )
# Walk the request window in 90-day chunks (SimpleFIN's per-call cap).
transactions: list[TransactionData] = []
seen_ids: set[str] = set()
cursor = start_date
- while cursor <= end_date:
- chunk_end = min(cursor + timedelta(days=SIMPLEFIN_MAX_WINDOW_DAYS), end_date)
+ end_exclusive = end_date + timedelta(days=1)
+ while cursor < end_exclusive:
+ chunk_end = min(
+ cursor + timedelta(days=SIMPLEFIN_MAX_WINDOW_DAYS),
+ end_exclusive,
+ )
payload = await self._fetch_accounts(
credentials,
start_date=cursor,
- end_date=chunk_end + timedelta(days=1),
+ end_date=chunk_end,
account_id=account_external_id,
pending=True,
)
- _surface_errors(payload.get("errlist") or [], context="get_transactions")
- for raw_acc in payload.get("accounts") or []:
- if str(raw_acc.get("id") or "") != account_external_id:
- continue
+ matching_accounts = [
+ raw_acc for raw_acc in payload.get("accounts") or []
+ if str(raw_acc.get("id") or "") == account_external_id
+ ]
+ warnings = _surface_errors(
+ payload.get("errlist") or [],
+ context="get_transactions",
+ has_usable_data=bool(matching_accounts),
+ )
+ self.action_required_warnings.extend(warnings)
+ for raw_acc in matching_accounts:
for raw_txn in raw_acc.get("transactions") or []:
parsed = self._build_transaction(raw_txn, payee_source)
if parsed and parsed.external_id not in seen_ids:
seen_ids.add(parsed.external_id)
transactions.append(parsed)
- cursor = chunk_end + timedelta(days=1)
+ cursor = chunk_end
return transactions
@staticmethod
@@ -524,7 +553,12 @@ class SimpleFinProvider(BankProvider):
async def get_holdings(self, credentials: dict) -> list[HoldingData]:
payload = await self._fetch_accounts(credentials, pending=False)
- _surface_errors(payload.get("errlist") or [], context="get_holdings")
+ warnings = _surface_errors(
+ payload.get("errlist") or [],
+ context="get_holdings",
+ has_usable_data=bool(payload.get("accounts") or []),
+ )
+ self.action_required_warnings.extend(warnings)
holdings: list[HoldingData] = []
for raw_acc in payload.get("accounts") or []:
acc_currency = raw_acc.get("currency") or "USD"
diff --git a/backend/app/services/connection_service.py b/backend/app/services/connection_service.py
index 9263e635..efd0c6f4 100644
--- a/backend/app/services/connection_service.py
+++ b/backend/app/services/connection_service.py
@@ -52,9 +52,12 @@ from app.services.rule_service import apply_rules_to_transaction, preview_rules_
from app.services.transfer_detection_service import detect_transfer_pairs
from app.services.fx_rate_service import stamp_primary_amount
from app.services.payee_service import get_or_create_payee
+from app.services.transaction_match_service import find_unique_transaction_match
logger = logging.getLogger(__name__)
+LOCAL_IMPORT_SOURCES = {"import", "csv", "ofx", "qif", "camt"}
+
settings = get_settings()
_PROVIDER_SELL_DATE_METADATA_KEY = "_securo_provider_sell_date"
@@ -895,13 +898,20 @@ async def get_connections(session: AsyncSession, workspace_id: uuid.UUID) -> lis
async def get_connection(
- session: AsyncSession, connection_id: uuid.UUID, workspace_id: uuid.UUID
+ session: AsyncSession,
+ connection_id: uuid.UUID,
+ workspace_id: uuid.UUID,
+ *,
+ for_update: bool = False,
) -> Optional[BankConnection]:
- result = await session.execute(
+ statement = (
select(BankConnection)
.where(BankConnection.id == connection_id, BankConnection.workspace_id == workspace_id)
.options(selectinload(BankConnection.accounts))
)
+ if for_update:
+ statement = statement.with_for_update()
+ result = await session.execute(statement)
return result.scalar_one_or_none()
@@ -1133,6 +1143,7 @@ async def handle_oauth_callback(
transactions_data = await provider.get_transactions(
connection_data.credentials, acc_data.external_id, None
)
+ incoming_external_ids = {txn.external_id for txn in transactions_data}
seen_external_ids: set[str] = set()
for txn_data in transactions_data:
# Providers should return each external transaction once, but an
@@ -1147,14 +1158,19 @@ async def handle_oauth_callback(
# When the same logical operation comes back under a new external
# id with a different status, fingerprint match prevents the
# second copy from landing.
- synced_dup = await _find_synced_duplicate(session, account.id, txn_data)
+ synced_dup = await _find_synced_duplicate(
+ session, account.id, txn_data, incoming_external_ids
+ )
if synced_dup:
if synced_dup.original_description is None:
synced_dup.original_description = txn_data.description
if synced_dup.status == "pending" and txn_data.status == "posted":
synced_dup.status = "posted"
- synced_dup.external_id = txn_data.external_id
- synced_dup.raw_data = txn_data.raw_data
+ _merge_sync_metadata(
+ synced_dup,
+ txn_data,
+ replace_external_id=synced_dup.source == "sync",
+ )
if (
txn_data.bill_external_id
and synced_dup.effective_bill_date is None
@@ -1165,6 +1181,12 @@ async def handle_oauth_callback(
apply_effective_date(
synced_dup, account, bill_due_date=bill.due_date
)
+ elif synced_dup.external_id != txn_data.external_id:
+ _merge_sync_metadata(
+ synced_dup,
+ txn_data,
+ replace_external_id=synced_dup.source == "sync",
+ )
continue
category_id = await _find_installment_category(
@@ -1270,6 +1292,80 @@ async def handle_oauth_callback(
return connection
+def _normalized_account_name(name: str | None) -> str:
+ """Normalize provider account names for fallback matching.
+
+ Some SimpleFIN bridges re-key account ids, but the display name / masked
+ suffix stays stable (e.g. "High Yield Savings Account (9402)").
+ """
+ return " ".join((name or "").casefold().split())
+
+
+async def _find_existing_connected_account(
+ session: AsyncSession,
+ connection: BankConnection,
+ acc_data: AccountData,
+ institution: Optional[Institution],
+ incoming_external_ids: set[str],
+) -> Optional[Account]:
+ """Find an existing account for incoming provider account data.
+
+ Primary identity is provider external_id. For SimpleFIN only, fall back to
+ stable account name + currency because some bridges emit fresh UUID-like
+ account ids on each pull; when that happens, blindly keying by id creates a
+ new app account on every sync.
+ """
+ result = await session.execute(
+ select(Account).where(
+ Account.connection_id == connection.id,
+ Account.external_id == acc_data.external_id,
+ )
+ )
+ account = result.scalar_one_or_none()
+ if account or connection.provider != "simplefin":
+ return account
+
+ normalized_name = _normalized_account_name(acc_data.name)
+ if not normalized_name:
+ return None
+
+ candidates_result = await session.execute(
+ select(Account).where(
+ Account.connection_id == connection.id,
+ Account.currency == acc_data.currency,
+ )
+ )
+ candidates = [
+ candidate
+ for candidate in candidates_result.scalars().all()
+ if candidate.external_id not in incoming_external_ids
+ and _normalized_account_name(candidate.name) == normalized_name
+ ]
+ if institution is not None:
+ matched_institution = [
+ candidate for candidate in candidates
+ if candidate.institution_id == institution.id
+ ]
+ if matched_institution:
+ candidates = matched_institution
+ else:
+ candidates = [
+ candidate for candidate in candidates
+ if candidate.institution_id is None
+ ]
+ else:
+ candidates = [
+ candidate for candidate in candidates
+ if candidate.institution_id is None
+ ]
+ if len(candidates) != 1:
+ return None
+
+ account = candidates[0]
+ account.external_id = acc_data.external_id
+ return account
+
+
async def _fuzzy_match_manual(
session: AsyncSession,
account_id: uuid.UUID,
@@ -1310,12 +1406,41 @@ async def _fuzzy_match_manual(
return None
+def _merge_sync_metadata(
+ transaction: Transaction,
+ txn_data,
+ *,
+ replace_external_id: bool = False,
+) -> None:
+ if transaction.source == "sync" and transaction.date.year <= 1970:
+ transaction.date = txn_data.date
+ if transaction.effective_date.year <= 1970:
+ transaction.effective_date = txn_data.date
+ if transaction.source in LOCAL_IMPORT_SOURCES:
+ transaction.import_id = None
+ replace_external_id = True
+ if txn_data.external_id and (replace_external_id or not transaction.external_id):
+ transaction.external_id = txn_data.external_id
+ if txn_data.raw_data:
+ if not transaction.raw_data:
+ transaction.raw_data = txn_data.raw_data
+ elif isinstance(transaction.raw_data, dict) and isinstance(txn_data.raw_data, dict):
+ merged = {**transaction.raw_data, **txn_data.raw_data}
+ if merged != transaction.raw_data:
+ transaction.raw_data = merged
+ if transaction.original_description is None:
+ transaction.original_description = txn_data.description
+ if txn_data.payee and not transaction.payee:
+ transaction.payee = txn_data.payee
+
+
async def _find_synced_duplicate(
session: AsyncSession,
account_id: uuid.UUID,
txn_data,
+ incoming_external_ids: set[str],
) -> Optional[Transaction]:
- """Find an existing synced row that the incoming `txn_data` is a twin of.
+ """Find an existing row that the incoming `txn_data` is a twin of.
The `(account_id, external_id)` lookup only catches the case where a
provider keeps the same id while a row's `status` flips pending→posted.
@@ -1331,8 +1456,8 @@ async def _find_synced_duplicate(
`(purchase_date, number, total, amount, type)`.
Returns the existing Transaction the caller should reuse; the caller
- decides whether to upgrade its status (pending→posted + swap external_id)
- or skip the incoming insert. Synthetic bill-charge rows
+ decides whether to upgrade its status (pending→posted + swap external_id),
+ enrich an imported row, or skip the incoming insert. Synthetic bill-charge rows
(`bill_charge:*`) are excluded — they have their own idempotency keys.
"""
# Path 1: installment fingerprint. Highly specific, so we don't require a
@@ -1385,7 +1510,68 @@ async def _find_synced_duplicate(
) >= 0.7:
return candidate
- return None
+ # Path 3: exact posted/transacted timestamp fingerprint. Some SimpleFIN
+ # bridges re-key already-posted rows on later pulls, so status does not
+ # differ. The raw bank timestamps plus same account/date/amount/type and a
+ # near-identical description are specific enough to collapse the re-keyed
+ # row while avoiding broad same-day/same-amount merchant dedupe.
+ raw = txn_data.raw_data if isinstance(txn_data.raw_data, dict) else {}
+ posted = raw.get("posted")
+ transacted_at = raw.get("transacted_at")
+ if posted is not None or transacted_at is not None:
+ result = await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ or_(
+ Transaction.source == "sync",
+ (
+ Transaction.source.in_(LOCAL_IMPORT_SOURCES)
+ & Transaction.raw_data.is_not(None)
+ ),
+ ),
+ Transaction.date >= txn_data.date - timedelta(days=3),
+ Transaction.date <= txn_data.date + timedelta(days=3),
+ Transaction.amount == txn_data.amount,
+ Transaction.type == txn_data.type,
+ Transaction.status == txn_data.status,
+ Transaction.external_id != txn_data.external_id,
+ )
+ )
+ for candidate in result.scalars():
+ if candidate.external_id and candidate.external_id.startswith("bill_charge:"):
+ continue
+ if candidate.external_id in incoming_external_ids:
+ continue
+ candidate_raw = candidate.raw_data if isinstance(candidate.raw_data, dict) else {}
+ candidate_descriptions = (
+ candidate_raw.get("description"), candidate.payee, candidate.description,
+ )
+ incoming_descriptions = (
+ raw.get("description"), txn_data.payee, txn_data.description,
+ )
+ description_matches = any(
+ left and right and token_overlap(left, right) >= 0.9
+ for left in candidate_descriptions
+ for right in incoming_descriptions
+ )
+ if (
+ candidate_raw.get("posted") == posted
+ and candidate_raw.get("transacted_at") == transacted_at
+ and description_matches
+ ):
+ return candidate
+
+ # Path 5: local import history from before the account was connected.
+ # Posting lag and statement exports can shift the date or shorten the
+ # merchant description, so accept only one exact normalized merchant/payee.
+ return await find_unique_transaction_match(
+ session,
+ account_id,
+ txn_data,
+ LOCAL_IMPORT_SOURCES,
+ unclaimed_only=True,
+ exclude_external_ids=incoming_external_ids,
+ )
async def _cleanup_phantom_duplicates(
@@ -1689,7 +1875,9 @@ async def sync_connection(
requesting_user_id: uuid.UUID,
trigger_provider_refresh: bool = False,
) -> tuple[BankConnection, int]:
- connection = await get_connection(session, connection_id, workspace_id)
+ connection = await get_connection(
+ session, connection_id, workspace_id, for_update=True
+ )
if not connection:
raise ValueError("Connection not found")
if not connection.credentials:
@@ -1750,14 +1938,19 @@ async def sync_connection(
# read. Providers that don't expose an on-demand refresh return
# "skipped" via the default implementation and we proceed normally.
if trigger_provider_refresh:
+ connection.settings = {
+ **conn_settings,
+ "last_provider_refresh_at": datetime.now(timezone.utc).isoformat(),
+ }
outcome = await provider.trigger_refresh(credentials)
if outcome == "needs_user_action":
# Surfacing reconnect immediately is better than silently
# reading stale data the user knows is stale.
connection.status = "error"
await session.commit()
- raise RuntimeError(
- "Provider needs the user to reconnect before fetching fresh data"
+ raise ProviderUserActionRequired(
+ "Provider needs the user to reconnect before fetching fresh data",
+ code="credentials_invalid",
)
# "refreshed", "skipped", or "failed" all fall through to a read.
# On "failed" we read whatever cached copy the provider has —
@@ -1769,15 +1962,19 @@ async def sync_connection(
new_tx_ids: list[uuid.UUID] = []
merged_count = 0
accounts_data = await provider.get_accounts(credentials)
+ incoming_external_ids = {acc.external_id for acc in accounts_data}
institution_cache: dict[str, Institution] = {}
for acc_data in accounts_data:
- result = await session.execute(
- select(Account).where(
- Account.connection_id == connection.id,
- Account.external_id == acc_data.external_id,
- )
+ institution = await _resolve_institution(
+ session, connection.id, institution_cache, acc_data
+ )
+ account = await _find_existing_connected_account(
+ session,
+ connection,
+ acc_data,
+ institution,
+ incoming_external_ids,
)
- account = result.scalar_one_or_none()
if account is not None:
account.user_id = user_id
@@ -1796,10 +1993,6 @@ async def sync_connection(
.values(user_id=user_id, workspace_id=connection.workspace_id)
)
- institution = await _resolve_institution(
- session, connection.id, institution_cache, acc_data
- )
-
# Honor user intent: a closed connected account stays closed and is
# not touched by sync. The row is left alone (no balance/name
# rewrite, no new transactions) but the connection link is kept so
@@ -1860,6 +2053,7 @@ async def sync_connection(
is_cc = acc_data.type == "credit_card"
account = Account(
user_id=user_id,
+ workspace_id=workspace_id,
connection_id=connection.id,
external_id=acc_data.external_id,
name=acc_data.name,
@@ -1906,6 +2100,7 @@ async def sync_connection(
if not import_pending:
transactions_data = [t for t in transactions_data if t.status != "pending"]
+ incoming_external_ids = {txn.external_id for txn in transactions_data}
for txn_data in transactions_data:
existing = await session.execute(
select(Transaction)
@@ -1928,8 +2123,7 @@ async def sync_connection(
# a re-sync can't revive a transaction the user hid.
if existing_tx.is_ignored:
continue
- if existing_tx.original_description is None:
- existing_tx.original_description = txn_data.description
+ _merge_sync_metadata(existing_tx, txn_data)
if existing_tx.status == "pending" and txn_data.status == "posted":
existing_tx.status = "posted"
# Self-heal bill linkage: a tx that pre-dates the bills
@@ -1959,13 +2153,8 @@ async def sync_connection(
if fuzzy_match:
if fuzzy_match.is_ignored:
continue
- fuzzy_match.external_id = txn_data.external_id
+ _merge_sync_metadata(fuzzy_match, txn_data)
fuzzy_match.source = "sync"
- fuzzy_match.raw_data = txn_data.raw_data
- if fuzzy_match.original_description is None:
- fuzzy_match.original_description = txn_data.description
- if not fuzzy_match.payee and txn_data.payee:
- fuzzy_match.payee = txn_data.payee
merged_count += 1
continue
@@ -1975,7 +2164,7 @@ async def sync_connection(
# status, fingerprint match collapses it instead of letting
# both rows land.
synced_dup = await _find_synced_duplicate(
- session, account.id, txn_data
+ session, account.id, txn_data, incoming_external_ids
)
if synced_dup:
if synced_dup.original_description is None:
@@ -1984,8 +2173,11 @@ async def sync_connection(
# Posted truth wins: swap in the new id so subsequent
# syncs match by external_id and update raw_data.
synced_dup.status = "posted"
- synced_dup.external_id = txn_data.external_id
- synced_dup.raw_data = txn_data.raw_data
+ _merge_sync_metadata(
+ synced_dup,
+ txn_data,
+ replace_external_id=synced_dup.source == "sync",
+ )
if (
txn_data.bill_external_id
and synced_dup.effective_bill_date is None
@@ -1996,6 +2188,15 @@ async def sync_connection(
apply_effective_date(
synced_dup, account, bill_due_date=bill.due_date
)
+ elif synced_dup.external_id != txn_data.external_id:
+ # Same logical posted row re-keyed by a provider such
+ # as SimpleFIN. Keep the user's row and move its
+ # idempotency key forward instead of inserting a twin.
+ _merge_sync_metadata(
+ synced_dup,
+ txn_data,
+ replace_external_id=synced_dup.source == "sync",
+ )
continue
incoming_currency = (
@@ -2199,7 +2400,17 @@ async def sync_connection(
await session.delete(orphan)
connection.last_sync_at = datetime.now(timezone.utc)
- connection.status = "active"
+ action_required_warnings = getattr(provider, "action_required_warnings", None)
+ if isinstance(action_required_warnings, list) and action_required_warnings:
+ logger.warning(
+ "Provider %s synced with %d user-action warning(s) for connection %s",
+ connection.provider,
+ len(action_required_warnings),
+ connection.id,
+ )
+ connection.status = "error"
+ else:
+ connection.status = "active"
await session.commit()
await session.refresh(connection)
return connection, merged_count
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.py
index 438ca6d1..45071117 100644
--- a/backend/app/services/import_service.py
+++ b/backend/app/services/import_service.py
@@ -15,6 +15,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import get_settings
from app.models.account import Account
+from app.models.bank_connection import BankConnection
from app.models.category import Category
from app.models.rule import Rule
from app.models.transaction import Transaction
@@ -26,6 +27,7 @@ from app.services.rule_engine import apply_rule_actions, evaluate_conditions, me
from app.services.rule_service import apply_rules_to_transaction, preview_rules_for_transaction
from app.services.fx_rate_service import stamp_primary_amount
from app.services.payee_service import get_or_create_payee
+from app.services.transaction_match_service import find_unique_transaction_match
# Descriptions used by some Brazilian banks (e.g. Banco do Brasil) for
@@ -743,10 +745,19 @@ async def import_transactions(
await session.flush() # Get the import_log.id
# Look up account currency for fallback
- account_result = await session.execute(
- select(Account).where(Account.id == account_id)
- )
+ account_result = await session.execute(select(Account).where(Account.id == account_id))
account = account_result.scalar_one_or_none()
+ if account and account.connection_id:
+ await session.execute(
+ select(BankConnection.id)
+ .where(BankConnection.id == account.connection_id)
+ .with_for_update()
+ )
+ if account:
+ account_result = await session.execute(
+ select(Account).where(Account.id == account_id).with_for_update()
+ )
+ account = account_result.scalar_one()
account_currency = account.currency if account else get_settings().default_currency
# Build category name → id map scoped to the workspace, on the same terms
@@ -767,6 +778,7 @@ async def import_transactions(
imported = 0
landed: list[Transaction] = []
skipped = 0
+ matched_existing_ids: set[uuid.UUID] = set()
effective_format = (detected_format or source or "").lower()
should_detect_duplicates = detect_duplicates if effective_format == "csv" else True
@@ -780,30 +792,45 @@ async def import_transactions(
# Formats without unique IDs fall back to transaction fields; compare
# both descriptions because rules may have changed the displayed one.
if txn_data.external_id:
- existing = await session.execute(
- select(Transaction).where(
- Transaction.account_id == account_id,
- Transaction.external_id == txn_data.external_id,
- Transaction.date == txn_data.date,
- )
+ existing_statement = select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.external_id == txn_data.external_id,
+ Transaction.date == txn_data.date,
)
else:
- existing = await session.execute(
- select(Transaction).where(
- Transaction.account_id == account_id,
- Transaction.date == txn_data.date,
- Transaction.amount == txn_data.amount,
- Transaction.type == txn_data.type,
- or_(
- Transaction.description == txn_data.description,
- Transaction.original_description == txn_data.description,
- ),
- )
+ existing_statement = select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.date == txn_data.date,
+ Transaction.amount == txn_data.amount,
+ Transaction.type == txn_data.type,
+ or_(
+ Transaction.description == txn_data.description,
+ Transaction.original_description == txn_data.description,
+ ),
+ )
+ # `.first()` rather than `.scalar_one_or_none()`: the dedup key can
+ # legitimately match more than one row (e.g. a prior sync/import race
+ # left a duplicate, or a bank reuses one FITID across statements),
+ # and we only need to know whether *any* match exists. Requiring
+ # exactly one would raise MultipleResultsFound and abort the import.
+ if matched_existing_ids and not txn_data.external_id:
+ existing_statement = existing_statement.where(
+ Transaction.id.not_in(matched_existing_ids)
+ )
+ existing = await session.execute(
+ existing_statement.order_by(Transaction.created_at, Transaction.id)
+ )
+ duplicate = existing.scalars().first()
+ if not duplicate:
+ duplicate = await find_unique_transaction_match(
+ session,
+ account_id,
+ txn_data,
+ {"sync"},
+ exclude_ids=matched_existing_ids,
)
- # `.first()` is intentional: duplicate keys can legitimately match
- # multiple rows after an import/sync race or reused bank identifier,
- # and duplicate detection only needs to establish that any row exists.
- if existing.scalars().first() is not None:
+ if duplicate:
+ matched_existing_ids.add(duplicate.id)
skipped += 1
continue
@@ -913,6 +940,8 @@ async def import_transactions(
session.add(incoming)
await session.flush()
+ if should_detect_duplicates and not txn_data.external_id:
+ matched_existing_ids.add(incoming.id)
if recurring_link is not None:
recurring_match_service.advance_past(recurring_link, txn_data.date)
diff --git a/backend/app/services/transaction_match_service.py b/backend/app/services/transaction_match_service.py
new file mode 100644
index 00000000..2e62fe69
--- /dev/null
+++ b/backend/app/services/transaction_match_service.py
@@ -0,0 +1,102 @@
+import re
+import uuid
+from collections.abc import Collection
+from datetime import timedelta
+from typing import Optional
+
+from sqlalchemy import or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.transaction import Transaction
+
+
+def _normalize_description(value: str | None) -> str:
+ return " ".join(re.sub(r"[^\w]+", " ", (value or "").casefold()).split())
+
+
+def descriptions_match(a: str | None, b: str | None) -> bool:
+ """Match only normalized merchant text; never infer from shared fragments."""
+ normalized = [_normalize_description(value) for value in (a, b)]
+ return bool(normalized[0]) and normalized[0] == normalized[1]
+
+
+async def find_unique_transaction_match(
+ session: AsyncSession,
+ account_id: uuid.UUID,
+ txn_data,
+ sources: Collection[str],
+ *,
+ date_tolerance_days: int = 3,
+ unclaimed_only: bool = False,
+ exclude_ids: Collection[uuid.UUID] = (),
+ exclude_external_ids: Collection[str] = (),
+) -> Optional[Transaction]:
+ """Return one unambiguous imported/provider counterpart, never a guess."""
+ date_lo = txn_data.date - timedelta(days=date_tolerance_days)
+ date_hi = txn_data.date + timedelta(days=date_tolerance_days)
+ statement = select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.source.in_(sources),
+ Transaction.amount == txn_data.amount,
+ Transaction.type == txn_data.type,
+ Transaction.date >= date_lo,
+ Transaction.date <= date_hi,
+ )
+ if unclaimed_only:
+ statement = statement.where(Transaction.raw_data.is_(None))
+ if exclude_ids:
+ statement = statement.where(Transaction.id.not_in(exclude_ids))
+ if exclude_external_ids:
+ statement = statement.where(
+ or_(
+ Transaction.external_id.is_(None),
+ Transaction.external_id.not_in(exclude_external_ids),
+ )
+ )
+ result = await session.execute(statement)
+
+ incoming_descriptions = (
+ txn_data.description,
+ getattr(txn_data, "payee", None),
+ getattr(txn_data, "payee_raw", None),
+ )
+ matches = [
+ candidate
+ for candidate in result.scalars().all()
+ if any(
+ descriptions_match(candidate_description, incoming_description)
+ for candidate_description in (
+ candidate.description,
+ candidate.original_description,
+ candidate.payee,
+ )
+ for incoming_description in incoming_descriptions
+ )
+ ]
+ if len(matches) <= 1:
+ return matches[0] if matches else None
+
+ incoming_exact = {
+ normalized
+ for value in incoming_descriptions
+ if (normalized := _normalize_description(value))
+ }
+ exact_matches = [
+ candidate
+ for candidate in matches
+ if incoming_exact
+ & {
+ normalized
+ for value in (
+ candidate.description,
+ candidate.original_description,
+ candidate.payee,
+ )
+ if (normalized := _normalize_description(value))
+ }
+ ]
+ if exact_matches:
+ # Indistinguishable exact candidates form a multiset. Consume one
+ # deterministically; the caller's exclusions prevent reusing it.
+ return min(exact_matches, key=lambda candidate: str(candidate.id))
+ return None
diff --git a/backend/app/tasks/sync_tasks.py b/backend/app/tasks/sync_tasks.py
index f562e872..96a4ca8e 100644
--- a/backend/app/tasks/sync_tasks.py
+++ b/backend/app/tasks/sync_tasks.py
@@ -15,6 +15,20 @@ from app.services import connection_service
logger = logging.getLogger(__name__)
STALE_THRESHOLD = timedelta(hours=4)
+PROVIDER_REFRESH_THRESHOLD = timedelta(hours=20)
+
+
+def _should_trigger_provider_refresh(settings: dict | None, now: datetime) -> bool:
+ value = (settings or {}).get("last_provider_refresh_at")
+ if not isinstance(value, str):
+ return True
+ try:
+ last_refresh = datetime.fromisoformat(value)
+ except ValueError:
+ return True
+ if last_refresh.tzinfo is None:
+ return True
+ return now - last_refresh >= PROVIDER_REFRESH_THRESHOLD
def _make_session_maker():
@@ -33,7 +47,10 @@ async def _sync_all() -> int:
async with session_maker() as session:
result = await session.execute(
select(
- BankConnection.id, BankConnection.user_id, BankConnection.last_sync_at
+ BankConnection.id,
+ BankConnection.user_id,
+ BankConnection.last_sync_at,
+ BankConnection.settings,
).where(
BankConnection.status.in_(["active", "error"]),
(BankConnection.last_sync_at < cutoff)
@@ -48,10 +65,17 @@ async def _sync_all() -> int:
cutoff.isoformat(),
)
- for conn_id, user_id, last_sync in connections:
+ for conn_id, user_id, last_sync, settings in connections:
try:
logger.info("Syncing connection %s (last_sync=%s)", conn_id, last_sync)
- await _sync_one(session_maker, conn_id, user_id)
+ await _sync_one(
+ session_maker,
+ conn_id,
+ user_id,
+ trigger_provider_refresh=_should_trigger_provider_refresh(
+ settings, datetime.now(timezone.utc)
+ ),
+ )
synced += 1
except ProviderNotConfiguredError as exc:
# Actionable one-liner instead of a buried traceback: this
@@ -65,7 +89,13 @@ async def _sync_all() -> int:
await engine.dispose()
-async def _sync_one(session_maker, connection_id: uuid.UUID, user_id: uuid.UUID) -> None:
+async def _sync_one(
+ session_maker,
+ connection_id: uuid.UUID,
+ user_id: uuid.UUID,
+ *,
+ trigger_provider_refresh: bool = False,
+) -> None:
"""Sync a single connection. Error status is set by sync_connection itself."""
async with session_maker() as session:
workspace_id = await session.scalar(
@@ -75,7 +105,11 @@ async def _sync_one(session_maker, connection_id: uuid.UUID, user_id: uuid.UUID)
logger.warning("Connection %s has no workspace; skipping sync", connection_id)
return
await connection_service.sync_connection(
- session, connection_id, workspace_id, user_id
+ session,
+ connection_id,
+ workspace_id,
+ user_id,
+ trigger_provider_refresh=trigger_provider_refresh,
)
diff --git a/backend/tests/test_connection_service.py b/backend/tests/test_connection_service.py
index c3bcc097..31e82215 100644
--- a/backend/tests/test_connection_service.py
+++ b/backend/tests/test_connection_service.py
@@ -1,16 +1,19 @@
import uuid
from datetime import date, datetime, timezone
from decimal import Decimal
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlalchemy import select
+from sqlalchemy.dialects import postgresql
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.account import Account
from app.models.asset import Asset
from app.models.bank_connection import BankConnection
from app.models.category import Category
+from app.models.import_log import ImportLog
+from app.models.institution import Institution
from app.models.transaction import Transaction
from app.schemas.rule import RuleAction, RuleCondition, RuleCreate
from app.providers.base import (
@@ -24,6 +27,8 @@ from app.providers.base import (
)
from app.services.text_similarity import token_overlap
from app.services.connection_service import (
+ _find_existing_connected_account,
+ _merge_sync_metadata,
_match_pluggy_category,
create_connect_token,
delete_connection,
@@ -90,6 +95,62 @@ def test_description_similarity_no_overlap():
assert token_overlap("abc", "xyz") == 0.0
+def test_merge_sync_metadata_repairs_invalid_epoch_date():
+ tx = Transaction(
+ id=uuid.uuid4(),
+ user_id=uuid.uuid4(),
+ account_id=uuid.uuid4(),
+ description="Pending",
+ amount=Decimal("5"),
+ date=date(1970, 1, 1),
+ effective_date=date(1970, 1, 1),
+ type="debit",
+ source="sync",
+ status="pending",
+ )
+ txn_data = TransactionData(
+ external_id="provider-id",
+ description="Pending",
+ amount=Decimal("5"),
+ date=date(2026, 6, 27),
+ type="debit",
+ )
+
+ _merge_sync_metadata(tx, txn_data)
+
+ assert tx.date == date(2026, 6, 27)
+ assert tx.effective_date == date(2026, 6, 27)
+
+
+def test_merge_sync_metadata_rekey_preserves_old_raw_fields():
+ tx = Transaction(
+ id=uuid.uuid4(),
+ user_id=uuid.uuid4(),
+ account_id=uuid.uuid4(),
+ external_id="old-id",
+ description="Coffee",
+ amount=Decimal("5"),
+ date=date(2026, 6, 27),
+ type="debit",
+ source="sync",
+ status="posted",
+ raw_data={"posted": 1, "old_only": True},
+ )
+ txn_data = TransactionData(
+ external_id="new-id",
+ description="Coffee",
+ amount=Decimal("5"),
+ date=date(2026, 6, 27),
+ type="debit",
+ raw_data={"posted": 2, "new_only": True},
+ )
+
+ _merge_sync_metadata(tx, txn_data, replace_external_id=True)
+
+ assert tx.external_id == "new-id"
+ assert tx.raw_data == {"posted": 2, "old_only": True, "new_only": True}
+
+
def test_description_similarity_none():
assert token_overlap(None, "hello") == 0.0
assert token_overlap("hello", None) == 0.0
@@ -764,6 +825,472 @@ async def test_sync_connection_new_transactions(session: AsyncSession, test_user
assert transaction.original_description == "GROCERY"
+async def test_sync_connection_partial_user_action_warning_marks_connection_error(
+ session: AsyncSession, test_user, test_workspace,
+):
+ conn = await _make_connection(session, test_user.id, "Partial SimpleFIN")
+ mock_provider = AsyncMock()
+ mock_provider.action_required_warnings = [
+ {"code": "credentials_invalid", "message": "Apple Card auth required"}
+ ]
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "refreshed"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="partial-acc-1", name="Checking",
+ type="checking", balance=Decimal("2000"), currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="partial-tx-1", description="GROCERY",
+ amount=Decimal("80"), date=date.today(), type="debit", currency="BRL",
+ ),
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ result_conn, merged = await sync_connection(session, conn.id, test_workspace.id, test_user.id)
+
+ assert result_conn.status == "error"
+ assert merged == 0
+ synced_tx = (await session.execute(
+ select(Transaction).where(Transaction.external_id == "partial-tx-1")
+ )).scalar_one()
+ assert synced_tx.description == "GROCERY"
+
+
+async def test_sync_connection_simplefin_rekey_does_not_duplicate_account_or_tx(
+ session: AsyncSession, test_user, test_workspace,
+):
+ conn = await _make_connection(session, test_user.id, "SimpleFIN Bank")
+ conn.provider = "simplefin"
+
+ account = Account(
+ id=uuid.uuid4(),
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ connection_id=conn.id,
+ external_id="old-account-id",
+ name="High Yield Savings Account (9402)",
+ display_name="Albert · Amex HYSA",
+ type="savings",
+ balance=Decimal("100.00"),
+ currency="USD",
+ )
+ session.add(account)
+ existing = Transaction(
+ id=uuid.uuid4(),
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ account_id=account.id,
+ external_id="old-tx-id",
+ description="ROBINHOOD FUNDS",
+ amount=Decimal("3086.00"),
+ date=date(2026, 6, 16),
+ type="credit",
+ currency="USD",
+ source="sync",
+ status="posted",
+ raw_data={"posted": 1781618248, "transacted_at": 1781611200},
+ created_at=datetime.now(timezone.utc),
+ )
+ session.add(existing)
+ await session.commit()
+
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "refreshed"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="new-account-id",
+ name="High Yield Savings Account (9402)",
+ type="checking",
+ balance=Decimal("100.00"),
+ currency="USD",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="new-tx-id",
+ description="ROBINHOOD FUNDS",
+ amount=Decimal("3086.00"),
+ date=date(2026, 6, 16),
+ type="credit",
+ currency="USD",
+ status="posted",
+ payee="Robinhood",
+ raw_data={"posted": 1781618248, "transacted_at": 1781611200},
+ ),
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ result_conn, merged = await sync_connection(session, conn.id, test_workspace.id, test_user.id)
+
+ assert result_conn.status == "active"
+ assert merged == 0
+
+ accounts = (await session.execute(
+ select(Account).where(Account.connection_id == conn.id)
+ )).scalars().all()
+ assert len(accounts) == 1
+ assert accounts[0].id == account.id
+ assert accounts[0].external_id == "new-account-id"
+ assert accounts[0].display_name == "Albert · Amex HYSA"
+ assert accounts[0].type == "savings"
+
+ txs = (await session.execute(
+ select(Transaction).where(Transaction.account_id == account.id)
+ )).scalars().all()
+ sync_txs = [tx for tx in txs if tx.source == "sync"]
+ assert len(sync_txs) == 1
+ assert sync_txs[0].id == existing.id
+ assert sync_txs[0].external_id == "new-tx-id"
+
+
+async def test_simplefin_rekey_respects_institution_and_closed_account(
+ session: AsyncSession, test_user, test_workspace,
+):
+ conn = await _make_connection(session, test_user.id, "SimpleFIN")
+ conn.provider = "simplefin"
+ bank_a = Institution(
+ connection_id=conn.id, external_id="bank-a", name="Bank A"
+ )
+ bank_b = Institution(
+ connection_id=conn.id, external_id="bank-b", name="Bank B"
+ )
+ session.add_all([bank_a, bank_b])
+ await session.flush()
+ closed_bank_a = Account(
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ connection_id=conn.id,
+ institution_id=bank_a.id,
+ external_id="old-bank-a",
+ name="Checking",
+ type="checking",
+ balance=Decimal("10"),
+ currency="USD",
+ is_closed=True,
+ )
+ customized_bank_b = Account(
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ connection_id=conn.id,
+ institution_id=bank_b.id,
+ external_id="old-bank-b",
+ name="Checking",
+ display_name="My Bank B",
+ type="checking",
+ balance=Decimal("20"),
+ currency="USD",
+ )
+ session.add_all([closed_bank_a, customized_bank_b])
+ await session.flush()
+
+ matched = await _find_existing_connected_account(
+ session,
+ conn,
+ AccountData(
+ external_id="new-bank-a",
+ name="Checking",
+ type="checking",
+ balance=Decimal("10"),
+ currency="USD",
+ institution_external_id="bank-a",
+ institution_name="Bank A",
+ ),
+ bank_a,
+ {"new-bank-a"},
+ )
+
+ assert matched is closed_bank_a
+ assert closed_bank_a.external_id == "new-bank-a"
+ assert customized_bank_b.external_id == "old-bank-b"
+
+
+async def test_simplefin_rekey_reserves_ids_from_later_accounts(
+ session: AsyncSession, test_user, test_workspace,
+):
+ conn = await _make_connection(session, test_user.id, "SimpleFIN")
+ conn.provider = "simplefin"
+ later_account = Account(
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ connection_id=conn.id,
+ external_id="later-account-id",
+ name="Checking",
+ type="checking",
+ balance=Decimal("10"),
+ currency="USD",
+ )
+ session.add(later_account)
+ await session.flush()
+
+ matched = await _find_existing_connected_account(
+ session,
+ conn,
+ AccountData(
+ external_id="new-account-id",
+ name="Checking",
+ type="checking",
+ balance=Decimal("10"),
+ currency="USD",
+ ),
+ None,
+ {"new-account-id", "later-account-id"},
+ )
+
+ assert matched is None
+ assert later_account.external_id == "later-account-id"
+
+
+async def test_simplefin_rekey_refuses_same_institution_ambiguity(
+ session: AsyncSession, test_user, test_workspace,
+):
+ conn = await _make_connection(session, test_user.id, "SimpleFIN")
+ conn.provider = "simplefin"
+ institution = Institution(
+ connection_id=conn.id, external_id="bank-a", name="Bank A"
+ )
+ session.add(institution)
+ await session.flush()
+ accounts = [
+ Account(
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ connection_id=conn.id,
+ institution_id=institution.id,
+ external_id=f"old-{index}",
+ name="Checking",
+ type="checking",
+ balance=Decimal("10"),
+ currency="USD",
+ )
+ for index in range(2)
+ ]
+ session.add_all(accounts)
+ await session.flush()
+
+ matched = await _find_existing_connected_account(
+ session,
+ conn,
+ AccountData(
+ external_id="rotated-id",
+ name="Checking",
+ type="checking",
+ balance=Decimal("10"),
+ currency="USD",
+ institution_external_id="bank-a",
+ institution_name="Bank A",
+ ),
+ institution,
+ {"rotated-id"},
+ )
+
+ assert matched is None
+ assert {account.external_id for account in accounts} == {"old-0", "old-1"}
+
+
+async def test_sync_upserts_matching_csv_import_without_overwriting_user_fields(
+ session: AsyncSession, test_user, test_workspace,
+):
+ conn = await _make_connection(session, test_user.id, "Imported History Bank")
+ account_id = uuid.uuid4()
+ category_id = uuid.uuid4()
+ import_log_id = uuid.uuid4()
+ imported_id = uuid.uuid4()
+ second_imported_id = uuid.uuid4()
+ category = Category(
+ id=category_id,
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ name="Already Categorized",
+ icon="tag",
+ color="#000000",
+ is_system=False,
+ )
+ account = Account(
+ id=account_id,
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ connection_id=conn.id,
+ external_id="import-acc-1",
+ name="Checking",
+ type="checking",
+ balance=Decimal("-42.00"),
+ currency="BRL",
+ )
+ import_log = ImportLog(
+ id=import_log_id,
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ account_id=account_id,
+ filename="history.csv",
+ format="csv",
+ transaction_count=2,
+ total_debit=Decimal("42.00"),
+ )
+ imported = Transaction(
+ id=imported_id,
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ account_id=account_id,
+ external_id="csv-row-1",
+ description="Spotify",
+ amount=Decimal("42.00"),
+ date=date(2026, 6, 21),
+ type="debit",
+ currency="BRL",
+ source="import",
+ import_id=import_log_id,
+ category_id=category_id,
+ )
+ second_imported = Transaction(
+ id=second_imported_id,
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ account_id=account_id,
+ external_id="csv-row-2",
+ description="Spotify",
+ amount=Decimal("42.00"),
+ date=date(2026, 6, 21),
+ type="debit",
+ currency="BRL",
+ source="import",
+ import_id=import_log_id,
+ category_id=category_id,
+ )
+ session.add_all([category, account, import_log, imported, second_imported])
+ await session.commit()
+
+ mock_provider = AsyncMock()
+ mock_provider.refresh_credentials = AsyncMock(return_value={"token": "refreshed"})
+ mock_provider.get_accounts = AsyncMock(return_value=[
+ AccountData(
+ external_id="import-acc-1",
+ name="Checking",
+ type="checking",
+ balance=Decimal("-42.00"),
+ currency="BRL",
+ ),
+ ])
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="provider-tx-1",
+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ amount=Decimal("42.00"),
+ date=date(2026, 6, 20),
+ type="debit",
+ currency="BRL",
+ payee="Spotify",
+ raw_data={
+ "posted": 1781913600,
+ "description": "SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ },
+ ),
+ TransactionData(
+ external_id="provider-tx-2",
+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ amount=Decimal("42.00"),
+ date=date(2026, 6, 20),
+ type="debit",
+ currency="BRL",
+ payee="Spotify",
+ raw_data={
+ "posted": 1781913600,
+ "description": "SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ },
+ ),
+ TransactionData(
+ external_id="provider-tx-3",
+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ amount=Decimal("42.00"),
+ date=date(2026, 6, 20),
+ type="debit",
+ currency="BRL",
+ payee="Spotify",
+ raw_data={
+ "posted": 1781913600,
+ "description": "SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ },
+ ),
+ ])
+
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock) as apply_rules:
+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)
+
+ rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.source != "opening_balance",
+ )
+ )).scalars().all()
+ assert len(rows) == 3
+ claimed_rows = [
+ row for row in rows if row.id in {imported_id, second_imported_id}
+ ]
+ assert len(claimed_rows) == 2
+ assert {row.external_id for row in claimed_rows} == {
+ "provider-tx-1", "provider-tx-2",
+ }
+ assert all(row.source == "import" for row in claimed_rows)
+ assert all(row.import_id is None for row in claimed_rows)
+ assert all(row.category_id == category_id for row in claimed_rows)
+ assert all(row.description == "Spotify" for row in claimed_rows)
+ assert all(row.date == date(2026, 6, 21) for row in claimed_rows)
+ assert all(row.payee == "Spotify" for row in claimed_rows)
+ assert {row.external_id for row in rows} == {
+ "provider-tx-1", "provider-tx-2", "provider-tx-3",
+ }
+ assert apply_rules.await_count == 1
+
+ mock_provider.get_transactions = AsyncMock(return_value=[
+ TransactionData(
+ external_id="provider-tx-4",
+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ amount=Decimal("42.00"),
+ date=date(2026, 6, 20),
+ type="debit",
+ currency="BRL",
+ payee="Spotify",
+ raw_data={
+ "posted": 1781913600,
+ "description": "SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ },
+ ),
+ *mock_provider.get_transactions.return_value[1:],
+ ])
+ with patch("app.services.connection_service.get_provider", return_value=mock_provider), \
+ patch("app.services.connection_service.detect_transfer_pairs", new_callable=AsyncMock), \
+ patch("app.services.connection_service.stamp_primary_amount", new_callable=AsyncMock), \
+ patch("app.services.connection_service.apply_rules_to_transaction", new_callable=AsyncMock):
+ await sync_connection(session, conn.id, test_workspace.id, test_user.id)
+
+ rekeyed_rows = (await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.source != "opening_balance",
+ )
+ )).scalars().all()
+ assert len(rekeyed_rows) == 3
+ assert {row.external_id for row in rekeyed_rows} == {
+ "provider-tx-4", "provider-tx-2", "provider-tx-3",
+ }
+
+
@pytest.mark.asyncio
async def test_sync_connection_tolerates_duplicate_transaction_rows(
session: AsyncSession, test_user, test_workspace
@@ -830,10 +1357,34 @@ async def test_sync_connection_tolerates_duplicate_transaction_rows(
assert any(r.status == "posted" for r in rows)
+
@pytest.mark.asyncio
async def test_sync_connection_not_found(session: AsyncSession, test_user, test_workspace):
- with pytest.raises(ValueError, match="not found"):
- await sync_connection(session, uuid.uuid4(), test_workspace.id, test_user.id)
+ connection_id = uuid.uuid4()
+ with patch(
+ "app.services.connection_service.get_connection", wraps=get_connection
+ ) as locked_get_connection:
+ with pytest.raises(ValueError, match="not found"):
+ await sync_connection(session, connection_id, test_workspace.id, test_user.id)
+ locked_get_connection.assert_awaited_once_with(
+ session, connection_id, test_workspace.id, for_update=True
+ )
+
+
+async def test_get_connection_can_lock_row_for_update():
+ session = AsyncMock(spec=AsyncSession)
+ result = MagicMock()
+ result.scalar_one_or_none.return_value = None
+ session.execute.return_value = result
+
+ await get_connection(
+ session, uuid.uuid4(), uuid.uuid4(), for_update=True
+ )
+
+ statement = session.execute.await_args.args[0]
+ sql = str(statement.compile(dialect=postgresql.dialect()))
+ assert sql.rstrip().endswith("FOR UPDATE")
@pytest.mark.asyncio
@@ -2751,12 +3302,14 @@ async def test_sync_keeps_genuine_same_day_repeats(
description="UBER TRIP",
amount=Decimal("25.00"), date=date(2026, 4, 20),
type="debit", currency="BRL", status="posted",
+ raw_data={"posted": 1776643200, "transacted_at": 1776643200},
),
TransactionData(
external_id="uber-2",
description="UBER TRIP",
amount=Decimal("25.00"), date=date(2026, 4, 20),
type="debit", currency="BRL", status="posted",
+ raw_data={"posted": 1776643200, "transacted_at": 1776643200},
),
])
@@ -2772,7 +3325,7 @@ async def test_sync_keeps_genuine_same_day_repeats(
Transaction.source == "sync",
)
)).scalars().all()
- assert len(rows) == 2, "identical-description same-day repeats must be kept"
+ assert {row.external_id for row in rows} == {"uber-1", "uber-2"}
# ---------------------------------------------------------------------------
diff --git a/backend/tests/test_connection_service_coverage.py b/backend/tests/test_connection_service_coverage.py
index 94854a8c..55414276 100644
--- a/backend/tests/test_connection_service_coverage.py
+++ b/backend/tests/test_connection_service_coverage.py
@@ -487,16 +487,19 @@ async def test_sync_trigger_refresh_needs_user_action(session: AsyncSession, tes
mock_provider.trigger_refresh = AsyncMock(return_value="needs_user_action")
with patch("app.services.connection_service.get_provider", return_value=mock_provider):
- with pytest.raises(RuntimeError, match="reconnect"):
+ with pytest.raises(ProviderUserActionRequired, match="reconnect") as exc:
await sync_connection(
session, conn.id, test_workspace.id, test_user.id,
trigger_provider_refresh=True,
)
+ assert exc.value.code == "credentials_invalid"
# status marked error and committed
refreshed = (await session.execute(
select(BankConnection).where(BankConnection.id == conn.id)
)).scalar_one()
assert refreshed.status == "error"
+ assert refreshed.settings is not None
+ assert "last_provider_refresh_at" in refreshed.settings
@pytest.mark.asyncio
@@ -519,6 +522,8 @@ async def test_sync_trigger_refresh_refreshed_then_reads(session: AsyncSession,
trigger_provider_refresh=True,
)
assert result.status == "active"
+ assert result.settings is not None
+ assert "last_provider_refresh_at" in result.settings
mock_provider.trigger_refresh.assert_awaited_once()
diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.py
index c5349b1d..772eb5ac 100644
--- a/backend/tests/test_import_service.py
+++ b/backend/tests/test_import_service.py
@@ -2047,13 +2047,130 @@ class TestOfxInstallmentDedup:
)
await import_transactions(session, test_workspace.id, test_user.id, test_account.id, [txn], "ofx")
imported, skipped, _, _ = await import_transactions(
- session, test_workspace.id, test_user.id, test_account.id, [txn], "ofx",
+ session,
+ test_workspace.id,
+ test_user.id,
+ test_account.id,
+ [txn, txn.model_copy()],
+ "ofx",
)
assert imported == 0
- assert skipped == 1
+ assert skipped == 2
class TestCsvDuplicateDetectionToggle:
+ @pytest.mark.asyncio
+ async def test_csv_identical_new_rows_remain_distinct(
+ self, session: AsyncSession, test_user: User, test_workspace, test_account: Account,
+ ):
+ from app.schemas.transaction import TransactionImport
+
+ row = TransactionImport(
+ description="Coffee Shop",
+ amount=Decimal("8.50"),
+ date=date(2026, 6, 15),
+ type="debit",
+ )
+ imported, skipped, _, _ = await import_transactions(
+ session,
+ test_workspace.id,
+ test_user.id,
+ test_account.id,
+ [row, row.model_copy()],
+ "csv",
+ )
+
+ assert (imported, skipped) == (2, 0)
+
+ @pytest.mark.asyncio
+ async def test_csv_exact_match_consumes_one_synced_row_once(
+ self, session: AsyncSession, test_user: User, test_workspace, test_account: Account,
+ ):
+ from app.models.transaction import Transaction
+ from app.schemas.transaction import TransactionImport
+
+ session.add(Transaction(
+ id=uuid.uuid4(),
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ account_id=test_account.id,
+ external_id="provider-coffee",
+ description="Coffee Shop",
+ amount=Decimal("8.50"),
+ date=date(2026, 6, 15),
+ type="debit",
+ source="sync",
+ status="posted",
+ ))
+ await session.commit()
+
+ row = TransactionImport(
+ description="Coffee Shop",
+ amount=Decimal("8.50"),
+ date=date(2026, 6, 15),
+ type="debit",
+ )
+ imported, skipped, _, _ = await import_transactions(
+ session,
+ test_workspace.id,
+ test_user.id,
+ test_account.id,
+ [row, row.model_copy()],
+ "csv",
+ )
+
+ assert (imported, skipped) == (1, 1)
+
+ @pytest.mark.asyncio
+ async def test_csv_payee_match_consumes_one_synced_row_once(
+ self, session: AsyncSession, test_user: User, test_workspace, test_account: Account,
+ ):
+ from app.models.transaction import Transaction
+ from app.schemas.transaction import TransactionImport
+
+ synced = Transaction(
+ id=uuid.uuid4(),
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ account_id=test_account.id,
+ external_id="provider-spotify",
+ description="SPOTIFY USA 45 W. 18TH STREET NEW YORK",
+ payee="Spotify",
+ amount=Decimal("12.99"),
+ date=date(2026, 6, 15),
+ type="debit",
+ source="sync",
+ status="posted",
+ )
+ session.add(synced)
+ await session.commit()
+
+ imported, skipped, _, _ = await import_transactions(
+ session,
+ test_workspace.id,
+ test_user.id,
+ test_account.id,
+ [
+ TransactionImport(
+ description="Spotify",
+ payee_raw="Spotify",
+ amount=Decimal("12.99"),
+ date=date(2026, 6, 16),
+ type="debit",
+ ),
+ TransactionImport(
+ description="Spotify",
+ payee_raw="Spotify",
+ amount=Decimal("12.99"),
+ date=date(2026, 6, 16),
+ type="debit",
+ ),
+ ],
+ "csv",
+ )
+
+ assert (imported, skipped) == (1, 1)
+
@pytest.mark.asyncio
async def test_csv_detect_duplicates_false_allows_duplicates(
self, session: AsyncSession, test_user: User, test_workspace, test_account: Account,
@@ -2513,6 +2630,48 @@ async def test_import_tolerates_duplicate_external_id_rows(
assert len(remaining) == 2
+async def test_import_external_id_reconciles_matching_synced_transaction(
+ session: AsyncSession, test_user: User, test_workspace, test_account: Account,
+):
+ from app.models.transaction import Transaction
+ from app.schemas.transaction import TransactionImport
+
+ session.add(Transaction(
+ id=uuid.uuid4(),
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ account_id=test_account.id,
+ external_id="provider-id",
+ description="Music subscription",
+ original_description="SPOTIFY",
+ amount=Decimal("23.90"),
+ date=date(2026, 1, 15),
+ type="debit",
+ source="sync",
+ ))
+ await session.commit()
+
+ imported, skipped, _, _ = await import_transactions(
+ session,
+ test_workspace.id,
+ test_user.id,
+ test_account.id,
+ [TransactionImport(
+ external_id="ofx-fitid",
+ description="SPOTIFY",
+ amount=Decimal("23.90"),
+ date=date(2026, 1, 15),
+ type="debit",
+ )],
+ "ofx",
+ detected_format="ofx",
+ )
+
+ assert imported == 0
+ assert skipped == 1
+
+
def test_normalize_amount_swiss_single_quotes():
from app.services.import_service import normalize_amount
assert normalize_amount("1'050.45") == "1050.45"
diff --git a/backend/tests/test_providers_simplefin.py b/backend/tests/test_providers_simplefin.py
index 6dd00b41..ffd2f3bc 100644
--- a/backend/tests/test_providers_simplefin.py
+++ b/backend/tests/test_providers_simplefin.py
@@ -182,6 +182,38 @@ async def test_auth_errlist_raises_user_action_required():
assert exc.value.code == "credentials_invalid"
+async def test_auth_errlist_with_usable_accounts_is_soft_warning(caplog):
+ """A stale SimpleFIN sub-connection must not block healthy accounts."""
+ import logging as stdlogging
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(
+ 200,
+ json={
+ "errlist": [
+ {"code": "con.auth", "msg": "Apple Card auth required", "conn_id": "APPLE"}
+ ],
+ "accounts": [
+ {
+ "id": "acc-healthy",
+ "name": "Healthy Checking",
+ "currency": "USD",
+ "balance": "42.00",
+ }
+ ],
+ },
+ )
+
+ creds = {"access_url": "https://u:[email protected]/simplefin"}
+ provider = SimpleFinProvider()
+ with caplog.at_level(stdlogging.WARNING, logger="app.providers.simplefin"), _patched_client(handler):
+ accounts = await provider.get_accounts(creds)
+
+ assert [acc.external_id for acc in accounts] == ["acc-healthy"]
+ assert any("con.auth" in rec.getMessage() for rec in caplog.records)
+
+
@pytest.mark.asyncio
async def test_act_failed_is_soft_warning(caplog):
"""``act.failed`` is transient — keep going, just log."""
@@ -242,6 +274,45 @@ async def test_missing_access_url_raises_session_expired():
await SimpleFinProvider().get_accounts({})
+async def test_get_transactions_auth_errlist_with_requested_account_is_soft_warning(caplog):
+ """A top-level auth warning must not block requested-account transactions."""
+ import logging as stdlogging
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ assert request.url.params.get("account") == "acc-healthy"
+ return httpx.Response(
+ 200,
+ json={
+ "errlist": [
+ {"code": "con.auth", "msg": "Apple Card auth required", "conn_id": "APPLE"}
+ ],
+ "accounts": [
+ {
+ "id": "acc-healthy",
+ "transactions": [
+ {
+ "id": "tx-healthy",
+ "amount": "-12.34",
+ "posted": 1672531200,
+ "description": "Coffee",
+ }
+ ],
+ }
+ ],
+ },
+ )
+
+ creds = {"access_url": "https://u:[email protected]/simplefin"}
+ with caplog.at_level(stdlogging.WARNING, logger="app.providers.simplefin"), _patched_client(handler):
+ txns = await SimpleFinProvider().get_transactions(
+ creds, "acc-healthy", since=date(2023, 1, 1)
+ )
+
+ assert [txn.external_id for txn in txns] == ["tx-healthy"]
+ assert any("con.auth" in rec.getMessage() for rec in caplog.records)
+
+
# ----- transactions -----------------------------------------------------------
@@ -363,6 +434,29 @@ async def test_get_transactions_chunks_long_windows():
assert len(calls) >= 3 # 200 days / 90-day window
+async def test_get_transactions_followup_rewinds_one_valid_full_window():
+ calls: list[tuple[int, int]] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append((
+ int(request.url.params["start-date"]),
+ int(request.url.params["end-date"]),
+ ))
+ return httpx.Response(
+ 200, json={"accounts": [{"id": "acc-1", "transactions": []}]}
+ )
+
+ creds = {"access_url": "https://u:[email protected]/simplefin"}
+ with _patched_client(handler):
+ await SimpleFinProvider().get_transactions(
+ creds, "acc-1", since=date.today() - timedelta(days=14)
+ )
+
+ assert len(calls) == 1
+ assert calls[0][1] - calls[0][0] == 45 * 24 * 60 * 60
+
+
# ----- holdings ---------------------------------------------------------------
diff --git a/backend/tests/test_sync_tasks.py b/backend/tests/test_sync_tasks.py
new file mode 100644
index 00000000..2964582d
--- /dev/null
+++ b/backend/tests/test_sync_tasks.py
@@ -0,0 +1,61 @@
+import uuid
+from datetime import datetime, timedelta, timezone
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from app.models.bank_connection import BankConnection
+from app.tasks.sync_tasks import _should_trigger_provider_refresh, _sync_one
+
+
+def test_should_trigger_provider_refresh_daily():
+ now = datetime(2026, 6, 29, tzinfo=timezone.utc)
+
+ assert _should_trigger_provider_refresh(None, now)
+ assert _should_trigger_provider_refresh({"last_provider_refresh_at": "invalid"}, now)
+ assert _should_trigger_provider_refresh(
+ {"last_provider_refresh_at": (now - timedelta(hours=20)).isoformat()}, now
+ )
+ assert not _should_trigger_provider_refresh(
+ {"last_provider_refresh_at": (now - timedelta(hours=4)).isoformat()}, now
+ )
+
+
+async def test_sync_one_forwards_provider_refresh(session, test_user, test_workspace):
+ conn = BankConnection(
+ id=uuid.uuid4(),
+ user_id=test_user.id,
+ workspace_id=test_workspace.id,
+ provider="test",
+ external_id="provider-conn",
+ institution_name="Test Bank",
+ credentials={"token": "fake"},
+ status="active",
+ last_sync_at=datetime.now(timezone.utc),
+ created_at=datetime.now(timezone.utc),
+ )
+ session.add(conn)
+ await session.commit()
+
+ def session_maker():
+ return session
+
+ with patch(
+ "app.tasks.sync_tasks.connection_service.sync_connection",
+ new_callable=AsyncMock,
+ ) as sync_connection:
+ await _sync_one(
+ session_maker,
+ conn.id,
+ test_user.id,
+ trigger_provider_refresh=True,
+ )
+
+ sync_connection.assert_awaited_once_with(
+ session,
+ conn.id,
+ test_workspace.id,
+ test_user.id,
+ trigger_provider_refresh=True,
+ )