Open-source personal finance manager. Self-hosted, privacy-first.
feat: make duplicate detection optional for CSV transaction imports (#145)
Gerson Caldeirac5d1ba466e59
diff --git a/backend/app/api/import_transactions.py b/backend/app/api/import_transactions.pyindex 91166122..95f4d84c 100644--- a/backend/app/api/import_transactions.py+++ b/backend/app/api/import_transactions.py@@ -103,6 +103,7 @@ async def import_transactions(imported, skipped, import_log_id = await import_service.import_transactions(session, user.id, data.account_id, data.transactions, "import",filename=data.filename, detected_format=data.detected_format,+ detect_duplicates=data.detect_duplicates,)return {"imported": imported, "skipped": skipped, "import_log_id": str(import_log_id)}diff --git a/backend/app/schemas/transaction.py b/backend/app/schemas/transaction.pyindex 4a56fa0d..853eb464 100644--- a/backend/app/schemas/transaction.py+++ b/backend/app/schemas/transaction.py@@ -114,3 +114,4 @@ class TransactionImportRequest(BaseModel):transactions: list[TransactionImport]filename: str = ""detected_format: str = ""+ detect_duplicates: bool = Truediff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.pyindex 1c2eed8e..c4837b8b 100644--- a/backend/app/services/import_service.py+++ b/backend/app/services/import_service.py@@ -436,6 +436,7 @@ async def import_transactions(source: str,filename: str = "",detected_format: str = "",+ detect_duplicates: bool = True,) -> tuple[int, int, uuid.UUID]:"""Import transactions into an account. Returns (imported, skipped, import_log_id)."""from app.models.import_log import ImportLog@@ -472,37 +473,41 @@ async def import_transactions(imported = 0skipped = 0+ effective_format = (detected_format or source or "").lower()+ should_detect_duplicates = detect_duplicates if effective_format == "csv" else True+for txn_data in transactions:# Resolve currency: CSV value > account currencytxn_currency = txn_data.currency or account_currency- # Duplicate detection: use external_id when available (OFX FITID),- # fall back to field-based matching for formats without unique IDs.- # When matching by external_id, also require the same `date` so that- # Brazilian credit-card installments — where some banks reuse one- # purchase FITID across every monthly statement — don't get skipped- # as duplicates from later monthly imports (issue #98).- if txn_data.external_id:- existing = await session.execute(- select(Transaction).where(- Transaction.account_id == account_id,- Transaction.external_id == txn_data.external_id,- Transaction.date == txn_data.date,+ if should_detect_duplicates:+ # Duplicate detection: use external_id when available (OFX FITID),+ # fall back to field-based matching for formats without unique IDs.+ # When matching by external_id, also require the same `date` so that+ # Brazilian credit-card installments — where some banks reuse one+ # purchase FITID across every monthly statement — don't get skipped+ # as duplicates from later monthly imports (issue #98).+ if txn_data.external_id:+ existing = await session.execute(+ select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.external_id == txn_data.external_id,+ Transaction.date == txn_data.date,+ ))- )- else:- existing = await session.execute(- select(Transaction).where(- Transaction.account_id == account_id,- Transaction.date == txn_data.date,- Transaction.amount == txn_data.amount,- Transaction.type == txn_data.type,- Transaction.description == txn_data.description,+ else:+ existing = await session.execute(+ select(Transaction).where(+ Transaction.account_id == account_id,+ Transaction.date == txn_data.date,+ Transaction.amount == txn_data.amount,+ Transaction.type == txn_data.type,+ Transaction.description == txn_data.description,+ ))- )- if existing.scalar_one_or_none():- skipped += 1- continue+ if existing.scalar_one_or_none():+ skipped += 1+ continue# Resolve payee entity from raw payee text (OFX/QIF)import_payee_id = Nonediff --git a/backend/tests/test_import_api.py b/backend/tests/test_import_api.pyindex 75fe4b59..7a386b6e 100644--- a/backend/tests/test_import_api.py+++ b/backend/tests/test_import_api.py@@ -132,6 +132,31 @@ async def test_import_creates_log(client: AsyncClient, auth_headers, test_accounassert "import_log_id" in data+async def test_import_csv_with_duplicate_detection_disabled_allows_reimport(+ client: AsyncClient, auth_headers, test_account: Account,+):+ payload = {+ "account_id": str(test_account.id),+ "transactions": [+ {"description": "REPEATED CSV", "amount": "10.00", "date": "2026-03-01", "type": "debit"},+ ],+ "filename": "repeated.csv",+ "detected_format": "csv",+ "detect_duplicates": False,+ }++ first = await client.post("/api/transactions/import", headers=auth_headers, json=payload)+ assert first.status_code == 201+ assert first.json()["imported"] == 1+ assert first.json()["skipped"] == 0++ second = await client.post("/api/transactions/import", headers=auth_headers, json=payload)+ assert second.status_code == 201+ assert second.json()["imported"] == 1+ assert second.json()["skipped"] == 0++@pytest.mark.asyncioasync def test_list_import_logs(client: AsyncClient, auth_headers, test_account: Account):# Create an import firstdiff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.pyindex 9194a0f6..cbdd6d3e 100644--- a/backend/tests/test_import_service.py+++ b/backend/tests/test_import_service.py@@ -1425,3 +1425,74 @@ class TestOfxInstallmentDedup:)assert imported == 0assert skipped == 1+++class TestCsvDuplicateDetectionToggle:+ @pytest.mark.asyncio+ async def test_csv_detect_duplicates_false_allows_duplicates(+ self, session: AsyncSession, test_user: User, test_account: Account,+ ):+ from app.schemas.transaction import TransactionBase++ txn = TransactionBase(+ description="CSV DUP TOGGLE",+ amount=Decimal("42.00"),+ date=date(2026, 3, 10),+ type="debit",+ )++ await import_transactions(+ session,+ test_user.id,+ test_account.id,+ [txn],+ "csv",+ detected_format="csv",+ detect_duplicates=False,+ )+ imported, skipped, _ = await import_transactions(+ session,+ test_user.id,+ test_account.id,+ [txn],+ "csv",+ detected_format="csv",+ detect_duplicates=False,+ )+ assert imported == 1+ assert skipped == 0++ @pytest.mark.asyncio+ async def test_non_csv_ignores_toggle_and_still_dedups(+ self, session: AsyncSession, test_user: User, test_account: Account,+ ):+ from app.schemas.transaction import TransactionBase++ txn = TransactionBase(+ description="OFX DUP",+ amount=Decimal("15.00"),+ date=date(2026, 3, 11),+ type="debit",+ external_id="OFX_DUP_01",+ )++ await import_transactions(+ session,+ test_user.id,+ test_account.id,+ [txn],+ "ofx",+ detected_format="ofx",+ detect_duplicates=False,+ )+ imported, skipped, _ = await import_transactions(+ session,+ test_user.id,+ test_account.id,+ [txn],+ "ofx",+ detected_format="ofx",+ detect_duplicates=False,+ )+ assert imported == 0+ assert skipped == 1diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.tsindex a88f1373..83d77d8c 100644--- a/frontend/src/lib/api.ts+++ b/frontend/src/lib/api.ts@@ -341,8 +341,26 @@ export const transactions = {const { data } = await api.post('/transactions/import/preview', formData)return data},- import: async (account_id: string, transactions: Transaction[], filename: string, detected_format: string): Promise<{ imported: number; skipped: number; import_log_id: string }> => {- const { data } = await api.post('/transactions/import', { account_id, transactions, filename, detected_format })+ import: async (+ account_id: string,+ transactions: Transaction[],+ filename: string,+ detected_format: string,+ options?: { detect_duplicates?: boolean },+ ): Promise<{ imported: number; skipped: number; import_log_id: string }> => {+ const payload: {+ account_id: string+ transactions: Transaction[]+ filename: string+ detected_format: string+ detect_duplicates?: boolean+ } = { account_id, transactions, filename, detected_format }++ if (typeof options?.detect_duplicates === 'boolean') {+ payload.detect_duplicates = options.detect_duplicates+ }++ const { data } = await api.post('/transactions/import', payload)return data},export: async (params?: {diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.jsonindex 2a6e6113..c8122283 100644--- a/frontend/src/locales/en.json+++ b/frontend/src/locales/en.json@@ -586,6 +586,7 @@"dateFormat": "Date format","dateFormatAuto": "Auto (detect)","flipAmounts": "Negate amounts (swap income/expense)",+ "detectDuplicates": "Skip duplicate transactions","splitColumns": "Split inflow/outflow columns","inflowColumn": "Inflow column","outflowColumn": "Outflow column",diff --git a/frontend/src/locales/pt-BR.json b/frontend/src/locales/pt-BR.jsonindex a3654316..75a97d50 100644--- a/frontend/src/locales/pt-BR.json+++ b/frontend/src/locales/pt-BR.json@@ -586,6 +586,7 @@"dateFormat": "Formato da data","dateFormatAuto": "Automático (detectar)","flipAmounts": "Inverter valores (trocar entrada/saída)",+ "detectDuplicates": "Ignorar transações duplicadas","splitColumns": "Separar colunas entrada/saída","inflowColumn": "Coluna de entrada","outflowColumn": "Coluna de saída",diff --git a/frontend/src/pages/import.tsx b/frontend/src/pages/import.tsxindex b5a068e9..6d02dee6 100644--- a/frontend/src/pages/import.tsx+++ b/frontend/src/pages/import.tsx@@ -50,6 +50,7 @@ export default function ImportPage() {// CSV optionsconst [csvDateFormat, setCsvDateFormat] = useState('')const [csvFlipAmount, setCsvFlipAmount] = useState(false)+ const [csvDetectDuplicates, setCsvDetectDuplicates] = useState(true)const [csvSplitColumns, setCsvSplitColumns] = useState(false)const [csvInflowColumn, setCsvInflowColumn] = useState('')const [csvOutflowColumn, setCsvOutflowColumn] = useState('')@@ -75,7 +76,13 @@ export default function ImportPage() {})const importMutation = useMutation({- mutationFn: () => transactionsApi.import(selectedAccount, previewData!.transactions, fileName ?? '', previewData!.detected_format),+ mutationFn: () => transactionsApi.import(+ selectedAccount,+ previewData!.transactions,+ fileName ?? '',+ previewData!.detected_format,+ isCsvFile ? { detect_duplicates: csvDetectDuplicates } : undefined,+ ),onSuccess: (data) => {invalidateFinancialQueries(queryClient)queryClient.invalidateQueries({ queryKey: ['import-logs'] })@@ -108,6 +115,7 @@ export default function ImportPage() {function resetCsvOptions() {setCsvDateFormat('')setCsvFlipAmount(false)+ setCsvDetectDuplicates(true)setCsvSplitColumns(false)setCsvInflowColumn('')setCsvOutflowColumn('')@@ -347,6 +355,20 @@ export default function ImportPage() {{t('import.splitColumns')}</Label></div>++ {/* Duplicate detection toggle */}+ <div className="flex items-center gap-2 pt-4">+ <input+ type="checkbox"+ id="detect-duplicates"+ checked={csvDetectDuplicates}+ onChange={(e) => setCsvDetectDuplicates(e.target.checked)}+ className="rounded border-border text-primary focus:ring-primary"+ />+ <Label htmlFor="detect-duplicates" className="text-sm text-muted-foreground cursor-pointer">+ {t('import.detectDuplicates')}+ </Label>+ </div></div>{/* Split column selectors */}