Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

feat: make duplicate detection optional for CSV transaction imports (#145)

Gerson Caldeirac5d1ba466e59
diff --git a/backend/app/api/import_transactions.py b/backend/app/api/import_transactions.py
index 91166122..95f4d84c 100644
--- a/backend/app/api/import_transactions.py
+++ b/backend/app/api/import_transactions.py
@@ -103,6 +103,7 @@ async def import_transactions(
imported, skipped, import_log_id = await import_service.import_transactions(
session, user.id, data.account_id, data.transactions, "import",
filename=data.filename, detected_format=data.detected_format,
+ detect_duplicates=data.detect_duplicates,
)
return {"imported": imported, "skipped": skipped, "import_log_id": str(import_log_id)}
diff --git a/backend/app/schemas/transaction.py b/backend/app/schemas/transaction.py
index 4a56fa0d..853eb464 100644
--- a/backend/app/schemas/transaction.py
+++ b/backend/app/schemas/transaction.py
@@ -114,3 +114,4 @@ class TransactionImportRequest(BaseModel):
transactions: list[TransactionImport]
filename: str = ""
detected_format: str = ""
+ detect_duplicates: bool = True
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.py
index 1c2eed8e..c4837b8b 100644
--- a/backend/app/services/import_service.py
+++ b/backend/app/services/import_service.py
@@ -436,6 +436,7 @@ async def import_transactions(
source: str,
filename: str = "",
detected_format: str = "",
+ detect_duplicates: bool = True,
) -> tuple[int, int, uuid.UUID]:
"""Import transactions into an account. Returns (imported, skipped, import_log_id)."""
from app.models.import_log import ImportLog
@@ -472,37 +473,41 @@ async def import_transactions(
imported = 0
skipped = 0
+ effective_format = (detected_format or source or "").lower()
+ should_detect_duplicates = detect_duplicates if effective_format == "csv" else True
+
for txn_data in transactions:
# Resolve currency: CSV value > account currency
txn_currency = txn_data.currency or account_currency
- # Duplicate detection: use external_id when available (OFX FITID),
- # fall back to field-based matching for formats without unique IDs.
- # When matching by external_id, also require the same `date` so that
- # Brazilian credit-card installments — where some banks reuse one
- # purchase FITID across every monthly statement — don't get skipped
- # as duplicates from later monthly imports (issue #98).
- if txn_data.external_id:
- existing = await session.execute(
- select(Transaction).where(
- Transaction.account_id == account_id,
- Transaction.external_id == txn_data.external_id,
- Transaction.date == txn_data.date,
+ if should_detect_duplicates:
+ # Duplicate detection: use external_id when available (OFX FITID),
+ # fall back to field-based matching for formats without unique IDs.
+ # When matching by external_id, also require the same `date` so that
+ # Brazilian credit-card installments — where some banks reuse one
+ # purchase FITID across every monthly statement — don't get skipped
+ # as duplicates from later monthly imports (issue #98).
+ if txn_data.external_id:
+ existing = await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.external_id == txn_data.external_id,
+ Transaction.date == txn_data.date,
+ )
)
- )
- else:
- existing = await session.execute(
- select(Transaction).where(
- Transaction.account_id == account_id,
- Transaction.date == txn_data.date,
- Transaction.amount == txn_data.amount,
- Transaction.type == txn_data.type,
- Transaction.description == txn_data.description,
+ else:
+ existing = await session.execute(
+ select(Transaction).where(
+ Transaction.account_id == account_id,
+ Transaction.date == txn_data.date,
+ Transaction.amount == txn_data.amount,
+ Transaction.type == txn_data.type,
+ Transaction.description == txn_data.description,
+ )
)
- )
- if existing.scalar_one_or_none():
- skipped += 1
- continue
+ if existing.scalar_one_or_none():
+ skipped += 1
+ continue
# Resolve payee entity from raw payee text (OFX/QIF)
import_payee_id = None
diff --git a/backend/tests/test_import_api.py b/backend/tests/test_import_api.py
index 75fe4b59..7a386b6e 100644
--- a/backend/tests/test_import_api.py
+++ b/backend/tests/test_import_api.py
@@ -132,6 +132,31 @@ async def test_import_creates_log(client: AsyncClient, auth_headers, test_accoun
assert "import_log_id" in data
+async def test_import_csv_with_duplicate_detection_disabled_allows_reimport(
+ client: AsyncClient, auth_headers, test_account: Account,
+):
+ payload = {
+ "account_id": str(test_account.id),
+ "transactions": [
+ {"description": "REPEATED CSV", "amount": "10.00", "date": "2026-03-01", "type": "debit"},
+ ],
+ "filename": "repeated.csv",
+ "detected_format": "csv",
+ "detect_duplicates": False,
+ }
+
+ first = await client.post("/api/transactions/import", headers=auth_headers, json=payload)
+ assert first.status_code == 201
+ assert first.json()["imported"] == 1
+ assert first.json()["skipped"] == 0
+
+ second = await client.post("/api/transactions/import", headers=auth_headers, json=payload)
+ assert second.status_code == 201
+ assert second.json()["imported"] == 1
+ assert second.json()["skipped"] == 0
+
+
@pytest.mark.asyncio
async def test_list_import_logs(client: AsyncClient, auth_headers, test_account: Account):
# Create an import first
diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.py
index 9194a0f6..cbdd6d3e 100644
--- a/backend/tests/test_import_service.py
+++ b/backend/tests/test_import_service.py
@@ -1425,3 +1425,74 @@ class TestOfxInstallmentDedup:
)
assert imported == 0
assert skipped == 1
+
+
+class TestCsvDuplicateDetectionToggle:
+ @pytest.mark.asyncio
+ async def test_csv_detect_duplicates_false_allows_duplicates(
+ self, session: AsyncSession, test_user: User, test_account: Account,
+ ):
+ from app.schemas.transaction import TransactionBase
+
+ txn = TransactionBase(
+ description="CSV DUP TOGGLE",
+ amount=Decimal("42.00"),
+ date=date(2026, 3, 10),
+ type="debit",
+ )
+
+ await import_transactions(
+ session,
+ test_user.id,
+ test_account.id,
+ [txn],
+ "csv",
+ detected_format="csv",
+ detect_duplicates=False,
+ )
+ imported, skipped, _ = await import_transactions(
+ session,
+ test_user.id,
+ test_account.id,
+ [txn],
+ "csv",
+ detected_format="csv",
+ detect_duplicates=False,
+ )
+ assert imported == 1
+ assert skipped == 0
+
+ @pytest.mark.asyncio
+ async def test_non_csv_ignores_toggle_and_still_dedups(
+ self, session: AsyncSession, test_user: User, test_account: Account,
+ ):
+ from app.schemas.transaction import TransactionBase
+
+ txn = TransactionBase(
+ description="OFX DUP",
+ amount=Decimal("15.00"),
+ date=date(2026, 3, 11),
+ type="debit",
+ external_id="OFX_DUP_01",
+ )
+
+ await import_transactions(
+ session,
+ test_user.id,
+ test_account.id,
+ [txn],
+ "ofx",
+ detected_format="ofx",
+ detect_duplicates=False,
+ )
+ imported, skipped, _ = await import_transactions(
+ session,
+ test_user.id,
+ test_account.id,
+ [txn],
+ "ofx",
+ detected_format="ofx",
+ detect_duplicates=False,
+ )
+ assert imported == 0
+ assert skipped == 1
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index a88f1373..83d77d8c 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -341,8 +341,26 @@ export const transactions = {
const { data } = await api.post('/transactions/import/preview', formData)
return data
},
- import: async (account_id: string, transactions: Transaction[], filename: string, detected_format: string): Promise<{ imported: number; skipped: number; import_log_id: string }> => {
- const { data } = await api.post('/transactions/import', { account_id, transactions, filename, detected_format })
+ import: async (
+ account_id: string,
+ transactions: Transaction[],
+ filename: string,
+ detected_format: string,
+ options?: { detect_duplicates?: boolean },
+ ): Promise<{ imported: number; skipped: number; import_log_id: string }> => {
+ const payload: {
+ account_id: string
+ transactions: Transaction[]
+ filename: string
+ detected_format: string
+ detect_duplicates?: boolean
+ } = { account_id, transactions, filename, detected_format }
+
+ if (typeof options?.detect_duplicates === 'boolean') {
+ payload.detect_duplicates = options.detect_duplicates
+ }
+
+ const { data } = await api.post('/transactions/import', payload)
return data
},
export: async (params?: {
diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.json
index 2a6e6113..c8122283 100644
--- a/frontend/src/locales/en.json
+++ b/frontend/src/locales/en.json
@@ -586,6 +586,7 @@
"dateFormat": "Date format",
"dateFormatAuto": "Auto (detect)",
"flipAmounts": "Negate amounts (swap income/expense)",
+ "detectDuplicates": "Skip duplicate transactions",
"splitColumns": "Split inflow/outflow columns",
"inflowColumn": "Inflow column",
"outflowColumn": "Outflow column",
diff --git a/frontend/src/locales/pt-BR.json b/frontend/src/locales/pt-BR.json
index a3654316..75a97d50 100644
--- a/frontend/src/locales/pt-BR.json
+++ b/frontend/src/locales/pt-BR.json
@@ -586,6 +586,7 @@
"dateFormat": "Formato da data",
"dateFormatAuto": "Automático (detectar)",
"flipAmounts": "Inverter valores (trocar entrada/saída)",
+ "detectDuplicates": "Ignorar transações duplicadas",
"splitColumns": "Separar colunas entrada/saída",
"inflowColumn": "Coluna de entrada",
"outflowColumn": "Coluna de saída",
diff --git a/frontend/src/pages/import.tsx b/frontend/src/pages/import.tsx
index b5a068e9..6d02dee6 100644
--- a/frontend/src/pages/import.tsx
+++ b/frontend/src/pages/import.tsx
@@ -50,6 +50,7 @@ export default function ImportPage() {
// CSV options
const [csvDateFormat, setCsvDateFormat] = useState('')
const [csvFlipAmount, setCsvFlipAmount] = useState(false)
+ const [csvDetectDuplicates, setCsvDetectDuplicates] = useState(true)
const [csvSplitColumns, setCsvSplitColumns] = useState(false)
const [csvInflowColumn, setCsvInflowColumn] = useState('')
const [csvOutflowColumn, setCsvOutflowColumn] = useState('')
@@ -75,7 +76,13 @@ export default function ImportPage() {
})
const importMutation = useMutation({
- mutationFn: () => transactionsApi.import(selectedAccount, previewData!.transactions, fileName ?? '', previewData!.detected_format),
+ mutationFn: () => transactionsApi.import(
+ selectedAccount,
+ previewData!.transactions,
+ fileName ?? '',
+ previewData!.detected_format,
+ isCsvFile ? { detect_duplicates: csvDetectDuplicates } : undefined,
+ ),
onSuccess: (data) => {
invalidateFinancialQueries(queryClient)
queryClient.invalidateQueries({ queryKey: ['import-logs'] })
@@ -108,6 +115,7 @@ export default function ImportPage() {
function resetCsvOptions() {
setCsvDateFormat('')
setCsvFlipAmount(false)
+ setCsvDetectDuplicates(true)
setCsvSplitColumns(false)
setCsvInflowColumn('')
setCsvOutflowColumn('')
@@ -347,6 +355,20 @@ export default function ImportPage() {
{t('import.splitColumns')}
</Label>
</div>
+
+ {/* Duplicate detection toggle */}
+ <div className="flex items-center gap-2 pt-4">
+ <input
+ type="checkbox"
+ id="detect-duplicates"
+ checked={csvDetectDuplicates}
+ onChange={(e) => setCsvDetectDuplicates(e.target.checked)}
+ className="rounded border-border text-primary focus:ring-primary"
+ />
+ <Label htmlFor="detect-duplicates" className="text-sm text-muted-foreground cursor-pointer">
+ {t('import.detectDuplicates')}
+ </Label>
+ </div>
</div>
{/* Split column selectors */}