Open-source personal finance manager. Self-hosted, privacy-first.
fix: improved csv import resilience and added support to map payee external id and notes (#474)
jorishaenseler15cc61e742f85c
diff --git a/backend/app/schemas/payee.py b/backend/app/schemas/payee.pyindex b170b842..f285536e 100644--- a/backend/app/schemas/payee.py+++ b/backend/app/schemas/payee.py@@ -3,22 +3,22 @@ from datetime import date as _Date, datetimefrom decimal import Decimalfrom typing import Optional-from pydantic import BaseModel, ConfigDict+from pydantic import BaseModel, ConfigDict, Fieldfrom app.schemas.category import CategoryReadclass PayeeCreate(BaseModel):- name: str+ name: str = Field(..., max_length=255)type: str = "merchant"- notes: Optional[str] = None+ notes: Optional[str] = Field(None, max_length=1000)class PayeeUpdate(BaseModel):- name: Optional[str] = None+ name: Optional[str] = Field(None, max_length=255)type: Optional[str] = Noneis_favorite: Optional[bool] = None- notes: Optional[str] = None+ notes: Optional[str] = Field(None, max_length=1000)class PayeeRead(BaseModel):diff --git a/backend/app/schemas/transaction.py b/backend/app/schemas/transaction.pyindex 14615551..2d43cdc1 100644--- a/backend/app/schemas/transaction.py+++ b/backend/app/schemas/transaction.py@@ -163,6 +163,7 @@ class TransactionImport(TransactionBase):excluded: bool = Falsecategory_id: Optional[uuid.UUID] = Noneforce_uncategorized: bool = False+ notes: Optional[str] = Noneclass TransactionImportPreview(BaseModel):diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.pyindex a0692ef2..77d085eb 100644--- a/backend/app/services/import_service.py+++ b/backend/app/services/import_service.py@@ -336,6 +336,7 @@ DATE_FORMAT_MAP = {CSV_MAPPABLE_FIELDS = ('date', 'description', 'amount', 'type','category', 'currency', 'fx_rate', 'inflow', 'outflow',+ 'payee', 'external_id', 'notes',)@@ -385,7 +386,7 @@ def parse_csv(reader = csv.DictReader(io.StringIO(text), dialect=dialect)# Normalize field names- fieldnames = [f.lower().strip() for f in (reader.fieldnames or [])]+ fieldnames = [f.lower().strip() if f is not None else "" for f in (reader.fieldnames or [])]# Map common column namesdate_cols = ['date', 'data', 'dt', 'transaction_date', 'data_transacao']@@ -395,6 +396,9 @@ def parse_csv(category_cols = ['category', 'categoria']currency_cols = ['currency', 'moeda', 'currency_code']fx_rate_cols = ['fx_rate', 'fx_rate_used', 'taxa_cambio', 'exchange_rate', 'taxa']+ payee_cols = ['payee', 'merchant', 'beneficiary', 'beneficiario', 'pagador']+ external_id_cols = [] # External ID must be mapped explicitly+ notes_cols = ['notes', 'nota', 'observacao']# Normalize the user-supplied column mapping (Securo field -> CSV header).mapping = {@@ -445,6 +449,9 @@ def parse_csv(category_col = resolve_col('category', category_cols)currency_col = resolve_col('currency', currency_cols)fx_rate_col = resolve_col('fx_rate', fx_rate_cols)+ payee_col = resolve_col('payee', payee_cols)+ external_id_col = resolve_col('external_id', external_id_cols)+ notes_col = resolve_col('notes', notes_cols)if not date_col or not desc_col:raise ValueError(@@ -461,12 +468,12 @@ def parse_csv(if date_format and date_format in DATE_FORMAT_MAP:date_formats = [DATE_FORMAT_MAP[date_format]]else:- date_formats = ['%Y-%m-%d', '%d/%m/%Y', '%d-%m-%Y', '%m/%d/%Y']+ date_formats = ['%Y-%m-%d', '%d/%m/%Y', '%d-%m-%Y', '%m/%d/%Y', '%d.%m.%Y']transactions = []for row in reader:# Normalize row keys- row = {k.lower().strip(): v for k, v in row.items()}+ row = {k.lower().strip() if k is not None else "": v for k, v in row.items()}# Parse datedate_str = row[date_col].strip()@@ -534,6 +541,10 @@ def parse_csv(except Exception:pass+ txn_payee = row[payee_col].strip() if payee_col and row.get(payee_col) else None+ txn_external_id = row[external_id_col].strip() if external_id_col and row.get(external_id_col) else None+ txn_notes = row[notes_col].strip() if notes_col and row.get(notes_col) else None+transactions.append(TransactionImport(description=row[desc_col].strip(),amount=abs(amount),@@ -542,6 +553,9 @@ def parse_csv(currency=txn_currency,fx_rate=txn_fx_rate,category_name=category_name,+ payee_raw=txn_payee,+ external_id=txn_external_id,+ notes=txn_notes,))return transactions@@ -741,6 +755,7 @@ async def import_transactions(payee=import_payee_raw,payee_id=import_payee_id,category_id=category_id,+ notes=getattr(txn_data, "notes", None),recurring_transaction_id=recurring_link.id if recurring_link else None,)apply_effective_date(transaction, account)@@ -768,7 +783,7 @@ async def import_transactions(await session.commit()return imported, skipped, excluded_count, import_log.id-def normalize_amount(amount_str: str) -> str:+def normalize_amount(amount_str: str | None) -> str:"""Normalize monetary string into a standard decimal format compatible with Decimal.@@ -776,8 +791,10 @@ def normalize_amount(amount_str: str) -> str:1.442,20 -> 1442.201,442.20 -> 1442.20"""+ if not amount_str:+ return ""- amount_str = amount_str.replace('R$', '').strip()+ amount_str = str(amount_str).replace('R$', '').strip()if ',' in amount_str and '.' in amount_str:if amount_str.rfind(',') > amount_str.rfind('.'):diff --git a/backend/app/services/payee_service.py b/backend/app/services/payee_service.pyindex dd05cac8..04e09373 100644--- a/backend/app/services/payee_service.py+++ b/backend/app/services/payee_service.py@@ -81,6 +81,9 @@ async def get_or_create_payee(if not name:raise ValueError("Payee name cannot be empty")+ if len(name) > 255:+ name = name[:255]+lookup = select(Payee).where(func.lower(Payee.name) == name.lower())if workspace_id is not None:lookup = lookup.where(Payee.workspace_id == workspace_id)diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.pyindex b1158b9a..49ba012d 100644--- a/backend/tests/test_import_service.py+++ b/backend/tests/test_import_service.py@@ -256,6 +256,17 @@ class TestParseCsv:assert transactions[0].amount == Decimal("5000.00")assert transactions[1].amount == Decimal("1200.00")+ def test_parse_csv_autodetects_payee_and_notes(self):+ """CSV with common headers for payee and notes should auto-detect them. external_id is strictly explicit."""+ csv_content = (+ "date,description,amount,merchant,transaction_id,notes\n"+ "2026-05-01,AMAZON,-25.00,Amazon.com,txn_123,Gift for John\n"+ )+ transactions = parse_csv(csv_content.encode("utf-8"))+ assert len(transactions) == 1+ assert transactions[0].payee_raw == "Amazon.com"+ assert transactions[0].external_id is None+ assert transactions[0].notes == "Gift for John"class TestParseCsvColumnMapping:"""Tests for customizable CSV column mapping (issue #201)."""@@ -445,6 +456,28 @@ class TestParseCsvColumnMapping:assert transactions[1].amount == Decimal("200.00")assert transactions[1].type == "debit"+ def test_column_mapping_payee_external_id_notes(self):+ """Explicitly mapping payee, external_id, and notes should extract them correctly."""+ csv_content = (+ "Txn Date,Memo,Value,Counterparty,ExtRef,Comment\n"+ "2026-08-01,Purchase,-15.00,Target,ref999,Groceries\n"+ )+ transactions = parse_csv(+ csv_content.encode("utf-8"),+ column_mapping={+ "date": "Txn Date",+ "description": "Memo",+ "amount": "Value",+ "payee": "Counterparty",+ "external_id": "ExtRef",+ "notes": "Comment",+ },+ )+ assert len(transactions) == 1+ assert transactions[0].payee_raw == "Target"+ assert transactions[0].external_id == "ref999"+ assert transactions[0].notes == "Groceries"+class TestDetectCsvColumns:"""Tests for detect_csv_columns — used to drive the import-UI mapping dropdowns."""@@ -1266,6 +1299,32 @@ class TestImportTransactionsFx:assert payee.workspace_id == test_workspace.idassert tx.workspace_id == test_workspace.id+ @pytest.mark.asyncio+ async def test_import_csv_preserves_notes_and_external_id(self, session: AsyncSession, test_user: User, test_workspace, test_account: Account):+ from app.models.transaction import Transaction+ from app.schemas.transaction import TransactionImport+ from sqlalchemy import select++ txns = [+ TransactionImport(+ description="Hardware Store",+ amount=Decimal("50.00"),+ date=date(2026, 2, 10),+ type="debit",+ currency=test_account.currency,+ external_id="ext_456",+ notes="Tools for repair",+ ),+ ]++ imported, _, _, _ = await import_transactions(+ session, test_workspace.id, test_user.id, test_account.id, txns, "csv",+ )++ assert imported == 1+ tx = (await session.execute(select(Transaction).where(Transaction.external_id == "ext_456"))).scalar_one()+ assert tx.notes == "Tools for repair"+@pytest.mark.asyncio@patch("app.services.fx_rate_service._provider")async def test_import_foreign_currency_without_fx_rate_auto_converts(diff --git a/frontend/src/locales/de.json b/frontend/src/locales/de.jsonindex 21a5553c..2d6cd509 100644--- a/frontend/src/locales/de.json+++ b/frontend/src/locales/de.json@@ -983,7 +983,10 @@"balanceImpact": "Auswirkung auf den Saldo","importedWithExcluded": "{{imported}} importiert, {{skipped}} übersprungen, {{excluded}} ausgeschlossen","suggested": "Vorgeschlagen",- "page": "Seite {{current}} von {{total}}"+ "page": "Seite {{current}} von {{total}}",+ "mapPayee": "Spalte Zahlungsempfänger",+ "mapExternalId": "Spalte Transaktions-ID",+ "mapNotes": "Spalte Notizen"},"assets": {"title": "Vermögenswerte",diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.jsonindex b0ac7a13..7a3baaad 100644--- a/frontend/src/locales/en.json+++ b/frontend/src/locales/en.json@@ -983,7 +983,10 @@"balanceImpact": "Balance impact","importedWithExcluded": "{{imported}} imported, {{skipped}} skipped, {{excluded}} excluded","suggested": "Suggested",- "page": "Page {{current}} of {{total}}"+ "page": "Page {{current}} of {{total}}",+ "mapPayee": "Payee column",+ "mapExternalId": "Transaction ID column",+ "mapNotes": "Notes column"},"assets": {"title": "Assets",diff --git a/frontend/src/locales/es.json b/frontend/src/locales/es.jsonindex a11e72d6..536d0441 100644--- a/frontend/src/locales/es.json+++ b/frontend/src/locales/es.json@@ -983,7 +983,10 @@"balanceImpact": "Impacto en el saldo","importedWithExcluded": "{{imported}} importadas, {{skipped}} omitidas, {{excluded}} excluidas","suggested": "Sugerido",- "page": "Página {{current}} de {{total}}"+ "page": "Página {{current}} de {{total}}",+ "mapPayee": "Columna de beneficiario",+ "mapExternalId": "Columna de ID de transacción",+ "mapNotes": "Columna de notas"},"assets": {"title": "Activos",diff --git a/frontend/src/locales/fr.json b/frontend/src/locales/fr.jsonindex 47cd4fa5..e1ef5e45 100644--- a/frontend/src/locales/fr.json+++ b/frontend/src/locales/fr.json@@ -983,7 +983,10 @@"balanceImpact": "Impact sur le solde","importedWithExcluded": "{{imported}} importées, {{skipped}} ignorées, {{excluded}} exclues","suggested": "Suggérée",- "page": "Page {{current}} sur {{total}}"+ "page": "Page {{current}} sur {{total}}",+ "mapPayee": "Colonne du bénéficiaire",+ "mapExternalId": "Colonne d'ID de transaction",+ "mapNotes": "Colonne des notes"},"assets": {"title": "Actifs",diff --git a/frontend/src/locales/it.json b/frontend/src/locales/it.jsonindex a667ccea..c39a9cdc 100644--- a/frontend/src/locales/it.json+++ b/frontend/src/locales/it.json@@ -983,7 +983,10 @@"balanceImpact": "Impatto sul saldo","importedWithExcluded": "{{imported}} importate, {{skipped}} saltate, {{excluded}} escluse","suggested": "Suggerita",- "page": "Pagina {{current}} di {{total}}"+ "page": "Pagina {{current}} di {{total}}",+ "mapPayee": "Colonna del beneficiario",+ "mapExternalId": "Colonna dell'ID transazione",+ "mapNotes": "Colonna delle note"},"assets": {"title": "Beni",diff --git a/frontend/src/locales/pl.json b/frontend/src/locales/pl.jsonindex bcee924c..a06358b4 100644--- a/frontend/src/locales/pl.json+++ b/frontend/src/locales/pl.json@@ -1052,7 +1052,10 @@"balanceImpact": "Wpływ na saldo","importedWithExcluded": "Zaimportowano: {{imported}}, pominięto: {{skipped}}, wykluczono: {{excluded}}","suggested": "Sugerowane",- "page": "Strona {{current}} z {{total}}"+ "page": "Strona {{current}} z {{total}}",+ "mapPayee": "Kolumna odbiorcy",+ "mapExternalId": "Kolumna ID transakcji",+ "mapNotes": "Kolumna notatek"},"assets": {"title": "Aktywa",diff --git a/frontend/src/locales/pt-BR.json b/frontend/src/locales/pt-BR.jsonindex 0bd5fbfb..9586c009 100644--- a/frontend/src/locales/pt-BR.json+++ b/frontend/src/locales/pt-BR.json@@ -983,7 +983,10 @@"balanceImpact": "Impacto no saldo","importedWithExcluded": "{{imported}} importadas, {{skipped}} ignoradas, {{excluded}} excluídas","suggested": "Sugerida",- "page": "Página {{current}} de {{total}}"+ "page": "Página {{current}} de {{total}}",+ "mapPayee": "Coluna de beneficiário",+ "mapExternalId": "Coluna de ID da transação",+ "mapNotes": "Coluna de notas"},"assets": {"title": "Patrimônio",diff --git a/frontend/src/locales/pt-PT.json b/frontend/src/locales/pt-PT.jsonindex c8a047cd..b1ab5a3c 100644--- a/frontend/src/locales/pt-PT.json+++ b/frontend/src/locales/pt-PT.json@@ -940,6 +940,9 @@"mapCategory": "Coluna de categoria","mapCurrency": "Coluna de moeda","mapFxRate": "Coluna de câmbio",+ "mapPayee": "Coluna de beneficiário",+ "mapExternalId": "Coluna de ID da transação",+ "mapNotes": "Coluna de notas","title": "Importar","subtitle": "Extrato bancário","processing": "A processar ficheiro…",diff --git a/frontend/src/locales/ru.json b/frontend/src/locales/ru.jsonindex 06dab85c..1504f3e5 100644--- a/frontend/src/locales/ru.json+++ b/frontend/src/locales/ru.json@@ -983,7 +983,10 @@"balanceImpact": "Влияние на баланс","importedWithExcluded": "{{imported}} импортировано, {{skipped}} пропущено, {{excluded}} исключено","suggested": "Предложено",- "page": "Страница {{current}} из {{total}}"+ "page": "Страница {{current}} из {{total}}",+ "mapPayee": "Колонка получателя",+ "mapExternalId": "Колонка ID транзакции",+ "mapNotes": "Колонка заметок"},"assets": {"title": "Активы",diff --git a/frontend/src/locales/uk.json b/frontend/src/locales/uk.jsonindex 44a9b812..7eb48288 100644--- a/frontend/src/locales/uk.json+++ b/frontend/src/locales/uk.json@@ -983,7 +983,10 @@"balanceImpact": "Вплив на баланс","importedWithExcluded": "{{imported}} імпортовано, {{skipped}} пропущено, {{excluded}} виключено","suggested": "Запропоновано",- "page": "Сторінка {{current}} із {{total}}"+ "page": "Сторінка {{current}} із {{total}}",+ "mapPayee": "Стовпець одержувача",+ "mapExternalId": "Стовпець ID транзакції",+ "mapNotes": "Стовпець приміток"},"assets": {"title": "Активи",diff --git a/frontend/src/pages/import.tsx b/frontend/src/pages/import.tsxindex bdde682c..40affa8c 100644--- a/frontend/src/pages/import.tsx+++ b/frontend/src/pages/import.tsx@@ -34,6 +34,9 @@ const CSV_MAPPING_FIELDS = [{ key: 'category', label: 'import.mapCategory' },{ key: 'currency', label: 'import.mapCurrency' },{ key: 'fx_rate', label: 'import.mapFxRate' },+ { key: 'payee', label: 'import.mapPayee' },+ { key: 'external_id', label: 'import.mapExternalId' },+ { key: 'notes', label: 'import.mapNotes' },] as constfunction toReviewTransactions(txns: ImportPreviewTransaction[]): ImportReviewTransaction[] {@@ -127,6 +130,7 @@ export default function ImportPage() {currency: rt.currency ?? undefined,fx_rate: rt.fx_rate ?? undefined,payee_raw: rt.payee_raw ?? undefined,+ notes: rt.notes ?? undefined,category_name: rt.category_name ?? undefined,excluded: rt.excluded,category_id: rt.selected_category_id !== undefined@@ -145,6 +149,8 @@ export default function ImportPage() {onSuccess: (data) => {invalidateFinancialQueries(queryClient)queryClient.invalidateQueries({ queryKey: ['import-logs'] })+ queryClient.invalidateQueries({ queryKey: ['payees'] })+ queryClient.invalidateQueries({ queryKey: ['categories'] })const hasSkippedOrExcluded = (data.skipped ?? 0) > 0 || (data.excluded ?? 0) > 0const msg = hasSkippedOrExcluded? t('import.importedWithExcluded', { imported: data.imported, skipped: data.skipped ?? 0, excluded: data.excluded ?? 0 })diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.tsindex fe58bc3a..85f4c28b 100644--- a/frontend/src/types/index.ts+++ b/frontend/src/types/index.ts@@ -413,6 +413,7 @@ export interface ImportPreviewTransaction {excluded?: booleancategory_id?: string | nullforce_uncategorized?: boolean+ notes?: string | null}export interface ImportReviewTransaction extends ImportPreviewTransaction {