Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix: improved csv import resilience and added support to map payee external id and notes (#474)

jorishaenseler15cc61e742f85c
diff --git a/backend/app/schemas/payee.py b/backend/app/schemas/payee.py
index b170b842..f285536e 100644
--- a/backend/app/schemas/payee.py
+++ b/backend/app/schemas/payee.py
@@ -3,22 +3,22 @@ from datetime import date as _Date, datetime
from decimal import Decimal
from typing import Optional
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
from app.schemas.category import CategoryRead
class PayeeCreate(BaseModel):
- name: str
+ name: str = Field(..., max_length=255)
type: str = "merchant"
- notes: Optional[str] = None
+ notes: Optional[str] = Field(None, max_length=1000)
class PayeeUpdate(BaseModel):
- name: Optional[str] = None
+ name: Optional[str] = Field(None, max_length=255)
type: Optional[str] = None
is_favorite: Optional[bool] = None
- notes: Optional[str] = None
+ notes: Optional[str] = Field(None, max_length=1000)
class PayeeRead(BaseModel):
diff --git a/backend/app/schemas/transaction.py b/backend/app/schemas/transaction.py
index 14615551..2d43cdc1 100644
--- a/backend/app/schemas/transaction.py
+++ b/backend/app/schemas/transaction.py
@@ -163,6 +163,7 @@ class TransactionImport(TransactionBase):
excluded: bool = False
category_id: Optional[uuid.UUID] = None
force_uncategorized: bool = False
+ notes: Optional[str] = None
class TransactionImportPreview(BaseModel):
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.py
index a0692ef2..77d085eb 100644
--- a/backend/app/services/import_service.py
+++ b/backend/app/services/import_service.py
@@ -336,6 +336,7 @@ DATE_FORMAT_MAP = {
CSV_MAPPABLE_FIELDS = (
'date', 'description', 'amount', 'type',
'category', 'currency', 'fx_rate', 'inflow', 'outflow',
+ 'payee', 'external_id', 'notes',
)
@@ -385,7 +386,7 @@ def parse_csv(
reader = csv.DictReader(io.StringIO(text), dialect=dialect)
# Normalize field names
- fieldnames = [f.lower().strip() for f in (reader.fieldnames or [])]
+ fieldnames = [f.lower().strip() if f is not None else "" for f in (reader.fieldnames or [])]
# Map common column names
date_cols = ['date', 'data', 'dt', 'transaction_date', 'data_transacao']
@@ -395,6 +396,9 @@ def parse_csv(
category_cols = ['category', 'categoria']
currency_cols = ['currency', 'moeda', 'currency_code']
fx_rate_cols = ['fx_rate', 'fx_rate_used', 'taxa_cambio', 'exchange_rate', 'taxa']
+ payee_cols = ['payee', 'merchant', 'beneficiary', 'beneficiario', 'pagador']
+ external_id_cols = [] # External ID must be mapped explicitly
+ notes_cols = ['notes', 'nota', 'observacao']
# Normalize the user-supplied column mapping (Securo field -> CSV header).
mapping = {
@@ -445,6 +449,9 @@ def parse_csv(
category_col = resolve_col('category', category_cols)
currency_col = resolve_col('currency', currency_cols)
fx_rate_col = resolve_col('fx_rate', fx_rate_cols)
+ payee_col = resolve_col('payee', payee_cols)
+ external_id_col = resolve_col('external_id', external_id_cols)
+ notes_col = resolve_col('notes', notes_cols)
if not date_col or not desc_col:
raise ValueError(
@@ -461,12 +468,12 @@ def parse_csv(
if date_format and date_format in DATE_FORMAT_MAP:
date_formats = [DATE_FORMAT_MAP[date_format]]
else:
- date_formats = ['%Y-%m-%d', '%d/%m/%Y', '%d-%m-%Y', '%m/%d/%Y']
+ date_formats = ['%Y-%m-%d', '%d/%m/%Y', '%d-%m-%Y', '%m/%d/%Y', '%d.%m.%Y']
transactions = []
for row in reader:
# Normalize row keys
- row = {k.lower().strip(): v for k, v in row.items()}
+ row = {k.lower().strip() if k is not None else "": v for k, v in row.items()}
# Parse date
date_str = row[date_col].strip()
@@ -534,6 +541,10 @@ def parse_csv(
except Exception:
pass
+ txn_payee = row[payee_col].strip() if payee_col and row.get(payee_col) else None
+ txn_external_id = row[external_id_col].strip() if external_id_col and row.get(external_id_col) else None
+ txn_notes = row[notes_col].strip() if notes_col and row.get(notes_col) else None
+
transactions.append(TransactionImport(
description=row[desc_col].strip(),
amount=abs(amount),
@@ -542,6 +553,9 @@ def parse_csv(
currency=txn_currency,
fx_rate=txn_fx_rate,
category_name=category_name,
+ payee_raw=txn_payee,
+ external_id=txn_external_id,
+ notes=txn_notes,
))
return transactions
@@ -741,6 +755,7 @@ async def import_transactions(
payee=import_payee_raw,
payee_id=import_payee_id,
category_id=category_id,
+ notes=getattr(txn_data, "notes", None),
recurring_transaction_id=recurring_link.id if recurring_link else None,
)
apply_effective_date(transaction, account)
@@ -768,7 +783,7 @@ async def import_transactions(
await session.commit()
return imported, skipped, excluded_count, import_log.id
-def normalize_amount(amount_str: str) -> str:
+def normalize_amount(amount_str: str | None) -> str:
"""
Normalize monetary string into a standard decimal format compatible with Decimal.
@@ -776,8 +791,10 @@ def normalize_amount(amount_str: str) -> str:
1.442,20 -> 1442.20
1,442.20 -> 1442.20
"""
+ if not amount_str:
+ return ""
- amount_str = amount_str.replace('R$', '').strip()
+ amount_str = str(amount_str).replace('R$', '').strip()
if ',' in amount_str and '.' in amount_str:
if amount_str.rfind(',') > amount_str.rfind('.'):
diff --git a/backend/app/services/payee_service.py b/backend/app/services/payee_service.py
index dd05cac8..04e09373 100644
--- a/backend/app/services/payee_service.py
+++ b/backend/app/services/payee_service.py
@@ -81,6 +81,9 @@ async def get_or_create_payee(
if not name:
raise ValueError("Payee name cannot be empty")
+ if len(name) > 255:
+ name = name[:255]
+
lookup = select(Payee).where(func.lower(Payee.name) == name.lower())
if workspace_id is not None:
lookup = lookup.where(Payee.workspace_id == workspace_id)
diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.py
index b1158b9a..49ba012d 100644
--- a/backend/tests/test_import_service.py
+++ b/backend/tests/test_import_service.py
@@ -256,6 +256,17 @@ class TestParseCsv:
assert transactions[0].amount == Decimal("5000.00")
assert transactions[1].amount == Decimal("1200.00")
+ def test_parse_csv_autodetects_payee_and_notes(self):
+ """CSV with common headers for payee and notes should auto-detect them. external_id is strictly explicit."""
+ csv_content = (
+ "date,description,amount,merchant,transaction_id,notes\n"
+ "2026-05-01,AMAZON,-25.00,Amazon.com,txn_123,Gift for John\n"
+ )
+ transactions = parse_csv(csv_content.encode("utf-8"))
+ assert len(transactions) == 1
+ assert transactions[0].payee_raw == "Amazon.com"
+ assert transactions[0].external_id is None
+ assert transactions[0].notes == "Gift for John"
class TestParseCsvColumnMapping:
"""Tests for customizable CSV column mapping (issue #201)."""
@@ -445,6 +456,28 @@ class TestParseCsvColumnMapping:
assert transactions[1].amount == Decimal("200.00")
assert transactions[1].type == "debit"
+ def test_column_mapping_payee_external_id_notes(self):
+ """Explicitly mapping payee, external_id, and notes should extract them correctly."""
+ csv_content = (
+ "Txn Date,Memo,Value,Counterparty,ExtRef,Comment\n"
+ "2026-08-01,Purchase,-15.00,Target,ref999,Groceries\n"
+ )
+ transactions = parse_csv(
+ csv_content.encode("utf-8"),
+ column_mapping={
+ "date": "Txn Date",
+ "description": "Memo",
+ "amount": "Value",
+ "payee": "Counterparty",
+ "external_id": "ExtRef",
+ "notes": "Comment",
+ },
+ )
+ assert len(transactions) == 1
+ assert transactions[0].payee_raw == "Target"
+ assert transactions[0].external_id == "ref999"
+ assert transactions[0].notes == "Groceries"
+
class TestDetectCsvColumns:
"""Tests for detect_csv_columns — used to drive the import-UI mapping dropdowns."""
@@ -1266,6 +1299,32 @@ class TestImportTransactionsFx:
assert payee.workspace_id == test_workspace.id
assert tx.workspace_id == test_workspace.id
+ @pytest.mark.asyncio
+ async def test_import_csv_preserves_notes_and_external_id(self, session: AsyncSession, test_user: User, test_workspace, test_account: Account):
+ from app.models.transaction import Transaction
+ from app.schemas.transaction import TransactionImport
+ from sqlalchemy import select
+
+ txns = [
+ TransactionImport(
+ description="Hardware Store",
+ amount=Decimal("50.00"),
+ date=date(2026, 2, 10),
+ type="debit",
+ currency=test_account.currency,
+ external_id="ext_456",
+ notes="Tools for repair",
+ ),
+ ]
+
+ imported, _, _, _ = await import_transactions(
+ session, test_workspace.id, test_user.id, test_account.id, txns, "csv",
+ )
+
+ assert imported == 1
+ tx = (await session.execute(select(Transaction).where(Transaction.external_id == "ext_456"))).scalar_one()
+ assert tx.notes == "Tools for repair"
+
@pytest.mark.asyncio
@patch("app.services.fx_rate_service._provider")
async def test_import_foreign_currency_without_fx_rate_auto_converts(
diff --git a/frontend/src/locales/de.json b/frontend/src/locales/de.json
index 21a5553c..2d6cd509 100644
--- a/frontend/src/locales/de.json
+++ b/frontend/src/locales/de.json
@@ -983,7 +983,10 @@
"balanceImpact": "Auswirkung auf den Saldo",
"importedWithExcluded": "{{imported}} importiert, {{skipped}} übersprungen, {{excluded}} ausgeschlossen",
"suggested": "Vorgeschlagen",
- "page": "Seite {{current}} von {{total}}"
+ "page": "Seite {{current}} von {{total}}",
+ "mapPayee": "Spalte Zahlungsempfänger",
+ "mapExternalId": "Spalte Transaktions-ID",
+ "mapNotes": "Spalte Notizen"
},
"assets": {
"title": "Vermögenswerte",
diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.json
index b0ac7a13..7a3baaad 100644
--- a/frontend/src/locales/en.json
+++ b/frontend/src/locales/en.json
@@ -983,7 +983,10 @@
"balanceImpact": "Balance impact",
"importedWithExcluded": "{{imported}} imported, {{skipped}} skipped, {{excluded}} excluded",
"suggested": "Suggested",
- "page": "Page {{current}} of {{total}}"
+ "page": "Page {{current}} of {{total}}",
+ "mapPayee": "Payee column",
+ "mapExternalId": "Transaction ID column",
+ "mapNotes": "Notes column"
},
"assets": {
"title": "Assets",
diff --git a/frontend/src/locales/es.json b/frontend/src/locales/es.json
index a11e72d6..536d0441 100644
--- a/frontend/src/locales/es.json
+++ b/frontend/src/locales/es.json
@@ -983,7 +983,10 @@
"balanceImpact": "Impacto en el saldo",
"importedWithExcluded": "{{imported}} importadas, {{skipped}} omitidas, {{excluded}} excluidas",
"suggested": "Sugerido",
- "page": "Página {{current}} de {{total}}"
+ "page": "Página {{current}} de {{total}}",
+ "mapPayee": "Columna de beneficiario",
+ "mapExternalId": "Columna de ID de transacción",
+ "mapNotes": "Columna de notas"
},
"assets": {
"title": "Activos",
diff --git a/frontend/src/locales/fr.json b/frontend/src/locales/fr.json
index 47cd4fa5..e1ef5e45 100644
--- a/frontend/src/locales/fr.json
+++ b/frontend/src/locales/fr.json
@@ -983,7 +983,10 @@
"balanceImpact": "Impact sur le solde",
"importedWithExcluded": "{{imported}} importées, {{skipped}} ignorées, {{excluded}} exclues",
"suggested": "Suggérée",
- "page": "Page {{current}} sur {{total}}"
+ "page": "Page {{current}} sur {{total}}",
+ "mapPayee": "Colonne du bénéficiaire",
+ "mapExternalId": "Colonne d'ID de transaction",
+ "mapNotes": "Colonne des notes"
},
"assets": {
"title": "Actifs",
diff --git a/frontend/src/locales/it.json b/frontend/src/locales/it.json
index a667ccea..c39a9cdc 100644
--- a/frontend/src/locales/it.json
+++ b/frontend/src/locales/it.json
@@ -983,7 +983,10 @@
"balanceImpact": "Impatto sul saldo",
"importedWithExcluded": "{{imported}} importate, {{skipped}} saltate, {{excluded}} escluse",
"suggested": "Suggerita",
- "page": "Pagina {{current}} di {{total}}"
+ "page": "Pagina {{current}} di {{total}}",
+ "mapPayee": "Colonna del beneficiario",
+ "mapExternalId": "Colonna dell'ID transazione",
+ "mapNotes": "Colonna delle note"
},
"assets": {
"title": "Beni",
diff --git a/frontend/src/locales/pl.json b/frontend/src/locales/pl.json
index bcee924c..a06358b4 100644
--- a/frontend/src/locales/pl.json
+++ b/frontend/src/locales/pl.json
@@ -1052,7 +1052,10 @@
"balanceImpact": "Wpływ na saldo",
"importedWithExcluded": "Zaimportowano: {{imported}}, pominięto: {{skipped}}, wykluczono: {{excluded}}",
"suggested": "Sugerowane",
- "page": "Strona {{current}} z {{total}}"
+ "page": "Strona {{current}} z {{total}}",
+ "mapPayee": "Kolumna odbiorcy",
+ "mapExternalId": "Kolumna ID transakcji",
+ "mapNotes": "Kolumna notatek"
},
"assets": {
"title": "Aktywa",
diff --git a/frontend/src/locales/pt-BR.json b/frontend/src/locales/pt-BR.json
index 0bd5fbfb..9586c009 100644
--- a/frontend/src/locales/pt-BR.json
+++ b/frontend/src/locales/pt-BR.json
@@ -983,7 +983,10 @@
"balanceImpact": "Impacto no saldo",
"importedWithExcluded": "{{imported}} importadas, {{skipped}} ignoradas, {{excluded}} excluídas",
"suggested": "Sugerida",
- "page": "Página {{current}} de {{total}}"
+ "page": "Página {{current}} de {{total}}",
+ "mapPayee": "Coluna de beneficiário",
+ "mapExternalId": "Coluna de ID da transação",
+ "mapNotes": "Coluna de notas"
},
"assets": {
"title": "Patrimônio",
diff --git a/frontend/src/locales/pt-PT.json b/frontend/src/locales/pt-PT.json
index c8a047cd..b1ab5a3c 100644
--- a/frontend/src/locales/pt-PT.json
+++ b/frontend/src/locales/pt-PT.json
@@ -940,6 +940,9 @@
"mapCategory": "Coluna de categoria",
"mapCurrency": "Coluna de moeda",
"mapFxRate": "Coluna de câmbio",
+ "mapPayee": "Coluna de beneficiário",
+ "mapExternalId": "Coluna de ID da transação",
+ "mapNotes": "Coluna de notas",
"title": "Importar",
"subtitle": "Extrato bancário",
"processing": "A processar ficheiro…",
diff --git a/frontend/src/locales/ru.json b/frontend/src/locales/ru.json
index 06dab85c..1504f3e5 100644
--- a/frontend/src/locales/ru.json
+++ b/frontend/src/locales/ru.json
@@ -983,7 +983,10 @@
"balanceImpact": "Влияние на баланс",
"importedWithExcluded": "{{imported}} импортировано, {{skipped}} пропущено, {{excluded}} исключено",
"suggested": "Предложено",
- "page": "Страница {{current}} из {{total}}"
+ "page": "Страница {{current}} из {{total}}",
+ "mapPayee": "Колонка получателя",
+ "mapExternalId": "Колонка ID транзакции",
+ "mapNotes": "Колонка заметок"
},
"assets": {
"title": "Активы",
diff --git a/frontend/src/locales/uk.json b/frontend/src/locales/uk.json
index 44a9b812..7eb48288 100644
--- a/frontend/src/locales/uk.json
+++ b/frontend/src/locales/uk.json
@@ -983,7 +983,10 @@
"balanceImpact": "Вплив на баланс",
"importedWithExcluded": "{{imported}} імпортовано, {{skipped}} пропущено, {{excluded}} виключено",
"suggested": "Запропоновано",
- "page": "Сторінка {{current}} із {{total}}"
+ "page": "Сторінка {{current}} із {{total}}",
+ "mapPayee": "Стовпець одержувача",
+ "mapExternalId": "Стовпець ID транзакції",
+ "mapNotes": "Стовпець приміток"
},
"assets": {
"title": "Активи",
diff --git a/frontend/src/pages/import.tsx b/frontend/src/pages/import.tsx
index bdde682c..40affa8c 100644
--- a/frontend/src/pages/import.tsx
+++ b/frontend/src/pages/import.tsx
@@ -34,6 +34,9 @@ const CSV_MAPPING_FIELDS = [
{ key: 'category', label: 'import.mapCategory' },
{ key: 'currency', label: 'import.mapCurrency' },
{ key: 'fx_rate', label: 'import.mapFxRate' },
+ { key: 'payee', label: 'import.mapPayee' },
+ { key: 'external_id', label: 'import.mapExternalId' },
+ { key: 'notes', label: 'import.mapNotes' },
] as const
function toReviewTransactions(txns: ImportPreviewTransaction[]): ImportReviewTransaction[] {
@@ -127,6 +130,7 @@ export default function ImportPage() {
currency: rt.currency ?? undefined,
fx_rate: rt.fx_rate ?? undefined,
payee_raw: rt.payee_raw ?? undefined,
+ notes: rt.notes ?? undefined,
category_name: rt.category_name ?? undefined,
excluded: rt.excluded,
category_id: rt.selected_category_id !== undefined
@@ -145,6 +149,8 @@ export default function ImportPage() {
onSuccess: (data) => {
invalidateFinancialQueries(queryClient)
queryClient.invalidateQueries({ queryKey: ['import-logs'] })
+ queryClient.invalidateQueries({ queryKey: ['payees'] })
+ queryClient.invalidateQueries({ queryKey: ['categories'] })
const hasSkippedOrExcluded = (data.skipped ?? 0) > 0 || (data.excluded ?? 0) > 0
const msg = hasSkippedOrExcluded
? t('import.importedWithExcluded', { imported: data.imported, skipped: data.skipped ?? 0, excluded: data.excluded ?? 0 })
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts
index fe58bc3a..85f4c28b 100644
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -413,6 +413,7 @@ export interface ImportPreviewTransaction {
excluded?: boolean
category_id?: string | null
force_uncategorized?: boolean
+ notes?: string | null
}
export interface ImportReviewTransaction extends ImportPreviewTransaction {