Open-source personal finance manager. Self-hosted, privacy-first.
fix(import): parse CSV amounts with thousands separators and currency symbols (#1008)
Tássio Noronhacfe2f5da25f3
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.pyindex 046baf03..6408e8ea 100644--- a/backend/app/services/import_service.py+++ b/backend/app/services/import_service.py@@ -459,7 +459,7 @@ def parse_csv(date_cols = ['date', 'data', 'dt', 'transaction_date', 'data_transacao']desc_cols = ['description', 'descricao', 'desc', 'memo', 'historico', 'lancamento']amount_cols = ['amount', 'valor', 'value', 'quantia']- type_cols = ['type', 'tipo']+ type_cols = ['type', 'tipo', 'transaction type', 'transaction_type']category_cols = ['category', 'categoria']currency_cols = ['currency', 'moeda', 'currency_code']fx_rate_cols = ['fx_rate', 'fx_rate_used', 'taxa_cambio', 'exchange_rate', 'taxa']@@ -537,6 +537,17 @@ def parse_csv(else:date_formats = ['%Y-%m-%d', '%d/%m/%Y', '%d-%m-%Y', '%m/%d/%Y', '%d.%m.%Y']+ # Decide the decimal separator once per file from every amount cell, so+ # "25,000" next to "1,500.50" reads as twenty-five thousand rather than+ # being guessed in isolation.+ amount_fields = [c for c in (inflow_col, outflow_col) if c] if use_split else [amount_col]+ decimal_separator = infer_decimal_separator(+ v+ for r in csv.DictReader(io.StringIO(text), dialect=dialect)+ for k, v in r.items()+ if k is not None and k.lower().strip() in amount_fields+ )+transactions = []failed_rows = []for row in reader:@@ -567,8 +578,8 @@ def parse_csv(# Parse amountif use_split:- inflow_str = normalize_amount(row.get(inflow_col, ""))- outflow_str = normalize_amount(row.get(outflow_col, ""))+ inflow_str = normalize_amount(row.get(inflow_col, ""), decimal_separator)+ outflow_str = normalize_amount(row.get(outflow_col, ""), decimal_separator)try:inflow = Decimal(inflow_str) if inflow_str else Decimal('0')@@ -590,7 +601,7 @@ def parse_csv(failed_rows.append(FailedRow(line_number=reader.line_num, description=row.get(desc_col, "").strip(), raw_value=raw_val, error_reason="no_amount"))continue # Skip rows with no amountelse:- amount_str = normalize_amount(row[amount_col])+ amount_str = normalize_amount(row[amount_col], decimal_separator)try:amount = Decimal(amount_str)@@ -601,8 +612,9 @@ def parse_csv(if flip_amount:amount = -amount- if type_col and row.get(type_col, '').strip() in ('credit', 'debit'):- txn_type = row[type_col].strip()+ raw_type = row.get(type_col, '').strip().lower() if type_col else ''+ if raw_type in ('credit', 'debit'):+ txn_type = raw_typeelse:txn_type = "credit" if amount > 0 else "debit"amount = abs(amount)@@ -976,21 +988,112 @@ async def import_transactions(await session.commit()return imported, skipped, excluded_count, import_log.id-def normalize_amount(amount_str: str | None) -> str:+# Currency symbols and ISO codes around a number ("R$", "$", "NGN ", " EUR").+_AMOUNT_EDGE_LEADING = re.compile(r'^[^\d\-+(),.]+')+_AMOUNT_EDGE_TRAILING = re.compile(r'[^\d\-+(),.]+$')+_COMMA_DECIMAL_TAIL = re.compile(r',\d{1,2}$')+_DOT_DECIMAL_TAIL = re.compile(r'\.\d{1,2}$')+_COMMA_THOUSANDS = re.compile(r'^\d{1,3}(,\d{3})+$')+_ZERO_COMMA_DECIMAL = re.compile(r'^0,\d+$')+_DR_CR_SUFFIX = re.compile(r'(?i)(?<![a-z])(dr|cr)\.?$')+# U+2212 minus, U+2012 figure dash, U+2013 en dash, U+FE63 small and+# U+FF0D fullwidth hyphen-minus: spreadsheet exports use them as a minus.+_UNICODE_MINUS = str.maketrans({c: '-' for c in '\u2212\u2012\u2013\ufe63\uff0d'})+++def _strip_amount_decorations(amount_str: str) -> tuple[str, bool]:+ """Remove currency symbols, codes, whitespace and sign markers.++ Returns the bare number and whether it was negative. A leading minus+ (ASCII or a Unicode minus sign), accounting parentheses, e.g. "(12.50)",+ or a trailing "DR" mark a negative amount; a trailing "CR" marks a+ positive one.+ """+ s = re.sub(r"[\s'\u00a0\u202f]", "", amount_str).translate(_UNICODE_MINUS)+ # A standalone DR/CR suffix carries the sign, so read it before the edge+ # strip below would drop it as a currency code. Codes such as "XDR" or+ # "CRC" do not match.+ marker = _DR_CR_SUFFIX.search(s)+ if marker:+ s = s[: marker.start()]+ negative = False+ while True:+ before = s+ s = _AMOUNT_EDGE_LEADING.sub("", s)+ s = _AMOUNT_EDGE_TRAILING.sub("", s)+ if len(s) >= 2 and s[0] == "(" and s[-1] == ")":+ negative = not negative+ s = s[1:-1]+ elif s[:1] == "-":+ negative = not negative+ s = s[1:]+ elif s[:1] == "+":+ s = s[1:]+ if s == before:+ if marker:+ negative = marker.group(1).lower() == "dr"+ return s, negative+++def infer_decimal_separator(values) -> str | None:+ """Infer the decimal separator used by a whole column of amounts.++ Returns "." or ",", or None when the values give no clear signal, in+ which case normalize_amount falls back to deciding cell by cell.+ """+ votes: set[str] = set()+ comma_thousands = False+ for raw in values:+ if not raw:+ continue+ s, _ = _strip_amount_decorations(str(raw))+ if not s:+ continue+ if ',' in s and '.' in s:+ votes.add(',' if s.rfind(',') > s.rfind('.') else '.')+ elif _COMMA_DECIMAL_TAIL.search(s):+ votes.add(',')+ elif _DOT_DECIMAL_TAIL.search(s):+ votes.add('.')+ elif _COMMA_THOUSANDS.match(s) and not s.startswith('0,'):+ comma_thousands = True+ if len(votes) == 1:+ return votes.pop()+ if not votes and comma_thousands:+ return '.'+ return None+++def normalize_amount(amount_str: str | None, decimal_separator: str | None = None) -> str:"""Normalize monetary string into a standard decimal format compatible with Decimal.+ Currency symbols and codes around the number are dropped, and accounting+ parentheses read as a negative amount. When decimal_separator is known+ for the column ("." or ","), the other separator is treated as grouping;+ otherwise the separator is guessed from the cell alone.+Example:1.442,20 -> 1442.201,442.20 -> 1442.20+ $40.00 -> 40.00+ 25,000 (decimal_separator=".") -> 25000"""if not amount_str:return ""- # Strip currency prefix and Swiss thousands separators (single quote)- amount_str = str(amount_str).replace('R$', '').replace("'", "").strip()+ amount_str, negative = _strip_amount_decorations(str(amount_str))+ if not amount_str:+ return ""- if ',' in amount_str and '.' in amount_str:+ if decimal_separator == ',':+ amount_str = amount_str.replace('.', '').replace(',', '.')+ elif decimal_separator == '.':+ if _ZERO_COMMA_DECIMAL.match(amount_str):+ amount_str = amount_str.replace(',', '.')+ else:+ amount_str = amount_str.replace(',', '')+ elif ',' in amount_str and '.' in amount_str:if amount_str.rfind(',') > amount_str.rfind('.'):amount_str = amount_str.replace('.', '').replace(',', '.')else:@@ -998,4 +1101,4 @@ def normalize_amount(amount_str: str | None) -> str:elif ',' in amount_str:amount_str = amount_str.replace(',', '.')- return amount_str+ return f"-{amount_str}" if negative else amount_strdiff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.pyindex 6ab493c2..7366a030 100644--- a/backend/tests/test_import_service.py+++ b/backend/tests/test_import_service.py@@ -2769,3 +2769,127 @@ def test_parse_csv_reports_short_rows_instead_of_raising():assert failed_rows[2].line_number == 5assert failed_rows[2].error_reason == "invalid_date"assert failed_rows[2].raw_value == "invalid_date"+++def test_parse_csv_comma_thousands_inferred_per_column():+ from app.services.import_service import parse_csv+ csv_content = (+ "Date,Transaction Type,Amount,Description\n"+ '2026-08-01,Credit,"375,000.00",SALARY\n'+ '2026-08-02,Debit,"25,000",TRANSFER\n'+ '2026-08-03,DEBIT,"1,500.50",AIRTIME\n'+ '2026-08-05,debit,"3,000",POS\n'+ )+ transactions, failed_rows = parse_csv(csv_content.encode("utf-8"))++ assert failed_rows == []+ assert [t.amount for t in transactions] == [+ Decimal("375000.00"), Decimal("25000"), Decimal("1500.50"), Decimal("3000"),+ ]+ assert [t.type for t in transactions] == ["credit", "debit", "debit", "debit"]+++def test_parse_csv_comma_thousands_only_column():+ from app.services.import_service import parse_csv+ csv_content = (+ "date,description,amount\n"+ '2026-08-02,Transfer,"-25,000"\n'+ '2026-08-03,Salary,"1,234,567"\n'+ )+ transactions, _ = parse_csv(csv_content.encode("utf-8"))+ assert [t.amount for t in transactions] == [Decimal("25000"), Decimal("1234567")]+ assert [t.type for t in transactions] == ["debit", "credit"]+++def test_parse_csv_brazilian_column_still_parses():+ from app.services.import_service import parse_csv+ csv_content = (+ "data;descricao;valor\n"+ "01/08/2026;Mercado;-1.234,56\n"+ "02/08/2026;Padaria;-12,50\n"+ "03/08/2026;Aluguel;-2.000\n"+ )+ transactions, _ = parse_csv(csv_content.encode("utf-8"))+ assert [t.amount for t in transactions] == [+ Decimal("1234.56"), Decimal("12.50"), Decimal("2000"),+ ]+ assert all(t.type == "debit" for t in transactions)+++def test_parse_csv_zero_comma_is_decimal():+ from app.services.import_service import parse_csv+ csv_content = 'date,description,amount\n2026-08-01,Fee,"0,125"\n'+ transactions, _ = parse_csv(csv_content.encode("utf-8"))+ assert transactions[0].amount == Decimal("0.125")+++def test_parse_csv_strips_currency_symbols_and_codes():+ from app.services.import_service import parse_csv+ csv_content = (+ "date,description,amount\n"+ "2026-08-01,A,$40.00\n"+ "2026-08-02,B,NGN 2300.50\n"+ "2026-08-03,C,USD 10\n"+ "2026-08-04,D,-₦1500.00\n"+ )+ transactions, failed_rows = parse_csv(csv_content.encode("utf-8"))+ assert failed_rows == []+ assert [t.amount for t in transactions] == [+ Decimal("40.00"), Decimal("2300.50"), Decimal("10"), Decimal("1500.00"),+ ]+ assert transactions[3].type == "debit"+++def test_normalize_amount_currency_and_separators():+ from app.services.import_service import normalize_amount+ assert normalize_amount("$40.00") == "40.00"+ assert normalize_amount("€12,50") == "12.50"+ assert normalize_amount("NGN 2,300.50") == "2300.50"+ assert normalize_amount("₦1,500.00") == "1500.00"+ assert normalize_amount("USD 10") == "10"+ assert normalize_amount("12,50 EUR") == "12.50"+ assert normalize_amount("R$ 1.234,56") == "1234.56"+ assert normalize_amount("-$40.00") == "-40.00"+ assert normalize_amount("(12.50)") == "-12.50"+ assert normalize_amount("25,000", ".") == "25000"+ assert normalize_amount("0,125", ".") == "0.125"+ assert normalize_amount("1.234", ",") == "1234"+ # Without a column hint the per-cell behaviour is unchanged.+ assert normalize_amount("0,125") == "0.125"+ assert normalize_amount("12,50") == "12.50"+++def test_infer_decimal_separator():+ from app.services.import_service import infer_decimal_separator+ assert infer_decimal_separator(["25,000", "1,500.50"]) == "."+ assert infer_decimal_separator(["1.234,56", "12,50"]) == ","+ assert infer_decimal_separator(["25,000", "3,000"]) == "."+ assert infer_decimal_separator(["0,125"]) is None+ assert infer_decimal_separator(["10", ""]) is None+++def test_parse_csv_dr_cr_and_unicode_minus_keep_the_sign():+ from app.services.import_service import parse_csv+ csv_content = (+ "date,description,amount\n"+ "2026-08-01,Card,100.00 DR\n"+ "2026-08-02,Refund,50.00 CR\n"+ "2026-08-03,Fee,\u221240.00\n"+ "2026-08-04,Costa Rica,CRC 10.00\n"+ )+ transactions, failed_rows = parse_csv(csv_content.encode("utf-8"))+ assert failed_rows == []+ assert [(t.type, t.amount) for t in transactions] == [+ ("debit", Decimal("100.00")),+ ("credit", Decimal("50.00")),+ ("debit", Decimal("40.00")),+ ("credit", Decimal("10.00")),+ ]+++def test_normalize_amount_dr_cr_markers():+ from app.services.import_service import normalize_amount+ assert normalize_amount("1,234.56DR") == "-1234.56"+ assert normalize_amount("-10.00 CR") == "10.00"+ assert normalize_amount("10.00 XDR") == "10.00"+ assert normalize_amount("\u20131.50") == "-1.50"