Open-source personal finance manager. Self-hosted, privacy-first.
fix: decode OFX 2.x/XML imports without a legacy SGML header (#480)
Jyasapara3e67fa8a59ba
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.pyindex 2216a5a4..a0692ef2 100644--- a/backend/app/services/import_service.py+++ b/backend/app/services/import_service.py@@ -38,7 +38,18 @@ _OFX_BALANCE_ROW_DESCRIPTIONS = ()-def _preprocess_ofx_for_empty_fitid(content: bytes) -> bytes:+def _decode_ofx_bytes(content: bytes) -> tuple[str, str]:+ """Decode OFX content to text, trying UTF-8 then falling back to Latin-1.++ Returns (text, encoding) so callers can re-encode consistently later.+ """+ try:+ return content.decode("utf-8"), "utf-8"+ except UnicodeDecodeError:+ return content.decode("latin-1"), "latin-1"+++def _patch_empty_fitids(text: str) -> str:"""Synthesize a FITID for STMTTRN blocks that have an empty/missing one.Banco do Brasil (and a few other Brazilian banks) emit balance-summary@@ -47,13 +58,6 @@ def _preprocess_ofx_for_empty_fitid(content: bytes) -> bytes:each affected block with a deterministic synthetic FITID so parsingsucceeds; balance rows are filtered out later by description."""- try:- text = content.decode("utf-8")- original_encoding = "utf-8"- except UnicodeDecodeError:- text = content.decode("latin-1")- original_encoding = "latin-1"-def _replace(match: re.Match) -> str:block = match.group(0)fitid_match = re.search(r"<FITID>([^<\r\n]*)", block, re.IGNORECASE)@@ -74,13 +78,56 @@ def _preprocess_ofx_for_empty_fitid(content: bytes) -> bytes:flags=re.IGNORECASE,)- patched = re.sub(+ return re.sub(r"<STMTTRN>.*?</STMTTRN>",_replace,text,flags=re.IGNORECASE | re.DOTALL,)- return patched.encode(original_encoding, errors="replace")+++def _ensure_ofx_sgml_header(text: str, encoding: str) -> str:+ """Prepend a legacy OFX 1.x SGML header for OFX 2.x files that omit it.++ OFX 2.x is plain XML that goes straight into its first tag, with no+ colon-delimited SGML header block (e.g. Erste Bank's "MS Money Sunset+ Deluxe" export). ofxparse only looks for encoding hints in the bytes+ preceding the file's first "<"; when that's empty it silently assumes+ ASCII and crashes on any non-ASCII byte. Prepending a synthetic SGML+ header routes the file through ofxparse's existing, correctly-working+ SGML decode path instead of its broken auto-detection (see+ https://github.com/jseutter/ofxparse/issues/133).++ The trigger is therefore "nothing precedes the first tag", not "starts+ with <?xml": the XML declaration is optional in XML 1.0, so an OFX 2.x+ file may open with just its <?OFX ... ?> instruction, or with <OFX>+ itself, and those hit the same ofxparse bug. Anything else already has a+ legacy header, and a file with no tag at all is left for ofxparse to+ reject on its own terms.++ `encoding` must match whatever the caller will re-encode `text` with,+ so the declared header and the actual bytes stay consistent.+ """+ preamble, first_tag, _ = text.lstrip("\ufeff \t\r\n").partition("<")+ if not first_tag or preamble.strip():+ return text+ if encoding == "latin-1":+ enc_lines = "ENCODING:USASCII\r\nCHARSET:8859-1\r\n"+ else:+ enc_lines = "ENCODING:UTF-8\r\nCHARSET:NONE\r\n"+ header = (+ f"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"+ f"{enc_lines}COMPRESSION:NONE\r\nOLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"+ )+ return header + text+++def _preprocess_ofx(content: bytes) -> bytes:+ """Apply text-level fixups ofxparse needs before it can parse the file."""+ text, encoding = _decode_ofx_bytes(content)+ text = _patch_empty_fitids(text)+ text = _ensure_ofx_sgml_header(text, encoding)+ return text.encode(encoding, errors="replace")def _is_balance_summary_row(description: str | None) -> bool:@@ -92,7 +139,7 @@ def _is_balance_summary_row(description: str | None) -> bool:def parse_ofx(content: bytes) -> list[TransactionImport]:"""Parse OFX file content and return transactions."""- content = _preprocess_ofx_for_empty_fitid(content)+ content = _preprocess_ofx(content)ofx = OfxParser.parse(io.BytesIO(content))transactions = []diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.pyindex ac6587b4..b1158b9a 100644--- a/backend/tests/test_import_service.py+++ b/backend/tests/test_import_service.py@@ -10,6 +10,7 @@ from app.models.account import Accountfrom app.models.fx_rate import FxRatefrom app.models.user import Userfrom app.services.import_service import (+ _preprocess_ofx,detect_csv_columns,parse_csv,parse_ofx,@@ -966,6 +967,92 @@ class TestParseOfx:assert transactions[0].description == "UBER TRIP"assert transactions[0].amount == Decimal("100.00")+ def _make_ofx_xml(+ self, memo: str, xml_encoding: str, *, prolog: bool = True, ofx_pi: bool = True+ ) -> bytes:+ """Helper to build an OFX 2.x document: XML prolog, no SGML header+ block (e.g. Erste Bank's "MS Money Sunset Deluxe" export).++ The XML declaration is optional in XML 1.0 and the <?OFX ?> instruction+ can be dropped too, so `prolog` and `ofx_pi` cover the headerless+ variants that reach ofxparse with nothing before the first tag.+ """+ text = (+ (f'<?xml version="1.0" encoding="{xml_encoding}" ?>' if prolog else "")+ + ('<?OFX OFXHEADER="200" VERSION="202" SECURITY="NONE" '+ 'OLDFILEUID="NONE" NEWFILEUID="NONE"?>' if ofx_pi else "")+ + "<OFX><BANKMSGSRSV1><STMTTRNRS><STMTRS><CURDEF>EUR</CURDEF>"+ "<BANKACCTFROM><BANKID>1</BANKID><ACCTID>1</ACCTID>"+ "<ACCTTYPE>CHECKING</ACCTTYPE></BANKACCTFROM>"+ "<BANKTRANLIST><DTSTART>20260101</DTSTART><DTEND>20260131</DTEND>"+ "<STMTTRN><TRNTYPE>DEBIT</TRNTYPE><DTPOSTED>20260115</DTPOSTED>"+ "<TRNAMT>-10.00</TRNAMT><FITID>1</FITID>"+ f"<MEMO>{memo}</MEMO></STMTTRN>"+ "</BANKTRANLIST></STMTRS></STMTTRNRS></BANKMSGSRSV1></OFX>"+ )+ return text.encode(xml_encoding.lower().replace("iso-8859-1", "latin-1"))++ def test_parse_ofx_xml_format_without_sgml_header(self):+ """OFX 2.x/XML files (no SGML header block) with UTF-8 non-ASCII+ characters should parse without raising UnicodeDecodeError."""+ ofx = self._make_ofx_xml("1 Aufladung(en) für Konto", "utf-8")+ transactions = parse_ofx(ofx)++ assert len(transactions) == 1+ assert transactions[0].description == "1 Aufladung(en) für Konto"+ assert transactions[0].amount == Decimal("10.00")+ assert transactions[0].type == "debit"++ def test_parse_ofx_xml_format_latin1(self):+ """OFX 2.x/XML files declaring and using Latin-1 should decode+ correctly, proving the fix doesn't hardcode UTF-8."""+ ofx = self._make_ofx_xml("Café Paris", "ISO-8859-1")+ transactions = parse_ofx(ofx)++ assert len(transactions) == 1+ assert transactions[0].description == "Café Paris"++ def test_parse_ofx_xml_format_ascii_only(self):+ """OFX 2.x/XML files with pure-ASCII content should still parse+ correctly after the SGML header injection."""+ ofx = self._make_ofx_xml("Grocery Store", "utf-8")+ transactions = parse_ofx(ofx)++ assert len(transactions) == 1+ assert transactions[0].description == "Grocery Store"+ assert transactions[0].amount == Decimal("10.00")++ def test_parse_ofx_xml_format_without_xml_prolog(self):+ """The XML declaration is optional, so an OFX 2.x file may open with+ just its <?OFX ?> instruction. It reaches ofxparse with nothing before+ the first tag, so it hits the same bug and needs the same header."""+ ofx = self._make_ofx_xml("Aufladung für Konto", "utf-8", prolog=False)+ transactions = parse_ofx(ofx)++ assert len(transactions) == 1+ assert transactions[0].description == "Aufladung für Konto"++ def test_parse_ofx_xml_format_bare_ofx_element(self):+ """Neither declaration is required: a document opening straight on+ <OFX> is the same headerless case."""+ ofx = self._make_ofx_xml("Café Paris", "utf-8", prolog=False, ofx_pi=False)+ transactions = parse_ofx(ofx)++ assert len(transactions) == 1+ assert transactions[0].description == "Café Paris"++ def test_parse_ofx_sgml_header_is_not_duplicated(self):+ """A real OFX 1.x file already has a header before its first tag, so it+ must be left untouched rather than given a second one."""+ ofx = self._make_ofx_xml("Grocery Store", "utf-8", prolog=False, ofx_pi=False)+ sgml = (+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"+ b"ENCODING:UTF-8\r\nCHARSET:NONE\r\nCOMPRESSION:NONE\r\n"+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n" + ofx+ )+ assert _preprocess_ofx(sgml).count(b"OFXHEADER:") == 1+ assert parse_ofx(sgml)[0].description == "Grocery Store"+# ═══════════════════════════════════════════════════════════════════════════# MULTI-CURRENCY PARSING TESTS