Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix: decode OFX 2.x/XML imports without a legacy SGML header (#480)

Jyasapara3e67fa8a59ba
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.py
index 2216a5a4..a0692ef2 100644
--- a/backend/app/services/import_service.py
+++ b/backend/app/services/import_service.py
@@ -38,7 +38,18 @@ _OFX_BALANCE_ROW_DESCRIPTIONS = (
)
-def _preprocess_ofx_for_empty_fitid(content: bytes) -> bytes:
+def _decode_ofx_bytes(content: bytes) -> tuple[str, str]:
+ """Decode OFX content to text, trying UTF-8 then falling back to Latin-1.
+
+ Returns (text, encoding) so callers can re-encode consistently later.
+ """
+ try:
+ return content.decode("utf-8"), "utf-8"
+ except UnicodeDecodeError:
+ return content.decode("latin-1"), "latin-1"
+
+
+def _patch_empty_fitids(text: str) -> str:
"""Synthesize a FITID for STMTTRN blocks that have an empty/missing one.
Banco do Brasil (and a few other Brazilian banks) emit balance-summary
@@ -47,13 +58,6 @@ def _preprocess_ofx_for_empty_fitid(content: bytes) -> bytes:
each affected block with a deterministic synthetic FITID so parsing
succeeds; balance rows are filtered out later by description.
"""
- try:
- text = content.decode("utf-8")
- original_encoding = "utf-8"
- except UnicodeDecodeError:
- text = content.decode("latin-1")
- original_encoding = "latin-1"
-
def _replace(match: re.Match) -> str:
block = match.group(0)
fitid_match = re.search(r"<FITID>([^<\r\n]*)", block, re.IGNORECASE)
@@ -74,13 +78,56 @@ def _preprocess_ofx_for_empty_fitid(content: bytes) -> bytes:
flags=re.IGNORECASE,
)
- patched = re.sub(
+ return re.sub(
r"<STMTTRN>.*?</STMTTRN>",
_replace,
text,
flags=re.IGNORECASE | re.DOTALL,
)
- return patched.encode(original_encoding, errors="replace")
+
+
+def _ensure_ofx_sgml_header(text: str, encoding: str) -> str:
+ """Prepend a legacy OFX 1.x SGML header for OFX 2.x files that omit it.
+
+ OFX 2.x is plain XML that goes straight into its first tag, with no
+ colon-delimited SGML header block (e.g. Erste Bank's "MS Money Sunset
+ Deluxe" export). ofxparse only looks for encoding hints in the bytes
+ preceding the file's first "<"; when that's empty it silently assumes
+ ASCII and crashes on any non-ASCII byte. Prepending a synthetic SGML
+ header routes the file through ofxparse's existing, correctly-working
+ SGML decode path instead of its broken auto-detection (see
+ https://github.com/jseutter/ofxparse/issues/133).
+
+ The trigger is therefore "nothing precedes the first tag", not "starts
+ with <?xml": the XML declaration is optional in XML 1.0, so an OFX 2.x
+ file may open with just its <?OFX ... ?> instruction, or with <OFX>
+ itself, and those hit the same ofxparse bug. Anything else already has a
+ legacy header, and a file with no tag at all is left for ofxparse to
+ reject on its own terms.
+
+ `encoding` must match whatever the caller will re-encode `text` with,
+ so the declared header and the actual bytes stay consistent.
+ """
+ preamble, first_tag, _ = text.lstrip("\ufeff \t\r\n").partition("<")
+ if not first_tag or preamble.strip():
+ return text
+ if encoding == "latin-1":
+ enc_lines = "ENCODING:USASCII\r\nCHARSET:8859-1\r\n"
+ else:
+ enc_lines = "ENCODING:UTF-8\r\nCHARSET:NONE\r\n"
+ header = (
+ f"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"
+ f"{enc_lines}COMPRESSION:NONE\r\nOLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"
+ )
+ return header + text
+
+
+def _preprocess_ofx(content: bytes) -> bytes:
+ """Apply text-level fixups ofxparse needs before it can parse the file."""
+ text, encoding = _decode_ofx_bytes(content)
+ text = _patch_empty_fitids(text)
+ text = _ensure_ofx_sgml_header(text, encoding)
+ return text.encode(encoding, errors="replace")
def _is_balance_summary_row(description: str | None) -> bool:
@@ -92,7 +139,7 @@ def _is_balance_summary_row(description: str | None) -> bool:
def parse_ofx(content: bytes) -> list[TransactionImport]:
"""Parse OFX file content and return transactions."""
- content = _preprocess_ofx_for_empty_fitid(content)
+ content = _preprocess_ofx(content)
ofx = OfxParser.parse(io.BytesIO(content))
transactions = []
diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.py
index ac6587b4..b1158b9a 100644
--- a/backend/tests/test_import_service.py
+++ b/backend/tests/test_import_service.py
@@ -10,6 +10,7 @@ from app.models.account import Account
from app.models.fx_rate import FxRate
from app.models.user import User
from app.services.import_service import (
+ _preprocess_ofx,
detect_csv_columns,
parse_csv,
parse_ofx,
@@ -966,6 +967,92 @@ class TestParseOfx:
assert transactions[0].description == "UBER TRIP"
assert transactions[0].amount == Decimal("100.00")
+ def _make_ofx_xml(
+ self, memo: str, xml_encoding: str, *, prolog: bool = True, ofx_pi: bool = True
+ ) -> bytes:
+ """Helper to build an OFX 2.x document: XML prolog, no SGML header
+ block (e.g. Erste Bank's "MS Money Sunset Deluxe" export).
+
+ The XML declaration is optional in XML 1.0 and the <?OFX ?> instruction
+ can be dropped too, so `prolog` and `ofx_pi` cover the headerless
+ variants that reach ofxparse with nothing before the first tag.
+ """
+ text = (
+ (f'<?xml version="1.0" encoding="{xml_encoding}" ?>' if prolog else "")
+ + ('<?OFX OFXHEADER="200" VERSION="202" SECURITY="NONE" '
+ 'OLDFILEUID="NONE" NEWFILEUID="NONE"?>' if ofx_pi else "")
+ + "<OFX><BANKMSGSRSV1><STMTTRNRS><STMTRS><CURDEF>EUR</CURDEF>"
+ "<BANKACCTFROM><BANKID>1</BANKID><ACCTID>1</ACCTID>"
+ "<ACCTTYPE>CHECKING</ACCTTYPE></BANKACCTFROM>"
+ "<BANKTRANLIST><DTSTART>20260101</DTSTART><DTEND>20260131</DTEND>"
+ "<STMTTRN><TRNTYPE>DEBIT</TRNTYPE><DTPOSTED>20260115</DTPOSTED>"
+ "<TRNAMT>-10.00</TRNAMT><FITID>1</FITID>"
+ f"<MEMO>{memo}</MEMO></STMTTRN>"
+ "</BANKTRANLIST></STMTRS></STMTTRNRS></BANKMSGSRSV1></OFX>"
+ )
+ return text.encode(xml_encoding.lower().replace("iso-8859-1", "latin-1"))
+
+ def test_parse_ofx_xml_format_without_sgml_header(self):
+ """OFX 2.x/XML files (no SGML header block) with UTF-8 non-ASCII
+ characters should parse without raising UnicodeDecodeError."""
+ ofx = self._make_ofx_xml("1 Aufladung(en) für Konto", "utf-8")
+ transactions = parse_ofx(ofx)
+
+ assert len(transactions) == 1
+ assert transactions[0].description == "1 Aufladung(en) für Konto"
+ assert transactions[0].amount == Decimal("10.00")
+ assert transactions[0].type == "debit"
+
+ def test_parse_ofx_xml_format_latin1(self):
+ """OFX 2.x/XML files declaring and using Latin-1 should decode
+ correctly, proving the fix doesn't hardcode UTF-8."""
+ ofx = self._make_ofx_xml("Café Paris", "ISO-8859-1")
+ transactions = parse_ofx(ofx)
+
+ assert len(transactions) == 1
+ assert transactions[0].description == "Café Paris"
+
+ def test_parse_ofx_xml_format_ascii_only(self):
+ """OFX 2.x/XML files with pure-ASCII content should still parse
+ correctly after the SGML header injection."""
+ ofx = self._make_ofx_xml("Grocery Store", "utf-8")
+ transactions = parse_ofx(ofx)
+
+ assert len(transactions) == 1
+ assert transactions[0].description == "Grocery Store"
+ assert transactions[0].amount == Decimal("10.00")
+
+ def test_parse_ofx_xml_format_without_xml_prolog(self):
+ """The XML declaration is optional, so an OFX 2.x file may open with
+ just its <?OFX ?> instruction. It reaches ofxparse with nothing before
+ the first tag, so it hits the same bug and needs the same header."""
+ ofx = self._make_ofx_xml("Aufladung für Konto", "utf-8", prolog=False)
+ transactions = parse_ofx(ofx)
+
+ assert len(transactions) == 1
+ assert transactions[0].description == "Aufladung für Konto"
+
+ def test_parse_ofx_xml_format_bare_ofx_element(self):
+ """Neither declaration is required: a document opening straight on
+ <OFX> is the same headerless case."""
+ ofx = self._make_ofx_xml("Café Paris", "utf-8", prolog=False, ofx_pi=False)
+ transactions = parse_ofx(ofx)
+
+ assert len(transactions) == 1
+ assert transactions[0].description == "Café Paris"
+
+ def test_parse_ofx_sgml_header_is_not_duplicated(self):
+ """A real OFX 1.x file already has a header before its first tag, so it
+ must be left untouched rather than given a second one."""
+ ofx = self._make_ofx_xml("Grocery Store", "utf-8", prolog=False, ofx_pi=False)
+ sgml = (
+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"
+ b"ENCODING:UTF-8\r\nCHARSET:NONE\r\nCOMPRESSION:NONE\r\n"
+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n" + ofx
+ )
+ assert _preprocess_ofx(sgml).count(b"OFXHEADER:") == 1
+ assert parse_ofx(sgml)[0].description == "Grocery Store"
+
# ═══════════════════════════════════════════════════════════════════════════
# MULTI-CURRENCY PARSING TESTS