Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

fix: normalize OFX SGML ENCODING header to prevent ofxparse UnboundLocalError (#794)

Samuel Barbosa612c5974708a
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.py
index 6408e8ea..a7ec9219 100644
--- a/backend/app/services/import_service.py
+++ b/backend/app/services/import_service.py
@@ -89,6 +89,77 @@ def _patch_empty_fitids(text: str) -> str:
)
+_OFX_SGML_ENCODING_RE = re.compile(
+ r"(^|\r?\n)\s*ENCODING\s*:\s*([^\r\n]*)", re.IGNORECASE
+)
+
+_OFX_SGML_CHARSET_RE = re.compile(
+ r"(^|\r?\n)\s*CHARSET\s*:\s*([^\r\n]*)", re.IGNORECASE
+)
+
+# Values of ENCODING that ofxparse 0.21 handles without crashing.
+_OFX_USASCII_VARIANTS = frozenset({
+ "USASCII", "ISO88591", "ISO8859-1", "ISO885915", "ISO8859-15",
+ "CP1252", "WINDOWS-1252", "WINDOWS1252",
+})
+_OFX_UTF8_VARIANTS = frozenset({
+ "UNICODE", "UTF8", "UTF-8",
+})
+
+
+def _normalize_ofx_encoding(text: str, encoding: str) -> str:
+ """Fix SGML headers with ENCODING values that crash ofxparse.
+
+ ofxparse 0.21 only handles USASCII, UNICODE and UTF-8. Any other value
+ (e.g. ISO-8859-1, WINDOWS-1252, or a misspelling) leaves its local
+ ``encoding`` variable unbound, raising ``UnboundLocalError`` inside
+ ``handle_encoding()``.
+
+ This function normalises **only the preamble** (everything before the
+ first ``<``) so transaction memo content is never modified. Latin-1
+ decoded files get ``ENCODING:USASCII``; the ``CHARSET`` distinguishes
+ the two byte layouts Python's ``latin-1`` codec can represent, so
+ ofxparse decodes the same bytes the same way the source declared:
+ ``CHARSET:1252`` when the file declared WINDOWS-1252/CP1252 (whose
+ 0x80-0x9F range holds real typographic characters — em dash, curly
+ quotes — that ISO-8859-1 leaves as undefined control codes), and
+ ``CHARSET:8859-1`` otherwise. UTF-8 files get ``ENCODING:UTF-8`` with
+ ``CHARSET:NONE``.
+ """
+ preamble, first_tag, body = text.lstrip("\ufeff \t\r\n").partition("<")
+ if not first_tag or not preamble.strip():
+ return text
+
+ raw_match = _OFX_SGML_ENCODING_RE.search(preamble)
+ if not raw_match:
+ return text
+
+ raw_value = raw_match.group(2).strip().upper().replace("-", "").replace(" ", "")
+
+ # Only the three exact values that ofxparse 0.21 handles are left alone.
+ # Everything else — including valid IANA names like ISO-8859-1 — must be
+ # rewritten so ofxparse doesn't crash on the UnboundLocalError.
+ if raw_value in ("USASCII", "UNICODE", "UTF8"):
+ return text
+
+ # Unknown or unsupported encoding — rewrite to match the actual byte
+ # encoding so ofxparse decodes consistently.
+ if encoding == "latin-1":
+ new_encoding = "USASCII"
+ new_charset = "1252" if raw_value in ("CP1252", "WINDOWS1252") else "8859-1"
+ else:
+ new_encoding = "UTF-8"
+ new_charset = "NONE"
+
+ new_preamble = _OFX_SGML_ENCODING_RE.sub(
+ lambda m: f"{m.group(1)}ENCODING:{new_encoding}", preamble, count=1,
+ )
+ new_preamble = _OFX_SGML_CHARSET_RE.sub(
+ lambda m: f"{m.group(1)}CHARSET:{new_charset}", new_preamble, count=1,
+ )
+ return new_preamble + first_tag + body
+
+
def _ensure_ofx_sgml_header(text: str, encoding: str) -> str:
"""Prepend a legacy OFX 1.x SGML header for OFX 2.x files that omit it.
@@ -112,8 +183,14 @@ def _ensure_ofx_sgml_header(text: str, encoding: str) -> str:
so the declared header and the actual bytes stay consistent.
"""
preamble, first_tag, _ = text.lstrip("\ufeff \t\r\n").partition("<")
- if not first_tag or preamble.strip():
+ if not first_tag:
return text
+ if preamble.strip():
+ # File already has a header — normalise the ENCODING value so
+ # ofxparse doesn't crash on values it doesn't handle (e.g.
+ # ISO-8859-1, WINDOWS-1252). Only the preamble is touched;
+ # transaction body content is preserved verbatim.
+ return _normalize_ofx_encoding(text, encoding)
if encoding == "latin-1":
enc_lines = "ENCODING:USASCII\r\nCHARSET:8859-1\r\n"
else:
diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.py
index 7366a030..b4693bfb 100644
--- a/backend/tests/test_import_service.py
+++ b/backend/tests/test_import_service.py
@@ -1141,6 +1141,151 @@ class TestParseOfx:
assert _preprocess_ofx(sgml).count(b"OFXHEADER:") == 1
assert parse_ofx(sgml)[0].description == "Grocery Store"
+ def test_parse_ofx_sgml_header_iso_8859_1_encoding(self):
+ """OFX 1.x files with ENCODING:ISO-8859-1 in the SGML header should
+ parse without UnboundLocalError (ofxparse only handles USASCII,
+ UNICODE and UTF-8)."""
+ sgml = (
+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"
+ b"ENCODING:ISO-8859-1\r\nCHARSET:8859-1\r\nCOMPRESSION:NONE\r\n"
+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"
+ b"<OFX>\r\n"
+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"
+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"
+ b"<BANKMSGSRSV1>\r\n"
+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"
+ b"<BANKTRANLIST>\r\n"
+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-25.50<FITID>TX001"
+ b"<NAME>Caf\xe9 Express</NAME></STMTTRN>\r\n"
+ b"</BANKTRANLIST>\r\n"
+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"
+ b"</STMTRS></STMTTRNRS>\r\n"
+ b"</BANKMSGSRSV1>\r\n"
+ b"</OFX>"
+ )
+ transactions = parse_ofx(sgml)
+ assert len(transactions) == 1
+ assert transactions[0].description == "Caf\xe9 Express"
+
+ def test_parse_ofx_sgml_header_windows_1252_encoding(self):
+ """OFX 1.x files with ENCODING:WINDOWS-1252 should parse correctly."""
+ sgml = (
+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"
+ b"ENCODING:WINDOWS-1252\r\nCHARSET:1252\r\nCOMPRESSION:NONE\r\n"
+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"
+ b"<OFX>\r\n"
+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"
+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"
+ b"<BANKMSGSRSV1>\r\n"
+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"
+ b"<BANKTRANLIST>\r\n"
+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-50.00<FITID>TX002"
+ b"<NAME>Loj\xe3 Center</NAME></STMTTRN>\r\n"
+ b"</BANKTRANLIST>\r\n"
+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"
+ b"</STMTRS></STMTTRNRS>\r\n"
+ b"</BANKMSGSRSV1>\r\n"
+ b"</OFX>"
+ )
+ transactions = parse_ofx(sgml)
+ assert len(transactions) == 1
+ assert transactions[0].description == "Loj\xe3 Center"
+
+ def test_parse_ofx_windows_1252_charset_decodes_typographic_characters(self):
+ """WINDOWS-1252/CP1252 must normalise to CHARSET:1252, not 8859-1.
+
+ Byte 0x96 is an en dash (U+2013) in Windows-1252 but an undefined
+ C1 control code in ISO-8859-1 — declaring the wrong CHARSET would
+ have ofxparse decode this byte as the control code instead of the
+ dash the source bank actually meant.
+ """
+ sgml = (
+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"
+ b"ENCODING:WINDOWS-1252\r\nCHARSET:1252\r\nCOMPRESSION:NONE\r\n"
+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"
+ b"<OFX>\r\n"
+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"
+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"
+ b"<BANKMSGSRSV1>\r\n"
+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"
+ b"<BANKTRANLIST>\r\n"
+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-15.00<FITID>TX005"
+ b"<NAME>Loja A \x96 Filial B</NAME></STMTTRN>\r\n"
+ b"</BANKTRANLIST>\r\n"
+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"
+ b"</STMTRS></STMTTRNRS>\r\n"
+ b"</BANKMSGSRSV1>\r\n"
+ b"</OFX>"
+ )
+ transactions = parse_ofx(sgml)
+ assert len(transactions) == 1
+ assert transactions[0].description == "Loja A – Filial B"
+
+ def test_parse_ofx_sgml_header_unknown_encoding_fallback(self):
+ """An OFX file with a completely unknown ENCODING value should fall
+ back to UTF-8 without crashing."""
+ sgml = (
+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"
+ b"ENCODING:UNKNOWN-CODEC\r\nCHARSET:8859-1\r\nCOMPRESSION:NONE\r\n"
+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"
+ b"<OFX>\r\n"
+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"
+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"
+ b"<BANKMSGSRSV1>\r\n"
+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"
+ b"<BANKTRANLIST>\r\n"
+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-10.00<FITID>TX003"
+ b"<NAME>Simple Transaction</NAME></STMTTRN>\r\n"
+ b"</BANKTRANLIST>\r\n"
+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"
+ b"</STMTRS></STMTTRNRS>\r\n"
+ b"</BANKMSGSRSV1>\r\n"
+ b"</OFX>"
+ )
+ transactions = parse_ofx(sgml)
+ assert len(transactions) == 1
+ assert transactions[0].description == "Simple Transaction"
+
+ def test_parse_ofx_body_encoding_line_not_corrupted(self):
+ """Transaction memo content that happens to contain 'ENCODING:' on a
+ new line must not be rewritten by the header normalisation."""
+ sgml = (
+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"
+ b"ENCODING:ISO-8859-1\r\nCHARSET:8859-1\r\nCOMPRESSION:NONE\r\n"
+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"
+ b"<OFX>\r\n"
+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"
+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"
+ b"<BANKMSGSRSV1>\r\n"
+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"
+ b"<BANKTRANLIST>\r\n"
+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-30.00<FITID>TX004"
+ b"<NAME>Rate: ENCODING:WINDOWS-1252 promo</NAME></STMTTRN>\r\n"
+ b"</BANKTRANLIST>\r\n"
+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"
+ b"</STMTRS></STMTTRNRS>\r\n"
+ b"</BANKMSGSRSV1>\r\n"
+ b"</OFX>"
+ )
+ transactions = parse_ofx(sgml)
+ assert len(transactions) == 1
+ assert "ENCODING:WINDOWS-1252" in transactions[0].description
+
+ def test_normalize_ofx_encoding_only_affects_preamble(self):
+ """_normalize_ofx_encoding must only touch the preamble before the
+ first '<', never the body."""
+ from app.services.import_service import _normalize_ofx_encoding
+
+ text = (
+ "ENCODING:ISO-8859-1\r\nCHARSET:8859-1\r\n\r\n"
+ "<OFX><MEMO>\r\nENCODING:WINDOWS-1252 line</MEMO>"
+ )
+ result = _normalize_ofx_encoding(text, "latin-1")
+ # Preamble should be normalised
+ assert "ENCODING:USASCII" in result.split("<")[0]
+ # Body should be untouched
+ assert "ENCODING:WINDOWS-1252" in result.split("<", 1)[1]
+
# ═══════════════════════════════════════════════════════════════════════════
# MULTI-CURRENCY PARSING TESTS