Open-source personal finance manager. Self-hosted, privacy-first.
fix: normalize OFX SGML ENCODING header to prevent ofxparse UnboundLocalError (#794)
Samuel Barbosa612c5974708a
diff --git a/backend/app/services/import_service.py b/backend/app/services/import_service.pyindex 6408e8ea..a7ec9219 100644--- a/backend/app/services/import_service.py+++ b/backend/app/services/import_service.py@@ -89,6 +89,77 @@ def _patch_empty_fitids(text: str) -> str:)+_OFX_SGML_ENCODING_RE = re.compile(+ r"(^|\r?\n)\s*ENCODING\s*:\s*([^\r\n]*)", re.IGNORECASE+)++_OFX_SGML_CHARSET_RE = re.compile(+ r"(^|\r?\n)\s*CHARSET\s*:\s*([^\r\n]*)", re.IGNORECASE+)++# Values of ENCODING that ofxparse 0.21 handles without crashing.+_OFX_USASCII_VARIANTS = frozenset({+ "USASCII", "ISO88591", "ISO8859-1", "ISO885915", "ISO8859-15",+ "CP1252", "WINDOWS-1252", "WINDOWS1252",+})+_OFX_UTF8_VARIANTS = frozenset({+ "UNICODE", "UTF8", "UTF-8",+})+++def _normalize_ofx_encoding(text: str, encoding: str) -> str:+ """Fix SGML headers with ENCODING values that crash ofxparse.++ ofxparse 0.21 only handles USASCII, UNICODE and UTF-8. Any other value+ (e.g. ISO-8859-1, WINDOWS-1252, or a misspelling) leaves its local+ ``encoding`` variable unbound, raising ``UnboundLocalError`` inside+ ``handle_encoding()``.++ This function normalises **only the preamble** (everything before the+ first ``<``) so transaction memo content is never modified. Latin-1+ decoded files get ``ENCODING:USASCII``; the ``CHARSET`` distinguishes+ the two byte layouts Python's ``latin-1`` codec can represent, so+ ofxparse decodes the same bytes the same way the source declared:+ ``CHARSET:1252`` when the file declared WINDOWS-1252/CP1252 (whose+ 0x80-0x9F range holds real typographic characters — em dash, curly+ quotes — that ISO-8859-1 leaves as undefined control codes), and+ ``CHARSET:8859-1`` otherwise. UTF-8 files get ``ENCODING:UTF-8`` with+ ``CHARSET:NONE``.+ """+ preamble, first_tag, body = text.lstrip("\ufeff \t\r\n").partition("<")+ if not first_tag or not preamble.strip():+ return text++ raw_match = _OFX_SGML_ENCODING_RE.search(preamble)+ if not raw_match:+ return text++ raw_value = raw_match.group(2).strip().upper().replace("-", "").replace(" ", "")++ # Only the three exact values that ofxparse 0.21 handles are left alone.+ # Everything else — including valid IANA names like ISO-8859-1 — must be+ # rewritten so ofxparse doesn't crash on the UnboundLocalError.+ if raw_value in ("USASCII", "UNICODE", "UTF8"):+ return text++ # Unknown or unsupported encoding — rewrite to match the actual byte+ # encoding so ofxparse decodes consistently.+ if encoding == "latin-1":+ new_encoding = "USASCII"+ new_charset = "1252" if raw_value in ("CP1252", "WINDOWS1252") else "8859-1"+ else:+ new_encoding = "UTF-8"+ new_charset = "NONE"++ new_preamble = _OFX_SGML_ENCODING_RE.sub(+ lambda m: f"{m.group(1)}ENCODING:{new_encoding}", preamble, count=1,+ )+ new_preamble = _OFX_SGML_CHARSET_RE.sub(+ lambda m: f"{m.group(1)}CHARSET:{new_charset}", new_preamble, count=1,+ )+ return new_preamble + first_tag + body++def _ensure_ofx_sgml_header(text: str, encoding: str) -> str:"""Prepend a legacy OFX 1.x SGML header for OFX 2.x files that omit it.@@ -112,8 +183,14 @@ def _ensure_ofx_sgml_header(text: str, encoding: str) -> str:so the declared header and the actual bytes stay consistent."""preamble, first_tag, _ = text.lstrip("\ufeff \t\r\n").partition("<")- if not first_tag or preamble.strip():+ if not first_tag:return text+ if preamble.strip():+ # File already has a header — normalise the ENCODING value so+ # ofxparse doesn't crash on values it doesn't handle (e.g.+ # ISO-8859-1, WINDOWS-1252). Only the preamble is touched;+ # transaction body content is preserved verbatim.+ return _normalize_ofx_encoding(text, encoding)if encoding == "latin-1":enc_lines = "ENCODING:USASCII\r\nCHARSET:8859-1\r\n"else:diff --git a/backend/tests/test_import_service.py b/backend/tests/test_import_service.pyindex 7366a030..b4693bfb 100644--- a/backend/tests/test_import_service.py+++ b/backend/tests/test_import_service.py@@ -1141,6 +1141,151 @@ class TestParseOfx:assert _preprocess_ofx(sgml).count(b"OFXHEADER:") == 1assert parse_ofx(sgml)[0].description == "Grocery Store"+ def test_parse_ofx_sgml_header_iso_8859_1_encoding(self):+ """OFX 1.x files with ENCODING:ISO-8859-1 in the SGML header should+ parse without UnboundLocalError (ofxparse only handles USASCII,+ UNICODE and UTF-8)."""+ sgml = (+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"+ b"ENCODING:ISO-8859-1\r\nCHARSET:8859-1\r\nCOMPRESSION:NONE\r\n"+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"+ b"<OFX>\r\n"+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"+ b"<BANKMSGSRSV1>\r\n"+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"+ b"<BANKTRANLIST>\r\n"+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-25.50<FITID>TX001"+ b"<NAME>Caf\xe9 Express</NAME></STMTTRN>\r\n"+ b"</BANKTRANLIST>\r\n"+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"+ b"</STMTRS></STMTTRNRS>\r\n"+ b"</BANKMSGSRSV1>\r\n"+ b"</OFX>"+ )+ transactions = parse_ofx(sgml)+ assert len(transactions) == 1+ assert transactions[0].description == "Caf\xe9 Express"++ def test_parse_ofx_sgml_header_windows_1252_encoding(self):+ """OFX 1.x files with ENCODING:WINDOWS-1252 should parse correctly."""+ sgml = (+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"+ b"ENCODING:WINDOWS-1252\r\nCHARSET:1252\r\nCOMPRESSION:NONE\r\n"+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"+ b"<OFX>\r\n"+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"+ b"<BANKMSGSRSV1>\r\n"+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"+ b"<BANKTRANLIST>\r\n"+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-50.00<FITID>TX002"+ b"<NAME>Loj\xe3 Center</NAME></STMTTRN>\r\n"+ b"</BANKTRANLIST>\r\n"+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"+ b"</STMTRS></STMTTRNRS>\r\n"+ b"</BANKMSGSRSV1>\r\n"+ b"</OFX>"+ )+ transactions = parse_ofx(sgml)+ assert len(transactions) == 1+ assert transactions[0].description == "Loj\xe3 Center"++ def test_parse_ofx_windows_1252_charset_decodes_typographic_characters(self):+ """WINDOWS-1252/CP1252 must normalise to CHARSET:1252, not 8859-1.++ Byte 0x96 is an en dash (U+2013) in Windows-1252 but an undefined+ C1 control code in ISO-8859-1 — declaring the wrong CHARSET would+ have ofxparse decode this byte as the control code instead of the+ dash the source bank actually meant.+ """+ sgml = (+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"+ b"ENCODING:WINDOWS-1252\r\nCHARSET:1252\r\nCOMPRESSION:NONE\r\n"+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"+ b"<OFX>\r\n"+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"+ b"<BANKMSGSRSV1>\r\n"+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"+ b"<BANKTRANLIST>\r\n"+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-15.00<FITID>TX005"+ b"<NAME>Loja A \x96 Filial B</NAME></STMTTRN>\r\n"+ b"</BANKTRANLIST>\r\n"+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"+ b"</STMTRS></STMTTRNRS>\r\n"+ b"</BANKMSGSRSV1>\r\n"+ b"</OFX>"+ )+ transactions = parse_ofx(sgml)+ assert len(transactions) == 1+ assert transactions[0].description == "Loja A – Filial B"++ def test_parse_ofx_sgml_header_unknown_encoding_fallback(self):+ """An OFX file with a completely unknown ENCODING value should fall+ back to UTF-8 without crashing."""+ sgml = (+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"+ b"ENCODING:UNKNOWN-CODEC\r\nCHARSET:8859-1\r\nCOMPRESSION:NONE\r\n"+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"+ b"<OFX>\r\n"+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"+ b"<BANKMSGSRSV1>\r\n"+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"+ b"<BANKTRANLIST>\r\n"+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-10.00<FITID>TX003"+ b"<NAME>Simple Transaction</NAME></STMTTRN>\r\n"+ b"</BANKTRANLIST>\r\n"+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"+ b"</STMTRS></STMTTRNRS>\r\n"+ b"</BANKMSGSRSV1>\r\n"+ b"</OFX>"+ )+ transactions = parse_ofx(sgml)+ assert len(transactions) == 1+ assert transactions[0].description == "Simple Transaction"++ def test_parse_ofx_body_encoding_line_not_corrupted(self):+ """Transaction memo content that happens to contain 'ENCODING:' on a+ new line must not be rewritten by the header normalisation."""+ sgml = (+ b"OFXHEADER:100\r\nDATA:OFXSGML\r\nVERSION:102\r\nSECURITY:NONE\r\n"+ b"ENCODING:ISO-8859-1\r\nCHARSET:8859-1\r\nCOMPRESSION:NONE\r\n"+ b"OLDFILEUID:NONE\r\nNEWFILEUID:NONE\r\n\r\n"+ b"<OFX>\r\n"+ b"<SIGNONMSGSRSV1>\r\n<SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS>"+ b"<DTSERVER>20260115<LANGUAGE>POR</SONRS></SIGNONMSGSRSV1>\r\n"+ b"<BANKMSGSRSV1>\r\n"+ b"<STMTTRNRS><STMTRS><CURDEF>BRL>\r\n"+ b"<BANKTRANLIST>\r\n"+ b"<STMTTRN><TRNTYPE>POS<DTPOSTED>20260115<TRNAMT>-30.00<FITID>TX004"+ b"<NAME>Rate: ENCODING:WINDOWS-1252 promo</NAME></STMTTRN>\r\n"+ b"</BANKTRANLIST>\r\n"+ b"<LEDGERBAL><BALAMT>1000.00<DTASOF>20260115</LEDGERBAL>\r\n"+ b"</STMTRS></STMTTRNRS>\r\n"+ b"</BANKMSGSRSV1>\r\n"+ b"</OFX>"+ )+ transactions = parse_ofx(sgml)+ assert len(transactions) == 1+ assert "ENCODING:WINDOWS-1252" in transactions[0].description++ def test_normalize_ofx_encoding_only_affects_preamble(self):+ """_normalize_ofx_encoding must only touch the preamble before the+ first '<', never the body."""+ from app.services.import_service import _normalize_ofx_encoding++ text = (+ "ENCODING:ISO-8859-1\r\nCHARSET:8859-1\r\n\r\n"+ "<OFX><MEMO>\r\nENCODING:WINDOWS-1252 line</MEMO>"+ )+ result = _normalize_ofx_encoding(text, "latin-1")+ # Preamble should be normalised+ assert "ENCODING:USASCII" in result.split("<")[0]+ # Body should be untouched+ assert "ENCODING:WINDOWS-1252" in result.split("<", 1)[1]+# ═══════════════════════════════════════════════════════════════════════════# MULTI-CURRENCY PARSING TESTS