A local-first personal finance app
[AI] Fix character encoding corruption when importing CAMT files (#8912) (#8913)
Fabian Markert4a7ba9687737
diff --git a/packages/loot-core/src/mocks/files/camt/camt.latin1.xml b/packages/loot-core/src/mocks/files/camt/camt.latin1.xmlnew file mode 100644index 0000000000..f966d4eef4--- /dev/null+++ b/packages/loot-core/src/mocks/files/camt/camt.latin1.xml@@ -0,0 +1 @@+<?xml version="1.0" encoding="ISO-8859-1" ?><Document xmlns="urn:iso:std:iso:20022:tech:xsd:camt.052.001.08" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:iso:std:iso:20022:tech:xsd:camt.052.001.08 camt.052.001.08.xsd"><BkToCstmrAcctRpt><GrpHdr><MsgId>52D20250909T0000000000000000000000</MsgId><CreDtTm>2025-09-09T00:00:00.0+02:00</CreDtTm></GrpHdr><Rpt><Id>0752C0020250909000000N000000000</Id><RptPgntn><PgNb>1</PgNb><LastPgInd>true</LastPgInd></RptPgntn><ElctrncSeqNb>000000000</ElctrncSeqNb><CreDtTm>2025-09-09T00:00:00.0+02:00</CreDtTm><Acct><Id><IBAN>DE00000000000000000000</IBAN></Id><Ccy>EUR</Ccy><Ownr><Nm>Max Mustermann</Nm></Ownr></Acct><Ntry><Amt Ccy="EUR">100.00</Amt><CdtDbtInd>CRDT</CdtDbtInd><Sts><Cd>BOOK</Cd></Sts><BookgDt><Dt>2025-09-09</Dt></BookgDt><ValDt><Dt>2025-09-09</Dt></ValDt><AcctSvcrRef>2025090900000000</AcctSvcrRef><NtryDtls><TxDtls><RmtInf><Ustrd>M-�berschusssparen</Ustrd></RmtInf></TxDtls></NtryDtls><AddtlNtryInf>DAUERAUFTRAG</AddtlNtryInf></Ntry><Ntry><Amt Ccy="EUR">5.00</Amt><CdtDbtInd>DBIT</CdtDbtInd><Sts><Cd>BOOK</Cd></Sts><BookgDt><Dt>2025-09-09</Dt></BookgDt><ValDt><Dt>2025-09-09</Dt></ValDt><AcctSvcrRef>2025090900000001</AcctSvcrRef><NtryDtls><TxDtls><RltdPties><Cdtr><Pty><Nm>Gr��e aus M�nchen</Nm></Pty></Cdtr></RltdPties><RmtInf><Ustrd>Restaurant Schr�dingers Katze und Caf� �nique</Ustrd></RmtInf></TxDtls></NtryDtls><AddtlNtryInf>UEBERWEISUNG</AddtlNtryInf></Ntry></Rpt></BkToCstmrAcctRpt></Document>diff --git a/packages/loot-core/src/server/transactions/import/parse-file.test.ts b/packages/loot-core/src/server/transactions/import/parse-file.test.tsindex 335f8594ba..f357a2898d 100644--- a/packages/loot-core/src/server/transactions/import/parse-file.test.ts+++ b/packages/loot-core/src/server/transactions/import/parse-file.test.ts@@ -262,4 +262,19 @@ describe('File import', () => {expect(errors.length).toBe(0);expect(await getTransactions('one')).toMatchSnapshot();});++ test('CAMT import respects ISO-8859-1 encoding', async () => {+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/camt/camt.latin1.xml',+ { importNotes: true },+ );+ expect(errors.length).toBe(0);+ expect(transactions).toMatchObject([+ { notes: 'M-Überschusssparen' },+ {+ payee_name: 'Grüße aus München',+ notes: 'Restaurant Schrödingers Katze und Café Ünique',+ },+ ]);+ });});diff --git a/packages/loot-core/src/server/transactions/import/parse-file.ts b/packages/loot-core/src/server/transactions/import/parse-file.tsindex 26bffc74cd..d9194d1f26 100644--- a/packages/loot-core/src/server/transactions/import/parse-file.ts+++ b/packages/loot-core/src/server/transactions/import/parse-file.ts@@ -254,7 +254,9 @@ async function parseCAMT(options: ParseFileOptions = {},): Promise<ParseFileResult> {const errors = Array<ParseError>();- const contents = await fs.readFile(filepath);+ // Read the raw bytes so xmlCAMT2json can honor the encoding declared in+ // the XML header instead of decoding the file as UTF-8.+ const contents = await fs.readFile(filepath, 'binary');let data: Awaited<ReturnType<typeof xmlCAMT2json>>;try {diff --git a/packages/loot-core/src/server/transactions/import/xmlcamt2json.ts b/packages/loot-core/src/server/transactions/import/xmlcamt2json.tsindex 6433a1fb1d..7f1cfb99e4 100644--- a/packages/loot-core/src/server/transactions/import/xmlcamt2json.ts+++ b/packages/loot-core/src/server/transactions/import/xmlcamt2json.ts@@ -95,10 +95,35 @@ function getDtOrDtTm(Date: DateRef | null): string | null {return Date?.Dt;}+function decodeXmlContent(content: Uint8Array): string {+ const utf8Decoder = new TextDecoder('utf-8');++ // The XML declaration (up to the first `>`) is ASCII, so it can be+ // inspected to find the declared encoding before decoding the rest of+ // the document. Decoding the header as latin1 keeps the raw byte+ // values, which is correct for the ASCII-only declaration.+ const header = new TextDecoder('latin1').decode(content.subarray(0, 500));+ const match = header.match(/encoding\s*=\s*["']([^"']+)["']/i);+ const declaredEncoding = match?.[1];++ if (declaredEncoding && !/^utf-?8$/i.test(declaredEncoding)) {+ try {+ return new TextDecoder(declaredEncoding).decode(content);+ } catch {+ // Unsupported or unknown encoding label; fall back to UTF-8.+ }+ }++ return utf8Decoder.decode(content);+}+export async function xmlCAMT2json(- content: string,+ content: string | Uint8Array,): Promise<TransactionCAMT[]> {- const data = await parseStringPromise(content, { explicitArray: false });+ const data = await parseStringPromise(+ content instanceof Uint8Array ? decodeXmlContent(content) : content,+ { explicitArray: false },+ );const entries = findKeys(data, 'Ntry') as Ntry[];const transactions: TransactionCAMT[] = [];diff --git a/upcoming-release-notes/fix-camt-file-encoding.md b/upcoming-release-notes/fix-camt-file-encoding.mdnew file mode 100644index 0000000000..b25ba2f3ff--- /dev/null+++ b/upcoming-release-notes/fix-camt-file-encoding.md@@ -0,0 +1,6 @@+---+category: Bugfix+authors: [P1tt187]+---++Fix special characters (ä, ö, ü) being garbled when importing CAMT bank files