Skip to content

A local-first personal finance app

[AI] Fix character encoding corruption when importing CAMT files (#8912) (#8913)

Fabian Markert4a7ba9687737
diff --git a/packages/loot-core/src/mocks/files/camt/camt.latin1.xml b/packages/loot-core/src/mocks/files/camt/camt.latin1.xml
new file mode 100644
index 0000000000..f966d4eef4
--- /dev/null
+++ b/packages/loot-core/src/mocks/files/camt/camt.latin1.xml
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?><Document xmlns="urn:iso:std:iso:20022:tech:xsd:camt.052.001.08" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:iso:std:iso:20022:tech:xsd:camt.052.001.08 camt.052.001.08.xsd"><BkToCstmrAcctRpt><GrpHdr><MsgId>52D20250909T0000000000000000000000</MsgId><CreDtTm>2025-09-09T00:00:00.0+02:00</CreDtTm></GrpHdr><Rpt><Id>0752C0020250909000000N000000000</Id><RptPgntn><PgNb>1</PgNb><LastPgInd>true</LastPgInd></RptPgntn><ElctrncSeqNb>000000000</ElctrncSeqNb><CreDtTm>2025-09-09T00:00:00.0+02:00</CreDtTm><Acct><Id><IBAN>DE00000000000000000000</IBAN></Id><Ccy>EUR</Ccy><Ownr><Nm>Max Mustermann</Nm></Ownr></Acct><Ntry><Amt Ccy="EUR">100.00</Amt><CdtDbtInd>CRDT</CdtDbtInd><Sts><Cd>BOOK</Cd></Sts><BookgDt><Dt>2025-09-09</Dt></BookgDt><ValDt><Dt>2025-09-09</Dt></ValDt><AcctSvcrRef>2025090900000000</AcctSvcrRef><NtryDtls><TxDtls><RmtInf><Ustrd>M-�berschusssparen</Ustrd></RmtInf></TxDtls></NtryDtls><AddtlNtryInf>DAUERAUFTRAG</AddtlNtryInf></Ntry><Ntry><Amt Ccy="EUR">5.00</Amt><CdtDbtInd>DBIT</CdtDbtInd><Sts><Cd>BOOK</Cd></Sts><BookgDt><Dt>2025-09-09</Dt></BookgDt><ValDt><Dt>2025-09-09</Dt></ValDt><AcctSvcrRef>2025090900000001</AcctSvcrRef><NtryDtls><TxDtls><RltdPties><Cdtr><Pty><Nm>Gr��e aus M�nchen</Nm></Pty></Cdtr></RltdPties><RmtInf><Ustrd>Restaurant Schr�dingers Katze und Caf� �nique</Ustrd></RmtInf></TxDtls></NtryDtls><AddtlNtryInf>UEBERWEISUNG</AddtlNtryInf></Ntry></Rpt></BkToCstmrAcctRpt></Document>
diff --git a/packages/loot-core/src/server/transactions/import/parse-file.test.ts b/packages/loot-core/src/server/transactions/import/parse-file.test.ts
index 335f8594ba..f357a2898d 100644
--- a/packages/loot-core/src/server/transactions/import/parse-file.test.ts
+++ b/packages/loot-core/src/server/transactions/import/parse-file.test.ts
@@ -262,4 +262,19 @@ describe('File import', () => {
expect(errors.length).toBe(0);
expect(await getTransactions('one')).toMatchSnapshot();
});
+
+ test('CAMT import respects ISO-8859-1 encoding', async () => {
+ const { errors, transactions } = await parseFile(
+ __dirname + '/../../../mocks/files/camt/camt.latin1.xml',
+ { importNotes: true },
+ );
+ expect(errors.length).toBe(0);
+ expect(transactions).toMatchObject([
+ { notes: 'M-Überschusssparen' },
+ {
+ payee_name: 'Grüße aus München',
+ notes: 'Restaurant Schrödingers Katze und Café Ünique',
+ },
+ ]);
+ });
});
diff --git a/packages/loot-core/src/server/transactions/import/parse-file.ts b/packages/loot-core/src/server/transactions/import/parse-file.ts
index 26bffc74cd..d9194d1f26 100644
--- a/packages/loot-core/src/server/transactions/import/parse-file.ts
+++ b/packages/loot-core/src/server/transactions/import/parse-file.ts
@@ -254,7 +254,9 @@ async function parseCAMT(
options: ParseFileOptions = {},
): Promise<ParseFileResult> {
const errors = Array<ParseError>();
- const contents = await fs.readFile(filepath);
+ // Read the raw bytes so xmlCAMT2json can honor the encoding declared in
+ // the XML header instead of decoding the file as UTF-8.
+ const contents = await fs.readFile(filepath, 'binary');
let data: Awaited<ReturnType<typeof xmlCAMT2json>>;
try {
diff --git a/packages/loot-core/src/server/transactions/import/xmlcamt2json.ts b/packages/loot-core/src/server/transactions/import/xmlcamt2json.ts
index 6433a1fb1d..7f1cfb99e4 100644
--- a/packages/loot-core/src/server/transactions/import/xmlcamt2json.ts
+++ b/packages/loot-core/src/server/transactions/import/xmlcamt2json.ts
@@ -95,10 +95,35 @@ function getDtOrDtTm(Date: DateRef | null): string | null {
return Date?.Dt;
}
+function decodeXmlContent(content: Uint8Array): string {
+ const utf8Decoder = new TextDecoder('utf-8');
+
+ // The XML declaration (up to the first `>`) is ASCII, so it can be
+ // inspected to find the declared encoding before decoding the rest of
+ // the document. Decoding the header as latin1 keeps the raw byte
+ // values, which is correct for the ASCII-only declaration.
+ const header = new TextDecoder('latin1').decode(content.subarray(0, 500));
+ const match = header.match(/encoding\s*=\s*["']([^"']+)["']/i);
+ const declaredEncoding = match?.[1];
+
+ if (declaredEncoding && !/^utf-?8$/i.test(declaredEncoding)) {
+ try {
+ return new TextDecoder(declaredEncoding).decode(content);
+ } catch {
+ // Unsupported or unknown encoding label; fall back to UTF-8.
+ }
+ }
+
+ return utf8Decoder.decode(content);
+}
+
export async function xmlCAMT2json(
- content: string,
+ content: string | Uint8Array,
): Promise<TransactionCAMT[]> {
- const data = await parseStringPromise(content, { explicitArray: false });
+ const data = await parseStringPromise(
+ content instanceof Uint8Array ? decodeXmlContent(content) : content,
+ { explicitArray: false },
+ );
const entries = findKeys(data, 'Ntry') as Ntry[];
const transactions: TransactionCAMT[] = [];
diff --git a/upcoming-release-notes/fix-camt-file-encoding.md b/upcoming-release-notes/fix-camt-file-encoding.md
new file mode 100644
index 0000000000..b25ba2f3ff
--- /dev/null
+++ b/upcoming-release-notes/fix-camt-file-encoding.md
@@ -0,0 +1,6 @@
+---
+category: Bugfix
+authors: [P1tt187]
+---
+
+Fix special characters (ä, ö, ü) being garbled when importing CAMT bank files