A local-first personal finance app
[AI] Detect CSV file encoding and add a per-account encoding selector (#8924)
flafleur4e263d983da8
diff --git a/.gitattributes b/.gitattributesindex cfb1e6b81b..3990105915 100644--- a/.gitattributes+++ b/.gitattributes@@ -22,3 +22,4 @@ packages/mobile-client/android/gradlew.bat text eol=crlf# Denote all files that are truly binary and should not be modified.*.png binary*.jpg binary+packages/loot-core/src/mocks/files/utf-16*.csv binarydiff --git a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-1-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-1-chromium-linux.pngindex dd7aea0f5d..57de7f15c6 100644Binary files a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-1-chromium-linux.png and b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-1-chromium-linux.png differdiff --git a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-2-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-2-chromium-linux.pngindex 082448f1e0..fa03e90777 100644Binary files a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-2-chromium-linux.png and b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-2-chromium-linux.png differdiff --git a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-3-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-3-chromium-linux.pngindex e821172a4e..e1c91a0e2b 100644Binary files a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-3-chromium-linux.png and b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-import-csv-file-twice-3-chromium-linux.png differdiff --git a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-1-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-1-chromium-linux.pngindex 806e6d6329..912f3abde9 100644Binary files a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-1-chromium-linux.png and b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-1-chromium-linux.png differdiff --git a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-2-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-2-chromium-linux.pngindex b001468f27..84a942efb2 100644Binary files a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-2-chromium-linux.png and b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-2-chromium-linux.png differdiff --git a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-3-chromium-linux.png b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-3-chromium-linux.pngindex aa2814079b..b433d9468c 100644Binary files a/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-3-chromium-linux.png and b/packages/desktop-client/e2e/accounts.test.ts-snapshots/Accounts-Import-Transactions-imports-transactions-from-a-CSV-file-3-chromium-linux.png differdiff --git a/packages/desktop-client/src/components/modals/ImportTransactionsModal/ImportTransactionsModal.tsx b/packages/desktop-client/src/components/modals/ImportTransactionsModal/ImportTransactionsModal.tsxindex d4bba99d7a..5fd2b59d73 100644--- a/packages/desktop-client/src/components/modals/ImportTransactionsModal/ImportTransactionsModal.tsx+++ b/packages/desktop-client/src/components/modals/ImportTransactionsModal/ImportTransactionsModal.tsx@@ -176,6 +176,7 @@ type LastParse = {const parseOptionKeys = ['hasHeaderRow','delimiter',+ 'encoding','fallbackMissingPayeeToMemo','swapPayeeAndMemo','skipStartLines',@@ -242,6 +243,9 @@ export function ImportTransactionsModal({prefs[`csv-delimiter-${accountId}`] ||(filename.endsWith('.tsv') ? '\t' : ','),);+ const [csvEncoding, setCsvEncoding] = useState(+ prefs[`csv-encoding-${accountId}`] || 'auto',+ );const [skipStartLines, setSkipStartLines] = useState(parseInt(prefs[`csv-skip-start-lines-${accountId}`], 10) || 0,);@@ -477,6 +481,7 @@ export function ImportTransactionsModal({const fileType = getFileType(originalFileName);const parseOptions = getParseOptions(fileType, {delimiter,+ encoding: csvEncoding,hasHeaderRow,skipStartLines,skipEndLines,@@ -509,6 +514,7 @@ export function ImportTransactionsModal({}, [originalFileName,delimiter,+ csvEncoding,hasHeaderRow,skipStartLines,skipEndLines,@@ -559,6 +565,7 @@ export function ImportTransactionsModal({const fileType = getFileType(res[0]);const parseOptions = getParseOptions(fileType, {delimiter,+ encoding: csvEncoding,hasHeaderRow,skipStartLines,skipEndLines,@@ -733,6 +740,7 @@ export function ImportTransactionsModal({[`csv-mappings-${accountId}`]: JSON.stringify(fieldMappings),});savePrefs({ [`csv-delimiter-${accountId}`]: delimiter });+ savePrefs({ [`csv-encoding-${accountId}`]: csvEncoding });savePrefs({ [`csv-has-header-${accountId}`]: String(hasHeaderRow) });savePrefs({[`csv-skip-start-lines-${accountId}`]: String(skipStartLines),@@ -1188,6 +1196,34 @@ export function ImportTransactionsModal({style={{ width: 50 }}/></label>+ <label+ htmlFor="csv-encoding-select"+ style={{+ display: 'flex',+ flexDirection: 'row',+ gap: 5,+ alignItems: 'baseline',+ }}+ >+ <Trans>Encoding:</Trans>+ <Select+ id="csv-encoding-select"+ options={[+ ['auto', t('Auto (detect)')],+ ['utf-8', t('UTF-8')],+ ['utf-16le', t('UTF-16 LE')],+ ['utf-16be', t('UTF-16 BE')],+ ['windows-1252', t('Windows-1252')],+ ['windows-1250', t('Windows-1250')],+ ['iso-8859-2', t('ISO-8859-2')],+ ]}+ value={csvEncoding}+ onChange={value => {+ setCsvEncoding(value);+ }}+ style={{ width: 130 }}+ />+ </label><labelhtmlFor="csv-skip-start-lines"style={{@@ -1355,8 +1391,9 @@ export function ImportTransactionsModal({function getParseOptions(fileType: string, options: ParseFileOptions = {}) {if (fileType === 'csv') {- const { delimiter, hasHeaderRow, skipStartLines, skipEndLines } = options;- return { delimiter, hasHeaderRow, skipStartLines, skipEndLines };+ const { delimiter, encoding, hasHeaderRow, skipStartLines, skipEndLines } =+ options;+ return { delimiter, encoding, hasHeaderRow, skipStartLines, skipEndLines };}if (isOfxFile(fileType)) {const { fallbackMissingPayeeToMemo, importNotes, swapPayeeAndMemo } =diff --git a/packages/loot-core/src/mocks/files/utf-16be.csv b/packages/loot-core/src/mocks/files/utf-16be.csvnew file mode 100644index 0000000000..5986cd6253Binary files /dev/null and b/packages/loot-core/src/mocks/files/utf-16be.csv differdiff --git a/packages/loot-core/src/mocks/files/utf-16le.csv b/packages/loot-core/src/mocks/files/utf-16le.csvnew file mode 100644index 0000000000..7418772acaBinary files /dev/null and b/packages/loot-core/src/mocks/files/utf-16le.csv differdiff --git a/packages/loot-core/src/mocks/files/utf-8-bom.csv b/packages/loot-core/src/mocks/files/utf-8-bom.csvnew file mode 100644index 0000000000..8e031acccf--- /dev/null+++ b/packages/loot-core/src/mocks/files/utf-8-bom.csv@@ -0,0 +1,3 @@+"Felhasználónév","Számlaszám","Könyvelés dátuma","Összeg","Devizanem","Partner név","Partner IBAN száma","Partner számlaszáma","Partner bankkódja","Könyvelési információk","Tranzakcióazonosító","Hitel azonosító","Közlemény","Partner címe","Küldő címe","Megjegyzés","Kedvenc","Dátum","Tranzakció dátuma és ideje","Kategória","Kártyaszám","Cím","Tranzakció elfogadásának típusa","Kártya típusa","Termék típusa","Digitalizált kártya száma","Mobilapplikáció","Tranzakció típusa","Számlaegyenleg"+"tariff package name","11600006-XXXXXXXX-XXXXXXXX","2025.12.04","100","HUF","","","","","","","","","","","","0","","","","","","","","","","","",""+"tariff package name","11600006-XXXXXXXX-XXXXXXXX","2025.12.01","100","HUF","","","","","","","","","","","","0","","","","","","","","","","","",""diff --git a/packages/loot-core/src/mocks/files/windows-1252-euro.csv b/packages/loot-core/src/mocks/files/windows-1252-euro.csvnew file mode 100644index 0000000000..a794496bfb--- /dev/null+++ b/packages/loot-core/src/mocks/files/windows-1252-euro.csv@@ -0,0 +1,2 @@+"Date","Payee","Amount"+"2025.12.04","Caf� �R�my","100.25"diff --git a/packages/loot-core/src/mocks/files/windows-1252.csv b/packages/loot-core/src/mocks/files/windows-1252.csvnew file mode 100644index 0000000000..ac185a92ca--- /dev/null+++ b/packages/loot-core/src/mocks/files/windows-1252.csv@@ -0,0 +1,3 @@+"Date","Payee","Amount"+"2025.12.04","Caf� R�my","100.25"+"2025.12.05","Boulangerie M�ller","-42.10"diff --git a/packages/loot-core/src/server/transactions/import/parse-file.test.ts b/packages/loot-core/src/server/transactions/import/parse-file.test.tsindex f357a2898d..bd5aca8f96 100644--- a/packages/loot-core/src/server/transactions/import/parse-file.test.ts+++ b/packages/loot-core/src/server/transactions/import/parse-file.test.ts@@ -263,6 +263,127 @@ describe('File import', () => {expect(await getTransactions('one')).toMatchSnapshot();});+ test('csv import works (utf-16le bank export)', async () => {+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/utf-16le.csv',+ { hasHeaderRow: true },+ );++ expect(errors.length).toBe(0);+ expect(transactions).toHaveLength(2);+ expect(transactions[0]).toMatchObject({+ 'Könyvelés dátuma': '2025.12.04',+ Összeg: '100',+ Devizanem: 'HUF',+ });+ });++ test('csv import works (utf-16be)', async () => {+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/utf-16be.csv',+ { hasHeaderRow: true },+ );++ expect(errors.length).toBe(0);+ expect(transactions).toHaveLength(2);+ expect(transactions[0]).toMatchObject({+ 'Könyvelés dátuma': '2025.12.04',+ Összeg: '100',+ Devizanem: 'HUF',+ });+ });++ test('csv import works (utf-8 with bom)', async () => {+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/utf-8-bom.csv',+ { hasHeaderRow: true },+ );++ expect(errors.length).toBe(0);+ expect(transactions).toHaveLength(2);+ expect(transactions[0]).toMatchObject({+ 'Könyvelés dátuma': '2025.12.04',+ Összeg: '100',+ Devizanem: 'HUF',+ });+ });++ test('csv import skips start lines on utf-16le files', async () => {+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/utf-16le.csv',+ { hasHeaderRow: false, skipStartLines: 1 },+ );++ expect(errors.length).toBe(0);+ expect(transactions).toHaveLength(2);+ const rows = transactions as string[][];+ expect(rows[0][0]).toBe('tariff package name');+ });++ test('csv import respects manual encoding override', async () => {+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/windows-1252.csv',+ { hasHeaderRow: true, encoding: 'windows-1252' },+ );++ expect(errors.length).toBe(0);+ expect(transactions).toHaveLength(2);+ expect(transactions[0]).toMatchObject({+ Date: '2025.12.04',+ Payee: 'Café Rémy',+ Amount: '100.25',+ });+ expect(transactions[1]).toMatchObject({+ Date: '2025.12.05',+ Payee: 'Boulangerie Müller',+ Amount: '-42.10',+ });+ });++ test('csv import accepts iso-8859-1 as a windows-1252 alias', async () => {+ // TextDecoder resolves the iso-8859-1 label to windows-1252 per the+ // WHATWG encoding spec. The € byte (0x80) proves it: windows-1252 decodes+ // it as €, while a true ISO-8859-1 decoder would yield a C1 control char.+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/windows-1252-euro.csv',+ { hasHeaderRow: true, encoding: 'iso-8859-1' },+ );++ expect(errors.length).toBe(0);+ expect(transactions).toHaveLength(1);+ expect(transactions[0]).toMatchObject({+ Date: '2025.12.04',+ Payee: 'Café €Rémy',+ Amount: '100.25',+ });+ });++ test('csv import treats explicit auto encoding as detection', async () => {+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/utf-16le.csv',+ { hasHeaderRow: true, encoding: 'auto' },+ );++ expect(errors.length).toBe(0);+ expect(transactions).toHaveLength(2);+ expect(transactions[0]).toMatchObject({+ 'Könyvelés dátuma': '2025.12.04',+ Összeg: '100',+ Devizanem: 'HUF',+ });+ });++ test('csv import rejects an invalid encoding', async () => {+ const { errors, transactions } = await parseFile(+ __dirname + '/../../../mocks/files/utf-8-bom.csv',+ { hasHeaderRow: true, encoding: 'not-an-encoding' },+ );++ expect(errors.length).toBe(1);+ expect(transactions).toHaveLength(0);+ expect(errors[0].message).toContain('Failed parsing');+ });+test('CAMT import respects ISO-8859-1 encoding', async () => {const { errors, transactions } = await parseFile(__dirname + '/../../../mocks/files/camt/camt.latin1.xml',diff --git a/packages/loot-core/src/server/transactions/import/parse-file.ts b/packages/loot-core/src/server/transactions/import/parse-file.tsindex d9194d1f26..ef7a63f8d5 100644--- a/packages/loot-core/src/server/transactions/import/parse-file.ts+++ b/packages/loot-core/src/server/transactions/import/parse-file.ts@@ -54,6 +54,31 @@ type StructuredTransaction = {category?: string | null;};+/**+ * Decode raw CSV file bytes into a string. A user-provided encoding always+ * wins; otherwise the byte order mark selects UTF-16 LE/BE and everything+ * else decodes as UTF-8. Files in other encodings can be selected manually+ * in the import dialog.+ */+function decodeCsvBytes(bytes: Uint8Array, encoding = 'auto'): string {+ if (encoding !== 'auto') {+ // Per the WHATWG encoding spec, the iso-8859-1 label resolves to the+ // windows-1252 decoder; no browser provides a true ISO-8859-1 decoder,+ // and windows-1252 gives better results for legacy CSV content anyway.+ return new TextDecoder(encoding).decode(bytes);+ }++ if (bytes[0] === 0xff && bytes[1] === 0xfe) {+ return new TextDecoder('utf-16le').decode(bytes);+ }++ if (bytes[0] === 0xfe && bytes[1] === 0xff) {+ return new TextDecoder('utf-16be').decode(bytes);+ }++ return new TextDecoder('utf-8').decode(bytes);+}+// CSV files return raw data that are not guaranteed to be StructuredTransactionstype CsvTransaction = Record<string, string> | string[];@@ -73,6 +98,7 @@ export type ParseFileOptions = {skipStartLines?: number;skipEndLines?: number;importNotes?: boolean;+ encoding?: string;};export async function parseFile(@@ -112,7 +138,18 @@ async function parseCSV(options: ParseFileOptions,): Promise<ParseFileResult> {const errors = Array<ParseError>();- let contents = await fs.readFile(filepath);+ const bytes = await fs.readFile(filepath, 'binary');++ let contents: string;+ try {+ contents = decodeCsvBytes(bytes, options.encoding);+ } catch (err) {+ errors.push({+ message: 'Failed parsing: ' + err.message,+ internal: err.message,+ });+ return { errors, transactions: [] };+ }const skipStart = Math.max(0, options.skipStartLines || 0);const skipEnd = Math.max(0, options.skipEndLines || 0);diff --git a/upcoming-release-notes/fix-csv-utf16-import.md b/upcoming-release-notes/fix-csv-utf16-import.mdnew file mode 100644index 0000000000..e4555aea1f--- /dev/null+++ b/upcoming-release-notes/fix-csv-utf16-import.md@@ -0,0 +1,6 @@+---+category: Bugfix+authors: [flafleur]+---++Fix CSV transaction import failing on UTF-16 encoded bank export files, automatically detect their byte order mark, and add a per-account encoding selector for less common encodings