A local-first personal finance app
Add experimental new OFX importer (#921)
Davis Silvermana1d321d65e59
diff --git a/packages/desktop-client/src/components/modals/ImportTransactions.js b/packages/desktop-client/src/components/modals/ImportTransactions.jsindex 2b40a8471a..897a810372 100644--- a/packages/desktop-client/src/components/modals/ImportTransactions.js+++ b/packages/desktop-client/src/components/modals/ImportTransactions.js@@ -555,6 +555,7 @@ export function ImportTransactions({importTransactions,getPayees,savePrefs,+ addNotification,}) {let [multiplierAmount, setMultiplierAmount] = useState('');let [loadingState, setLoadingState] = useState('parsing');@@ -586,11 +587,67 @@ export function ImportTransactions({setFilename(filename);setFileType(filetype);- let { errors, transactions } = await parseTransactions(filename, options);-+ let results = await parseTransactions(filename, options);+ let errors, transactions;setLoadingState(null);setError(null);+ /// Do fine grained reporting between the old and new OFX importers.+ if (results.ofxParser) {+ if (results.which === 'both') {+ // There were errors in the both parsers.+ // Either a bad file, or a bug that must be fixed in the new!+ // Show the new errors here.+ console.log('Old errors: ', results.oldErrors);+ console.log('New errors: ', results.newErrors);+ errors = results.newErrors;+ transactions = {};+ } else if (results.which === 'old') {+ // There were errors in the old, but not the new+ // So the transactions here are from the new parser.+ addNotification({+ type: 'warning',+ sticky: 'false',+ message:+ 'Import was successful, but please [file a bug report](https://github.com/actualbudget/actual/issues/new?assignees=&labels=bug%2Cneeds+triage&template=bug-report.yml&title=%5BBug%5D%3A+New+OFX+Importer:+bad+old+parse:) and attach a redacted version of the file so we can fix our new algorithm.',+ });+ errors = [];+ transactions = results.transactions;+ } else if (results.which === 'new') {+ // There were errors in the new, but not the old.+ // So the transactions here are from the old parser.+ // THIS IS A BUG AND SHOULD BE FIXED IN THE NEW PARSER!+ addNotification({+ type: 'warning',+ sticky: 'false',+ message:+ 'Import was successful, but please [file a bug report](https://github.com/actualbudget/actual/issues/new?assignees=&labels=bug%2Cneeds+triage&template=bug-report.yml&title=%5BBug%5D%3A+New+OFX+Importer:+bad+new+parse:) and attach a redacted version of the file so we can fix our new algorithm.',+ });+ errors = [];+ transactions = results.transactions;+ } else if (results.which === 'none') {+ // Results were the same between the two!+ errors = [];+ transactions = results.transactions;+ } else if (results.which === 'diff') {+ // There was a difference in results between the two.+ // use the old importer to be safe.+ console.log('Different parse results');+ console.log('Old OFX importer: ', results.oldTrans);+ console.log('New OFX importer: ', results.newTrans);+ addNotification({+ type: 'warning',+ sticky: 'false',+ message:+ 'possible error importing file, please [file a bug report](https://github.com/actualbudget/actual/issues/new?assignees=&labels=bug%2Cneeds+triage&template=bug-report.yml&title=%5BBug%5D%3A+New+OFX+Importer:) and attach a redacted version of the file.',+ });+ transactions = results.oldTrans;+ errors = [];+ }+ } else {+ transactions = results.transactions;+ errors = results.errors;+ }if (errors.length > 0) {setError({parsed: true,diff --git a/packages/loot-core/package.json b/packages/loot-core/package.jsonindex c580ecc043..36e6d6543e 100644--- a/packages/loot-core/package.json+++ b/packages/loot-core/package.json@@ -23,6 +23,7 @@"@rschedule/ical-tools": "^1.2.0","@rschedule/json-tools": "^1.2.0","@rschedule/standard-date-adapter": "^1.2.0",+ "@wademason/ofx": "1.0.1","absurd-sql": "0.0.53","better-sqlite3": "^8.2.0","core-js": "^3.8.3",diff --git a/packages/loot-core/src/server/accounts/parse-file.js b/packages/loot-core/src/server/accounts/parse-file.jsindex 750cecedef..9f2b8211ed 100644--- a/packages/loot-core/src/server/accounts/parse-file.js+++ b/packages/loot-core/src/server/accounts/parse-file.js@@ -1,3 +1,4 @@+import ofx from '@wademason/ofx';import csv2json from 'csv-parse/lib/sync';import * as fs from '../../platform/server/fs';@@ -85,7 +86,147 @@ async function parseQIF(filepath) {};}+// Extracts the YYYY-MM-DD from the following formats:+/// * YYYYMMDDHHMMSS.XXX[gmt offset:tz name] (official OFX format)+/// * 20190123120000+/// If an OFX/QFX file is seen with a different format, it should+/// be supported by this function, even if it is not spec-compliant.+function ofxDateParse(raw) {+ const longRx =+ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})(\.\d{3})?\[(.+):(.+)\]$/;+ const shortRx = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/;+ let res = longRx.exec(raw);+ if (res) {+ const [_, y, m, day, _hr, _min, _sec, _milli, _offsetStr, _tz] = res;+ return `${y}-${m}-${day}`;+ } else {+ res = shortRx.exec(raw);+ if (res) {+ const [_, y, m, day, _hr, _min, _sec] = res;+ return `${y}-${m}-${day}`;+ }+ }+ console.warn('Not able to parse date', raw);+ return null;+}+async function parseOFX(filepath) {+ let objectEq = (arg1, arg2) => {+ if (+ Object.prototype.toString.call(arg1) ===+ Object.prototype.toString.call(arg2)+ ) {+ if (+ Object.prototype.toString.call(arg1) === '[object Object]' ||+ Object.prototype.toString.call(arg1) === '[object Array]'+ ) {+ if (Object.keys(arg1).length !== Object.keys(arg2).length) {+ return false;+ }+ return Object.keys(arg1).every(function (key) {+ return objectEq(arg1[key], arg2[key]);+ });+ } else if (typeof arg1 === 'number' && typeof arg2 === 'number') {+ return Math.abs(arg1 - arg2) < Number.EPSILON;+ }+ return arg1 === arg2;+ } else if (+ (arg1 === undefined || arg1 === null) &&+ (arg2 === undefined || arg2 === null)+ ) {+ return true;+ }+ return false;+ };+ const { transactions: oldTrans, errors: oldErrors } =+ await parseOfxNodeLibofx(filepath);+ const { transactions: newTrans, errors: newErrors } =+ await parseOfxJavascript(filepath);+ // send fine-grained information about how the parsing went, comparatively.+ if (oldErrors.length > 0 && newErrors.length > 0) {+ return {+ ofxParser: true,+ which: 'both',+ errors: {+ oldErrors,+ newErrors,+ },+ };+ } else if (oldErrors.length > 0) {+ return {+ ofxParser: true,+ which: 'old',+ errors: {+ oldErrors,+ },+ transactions: newTrans,+ };+ } else if (newErrors.length > 0) {+ return {+ ofxParser: true,+ which: 'new',+ errors: {+ newErrors,+ },+ transactions: oldTrans,+ };+ } else {+ if (objectEq(oldTrans, newTrans)) {+ return {+ ofxParser: true,+ which: 'none',+ errors: [],+ transactions: oldTrans,+ };+ } else {+ return {+ ofxParser: true,+ which: 'diff',+ errors: [],+ oldTrans: oldTrans,+ newTrans: newTrans,+ };+ }+ }+}++async function parseOfxJavascript(filepath) {+ let errors = [];+ // 'binary' should be equal to latin1 in node.js+ // not sure about browser. We want latin1 and not utf8.+ // For some reason, utf8 does not parse ofx files correctly here.+ const contents = new TextDecoder('latin1').decode(+ await fs.readFile(filepath, 'binary'),+ );++ let data;+ try {+ data = ofx.parse(contents);+ } catch (err) {+ errors.push({+ message: 'Failed importing file',+ internal: err.stack,+ });+ return { errors };+ }+ // .STMTTRN may be a list or a single object.+ const transactions = [+ data.body.OFX.BANKMSGSRSV1.STMTTRNRS.STMTRS.BANKTRANLIST.STMTTRN,+ ].flat();+ return {+ errors,+ transactions: transactions.map(trans => ({+ amount: parseFloat(trans.TRNAMT._text),+ imported_id: trans.FITID._text,+ date: trans.DTPOSTED ? ofxDateParse(trans.DTPOSTED._text) : null,+ payee_name: trans.NAME?._text,+ imported_payee: trans.NAME?._text,+ notes: trans.MEMO?._text,+ })),+ };+}++async function parseOfxNodeLibofx(filepath) {let { getOFXTransactions, initModule } = await import(/* webpackChunkName: 'xfo' */ 'node-libofx');diff --git a/upcoming-release-notes/921.md b/upcoming-release-notes/921.mdnew file mode 100644index 0000000000..7e1d900ebf--- /dev/null+++ b/upcoming-release-notes/921.md@@ -0,0 +1,6 @@+---+category: Features+authors: [sinistersnare]+---++Add experimental OFX importer written in pure javascript.diff --git a/yarn.lock b/yarn.lockindex 054995e0d0..f15ab08ae9 100644--- a/yarn.lock+++ b/yarn.lock@@ -4302,6 +4302,15 @@ __metadata:languageName: nodelinkType: hard+"@wademason/ofx@npm:1.0.1":+ version: 1.0.1+ resolution: "@wademason/ofx@npm:1.0.1"+ dependencies:+ xml-js: ^1.6.11+ checksum: 704f33ec70bb44f1612723a1ac216a938dcd47399414d5c39a7d95c365ffedbe033eb5812f732ab9e6f5dc6fcce318e6145a5ca058d4dad2d34b621bd786001e+ languageName: node+ linkType: hard+"@webassemblyjs/ast@npm:1.11.1":version: 1.11.1resolution: "@webassemblyjs/ast@npm:1.11.1"@@ -13429,6 +13438,7 @@ __metadata:"@types/jlongster__sql.js": "npm:@types/sql.js@latest""@types/node-ipc": ^9.2.0"@types/webpack": ^5.28.0+ "@wademason/ofx": 1.0.1absurd-sql: 0.0.53adm-zip: ^0.5.9babel-loader: ^8.0.6@@ -21465,6 +21475,17 @@ __metadata:languageName: nodelinkType: hard+"xml-js@npm:^1.6.11":+ version: 1.6.11+ resolution: "xml-js@npm:1.6.11"+ dependencies:+ sax: ^1.2.4+ bin:+ xml-js: ./bin/cli.js+ checksum: 24a55479919413687105fc2d8ab05e613ebedb1c1bc12258a108e07cff5ef793779297db854800a4edf0281303ebd1f177bc4a588442f5344e62b3dddda26c2b+ languageName: node+ linkType: hard+"xml-name-validator@npm:^3.0.0":version: 3.0.0resolution: "xml-name-validator@npm:3.0.0"