A local-first personal finance app
Decide on transaction date during GoCardless transaction normalization (#243)
Johannes Löthberg95c7d5bc2c6a
diff --git a/package.json b/package.jsonindex 07055bfbf6..362034582f 100644--- a/package.json+++ b/package.json@@ -21,6 +21,7 @@"better-sqlite3": "^8.2.0","body-parser": "^1.20.1","cors": "^2.8.5",+ "date-fns": "^2.30.0","debug": "^4.3.4","express": "4.18.2","express-actuator": "1.8.4",diff --git a/src/app-gocardless/banks/american-express-aesudef1.js b/src/app-gocardless/banks/american-express-aesudef1.jsindex 0b0842de6c..2339c5180c 100644--- a/src/app-gocardless/banks/american-express-aesudef1.js+++ b/src/app-gocardless/banks/american-express-aesudef1.js@@ -21,14 +21,10 @@ export default {},normalizeTransaction(transaction, _booked) {- /**- * The American Express Europe integration sends the actual date of- * purchase as `bookingDate`, and `valueDate` appears to contain a date- * related to the actual booking date, though sometimes offset by a day- * compared to the American Express website.- */- delete transaction.valueDate;- return transaction;+ return {+ ...transaction,+ date: transaction.bookingDate,+ };},sortTransactions(transactions = []) {@@ -36,12 +32,12 @@ export default {},/**- * For SANDBOXFINANCE_SFIN0000 we don't know what balance was+ * For AMERICAN_EXPRESS_AESUDEF1 we don't know what balance was* after each transaction so we have to calculate it by getting* current balance from the account and subtract all the transactions*- * As a current balance we use `interimBooked` balance type because- * it includes transaction placed during current day+ * As a current balance we use the non-standard `information` balance type+ * which is the only one provided for American Express.*/calculateStartingBalance(sortedTransactions = [], balances = []) {const currentBalance = balances.find(diff --git a/src/app-gocardless/banks/bank.interface.ts b/src/app-gocardless/banks/bank.interface.tsindex b01e00b3e4..777847f389 100644--- a/src/app-gocardless/banks/bank.interface.ts+++ b/src/app-gocardless/banks/bank.interface.ts@@ -15,11 +15,16 @@ export interface IBank {/*** Returns a normalized transaction object+ *+ * The GoCardless integrations with different banks are very inconsistent in+ * what each of the different date fields actually mean, so this function is+ * expected to set a `date` field which corresponds to the expected+ * transaction date.*/normalizeTransaction: (transaction: Transaction,booked: boolean,- ) => Transaction | null;+ ) => (Transaction & { date?: string }) | null;/*** Function sorts an array of transactions from newest to oldestdiff --git a/src/app-gocardless/banks/fintro-be-gebabebb.js b/src/app-gocardless/banks/fintro-be-gebabebb.jsindex da2dbfb8b3..8a56879958 100644--- a/src/app-gocardless/banks/fintro-be-gebabebb.js+++ b/src/app-gocardless/banks/fintro-be-gebabebb.js@@ -19,11 +19,6 @@ export default {institutionIds: ['FINTRO_BE_GEBABEBB'],normalizeAccount(account) {- console.log(- 'Available account properties for new institution integration',- { account: JSON.stringify(account) },- );-return {account_id: account.id,institution: account.institution,@@ -69,28 +64,18 @@ export default {].filter(Boolean);}}- return transaction;++ return {+ ...transaction,+ date: transaction.valueDate,+ };},sortTransactions(transactions = []) {- console.log(- 'Available (first 10) transactions properties for new integration of institution in sortTransactions function',- { top10Transactions: JSON.stringify(transactions.slice(0, 10)) },- );return sortByBookingDateOrValueDate(transactions);},calculateStartingBalance(sortedTransactions = [], balances = []) {- console.log(- 'Available (first 10) transactions properties for new integration of institution in calculateStartingBalance function',- {- balances: JSON.stringify(balances),- top10SortedTransactions: JSON.stringify(- sortedTransactions.slice(0, 10),- ),- },- );-const currentBalance = balances.filter((item) => SORTED_BALANCE_TYPE_LIST.includes(item.balanceType)).sort(diff --git a/src/app-gocardless/banks/ing-pl-ingbplpw.js b/src/app-gocardless/banks/ing-pl-ingbplpw.jsindex c7659f0560..7e30a868f5 100644--- a/src/app-gocardless/banks/ing-pl-ingbplpw.js+++ b/src/app-gocardless/banks/ing-pl-ingbplpw.js@@ -17,7 +17,10 @@ export default {},normalizeTransaction(transaction, _booked) {- return transaction;+ return {+ ...transaction,+ date: transaction.bookingDate || transaction.valueDate,+ };},sortTransactions(transactions = []) {diff --git a/src/app-gocardless/banks/integration-bank.js b/src/app-gocardless/banks/integration-bank.jsindex a6165a0b49..53743d00a9 100644--- a/src/app-gocardless/banks/integration-bank.js+++ b/src/app-gocardless/banks/integration-bank.js@@ -1,3 +1,4 @@+import * as d from 'date-fns';import {sortByBookingDateOrValueDate,amountToInteger,@@ -37,7 +38,21 @@ export default {},normalizeTransaction(transaction, _booked) {- return transaction;+ const date =+ transaction.bookingDate ||+ transaction.bookingDateTime ||+ transaction.valueDate ||+ transaction.valueDateTime;+ // If we couldn't find a valid date field we filter out this transaction+ // and hope that we will import it again once the bank has processed the+ // transaction further.+ if (!date) {+ return null;+ }+ return {+ ...transaction,+ date: d.format(d.parseISO(date), 'yyyy-MM-dd'),+ };},sortTransactions(transactions = []) {diff --git a/src/app-gocardless/banks/mbank-retail-brexplpw.js b/src/app-gocardless/banks/mbank-retail-brexplpw.jsindex f93cad39f1..d388ddf2e0 100644--- a/src/app-gocardless/banks/mbank-retail-brexplpw.js+++ b/src/app-gocardless/banks/mbank-retail-brexplpw.js@@ -17,7 +17,10 @@ export default {},normalizeTransaction(transaction, _booked) {- return transaction;+ return {+ ...transaction,+ date: transaction.bookingDate || transaction.valueDate,+ };},sortTransactions(transactions = []) {diff --git a/src/app-gocardless/banks/norwegian-xx-norwnok1.js b/src/app-gocardless/banks/norwegian-xx-norwnok1.jsindex f151d412a7..45d45dbc4b 100644--- a/src/app-gocardless/banks/norwegian-xx-norwnok1.js+++ b/src/app-gocardless/banks/norwegian-xx-norwnok1.js@@ -28,50 +28,45 @@ export default {},normalizeTransaction(transaction, booked) {- /**- * The way Bank Norwegian handles the date fields is rather strange and- * countrary to GoCardless's documentation.- *- * For booked transactions Bank Norwegian sends a `valueDate` field that- * doesn't match the NextGenPSD2 definition of `valueDate` which is what we- * expect to receive from GoCardless. Therefore we remove the incorrect- * field so that transactions are correctly imported.- */if (booked) {- delete transaction.valueDate;- return transaction;+ return {+ ...transaction,+ date: transaction.bookingDate,+ };}/**- * For pending transactions there are two possibilities. Either the- * transaction has a `valueDate`, in which case the `valueDate` we receive- * corresponds to when the transaction actually occurred, and so we simply- * return the transaction as-is.+ * For pending transactions there are two possibilities:+ *+ * - Either a `valueDate` was set, in which case it corresponds to when the+ * transaction actually occurred, or+ * - There is no date field, in which case we try to parse the correct date+ * out of the `remittanceInformationStructured` field.+ *+ * If neither case succeeds then we return `null` causing this transaction+ * to be filtered out for now, and hopefully we'll be able to import it+ * once the bank has processed it further.*/if (transaction.valueDate !== undefined) {- return transaction;+ return {+ ...transaction,+ date: transaction.valueDate,+ };}- /**- * If the pending transaction didn't have a `valueDate` field then it- * should have a `remittanceInformationStructured` field which contains the- * date we expect to receive as the `valueDate`. In this case we extract- * the date from that field and set it as `valueDate`.- */if (transaction.remittanceInformationStructured) {const remittanceInfoRegex = / (\d{4}-\d{2}-\d{2}) /;const matches =transaction.remittanceInformationStructured.match(remittanceInfoRegex);if (matches) {transaction.valueDate = matches[1];- return transaction;+ return {+ ...transaction,+ date: matches[1],+ };}}- /**- * If neither pending case is true we return `null` and ignore the- * transaction until it's been further processed by the bank.- */return null;},diff --git a/src/app-gocardless/banks/sandboxfinance-sfin0000.js b/src/app-gocardless/banks/sandboxfinance-sfin0000.jsindex bf00a14e61..5a0bf56bba 100644--- a/src/app-gocardless/banks/sandboxfinance-sfin0000.js+++ b/src/app-gocardless/banks/sandboxfinance-sfin0000.js@@ -1,4 +1,8 @@-import { printIban, amountToInteger } from '../utils.js';+import {+ printIban,+ amountToInteger,+ sortByBookingDateOrValueDate,+} from '../utils.js';/** @type {import('./bank.interface.js').IBank} */export default {@@ -16,17 +20,23 @@ export default {};},+ /**+ * Following the GoCardless documentation[0] we should prefer `bookingDate`+ * here, though some of their bank integrations uses the date field+ * differently from what's describen in their documentation and so it's+ * sometimes necessary to use `valueDate` instead.+ *+ * [0]: https://nordigen.zendesk.com/hc/en-gb/articles/7899367372829-valueDate-and-bookingDate-for-transactions+ */normalizeTransaction(transaction, _booked) {- return transaction;+ return {+ ...transaction,+ date: transaction.bookingDate || transaction.valueDate,+ };},sortTransactions(transactions = []) {- return transactions.sort((a, b) => {- const [aTime, aSeq] = a.transactionId.split('-');- const [bTime, bSeq] = b.transactionId.split('-');-- return Number(bTime) - Number(aTime) || Number(bSeq) - Number(aSeq);- });+ return sortByBookingDateOrValueDate(transactions);},/**diff --git a/src/app-gocardless/banks/tests/sandboxfinance-sfin0000.spec.js b/src/app-gocardless/banks/tests/sandboxfinance-sfin0000.spec.jsindex ab885167e1..ba705f7e73 100644--- a/src/app-gocardless/banks/tests/sandboxfinance-sfin0000.spec.js+++ b/src/app-gocardless/banks/tests/sandboxfinance-sfin0000.spec.js@@ -1,5 +1,4 @@import SandboxfinanceSfin0000 from '../sandboxfinance-sfin0000.js';-import { mockTransactionAmount } from '../../services/tests/fixtures.js';describe('SandboxfinanceSfin0000', () => {describe('#normalizeAccount', () => {@@ -58,55 +57,6 @@ describe('SandboxfinanceSfin0000', () => {});describe('#sortTransactions', () => {- it('sorts transactions by time and sequence from newest to oldest', () => {- const transactions = [- {- transactionId: '2023012301927902-2',- transactionAmount: mockTransactionAmount,- },- {- transactionId: '2023012301927902-1',- transactionAmount: mockTransactionAmount,- },- {- transactionId: '2023012301927900-2',- transactionAmount: mockTransactionAmount,- },- {- transactionId: '2023012301927900-1',- transactionAmount: mockTransactionAmount,- },- {- transactionId: '2023012301927900-3',- transactionAmount: mockTransactionAmount,- },- ];- const sortedTransactions =- SandboxfinanceSfin0000.sortTransactions(transactions);- expect(sortedTransactions).toEqual([- {- transactionId: '2023012301927902-2',- transactionAmount: mockTransactionAmount,- },- {- transactionId: '2023012301927902-1',- transactionAmount: mockTransactionAmount,- },- {- transactionId: '2023012301927900-3',- transactionAmount: mockTransactionAmount,- },- {- transactionId: '2023012301927900-2',- transactionAmount: mockTransactionAmount,- },- {- transactionId: '2023012301927900-1',- transactionAmount: mockTransactionAmount,- },- ]);- });-it('handles empty arrays', () => {const transactions = [];const sortedTransactions =diff --git a/src/app-gocardless/services/gocardless-service.js b/src/app-gocardless/services/gocardless-service.jsindex 9c0e7adeb6..66f4642709 100644--- a/src/app-gocardless/services/gocardless-service.js+++ b/src/app-gocardless/services/gocardless-service.js@@ -439,14 +439,12 @@ export const goCardlessService = {handleGoCardlessError(response);- const bankAccount = BankFactory(institutionId);+ const bank = BankFactory(institutionId);response.transactions.booked = response.transactions.booked- .map((transaction) => bankAccount.normalizeTransaction(transaction, true))+ .map((transaction) => bank.normalizeTransaction(transaction, true)).filter((transaction) => transaction);response.transactions.pending = response.transactions.pending- .map((transaction) =>- bankAccount.normalizeTransaction(transaction, false),- )+ .map((transaction) => bank.normalizeTransaction(transaction, false)).filter((transaction) => transaction);return response;diff --git a/src/app-gocardless/services/tests/gocardless-service.spec.js b/src/app-gocardless/services/tests/gocardless-service.spec.jsindex 346ead5b1e..351031473e 100644--- a/src/app-gocardless/services/tests/gocardless-service.spec.js+++ b/src/app-gocardless/services/tests/gocardless-service.spec.js@@ -489,6 +489,7 @@ describe('goCardlessService', () => {{"bankTransactionCode": "string","bookingDate": "date",+ "date": "date","debtorAccount": {"iban": "string",},@@ -503,6 +504,7 @@ describe('goCardlessService', () => {{"bankTransactionCode": "string","bookingDate": "date",+ "date": "date","transactionAmount": {"amount": "947.26","currency": "EUR",@@ -513,6 +515,7 @@ describe('goCardlessService', () => {],"pending": [{+ "date": "date","transactionAmount": {"amount": "947.26","currency": "EUR",diff --git a/upcoming-release-notes/243.md b/upcoming-release-notes/243.mdnew file mode 100644index 0000000000..91973b6cd7--- /dev/null+++ b/upcoming-release-notes/243.md@@ -0,0 +1,6 @@+---+category: Enhancements+authors: [kyrias]+---++Decide on transaction date during GoCardless transaction normalization.diff --git a/yarn.lock b/yarn.lockindex b5d5d3b384..c970e761f9 100644--- a/yarn.lock+++ b/yarn.lock@@ -575,6 +575,15 @@ __metadata:languageName: nodelinkType: hard+"@babel/runtime@npm:^7.21.0":+ version: 7.22.10+ resolution: "@babel/runtime@npm:7.22.10"+ dependencies:+ regenerator-runtime: ^0.14.0+ checksum: 524d41517e68953dbc73a4f3616b8475e5813f64e28ba89ff5fca2c044d535c2ea1a3f310df1e5bb06162e1f0b401b5c4af73fe6e2519ca2450d9d8c44cf268d+ languageName: node+ linkType: hard+"@babel/template@npm:^7.18.10, @babel/template@npm:^7.20.7, @babel/template@npm:^7.3.3":version: 7.20.7resolution: "@babel/template@npm:7.20.7"@@ -1573,6 +1582,7 @@ __metadata:better-sqlite3: ^8.2.0body-parser: ^1.20.1cors: ^2.8.5+ date-fns: ^2.30.0debug: ^4.3.4eslint: ^8.33.0eslint-plugin-prettier: ^4.2.1@@ -2293,6 +2303,15 @@ __metadata:languageName: nodelinkType: hard+"date-fns@npm:^2.30.0":+ version: 2.30.0+ resolution: "date-fns@npm:2.30.0"+ dependencies:+ "@babel/runtime": ^7.21.0+ checksum: f7be01523282e9bb06c0cd2693d34f245247a29098527d4420628966a2d9aad154bd0e90a6b1cf66d37adcb769cd108cf8a7bd49d76db0fb119af5cdd13644f4+ languageName: node+ linkType: hard+"dayjs@npm:^1.11.3":version: 1.11.7resolution: "dayjs@npm:1.11.7"@@ -5162,6 +5181,13 @@ __metadata:languageName: nodelinkType: hard+"regenerator-runtime@npm:^0.14.0":+ version: 0.14.0+ resolution: "regenerator-runtime@npm:0.14.0"+ checksum: 1c977ad82a82a4412e4f639d65d22be376d3ebdd30da2c003eeafdaaacd03fc00c2320f18120007ee700900979284fc78a9f00da7fb593f6e6eeebc673fba9a3+ languageName: node+ linkType: hard+"regexpp@npm:^3.2.0":version: 3.2.0resolution: "regexpp@npm:3.2.0"