A local-first personal finance app
implement SimpleFin batch sync (#3581)
Matt Fiddaman815f69a05194
diff --git a/packages/loot-core/src/client/actions/account.ts b/packages/loot-core/src/client/actions/account.tsindex 426f63a971..54330402ab 100644--- a/packages/loot-core/src/client/actions/account.ts+++ b/packages/loot-core/src/client/actions/account.ts@@ -89,6 +89,55 @@ export function linkAccountSimpleFin(};}+function handleSyncResponse(+ accountId,+ res,+ dispatch,+ resNewTransactions,+ resMatchedTransactions,+ resUpdatedAccounts,+) {+ const { errors, newTransactions, matchedTransactions, updatedAccounts } = res;++ // Mark the account as failed or succeeded (depending on sync output)+ const [error] = errors;+ if (error) {+ // We only want to mark the account as having problem if it+ // was a real syncing error.+ if (error.type === 'SyncError') {+ dispatch(markAccountFailed(accountId, error.category, error.code));+ }+ } else {+ dispatch(markAccountSuccess(accountId));+ }++ // Dispatch errors (if any)+ errors.forEach(error => {+ if (error.type === 'SyncError') {+ dispatch(+ addNotification({+ type: 'error',+ message: error.message,+ }),+ );+ } else {+ dispatch(+ addNotification({+ type: 'error',+ message: error.message,+ internal: error.internal,+ }),+ );+ }+ });++ resNewTransactions.push(...newTransactions);+ resMatchedTransactions.push(...matchedTransactions);+ resUpdatedAccounts.push(...updatedAccounts);++ return newTransactions.length > 0 || matchedTransactions.length > 0;+}+export function syncAccounts(id?: string) {return async (dispatch: Dispatch, getState: GetState) => {// Disallow two parallel sync operations@@ -96,9 +145,11 @@ export function syncAccounts(id?: string) {return false;}+ const batchSync = !id;+// Build an array of IDs for accounts to sync.. if no `id` provided// then we assume that all accounts should be synced- const accountIdsToSync = id+ let accountIdsToSync = !batchSync? [id]: getState().queries.accounts.filter(@@ -113,67 +164,73 @@ export function syncAccounts(id?: string) {dispatch(setAccountsSyncing(accountIdsToSync));+ const accountsData = await send('accounts-get');+ const simpleFinAccounts = accountsData.filter(+ a => a.account_sync_source === 'simpleFin',+ );+let isSyncSuccess = false;+ const newTransactions = [];+ const matchedTransactions = [];+ const updatedAccounts = [];++ if (batchSync && simpleFinAccounts.length > 0) {+ console.log('Using SimpleFin batch sync');++ const res = await send('simplefin-batch-sync', {+ ids: simpleFinAccounts.map(a => a.id),+ });++ for (const account of res) {+ const success = handleSyncResponse(+ account.accountId,+ account.res,+ dispatch,+ newTransactions,+ matchedTransactions,+ updatedAccounts,+ );+ if (success) isSyncSuccess = true;+ }++ accountIdsToSync = accountIdsToSync.filter(+ id => !simpleFinAccounts.find(sfa => sfa.id === id),+ );+ }// Loop through the accounts and perform sync operation.. one by onefor (let idx = 0; idx < accountIdsToSync.length; idx++) {const accountId = accountIdsToSync[idx];// Perform sync operation- const { errors, newTransactions, matchedTransactions, updatedAccounts } =- await send('accounts-bank-sync', {- id: accountId,- });-- // Mark the account as failed or succeeded (depending on sync output)- const [error] = errors;- if (error) {- // We only want to mark the account as having problem if it- // was a real syncing error.- if (error.type === 'SyncError') {- dispatch(markAccountFailed(accountId, error.category, error.code));- }- } else {- dispatch(markAccountSuccess(accountId));- }-- // Dispatch errors (if any)- errors.forEach(error => {- if (error.type === 'SyncError') {- dispatch(- addNotification({- type: 'error',- message: error.message,- }),- );- } else {- dispatch(- addNotification({- type: 'error',- message: error.message,- internal: error.internal,- }),- );- }+ const res = await send('accounts-bank-sync', {+ id: accountId,});- // Set new transactions- dispatch({- type: constants.SET_NEW_TRANSACTIONS,+ const success = handleSyncResponse(+ accountId,+ res,+ dispatch,newTransactions,matchedTransactions,updatedAccounts,- });+ );++ if (success) isSyncSuccess = true;// Dispatch the ids for the accounts that are yet to be synceddispatch(setAccountsSyncing(accountIdsToSync.slice(idx + 1)));-- if (newTransactions.length > 0 || matchedTransactions.length > 0) {- isSyncSuccess = true;- }}- // Rest the sync state back to empty (fallback in case something breaks+ // Set new transactions+ dispatch({+ type: constants.SET_NEW_TRANSACTIONS,+ newTransactions,+ matchedTransactions,+ updatedAccounts,+ });++ // Reset the sync state back to empty (fallback in case something breaks// in the logic above)dispatch(setAccountsSyncing([]));return isSyncSuccess;diff --git a/packages/loot-core/src/server/accounts/sync.ts b/packages/loot-core/src/server/accounts/sync.tsindex a861b37407..80def5788c 100644--- a/packages/loot-core/src/server/accounts/sync.ts+++ b/packages/loot-core/src/server/accounts/sync.ts@@ -4,6 +4,7 @@ import { v4 as uuidv4 } from 'uuid';import * as asyncStorage from '../../platform/server/asyncStorage';import * as monthUtils from '../../shared/months';+import { q } from '../../shared/query';import {makeChild as makeChildTransaction,recalculateSplit,@@ -13,6 +14,13 @@ import {amountToInteger,integerToAmount,} from '../../shared/util';+import {+ AccountEntity,+ BankSyncResponse,+ SimpleFinBatchSyncResponse,+ TransactionEntity,+} from '../../types/models';+import { runQuery } from '../aql';import * as db from '../db';import { runMutator } from '../mutators';import { post } from '../post';@@ -61,6 +69,35 @@ async function updateAccountBalance(id, balance) {]);}+async function getAccountOldestTransaction(id): Promise<TransactionEntity> {+ return (+ await runQuery(+ q('transactions')+ .filter({+ account: id,+ date: { $lte: monthUtils.currentDay() },+ })+ .select('date')+ .orderBy('date')+ .limit(1),+ )+ ).data?.[0];+}++async function getAccountSyncStartDate(id) {+ // Many GoCardless integrations do not support getting more than 90 days+ // worth of data, so make that the earliest possible limit.+ const dates = [monthUtils.subDays(monthUtils.currentDay(), 90)];++ const oldestTransaction = await getAccountOldestTransaction(id);++ if (oldestTransaction) dates.push(oldestTransaction.date);++ return monthUtils.dayFromDate(+ dateFns.max(dates.map(d => monthUtils.parseDate(d))),+ );+}+export async function getGoCardlessAccounts(userId, userKey, id) {const userToken = await asyncStorage.getItem('user-token');if (!userToken) return;@@ -141,10 +178,15 @@ async function downloadGoCardlessTransactions(}}-async function downloadSimpleFinTransactions(acctId, since) {+async function downloadSimpleFinTransactions(+ acctId: AccountEntity['id'] | AccountEntity['id'][],+ since: string | string[],+) {const userToken = await asyncStorage.getItem('user-token');if (!userToken) return;+ const batchSync = Array.isArray(acctId);+console.log('Pulling transactions from SimpleFin');const res = await post(@@ -163,19 +205,37 @@ async function downloadSimpleFinTransactions(acctId, since) {throw BankSyncError(res.error_type, res.error_code);}- const {- transactions: { all },- balances,- startingBalance,- } = res;+ let retVal = {};+ if (batchSync) {+ for (const [accountId, data] of Object.entries(+ res as SimpleFinBatchSyncResponse,+ )) {+ if (accountId === 'errors') continue;- console.log('Response:', res);+ const error = res?.errors?.[accountId]?.[0];- return {- transactions: all,- accountBalance: balances,- startingBalance,- };+ retVal[accountId] = {+ transactions: data?.transactions?.all,+ accountBalance: data?.balances,+ startingBalance: data?.startingBalance,+ };++ if (error) {+ retVal[accountId].error_type = error.error_type;+ retVal[accountId].error_code = error.error_code;+ }+ }+ } else {+ const singleRes = res as BankSyncResponse;+ retVal = {+ transactions: singleRes.transactions.all,+ accountBalance: singleRes.balances,+ startingBalance: singleRes.startingBalance,+ };+ }++ console.log('Response:', retVal);+ return retVal;}async function resolvePayee(trans, payeeName, payeesToCreate) {@@ -646,106 +706,18 @@ export async function addTransactions(return newTransactions;}-export async function syncAccount(- userId: string,- userKey: string,- id: string,- acctId: string,- bankId: string,+async function processBankSyncDownload(+ download,+ id,+ acctRow,+ initialSync = false,) {- // TODO: Handle the case where transactions exist in the future- // (that will make start date after end date)- const latestTransaction = await db.first(- 'SELECT * FROM v_transactions WHERE account = ? ORDER BY date DESC LIMIT 1',- [id],- );-- const acctRow = await db.select('accounts', id);// If syncing an account from sync source it must not use strictIdChecking. This allows// the fuzzy search to match transactions where the import IDs are different. It is a known quirk// that account sync sources can give two different transaction IDs even though it's the same transaction.const useStrictIdChecking = !acctRow.account_sync_source;- if (latestTransaction) {- const startingTransaction = await db.first(- 'SELECT date FROM v_transactions WHERE account = ? ORDER BY date ASC LIMIT 1',- [id],- );- const startingDate = db.fromDateRepr(startingTransaction.date);- // assert(startingTransaction)-- const startDate = monthUtils.dayFromDate(- dateFns.max([- // Many GoCardless integrations do not support getting more than 90 days- // worth of data, so make that the earliest possible limit.- monthUtils.parseDate(monthUtils.subDays(monthUtils.currentDay(), 90)),-- // Never download transactions before the starting date.- monthUtils.parseDate(startingDate),- ]),- );-- let download;-- if (acctRow.account_sync_source === 'simpleFin') {- download = await downloadSimpleFinTransactions(acctId, startDate);- } else if (acctRow.account_sync_source === 'goCardless') {- download = await downloadGoCardlessTransactions(- userId,- userKey,- acctId,- bankId,- startDate,- false,- );- } else {- throw new Error(- `Unrecognized bank-sync provider: ${acctRow.account_sync_source}`,- );- }-- const { transactions: originalTransactions, accountBalance } = download;-- if (originalTransactions.length === 0) {- return { added: [], updated: [] };- }-- const transactions = originalTransactions.map(trans => ({- ...trans,- account: id,- }));-- return runMutator(async () => {- const result = await reconcileTransactions(- id,- transactions,- true,- useStrictIdChecking,- );-- if (accountBalance) await updateAccountBalance(id, accountBalance);-- return result;- });- } else {- let download;-- // Otherwise, download transaction for the past 90 days- const startingDay = monthUtils.subDays(monthUtils.currentDay(), 90);-- if (acctRow.account_sync_source === 'simpleFin') {- download = await downloadSimpleFinTransactions(acctId, startingDay);- } else if (acctRow.account_sync_source === 'goCardless') {- download = await downloadGoCardlessTransactions(- userId,- userKey,- acctId,- bankId,- startingDay,- true,- );- }-+ if (initialSync) {const { transactions } = download;let balanceToUse = download.startingBalance;@@ -791,4 +763,110 @@ export async function syncAccount(};});}++ const { transactions: originalTransactions, accountBalance } = download;++ if (originalTransactions.length === 0) {+ return { added: [], updated: [] };+ }++ const transactions = originalTransactions.map(trans => ({+ ...trans,+ account: id,+ }));++ return runMutator(async () => {+ const result = await reconcileTransactions(+ id,+ transactions,+ true,+ useStrictIdChecking,+ );++ if (accountBalance) await updateAccountBalance(id, accountBalance);++ return result;+ });+}++export async function syncAccount(+ userId: string,+ userKey: string,+ id: string,+ acctId: string,+ bankId: string,+) {+ const acctRow = await db.select('accounts', id);++ const syncStartDate = await getAccountSyncStartDate(id);+ const oldestTransaction = await getAccountOldestTransaction(id);+ const newAccount = oldestTransaction == null;++ let download;+ if (acctRow.account_sync_source === 'simpleFin') {+ download = await downloadSimpleFinTransactions(acctId, syncStartDate);+ } else if (acctRow.account_sync_source === 'goCardless') {+ download = await downloadGoCardlessTransactions(+ userId,+ userKey,+ acctId,+ bankId,+ syncStartDate,+ newAccount,+ );+ } else {+ throw new Error(+ `Unrecognized bank-sync provider: ${acctRow.account_sync_source}`,+ );+ }++ return processBankSyncDownload(download, id, acctRow, newAccount);+}++export async function SimpleFinBatchSync(+ accounts: {+ id: AccountEntity['id'];+ accountId: AccountEntity['account_id'];+ }[],+) {+ const startDates = await Promise.all(+ accounts.map(async a => getAccountSyncStartDate(a.id)),+ );++ const res = await downloadSimpleFinTransactions(+ accounts.map(a => a.accountId),+ startDates,+ );++ const promises = [];+ for (let i = 0; i < accounts.length; i++) {+ const account = accounts[i];+ const download = res[account.accountId];++ const acctRow = await db.select('accounts', account.id);+ const oldestTransaction = await getAccountOldestTransaction(account.id);+ const newAccount = oldestTransaction == null;++ if (download.error_code) {+ promises.push(+ Promise.resolve({+ accountId: account.id,+ res: download,+ }),+ );++ continue;+ }++ promises.push(+ processBankSyncDownload(download, account.id, acctRow, newAccount).then(+ res => ({+ accountId: account.id,+ res,+ }),+ ),+ );+ }++ return await Promise.all(promises);}diff --git a/packages/loot-core/src/server/main.ts b/packages/loot-core/src/server/main.tsindex a7979f0378..669c4e8aa3 100644--- a/packages/loot-core/src/server/main.ts+++ b/packages/loot-core/src/server/main.ts@@ -1056,6 +1056,51 @@ handlers['gocardless-create-web-token'] = async function ({}};+function handleSyncResponse(+ res,+ acct,+ newTransactions,+ matchedTransactions,+ updatedAccounts,+) {+ const { added, updated } = res;++ newTransactions.push(...added);+ matchedTransactions.push(...updated);++ if (added.length > 0 || updated.length > 0) {+ updatedAccounts.push(acct.id);+ }+}++function handleSyncError(err, acct) {+ if (err.type === 'BankSyncError') {+ return {+ type: 'SyncError',+ accountId: acct.id,+ message: 'Failed syncing account “' + acct.name + '.”',+ category: err.category,+ code: err.code,+ };+ }++ if (err instanceof PostError && err.reason !== 'internal') {+ return {+ accountId: acct.id,+ message: err.reason+ ? err.reason+ : `Account “${acct.name}” is not linked properly. Please link it again.`,+ };+ }++ return {+ accountId: acct.id,+ message:+ 'There was an internal error. Please get in touch https://actualbudget.org/contact for support.',+ internal: err.stack,+ };+}+handlers['accounts-bank-sync'] = async function ({ id }) {const [[, userId], [, userKey]] = await asyncStorage.multiGet(['user-id',@@ -1071,9 +1116,9 @@ handlers['accounts-bank-sync'] = async function ({ id }) {);const errors = [];- let newTransactions = [];- let matchedTransactions = [];- let updatedAccounts = [];+ const newTransactions = [];+ const matchedTransactions = [];+ const updatedAccounts = [];for (let i = 0; i < accounts.length; i++) {const acct = accounts[i];@@ -1088,39 +1133,15 @@ handlers['accounts-bank-sync'] = async function ({ id }) {acct.bankId,);- const { added, updated } = res;-- newTransactions = newTransactions.concat(added);- matchedTransactions = matchedTransactions.concat(updated);-- if (added.length > 0 || updated.length > 0) {- updatedAccounts = updatedAccounts.concat(acct.id);- }+ handleSyncResponse(+ res,+ acct,+ newTransactions,+ matchedTransactions,+ updatedAccounts,+ );} catch (err) {- if (err.type === 'BankSyncError') {- errors.push({- type: 'SyncError',- accountId: acct.id,- message: 'Failed syncing account “' + acct.name + '.”',- category: err.category,- code: err.code,- });- } else if (err instanceof PostError && err.reason !== 'internal') {- errors.push({- accountId: acct.id,- message: err.reason- ? err.reason- : `Account “${acct.name}” is not linked properly. Please link it again.`,- });- } else {- errors.push({- accountId: acct.id,- message:- 'There was an internal error. Please get in touch https://actualbudget.org/contact for support.',- internal: err.stack,- });- }-+ errors.push(handleSyncError(err, acct));err.message = 'Failed syncing account “' + acct.name + '.”';captureException(err);} finally {@@ -1139,6 +1160,70 @@ handlers['accounts-bank-sync'] = async function ({ id }) {return { errors, newTransactions, matchedTransactions, updatedAccounts };};+handlers['simplefin-batch-sync'] = async function ({ ids = [] }) {+ const accounts = await db.runQuery(+ `SELECT a.*, b.bank_id as bankId FROM accounts a+ LEFT JOIN banks b ON a.bank = b.id+ WHERE a.tombstone = 0 AND a.closed = 0 ${ids.length ? `AND a.id IN (${ids.map(() => '?').join(', ')})` : ''}+ ORDER BY a.offbudget, a.sort_order`,+ ids.length ? ids : [],+ true,+ );++ console.group('Bank Sync operation for all SimpleFin accounts');+ const res = await bankSync.SimpleFinBatchSync(+ accounts.map(a => ({+ id: a.id,+ accountId: a.account_id,+ })),+ );++ const retVal = [];+ for (const account of res) {+ const errors = [];+ const newTransactions = [];+ const matchedTransactions = [];+ const updatedAccounts = [];++ if (account.res.error_code) {+ errors.push(+ handleSyncError(+ {+ type: 'BankSyncError',+ category: account.res.error_type,+ code: account.res.error_code,+ },+ accounts.find(a => a.id === account.accountId),+ ),+ );+ } else {+ handleSyncResponse(+ account.res,+ accounts.find(a => a.id === account.accountId),+ newTransactions,+ matchedTransactions,+ updatedAccounts,+ );+ }++ retVal.push({+ accountId: account.accountId,+ res: { errors, newTransactions, matchedTransactions, updatedAccounts },+ });+ }++ if (retVal.some(a => a.res.updatedAccounts.length > 0)) {+ connection.send('sync-event', {+ type: 'success',+ tables: ['transactions'],+ });+ }++ console.groupEnd();++ return retVal;+};+handlers['transactions-import'] = mutator(function ({accountId,transactions,diff --git a/packages/loot-core/src/types/models/bank-sync.d.ts b/packages/loot-core/src/types/models/bank-sync.d.tsnew file mode 100644index 0000000000..279a23cfae--- /dev/null+++ b/packages/loot-core/src/types/models/bank-sync.d.ts@@ -0,0 +1,21 @@+import {+ GoCardlessAmount,+ GoCardlessBalance,+ GoCardlessTransaction,+} from './gocardless';++export type BankSyncBalance = GoCardlessBalance;+export type BankSyncAmount = GoCardlessAmount;+export type BankSyncTransaction = GoCardlessTransaction;++export type BankSyncResponse = {+ transactions: {+ all: BankSyncTransaction[];+ booked: BankSyncTransaction[];+ pending: BankSyncTransaction[];+ };+ balances: BankSyncBalance[];+ startingBalance: number;+ error_type: string;+ error_code: string;+};diff --git a/packages/loot-core/src/types/models/gocardless.d.ts b/packages/loot-core/src/types/models/gocardless.d.tsindex a15bed650c..b1003e8836 100644--- a/packages/loot-core/src/types/models/gocardless.d.ts+++ b/packages/loot-core/src/types/models/gocardless.d.ts@@ -12,3 +12,64 @@ export type GoCardlessInstitution = {logo: string;identification_codes: string[];};++export type GoCardlessBalance = {+ balanceAmount: GoCardlessAmount;+ balanceType:+ | 'closingBooked'+ | 'expected'+ | 'forwardAvailable'+ | 'interimAvailable'+ | 'interimBooked'+ | 'nonInvoiced'+ | 'openingBooked';+ creditLimitIncluded?: boolean;+ lastChangeDateTime?: string;+ lastCommittedTransaction?: string;+ referenceDate?: string;+};++export type GoCardlessAmount = {+ amount: string;+ currency: string;+};++export type GoCardlessTransaction = {+ additionalInformation?: string;+ bookingStatus?: string;+ balanceAfterTransaction?: Pick<+ GoCardlessBalance,+ 'balanceType' | 'balanceAmount'+ >;+ bankTransactionCode?: string;+ bookingDate?: string;+ bookingDateTime?: string;+ checkId?: string;+ creditorAccount?: string;+ creditorAgent?: string;+ creditorId?: string;+ creditorName?: string;+ currencyExchange?: string[];+ debtorAccount?: {+ iban: string;+ };+ debtorAgent?: string;+ debtorName?: string;+ endToEndId?: string;+ entryReference?: string;+ internalTransactionId?: string;+ mandateId?: string;+ merchantCategoryCode?: string;+ proprietaryBankTransactionCode?: string;+ purposeCode?: string;+ remittanceInformationStructured?: string;+ remittanceInformationStructuredArray?: string[];+ remittanceInformationUnstructured?: string;+ remittanceInformationUnstructuredArray?: string[];+ transactionAmount: GoCardlessAmount;+ transactionId?: string;+ ultimateCreditor?: string;+ ultimateDebtor?: string;+ valueDate?: string;+ valueDateTime?: string;+};diff --git a/packages/loot-core/src/types/models/index.d.ts b/packages/loot-core/src/types/models/index.d.tsindex 3e64570feb..b4ba56346c 100644--- a/packages/loot-core/src/types/models/index.d.ts+++ b/packages/loot-core/src/types/models/index.d.ts@@ -1,4 +1,5 @@export type * from './account';+export type * from './bank-sync';export type * from './category';export type * from './category-group';export type * from './dashboard';diff --git a/packages/loot-core/src/types/models/simplefin.d.ts b/packages/loot-core/src/types/models/simplefin.d.tsindex 5eddc6680a..c4c7f97579 100644--- a/packages/loot-core/src/types/models/simplefin.d.ts+++ b/packages/loot-core/src/types/models/simplefin.d.ts@@ -1,3 +1,6 @@+import { AccountEntity } from './account';+import { BankSyncResponse } from './bank-sync';+export type SimpleFinOrganization = {name: string;domain: string;@@ -8,3 +11,7 @@ export type SimpleFinAccount = {name: string;org: SimpleFinOrganization;};++export interface SimpleFinBatchSyncResponse {+ [accountId: AccountEntity['account_id']]: BankSyncResponse;+}diff --git a/packages/loot-core/src/types/server-handlers.d.ts b/packages/loot-core/src/types/server-handlers.d.tsindex 3536dada43..338b5e2f3f 100644--- a/packages/loot-core/src/types/server-handlers.d.ts+++ b/packages/loot-core/src/types/server-handlers.d.ts@@ -193,6 +193,16 @@ export interface ServerHandlers {'simplefin-accounts': () => Promise<{ accounts: SimpleFinAccount[] }>;+ 'simplefin-batch-sync': ({ ids }: { ids: string[] }) => Promise<+ {+ accountId: string;+ errors;+ newTransactions;+ matchedTransactions;+ updatedAccounts;+ }[]+ >;+'gocardless-get-banks': (country: string) => Promise<{data: GoCardlessInstitution[];error?: { reason: string };diff --git a/upcoming-release-notes/3581.md b/upcoming-release-notes/3581.mdnew file mode 100644index 0000000000..484e12a0cf--- /dev/null+++ b/upcoming-release-notes/3581.md@@ -0,0 +1,6 @@+---+category: Enhancements+authors: [matt-fidd]+---++Enable all SimpleFin accounts to be synced with a single request