A local-first personal finance app
Nordigen integration - account sync (#457)
Filip Stybel5bb4f4b27956
diff --git a/.gitignore b/.gitignoreindex 5ddc1fda8c..bade354f18 100644--- a/.gitignore+++ b/.gitignore@@ -17,6 +17,7 @@ bundle.desktop.js.mapbundle.mobile.jsbundle.mobile.js.mapexport-2020-01-10.csv+.idea**/*.logdiff --git a/packages/desktop-client/src/components/FinancesApp.js b/packages/desktop-client/src/components/FinancesApp.jsindex 5ec661d147..04a26115e8 100644--- a/packages/desktop-client/src/components/FinancesApp.js+++ b/packages/desktop-client/src/components/FinancesApp.js@@ -42,6 +42,7 @@ import FloatableSidebar, { SidebarProvider } from './FloatableSidebar';import GlobalKeys from './GlobalKeys';import { ManageRulesPage } from './ManageRulesPage';import Modals from './Modals';+import NordigenLink from './nordigen/NordigenLink';import Notifications from './Notifications';import { PageTypeProvider } from './Page';import { ManagePayeesPage } from './payees/ManagePayeesPage';@@ -100,6 +101,7 @@ function Routes({ isMobile, location }) {<Route path="/payees" exact component={ManagePayeesPage} /><Route path="/rules" exact component={ManageRulesPage} /><Route path="/settings" component={Settings} />+ <Route path="/nordigen/link" exact component={NordigenLink} /><Routepath="/accounts/:id"diff --git a/packages/desktop-client/src/components/Modals.js b/packages/desktop-client/src/components/Modals.jsindex ecaf7cb5e1..c5804e625f 100644--- a/packages/desktop-client/src/components/Modals.js+++ b/packages/desktop-client/src/components/Modals.js@@ -15,6 +15,7 @@ import CreateLocalAccount from 'loot-design/src/components/modals/CreateLocalAccimport EditField from 'loot-design/src/components/modals/EditField';import ImportTransactions from 'loot-design/src/components/modals/ImportTransactions';import LoadBackup from 'loot-design/src/components/modals/LoadBackup';+import NordigenExternalMsg from 'loot-design/src/components/modals/NordigenExternalMsg';import PlaidExternalMsg from 'loot-design/src/components/modals/PlaidExternalMsg';import SelectLinkedAccounts from 'loot-design/src/components/modals/SelectLinkedAccounts';@@ -82,10 +83,10 @@ function Modals({<Route path="/select-linked-accounts"><SelectLinkedAccountsmodalProps={modalProps}- institution={options.institution}- publicToken={options.publicToken}accounts={options.accounts}- upgradingId={options.upgradingId}+ requisitionId={options.requisitionId}+ actualAccounts={accounts.filter(acct => acct.closed === 0)}+ upgradingAccountId={options.upgradingAccountId}actions={actions}/></Route>@@ -208,6 +209,23 @@ function Modals({);}}/>+ <Route+ path="/nordigen-external-msg"+ render={() => {+ return (+ <NordigenExternalMsg+ modalProps={modalProps}+ actions={actions}+ onMoveExternal={options.onMoveExternal}+ onClose={() => {+ options.onClose && options.onClose();+ send('nordigen-poll-web-token-stop');+ }}+ onSuccess={options.onSuccess}+ />+ );+ }}+ /><Routepath="/create-encryption-key"diff --git a/packages/desktop-client/src/components/accounts/Account.js b/packages/desktop-client/src/components/accounts/Account.jsindex a3b58d6fba..ee12dd2558 100644--- a/packages/desktop-client/src/components/accounts/Account.js+++ b/packages/desktop-client/src/components/accounts/Account.js@@ -60,7 +60,7 @@ import Pencil1 from 'loot-design/src/svg/v2/Pencil1';import SvgRemove from 'loot-design/src/svg/v2/Remove';import SearchAlternate from 'loot-design/src/svg/v2/SearchAlternate';-import { authorizeBank } from '../../plaid';+import { authorizeBank } from '../../nordigen';import { useActiveLocation } from '../ActiveLocation';import AnimatedRefresh from '../AnimatedRefresh';@@ -1386,7 +1386,7 @@ class AccountInternal extends React.PureComponent {switch (item) {case 'link':- authorizeBank(this.props.pushModal, { upgradingId: accountId });+ authorizeBank(this.props.pushModal, { upgradingAccountId: accountId });break;case 'unlink':this.props.unlinkAccount(accountId);diff --git a/packages/desktop-client/src/components/accounts/AccountSyncCheck.js b/packages/desktop-client/src/components/accounts/AccountSyncCheck.jsindex f1345eb30b..0c954beb7c 100644--- a/packages/desktop-client/src/components/accounts/AccountSyncCheck.js+++ b/packages/desktop-client/src/components/accounts/AccountSyncCheck.js@@ -6,7 +6,7 @@ import { View, Button, Tooltip } from 'loot-design/src/components/common';import { colors } from 'loot-design/src/style';import ExclamationOutline from 'loot-design/src/svg/v1/ExclamationOutline';-import { reauthorizeBank } from '../../plaid';+import { authorizeBank } from '../../nordigen';function getErrorMessage(type, code) {switch (type.toUpperCase()) {@@ -88,21 +88,7 @@ function AccountSyncCheck({function reauth() {setOpen(false);- let modalOpened = reauthorizeBank(pushModal, account.bankId, () => {- closeModal();-- // Re-sync accounts. If there are multiple failed account, that- // means all accounts were synced to resync all of them.- // Multiple accounts can be tied to the same bank id.- syncAndDownload(failedAccounts.size > 1 ? null : account.id);- });-- if (!modalOpened) {- addNotification({- type: 'error',- message: 'Unable to process this item, sorry!',- });- }+ authorizeBank(pushModal, { upgradingAccountId: account.account_id });}async function unlink() {diff --git a/packages/desktop-client/src/components/modals/CreateAccount.js b/packages/desktop-client/src/components/modals/CreateAccount.jsindex 54428eac26..435062059b 100644--- a/packages/desktop-client/src/components/modals/CreateAccount.js+++ b/packages/desktop-client/src/components/modals/CreateAccount.js@@ -7,7 +7,7 @@ import * as actions from 'loot-core/src/client/actions';import { View, Text, Modal, Button } from 'loot-design/src/components/common';import { colors } from 'loot-design/src/style';-import { authorizeBank } from '../../plaid';+import { authorizeBank } from '../../nordigen';class CreateAccount extends React.Component {onConnect = async () => {diff --git a/packages/desktop-client/src/components/nordigen/NordigenLink.js b/packages/desktop-client/src/components/nordigen/NordigenLink.jsnew file mode 100644index 0000000000..815953cf39--- /dev/null+++ b/packages/desktop-client/src/components/nordigen/NordigenLink.js@@ -0,0 +1,21 @@+import React from 'react';++import { Modal, P, View } from 'loot-design/src/components/common';++export default function NordigenLink() {+ window.close();++ return (+ <Modal isCurrent={true} showClose={false} title="Account sync">+ {() => (+ <View style={{ maxWidth: 500 }}>+ <P>Please wait...</P>+ <P>+ The window should close automatically. If nothing happend you can+ close this window or tab.+ </P>+ </View>+ )}+ </Modal>+ );+}diff --git a/packages/desktop-client/src/nordigen.js b/packages/desktop-client/src/nordigen.jsnew file mode 100644index 0000000000..853a0af277--- /dev/null+++ b/packages/desktop-client/src/nordigen.js@@ -0,0 +1,40 @@+import { send } from 'loot-core/src/platform/client/fetch';++function _authorize(pushModal, upgradingAccountId, { onSuccess, onClose }) {+ pushModal('nordigen-external-msg', {+ onMoveExternal: async ({ institutionId }) => {+ const accessValidForDays = 30;+ const resp = await send('nordigen-create-web-token', {+ upgradingAccountId,+ institutionId,+ accessValidForDays,+ });++ if (resp.error) return resp;+ const { link, requisitionId } = resp;+ window.Actual.openURLInBrowser(link);++ let { error, data } = await send('nordigen-poll-web-token', {+ upgradingAccountId,+ requisitionId,+ });++ return { error, data };+ },++ onClose,+ onSuccess,+ });+}++export async function authorizeBank(pushModal, { upgradingAccountId } = {}) {+ _authorize(pushModal, upgradingAccountId, {+ onSuccess: async data => {+ pushModal('select-linked-accounts', {+ accounts: data.accounts,+ requisitionId: data.id,+ upgradingAccountId,+ });+ },+ });+}diff --git a/packages/loot-core/src/client/actions/account.js b/packages/loot-core/src/client/actions/account.jsindex dd8140de44..ede6c96ea6 100644--- a/packages/loot-core/src/client/actions/account.js+++ b/packages/loot-core/src/client/actions/account.js@@ -42,12 +42,11 @@ export function unlinkAccount(id) {};}-export function linkAccount(institution, publicToken, accountId, upgradingId) {+export function linkAccount(requisitionId, account, upgradingId) {return async dispatch => {- await send('accounts-link', {- institution,- publicToken,- accountId,+ await send('nordigen-accounts-link', {+ requisitionId,+ account,upgradingId,});await dispatch(getPayees());@@ -74,6 +73,25 @@ export function connectAccounts(};}+export function connectNordigenAccounts(+ institution,+ publicToken,+ accountIds,+ offbudgetIds,+) {+ return async dispatch => {+ let ids = await send('nordigen-accounts-connect', {+ institution,+ publicToken,+ accountIds,+ offbudgetIds,+ });+ await dispatch(getPayees());+ await dispatch(getAccounts());+ return ids;+ };+}+export function syncAccounts(id) {return async (dispatch, getState) => {if (getState().account.accountsSyncing) {@@ -88,7 +106,7 @@ export function syncAccounts(id) {}const { errors, newTransactions, matchedTransactions, updatedAccounts } =- await send('accounts-sync', { id });+ await send('nordigen-accounts-sync', { id });dispatch(setAccountsSyncing(null));if (id) {diff --git a/packages/loot-core/src/server/accounts/link.js b/packages/loot-core/src/server/accounts/link.jsindex d3aee3cd1f..e22a09c0c8 100644--- a/packages/loot-core/src/server/accounts/link.js+++ b/packages/loot-core/src/server/accounts/link.js@@ -42,6 +42,27 @@ export async function handoffPublicToken(institution, publicToken) {return id;}+export async function findOrCreateBank(institution, requisitionId) {+ let bank = await db.first(+ 'SELECT id, bank_id, name FROM banks WHERE bank_id = ?',+ [requisitionId],+ );++ if (bank) {+ return bank;+ }++ const bankData = {+ id: uuid.v4Sync(),+ bank_id: requisitionId,+ name: institution.name,+ };++ await db.insertWithUUID('banks', bankData);++ return bankData;+}+export async function addAccounts(bankId, accountIds, offbudgetIds = []) {let [[, userId], [, userKey]] = await asyncStorage.multiGet(['user-id',@@ -84,3 +105,56 @@ export async function addAccounts(bankId, accountIds, offbudgetIds = []) {}),);}++export async function addNordigenAccounts(+ bankId,+ accountIds,+ offbudgetIds = [],+) {+ let [[, userId], [, userKey]] = await asyncStorage.multiGet([+ 'user-id',+ 'user-key',+ ]);++ // Get all the available accounts+ let accounts = await bankSync.getNordigenAccounts(userId, userKey, bankId);++ // Only add the selected accounts+ accounts = accounts.filter(acct => accountIds.includes(acct.account_id));++ return Promise.all(+ accounts.map(async acct => {+ let id = await runMutator(async () => {+ let id = await db.insertAccount({+ account_id: acct.account_id,+ name: acct.name,+ official_name: acct.official_name,+ type: fromPlaidAccountType(acct.type),+ balance_current: amountToInteger(acct.balances.current),+ mask: acct.mask,+ bank: bankId,+ offbudget: offbudgetIds.includes(acct.account_id) ? 1 : 0,+ });++ // Create a transfer payee+ await db.insertPayee({+ name: '',+ transfer_acct: id,+ });++ return id;+ });++ // Do an initial sync+ await bankSync.syncNordigenAccount(+ userId,+ userKey,+ id,+ acct.account_id,+ bankId,+ );++ return id;+ }),+ );+}diff --git a/packages/loot-core/src/server/accounts/sync.js b/packages/loot-core/src/server/accounts/sync.jsindex 2dbe085ecf..7a64ebee0e 100644--- a/packages/loot-core/src/server/accounts/sync.js+++ b/packages/loot-core/src/server/accounts/sync.js@@ -1,3 +1,4 @@+import asyncStorage from '../../platform/server/asyncStorage';import * as monthUtils from '../../shared/months';import {makeChild as makeChildTransaction,@@ -75,6 +76,31 @@ export async function getAccounts(userId, userKey, id) {return accounts;}+export async function getNordigenAccounts(userId, userKey, id) {+ const userToken = await asyncStorage.getItem('user-token');+ if (userToken) {+ let res = await post(+ getServer().NORDIGEN_SERVER + '/accounts',+ {+ userId,+ key: userKey,+ item_id: id,+ },+ {+ 'X-ACTUAL-TOKEN': userToken,+ },+ );++ let { accounts } = res;++ accounts.forEach(acct => {+ acct.balances.current = getAccountBalance(acct);+ });++ return accounts;+ }+}+export function fromPlaid(trans) {return {imported_id: trans.transaction_id,@@ -149,6 +175,52 @@ async function downloadTransactions(};}+async function downloadNordigenTransactions(+ userId,+ userKey,+ acctId,+ bankId,+ since,+ count,+) {+ let userToken = await asyncStorage.getItem('user-token');+ if (userToken) {+ const endDate = new Date().toISOString().split('T')[0];++ const res = await post(+ getServer().NORDIGEN_SERVER + '/transactions',+ {+ userId: userId,+ key: userKey,+ requisitionId: bankId,+ accountId: acctId,+ startDate: since,+ endDate,+ },+ {+ 'X-ACTUAL-TOKEN': userToken,+ },+ );++ if (res.error_code) {+ throw BankSyncError(res.error_type, res.error_code);+ }++ const {+ transactions: { booked },+ balances,+ startingBalance,+ } = res;++ return {+ transactions: booked,+ accountBalances: balances,+ startingBalance,+ };+ }+ return;+}+async function resolvePayee(trans, payeeName, payeesToCreate) {if (trans.payee == null && payeeName) {// First check our registry of new payees (to avoid a db access)@@ -220,6 +292,90 @@ async function normalizeTransactions(return { normalized, payeesToCreate };}+async function normalizeNordigenTransactions(transactions, acctId) {+ let payeesToCreate = new Map();++ let normalized = [];+ for (let trans of transactions) {+ if (!trans.date) {+ trans.date = trans.valueDate;+ }++ if (!trans.amount) {+ trans.amount = trans.transactionAmount.amount;+ }++ // Validate the date because we do some stuff with it. The db+ // layer does better validation, but this will give nicer errors+ if (trans.date == null) {+ throw new Error('`date` is required when adding a transaction');+ }++ let payee_name;+ if (trans.amount >= 0) {+ const nameParts = [];+ nameParts.push(+ title(+ trans.debtorName || trans.remittanceInformationUnstructured || '',+ ),+ );+ if (trans.debtorAccount && trans.debtorAccount.iban) {+ nameParts.push(+ '(' ++ trans.debtorAccount.iban.slice(0, 4) ++ ' XXX ' ++ trans.debtorAccount.iban.slice(-4) ++ ')',+ );+ }+ payee_name = nameParts.join(' ');+ } else {+ const nameParts = [];+ nameParts.push(+ title(+ trans.creditorName || trans.remittanceInformationUnstructured || '',+ ),+ );+ if (trans.creditorAccount && trans.creditorAccount.iban) {+ nameParts.push(+ '(' ++ trans.creditorAccount.iban.slice(0, 4) ++ ' XXX ' ++ trans.creditorAccount.iban.slice(-4) ++ ')',+ );+ }+ payee_name = nameParts.join(' ');+ }++ trans.imported_payee = trans.imported_payee || payee_name;+ if (trans.imported_payee) {+ trans.imported_payee = trans.imported_payee.trim();+ }++ // It's important to resolve both the account and payee early so+ // when rules are run, they have the right data. Resolving payees+ // also simplifies the payee creation process+ trans.account = acctId;+ trans.payee = await resolvePayee(trans, payee_name, payeesToCreate);++ normalized.push({+ payee_name,+ trans: {+ amount: amountToInteger(trans.amount),+ payee: trans.payee,+ account: trans.account,+ date: trans.date,+ notes: trans.remittanceInformationUnstructured,+ imported_id: trans.transactionId,+ imported_payee: trans.imported_payee,+ },+ });+ }++ return { normalized, payeesToCreate };+}+async function createNewPayees(payeesToCreate, addsAndUpdates) {let usedPayeeIds = new Set(addsAndUpdates.map(t => t.payee));@@ -233,6 +389,153 @@ async function createNewPayees(payeesToCreate, addsAndUpdates) {});}+export async function reconcileNordigenTransactions(acctId, transactions) {+ const hasMatched = new Set();+ const updated = [];+ const added = [];++ let { normalized, payeesToCreate } = await normalizeNordigenTransactions(+ transactions,+ acctId,+ );++ // The first pass runs the rules, and preps data for fuzzy matching+ let transactionsStep1 = [];+ for (let { payee_name, trans, subtransactions } of normalized) {+ // Run the rules+ trans = runRules(trans);++ let match = null;+ let fuzzyDataset = null;++ // First, match with an existing transaction's imported_id. This+ // is the highest fidelity match and should always be attempted+ // first.+ if (trans.imported_id) {+ match = await db.first(+ 'SELECT * FROM v_transactions WHERE imported_id = ? AND account = ?',+ [trans.imported_id, acctId],+ );++ // TODO: Pending transactions++ if (match) {+ hasMatched.add(match.id);+ }+ }++ // If it didn't match, query data needed for fuzzy matching+ if (!match) {+ // Look 1 day ahead and 4 days back when fuzzy matching. This+ // needs to select all fields that need to be read from the+ // matched transaction. See the final pass below for the needed+ // fields.+ fuzzyDataset = await db.all(+ `SELECT id, date, imported_id, payee, category, notes FROM v_transactions+ WHERE date >= ? AND date <= ? AND amount = ? AND account = ? AND is_child = 0`,+ [+ db.toDateRepr(monthUtils.subDays(trans.date, 4)),+ db.toDateRepr(monthUtils.addDays(trans.date, 1)),+ trans.amount || 0,+ acctId,+ ],+ );+ }++ transactionsStep1.push({+ payee_name,+ trans,+ subtransactions,+ match,+ fuzzyDataset,+ });+ }++ // Next, do the fuzzy matching. This first pass matches based on the+ // payee id. We do this in multiple passes so that higher fidelity+ // matching always happens first, i.e. a transaction should match+ // match with low fidelity if a later transaction is going to match+ // the same one with high fidelity.+ let transactionsStep2 = transactionsStep1.map(data => {+ if (!data.match && data.fuzzyDataset) {+ // Try to find one where the payees match.+ let match = data.fuzzyDataset.find(+ row => !hasMatched.has(row.id) && data.trans.payee === row.payee,+ );++ if (match) {+ hasMatched.add(match.id);+ return { ...data, match };+ }+ }+ return data;+ });++ // The final fuzzy matching pass. This is the lowest fidelity+ // matching: it just find the first transaction that hasn't been+ // matched yet. Remember the the dataset only contains transactions+ // around the same date with the same amount.+ let transactionsStep3 = transactionsStep2.map(data => {+ if (!data.match && data.fuzzyDataset) {+ let match = data.fuzzyDataset.find(row => !hasMatched.has(row.id));+ if (match) {+ hasMatched.add(match.id);+ return { ...data, match };+ }+ }+ return data;+ });++ // Finally, generate & commit the changes+ for (let { trans, subtransactions, match } of transactionsStep3) {+ if (match) {+ // TODO: change the above sql query to use aql+ let existing = {+ ...match,+ cleared: match.cleared === 1,+ date: db.fromDateRepr(match.date),+ };++ // Update the transaction+ const updates = {+ date: trans.date,+ imported_id: trans.imported_id || null,+ payee: existing.payee || trans.payee || null,+ category: existing.category || trans.category || null,+ imported_payee: trans.imported_payee || null,+ notes: existing.notes || trans.notes || null,+ cleared: trans.cleared != null ? trans.cleared : true,+ };++ if (hasFieldsChanged(existing, updates, Object.keys(updates))) {+ updated.push({ id: existing.id, ...updates });+ }+ } else {+ // Insert a new transaction+ let finalTransaction = {+ ...trans,+ id: uuid.v4Sync(),+ category: trans.category || null,+ cleared: trans.cleared != null ? trans.cleared : true,+ };++ if (subtransactions && subtransactions.length > 0) {+ added.push(...makeSplitTransaction(finalTransaction, subtransactions));+ } else {+ added.push(finalTransaction);+ }+ }+ }++ await createNewPayees(payeesToCreate, [...added, ...updated]);+ await batchUpdateTransactions({ added, updated });++ return {+ added: added.map(trans => trans.id),+ updated: updated.map(trans => trans.id),+ };+}+export async function reconcileTransactions(acctId, transactions) {const hasMatched = new Set();const updated = [];@@ -430,6 +733,105 @@ export async function addTransactions(return newTransactions;}+export async function syncNordigenAccount(userId, userKey, id, acctId, bankId) {+ // TODO: Handle the case where transactions exist in the future+ // (that will make start date after end date)+ const latestTransaction = await db.first(+ 'SELECT * FROM v_transactions WHERE account = ? ORDER BY date DESC LIMIT 1',+ [id],+ );++ const acctRow = await db.select('accounts', id);++ if (latestTransaction) {+ const startingTransaction = await db.first(+ 'SELECT date FROM v_transactions WHERE account = ? ORDER BY date ASC LIMIT 1',+ [id],+ );+ const startingDate = db.fromDateRepr(startingTransaction.date);+ // assert(startingTransaction)++ // Get all transactions since the latest transaction, plus any 5+ // days before the latest transaction. This gives us a chance to+ // resolve any transactions that were entered manually.+ //+ // TODO: What this really should do is query the last imported_id+ // and since then+ let date = monthUtils.subDays(db.fromDateRepr(latestTransaction.date), 31);++ // Never download transactions before the starting date. This was+ // when the account was added to the system.+ if (date < startingDate) {+ date = startingDate;+ }++ let { transactions, accountBalance } = await downloadNordigenTransactions(+ userId,+ userKey,+ acctId,+ bankId,+ date,+ );++ if (transactions.length === 0) {+ return { added: [], updated: [] };+ }++ transactions = transactions.map(trans => ({ ...trans, account: id }));++ return runMutator(async () => {+ const result = await reconcileNordigenTransactions(id, transactions);+ await updateAccountBalance(id, accountBalance);+ return result;+ });+ } else {+ // Otherwise, download transaction for the past 30 days+ const startingDay = monthUtils.subDays(monthUtils.currentDay(), 30);++ const { transactions, startingBalance } =+ await downloadNordigenTransactions(+ userId,+ userKey,+ acctId,+ bankId,+ dateFns.format(dateFns.parseISO(startingDay), 'yyyy-MM-dd'),+ );++ // We need to add a transaction that represents the starting+ // balance for everything to balance out. In order to get balance+ // before the first imported transaction, we need to get the+ // current balance from the accounts table and subtract all the+ // imported transactions.++ const oldestTransaction = transactions[transactions.length - 1];++ const oldestDate =+ transactions.length > 0+ ? oldestTransaction.valueDate+ : monthUtils.currentDay();++ const payee = await getStartingBalancePayee();++ return runMutator(async () => {+ let initialId = await db.insertTransaction({+ account: id,+ amount: startingBalance,+ category: acctRow.offbudget === 0 ? payee.category : null,+ payee: payee.id,+ date: oldestDate,+ cleared: true,+ starting_balance_flag: true,+ });++ let result = await reconcileNordigenTransactions(id, transactions);+ return {+ ...result,+ added: [initialId, ...result.added],+ };+ });+ }+}+export async function syncAccount(userId, userKey, id, acctId, bankId) {// TODO: Handle the case where transactions exist in the future// (that will make start date after end date)diff --git a/packages/loot-core/src/server/main.js b/packages/loot-core/src/server/main.jsindex 2f0cc2b511..cb8ae00984 100644--- a/packages/loot-core/src/server/main.js+++ b/packages/loot-core/src/server/main.js@@ -799,7 +799,7 @@ handlers['accounts-link'] = async function ({]);// Get all the available accounts and find the selected one- let accounts = await bankSync.getAccounts(userId, userKey, bankId);+ let accounts = await bankSync.getNordigenAccounts(userId, userKey, bankId);let account = accounts.find(acct => acct.account_id === accountId);await db.update('accounts', {@@ -830,6 +830,57 @@ handlers['accounts-link'] = async function ({return 'ok';};+handlers['nordigen-accounts-link'] = async function ({+ requisitionId,+ account,+ upgradingId,+}) {+ let id;+ let bank = await link.findOrCreateBank(account.institution, requisitionId);++ if (upgradingId) {+ const accRow = await db.first('SELECT * FROM accounts WHERE id = ?', [+ upgradingId,+ ]);+ id = accRow.id;+ await db.update('accounts', {+ id,+ account_id: account.account_id,+ bank: bank.id,+ });+ } else {+ id = uuid.v4Sync();+ await db.insertWithUUID('accounts', {+ id,+ account_id: account.account_id,+ mask: account.mask,+ name: account.name,+ official_name: account.official_name,+ type: account.type,+ bank: bank.id,+ });+ await db.insertPayee({+ name: '',+ transfer_acct: id,+ });+ }++ await bankSync.syncNordigenAccount(+ undefined,+ undefined,+ id,+ account.account_id,+ bank.bank_id,+ );++ connection.send('sync-event', {+ type: 'success',+ tables: ['transactions'],+ });++ return 'ok';+};+handlers['accounts-connect'] = async function ({institution,publicToken,@@ -841,6 +892,17 @@ handlers['accounts-connect'] = async function ({return ids;};+handlers['nordigen-accounts-connect'] = async function ({+ institution,+ publicToken,+ accountIds,+ offbudgetIds,+}) {+ let bankId = await link.handoffPublicToken(institution, publicToken);+ let ids = await link.addNordigenAccounts(bankId, accountIds, offbudgetIds);+ return ids;+};+handlers['account-create'] = mutator(async function ({name,type,@@ -1122,6 +1184,176 @@ handlers['accounts-sync'] = async function ({ id }) {return { errors, newTransactions, matchedTransactions, updatedAccounts };};+handlers['nordigen-poll-web-token'] = async function ({+ upgradingAccountId,+ requisitionId,+}) {+ let userToken = await asyncStorage.getItem('user-token');++ if (userToken) {+ let startTime = Date.now();+ stopPolling = false;++ async function getData(cb) {+ if (stopPolling) {+ return;+ }++ if (Date.now() - startTime >= 1000 * 60 * 10) {+ cb('timeout');+ return;+ }++ let data = await post(+ getServer().NORDIGEN_SERVER + '/get-accounts',+ {+ upgradingAccountId,+ requisitionId,+ },+ {+ 'X-ACTUAL-TOKEN': userToken,+ },+ );++ if (data) {+ if (data.error) {+ cb('unknown');+ } else {+ cb(null, data);+ }+ } else {+ setTimeout(() => getData(cb), 3000);+ }+ }++ return new Promise(resolve => {+ getData((error, data) => {+ if (error) {+ resolve({ error });+ } else {+ resolve({ data });+ }+ });+ });+ }++ return null;+};++handlers['nordigen-poll-web-token-stop'] = async function () {+ stopPolling = true;+ return 'ok';+};++handlers['nordigen-create-web-token'] = async function ({+ upgradingAccountId,+ institutionId,+ accessValidForDays,+}) {+ let userToken = await asyncStorage.getItem('user-token');++ if (userToken) {+ try {+ return await post(+ getServer().NORDIGEN_SERVER + '/create-web-token',+ {+ upgradingAccountId,+ institutionId,+ accessValidForDays,+ },+ {+ 'X-ACTUAL-TOKEN': userToken,+ },+ );+ } catch (error) {+ console.error(error);+ return { error: 'failed' };+ }+ }+ return { error: 'unauthorized' };+};++handlers['nordigen-accounts-sync'] = async function ({ id }) {+ let [[, userId], [, userKey]] = await asyncStorage.multiGet([+ 'user-id',+ 'user-key',+ ]);+ let accounts = await db.runQuery(+ `SELECT a.*, b.bank_id as bankId FROM accounts a+ LEFT JOIN banks b ON a.bank = b.id+ WHERE a.tombstone = 0 AND a.closed = 0`,+ [],+ true,+ );++ if (id) {+ accounts = accounts.filter(acct => acct.id === id);+ }++ let errors = [];+ let newTransactions = [];+ let matchedTransactions = [];+ let updatedAccounts = [];++ for (var i = 0; i < accounts.length; i++) {+ const acct = accounts[i];+ if (acct.bankId) {+ try {+ const res = await bankSync.syncNordigenAccount(+ userId,+ userKey,+ acct.id,+ acct.account_id,+ acct.bankId,+ );+ let { added, updated } = res;++ newTransactions = newTransactions.concat(added);+ matchedTransactions = matchedTransactions.concat(updated);++ if (added.length > 0 || updated.length > 0) {+ updatedAccounts = updatedAccounts.concat(acct.id);+ }+ } catch (err) {+ if (err.type === 'BankSyncError') {+ errors.push({+ type: 'SyncError',+ accountId: acct.id,+ message: 'Failed syncing account "' + acct.name + '".',+ category: err.category,+ code: err.code,+ });+ } else if (err instanceof PostError && err.reason !== 'internal') {+ errors.push({+ accountId: acct.id,+ message: `Account "${acct.name}" is not linked properly. Please link it again`,+ });+ } else {+ errors.push({+ accountId: acct.id,+ message:+ 'There was an internal error. Please get in touch https://actualbudget.github.io/docs/Contact for support.',+ internal: err.stack,+ });++ err.message = 'Failed syncing account: ' + err.message;++ captureException(err);+ }+ }+ }+ }++ if (updatedAccounts.length > 0) {+ connection.send('sync-event', {+ type: 'success',+ tables: ['transactions'],+ });+ }++ return { errors, newTransactions, matchedTransactions, updatedAccounts };+};+handlers['transactions-import'] = mutator(function ({accountId,transactions,@@ -1167,20 +1399,28 @@ handlers['account-unlink'] = mutator(async function ({ id }) {[bankId],);- if (count === 0) {- // No more accounts are associated with this bank. We can remove- // it from Plaid.-- let [[, userId], [, key]] = await asyncStorage.multiGet([- 'user-id',- 'user-key',- ]);+ // No more accounts are associated with this bank. We can remove+ // it from Nordigen.+ let userToken = await asyncStorage.getItem('user-token');- await post(getServer().PLAID_SERVER + '/remove-access-token', {- userId,- key,- item_id: bankId,- });+ if (userToken && count === 0) {+ let { bank_id: requisitionId } = await db.first(+ 'SELECT bank_id FROM banks WHERE id = ?',+ [bankId],+ );+ try {+ await post(+ getServer().NORDIGEN_SERVER + '/remove-account',+ {+ requisitionId: requisitionId,+ },+ {+ 'X-ACTUAL-TOKEN': userToken,+ },+ );+ } catch (error) {+ console.log({ error });+ }}return 'ok';diff --git a/packages/loot-core/src/server/post.js b/packages/loot-core/src/server/post.jsindex 59fefe719d..104fb4d7ac 100644--- a/packages/loot-core/src/server/post.js+++ b/packages/loot-core/src/server/post.js@@ -19,7 +19,7 @@ function throwIfNot200(res, text) {}}-export async function post(url, data) {+export async function post(url, data, headers = {}) {let text;let res;@@ -28,6 +28,7 @@ export async function post(url, data) {method: 'POST',body: JSON.stringify(data),headers: {+ ...headers,'Content-Type': 'application/json',},});diff --git a/packages/loot-core/src/server/server-config.js b/packages/loot-core/src/server/server-config.jsindex f2c0058c69..8346db6a66 100644--- a/packages/loot-core/src/server/server-config.js+++ b/packages/loot-core/src/server/server-config.js@@ -24,6 +24,7 @@ export function getServer(url) {SYNC_SERVER: joinURL(url, '/sync'),SIGNUP_SERVER: joinURL(url, '/account'),PLAID_SERVER: joinURL(url, '/plaid'),+ NORDIGEN_SERVER: joinURL(url, '/nordigen'),};}return config;diff --git a/packages/loot-core/src/server/sync/index.js b/packages/loot-core/src/server/sync/index.jsindex afdf6e4cfd..1adf620c2a 100644--- a/packages/loot-core/src/server/sync/index.js+++ b/packages/loot-core/src/server/sync/index.js@@ -99,13 +99,15 @@ function apply(msg, prev) {}}+// TODO: convert to `whereIn`async function fetchAll(table, ids) {let results = [];- // TODO: convert to `whereIn`+ // was 500, but that caused a stack overflow in Safari+ let batchSize = 100;- for (let i = 0; i < ids.length; i += 500) {- let partIds = ids.slice(i, i + 500);+ for (let i = 0; i < ids.length; i += batchSize) {+ let partIds = ids.slice(i, i + batchSize);let sql;let column = `${table}.id`;@@ -131,7 +133,14 @@ async function fetchAll(table, ids) {let rows = await db.runQuery(sql, partIds, true);results = results.concat(rows);} catch (error) {- throw new SyncError('invalid-schema', { error, sql, params: partIds });+ throw new SyncError('invalid-schema', {+ error: {+ message: error.message,+ stack: error.stack,+ },+ sql,+ params: partIds,+ });}}diff --git a/packages/loot-design/src/components/common.js b/packages/loot-design/src/components/common.jsindex cd98c800cb..cc86b99627 100644--- a/packages/loot-design/src/components/common.js+++ b/packages/loot-design/src/components/common.js@@ -508,7 +508,13 @@ export const Select = React.forwardRef(},);-export function CustomSelect({ options, value, onChange, style }) {+export function CustomSelect({+ options,+ value,+ onChange,+ style,+ disabledKeys = [],+}) {return (<ListboxInputvalue={value}@@ -522,7 +528,11 @@ export function CustomSelect({ options, value, onChange, style }) {<ListboxPopover style={{ zIndex: 10000, outline: 0, borderRadius: 4 }}><ListboxList>{options.map(([value, label]) => (- <ListboxOption key={value} value={value}>+ <ListboxOption+ key={value}+ value={value}+ disabled={disabledKeys.includes(value)}+ >{label}</ListboxOption>))}diff --git a/packages/loot-design/src/components/modals/ConfigureLinkedAccounts.js b/packages/loot-design/src/components/modals/ConfigureLinkedAccounts.jsindex d829175eee..54ba57dedf 100644--- a/packages/loot-design/src/components/modals/ConfigureLinkedAccounts.js+++ b/packages/loot-design/src/components/modals/ConfigureLinkedAccounts.js@@ -93,7 +93,7 @@ export default function ConfigureLinkedAccounts({}async function onNext() {- let ids = await actions.connectAccounts(+ let ids = await actions.connectNordigenAccounts(institution,publicToken,accounts.map(acct => acct.id),diff --git a/packages/loot-design/src/components/modals/NordigenExternalMsg.js b/packages/loot-design/src/components/modals/NordigenExternalMsg.jsnew file mode 100644index 0000000000..beb96ae03b--- /dev/null+++ b/packages/loot-design/src/components/modals/NordigenExternalMsg.js@@ -0,0 +1,156 @@+import React, { useState, useRef } from 'react';++import { colors } from '../../style';+import AnimatedLoading from '../../svg/AnimatedLoading';+import { Error } from '../alerts';+import {+ CustomSelect,+ View,+ Modal,+ Button,+ P,+ ModalButtons,+ Strong,+} from '../common';++function renderError(error) {+ return (+ <Error style={{ alignSelf: 'center' }}>+ {error === 'timeout'+ ? 'Timed out. Please try again.'+ : 'An error occurred while linking your account, sorry!'}+ </Error>+ );+}++export default function NordigenExternalMsg({+ modalProps,+ onMoveExternal,+ onSuccess,+ onClose: originalOnClose,+}) {+ let [waiting, setWaiting] = useState(null);+ let [success, setSuccess] = useState(false);+ let [institutionId, setInstitutionId] = useState('default');+ let [error, setError] = useState(null);+ let data = useRef(null);++ async function onJump() {+ setError(null);+ setWaiting('browser');++ let res = await onMoveExternal({ institutionId });+ if (res.error) {+ setError(res.error);+ setWaiting(null);+ return;+ }++ data.current = res.data;+ setWaiting(null);+ setSuccess(true);+ }++ function onClose() {+ originalOnClose && originalOnClose();+ modalProps.onClose();+ }++ async function onContinue() {+ setWaiting('accounts');+ await onSuccess(data.current);+ setWaiting(null);+ }++ const renderLinkButton = () => {+ return (+ <View>+ <Strong>Choose your banks:</Strong>+ <CustomSelect+ options={[+ ['default', 'Choose your bank'],+ ['ING_PL_INGBPLPW', 'ING PL'],+ ['MBANK_RETAIL_BREXPLPW', 'MBANK'],+ ['SANDBOXFINANCE_SFIN0000', 'DEMO - TEST'],+ ]}+ disabledKeys={['default']}+ onChange={val => {+ setInstitutionId(val);+ }}+ value={institutionId || 'default'}+ />+ <Button+ primary+ style={{+ padding: '10px 0',+ fontSize: 15,+ fontWeight: 600,+ marginTop: 10,+ }}+ onClick={onJump}+ disabled={institutionId === 'default'}+ >+ Link bank in browser →+ </Button>+ </View>+ );+ };++ return (+ <Modal+ title="Link Your Bank"+ {...modalProps}+ onClose={onClose}+ style={{ flex: 0 }}+ >+ {() => (+ <View>+ <P style={{ fontSize: 15 }}>+ To link your bank account, you will be redirected to a new tab where+ you will have the possibility to grant access to your bank for+ Nordigen.+ </P>++ {error && renderError(error)}++ {waiting ? (+ <View style={{ alignItems: 'center', marginTop: 15 }}>+ <AnimatedLoading+ color={colors.n1}+ style={{ width: 20, height: 20 }}+ />+ <View style={{ marginTop: 10, color: colors.n4 }}>+ {waiting === 'browser'+ ? 'Waiting on Nordigen...'+ : waiting === 'accounts'+ ? 'Loading accounts...'+ : null}+ </View>+ </View>+ ) : success ? (+ <Button+ primary+ style={{+ padding: '10px 0',+ fontSize: 15,+ fontWeight: 600,+ marginTop: 10,+ backgroundColor: colors.g4,+ borderColor: colors.g4,+ }}+ onClick={onContinue}+ >+ Success! Click to continue →+ </Button>+ ) : (+ renderLinkButton()+ )}++ <ModalButtons style={{ marginTop: 10 }}>+ <Button onClick={() => modalProps.onBack()}>Back</Button>+ </ModalButtons>+ </View>+ )}+ </Modal>+ );+}diff --git a/packages/loot-design/src/components/modals/SelectLinkedAccounts.js b/packages/loot-design/src/components/modals/SelectLinkedAccounts.jsindex 625409ca1f..842c552441 100644--- a/packages/loot-design/src/components/modals/SelectLinkedAccounts.js+++ b/packages/loot-design/src/components/modals/SelectLinkedAccounts.js@@ -1,129 +1,122 @@-import React, { useState } from 'react';+import React, { useEffect, useState } from 'react';-import {- fromPlaidAccountType,- prettyAccountType,-} from 'loot-core/src/shared/accounts';--import { styles, colors } from '../../style';-import { View, Text, Modal, P, Button } from '../common';--let selectedStyle = {- color: colors.n1,-};+import { View, Modal, P, Button, Strong, CustomSelect } from '../common';function EmptyMessage() {return null;}-function Account({ account, selected, onSelect }) {- return (- <View- style={[- { marginBottom: 8, flexShrink: 0, userSelect: 'none' },- styles.mediumText,- ]}- onClick={onSelect}- >- <View- style={[- {- padding: 12,- ...styles.shadow,- cursor: 'pointer',- transition: 'transform .20s',- fontSize: 14,- borderRadius: 4,- flexDirection: 'row',- alignItems: 'center',- border: '1px solid ' + colors.n10,- ':hover': selectedStyle,- },- selected && {- ...selectedStyle,- borderColor: colors.b9,- backgroundColor: colors.b10,- },- ]}- >- <View>- <View style={{ fontWeight: 600, flex: 1 }}>{account.name}</View>- <View- style={{- fontSize: 13,- color: colors.n5,- flexDirection: 'row',- }}- >- {prettyAccountType(- fromPlaidAccountType(account.type, account.subtype),- )}- <Text style={{ marginLeft: 4 }}>- ...- {account.mask}- </Text>- </View>- </View>- </View>- </View>- );-}-export default function SelectLinkedAccounts({- institution,- publicToken,- upgradingId,+ upgradingAccountId,modalProps,- accounts,+ requisitionId,+ accounts: importedAccounts,+ actualAccounts,actions,}) {- let [selectedAccounts, setSelectedAccounts] = useState([]);-- function toggleAccount(id) {- if (upgradingId) {- setSelectedAccounts([id]);- } else {- if (selectedAccounts.includes(id)) {- setSelectedAccounts(selectedAccounts.filter(x => x !== id));- } else {- setSelectedAccounts([...selectedAccounts, id]);- }- }- }+ let [chosenAccounts, setChosenAccounts] = useState([]);- async function onNext() {- if (selectedAccounts.length > 0) {- if (upgradingId) {- actions.linkAccount(- institution,- publicToken,- selectedAccounts[0],- upgradingId,- );- actions.closeModal();- } else {- actions.pushModal('configure-linked-accounts', {- institution,- publicToken,- accounts: selectedAccounts.map(id =>- accounts.find(acct => acct.id === id),- ),+ const addAccountOption = { id: 'new', name: 'Create new account' };++ const importedAccountsToSelect = importedAccounts.filter(+ account =>+ !chosenAccounts+ .map(acc => acc.chosenImportedAccountId)+ .includes(account.account_id),+ );++ const actualAccountsToSelect = [+ addAccountOption,+ ...actualAccounts.filter(+ account =>+ !chosenAccounts+ .map(acc => acc.chosenActualAccountId)+ .includes(account.id),+ ),+ ];++ useEffect(() => {+ const chosenAccountsToAdd = [];+ importedAccountsToSelect.forEach(importedAccount => {+ // Try to auto-match accounts based on account_id or mask+ // Add matched accounts to list of selected accounts+ const matchedActualAccount = actualAccountsToSelect.find(+ actualAccount => {+ return (+ actualAccount.account_id === importedAccount.account_id ||+ actualAccount.mask === importedAccount.mask+ );+ },+ );++ if (matchedActualAccount) {+ chosenAccountsToAdd.push({+ chosenImportedAccountId: importedAccount.account_id,+ chosenActualAccountId: matchedActualAccount.id,});}- }+ });++ setChosenAccounts([...chosenAccounts, ...chosenAccountsToAdd]);+ }, []);++ let [selectedImportAccountId, setSelectedImportAccountId] = useState(+ importedAccountsToSelect[0] && importedAccountsToSelect[0].account_id,+ );+ let [selectedAccountId, setSelectedAccountId] = useState(+ actualAccountsToSelect[0] && actualAccountsToSelect[0].id,+ );++ async function onNext() {+ chosenAccounts.forEach(chosenAccount => {+ const importedAccount = importedAccounts.find(+ account => account.account_id === chosenAccount.chosenImportedAccountId,+ );++ actions.linkAccount(+ requisitionId,+ importedAccount,+ chosenAccount.chosenActualAccountId !== addAccountOption.id+ ? chosenAccount.chosenActualAccountId+ : undefined,+ );+ });++ actions.closeModal();}+ function addToChosenAccounts() {+ setChosenAccounts([+ ...chosenAccounts,+ {+ chosenImportedAccountId: selectedImportAccountId,+ chosenActualAccountId: selectedAccountId,+ },+ ]);+ }++ // Update dropbox with available accounts to select+ useEffect(() => {+ const newSelectedImportAccountId =+ importedAccountsToSelect[0] && importedAccountsToSelect[0].account_id;+ const newSelectedAccountId =+ actualAccountsToSelect[0] && actualAccountsToSelect[0].id;+ setSelectedImportAccountId(newSelectedImportAccountId);+ setSelectedAccountId(newSelectedAccountId);+ }, [chosenAccounts]);++ const removeChoose = chosenAccount => {+ setChosenAccounts([...chosenAccounts.filter(acc => acc !== chosenAccount)]);+ };+return (- <Modal- title={upgradingId ? 'Link Account' : 'Link Accounts'}- {...modalProps}- >+ <Modal title={'Link Accounts'} {...modalProps}>{() => (<View style={{ maxWidth: 500 }}>- {upgradingId ? (+ {upgradingAccountId ? (<P>- We found the following accounts. Select the one you want to link- with:+ You allowed access to the following accounts. Select the one you+ want to link with:</P>) : (<P>@@ -144,21 +137,98 @@ export default function SelectLinkedAccounts({}}><View>- {accounts.length === 0 ? (+ {importedAccounts.length === 0 ? (<EmptyMessage />) : (- accounts.map(account => {- let selected = selectedAccounts.includes(account.id);-- return (- <Account- key={account.id}- account={account}- selected={selected}- onSelect={() => toggleAccount(account.id)}- />- );- })+ <View>+ {importedAccountsToSelect.length ? (+ <View+ style={{+ flexDirection: 'row',+ justifyContent: 'flex-center',+ margin: '30px 0',+ borderBottom: 'solid 1px',+ }}+ >+ <View>+ <Strong>Imported Account:</Strong>+ <CustomSelect+ options={importedAccountsToSelect.map(account => [+ account.account_id,+ account.name,+ ])}+ onChange={val => {+ setSelectedImportAccountId(val);+ }}+ value={selectedImportAccountId}+ />+ </View>++ <View>+ <Strong>Actual Budget Account:</Strong>+ <CustomSelect+ options={actualAccountsToSelect.map(account => [+ account.id,+ account.name,+ ])}+ onChange={val => {+ setSelectedAccountId(val);+ }}+ value={selectedAccountId}+ />+ </View>++ <Button+ primary+ style={{+ padding: '10px',+ fontSize: 15,+ margin: 10,+ }}+ onClick={addToChosenAccounts}+ >+ Link account →+ </Button>+ </View>+ ) : (+ ''+ )}+ {chosenAccounts.map(chosenAccount => {+ const { chosenImportedAccountId, chosenActualAccountId } =+ chosenAccount;+ const importedAccount = importedAccounts.find(+ acc => acc.account_id === chosenImportedAccountId,+ );+ const actualAccount = [+ addAccountOption,+ ...actualAccounts,+ ].find(acc => acc.id === chosenActualAccountId);++ return (+ <View+ key={chosenImportedAccountId}+ style={{+ flexDirection: 'row',+ justifyContent: 'flex-center',+ marginTop: 30,+ }}+ >+ {importedAccount.name} → {actualAccount.name}+ <Button+ primary+ style={{+ padding: '10px',+ fontSize: 15,+ margin: 10,+ }}+ onClick={() => removeChoose(chosenAccount)}+ >+ Remove →+ </Button>+ </View>+ );+ })}+ </View>)}</View></View>@@ -173,8 +243,8 @@ export default function SelectLinkedAccounts({<Button style={{ marginRight: 10 }} onClick={modalProps.onClose}>Cancel</Button>- <Button primary onClick={onNext}>- {upgradingId ? 'Link Account' : 'Next'}+ <Button primary onClick={onNext} disabled={!chosenAccounts.length}>+ Link accounts</Button></View></View>