A local-first personal finance app
:hedgehog: Speeding up large budgets by only running rules when needed (#8946)
Michael Clarkdf0cfe3489b0
diff --git a/packages/loot-core/src/server/transactions/transaction-rules.test.ts b/packages/loot-core/src/server/transactions/transaction-rules.test.tsindex 2f6b766d6a..1819f00971 100644--- a/packages/loot-core/src/server/transactions/transaction-rules.test.ts+++ b/packages/loot-core/src/server/transactions/transaction-rules.test.ts@@ -5,6 +5,7 @@ import { loadMappings } from '#server/db/mappings';import { q } from '#shared/query';import {+ actionsReferenceBalance,conditionsToAQL,deleteRule,getProbableCategory,@@ -1307,3 +1308,160 @@ describe('Learning categories', () => {// TODO: write tests for split transactions});++// Counts prepared statements that aggregate amounts, which is what the+// running-balance query does and nothing else in `runRules` does+function countRunningBalanceQueries() {+ const rawDb = db.getDatabase() as unknown as {+ prepare: (sql: string) => unknown;+ };+ const originalPrepare = rawDb.prepare.bind(rawDb);+ let count = 0;+ rawDb.prepare = (sql: string) => {+ if (/SUM\(/i.test(sql)) {+ count++;+ }+ return originalPrepare(sql);+ };+ return () => count;+}++describe('Running balance for rules', () => {+ test('actionsReferenceBalance only flags templates and formulas that use it', () => {+ expect(+ actionsReferenceBalance([+ { options: { template: 'Balance: {{balance}}' } },+ ]),+ ).toBe(true);+ expect(+ actionsReferenceBalance([+ { options: { template: '{{add balance amount}}' } },+ ]),+ ).toBe(true);+ expect(+ actionsReferenceBalance([{ options: { formula: '=balance * 2' } }]),+ ).toBe(true);++ expect(actionsReferenceBalance([])).toBe(false);+ expect(actionsReferenceBalance([{}])).toBe(false);+ expect(+ actionsReferenceBalance([{ options: { template: 'well balanced' } }]),+ ).toBe(false);+ expect(+ actionsReferenceBalance([+ { options: { formula: '=BALANCE_OF("Savings") * 2' } },+ ]),+ ).toBe(false);+ });++ test('a template that uses the balance still sees the running balance', async () => {+ await loadRules();+ const account = await db.insertAccount({ name: 'bank' });+ await db.insertTransaction({+ account,+ date: '2020-01-01',+ amount: 1000,+ });+ await db.insertTransaction({+ account,+ date: '2020-01-02',+ amount: 2500,+ });++ await insertRule({+ stage: null,+ conditionsOp: 'and',+ conditions: [+ { op: 'contains', field: 'imported_payee', value: 'kroger' },+ ],+ actions: [+ {+ op: 'set',+ field: 'notes',+ value: '',+ options: { template: 'Balance: {{balance}}' },+ },+ ],+ });++ const queries = countRunningBalanceQueries();+ const transaction = await runRules({+ imported_payee: 'kroger',+ account,+ date: '2020-01-03',+ amount: 50,+ });++ expect(transaction.notes).toBe('Balance: 3500');+ expect(transaction).not.toHaveProperty('balance');+ expect(queries()).toBe(1);+ });++ test('a rule that uses the balance but does not match runs no query', async () => {+ await loadRules();+ const account = await db.insertAccount({ name: 'bank' });+ await db.insertTransaction({+ account,+ date: '2020-01-01',+ amount: 1000,+ });++ await insertRule({+ stage: null,+ conditionsOp: 'and',+ conditions: [+ { op: 'contains', field: 'imported_payee', value: 'walmart' },+ ],+ actions: [+ {+ op: 'set',+ field: 'notes',+ value: '',+ options: { template: 'Balance: {{balance}}' },+ },+ ],+ });++ const queries = countRunningBalanceQueries();+ const transaction = await runRules({+ imported_payee: 'kroger',+ account,+ date: '2020-01-03',+ amount: 50,+ });++ expect(transaction.notes).toBeUndefined();+ expect(queries()).toBe(0);+ });++ test('rules that do not use the balance still apply normally', async () => {+ await loadRules();+ const account = await db.insertAccount({ name: 'bank' });+ await db.insertTransaction({+ account,+ date: '2020-01-01',+ amount: 1000,+ });++ await insertRule({+ stage: null,+ conditionsOp: 'and',+ conditions: [+ { op: 'contains', field: 'imported_payee', value: 'kroger' },+ ],+ actions: [{ op: 'set', field: 'notes', value: 'plain' }],+ });++ const queries = countRunningBalanceQueries();+ const transaction = await runRules({+ imported_payee: 'kroger',+ account,+ date: '2020-01-03',+ amount: 50,+ });++ expect(transaction.notes).toBe('plain');+ expect(transaction).not.toHaveProperty('balance');+ expect(queries()).toBe(0);+ });+});diff --git a/packages/loot-core/src/server/transactions/transaction-rules.ts b/packages/loot-core/src/server/transactions/transaction-rules.tsindex f4efc50a60..5e3081dd91 100644--- a/packages/loot-core/src/server/transactions/transaction-rules.ts+++ b/packages/loot-core/src/server/transactions/transaction-rules.ts@@ -333,9 +333,33 @@ export async function runRules(accountsMap = accounts;}- let finalTrans = await prepareTransactionForRules({ ...trans }, accountsMap);+ const rules = rankRules(+ fastSetMerge(+ firstcharIndexer.getApplicableRules(trans),+ payeeIndexer.getApplicableRules(trans),+ ),+ );++ let finalTrans = await prepareTransactionForRules({ ...trans }, accountsMap, {+ includeBalance: false,+ });let lastCategoryIdForGroup: string | null = finalTrans.category ?? null;+ // The running balance is a query over every earlier transaction in+ // the account, so fetch it at most once, and only for a rule that is+ // about to run and actually reads it. `rules` is only the candidates+ // from the indexers, so checking their actions up front isn't enough.+ let hasBalance = false;+ async function ensureBalanceFor(rule: { actions: Action[] }) {+ if (!hasBalance && trans.account && actionsReferenceBalance(rule.actions)) {+ finalTrans.balance = await getRunningBalanceBeforeTransaction(+ trans,+ trans.account,+ );+ hasBalance = true;+ }+ }+let scheduleRuleID = '';// Check if a schedule is attached to this transaction and if so get the rule ID attached to that schedule.if (trans.schedule != null) {@@ -348,13 +372,6 @@ export async function runRules(const RuleIdsLinkedToSchedules =await getAllRuleIdsFromSchedules(scheduleRuleID);- const rules = rankRules(- fastSetMerge(- firstcharIndexer.getApplicableRules(trans),- payeeIndexer.getApplicableRules(trans),- ),- );-const formulaStrings = rules.flatMap(rule =>collectFormulasFromActions(rule.actions),);@@ -369,6 +386,7 @@ export async function runRules(if (scheduleRuleID !== '') {if (rules[i].id === scheduleRuleID) {// bypass condition checking to run the rule even if the transaction date falls outside of the schedule's date range.+ await ensureBalanceFor(rules[i]);const changes = rules[i].execActions(finalTrans);finalTrans = Object.assign({}, finalTrans, changes);await resolvePayeeNameForRules(finalTrans);@@ -381,7 +399,14 @@ export async function runRules(continue;} else {// if a rule is not linked to a schedule, run it.- finalTrans = rules[i].apply(finalTrans);+ if (rules[i].evalConditions(finalTrans)) {+ await ensureBalanceFor(rules[i]);+ finalTrans = Object.assign(+ {},+ finalTrans,+ rules[i].execActions(finalTrans),+ );+ }await resolvePayeeNameForRules(finalTrans);lastCategoryIdForGroup = await refreshCategoryGroupIfChanged(finalTrans,@@ -390,7 +415,14 @@ export async function runRules(}} else {// if there is no scheduleRuleID then just run all rules.- finalTrans = rules[i].apply(finalTrans);+ if (rules[i].evalConditions(finalTrans)) {+ await ensureBalanceFor(rules[i]);+ finalTrans = Object.assign(+ {},+ finalTrans,+ rules[i].execActions(finalTrans),+ );+ }await resolvePayeeNameForRules(finalTrans);lastCategoryIdForGroup = await refreshCategoryGroupIfChanged(finalTrans,@@ -755,9 +787,10 @@ export async function applyActions(const accounts: db.DbAccount[] = await db.getAccounts();const accountsMap = new Map(accounts.map(account => [account.id, account]));+ const includeBalance = actionsReferenceBalance(parsedActions);const transactionsForRules = await Promise.all(- transactions.map(transactions =>- prepareTransactionForRules(transactions, accountsMap),+ transactions.map(transaction =>+ prepareTransactionForRules(transaction, accountsMap, { includeBalance }),),);@@ -1084,9 +1117,28 @@ export async function prefetchBalanceOfForTransaction(return map;}+const BALANCE_VARIABLE = /\bbalance\b/i;++// Whether any of these actions can read the transaction's running+// balance: Handlebars templates via `{{balance}}` and formulas via the+// `balance` variable. `BALANCE_OF(...)` is a different, prefetched value+// and does not count.+export function actionsReferenceBalance(+ actions: Array<{ options?: { template?: string; formula?: string } }>,+): boolean {+ return actions.some(action => {+ const { template, formula } = action.options ?? {};+ return (+ (typeof template === 'string' && BALANCE_VARIABLE.test(template)) ||+ (typeof formula === 'string' && BALANCE_VARIABLE.test(formula))+ );+ });+}+export async function prepareTransactionForRules(trans: TransactionEntity,accounts: Map<string, db.DbAccount> | null = null,+ { includeBalance = true }: { includeBalance?: boolean } = {},): Promise<TransactionForRules> {const r: TransactionForRules = { ...trans };if (trans.payee) {@@ -1107,7 +1159,12 @@ export async function prepareTransactionForRules(r._account_name = r._account?.name || '';}- r.balance = await getRunningBalanceBeforeTransaction(trans, trans.account);+ if (includeBalance) {+ r.balance = await getRunningBalanceBeforeTransaction(+ trans,+ trans.account,+ );+ }}if (trans.category) {diff --git a/upcoming-release-notes/skip-running-balance-query.md b/upcoming-release-notes/skip-running-balance-query.mdnew file mode 100644index 0000000000..ac6dfd8453--- /dev/null+++ b/upcoming-release-notes/skip-running-balance-query.md@@ -0,0 +1,6 @@+---+category: Enhancements+authors: [MikesGlitch]+---++Speed up editing transactions by only calculating the running balance when a rule uses it