Skip to content

A local-first personal finance app

:hedgehog: Speeding up large budgets by only running rules when needed (#8946)

Michael Clarkdf0cfe3489b0
diff --git a/packages/loot-core/src/server/transactions/transaction-rules.test.ts b/packages/loot-core/src/server/transactions/transaction-rules.test.ts
index 2f6b766d6a..1819f00971 100644
--- a/packages/loot-core/src/server/transactions/transaction-rules.test.ts
+++ b/packages/loot-core/src/server/transactions/transaction-rules.test.ts
@@ -5,6 +5,7 @@ import { loadMappings } from '#server/db/mappings';
import { q } from '#shared/query';
import {
+ actionsReferenceBalance,
conditionsToAQL,
deleteRule,
getProbableCategory,
@@ -1307,3 +1308,160 @@ describe('Learning categories', () => {
// TODO: write tests for split transactions
});
+
+// Counts prepared statements that aggregate amounts, which is what the
+// running-balance query does and nothing else in `runRules` does
+function countRunningBalanceQueries() {
+ const rawDb = db.getDatabase() as unknown as {
+ prepare: (sql: string) => unknown;
+ };
+ const originalPrepare = rawDb.prepare.bind(rawDb);
+ let count = 0;
+ rawDb.prepare = (sql: string) => {
+ if (/SUM\(/i.test(sql)) {
+ count++;
+ }
+ return originalPrepare(sql);
+ };
+ return () => count;
+}
+
+describe('Running balance for rules', () => {
+ test('actionsReferenceBalance only flags templates and formulas that use it', () => {
+ expect(
+ actionsReferenceBalance([
+ { options: { template: 'Balance: {{balance}}' } },
+ ]),
+ ).toBe(true);
+ expect(
+ actionsReferenceBalance([
+ { options: { template: '{{add balance amount}}' } },
+ ]),
+ ).toBe(true);
+ expect(
+ actionsReferenceBalance([{ options: { formula: '=balance * 2' } }]),
+ ).toBe(true);
+
+ expect(actionsReferenceBalance([])).toBe(false);
+ expect(actionsReferenceBalance([{}])).toBe(false);
+ expect(
+ actionsReferenceBalance([{ options: { template: 'well balanced' } }]),
+ ).toBe(false);
+ expect(
+ actionsReferenceBalance([
+ { options: { formula: '=BALANCE_OF("Savings") * 2' } },
+ ]),
+ ).toBe(false);
+ });
+
+ test('a template that uses the balance still sees the running balance', async () => {
+ await loadRules();
+ const account = await db.insertAccount({ name: 'bank' });
+ await db.insertTransaction({
+ account,
+ date: '2020-01-01',
+ amount: 1000,
+ });
+ await db.insertTransaction({
+ account,
+ date: '2020-01-02',
+ amount: 2500,
+ });
+
+ await insertRule({
+ stage: null,
+ conditionsOp: 'and',
+ conditions: [
+ { op: 'contains', field: 'imported_payee', value: 'kroger' },
+ ],
+ actions: [
+ {
+ op: 'set',
+ field: 'notes',
+ value: '',
+ options: { template: 'Balance: {{balance}}' },
+ },
+ ],
+ });
+
+ const queries = countRunningBalanceQueries();
+ const transaction = await runRules({
+ imported_payee: 'kroger',
+ account,
+ date: '2020-01-03',
+ amount: 50,
+ });
+
+ expect(transaction.notes).toBe('Balance: 3500');
+ expect(transaction).not.toHaveProperty('balance');
+ expect(queries()).toBe(1);
+ });
+
+ test('a rule that uses the balance but does not match runs no query', async () => {
+ await loadRules();
+ const account = await db.insertAccount({ name: 'bank' });
+ await db.insertTransaction({
+ account,
+ date: '2020-01-01',
+ amount: 1000,
+ });
+
+ await insertRule({
+ stage: null,
+ conditionsOp: 'and',
+ conditions: [
+ { op: 'contains', field: 'imported_payee', value: 'walmart' },
+ ],
+ actions: [
+ {
+ op: 'set',
+ field: 'notes',
+ value: '',
+ options: { template: 'Balance: {{balance}}' },
+ },
+ ],
+ });
+
+ const queries = countRunningBalanceQueries();
+ const transaction = await runRules({
+ imported_payee: 'kroger',
+ account,
+ date: '2020-01-03',
+ amount: 50,
+ });
+
+ expect(transaction.notes).toBeUndefined();
+ expect(queries()).toBe(0);
+ });
+
+ test('rules that do not use the balance still apply normally', async () => {
+ await loadRules();
+ const account = await db.insertAccount({ name: 'bank' });
+ await db.insertTransaction({
+ account,
+ date: '2020-01-01',
+ amount: 1000,
+ });
+
+ await insertRule({
+ stage: null,
+ conditionsOp: 'and',
+ conditions: [
+ { op: 'contains', field: 'imported_payee', value: 'kroger' },
+ ],
+ actions: [{ op: 'set', field: 'notes', value: 'plain' }],
+ });
+
+ const queries = countRunningBalanceQueries();
+ const transaction = await runRules({
+ imported_payee: 'kroger',
+ account,
+ date: '2020-01-03',
+ amount: 50,
+ });
+
+ expect(transaction.notes).toBe('plain');
+ expect(transaction).not.toHaveProperty('balance');
+ expect(queries()).toBe(0);
+ });
+});
diff --git a/packages/loot-core/src/server/transactions/transaction-rules.ts b/packages/loot-core/src/server/transactions/transaction-rules.ts
index f4efc50a60..5e3081dd91 100644
--- a/packages/loot-core/src/server/transactions/transaction-rules.ts
+++ b/packages/loot-core/src/server/transactions/transaction-rules.ts
@@ -333,9 +333,33 @@ export async function runRules(
accountsMap = accounts;
}
- let finalTrans = await prepareTransactionForRules({ ...trans }, accountsMap);
+ const rules = rankRules(
+ fastSetMerge(
+ firstcharIndexer.getApplicableRules(trans),
+ payeeIndexer.getApplicableRules(trans),
+ ),
+ );
+
+ let finalTrans = await prepareTransactionForRules({ ...trans }, accountsMap, {
+ includeBalance: false,
+ });
let lastCategoryIdForGroup: string | null = finalTrans.category ?? null;
+ // The running balance is a query over every earlier transaction in
+ // the account, so fetch it at most once, and only for a rule that is
+ // about to run and actually reads it. `rules` is only the candidates
+ // from the indexers, so checking their actions up front isn't enough.
+ let hasBalance = false;
+ async function ensureBalanceFor(rule: { actions: Action[] }) {
+ if (!hasBalance && trans.account && actionsReferenceBalance(rule.actions)) {
+ finalTrans.balance = await getRunningBalanceBeforeTransaction(
+ trans,
+ trans.account,
+ );
+ hasBalance = true;
+ }
+ }
+
let scheduleRuleID = '';
// Check if a schedule is attached to this transaction and if so get the rule ID attached to that schedule.
if (trans.schedule != null) {
@@ -348,13 +372,6 @@ export async function runRules(
const RuleIdsLinkedToSchedules =
await getAllRuleIdsFromSchedules(scheduleRuleID);
- const rules = rankRules(
- fastSetMerge(
- firstcharIndexer.getApplicableRules(trans),
- payeeIndexer.getApplicableRules(trans),
- ),
- );
-
const formulaStrings = rules.flatMap(rule =>
collectFormulasFromActions(rule.actions),
);
@@ -369,6 +386,7 @@ export async function runRules(
if (scheduleRuleID !== '') {
if (rules[i].id === scheduleRuleID) {
// bypass condition checking to run the rule even if the transaction date falls outside of the schedule's date range.
+ await ensureBalanceFor(rules[i]);
const changes = rules[i].execActions(finalTrans);
finalTrans = Object.assign({}, finalTrans, changes);
await resolvePayeeNameForRules(finalTrans);
@@ -381,7 +399,14 @@ export async function runRules(
continue;
} else {
// if a rule is not linked to a schedule, run it.
- finalTrans = rules[i].apply(finalTrans);
+ if (rules[i].evalConditions(finalTrans)) {
+ await ensureBalanceFor(rules[i]);
+ finalTrans = Object.assign(
+ {},
+ finalTrans,
+ rules[i].execActions(finalTrans),
+ );
+ }
await resolvePayeeNameForRules(finalTrans);
lastCategoryIdForGroup = await refreshCategoryGroupIfChanged(
finalTrans,
@@ -390,7 +415,14 @@ export async function runRules(
}
} else {
// if there is no scheduleRuleID then just run all rules.
- finalTrans = rules[i].apply(finalTrans);
+ if (rules[i].evalConditions(finalTrans)) {
+ await ensureBalanceFor(rules[i]);
+ finalTrans = Object.assign(
+ {},
+ finalTrans,
+ rules[i].execActions(finalTrans),
+ );
+ }
await resolvePayeeNameForRules(finalTrans);
lastCategoryIdForGroup = await refreshCategoryGroupIfChanged(
finalTrans,
@@ -755,9 +787,10 @@ export async function applyActions(
const accounts: db.DbAccount[] = await db.getAccounts();
const accountsMap = new Map(accounts.map(account => [account.id, account]));
+ const includeBalance = actionsReferenceBalance(parsedActions);
const transactionsForRules = await Promise.all(
- transactions.map(transactions =>
- prepareTransactionForRules(transactions, accountsMap),
+ transactions.map(transaction =>
+ prepareTransactionForRules(transaction, accountsMap, { includeBalance }),
),
);
@@ -1084,9 +1117,28 @@ export async function prefetchBalanceOfForTransaction(
return map;
}
+const BALANCE_VARIABLE = /\bbalance\b/i;
+
+// Whether any of these actions can read the transaction's running
+// balance: Handlebars templates via `{{balance}}` and formulas via the
+// `balance` variable. `BALANCE_OF(...)` is a different, prefetched value
+// and does not count.
+export function actionsReferenceBalance(
+ actions: Array<{ options?: { template?: string; formula?: string } }>,
+): boolean {
+ return actions.some(action => {
+ const { template, formula } = action.options ?? {};
+ return (
+ (typeof template === 'string' && BALANCE_VARIABLE.test(template)) ||
+ (typeof formula === 'string' && BALANCE_VARIABLE.test(formula))
+ );
+ });
+}
+
export async function prepareTransactionForRules(
trans: TransactionEntity,
accounts: Map<string, db.DbAccount> | null = null,
+ { includeBalance = true }: { includeBalance?: boolean } = {},
): Promise<TransactionForRules> {
const r: TransactionForRules = { ...trans };
if (trans.payee) {
@@ -1107,7 +1159,12 @@ export async function prepareTransactionForRules(
r._account_name = r._account?.name || '';
}
- r.balance = await getRunningBalanceBeforeTransaction(trans, trans.account);
+ if (includeBalance) {
+ r.balance = await getRunningBalanceBeforeTransaction(
+ trans,
+ trans.account,
+ );
+ }
}
if (trans.category) {
diff --git a/upcoming-release-notes/skip-running-balance-query.md b/upcoming-release-notes/skip-running-balance-query.md
new file mode 100644
index 0000000000..ac6dfd8453
--- /dev/null
+++ b/upcoming-release-notes/skip-running-balance-query.md
@@ -0,0 +1,6 @@
+---
+category: Enhancements
+authors: [MikesGlitch]
+---
+
+Speed up editing transactions by only calculating the running balance when a rule uses it