A local-first personal finance app
:zap: [AI] Speed up resetting the budget cache (#9143)
youngcw137922fd6643
diff --git a/packages/loot-core/src/server/budget/base.ts b/packages/loot-core/src/server/budget/base.tsindex 4e10a6f8c1..b2215dfef6 100644--- a/packages/loot-core/src/server/budget/base.ts+++ b/packages/loot-core/src/server/budget/base.ts@@ -38,6 +38,27 @@ export function getBudgetRange(start: string, end: string) {return { start, end, range: monthUtils.rangeInclusive(start, end) };}+// The day range spanning the given months. They aren't guaranteed to be+// sorted, so find the first and last explicitly ('YYYY-MM' strings compare+// chronologically).+function getMonthSpanBounds(months: string[]) {+ let firstMonth = months[0];+ let lastMonth = months[0];+ for (const month of months) {+ if (month < firstMonth) {+ firstMonth = month;+ }+ if (month > lastMonth) {+ lastMonth = month;+ }+ }++ return {+ rangeStart: monthUtils.bounds(firstMonth).start,+ rangeEnd: monthUtils.bounds(lastMonth).end,+ };+}+// Computes the spend total for every category in every month within the// given day range using a single grouped query. This is used to seed the// `sum-amount` cells on a cold build so we avoid running one@@ -70,6 +91,45 @@ function getSumAmountsByMonth(return sums;}+// Reseeds the `sum-amount` cells for the given months from the same single+// grouped query a cold build uses, and returns the names of the cells that+// were seeded. Resetting the budget cache recomputes every cell, which would+// otherwise run one `SELECT SUM(amount)` per category per month; the caller+// skips recomputing these and caches them directly instead.+export async function reseedSumAmounts(months: string[]): Promise<string[]> {+ if (months.length === 0) {+ return [];+ }++ const { rangeStart, rangeEnd } = getMonthSpanBounds(months);+ const sums = getSumAmountsByMonth(rangeStart, rangeEnd);++ const { data: groups }: { data: CategoryGroupEntity[] } = await aqlQuery(+ q('category_groups').select('*'),+ );+ const categories = groups.flatMap(group => group.categories);++ const seededCells: string[] = [];+ for (const month of months) {+ const sheetName = monthUtils.sheetForMonth(month);+ const dbMonth = parseInt(month.replace('-', ''));++ for (const cat of categories) {+ const name = resolveName(sheetName, `sum-amount-${cat.id}`);+ // A cell the spreadsheet doesn't have stays out of the seed set and is+ // left to be recomputed normally.+ if (!sheet.get().hasCell(name)) {+ continue;+ }++ sheet.get().load(name, sums.get(`${dbMonth}-${cat.id}`) || 0);+ seededCells.push(name);+ }+ }++ return seededCells;+}+export function createCategory(cat, sheetName, prevSheetName, start, end) {sheet.get().createDynamic(sheetName, 'sum-amount-' + cat.id, {initialValue: 0,@@ -292,22 +352,8 @@ export async function createBudget(months) {let sumAmounts: Map<string, number> | null = null;const getSumAmounts = () => {if (!sumAmounts) {- // `monthsToCreate` isn't guaranteed to be sorted, so find the span- // explicitly ('YYYY-MM' strings compare chronologically).- let firstMonth = monthsToCreate[0];- let lastMonth = monthsToCreate[0];- for (const month of monthsToCreate) {- if (month < firstMonth) {- firstMonth = month;- }- if (month > lastMonth) {- lastMonth = month;- }- }- sumAmounts = getSumAmountsByMonth(- monthUtils.bounds(firstMonth).start,- monthUtils.bounds(lastMonth).end,- );+ const { rangeStart, rangeEnd } = getMonthSpanBounds(monthsToCreate);+ sumAmounts = getSumAmountsByMonth(rangeStart, rangeEnd);}return sumAmounts;};diff --git a/packages/loot-core/src/server/budgetfiles/app.ts b/packages/loot-core/src/server/budgetfiles/app.tsindex fb68209e42..d0ae58d637 100644--- a/packages/loot-core/src/server/budgetfiles/app.ts+++ b/packages/loot-core/src/server/budgetfiles/app.ts@@ -146,10 +146,44 @@ async function getRemoteFiles() {}async function resetBudgetCache() {- // Recomputing everything will update the cache- await sheet.loadUserBudgets(db);- sheet.get().recomputeAll();- await sheet.waitOnSpreadsheet();+ const spreadsheet = sheet.get();++ // Hold the cache dirty for the whole reset so an interrupted one doesn't+ // leave a half-written cache looking clean.+ spreadsheet.startCacheBarrier();++ try {+ let seededCells: string[] = [];++ // One transaction around the whole reset. `loadUserBudgets` would+ // otherwise close its own, queueing a computation pass over everything+ // downstream of the budget inputs that `recomputeAll` then repeats.+ spreadsheet.startTransaction();+ try {+ await sheet.loadUserBudgets(db);++ // Seeding the spend totals from a single grouped query keeps the+ // recompute from running a `SELECT SUM(amount)` per category per month.+ seededCells = await budget.reseedSumAmounts([+ ...spreadsheet.meta().createdMonths,+ ]);++ // Recomputing everything will update the cache+ spreadsheet.recomputeAll(new Set(seededCells));+ } finally {+ // A transaction left open would stop the spreadsheet from queueing any+ // further computations, and make every later `waitOnSpreadsheet` throw.+ spreadsheet.endTransaction();+ }++ // The seeded cells were loaded rather than computed, so they aren't part+ // of the computation queue that normally gets cached.+ spreadsheet.saveCachedCells(seededCells);++ await sheet.waitOnSpreadsheet();+ } finally {+ spreadsheet.endCacheBarrier();+ }}async function uploadBudget({ id }: { id?: Budget['id'] } = {}): Promise<{diff --git a/packages/loot-core/src/server/main.test.ts b/packages/loot-core/src/server/main.test.tsindex 6145851e23..c94c5c8432 100644--- a/packages/loot-core/src/server/main.test.ts+++ b/packages/loot-core/src/server/main.test.ts@@ -281,6 +281,77 @@ describe('Budget', () => {changed.sort((a, b) => (a > b ? 1 : a < b ? -1 : 0)),).toMatchSnapshot();});++ test('resetting the budget cache computes each cell at most once', async () => {+ const spreadsheet = await sheet.loadSpreadsheet(db);++ let catId;+ await runMutator(async () => {+ await db.insertCategoryGroup({+ id: 'incomeGroup',+ name: 'incomeGroup',+ is_income: 1,+ });+ await db.insertCategoryGroup({ id: 'group1', name: 'group1' });+ catId = await db.insertCategory({ name: 'foo', cat_group: 'group1' });+ await db.insertCategory({ name: 'bar', cat_group: 'group1' });+ });++ // Creates the budget months, which is what populates `createdMonths`+ await runHandler(handlers['get-budget-bounds']);++ db.runQuery("INSERT INTO accounts (id, name) VALUES ('one', 'boa')");+ await runHandler(handlers['transaction-add'], {+ id: uuidv4(),+ date: '2017-01-05',+ amount: -3400,+ account: 'one',+ category: catId,+ });+ await runHandler(handlers['budget/budget-amount'], {+ month: '2017-01',+ category: catId,+ amount: 5000,+ });++ await sheet.waitOnSpreadsheet();++ const sheetName = monthUtils.sheetForMonth('2017-01');+ const valuesOf = () =>+ new Map(+ [...spreadsheet.getNodes()].map(([name, node]) => [name, node.value]),+ );+ const before = valuesOf();+ expect(before.get(`${sheetName}!sum-amount-${catId}`)).toBe(-3400);++ let computed = [];+ const remove = spreadsheet.addEventListener('change', ({ names }) => {+ computed = computed.concat(names);+ });+ await runHandler(handlers['reset-budget-cache']);+ remove();++ // Resetting the cache recomputes values, it doesn't change them+ expect(valuesOf()).toEqual(before);++ // Spend totals are seeded from a single grouped query rather than each+ // running its own `SELECT SUM(amount)`+ expect(computed.filter(name => name.includes('!sum-amount-'))).toEqual([]);++ // Everything downstream of the budget inputs used to be computed twice:+ // once when `loadUserBudgets` closed its own transaction, and again for+ // `recomputeAll`+ const seen = new Set();+ const computedTwice = computed.filter(name => {+ const duplicate = seen.has(name);+ seen.add(name);+ return duplicate;+ });+ expect(computedTwice).toEqual([]);++ // Everything else really was recomputed+ expect(computed).toContain(`${sheetName}!to-budget`);+ });});describe('Categories', () => {diff --git a/packages/loot-core/src/server/sheet.ts b/packages/loot-core/src/server/sheet.tsindex cf539503b1..011cb180df 100644--- a/packages/loot-core/src/server/sheet.ts+++ b/packages/loot-core/src/server/sheet.ts@@ -218,32 +218,39 @@ export async function loadUserBudgets(db: typeof DbModule): Promise<void> {sheet.startTransaction();- // Load all the budget amounts and carryover values- for (const budget of budgets) {- if (budget.month && budget.category) {- const sheetName = `budget${budget.month}`;- sheet.set(`${sheetName}!budget-${budget.category}`, budget.amount);- sheet.set(- `${sheetName}!carryover-${budget.category}`,- budget.carryover === 1 ? true : false,- );- sheet.set(`${sheetName}!goal-${budget.category}`, budget.goal);- sheet.set(`${sheetName}!long-goal-${budget.category}`, budget.long_goal);+ try {+ // Load all the budget amounts and carryover values+ for (const budget of budgets) {+ if (budget.month && budget.category) {+ const sheetName = `budget${budget.month}`;+ sheet.set(`${sheetName}!budget-${budget.category}`, budget.amount);+ sheet.set(+ `${sheetName}!carryover-${budget.category}`,+ budget.carryover === 1 ? true : false,+ );+ sheet.set(`${sheetName}!goal-${budget.category}`, budget.goal);+ sheet.set(+ `${sheetName}!long-goal-${budget.category}`,+ budget.long_goal,+ );+ }}- }- // For zero-based budgets, load the buffered amounts- if (budgetType !== 'tracking') {- const budgetMonths = await db.all<DbZeroBudgetMonth>(- 'SELECT * FROM zero_budget_months',- );- for (const budgetMonth of budgetMonths) {- const sheetName = sheetForMonth(budgetMonth.id);- sheet.set(`${sheetName}!buffered`, budgetMonth.buffered);+ // For zero-based budgets, load the buffered amounts+ if (budgetType !== 'tracking') {+ const budgetMonths = await db.all<DbZeroBudgetMonth>(+ 'SELECT * FROM zero_budget_months',+ );+ for (const budgetMonth of budgetMonths) {+ const sheetName = sheetForMonth(budgetMonth.id);+ sheet.set(`${sheetName}!buffered`, budgetMonth.buffered);+ }}+ } finally {+ // The query above runs inside the transaction, so a failure would+ // otherwise leave it open and wedge the spreadsheet for the session.+ sheet.endTransaction();}-- sheet.endTransaction();}export function getCell(sheet: string, name: string) {diff --git a/packages/loot-core/src/server/spreadsheet/spreadsheet.ts b/packages/loot-core/src/server/spreadsheet/spreadsheet.tsindex e6a9389cea..f90018c477 100644--- a/packages/loot-core/src/server/spreadsheet/spreadsheet.ts+++ b/packages/loot-core/src/server/spreadsheet/spreadsheet.ts@@ -348,10 +348,12 @@ export class Spreadsheet {});}- recomputeAll(): void {- // Recompute everything!+ // Recompute everything, except any cells in `skip` — those are expected to+ // have already been given their final value with `load`.+ recomputeAll(skip?: ReadonlySet<string>): void {this.transaction(() => {- this.dirtyCells = [...this.nodes.keys()];+ const names = [...this.nodes.keys()];+ this.dirtyCells = skip ? names.filter(name => !skip.has(name)) : names;});}diff --git a/upcoming-release-notes/faster-reset-budget-cache.md b/upcoming-release-notes/faster-reset-budget-cache.mdnew file mode 100644index 0000000000..819ed97425--- /dev/null+++ b/upcoming-release-notes/faster-reset-budget-cache.md@@ -0,0 +1,6 @@+---+category: Enhancements+authors: [youngcw]+---++Make resetting the budget cache much faster on large budgets