Skip to content

A local-first personal finance app

[AI] Fix schedule service shutdown race (#8791)

cakenie82ea4dc3594
diff --git a/packages/loot-core/src/server/budgetfiles/app.ts b/packages/loot-core/src/server/budgetfiles/app.ts
index d8ee90edb0..e2e573a471 100644
--- a/packages/loot-core/src/server/budgetfiles/app.ts
+++ b/packages/loot-core/src/server/budgetfiles/app.ts
@@ -23,7 +23,9 @@ import * as sheet from '#server/sheet';
import {
clearFullSyncTimeout,
initialFullSync,
+ resumeFullSync,
setSyncingMode,
+ stopFullSync,
} from '#server/sync';
import * as syncMigrations from '#server/sync/migrate';
import { replayPendingMessages } from '#server/sync/replay';
@@ -298,6 +300,15 @@ async function createDemoBudget() {
}
async function closeBudget() {
+ await stopFullSync();
+ try {
+ return await unloadBudget();
+ } finally {
+ resumeFullSync();
+ }
+}
+
+async function unloadBudget() {
captureBreadcrumb({ message: 'Closing budget' });
resetFormulaPreferencesCache();
diff --git a/packages/loot-core/src/server/main.test.ts b/packages/loot-core/src/server/main.test.ts
index c94c5c8432..cccf43d314 100644
--- a/packages/loot-core/src/server/main.test.ts
+++ b/packages/loot-core/src/server/main.test.ts
@@ -5,20 +5,24 @@ import { v4 as uuidv4 } from 'uuid';
import { expectSnapshotWithDiffer } from '#mocks/util';
import * as connection from '#platform/server/connection';
import * as fs from '#platform/server/fs';
+import { logger } from '#platform/server/log';
import * as monthUtils from '#shared/months';
import * as budgetActions from './budget/actions';
import * as budget from './budget/base';
import * as db from './db';
-import { handlers } from './main';
+import { SyncError } from './errors';
+import { handlers, lib } from './main';
import {
disableGlobalMutations,
enableGlobalMutations,
runHandler,
runMutator,
} from './mutators';
+import * as post from './post';
import * as prefs from './prefs';
import * as sheet from './sheet';
+import { batchMessages, scheduleFullSync, setSyncingMode } from './sync';
vi.mock('./post');
@@ -514,3 +518,126 @@ describe('Categories', () => {
expect(sheet.getCellValue(nextSheetName, 'to-budget')).toBe(toBudget);
});
});
+
+describe('Budget shutdown', () => {
+ beforeEach(async () => {
+ vi.useFakeTimers();
+ await prefs.loadPrefs();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllEnvs();
+ vi.restoreAllMocks();
+ fs._setDocumentDir(null);
+ setSyncingMode('disabled');
+ });
+
+ function deferred() {
+ let resolve: () => void;
+ const promise = new Promise<void>(resolvePromise => {
+ resolve = resolvePromise;
+ });
+ return { promise, resolve };
+ }
+
+ test('finishes sync and schedule persistence before unloading, without admitting new syncs', async () => {
+ await prefs.savePrefs({ groupId: 'group' });
+ fs._setDocumentDir(__dirname);
+ setSyncingMode('enabled');
+ const syncGate = deferred();
+ const syncStarted = deferred();
+ const writeGate = deferred();
+ const writeStarted = deferred();
+ const sheetGate = deferred();
+ const sheetStarted = deferred();
+ const postBinary = post.postBinary;
+ const postSpy = vi
+ .spyOn(post, 'postBinary')
+ .mockImplementationOnce(async (...args) => {
+ syncStarted.resolve();
+ await syncGate.promise;
+ return postBinary(...args);
+ });
+ let savedPrefs;
+ vi.spyOn(fs, 'writeFile').mockImplementation(async (_path, contents) => {
+ const metadata = JSON.parse(contents);
+ if (metadata.lastScheduleRun) {
+ writeStarted.resolve();
+ await writeGate.promise;
+ savedPrefs = metadata;
+ }
+ return true;
+ });
+ await lib.send('app-focused');
+ await syncStarted.promise;
+ const closePromise = lib.send('close-budget');
+
+ try {
+ await vi.runAllTimersAsync();
+ expect(db.getDatabase()).not.toBeNull();
+
+ vi.stubEnv('NODE_ENV', 'production');
+ syncGate.resolve();
+ await writeStarted.promise;
+ await vi.runAllTimersAsync();
+ expect(db.getDatabase()).not.toBeNull();
+
+ vi.spyOn(sheet, 'waitOnSpreadsheet').mockImplementationOnce(async () => {
+ sheetStarted.resolve();
+ await sheetGate.promise;
+ });
+ writeGate.resolve();
+ await sheetStarted.promise;
+ await lib.send('app-focused');
+ void scheduleFullSync();
+ await vi.runAllTimersAsync();
+ expect(postSpy).toHaveBeenCalledTimes(1);
+
+ sheetGate.resolve();
+ await vi.runAllTimersAsync();
+ await expect(closePromise).resolves.toBe('ok');
+ expect(savedPrefs).toMatchObject({
+ groupId: 'group',
+ lastScheduleRun: monthUtils.currentDay(),
+ });
+ expect(db.getDatabase()).toBeNull();
+ } finally {
+ syncGate.resolve();
+ writeGate.resolve();
+ sheetGate.resolve();
+ await vi.runAllTimersAsync();
+ await closePromise;
+ }
+ });
+
+ test('drains schedule work queued by a rejecting mutator during shutdown', async () => {
+ const errorLog = vi.spyOn(logger, 'error');
+ const gate = deferred();
+ const rejectingPromise = runMutator(() =>
+ batchMessages(async () => {
+ await gate.promise;
+ throw new SyncError('invalid-schema');
+ }),
+ );
+ const closePromise = lib.send('close-budget');
+
+ try {
+ await vi.runAllTimersAsync();
+ gate.resolve();
+ await expect(rejectingPromise).rejects.toThrow('invalid-schema');
+ await vi.runAllTimersAsync();
+ await expect(closePromise).resolves.toBe('ok');
+ expect(db.getDatabase()).toBeNull();
+ await runMutator(async () => undefined);
+ expect(errorLog).not.toHaveBeenCalledWith(
+ 'Failed to run schedule service after sync',
+ expect.anything(),
+ );
+ } finally {
+ gate.resolve();
+ await vi.runAllTimersAsync();
+ await Promise.allSettled([rejectingPromise, closePromise]);
+ }
+ });
+});
diff --git a/packages/loot-core/src/server/mutators.ts b/packages/loot-core/src/server/mutators.ts
index ff0400ea63..c154ea1370 100644
--- a/packages/loot-core/src/server/mutators.ts
+++ b/packages/loot-core/src/server/mutators.ts
@@ -4,6 +4,7 @@ import { sequential } from '#shared/async';
import type { HandlerFunctions, Handlers } from '#types/handlers';
const runningMethods = new Set();
+const runningMutators = new Set<Promise<unknown>>();
let currentContext = null;
const mutatingMethods = new WeakMap();
@@ -95,7 +96,26 @@ function _runMutator<T extends () => Promise<unknown>>(
}) as Promise<Awaited<ReturnType<T>>>;
}
// Type cast needed as TS looses types over nested generic returns
-export const runMutator = sequential(_runMutator) as typeof _runMutator;
+const runMutatorSequential = sequential(_runMutator) as typeof _runMutator;
+
+export function runMutator<T extends () => Promise<unknown>>(
+ func: T,
+ initialContext = {},
+): Promise<Awaited<ReturnType<T>>> {
+ const promise = runMutatorSequential(func, initialContext);
+ runningMutators.add(promise);
+ const remove = () => runningMutators.delete(promise);
+ void promise.then(remove, remove);
+ return promise;
+}
+
+export async function flushRunningMutators(): Promise<void> {
+ // A settling mutator can queue another, e.g. a sync error queues a
+ // schedule run, so wait until none are left
+ while (runningMutators.size > 0) {
+ await Promise.allSettled([...runningMutators]);
+ }
+}
export function withMutatorContext<T>(
context: { undoListening: boolean; undoTag?: unknown },
diff --git a/packages/loot-core/src/server/schedules/app.ts b/packages/loot-core/src/server/schedules/app.ts
index d83760866e..e8304ee524 100644
--- a/packages/loot-core/src/server/schedules/app.ts
+++ b/packages/loot-core/src/server/schedules/app.ts
@@ -868,15 +868,25 @@ app.events.on('sync', ({ type }) => {
return;
}
- const { lastScheduleRun } = prefs.getPrefs();
- if (lastScheduleRun !== currentDay()) {
- void runMutator(() => advanceSchedulesService(type === 'success'));
-
- // Only mark the day as done when sync succeeded, so that
- // schedule auto-posting is retried on subsequent successful syncs
- if (type === 'success') {
- void prefs.savePrefs({ lastScheduleRun: currentDay() });
- }
+ if (prefs.getPrefs().lastScheduleRun !== currentDay()) {
+ void runMutator(async () => {
+ const runDay = currentDay();
+ const { lastScheduleRun } = prefs.getPrefs();
+ // A run queued by an earlier sync may have already finished today
+ if (lastScheduleRun === runDay) {
+ return;
+ }
+
+ await advanceSchedulesService(type === 'success');
+
+ // Only mark the day as done when sync succeeded, so that
+ // schedule auto-posting is retried on subsequent successful syncs
+ if (type === 'success') {
+ await prefs.savePrefs({ lastScheduleRun: runDay });
+ }
+ }).catch(error => {
+ logger.error('Failed to run schedule service after sync', error);
+ });
}
}
});
diff --git a/packages/loot-core/src/server/sync/index.ts b/packages/loot-core/src/server/sync/index.ts
index 2517a6b559..70768d82f9 100644
--- a/packages/loot-core/src/server/sync/index.ts
+++ b/packages/loot-core/src/server/sync/index.ts
@@ -18,14 +18,14 @@ import {
import * as db from '#server/db';
import { PostError, SyncError } from '#server/errors';
import { app } from '#server/main-app';
-import { runMutator } from '#server/mutators';
+import { flushRunningMutators, runMutator } from '#server/mutators';
import { postBinary } from '#server/post';
import * as prefs from '#server/prefs';
import { getServer } from '#server/server-config';
import * as sheet from '#server/sheet';
import { resolveName } from '#server/spreadsheet/util';
import * as undo from '#server/undo';
-import { once, sequential } from '#shared/async';
+import { sequential } from '#shared/async';
import { isMissingSchemaError } from '#shared/errors';
import { getIn, setIn } from '#shared/util';
import type { MetadataPrefs } from '#types/prefs';
@@ -825,10 +825,36 @@ export async function initialFullSync(): Promise<{
return {};
}
-export const fullSync = once(async function (): Promise<
+type FullSyncResult =
| { messages: Message[] }
- | { error: { message: string; reason: string; meta: unknown } }
-> {
+ | { error: { message: string; reason: string; meta: unknown } };
+
+let runningFullSync: Promise<FullSyncResult> | null = null;
+let isFullSyncStopped = false;
+export function fullSync(): Promise<FullSyncResult> {
+ if (isFullSyncStopped) {
+ return Promise.resolve({ messages: [] });
+ }
+ runningFullSync ??= runFullSync().finally(() => {
+ runningFullSync = null;
+ });
+ return runningFullSync;
+}
+
+// Stops new syncs, then waits for the running sync and all queued mutators so
+// the budget can be closed.
+export async function stopFullSync(): Promise<void> {
+ isFullSyncStopped = true;
+ clearFullSyncTimeout();
+ await Promise.allSettled([runningFullSync]);
+ await flushRunningMutators();
+}
+
+export function resumeFullSync(): void {
+ isFullSyncStopped = false;
+}
+
+async function runFullSync(): Promise<FullSyncResult> {
app.events.emit('sync', { type: 'start' });
let messages;
@@ -899,7 +925,7 @@ export const fullSync = once(async function (): Promise<
syncDisabled: checkSyncingMode('disabled'),
});
return { messages };
-});
+}
async function _fullSync(
sinceTimestamp: string,
diff --git a/upcoming-release-notes/8791.md b/upcoming-release-notes/8791.md
new file mode 100644
index 0000000000..095f2b7e92
--- /dev/null
+++ b/upcoming-release-notes/8791.md
@@ -0,0 +1,6 @@
+---
+category: Bugfix
+authors: [cakeni]
+---
+
+Fix an occasional API crash on shutdown while schedules are being processed