Skip to content

A local-first personal finance app

[AI] api: support custom dataDir (#8397)

Matiss Janis Aboltinsdcff910de67e
diff --git a/packages/api/browser-worker.ts b/packages/api/browser-worker.ts
index 5953811231..0653fc3f15 100644
--- a/packages/api/browser-worker.ts
+++ b/packages/api/browser-worker.ts
@@ -67,7 +67,8 @@ self.fetch = function patchedFetch(
'api-browser/init'
] = async function (args?: unknown) {
const { config } = (args ?? {}) as { config?: InitConfig };
- await init(config ?? {});
+ // There is no meaningful `process.cwd()` to fall back to in a worker
+ await init({ dataDir: '/documents', ...config });
};
connection.init(self, handlers);
diff --git a/packages/api/e2e/browser.test.ts b/packages/api/e2e/browser.test.ts
index 7370c9abae..b637592579 100644
--- a/packages/api/e2e/browser.test.ts
+++ b/packages/api/e2e/browser.test.ts
@@ -92,3 +92,38 @@ test('boots, imports a budget, reads it back, and persists', async ({
});
expect(budgetCount).toBe(1);
});
+
+test('supports a custom dataDir and persists it', async ({ page }) => {
+ await page.goto('/e2e/harness.html');
+
+ const result = await page.evaluate(async () => {
+ const api = await window.apiReady;
+ await api.init({ dataDir: '/budget' });
+
+ await api.runImport('e2e-custom-dir-budget', async () => {
+ const accountId = await api.createAccount({ name: 'Savings' }, 0);
+ await api.addTransactions(accountId, [
+ { date: '2024-02-01', amount: 12345, notes: 'deposit' },
+ ]);
+ });
+
+ const accounts = await api.getAccounts();
+ return { accountNames: accounts.map(a => a.name) };
+ });
+ expect(result.accountNames).toEqual(['Savings']);
+
+ // Reload the page: the budget must survive in IndexedDB.
+ await page.evaluate(async () => {
+ const api = await window.apiReady;
+ await api.shutdown();
+ });
+ await page.reload();
+ const budgetCount = await page.evaluate(async () => {
+ const api = await window.apiReady;
+ await api.init({ dataDir: '/budget' });
+ const budgets = await api.getBudgets();
+ await api.shutdown();
+ return budgets.length;
+ });
+ expect(budgetCount).toBe(1);
+});
diff --git a/packages/docs/docs/api/index.md b/packages/docs/docs/api/index.md
index 4fa021a956..8dea344732 100644
--- a/packages/docs/docs/api/index.md
+++ b/packages/docs/docs/api/index.md
@@ -94,9 +94,6 @@ The package also ships a browser build. When you bundle your web app with a mode
import * as api from '@actual-app/api';
await api.init({
- // In the browser, budget data is stored in IndexedDB. This is a path
- // inside that virtual file system.
- dataDir: '/documents',
serverURL: 'https://your-server.example.com',
password: 'hunter2',
});
@@ -108,6 +105,8 @@ await api.shutdown();
Behind the scenes, `init` starts a Web Worker running the same budget engine the Actual web app uses, backed by SQLite compiled to WebAssembly. Your budget data is stored in the browser's IndexedDB and stays on the device.
+In the browser, `dataDir` is a path inside the worker's virtual file system rather than a folder on disk. It is optional: it defaults to `/documents`, and if you pass a custom path it is created automatically and persisted to IndexedDB just the same.
+
The browser build is fully self-contained: the Web Worker and its WebAssembly and data files are inlined into the package. There are no extra files to copy or serve, and no bundler configuration is required (no `optimizeDeps` tweaks, no worker or asset plugins). Importing the package and calling `init()` is all that's needed.
:::caution Cross-origin isolation is required
@@ -209,7 +208,7 @@ These are the public methods that you can use. The API also exports low-level fu
Call this before attempting to use any of the API methods. This will connect to the server using the provided password and load the budget data.
-`dataDir` defaults to the current working directory.
+`dataDir` defaults to the current working directory in Node.js, and to `/documents` in the [browser build](#using-the-api-in-a-browser).
If no `serverURL` is provided, no network connections will be made, and you'll only be able to access budget files already downloaded locally.
diff --git a/packages/loot-core/src/platform/server/connection/index.ts b/packages/loot-core/src/platform/server/connection/index.ts
index 57f9e6151e..2e87e3317b 100644
--- a/packages/loot-core/src/platform/server/connection/index.ts
+++ b/packages/loot-core/src/platform/server/connection/index.ts
@@ -6,6 +6,8 @@ import { isMutating, runHandler } from '#server/mutators';
import { postErrorReply } from './errors';
import type * as T from './index-types';
+import { safePost } from './shared';
+import type { Message } from './shared';
function getGlobalObject() {
const obj =
@@ -28,6 +30,10 @@ export const init: T.Init = function (serverChn, handlers) {
const serverChannel = serverChn as Window;
getGlobalObject().__globalServerChannel = serverChannel;
+ function post(msg: Message) {
+ safePost(m => serverChannel.postMessage(m), msg);
+ }
+
serverChannel.addEventListener(
'message',
e => {
@@ -50,7 +56,7 @@ export const init: T.Init = function (serverChn, handlers) {
if (handlers[name]) {
runHandler(handlers[name], args, { undoTag, name }).then(
result => {
- serverChannel.postMessage({
+ post({
type: 'reply',
id,
result: catchErrors ? { data: result, error: null } : result,
@@ -82,13 +88,13 @@ export const init: T.Init = function (serverChn, handlers) {
const unknownMethodError = APIError('Unknown server method: ' + name);
if (catchErrors) {
- serverChannel.postMessage({
+ post({
type: 'reply',
id,
result: { error: unknownMethodError, data: null },
});
} else {
- serverChannel.postMessage({
+ post({
type: 'error',
id,
error: unknownMethodError,
@@ -118,7 +124,7 @@ export const init: T.Init = function (serverChn, handlers) {
export const send: T.Send = function (name, args) {
const { __globalServerChannel } = getGlobalObject();
if (__globalServerChannel) {
- __globalServerChannel.postMessage({
+ safePost(msg => __globalServerChannel.postMessage(msg), {
type: 'push',
name,
args,
diff --git a/packages/loot-core/src/platform/server/connection/shared.test.ts b/packages/loot-core/src/platform/server/connection/shared.test.ts
new file mode 100644
index 0000000000..e6a406463f
--- /dev/null
+++ b/packages/loot-core/src/platform/server/connection/shared.test.ts
@@ -0,0 +1,93 @@
+import { safePost, serializeError } from './shared';
+
+describe('serializeError', () => {
+ test('serializes a plain Error', () => {
+ const error = new Error('boom');
+ const result = serializeError(error);
+ expect(result.name).toBe('Error');
+ expect(result.message).toBe('boom');
+ expect(typeof result.stack).toBe('string');
+ });
+
+ test('keeps primitive own properties like errnos', () => {
+ const error = new Error('FS error');
+ Object.assign(error, {
+ errno: 44,
+ code: 'ENOENT',
+ setErrno: () => undefined,
+ node: { some: 'object' },
+ });
+
+ const result = serializeError(error);
+ expect(result.errno).toBe(44);
+ expect(result.code).toBe('ENOENT');
+ expect(result.setErrno).toBeUndefined();
+ expect(result.node).toBeUndefined();
+ });
+
+ test('handles non-object errors', () => {
+ expect(serializeError('oops')).toEqual({ name: 'Error', message: 'oops' });
+ expect(serializeError(null)).toEqual({ name: 'Error', message: 'null' });
+ });
+});
+
+describe('safePost', () => {
+ test('posts the original message when it is cloneable', () => {
+ const posted: unknown[] = [];
+ const msg = { type: 'reply', id: '1', result: { data: 42, error: null } };
+
+ safePost(m => posted.push(m), msg);
+
+ expect(posted).toEqual([msg]);
+ });
+
+ test('falls back to a serialized message when cloning fails', () => {
+ const fsError: Record<string, unknown> = Object.create(Error.prototype);
+ fsError.message = 'FS error';
+ fsError.errno = 44;
+ fsError.setErrno = function (this: Record<string, unknown>, errno: number) {
+ this.errno = errno;
+ };
+
+ const msg = {
+ type: 'error',
+ id: '1',
+ error: { type: 'ServerError', message: 'FS error', cause: fsError },
+ };
+
+ const posted: unknown[] = [];
+ safePost(m => posted.push(structuredClone(m)), msg);
+
+ expect(posted).toHaveLength(1);
+ const fallback = posted[0] as typeof msg;
+ expect(fallback.type).toBe('error');
+ expect(fallback.id).toBe('1');
+ expect(fallback.error.type).toBe('ServerError');
+ expect(fallback.error.message).toBe('FS error');
+ const cause = fallback.error.cause as unknown as Record<string, unknown>;
+ expect(cause.message).toBe('FS error');
+ expect(cause.errno).toBe(44);
+ });
+
+ test('posts a generic error when the message cannot be serialized at all', () => {
+ const posted: unknown[] = [];
+ const msg = {
+ type: 'reply',
+ id: '1',
+ result: { fn: () => undefined, big: BigInt(1) },
+ };
+
+ safePost(m => posted.push(structuredClone(m)), msg);
+
+ expect(posted).toHaveLength(1);
+ const fallback = posted[0] as {
+ type: string;
+ id: string;
+ error: { type: string; message: string };
+ };
+ expect(fallback.type).toBe('error');
+ expect(fallback.id).toBe('1');
+ expect(fallback.error.type).toBe('ServerError');
+ expect(fallback.error.message).toContain('Failed to serialize');
+ });
+});
diff --git a/packages/loot-core/src/platform/server/connection/shared.ts b/packages/loot-core/src/platform/server/connection/shared.ts
new file mode 100644
index 0000000000..877ba7f3a4
--- /dev/null
+++ b/packages/loot-core/src/platform/server/connection/shared.ts
@@ -0,0 +1,73 @@
+// `postMessage` payloads must be structured-cloneable; errors from wasm code
+// are not always (e.g. Emscripten filesystem errors carry function properties)
+// and a failed clone masks the real error.
+
+export type Message = { type: string; id?: string } & Record<string, unknown>;
+
+export function serializeError(error: unknown): Record<string, unknown> {
+ if (error == null || typeof error !== 'object') {
+ return { name: 'Error', message: String(error) };
+ }
+
+ const errorObject = error as Record<string, unknown>;
+ const plain: Record<string, unknown> = {
+ name: typeof errorObject.name === 'string' ? errorObject.name : 'Error',
+ message:
+ typeof errorObject.message === 'string'
+ ? errorObject.message
+ : String(error),
+ };
+ if (typeof errorObject.stack === 'string') {
+ plain.stack = errorObject.stack;
+ }
+
+ for (const key of Object.keys(errorObject)) {
+ if (key === 'name' || key === 'message' || key === 'stack') {
+ continue;
+ }
+ const value = errorObject[key];
+ if (
+ value === null ||
+ typeof value === 'string' ||
+ typeof value === 'number' ||
+ typeof value === 'boolean'
+ ) {
+ plain[key] = value;
+ }
+ }
+ return plain;
+}
+
+function errorReplacer(_key: string, value: unknown): unknown {
+ if (value instanceof Error) {
+ return serializeError(value);
+ }
+ return value;
+}
+
+export function safePost(post: (msg: unknown) => void, msg: Message): void {
+ try {
+ post(msg);
+ } catch (postError) {
+ try {
+ post(JSON.parse(JSON.stringify(msg, errorReplacer)));
+ } catch {
+ try {
+ post({
+ type: 'error',
+ id: msg.id,
+ error: {
+ type: 'ServerError',
+ message:
+ 'Failed to serialize server response: ' +
+ (postError instanceof Error
+ ? postError.message
+ : String(postError)),
+ },
+ });
+ } catch {
+ // The channel itself is broken; there is no way to report anything
+ }
+ }
+ }
+}
diff --git a/packages/loot-core/src/platform/server/fs/index.test.ts b/packages/loot-core/src/platform/server/fs/index.test.ts
index 656c718556..d29b01e729 100644
--- a/packages/loot-core/src/platform/server/fs/index.test.ts
+++ b/packages/loot-core/src/platform/server/fs/index.test.ts
@@ -5,7 +5,15 @@ import { patchFetchForSqlJS } from '#mocks/util';
import * as idb from '#platform/server/indexeddb';
import * as sqlite from '#platform/server/sqlite';
-import { exists, init, join, pathToId, readFile, writeFile } from './index';
+import {
+ _setDocumentDir,
+ exists,
+ init,
+ join,
+ pathToId,
+ readFile,
+ writeFile,
+} from './index';
beforeAll(() => {
const baseURL = `${__dirname}/../../../../../../node_modules/@jlongster/sql.js/dist/`;
@@ -106,6 +114,57 @@ describe('web filesystem', () => {
'/blocked/' + pathToId('/documents/deep/nested/db.sqlite'),
);
});
+
+ test('files under a custom document dir are persisted', async () => {
+ await idb.openDatabase();
+ await sqlite.init();
+ await init();
+
+ _setDocumentDir('/budget');
+
+ expect(await exists('/budget')).toBe(true);
+
+ await writeFile('/budget/foo.txt', 'hello');
+ expect(await readFile('/budget/foo.txt')).toBe('hello');
+
+ const db = await idb.openDatabase();
+ const { store } = idb.getStore(db, 'files');
+ expect(await idb.get(store, '/budget/foo.txt')).toEqual({
+ filepath: '/budget/foo.txt',
+ contents: 'hello',
+ });
+
+ // sqlite files are symlinked into the blocked fs
+ await writeFile('/budget/db.sqlite', 'some junk');
+ expect(await readFile('/blocked/' + pathToId('/budget/db.sqlite'))).toBe(
+ 'some junk',
+ );
+
+ await idb.closeDatabase();
+ });
+
+ test('files under a custom document dir are restored from idb', async () => {
+ const db = await idb.openDatabase();
+ const { store } = idb.getStore(db, 'files');
+ await idb.set(store, {
+ filepath: '/custom/xyz/metadata.json',
+ contents: '{"id":"xyz"}',
+ });
+ await idb.set(store, {
+ filepath: '/custom/xyz/db.sqlite',
+ contents: 'this will be blank and just create a symlink',
+ });
+
+ await sqlite.init();
+ await init();
+ _setDocumentDir('/custom');
+
+ expect(await readFile('/custom/xyz/metadata.json')).toBe('{"id":"xyz"}');
+
+ const FS = sqlite._getModule().FS;
+ const { node } = FS.lookupPath('/custom/xyz/db.sqlite', {});
+ expect(node.link).toBe('/blocked/' + pathToId('/custom/xyz/db.sqlite'));
+ });
});
describe('join', () => {
diff --git a/packages/loot-core/src/platform/server/fs/index.ts b/packages/loot-core/src/platform/server/fs/index.ts
index 160f0c2a6c..ecda69d7a2 100644
--- a/packages/loot-core/src/platform/server/fs/index.ts
+++ b/packages/loot-core/src/platform/server/fs/index.ts
@@ -9,6 +9,8 @@ import { logger } from '#platform/server/log';
import { _getModule } from '#platform/server/sqlite';
import type { SqlJsModule } from '#platform/server/sqlite';
+import { _setDocumentDir as _setDocumentDirShared } from './shared';
+
let FS: SqlJsModule['FS'] = null;
let BFS = null;
const NO_PERSIST = false;
@@ -17,9 +19,44 @@ export const bundledDatabasePath: string = '/default-db.sqlite';
export const migrationsPath: string = '/migrations';
export const demoBudgetPath: string = '/demo-budget';
export { join };
-export { getDocumentDir, getBudgetDir, _setDocumentDir } from './shared';
+export { getDocumentDir, getBudgetDir } from './shared';
export const getDataDir = () => process.env.ACTUAL_DATA_DIR;
+// The active document dir is persisted alongside the built-in /documents
+let customPersistedRoot: string | null = null;
+
+function normalizePath(filepath: string): string {
+ const absolute = filepath.startsWith('/') ? filepath : '/' + filepath;
+ let end = absolute.length;
+ while (end > 0 && absolute[end - 1] === '/') {
+ end--;
+ }
+ return absolute.slice(0, end);
+}
+
+function isUnderRoot(path: string, root: string): boolean {
+ return path === root || path.startsWith(root + '/');
+}
+
+function isPersistedPath(filepath: string): boolean {
+ const path = normalizePath(filepath);
+ return (
+ isUnderRoot(path, '/documents') ||
+ (customPersistedRoot != null && isUnderRoot(path, customPersistedRoot))
+ );
+}
+
+export const _setDocumentDir = dir => {
+ const normalized = typeof dir === 'string' ? normalizePath(dir) : '';
+ if (normalized !== '') {
+ customPersistedRoot = normalized;
+ if (FS) {
+ _mkdirRecursively(normalized);
+ }
+ }
+ return _setDocumentDirShared(dir);
+};
+
export const pathToId = function (filepath: string): string {
return filepath.replace(/^\//, '').replace(/\//g, '-');
};
@@ -48,21 +85,25 @@ function _mkdirRecursively(dir) {
}
}
+function _createPersistedFile(filepath: string) {
+ if (filepath.endsWith('.sqlite')) {
+ // If it doesn't exist, we need to create a symlink
+ if (!_exists(filepath)) {
+ FS.symlink('/blocked/' + pathToId(filepath), filepath);
+ }
+ } else {
+ // The contents are actually stored in IndexedDB. We only write to
+ // the in-memory fs to take advantage of the file hierarchy
+ FS.writeFile(filepath, '!$@) this should never read !$@)');
+ }
+}
+
function _createFile(filepath: string) {
// This can create the file. Check if it exists, if not create a
// symlink if it's a sqlite file. Otherwise store in idb
- if (!NO_PERSIST && filepath.startsWith('/documents')) {
- if (filepath.endsWith('.sqlite')) {
- // If it doesn't exist, we need to create a symlink
- if (!_exists(filepath)) {
- FS.symlink('/blocked/' + pathToId(filepath), filepath);
- }
- } else {
- // The contents are actually stored in IndexedDB. We only write to
- // the in-memory fs to take advantage of the file hierarchy
- FS.writeFile(filepath, '!$@) this should never read !$@)');
- }
+ if (!NO_PERSIST && isPersistedPath(filepath)) {
+ _createPersistedFile(filepath);
}
return filepath;
@@ -85,7 +126,7 @@ async function _readFile(
// filesystem and will be handled in the BlockedFS
if (
!NO_PERSIST &&
- filepath.startsWith('/documents') &&
+ isPersistedPath(filepath) &&
!filepath.endsWith('.sqlite')
) {
if (!_exists(filepath)) {
@@ -139,7 +180,7 @@ async function _writeFile(filepath: string, contents): Promise<boolean> {
// setups up the file depending on its type
_createFile(filepath);
- if (!NO_PERSIST && filepath.startsWith('/documents')) {
+ if (!NO_PERSIST && isPersistedPath(filepath)) {
const isDb = filepath.endsWith('.sqlite');
// Write to IDB
@@ -211,7 +252,7 @@ async function _copySqlFile(
}
async function _removeFile(filepath: string) {
- if (!NO_PERSIST && filepath.startsWith('/documents')) {
+ if (!NO_PERSIST && isPersistedPath(filepath)) {
const isDb = filepath.endsWith('.sqlite');
// Remove from IDB
@@ -272,7 +313,7 @@ const populateFileHierarchy = async function () {
for (const path of paths) {
_mkdirRecursively(basename(path));
- _createFile(path);
+ _createPersistedFile(path);
}
};
diff --git a/packages/loot-core/src/server/main.ts b/packages/loot-core/src/server/main.ts
index b27be83cac..25826588fc 100644
--- a/packages/loot-core/src/server/main.ts
+++ b/packages/loot-core/src/server/main.ts
@@ -228,6 +228,13 @@ export async function initApp(isDev, socketName) {
}
type BaseInitConfig = {
+ /**
+ * Directory where budget data is stored. In Node this is a directory on
+ * disk (defaults to the current working directory) and must already exist.
+ * In the browser build it is a path inside the worker's virtual filesystem
+ * (persisted to IndexedDB); it defaults to `/documents` and is created
+ * automatically if missing.
+ */
dataDir?: string;
verbose?: boolean;
};
diff --git a/upcoming-release-notes/api-browser-custom-data-dir.md b/upcoming-release-notes/api-browser-custom-data-dir.md
new file mode 100644
index 0000000000..c44ccda78f
--- /dev/null
+++ b/upcoming-release-notes/api-browser-custom-data-dir.md
@@ -0,0 +1,6 @@
+---
+category: Bugfix
+authors: [MatissJanis]
+---
+
+Fix using a custom data directory in the browser version of the API, and show the real error message when something goes wrong in the background