Skip to content

A local-first personal finance app

[AI] Make file writes atomic (#9014)

Malcolm168e174b47c0
diff --git a/packages/loot-core/src/platform/server/fs/atomic-write.test.ts b/packages/loot-core/src/platform/server/fs/atomic-write.test.ts
new file mode 100644
index 0000000000..9a7816411d
--- /dev/null
+++ b/packages/loot-core/src/platform/server/fs/atomic-write.test.ts
@@ -0,0 +1,123 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+import { atomicWriteFile } from './atomic-write';
+
+// Skips the lock-retry backoff delays while letting real file I/O complete.
+async function settle(promise: Promise<void>) {
+ let isSettled = false;
+ promise.then(
+ () => (isSettled = true),
+ () => (isSettled = true),
+ );
+ while (!isSettled) {
+ await new Promise(resolve => setImmediate(resolve));
+ await vi.advanceTimersByTimeAsync(500);
+ }
+ return promise;
+}
+
+describe('atomicWriteFile', () => {
+ let dir: string;
+
+ beforeEach(() => {
+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'actual-fs-test-'));
+ vi.useFakeTimers({ toFake: ['setTimeout'] });
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ fs.rmSync(dir, { recursive: true, force: true });
+ });
+
+ const tempFiles = () => fs.readdirSync(dir).filter(f => f.endsWith('.tmp'));
+
+ test('replaces the contents and leaves no temp file behind', async () => {
+ const target = path.join(dir, 'db.sqlite');
+ fs.writeFileSync(target, 'original');
+ const bytes = new Uint8Array([0x00, 0xff, 0x53, 0x51, 0x4c, 0x80]);
+
+ await atomicWriteFile(target, bytes);
+
+ expect(new Uint8Array(fs.readFileSync(target))).toEqual(bytes);
+ expect(tempFiles()).toEqual([]);
+ });
+
+ test('keeps the original file when the write fails partway through', async () => {
+ const target = path.join(dir, 'metadata.json');
+ fs.writeFileSync(target, 'original');
+ vi.spyOn(fs.promises, 'writeFile').mockImplementation(async tmpPath => {
+ fs.writeFileSync(tmpPath as fs.PathLike, 'PARTIAL');
+ throw new Error('simulated disk failure');
+ });
+
+ await expect(settle(atomicWriteFile(target, 'new'))).rejects.toThrow(
+ 'simulated disk failure',
+ );
+
+ expect(fs.readFileSync(target, 'utf8')).toBe('original');
+ expect(tempFiles()).toEqual([]);
+ });
+
+ test('recovers when the file is locked only briefly', async () => {
+ const target = path.join(dir, 'metadata.json');
+ vi.spyOn(fs.promises, 'writeFile').mockRejectedValueOnce(
+ Object.assign(new Error('resource busy or locked'), { code: 'EBUSY' }),
+ );
+
+ await settle(atomicWriteFile(target, 'new'));
+
+ expect(fs.readFileSync(target, 'utf8')).toBe('new');
+ });
+
+ test('writes through a symlink instead of replacing it', async () => {
+ const realTarget = path.join(dir, 'real-metadata.json');
+ const symlinkPath = path.join(dir, 'metadata.json');
+ fs.writeFileSync(realTarget, 'original');
+ fs.symlinkSync(realTarget, symlinkPath);
+
+ await atomicWriteFile(symlinkPath, 'new');
+
+ expect(fs.lstatSync(symlinkPath).isSymbolicLink()).toBe(true);
+ expect(fs.readFileSync(realTarget, 'utf8')).toBe('new');
+ });
+
+ test('creates the target of a dangling symlink instead of replacing it', async () => {
+ // A relative link reached through a symlinked directory, so `..` must
+ // resolve from the real directory.
+ const realDir = path.join(dir, 'actual', 'nested');
+ fs.mkdirSync(realDir, { recursive: true });
+ fs.symlinkSync(realDir, path.join(dir, 'storage'));
+ const symlinkPath = path.join(realDir, 'metadata.json');
+ fs.symlinkSync(path.join('..', 'metadata.json'), symlinkPath);
+ fs.writeFileSync(path.join(dir, 'metadata.json'), 'unrelated');
+
+ await atomicWriteFile(path.join(dir, 'storage', 'metadata.json'), 'new');
+
+ expect(fs.lstatSync(symlinkPath).isSymbolicLink()).toBe(true);
+ expect(
+ fs.readFileSync(path.join(dir, 'actual', 'metadata.json'), 'utf8'),
+ ).toBe('new');
+ expect(fs.readFileSync(path.join(dir, 'metadata.json'), 'utf8')).toBe(
+ 'unrelated',
+ );
+ });
+
+ test(
+ 'keeps the permissions of the file it replaces',
+ { skip: process.platform === 'win32' },
+ async () => {
+ const target = path.join(dir, 'metadata.json');
+ fs.writeFileSync(target, 'original');
+ // Set after creation, as the umask would strip the group and other
+ // write bits.
+ fs.chmodSync(target, 0o666);
+
+ await atomicWriteFile(target, 'new');
+
+ expect(fs.statSync(target).mode & 0o777).toBe(0o666);
+ },
+ );
+});
diff --git a/packages/loot-core/src/platform/server/fs/atomic-write.ts b/packages/loot-core/src/platform/server/fs/atomic-write.ts
new file mode 100644
index 0000000000..4b5e1592d7
--- /dev/null
+++ b/packages/loot-core/src/platform/server/fs/atomic-write.ts
@@ -0,0 +1,78 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { logger } from '#platform/server/log';
+import { retry as promiseRetry } from '#shared/retry';
+
+let tempFileCounter = 0;
+
+// Follows symlinks, including dangling ones, so the rename replaces the
+// link's target rather than the link.
+async function resolveTarget(filepath: string): Promise<string> {
+ try {
+ return await fs.promises.realpath(filepath);
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
+ throw err;
+ }
+ }
+ const link = await fs.promises.readlink(filepath).catch(() => null);
+ if (link === null) {
+ return filepath;
+ }
+ // Not path.resolve, which collapses `..` before symlinked directories are
+ // followed.
+ return resolveTarget(
+ path.isAbsolute(link) ? link : path.dirname(filepath) + path.sep + link,
+ );
+}
+
+// Writes a temp file and renames it over the target, so a write that fails or
+// is killed partway leaves the original file intact.
+export async function atomicWriteFile(
+ filepath: string,
+ contents: string | NodeJS.ArrayBufferView,
+): Promise<void> {
+ const target = await resolveTarget(filepath);
+ const tmpPath = `${target}.${process.pid}.${tempFileCounter++}.tmp`;
+ const mode = await fs.promises.stat(target).then(
+ stats => stats.mode & 0o777,
+ () => undefined,
+ );
+
+ try {
+ await promiseRetry(
+ async (retry, attempt) => {
+ try {
+ await fs.promises.writeFile(tmpPath, contents, { flush: true, mode });
+ if (mode !== undefined) {
+ // Restore bits the umask stripped. Best-effort, as some
+ // filesystems refuse chmod.
+ await fs.promises.chmod(tmpPath, mode).catch(() => undefined);
+ }
+ await fs.promises.rename(tmpPath, target);
+ if (attempt > 1) {
+ logger.info(
+ `Successfully recovered from file lock. It took ${attempt} retries`,
+ );
+ }
+ } catch (err) {
+ logger.error(
+ `Failed to write to ${filepath}. Attempted ${attempt} times. Something is locking the file - potentially a virus scanner or backup software.`,
+ );
+ retry(err);
+ }
+ },
+ {
+ retries: 20,
+ minTimeout: 100,
+ maxTimeout: 500,
+ factor: 1.5,
+ },
+ );
+ } catch (err) {
+ logger.error(`Unable to recover from file lock on file ${filepath}`);
+ await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined);
+ throw err;
+ }
+}
diff --git a/packages/loot-core/src/platform/server/fs/index.api.ts b/packages/loot-core/src/platform/server/fs/index.api.ts
index 737bcf2824..a0bd8165d2 100644
--- a/packages/loot-core/src/platform/server/fs/index.api.ts
+++ b/packages/loot-core/src/platform/server/fs/index.api.ts
@@ -2,9 +2,7 @@
import * as fs from 'fs';
import * as path from 'path';
-import { logger } from '#platform/server/log';
-import { retry as promiseRetry } from '#shared/retry';
-
+import { atomicWriteFile } from './atomic-write';
import { getDocumentDir } from './shared';
import type * as T from './index';
@@ -113,40 +111,8 @@ export const readFile: typeof T.readFile = (
};
export const writeFile: typeof T.writeFile = async (filepath, contents) => {
- try {
- await promiseRetry(
- (retry, attempt) => {
- return new Promise((resolve, reject) => {
- fs.writeFile(filepath, contents, 'utf8', err => {
- if (err) {
- logger.error(
- `Failed to write to ${filepath}. Attempted ${attempt} times. Something is locking the file - potentially a virus scanner or backup software.`,
- );
- reject(err);
- } else {
- if (attempt > 1) {
- logger.info(
- `Successfully recovered from file lock. It took ${attempt} retries`,
- );
- }
- resolve(undefined);
- }
- });
- }).catch(retry);
- },
- {
- retries: 20,
- minTimeout: 100,
- maxTimeout: 500,
- factor: 1.5,
- },
- );
-
- return undefined;
- } catch (err) {
- logger.error(`Unable to recover from file lock on file ${filepath}`);
- throw err;
- }
+ await atomicWriteFile(filepath, contents);
+ return undefined;
};
export const removeFile: typeof T.removeFile = filepath => {
diff --git a/packages/loot-core/src/platform/server/fs/index.electron.ts b/packages/loot-core/src/platform/server/fs/index.electron.ts
index 852587ffce..c413c8bbe1 100644
--- a/packages/loot-core/src/platform/server/fs/index.electron.ts
+++ b/packages/loot-core/src/platform/server/fs/index.electron.ts
@@ -2,8 +2,7 @@
import * as fs from 'fs';
import * as path from 'path';
-import { logger } from '#platform/server/log';
-import { retry as promiseRetry } from '#shared/retry';
+import { atomicWriteFile } from './atomic-write';
import type * as T from './index';
@@ -126,40 +125,8 @@ export const readFile: typeof T.readFile = (
};
export const writeFile: typeof T.writeFile = async (filepath, contents) => {
- try {
- await promiseRetry(
- (retry, attempt) => {
- return new Promise((resolve, reject) => {
- fs.writeFile(filepath, contents, 'utf8', err => {
- if (err) {
- logger.error(
- `Failed to write to ${filepath}. Attempted ${attempt} times. Something is locking the file - potentially a virus scanner or backup software.`,
- );
- reject(err);
- } else {
- if (attempt > 1) {
- logger.info(
- `Successfully recovered from file lock. It took ${attempt} retries`,
- );
- }
- resolve(undefined);
- }
- });
- }).catch(retry);
- },
- {
- retries: 20,
- minTimeout: 100,
- maxTimeout: 500,
- factor: 1.5,
- },
- );
-
- return undefined;
- } catch (err) {
- logger.error(`Unable to recover from file lock on file ${filepath}`);
- throw err;
- }
+ await atomicWriteFile(filepath, contents);
+ return undefined;
};
export const removeFile: typeof T.removeFile = filepath => {
diff --git a/packages/sync-server/src/app-sync.test.ts b/packages/sync-server/src/app-sync.test.ts
index 6ca79b3478..76c2162534 100644
--- a/packages/sync-server/src/app-sync.test.ts
+++ b/packages/sync-server/src/app-sync.test.ts
@@ -1,6 +1,7 @@
// @ts-strict-ignore
import crypto from 'node:crypto';
import fs from 'node:fs';
+import fsPromises from 'node:fs/promises';
import { create, SyncRequestSchema, toBinary } from '@actual-app/crdt';
import request from 'supertest';
@@ -768,6 +769,50 @@ describe('/upload-user-file', () => {
]);
expect(rows[0].name).toEqual('admin-renamed.txt');
});
+
+ it('returns 500 and keeps the existing file when the write fails', async () => {
+ const fileId = generateFileId();
+ const groupId = 'write-failure-group-id';
+ const keyId = 'key-id';
+ const syncVersion = 2;
+ const encryptMeta = JSON.stringify({ keyId });
+ const originalContent = 'original good content';
+ const filePath = getPathForUserFile(fileId);
+
+ getAccountDb().mutate(
+ 'INSERT INTO files (id, group_id, sync_version, name, encrypt_meta, encrypt_keyid) VALUES (?, ?, ?, ?, ?, ?)',
+ [fileId, groupId, syncVersion, 'failing.txt', encryptMeta, keyId],
+ );
+ fs.writeFileSync(filePath, originalContent);
+ onTestFinished(() => {
+ try {
+ fs.unlinkSync(filePath);
+ } catch {}
+ });
+
+ // Some bytes land on disk before the write rejects.
+ const writeFileSpy = vi
+ .spyOn(fsPromises, 'writeFile')
+ .mockImplementationOnce(async path => {
+ fs.writeFileSync(path as fs.PathLike, 'PARTIAL-GARBAGE');
+ throw new Error('simulated disk failure');
+ });
+ onTestFinished(() => writeFileSpy.mockRestore());
+
+ const res = await request(app)
+ .post('/upload-user-file')
+ .set('Content-Type', 'application/encrypted-file')
+ .set('x-actual-token', 'valid-token')
+ .set('x-actual-name', 'failing.txt')
+ .set('x-actual-file-id', fileId)
+ .set('x-actual-group-id', groupId)
+ .set('x-actual-format', syncVersion.toString())
+ .set('x-actual-encrypt-meta', encryptMeta)
+ .send(Buffer.from('new content that should not land'));
+
+ expect(res.statusCode).toEqual(500);
+ expect(fs.readFileSync(filePath, 'utf8')).toEqual(originalContent);
+ });
});
describe('/download-user-file', () => {
diff --git a/packages/sync-server/src/app-sync.ts b/packages/sync-server/src/app-sync.ts
index 45bf77d049..e170482b72 100644
--- a/packages/sync-server/src/app-sync.ts
+++ b/packages/sync-server/src/app-sync.ts
@@ -29,6 +29,7 @@ import { config } from './load-config';
import { SecretName, secretsService } from './services/secrets-service';
import * as UserService from './services/user-service';
import * as simpleSync from './sync-simple';
+import { atomicWriteFile } from './util/atomic-write';
import {
errorMiddleware,
requestLoggerMiddleware,
@@ -365,7 +366,7 @@ app.post('/upload-user-file', async (req, res) => {
}
try {
- await fs.writeFile(getPathForUserFile(fileId), req.body);
+ await atomicWriteFile(getPathForUserFile(fileId), req.body);
} catch (err) {
console.log('Error writing file', err);
res.status(500).send({ status: 'error' });
diff --git a/packages/sync-server/src/util/atomic-write.test.ts b/packages/sync-server/src/util/atomic-write.test.ts
new file mode 100644
index 0000000000..4d8b5d47c2
--- /dev/null
+++ b/packages/sync-server/src/util/atomic-write.test.ts
@@ -0,0 +1,99 @@
+import fs from 'node:fs';
+import fsPromises from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+import { atomicWriteFile } from './atomic-write';
+
+describe('atomicWriteFile', () => {
+ let dir: string;
+
+ beforeEach(() => {
+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'actual-atomic-test-'));
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ fs.rmSync(dir, { recursive: true, force: true });
+ });
+
+ const tempFiles = () => fs.readdirSync(dir).filter(f => f.endsWith('.tmp'));
+
+ it('replaces the file and leaves no temp file behind', async () => {
+ const target = path.join(dir, 'file-abc.blob');
+ fs.writeFileSync(target, 'original');
+
+ await atomicWriteFile(target, Buffer.from('new'));
+
+ expect(fs.readFileSync(target, 'utf8')).toBe('new');
+ expect(tempFiles()).toEqual([]);
+ });
+
+ it('keeps the original file when the write fails partway through', async () => {
+ const target = path.join(dir, 'file-abc.blob');
+ fs.writeFileSync(target, 'original');
+ vi.spyOn(fsPromises, 'writeFile').mockImplementationOnce(async tmpPath => {
+ fs.writeFileSync(tmpPath as fs.PathLike, 'PARTIAL');
+ throw new Error('simulated disk failure');
+ });
+
+ await expect(atomicWriteFile(target, Buffer.from('new'))).rejects.toThrow(
+ 'simulated disk failure',
+ );
+
+ expect(fs.readFileSync(target, 'utf8')).toBe('original');
+ expect(tempFiles()).toEqual([]);
+ });
+
+ it('writes through a symlink instead of replacing it', async () => {
+ const realTarget = path.join(dir, 'real.blob');
+ const symlinkPath = path.join(dir, 'file-abc.blob');
+ fs.writeFileSync(realTarget, 'original');
+ fs.symlinkSync(realTarget, symlinkPath);
+
+ await atomicWriteFile(symlinkPath, Buffer.from('new'));
+
+ expect(fs.lstatSync(symlinkPath).isSymbolicLink()).toBe(true);
+ expect(fs.readFileSync(realTarget, 'utf8')).toBe('new');
+ });
+
+ it('creates the target of a dangling symlink instead of replacing it', async () => {
+ // A relative link reached through a symlinked directory, so `..` must
+ // resolve from the real directory.
+ const realDir = path.join(dir, 'actual', 'nested');
+ fs.mkdirSync(realDir, { recursive: true });
+ fs.symlinkSync(realDir, path.join(dir, 'storage'));
+ const symlinkPath = path.join(realDir, 'file-abc.blob');
+ fs.symlinkSync(path.join('..', 'file-abc.blob'), symlinkPath);
+ fs.writeFileSync(path.join(dir, 'file-abc.blob'), 'unrelated');
+
+ await atomicWriteFile(
+ path.join(dir, 'storage', 'file-abc.blob'),
+ Buffer.from('new'),
+ );
+
+ expect(fs.lstatSync(symlinkPath).isSymbolicLink()).toBe(true);
+ expect(
+ fs.readFileSync(path.join(dir, 'actual', 'file-abc.blob'), 'utf8'),
+ ).toBe('new');
+ expect(fs.readFileSync(path.join(dir, 'file-abc.blob'), 'utf8')).toBe(
+ 'unrelated',
+ );
+ });
+
+ it(
+ 'keeps the permissions of the file it replaces',
+ { skip: process.platform === 'win32' },
+ async () => {
+ const target = path.join(dir, 'file-abc.blob');
+ fs.writeFileSync(target, 'original');
+ // Set after creation, as the umask would strip the group and other
+ // write bits.
+ fs.chmodSync(target, 0o666);
+
+ await atomicWriteFile(target, Buffer.from('new'));
+
+ expect(fs.statSync(target).mode & 0o777).toBe(0o666);
+ },
+ );
+});
diff --git a/packages/sync-server/src/util/atomic-write.ts b/packages/sync-server/src/util/atomic-write.ts
new file mode 100644
index 0000000000..eab7770d17
--- /dev/null
+++ b/packages/sync-server/src/util/atomic-write.ts
@@ -0,0 +1,52 @@
+import fs from 'node:fs/promises';
+import { dirname, isAbsolute, sep } from 'node:path';
+
+let tempFileCounter = 0;
+
+// Follows symlinks, including dangling ones, so the rename replaces the
+// link's target rather than the link.
+async function resolveTarget(filepath: string): Promise<string> {
+ try {
+ return await fs.realpath(filepath);
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
+ throw err;
+ }
+ }
+ const link = await fs.readlink(filepath).catch(() => null);
+ if (link === null) {
+ return filepath;
+ }
+ // Not path.resolve, which collapses `..` before symlinked directories are
+ // followed.
+ return resolveTarget(
+ isAbsolute(link) ? link : dirname(filepath) + sep + link,
+ );
+}
+
+// Writes a temp file and renames it over the target, so a write that fails or
+// is killed partway leaves the original file intact.
+export async function atomicWriteFile(
+ filepath: string,
+ contents: string | NodeJS.ArrayBufferView,
+): Promise<void> {
+ const target = await resolveTarget(filepath);
+ const tmpPath = `${target}.${process.pid}.${tempFileCounter++}.tmp`;
+ const mode = await fs.stat(target).then(
+ stats => stats.mode & 0o777,
+ () => undefined,
+ );
+
+ try {
+ await fs.writeFile(tmpPath, contents, { flush: true, mode });
+ if (mode !== undefined) {
+ // Restore bits the umask stripped. Best-effort, as some
+ // filesystems refuse chmod.
+ await fs.chmod(tmpPath, mode).catch(() => undefined);
+ }
+ await fs.rename(tmpPath, target);
+ } catch (err) {
+ await fs.rm(tmpPath, { force: true }).catch(() => undefined);
+ throw err;
+ }
+}
diff --git a/packages/sync-server/src/util/paths.test.ts b/packages/sync-server/src/util/paths.test.ts
new file mode 100644
index 0000000000..1f630517e4
--- /dev/null
+++ b/packages/sync-server/src/util/paths.test.ts
@@ -0,0 +1,20 @@
+import { config } from '#load-config';
+
+import { getPathForUserFile } from './paths';
+import type { FileId } from './paths';
+
+describe('getPathForUserFile', () => {
+ it(
+ 'accepts a filesystem root as the user files directory',
+ { skip: process.platform === 'win32' },
+ () => {
+ const userFiles = config.get('userFiles');
+ config.set('userFiles', '/');
+ onTestFinished(() => {
+ config.set('userFiles', userFiles);
+ });
+
+ expect(getPathForUserFile('abc' as FileId)).toBe('/file-abc.blob');
+ },
+ );
+});
diff --git a/packages/sync-server/src/util/paths.ts b/packages/sync-server/src/util/paths.ts
index 49b13e7eca..dce3b742d3 100644
--- a/packages/sync-server/src/util/paths.ts
+++ b/packages/sync-server/src/util/paths.ts
@@ -1,4 +1,4 @@
-import { join, resolve } from 'node:path';
+import { join, resolve, sep } from 'node:path';
import { config } from '#load-config';
@@ -18,7 +18,15 @@ export function isValidGroupId(id: string): id is GroupId {
}
export function getPathForUserFile(fileId: FileId) {
- return join(resolve(config.get('userFiles')), `file-${fileId}.blob`);
+ const userFilesDir = resolve(config.get('userFiles'));
+ const filePath = resolve(userFilesDir, `file-${fileId}.blob`);
+ // isValidFileId already rules out traversal; this keeps the check where the
+ // path is built. Filesystem roots already end in sep.
+ const prefix = userFilesDir.endsWith(sep) ? userFilesDir : userFilesDir + sep;
+ if (!filePath.startsWith(prefix)) {
+ throw new Error('Invalid file id');
+ }
+ return filePath;
}
export function getPathForGroupFile(groupId: GroupId) {
diff --git a/upcoming-release-notes/atomic-user-file-write.md b/upcoming-release-notes/atomic-user-file-write.md
new file mode 100644
index 0000000000..838e7e51fa
--- /dev/null
+++ b/upcoming-release-notes/atomic-user-file-write.md
@@ -0,0 +1,6 @@
+---
+category: Bugfix
+authors: [laforcem]
+---
+
+Fix budget files being left half-written if saving fails or the desktop app or sync server crashes mid-save