A local-first personal finance app
[AI] Make file writes atomic (#9014)
Malcolm168e174b47c0
diff --git a/packages/loot-core/src/platform/server/fs/atomic-write.test.ts b/packages/loot-core/src/platform/server/fs/atomic-write.test.tsnew file mode 100644index 0000000000..9a7816411d--- /dev/null+++ b/packages/loot-core/src/platform/server/fs/atomic-write.test.ts@@ -0,0 +1,123 @@+import * as fs from 'fs';+import * as os from 'os';+import * as path from 'path';++import { atomicWriteFile } from './atomic-write';++// Skips the lock-retry backoff delays while letting real file I/O complete.+async function settle(promise: Promise<void>) {+ let isSettled = false;+ promise.then(+ () => (isSettled = true),+ () => (isSettled = true),+ );+ while (!isSettled) {+ await new Promise(resolve => setImmediate(resolve));+ await vi.advanceTimersByTimeAsync(500);+ }+ return promise;+}++describe('atomicWriteFile', () => {+ let dir: string;++ beforeEach(() => {+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'actual-fs-test-'));+ vi.useFakeTimers({ toFake: ['setTimeout'] });+ });++ afterEach(() => {+ vi.useRealTimers();+ vi.restoreAllMocks();+ fs.rmSync(dir, { recursive: true, force: true });+ });++ const tempFiles = () => fs.readdirSync(dir).filter(f => f.endsWith('.tmp'));++ test('replaces the contents and leaves no temp file behind', async () => {+ const target = path.join(dir, 'db.sqlite');+ fs.writeFileSync(target, 'original');+ const bytes = new Uint8Array([0x00, 0xff, 0x53, 0x51, 0x4c, 0x80]);++ await atomicWriteFile(target, bytes);++ expect(new Uint8Array(fs.readFileSync(target))).toEqual(bytes);+ expect(tempFiles()).toEqual([]);+ });++ test('keeps the original file when the write fails partway through', async () => {+ const target = path.join(dir, 'metadata.json');+ fs.writeFileSync(target, 'original');+ vi.spyOn(fs.promises, 'writeFile').mockImplementation(async tmpPath => {+ fs.writeFileSync(tmpPath as fs.PathLike, 'PARTIAL');+ throw new Error('simulated disk failure');+ });++ await expect(settle(atomicWriteFile(target, 'new'))).rejects.toThrow(+ 'simulated disk failure',+ );++ expect(fs.readFileSync(target, 'utf8')).toBe('original');+ expect(tempFiles()).toEqual([]);+ });++ test('recovers when the file is locked only briefly', async () => {+ const target = path.join(dir, 'metadata.json');+ vi.spyOn(fs.promises, 'writeFile').mockRejectedValueOnce(+ Object.assign(new Error('resource busy or locked'), { code: 'EBUSY' }),+ );++ await settle(atomicWriteFile(target, 'new'));++ expect(fs.readFileSync(target, 'utf8')).toBe('new');+ });++ test('writes through a symlink instead of replacing it', async () => {+ const realTarget = path.join(dir, 'real-metadata.json');+ const symlinkPath = path.join(dir, 'metadata.json');+ fs.writeFileSync(realTarget, 'original');+ fs.symlinkSync(realTarget, symlinkPath);++ await atomicWriteFile(symlinkPath, 'new');++ expect(fs.lstatSync(symlinkPath).isSymbolicLink()).toBe(true);+ expect(fs.readFileSync(realTarget, 'utf8')).toBe('new');+ });++ test('creates the target of a dangling symlink instead of replacing it', async () => {+ // A relative link reached through a symlinked directory, so `..` must+ // resolve from the real directory.+ const realDir = path.join(dir, 'actual', 'nested');+ fs.mkdirSync(realDir, { recursive: true });+ fs.symlinkSync(realDir, path.join(dir, 'storage'));+ const symlinkPath = path.join(realDir, 'metadata.json');+ fs.symlinkSync(path.join('..', 'metadata.json'), symlinkPath);+ fs.writeFileSync(path.join(dir, 'metadata.json'), 'unrelated');++ await atomicWriteFile(path.join(dir, 'storage', 'metadata.json'), 'new');++ expect(fs.lstatSync(symlinkPath).isSymbolicLink()).toBe(true);+ expect(+ fs.readFileSync(path.join(dir, 'actual', 'metadata.json'), 'utf8'),+ ).toBe('new');+ expect(fs.readFileSync(path.join(dir, 'metadata.json'), 'utf8')).toBe(+ 'unrelated',+ );+ });++ test(+ 'keeps the permissions of the file it replaces',+ { skip: process.platform === 'win32' },+ async () => {+ const target = path.join(dir, 'metadata.json');+ fs.writeFileSync(target, 'original');+ // Set after creation, as the umask would strip the group and other+ // write bits.+ fs.chmodSync(target, 0o666);++ await atomicWriteFile(target, 'new');++ expect(fs.statSync(target).mode & 0o777).toBe(0o666);+ },+ );+});diff --git a/packages/loot-core/src/platform/server/fs/atomic-write.ts b/packages/loot-core/src/platform/server/fs/atomic-write.tsnew file mode 100644index 0000000000..4b5e1592d7--- /dev/null+++ b/packages/loot-core/src/platform/server/fs/atomic-write.ts@@ -0,0 +1,78 @@+import * as fs from 'fs';+import * as path from 'path';++import { logger } from '#platform/server/log';+import { retry as promiseRetry } from '#shared/retry';++let tempFileCounter = 0;++// Follows symlinks, including dangling ones, so the rename replaces the+// link's target rather than the link.+async function resolveTarget(filepath: string): Promise<string> {+ try {+ return await fs.promises.realpath(filepath);+ } catch (err) {+ if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {+ throw err;+ }+ }+ const link = await fs.promises.readlink(filepath).catch(() => null);+ if (link === null) {+ return filepath;+ }+ // Not path.resolve, which collapses `..` before symlinked directories are+ // followed.+ return resolveTarget(+ path.isAbsolute(link) ? link : path.dirname(filepath) + path.sep + link,+ );+}++// Writes a temp file and renames it over the target, so a write that fails or+// is killed partway leaves the original file intact.+export async function atomicWriteFile(+ filepath: string,+ contents: string | NodeJS.ArrayBufferView,+): Promise<void> {+ const target = await resolveTarget(filepath);+ const tmpPath = `${target}.${process.pid}.${tempFileCounter++}.tmp`;+ const mode = await fs.promises.stat(target).then(+ stats => stats.mode & 0o777,+ () => undefined,+ );++ try {+ await promiseRetry(+ async (retry, attempt) => {+ try {+ await fs.promises.writeFile(tmpPath, contents, { flush: true, mode });+ if (mode !== undefined) {+ // Restore bits the umask stripped. Best-effort, as some+ // filesystems refuse chmod.+ await fs.promises.chmod(tmpPath, mode).catch(() => undefined);+ }+ await fs.promises.rename(tmpPath, target);+ if (attempt > 1) {+ logger.info(+ `Successfully recovered from file lock. It took ${attempt} retries`,+ );+ }+ } catch (err) {+ logger.error(+ `Failed to write to ${filepath}. Attempted ${attempt} times. Something is locking the file - potentially a virus scanner or backup software.`,+ );+ retry(err);+ }+ },+ {+ retries: 20,+ minTimeout: 100,+ maxTimeout: 500,+ factor: 1.5,+ },+ );+ } catch (err) {+ logger.error(`Unable to recover from file lock on file ${filepath}`);+ await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined);+ throw err;+ }+}diff --git a/packages/loot-core/src/platform/server/fs/index.api.ts b/packages/loot-core/src/platform/server/fs/index.api.tsindex 737bcf2824..a0bd8165d2 100644--- a/packages/loot-core/src/platform/server/fs/index.api.ts+++ b/packages/loot-core/src/platform/server/fs/index.api.ts@@ -2,9 +2,7 @@import * as fs from 'fs';import * as path from 'path';-import { logger } from '#platform/server/log';-import { retry as promiseRetry } from '#shared/retry';-+import { atomicWriteFile } from './atomic-write';import { getDocumentDir } from './shared';import type * as T from './index';@@ -113,40 +111,8 @@ export const readFile: typeof T.readFile = (};export const writeFile: typeof T.writeFile = async (filepath, contents) => {- try {- await promiseRetry(- (retry, attempt) => {- return new Promise((resolve, reject) => {- fs.writeFile(filepath, contents, 'utf8', err => {- if (err) {- logger.error(- `Failed to write to ${filepath}. Attempted ${attempt} times. Something is locking the file - potentially a virus scanner or backup software.`,- );- reject(err);- } else {- if (attempt > 1) {- logger.info(- `Successfully recovered from file lock. It took ${attempt} retries`,- );- }- resolve(undefined);- }- });- }).catch(retry);- },- {- retries: 20,- minTimeout: 100,- maxTimeout: 500,- factor: 1.5,- },- );-- return undefined;- } catch (err) {- logger.error(`Unable to recover from file lock on file ${filepath}`);- throw err;- }+ await atomicWriteFile(filepath, contents);+ return undefined;};export const removeFile: typeof T.removeFile = filepath => {diff --git a/packages/loot-core/src/platform/server/fs/index.electron.ts b/packages/loot-core/src/platform/server/fs/index.electron.tsindex 852587ffce..c413c8bbe1 100644--- a/packages/loot-core/src/platform/server/fs/index.electron.ts+++ b/packages/loot-core/src/platform/server/fs/index.electron.ts@@ -2,8 +2,7 @@import * as fs from 'fs';import * as path from 'path';-import { logger } from '#platform/server/log';-import { retry as promiseRetry } from '#shared/retry';+import { atomicWriteFile } from './atomic-write';import type * as T from './index';@@ -126,40 +125,8 @@ export const readFile: typeof T.readFile = (};export const writeFile: typeof T.writeFile = async (filepath, contents) => {- try {- await promiseRetry(- (retry, attempt) => {- return new Promise((resolve, reject) => {- fs.writeFile(filepath, contents, 'utf8', err => {- if (err) {- logger.error(- `Failed to write to ${filepath}. Attempted ${attempt} times. Something is locking the file - potentially a virus scanner or backup software.`,- );- reject(err);- } else {- if (attempt > 1) {- logger.info(- `Successfully recovered from file lock. It took ${attempt} retries`,- );- }- resolve(undefined);- }- });- }).catch(retry);- },- {- retries: 20,- minTimeout: 100,- maxTimeout: 500,- factor: 1.5,- },- );-- return undefined;- } catch (err) {- logger.error(`Unable to recover from file lock on file ${filepath}`);- throw err;- }+ await atomicWriteFile(filepath, contents);+ return undefined;};export const removeFile: typeof T.removeFile = filepath => {diff --git a/packages/sync-server/src/app-sync.test.ts b/packages/sync-server/src/app-sync.test.tsindex 6ca79b3478..76c2162534 100644--- a/packages/sync-server/src/app-sync.test.ts+++ b/packages/sync-server/src/app-sync.test.ts@@ -1,6 +1,7 @@// @ts-strict-ignoreimport crypto from 'node:crypto';import fs from 'node:fs';+import fsPromises from 'node:fs/promises';import { create, SyncRequestSchema, toBinary } from '@actual-app/crdt';import request from 'supertest';@@ -768,6 +769,50 @@ describe('/upload-user-file', () => {]);expect(rows[0].name).toEqual('admin-renamed.txt');});++ it('returns 500 and keeps the existing file when the write fails', async () => {+ const fileId = generateFileId();+ const groupId = 'write-failure-group-id';+ const keyId = 'key-id';+ const syncVersion = 2;+ const encryptMeta = JSON.stringify({ keyId });+ const originalContent = 'original good content';+ const filePath = getPathForUserFile(fileId);++ getAccountDb().mutate(+ 'INSERT INTO files (id, group_id, sync_version, name, encrypt_meta, encrypt_keyid) VALUES (?, ?, ?, ?, ?, ?)',+ [fileId, groupId, syncVersion, 'failing.txt', encryptMeta, keyId],+ );+ fs.writeFileSync(filePath, originalContent);+ onTestFinished(() => {+ try {+ fs.unlinkSync(filePath);+ } catch {}+ });++ // Some bytes land on disk before the write rejects.+ const writeFileSpy = vi+ .spyOn(fsPromises, 'writeFile')+ .mockImplementationOnce(async path => {+ fs.writeFileSync(path as fs.PathLike, 'PARTIAL-GARBAGE');+ throw new Error('simulated disk failure');+ });+ onTestFinished(() => writeFileSpy.mockRestore());++ const res = await request(app)+ .post('/upload-user-file')+ .set('Content-Type', 'application/encrypted-file')+ .set('x-actual-token', 'valid-token')+ .set('x-actual-name', 'failing.txt')+ .set('x-actual-file-id', fileId)+ .set('x-actual-group-id', groupId)+ .set('x-actual-format', syncVersion.toString())+ .set('x-actual-encrypt-meta', encryptMeta)+ .send(Buffer.from('new content that should not land'));++ expect(res.statusCode).toEqual(500);+ expect(fs.readFileSync(filePath, 'utf8')).toEqual(originalContent);+ });});describe('/download-user-file', () => {diff --git a/packages/sync-server/src/app-sync.ts b/packages/sync-server/src/app-sync.tsindex 45bf77d049..e170482b72 100644--- a/packages/sync-server/src/app-sync.ts+++ b/packages/sync-server/src/app-sync.ts@@ -29,6 +29,7 @@ import { config } from './load-config';import { SecretName, secretsService } from './services/secrets-service';import * as UserService from './services/user-service';import * as simpleSync from './sync-simple';+import { atomicWriteFile } from './util/atomic-write';import {errorMiddleware,requestLoggerMiddleware,@@ -365,7 +366,7 @@ app.post('/upload-user-file', async (req, res) => {}try {- await fs.writeFile(getPathForUserFile(fileId), req.body);+ await atomicWriteFile(getPathForUserFile(fileId), req.body);} catch (err) {console.log('Error writing file', err);res.status(500).send({ status: 'error' });diff --git a/packages/sync-server/src/util/atomic-write.test.ts b/packages/sync-server/src/util/atomic-write.test.tsnew file mode 100644index 0000000000..4d8b5d47c2--- /dev/null+++ b/packages/sync-server/src/util/atomic-write.test.ts@@ -0,0 +1,99 @@+import fs from 'node:fs';+import fsPromises from 'node:fs/promises';+import os from 'node:os';+import path from 'node:path';++import { atomicWriteFile } from './atomic-write';++describe('atomicWriteFile', () => {+ let dir: string;++ beforeEach(() => {+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'actual-atomic-test-'));+ });++ afterEach(() => {+ vi.restoreAllMocks();+ fs.rmSync(dir, { recursive: true, force: true });+ });++ const tempFiles = () => fs.readdirSync(dir).filter(f => f.endsWith('.tmp'));++ it('replaces the file and leaves no temp file behind', async () => {+ const target = path.join(dir, 'file-abc.blob');+ fs.writeFileSync(target, 'original');++ await atomicWriteFile(target, Buffer.from('new'));++ expect(fs.readFileSync(target, 'utf8')).toBe('new');+ expect(tempFiles()).toEqual([]);+ });++ it('keeps the original file when the write fails partway through', async () => {+ const target = path.join(dir, 'file-abc.blob');+ fs.writeFileSync(target, 'original');+ vi.spyOn(fsPromises, 'writeFile').mockImplementationOnce(async tmpPath => {+ fs.writeFileSync(tmpPath as fs.PathLike, 'PARTIAL');+ throw new Error('simulated disk failure');+ });++ await expect(atomicWriteFile(target, Buffer.from('new'))).rejects.toThrow(+ 'simulated disk failure',+ );++ expect(fs.readFileSync(target, 'utf8')).toBe('original');+ expect(tempFiles()).toEqual([]);+ });++ it('writes through a symlink instead of replacing it', async () => {+ const realTarget = path.join(dir, 'real.blob');+ const symlinkPath = path.join(dir, 'file-abc.blob');+ fs.writeFileSync(realTarget, 'original');+ fs.symlinkSync(realTarget, symlinkPath);++ await atomicWriteFile(symlinkPath, Buffer.from('new'));++ expect(fs.lstatSync(symlinkPath).isSymbolicLink()).toBe(true);+ expect(fs.readFileSync(realTarget, 'utf8')).toBe('new');+ });++ it('creates the target of a dangling symlink instead of replacing it', async () => {+ // A relative link reached through a symlinked directory, so `..` must+ // resolve from the real directory.+ const realDir = path.join(dir, 'actual', 'nested');+ fs.mkdirSync(realDir, { recursive: true });+ fs.symlinkSync(realDir, path.join(dir, 'storage'));+ const symlinkPath = path.join(realDir, 'file-abc.blob');+ fs.symlinkSync(path.join('..', 'file-abc.blob'), symlinkPath);+ fs.writeFileSync(path.join(dir, 'file-abc.blob'), 'unrelated');++ await atomicWriteFile(+ path.join(dir, 'storage', 'file-abc.blob'),+ Buffer.from('new'),+ );++ expect(fs.lstatSync(symlinkPath).isSymbolicLink()).toBe(true);+ expect(+ fs.readFileSync(path.join(dir, 'actual', 'file-abc.blob'), 'utf8'),+ ).toBe('new');+ expect(fs.readFileSync(path.join(dir, 'file-abc.blob'), 'utf8')).toBe(+ 'unrelated',+ );+ });++ it(+ 'keeps the permissions of the file it replaces',+ { skip: process.platform === 'win32' },+ async () => {+ const target = path.join(dir, 'file-abc.blob');+ fs.writeFileSync(target, 'original');+ // Set after creation, as the umask would strip the group and other+ // write bits.+ fs.chmodSync(target, 0o666);++ await atomicWriteFile(target, Buffer.from('new'));++ expect(fs.statSync(target).mode & 0o777).toBe(0o666);+ },+ );+});diff --git a/packages/sync-server/src/util/atomic-write.ts b/packages/sync-server/src/util/atomic-write.tsnew file mode 100644index 0000000000..eab7770d17--- /dev/null+++ b/packages/sync-server/src/util/atomic-write.ts@@ -0,0 +1,52 @@+import fs from 'node:fs/promises';+import { dirname, isAbsolute, sep } from 'node:path';++let tempFileCounter = 0;++// Follows symlinks, including dangling ones, so the rename replaces the+// link's target rather than the link.+async function resolveTarget(filepath: string): Promise<string> {+ try {+ return await fs.realpath(filepath);+ } catch (err) {+ if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {+ throw err;+ }+ }+ const link = await fs.readlink(filepath).catch(() => null);+ if (link === null) {+ return filepath;+ }+ // Not path.resolve, which collapses `..` before symlinked directories are+ // followed.+ return resolveTarget(+ isAbsolute(link) ? link : dirname(filepath) + sep + link,+ );+}++// Writes a temp file and renames it over the target, so a write that fails or+// is killed partway leaves the original file intact.+export async function atomicWriteFile(+ filepath: string,+ contents: string | NodeJS.ArrayBufferView,+): Promise<void> {+ const target = await resolveTarget(filepath);+ const tmpPath = `${target}.${process.pid}.${tempFileCounter++}.tmp`;+ const mode = await fs.stat(target).then(+ stats => stats.mode & 0o777,+ () => undefined,+ );++ try {+ await fs.writeFile(tmpPath, contents, { flush: true, mode });+ if (mode !== undefined) {+ // Restore bits the umask stripped. Best-effort, as some+ // filesystems refuse chmod.+ await fs.chmod(tmpPath, mode).catch(() => undefined);+ }+ await fs.rename(tmpPath, target);+ } catch (err) {+ await fs.rm(tmpPath, { force: true }).catch(() => undefined);+ throw err;+ }+}diff --git a/packages/sync-server/src/util/paths.test.ts b/packages/sync-server/src/util/paths.test.tsnew file mode 100644index 0000000000..1f630517e4--- /dev/null+++ b/packages/sync-server/src/util/paths.test.ts@@ -0,0 +1,20 @@+import { config } from '#load-config';++import { getPathForUserFile } from './paths';+import type { FileId } from './paths';++describe('getPathForUserFile', () => {+ it(+ 'accepts a filesystem root as the user files directory',+ { skip: process.platform === 'win32' },+ () => {+ const userFiles = config.get('userFiles');+ config.set('userFiles', '/');+ onTestFinished(() => {+ config.set('userFiles', userFiles);+ });++ expect(getPathForUserFile('abc' as FileId)).toBe('/file-abc.blob');+ },+ );+});diff --git a/packages/sync-server/src/util/paths.ts b/packages/sync-server/src/util/paths.tsindex 49b13e7eca..dce3b742d3 100644--- a/packages/sync-server/src/util/paths.ts+++ b/packages/sync-server/src/util/paths.ts@@ -1,4 +1,4 @@-import { join, resolve } from 'node:path';+import { join, resolve, sep } from 'node:path';import { config } from '#load-config';@@ -18,7 +18,15 @@ export function isValidGroupId(id: string): id is GroupId {}export function getPathForUserFile(fileId: FileId) {- return join(resolve(config.get('userFiles')), `file-${fileId}.blob`);+ const userFilesDir = resolve(config.get('userFiles'));+ const filePath = resolve(userFilesDir, `file-${fileId}.blob`);+ // isValidFileId already rules out traversal; this keeps the check where the+ // path is built. Filesystem roots already end in sep.+ const prefix = userFilesDir.endsWith(sep) ? userFilesDir : userFilesDir + sep;+ if (!filePath.startsWith(prefix)) {+ throw new Error('Invalid file id');+ }+ return filePath;}export function getPathForGroupFile(groupId: GroupId) {diff --git a/upcoming-release-notes/atomic-user-file-write.md b/upcoming-release-notes/atomic-user-file-write.mdnew file mode 100644index 0000000000..838e7e51fa--- /dev/null+++ b/upcoming-release-notes/atomic-user-file-write.md@@ -0,0 +1,6 @@+---+category: Bugfix+authors: [laforcem]+---++Fix budget files being left half-written if saving fails or the desktop app or sync server crashes mid-save