Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve backup guards through completion persistence

Adam Eivy89c9f4b78bac
diff --git a/server/services/backup.js b/server/services/backup.js
index 2ada6fd4d7..6a6d0129c4 100644
--- a/server/services/backup.js
+++ b/server/services/backup.js
@@ -617,6 +617,7 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
let changedFiles = [];
let manifest;
+ let previousSnapshotId;
const removeMarker = (path, marker) => unlink(path).catch(cause => {
if (cause.code === 'ENOENT') return;
@@ -655,7 +656,10 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
if (failureRecorded) await clearInProgressMarkers().catch(() => {});
releaseActiveSnapshot();
setBackupRunning(false, 'failure');
- const failureState = { lastRun: new Date().toISOString(), status: 'error', error: err.message, pgBackup: null };
+ const failureState = {
+ lastRun: new Date().toISOString(), status: 'error', error: err.message, pgBackup: null,
+ ...(previousSnapshotId !== undefined ? { lastSnapshotId: previousSnapshotId } : {}),
+ };
await saveState(failureState, (cause) => {
const code = ['EACCES', 'EPERM', 'EIO', 'ENOSPC', 'EROFS', 'ENOENT', 'EMFILE', 'ENFILE'].includes(cause.code) ? cause.code : 'UNKNOWN';
failedStateProjection = { ...failureState, error: `Backup failed; status persistence failed (${code}). See server logs.` };
@@ -745,13 +749,14 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
}
// A backup is not complete while its durable guards still block restore.
- // Keep activeSnapshotId and the run lock until success is persisted, but
- // settle these markers before publishing success or allowing retention.
+ // Keep the reservation guard until success is persisted: if both the
+ // status write and failure-marker write fail, it must survive a restart.
await removeMarker(failedMarkerPath(snapshotDir), 'failed');
- await clearInProgressMarkers();
+ await removeMarker(markerPath(snapshotDir), 'snapshot');
const status = backupStatusForPg(pgResult);
const lastRun = new Date().toISOString();
+ previousSnapshotId = (await getState()).lastSnapshotId;
await saveState({
lastRun,
lastSnapshotId: snapshotId,
@@ -762,6 +767,8 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
error: pgResult.status === 'failed' ? `DB dump ${pgResult.reason}` : null
});
+ await removeMarker(parentMarker, 'reservation');
+
if (io) io.emit('backup:completed', { snapshotId, filesChanged: changedFiles.length, status, pgBackup: pgResult });
// Loud-on-failure: surface a degraded DB dump as a warning toast, even on
diff --git a/server/services/backup.test.js b/server/services/backup.test.js
index 06e3796055..6b48834316 100644
--- a/server/services/backup.test.js
+++ b/server/services/backup.test.js
@@ -4471,6 +4471,37 @@ describe('runBackup lifecycle', () => {
expect(spawn).toHaveBeenCalledTimes(1);
});
+ it('retains a durable guard when completion state and failure-marker persistence both fail', async () => {
+ const fsp = await actualFs();
+ const { atomicWrite } = await import('../lib/fileUtils.js');
+ const io = { emit: vi.fn() };
+ const proc = fakeProc();
+ spawn.mockReturnValue(proc);
+ const pending = runBackup(destRoot, io);
+ await waitFor(() => spawn.mock.calls.length === 1, 'rsync spawn');
+ const snapshotId = io.emit.mock.calls[0][1].snapshotId;
+ const snapshotDir = await findSnapshotDir();
+ atomicWrite.mockImplementationOnce(async (path, ...args) => {
+ if (path.endsWith('manifest.json')) return (await vi.importActual('../lib/fileUtils.js')).atomicWrite(path, ...args);
+ throw new Error('unexpected write');
+ }).mockRejectedValueOnce(new Error('completion status unavailable'));
+ fs.writeFile.mockImplementation((path, ...args) => path === joinPath(snapshotDir, '.failed')
+ ? Promise.reject(Object.assign(new Error('failure marker unavailable'), { code: 'EACCES' }))
+ : fsp.writeFile(path, ...args));
+ const rejected = expect(pending).rejects.toThrow('completion status unavailable');
+ proc.emit('close', 0);
+ await rejected;
+ expect(isBackupInProgress()).toBe(false);
+ expect(io.emit.mock.calls.some(([event]) => event === 'backup:completed')).toBe(false);
+ await expect(fsp.access(joinPath(destRoot, 'snapshots', machineHost, `.${snapshotId}.in-progress`)))
+ .resolves.toBeUndefined();
+ // The on-disk guard, not process state, must block restore after restart.
+ vi.resetModules();
+ const { restoreSnapshot: restoreAfterRestart } = await import('./backup.js');
+ await expect(restoreAfterRestart(destRoot, snapshotId, { dryRun: true }))
+ .rejects.toMatchObject({ code: 'SNAPSHOT_INCOMPLETE' });
+ });
+
it('retains incomplete guards when the failed marker cannot be written while releasing the process lock', async () => {
const io = { emit: vi.fn() };
const proc = fakeProc();