Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: report backup snapshot finalization failures

Adam Eivycc510e514546
diff --git a/server/services/backup.js b/server/services/backup.js
index f017bab272..2ada6fd4d7 100644
--- a/server/services/backup.js
+++ b/server/services/backup.js
@@ -618,18 +618,26 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
let changedFiles = [];
let manifest;
+ const removeMarker = (path, marker) => unlink(path).catch(cause => {
+ if (cause.code === 'ENOENT') return;
+ // Raw filesystem errors can contain the private backup destination.
+ const code = ['EACCES', 'EPERM', 'EIO', 'ENOSPC', 'EROFS', 'ENOTDIR', 'EMFILE', 'ENFILE', 'EBUSY'].includes(cause.code) ? cause.code : 'UNKNOWN';
+ console.error(`❌ Backup snapshot finalization failed: snapshot=${snapshotId} marker=${marker} code=${code}`);
+ throw new ServerError(`Backup snapshot finalization failed (${marker} marker, ${code})`, {
+ code: 'BACKUP_SNAPSHOT_FINALIZATION_FAILED',
+ });
+ });
+
const clearInProgressMarkers = async () => {
- if (snapshotDir) await unlink(markerPath(snapshotDir)).catch(() => {});
- if (parentMarker) await unlink(parentMarker).catch(() => {});
+ if (snapshotDir) await removeMarker(markerPath(snapshotDir), 'snapshot');
+ if (parentMarker) await removeMarker(parentMarker, 'reservation');
};
const releaseActiveSnapshot = () => {
activeSnapshotId = null;
};
- const complete = async (result) => {
- if (snapshotDir) await unlink(failedMarkerPath(snapshotDir)).catch(() => {});
- await clearInProgressMarkers();
+ const complete = (result) => {
releaseActiveSnapshot();
setBackupRunning(false, 'completion');
return result;
@@ -642,7 +650,9 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
const failureRecorded = snapshotDir
? await writeFile(failedMarkerPath(snapshotDir), '').then(() => true, () => false)
: false;
- if (failureRecorded) await clearInProgressMarkers();
+ // Cleanup errors are already logged by removeMarker. The durable failure
+ // marker blocks restore even if cleanup fails; always release the run lock.
+ if (failureRecorded) await clearInProgressMarkers().catch(() => {});
releaseActiveSnapshot();
setBackupRunning(false, 'failure');
const failureState = { lastRun: new Date().toISOString(), status: 'error', error: err.message, pgBackup: null };
@@ -734,6 +744,12 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
releaseSnapshotCut();
}
+ // A backup is not complete while its durable guards still block restore.
+ // Keep activeSnapshotId and the run lock until success is persisted, but
+ // settle these markers before publishing success or allowing retention.
+ await removeMarker(failedMarkerPath(snapshotDir), 'failed');
+ await clearInProgressMarkers();
+
const status = backupStatusForPg(pgResult);
const lastRun = new Date().toISOString();
await saveState({
@@ -761,10 +777,10 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
}
}
- const result = await complete({ snapshotId, filesChanged: changedFiles.length, status, lastRun, manifest, pgBackup: pgResult, assetConsistency });
+ const result = complete({ snapshotId, filesChanged: changedFiles.length, status, lastRun, manifest, pgBackup: pgResult, assetConsistency });
- // Prune only after `complete()` has cleared this snapshot's own
- // `.in-progress` marker and released `activeSnapshotId` — pruning any
+ // Prune only after the snapshot markers are cleared and `complete()` has
+ // released `activeSnapshotId` — pruning any
// earlier would make the run's OWN fresh snapshot read as incomplete
// (via `snapshotState`) and drop out of the retention count entirely,
// silently keeping one snapshot MORE than configured. A run that throws
diff --git a/server/services/backup.test.js b/server/services/backup.test.js
index e09ef4e9eb..06e3796055 100644
--- a/server/services/backup.test.js
+++ b/server/services/backup.test.js
@@ -144,6 +144,7 @@ vi.mock('fs/promises', async (importOriginal) => {
stat: vi.fn(actual.stat),
readFile: vi.fn(actual.readFile),
access: vi.fn(actual.access),
+ unlink: vi.fn(actual.unlink),
writeFile: vi.fn(actual.writeFile),
};
});
@@ -181,6 +182,7 @@ import { EXCLUDE_PATTERN_MAX_LENGTH, anchorUserExclude, anchorUserExcludes, isSa
beforeEach(async () => {
const actual = await vi.importActual('fs/promises');
fs.access.mockImplementation(actual.access);
+ fs.unlink.mockImplementation(actual.unlink);
fs.writeFile.mockImplementation(actual.writeFile);
});
@@ -4424,6 +4426,51 @@ describe('runBackup lifecycle', () => {
stream.destroy();
});
+ it.each(['snapshot', 'reservation', 'failed'])('reports %s marker cleanup failure instead of publishing a completed backup', async (marker) => {
+ const fsp = await actualFs();
+ const { saveState, getState } = await import('./backup.js');
+ const oldIds = ['2020-01-01T00-00-00', '2020-01-02T00-00-00'];
+ const snapshotsRoot = joinPath(destRoot, 'snapshots', machineHost);
+ for (const id of oldIds) {
+ await fsp.mkdir(joinPath(snapshotsRoot, id, 'data'), { recursive: true });
+ await fsp.writeFile(joinPath(snapshotsRoot, id, 'manifest.json'), '{}');
+ }
+ await saveState({ status: 'ok', lastSnapshotId: oldIds[1], error: null });
+ const io = { emit: vi.fn() };
+ const proc = fakeProc();
+ spawn.mockReturnValue(proc);
+ const pending = runBackup(destRoot, io, { retentionCount: 1 });
+ await waitFor(() => spawn.mock.calls.length === 1, 'rsync spawn');
+ const snapshotId = io.emit.mock.calls.find(([event]) => event === 'backup:started')[1].snapshotId;
+ const snapshotDir = joinPath(snapshotsRoot, snapshotId);
+ const target = marker === 'reservation' ? joinPath(snapshotsRoot, `.${snapshotId}.in-progress`)
+ : joinPath(snapshotDir, marker === 'failed' ? '.failed' : '.in-progress');
+ fs.unlink.mockImplementation((path, ...args) => path === target
+ ? Promise.reject(Object.assign(new Error('private destination details'), { code: 'EACCES' }))
+ : fsp.unlink(path, ...args));
+ const rejection = expect(pending).rejects.toMatchObject({
+ code: 'BACKUP_SNAPSHOT_FINALIZATION_FAILED',
+ message: `Backup snapshot finalization failed (${marker} marker, EACCES)`,
+ });
+ proc.emit('close', 0);
+ await rejection;
+
+ expect(await getState()).toMatchObject({
+ status: 'error', lastSnapshotId: oldIds[1],
+ error: `Backup snapshot finalization failed (${marker} marker, EACCES)`,
+ });
+ expect(io.emit.mock.calls.some(([event]) => event === 'backup:completed')).toBe(false);
+ expect(io.emit).toHaveBeenCalledWith('backup:failed', {
+ snapshotId, error: `Backup snapshot finalization failed (${marker} marker, EACCES)`,
+ });
+ expect(isBackupInProgress()).toBe(false);
+ for (const id of oldIds) await expect(fsp.access(joinPath(snapshotsRoot, id))).resolves.toBeUndefined();
+ await expect(fsp.access(joinPath(snapshotDir, '.failed'))).resolves.toBeUndefined();
+ await expect(restoreSnapshot(destRoot, snapshotId, { dryRun: true }))
+ .rejects.toMatchObject({ code: 'SNAPSHOT_FAILED' });
+ expect(spawn).toHaveBeenCalledTimes(1);
+ });
+
it('retains incomplete guards when the failed marker cannot be written while releasing the process lock', async () => {
const io = { emit: vi.fn() };
const proc = fakeProc();