Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: report backup snapshot finalization failures
Adam Eivycc510e514546
diff --git a/server/services/backup.js b/server/services/backup.jsindex f017bab272..2ada6fd4d7 100644--- a/server/services/backup.js+++ b/server/services/backup.js@@ -618,18 +618,26 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabllet changedFiles = [];let manifest;+ const removeMarker = (path, marker) => unlink(path).catch(cause => {+ if (cause.code === 'ENOENT') return;+ // Raw filesystem errors can contain the private backup destination.+ const code = ['EACCES', 'EPERM', 'EIO', 'ENOSPC', 'EROFS', 'ENOTDIR', 'EMFILE', 'ENFILE', 'EBUSY'].includes(cause.code) ? cause.code : 'UNKNOWN';+ console.error(`❌ Backup snapshot finalization failed: snapshot=${snapshotId} marker=${marker} code=${code}`);+ throw new ServerError(`Backup snapshot finalization failed (${marker} marker, ${code})`, {+ code: 'BACKUP_SNAPSHOT_FINALIZATION_FAILED',+ });+ });+const clearInProgressMarkers = async () => {- if (snapshotDir) await unlink(markerPath(snapshotDir)).catch(() => {});- if (parentMarker) await unlink(parentMarker).catch(() => {});+ if (snapshotDir) await removeMarker(markerPath(snapshotDir), 'snapshot');+ if (parentMarker) await removeMarker(parentMarker, 'reservation');};const releaseActiveSnapshot = () => {activeSnapshotId = null;};- const complete = async (result) => {- if (snapshotDir) await unlink(failedMarkerPath(snapshotDir)).catch(() => {});- await clearInProgressMarkers();+ const complete = (result) => {releaseActiveSnapshot();setBackupRunning(false, 'completion');return result;@@ -642,7 +650,9 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disablconst failureRecorded = snapshotDir? await writeFile(failedMarkerPath(snapshotDir), '').then(() => true, () => false): false;- if (failureRecorded) await clearInProgressMarkers();+ // Cleanup errors are already logged by removeMarker. The durable failure+ // marker blocks restore even if cleanup fails; always release the run lock.+ if (failureRecorded) await clearInProgressMarkers().catch(() => {});releaseActiveSnapshot();setBackupRunning(false, 'failure');const failureState = { lastRun: new Date().toISOString(), status: 'error', error: err.message, pgBackup: null };@@ -734,6 +744,12 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disablreleaseSnapshotCut();}+ // A backup is not complete while its durable guards still block restore.+ // Keep activeSnapshotId and the run lock until success is persisted, but+ // settle these markers before publishing success or allowing retention.+ await removeMarker(failedMarkerPath(snapshotDir), 'failed');+ await clearInProgressMarkers();+const status = backupStatusForPg(pgResult);const lastRun = new Date().toISOString();await saveState({@@ -761,10 +777,10 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl}}- const result = await complete({ snapshotId, filesChanged: changedFiles.length, status, lastRun, manifest, pgBackup: pgResult, assetConsistency });+ const result = complete({ snapshotId, filesChanged: changedFiles.length, status, lastRun, manifest, pgBackup: pgResult, assetConsistency });- // Prune only after `complete()` has cleared this snapshot's own- // `.in-progress` marker and released `activeSnapshotId` — pruning any+ // Prune only after the snapshot markers are cleared and `complete()` has+ // released `activeSnapshotId` — pruning any// earlier would make the run's OWN fresh snapshot read as incomplete// (via `snapshotState`) and drop out of the retention count entirely,// silently keeping one snapshot MORE than configured. A run that throwsdiff --git a/server/services/backup.test.js b/server/services/backup.test.jsindex e09ef4e9eb..06e3796055 100644--- a/server/services/backup.test.js+++ b/server/services/backup.test.js@@ -144,6 +144,7 @@ vi.mock('fs/promises', async (importOriginal) => {stat: vi.fn(actual.stat),readFile: vi.fn(actual.readFile),access: vi.fn(actual.access),+ unlink: vi.fn(actual.unlink),writeFile: vi.fn(actual.writeFile),};});@@ -181,6 +182,7 @@ import { EXCLUDE_PATTERN_MAX_LENGTH, anchorUserExclude, anchorUserExcludes, isSabeforeEach(async () => {const actual = await vi.importActual('fs/promises');fs.access.mockImplementation(actual.access);+ fs.unlink.mockImplementation(actual.unlink);fs.writeFile.mockImplementation(actual.writeFile);});@@ -4424,6 +4426,51 @@ describe('runBackup lifecycle', () => {stream.destroy();});+ it.each(['snapshot', 'reservation', 'failed'])('reports %s marker cleanup failure instead of publishing a completed backup', async (marker) => {+ const fsp = await actualFs();+ const { saveState, getState } = await import('./backup.js');+ const oldIds = ['2020-01-01T00-00-00', '2020-01-02T00-00-00'];+ const snapshotsRoot = joinPath(destRoot, 'snapshots', machineHost);+ for (const id of oldIds) {+ await fsp.mkdir(joinPath(snapshotsRoot, id, 'data'), { recursive: true });+ await fsp.writeFile(joinPath(snapshotsRoot, id, 'manifest.json'), '{}');+ }+ await saveState({ status: 'ok', lastSnapshotId: oldIds[1], error: null });+ const io = { emit: vi.fn() };+ const proc = fakeProc();+ spawn.mockReturnValue(proc);+ const pending = runBackup(destRoot, io, { retentionCount: 1 });+ await waitFor(() => spawn.mock.calls.length === 1, 'rsync spawn');+ const snapshotId = io.emit.mock.calls.find(([event]) => event === 'backup:started')[1].snapshotId;+ const snapshotDir = joinPath(snapshotsRoot, snapshotId);+ const target = marker === 'reservation' ? joinPath(snapshotsRoot, `.${snapshotId}.in-progress`)+ : joinPath(snapshotDir, marker === 'failed' ? '.failed' : '.in-progress');+ fs.unlink.mockImplementation((path, ...args) => path === target+ ? Promise.reject(Object.assign(new Error('private destination details'), { code: 'EACCES' }))+ : fsp.unlink(path, ...args));+ const rejection = expect(pending).rejects.toMatchObject({+ code: 'BACKUP_SNAPSHOT_FINALIZATION_FAILED',+ message: `Backup snapshot finalization failed (${marker} marker, EACCES)`,+ });+ proc.emit('close', 0);+ await rejection;++ expect(await getState()).toMatchObject({+ status: 'error', lastSnapshotId: oldIds[1],+ error: `Backup snapshot finalization failed (${marker} marker, EACCES)`,+ });+ expect(io.emit.mock.calls.some(([event]) => event === 'backup:completed')).toBe(false);+ expect(io.emit).toHaveBeenCalledWith('backup:failed', {+ snapshotId, error: `Backup snapshot finalization failed (${marker} marker, EACCES)`,+ });+ expect(isBackupInProgress()).toBe(false);+ for (const id of oldIds) await expect(fsp.access(joinPath(snapshotsRoot, id))).resolves.toBeUndefined();+ await expect(fsp.access(joinPath(snapshotDir, '.failed'))).resolves.toBeUndefined();+ await expect(restoreSnapshot(destRoot, snapshotId, { dryRun: true }))+ .rejects.toMatchObject({ code: 'SNAPSHOT_FAILED' });+ expect(spawn).toHaveBeenCalledTimes(1);+ });+it('retains incomplete guards when the failed marker cannot be written while releasing the process lock', async () => {const io = { emit: vi.fn() };const proc = fakeProc();