Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: let a backup interrupted by a restart be deleted and pruned (#12618)

[._.]/ Adam Eivy565ae4eb3dd1
diff --git a/client/src/components/BackupWidget.jsx b/client/src/components/BackupWidget.jsx
index 93d09ae1c2..7b5469d926 100644
--- a/client/src/components/BackupWidget.jsx
+++ b/client/src/components/BackupWidget.jsx
@@ -269,7 +269,7 @@ function SnapshotList({ restoringSnapshotId, onRestoreStateChange }) {
// server-side fileCount recount or a middle-row mutation can't hide
// behind the head/tail id check.
compare: (prev, next) => prev != null && equalListByKeys(prev, next, [
- 'selectionKey', 'id', 'source', 'fileCount', 'incomplete', 'failed',
+ 'selectionKey', 'id', 'source', 'fileCount', 'incomplete', 'failed', 'interrupted',
]),
},
);
@@ -323,7 +323,9 @@ function SnapshotList({ restoringSnapshotId, onRestoreStateChange }) {
<div className="text-xs text-gray-300 font-mono truncate">{snap.id}</div>
<div className="text-xs text-gray-500 truncate">Source: {snapshotSourceLabel(snap)}</div>
<div className="text-xs text-gray-600">
- {snap.incomplete
+ {snap.interrupted
+ ? 'Interrupted'
+ : snap.incomplete
? 'Still being written…'
: snap.failed
? 'Backup failed — download available for salvage'
@@ -336,7 +338,7 @@ function SnapshotList({ restoringSnapshotId, onRestoreStateChange }) {
// Deliberately disabled across every row, not just this one:
// each download spawns a tar over the whole snapshot on the
// same external drive, so two at once only thrash the disk.
- disabled={downloadingId !== null || snap.incomplete}
+ disabled={downloadingId !== null || snap.incomplete || snap.interrupted}
aria-label={`Download snapshot ${snap.id} from ${snapshotSourceLabel(snap)}`}
className="flex items-center gap-1 px-2 py-1 text-xs text-port-accent hover:text-port-accent/80 transition-colors disabled:opacity-50 disabled:cursor-not-allowed min-h-[32px]"
>
@@ -345,8 +347,10 @@ function SnapshotList({ restoringSnapshotId, onRestoreStateChange }) {
</button>
<button
onClick={() => setSelectedId(selectedId === snapshotIdentity(snap) ? null : snapshotIdentity(snap))}
- disabled={snap.incomplete || snap.failed || restoringSnapshotId !== null}
- title={snap.failed ? 'Failed backup snapshots can only be downloaded for salvage' : undefined}
+ disabled={snap.incomplete || snap.failed || snap.interrupted || restoringSnapshotId !== null}
+ title={snap.interrupted
+ ? 'An interrupted backup snapshot cannot be restored'
+ : snap.failed ? 'Failed backup snapshots can only be downloaded for salvage' : undefined}
className="flex items-center gap-1 px-2 py-1 text-xs text-port-accent hover:text-port-accent/80 transition-colors disabled:opacity-50 disabled:cursor-not-allowed min-h-[32px]"
>
<RotateCcw size={12} />
@@ -354,7 +358,7 @@ function SnapshotList({ restoringSnapshotId, onRestoreStateChange }) {
</button>
</div>
</div>
- {selectedId === snapshotIdentity(snap) && !snap.incomplete && !snap.failed && (
+ {selectedId === snapshotIdentity(snap) && !snap.incomplete && !snap.failed && !snap.interrupted && (
<RestorePanel
snapshot={snap}
onClose={() => setSelectedId(null)}
diff --git a/client/src/components/BackupWidget.test.jsx b/client/src/components/BackupWidget.test.jsx
index 9d1ec400fb..bbd915655a 100644
--- a/client/src/components/BackupWidget.test.jsx
+++ b/client/src/components/BackupWidget.test.jsx
@@ -97,6 +97,19 @@ describe('BackupWidget snapshots', () => {
expect(screen.getByRole('button', { name: 'Restore' })).toBeDisabled();
});
+ it('labels an interrupted snapshot and offers neither download nor restore', async () => {
+ mockGetBackupSnapshots.mockResolvedValue([
+ { id: '2026-08-25T12-00-00', fileCount: 0, incomplete: false, failed: true, interrupted: true },
+ ]);
+ renderWidget();
+
+ fireEvent.click(await screen.findByRole('button', { name: 'Snapshots' }));
+ expect(await screen.findByText('Interrupted')).toBeInTheDocument();
+ expect(screen.queryByText('Still being written…')).not.toBeInTheDocument();
+ expect(screen.getByRole('button', { name: /Download snapshot/ })).toBeDisabled();
+ expect(screen.getByRole('button', { name: 'Restore' })).toBeDisabled();
+ });
+
it('keeps failed snapshots downloadable for salvage but disables restore', async () => {
mockGetBackupSnapshots.mockResolvedValue([
{ id: '2026-08-25T12-00-00', fileCount: 2, failed: true, incomplete: false },
diff --git a/client/src/components/settings/BackupTab.jsx b/client/src/components/settings/BackupTab.jsx
index 382303d6b0..125640e7a6 100644
--- a/client/src/components/settings/BackupTab.jsx
+++ b/client/src/components/settings/BackupTab.jsx
@@ -769,7 +769,9 @@ export function BackupTab() {
<span className="min-w-0">
<span className="block text-gray-300 truncate">{snap.id}</span>
<span className="block text-gray-500 truncate">Source: {snapshotSourceLabel(snap)}</span>
- {snap.failed && (
+ {snap.interrupted ? (
+ <span className="block text-port-warning">Interrupted</span>
+ ) : snap.failed && (
<span className="block text-port-error">Backup failed — download only</span>
)}
{snap.incomplete && (
@@ -779,8 +781,10 @@ export function BackupTab() {
<span className="flex shrink-0 items-center gap-1.5">
<button
onClick={() => handleRestoreDb(snap)}
- disabled={snap.failed || snap.incomplete}
- title={snap.failed ? 'Failed backup snapshots can only be downloaded for salvage' : undefined}
+ disabled={snap.failed || snap.incomplete || snap.interrupted}
+ title={snap.interrupted
+ ? 'An interrupted backup snapshot cannot be restored'
+ : snap.failed ? 'Failed backup snapshots can only be downloaded for salvage' : undefined}
className="px-2 py-2 min-h-[40px] bg-port-border hover:bg-port-border/70 text-white rounded transition-colors disabled:opacity-50 disabled:cursor-not-allowed"
>
Restore DB
diff --git a/client/src/components/settings/BackupTab.test.jsx b/client/src/components/settings/BackupTab.test.jsx
index c5e27a2897..a76bd8e98a 100644
--- a/client/src/components/settings/BackupTab.test.jsx
+++ b/client/src/components/settings/BackupTab.test.jsx
@@ -577,6 +577,16 @@ describe('BackupTab', () => {
expect(screen.getByText('snap-2026-06-09')).toBeInTheDocument();
});
+ it('labels an interrupted snapshot and keeps it deletable but not restorable', async () => {
+ getBackupSnapshots.mockResolvedValue([{ id: 'snap-interrupted', failed: true, interrupted: true }]);
+ await renderTab();
+
+ expect(await screen.findByText('Interrupted')).toBeInTheDocument();
+ expect(screen.queryByText('Backup failed — download only')).not.toBeInTheDocument();
+ expect(screen.getByRole('button', { name: /Restore DB/i })).toBeDisabled();
+ expect(screen.getByRole('button', { name: /Delete snapshot snap-interrupted/i })).toBeEnabled();
+ });
+
it('disables delete for a snapshot still being written', async () => {
getBackupSnapshots.mockResolvedValue([{ id: 'snap-incomplete', incomplete: true }]);
await renderTab();
diff --git a/docs/BACKUP.md b/docs/BACKUP.md
index 00a00c3e0f..c2ec24b7b4 100644
--- a/docs/BACKUP.md
+++ b/docs/BACKUP.md
@@ -497,7 +497,9 @@ Restore is two independent operations — restoring files and restoring the DB a
**Snapshot source lifetime (#10898).** A consumer that reads snapshot bytes holds a read lease on that snapshot for as long as it reads: a file restore or preview from before its restorability and integrity preflight until rsync's child closes (and, for execution, caches are reconciled); a database restore from its restorability check through dump admission and spooling; a download until the `tar` child closes — including after the client disconnects, since `tar` may still be reading. The lease is released on every exit path: success, refusal, failure, timeout and abort. Deletion needs the snapshot exclusively: explicit deletion of a snapshot in use is refused with `409 SNAPSHOT_IN_USE`, automatic retention skips it (logged, and pruned by a later run), and a reader that arrives while a deletion is in progress is refused with `409 SNAPSHOT_DELETING`. Deletion takes ownership *before* its final existence and in-progress checks, so a deletion that was admitted first cannot race a new reader. Ownership is keyed by the snapshot's canonical path, so aliases — an omitted versus an explicit current-machine source, a symlinked destination, a different letter case on a case-insensitive volume — share one owner. The lease coordinates this server's own requests and scheduled jobs; it is not a lock between machines that share a destination.
-A backup run that fails after creating its snapshot directory records a durable `.failed` marker before releasing its `.in-progress` guards. Failed snapshots are never eligible for file or database restore (`SNAPSHOT_FAILED`); they remain downloadable so their partial files can be inspected or recovered manually. If PortOS cannot write the failed marker, it keeps the existing incomplete markers instead, which also block restore. Snapshots created by older PortOS versions without a manifest or failure marker retain their legacy behavior because an unmarked historical failure cannot be distinguished reliably from a genuine pre-manifest snapshot.
+A backup run that fails after creating its snapshot directory records a durable `.failed` marker before releasing its `.in-progress` guards. Failed snapshots are never eligible for file or database restore (`SNAPSHOT_FAILED`); they remain downloadable so their partial files can be inspected or recovered manually. If PortOS cannot write the failed marker, it keeps the existing incomplete markers instead, which also block restore.
+
+**Interrupted snapshots (#12618).** A crash, `max_memory_restart` or manual restart can kill a run before either marker path runs. Each `.in-progress` marker therefore carries the writing server process's start id. In the current machine's namespace, a snapshot whose in-progress markers were written by another process (or are the legacy empty markers) and that is not this process's active run is reported as `interrupted`, not `incomplete`: restore and download refuse it with `409 SNAPSHOT_INTERRUPTED`, but it can be deleted. The next `runBackup()` converges each one on disk by writing `.failed` with the reason `interrupted by restart` and removing its in-progress markers; the snapshot stays `interrupted`. A marker whose owner cannot be read, and any snapshot in another machine's namespace, stays `incomplete`, because that run may still be live. Snapshots created by older PortOS versions without a manifest or failure marker retain their legacy behavior because an unmarked historical failure cannot be distinguished reliably from a genuine pre-manifest snapshot.
### Files — `restoreSnapshot()`
@@ -575,9 +577,9 @@ Left unbounded, `snapshots/<hostname>/` grows forever — every prior full DB du
- **New installs default to 30 completed snapshots per source.** The default ships as `backup.retentionCount: 30` in `data.reference/settings.json`, copied only into an install that has no `data/settings.json` yet — see the comment on `resolveRetentionCount()` for why this, not a migration, is what keeps an existing install from silently losing its archive. Operators choose 1–365, or Unlimited, from the Backup settings tab.
- **`retentionCount` absent or explicitly `null` both mean unlimited** — no pruning runs. This is why an install that predates this setting keeps every snapshot until the operator saves a choice: it has no stored value, and absence resolves to unlimited, not to the new-install default.
- **Pruning runs only after a run reaches a completed snapshot with a usable database dump** (`pg_dump` succeeded, or was explicitly skipped by the file-backend escape hatch). A degraded run whose database dump failed keeps older snapshots and reports zero pruned snapshots; retention resumes after a later successful dump. A run that fails before completion never prunes, and a prune failure is logged without failing an otherwise-successful backup.
-- **Pruning is scoped to the CURRENT machine's namespace** (`snapshots/<hostname>/`) and only ever deletes snapshots whose `snapshotState()` reports neither `incomplete` nor `failed`. It never touches another machine's namespace in a shared destination, the legacy pre-namespace root, or an in-progress/failed snapshot — those stay until the operator deletes them explicitly. It also skips a snapshot that a restore or download is still reading (see [Snapshot source lifetime](#how-restore-works)); the skipped snapshot is not counted as pruned and becomes a candidate again on a later run.
+- **Pruning is scoped to the CURRENT machine's namespace** (`snapshots/<hostname>/`) and only ever deletes snapshots whose `snapshotState()` reports neither `incomplete` nor `failed`, plus interrupted snapshots older than the newest healthy completed one. Interrupted snapshots never count toward `retentionCount` and are pruned even under unlimited retention, since they can never be restored or downloaded. It never touches another machine's namespace in a shared destination, the legacy pre-namespace root, or an in-progress/ordinary failed snapshot — those stay until the operator deletes them explicitly. It also skips a snapshot that a restore or download is still reading (see [Snapshot source lifetime](#how-restore-works)); the skipped snapshot is not counted as pruned and becomes a candidate again on a later run.
- **Unreadable or structurally invalid manifests are protected in addition to the retention count.** Retention uses the same field, count, path and SHA-256-shape validator as file restore, without hashing old payloads. Existing JSON `null`, arrays, malformed objects and invalid entries are not legacy absence. They never displace structurally valid completed snapshots, even when an incorrect clock gave a protected snapshot a future-dated ID. Snapshot IDs determine age; manifest timestamps do not change that ordering. Legacy completed snapshots with no manifest remain eligible for retention as before.
-- **`DELETE /api/backup/snapshots/:snapshotId?source=<source>`** (`backup.deleteSnapshot()`) permanently removes exactly the selected source/ID pair, immediately — there is no undo and it is never automatic. It shares `resolveSnapshotPath()`'s path-traversal and symlink guards with restore/download, and refuses a snapshot that is still being written (`SNAPSHOT_INCOMPLETE`) or that a restore or download is still reading (`SNAPSHOT_IN_USE`); unlike restore, a `.failed` snapshot **is** deletable. The Backup settings tab's snapshot history exposes this as a per-row delete action behind a confirmation dialog.
+- **`DELETE /api/backup/snapshots/:snapshotId?source=<source>`** (`backup.deleteSnapshot()`) permanently removes exactly the selected source/ID pair, immediately — there is no undo and it is never automatic. It shares `resolveSnapshotPath()`'s path-traversal and symlink guards with restore/download, and refuses a snapshot that is still being written (`SNAPSHOT_INCOMPLETE`) or that a restore or download is still reading (`SNAPSHOT_IN_USE`); unlike restore, a `.failed` or interrupted snapshot **is** deletable. The Backup settings tab's snapshot history exposes this as a per-row delete action behind a confirmation dialog.
## Scheduling & status
diff --git a/server/services/backup.js b/server/services/backup.js
index 950ecfa237..7ecf6ae4d7 100644
--- a/server/services/backup.js
+++ b/server/services/backup.js
@@ -9,7 +9,7 @@
import { dashboardEvents } from './dashboardEvents.js';
import { spawn } from '../lib/childProcess.js';
import { killWithEscalation } from '../lib/killWithEscalation.js';
-import { access, lstat, mkdir, mkdtemp, readdir, realpath, rm, stat, unlink, writeFile } from 'fs/promises';
+import { access, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rm, stat, unlink, writeFile } from 'fs/promises';
import { randomUUID } from 'node:crypto';
import { PassThrough } from 'node:stream';
import { hostname, tmpdir } from 'os';
@@ -75,8 +75,17 @@ const STATE_PATH = join(PATHS.data, 'backup', 'state.json');
// A finished failure replaces these guards with `.failed`: downloads remain
// available for manual salvage, while restore paths can distinguish the partial
// tree from both completed and legacy snapshots after a restart.
+// Each in-progress marker carries the writing process's start id. Only the
+// server process runs `runBackup`, so in this host's namespace a marker with
+// any other token (or the legacy empty marker) belongs to a killed run: that
+// snapshot is `interrupted` — never restorable or downloadable, but deletable,
+// pruned once a newer complete snapshot exists, and converged to `.failed`
+// with INTERRUPTED_FAILURE_REASON by the next run (#12618).
const SNAPSHOT_IN_PROGRESS_MARKER = '.in-progress';
const SNAPSHOT_FAILED_MARKER = '.failed';
+const BACKUP_PROCESS_START_ID = randomUUID();
+const INTERRUPTED_FAILURE_REASON = 'interrupted by restart';
+const MARKER_UNREADABLE = Symbol('marker-unreadable');
let activeSnapshotId = null;
const markerPath = (snapshotDir) => join(snapshotDir, SNAPSHOT_IN_PROGRESS_MARKER);
@@ -88,50 +97,64 @@ const markerExistsAt = (path) =>
() => true,
(err) => err?.code === 'ENOENT' || err?.code === 'ENOTDIR' ? false : true,
);
-const markerExists = (snapshotDir, snapshotId) =>
- Promise.all([
- markerExistsAt(markerPath(snapshotDir)),
- snapshotId
- ? markerExistsAt(parentMarkerPath(snapshotDir, snapshotId))
- : false,
- ]).then(([snapshotMarker, parentMarker]) => snapshotMarker || parentMarker);
-
-const failedMarkerExists = (snapshotDir) =>
- markerExistsAt(failedMarkerPath(snapshotDir));
+// null when absent, the marker's text when readable, MARKER_UNREADABLE when it
+// exists (or cannot be proven absent) but its owner cannot be read.
+const readMarkerAt = async (path) => {
+ if (!await markerExistsAt(path)) return null;
+ return readFile(path, 'utf8').catch(() => MARKER_UNREADABLE);
+};
async function snapshotState(snapshotDir, snapshotId, currentSource = true) {
- const [markedInProgress, failed] = await Promise.all([
- markerExists(snapshotDir, snapshotId),
- failedMarkerExists(snapshotDir),
+ const [snapshotMarker, parentMarker, failedMarker] = await Promise.all([
+ readMarkerAt(markerPath(snapshotDir)),
+ snapshotId ? readMarkerAt(parentMarkerPath(snapshotDir, snapshotId)) : null,
+ readMarkerAt(failedMarkerPath(snapshotDir)),
]);
+ const inProgressMarkers = [snapshotMarker, parentMarker].filter(marker => marker !== null);
+ const failed = failedMarker !== null;
+ const active = currentSource && snapshotId === activeSnapshotId;
+ // Another machine's namespace stays `incomplete`: its run may still be
+ // live. An unreadable owner, or one this process wrote (the reservation
+ // window before `activeSnapshotId` is set), is never presumed dead.
+ const abandoned = currentSource && !active && !failed && inProgressMarkers.length > 0
+ && inProgressMarkers.every(marker => marker !== MARKER_UNREADABLE && marker !== BACKUP_PROCESS_START_ID);
return {
failed,
// Once `.failed` exists the run is finished even if marker cleanup was
// interrupted. The durable failed state still blocks every restore.
- incomplete: (currentSource && snapshotId === activeSnapshotId) || (markedInProgress && !failed),
+ incomplete: active || (inProgressMarkers.length > 0 && !failed && !abandoned),
+ interrupted: abandoned || (!active && failedMarker === INTERRUPTED_FAILURE_REASON),
};
}
-/** Reject a snapshot that is still being written before any consumer reads it. */
+const snapshotInterruptedError = (snapshotId) =>
+ new ServerError(`Snapshot backup was interrupted by a restart: ${snapshotId}. Delete it or choose a completed backup.`, {
+ status: 409,
+ code: 'SNAPSHOT_INTERRUPTED',
+ });
+
+/** Reject a snapshot that is still being written, or was abandoned mid-write, before any consumer reads it. */
async function assertSnapshotComplete(snapshotDir, snapshotId, currentSource) {
- const { incomplete } = await snapshotState(snapshotDir, snapshotId, currentSource);
+ const { incomplete, interrupted } = await snapshotState(snapshotDir, snapshotId, currentSource);
if (incomplete) {
throw new ServerError(`Snapshot is still being written: ${snapshotId}`, {
status: 409,
code: 'SNAPSHOT_INCOMPLETE',
});
}
+ if (interrupted) throw snapshotInterruptedError(snapshotId);
}
/** Reject snapshots whose backup run finished unsuccessfully before restoring. */
async function assertSnapshotRestorable(snapshotDir, snapshotId, currentSource) {
- const { incomplete, failed } = await snapshotState(snapshotDir, snapshotId, currentSource);
+ const { incomplete, failed, interrupted } = await snapshotState(snapshotDir, snapshotId, currentSource);
if (incomplete) {
throw new ServerError(`Snapshot is still being written: ${snapshotId}`, {
status: 409,
code: 'SNAPSHOT_INCOMPLETE',
});
}
+ if (interrupted) throw snapshotInterruptedError(snapshotId);
if (failed) {
throw new ServerError(`Snapshot backup failed: ${snapshotId}. Choose a completed backup to restore.`, {
status: 409,
@@ -689,6 +712,11 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
const timestamp = new Date().toISOString().replace(/:/g, '-').replace(/\..+/, '');
const snapshotsRoot = join(destPath, 'snapshots', MACHINE_HOST);
await ensureDir(snapshotsRoot);
+ // Inside the single-run guard, so no run of this process can own a marker
+ // the sweep is converging.
+ await markInterruptedSnapshotsFailed(snapshotsRoot).catch(err => {
+ console.error(`❌ Backup interrupted-snapshot sweep failed: code=${err?.code ?? 'UNKNOWN'}`);
+ });
// Own the durable guard before exposing a directory. Never adopt an
// existing tree, even when its timestamp matches this run's clock.
for (let attempt = 0; attempt < 100; attempt += 1) {
@@ -700,7 +728,7 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
throw err;
});
if (exists) continue;
- const reserved = await writeFile(candidateMarker, '', { flag: 'wx' }).then(() => true, err => {
+ const reserved = await writeFile(candidateMarker, BACKUP_PROCESS_START_ID, { flag: 'wx' }).then(() => true, err => {
if (err.code === 'EEXIST') return false;
throw err;
});
@@ -725,7 +753,7 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
if (io) io.emit('backup:started', { snapshotId });
await ensureDir(dataDestDir);
activeSnapshotId = snapshotId;
- await writeFile(markerPath(snapshotDir), '');
+ await writeFile(markerPath(snapshotDir), BACKUP_PROCESS_START_ID);
dashboardEvents.emit('backup:changed');
// Freeze admitted file-plus-row publications across both stores. A writer
@@ -796,7 +824,7 @@ export async function runBackup(destPath, io = null, { excludePaths = [], disabl
// (via `snapshotState`) and drop out of the retention count entirely,
// silently keeping one snapshot MORE than configured. A run that throws
// before this point takes the `fail()` path below and never reaches
- // here, so a failed or in-progress snapshot is never a prune candidate.
+ // here, so this run's own snapshot is never pruned as failed or in-progress.
// Retention failures must not fail an otherwise-successful backup.
let pruned = { pruned: 0 };
if (pgResult.status === 'failed') {
@@ -1039,7 +1067,7 @@ function inventoryReadError(cause, operation) {
/**
* List all snapshots in the backup destination.
* @param {string} destPath - Path to external drive backup root
- * @returns {Array<{ id, source, selectionKey, createdAt, fileCount, incomplete, failed }>} sorted newest-first
+ * @returns {Array<{ id, source, selectionKey, createdAt, fileCount, incomplete, failed, interrupted }>} sorted newest-first
*/
export async function listSnapshots(destPath) {
if (!destPath) return [];
@@ -1089,7 +1117,7 @@ export async function listSnapshots(destPath) {
// Report a still-being-written snapshot rather than hiding it: the row is
// real and the user should see the run in flight, but download and restore
// must not be offered for it. Mirrors assertSnapshotComplete's two signals.
- const { incomplete, failed } = await snapshotState(snapshotDir, id, currentSource);
+ const { incomplete, failed, interrupted } = await snapshotState(snapshotDir, id, currentSource);
return {
id,
source,
@@ -1103,6 +1131,7 @@ export async function listSnapshots(destPath) {
fileCount: manifest?.fileCount ?? 0,
incomplete,
failed,
+ interrupted,
};
})
);
@@ -1116,24 +1145,57 @@ export async function listSnapshots(destPath) {
});
}
+/**
+ * Converge every interrupted snapshot in this host's namespace to a durable
+ * `.failed` (reason INTERRUPTED_FAILURE_REASON), then drop its in-progress
+ * markers — the same order `fail()` uses, so a failed write leaves the
+ * snapshot guarded. Holds the exclusive lease so a concurrent delete or
+ * reader cannot interleave with the marker writes.
+ */
+async function markInterruptedSnapshotsFailed(snapshotsRoot) {
+ const entries = await readdir(snapshotsRoot, { withFileTypes: true });
+ let marked = 0;
+ for (const entry of entries) {
+ if (!entry.isDirectory() || entry.name.startsWith('.') || !SNAPSHOT_ID_PATTERN.test(entry.name)) continue;
+ const id = entry.name;
+ const snapshotDir = join(snapshotsRoot, id);
+ const { interrupted, failed } = await snapshotState(snapshotDir, id, true);
+ if (!interrupted || failed) continue;
+ const { release, refusal } = await claimSnapshotLease(snapshotDir, 'delete');
+ if (refusal) continue;
+ const removeIfPresent = (path) => unlink(path).catch(err => { if (err.code !== 'ENOENT') throw err; });
+ await writeFile(failedMarkerPath(snapshotDir), INTERRUPTED_FAILURE_REASON)
+ .then(() => removeIfPresent(markerPath(snapshotDir)))
+ .then(() => removeIfPresent(parentMarkerPath(snapshotDir, id)))
+ .then(
+ () => { marked += 1; },
+ err => console.error(`❌ Backup: failed to mark interrupted snapshot ${id} failed: code=${err?.code ?? 'UNKNOWN'}`),
+ );
+ release();
+ }
+ if (marked) console.log(`💾 Backup: marked ${marked} interrupted snapshot(s) failed`);
+}
+
/**
* Delete the oldest COMPLETED snapshots in the CURRENT machine's source
- * namespace beyond `retentionCount`. Called only from `runBackup()` after a
- * run reaches a completed snapshot with a successful or explicitly skipped
- * database dump — a run that fails earlier never reaches this call, so an
- * in-progress or `.failed` snapshot is never a candidate; both are re-checked
- * here anyway via `snapshotState` in case an unrelated run left one behind.
- * Only ever walks `snapshots/<MACHINE_HOST>/` — never another machine's
- * namespace and never the legacy pre-namespace root, both of which this
- * install has no authority to prune.
- * `retentionCount` of `null`/`undefined` means unlimited: no-op.
+ * namespace beyond `retentionCount`, plus every interrupted snapshot older
+ * than the newest healthy completed one. Interrupted snapshots never count
+ * toward the budget and are pruned even under unlimited retention: they can
+ * never be restored or downloaded, so they only fill the drive.
+ * Called only from `runBackup()` after a run reaches a completed snapshot
+ * with a successful or explicitly skipped database dump — a run that fails
+ * earlier never reaches this call, so an in-progress snapshot is never a
+ * candidate; state is re-checked here anyway via `snapshotState` in case an
+ * unrelated run left one behind. Ordinary `.failed` snapshots are kept for
+ * salvage. Only ever walks `snapshots/<MACHINE_HOST>/` — never another
+ * machine's namespace and never the legacy pre-namespace root, both of which
+ * this install has no authority to prune.
+ * `retentionCount` of `null`/`undefined` means unlimited completed snapshots.
* @param {string} destPath
* @param {number|null} retentionCount
* @returns {Promise<{ pruned: number }>}
*/
async function pruneOldSnapshots(destPath, retentionCount) {
- if (retentionCount === null || retentionCount === undefined) return { pruned: 0 };
-
const sourceRoot = join(destPath, 'snapshots', MACHINE_HOST);
// A completed snapshot was just written here; even ENOENT is a failed
// inventory, not proof that there is nothing to prune. Keep paths private.
@@ -1146,7 +1208,8 @@ async function pruneOldSnapshots(destPath, retentionCount) {
const descriptors = (await Promise.all(candidateIds.map(async (id) => {
const snapshotDir = join(sourceRoot, id);
- const { incomplete, failed } = await snapshotState(snapshotDir, id, true);
+ const { incomplete, failed, interrupted } = await snapshotState(snapshotDir, id, true);
+ if (interrupted) return { id, snapshotDir, interrupted: true };
if (incomplete || failed) return null;
// Share restore's structural admission, without hashing old payloads.
// Only confirmed absence is legacy; JSON null is an invalid manifest.
@@ -1162,13 +1225,20 @@ async function pruneOldSnapshots(destPath, retentionCount) {
// Keep protected snapshots in addition to the readable retention budget,
// even when a bad clock placed a protected snapshot ahead of healthy ones.
- const toDelete = descriptors.filter(d => {
+ const healthy = descriptors.filter(d => {
+ if (d.interrupted) return false;
if (d.protectedFromPrune) {
console.warn(`⚠️ Backup retention: kept snapshot ${d.id} (manifest unreadable or invalid)`);
return false;
}
return true;
- }).slice(retentionCount);
+ });
+ // An interrupted snapshot is only dropped once a newer healthy one exists.
+ const newestHealthyId = healthy[0]?.id;
+ const toDelete = [
+ ...(newestHealthyId ? descriptors.filter(d => d.interrupted && d.id.localeCompare(newestHealthyId) < 0) : []),
+ ...(retentionCount === null || retentionCount === undefined ? [] : healthy.slice(retentionCount)),
+ ];
let pruned = 0;
for (const { snapshotDir, id } of toDelete) {
// A snapshot a restore or download is reading stays; a later run prunes it
@@ -1185,7 +1255,7 @@ async function pruneOldSnapshots(destPath, retentionCount) {
release();
}
if (pruned) {
- console.log(`💾 Backup retention: pruned ${pruned} snapshot(s), keeping ${retentionCount}`);
+ console.log(`💾 Backup retention: pruned ${pruned} snapshot(s), keeping ${retentionCount ?? 'all'} completed`);
}
return { pruned };
}
@@ -1267,11 +1337,11 @@ async function assertExplicitSnapshotSourceSafe({
* action, never automatic retention (see `pruneOldSnapshots`, which shares the
* traversal/symlink guards through `resolveSnapshotPath` but is driven by
* `runBackup()` instead). Refuses a snapshot that is still being written
- * (`.in-progress`), matching the guard that blocks restore and download,
+ * (`incomplete`), matching the guard that blocks restore and download,
* because the partial directory may still be owned by an in-flight
- * `runBackup()`. A `.failed` snapshot IS deletable — unlike restore, deletion
- * has no reason to require a successful backup. A snapshot a restore or
- * download is still reading is refused with `SNAPSHOT_IN_USE`.
+ * `runBackup()`. A `.failed` or interrupted snapshot IS deletable — unlike
+ * restore, deletion has no reason to require a successful backup. A snapshot
+ * a restore or download is still reading is refused with `SNAPSHOT_IN_USE`.
* @param {string} destPath - Path to external drive backup root
* @param {string} snapshotId - Snapshot ID to delete
* @param {{ source?: string }} [options]
@@ -2291,13 +2361,12 @@ export function getNextRunTime() {
* system-idle gate. Deliberately reads the in-process `isRunning` flag rather
* than scanning for an on-disk `.in-progress` marker: that marker is designed
* to OUTLIVE a crash/PM2 restart so a restore never mistakes a partial
- * snapshot for a complete one, but nothing ever clears an orphaned one on its
- * own (the next run writes a fresh snapshot id) — gating the updater on its
- * mere existence would block it forever after a single interrupted backup on
- * a drive that might not even be mounted. The gate only needs to know whether
- * restarting would kill a live rsync/pg_dump in this server instance, and
- * `isRunning` answers exactly that, with no I/O and nothing that can fail to
- * read.
+ * snapshot for a complete one, and only the next run converges an orphaned
+ * one — gating the updater on its mere existence would block it until then
+ * after a single interrupted backup on a drive that might not even be
+ * mounted. The gate only needs to know whether restarting would kill a live
+ * rsync/pg_dump in this server instance, and `isRunning` answers exactly
+ * that, with no I/O and nothing that can fail to read.
*/
export function isBackupInProgress() {
return isRunning;
diff --git a/server/services/backup.test.js b/server/services/backup.test.js
index 1c7d5ea49b..84dcc66bc6 100644
--- a/server/services/backup.test.js
+++ b/server/services/backup.test.js
@@ -184,6 +184,7 @@ beforeEach(async () => {
const actual = await vi.importActual('fs/promises');
fs.readdir.mockImplementation(actual.readdir);
fs.access.mockImplementation(actual.access);
+ fs.readFile.mockImplementation(actual.readFile);
fs.unlink.mockImplementation(actual.unlink);
fs.writeFile.mockImplementation(actual.writeFile);
});
@@ -651,11 +652,11 @@ describe('deleteSnapshot', () => {
await fs.rm(destRoot, { recursive: true, force: true });
});
- async function makeSnapshot(source, id, { marker } = {}) {
+ async function makeSnapshot(source, id, { marker, markerContent = '' } = {}) {
const dir = joinPath(destRoot, 'snapshots', source, id);
await fs.mkdir(joinPath(dir, 'data'), { recursive: true });
await fs.writeFile(joinPath(dir, 'manifest.json'), JSON.stringify({ generatedAt: '2026-01-01T00:00:00Z', fileCount: 0 }));
- if (marker) await fs.writeFile(joinPath(dir, marker), '');
+ if (marker) await fs.writeFile(joinPath(dir, marker), markerContent);
return dir;
}
@@ -681,9 +682,32 @@ describe('deleteSnapshot', () => {
.rejects.toMatchObject({ status: 404, code: 'NOT_FOUND' });
});
- it('refuses a snapshot that is still being written (409 SNAPSHOT_INCOMPLETE), leaving it on disk', async () => {
- const dir = await makeSnapshot(machineHost, '2026-06-08T15-18-34', { marker: '.in-progress' });
- await expect(deleteSnapshot(destRoot, '2026-06-08T15-18-34', { source: machineHost }))
+ // Only the server process runs runBackup, so in this machine's namespace an
+ // in-progress marker another process wrote (or the legacy empty one) is a
+ // run that a restart killed (#12618).
+ it.each([
+ ['the legacy empty marker', ''],
+ ['another process token', 'previous-process-token'],
+ ])('reports a dead run left by %s as interrupted: never restorable or downloadable, but deletable', async (_label, markerContent) => {
+ const id = '2026-06-08T15-18-34';
+ const dir = await makeSnapshot(machineHost, id, { marker: '.in-progress', markerContent });
+
+ const [snapshot] = await listSnapshots(destRoot);
+ expect(snapshot).toMatchObject({ id, incomplete: false, failed: false, interrupted: true });
+ await expect(openSnapshotStream(destRoot, id, { source: machineHost }))
+ .rejects.toMatchObject({ status: 409, code: 'SNAPSHOT_INTERRUPTED' });
+ await expect(restoreSnapshot(destRoot, id, { source: machineHost, dryRun: true }))
+ .rejects.toMatchObject({ status: 409, code: 'SNAPSHOT_INTERRUPTED' });
+
+ await deleteSnapshot(destRoot, id, { source: machineHost });
+ await expect(fs.stat(dir)).rejects.toThrow();
+ });
+
+ it("keeps another machine's in-progress snapshot incomplete (409 SNAPSHOT_INCOMPLETE), leaving it on disk", async () => {
+ const dir = await makeSnapshot('previous-machine', '2026-06-08T15-18-34', { marker: '.in-progress' });
+ const [snapshot] = await listSnapshots(destRoot);
+ expect(snapshot).toMatchObject({ incomplete: true, interrupted: false });
+ await expect(deleteSnapshot(destRoot, '2026-06-08T15-18-34', { source: 'previous-machine' }))
.rejects.toMatchObject({ status: 409, code: 'SNAPSHOT_INCOMPLETE' });
await expect(fs.stat(dir)).resolves.toBeDefined();
});
@@ -730,12 +754,14 @@ describe('openSnapshotStream', () => {
return proc;
}
- // Same directory, but a marker left behind by a crashed run.
- function crashedSnapshot() {
+ // Same directory, but an in-progress marker whose owner cannot be read: it
+ // is never presumed dead, so it stays incomplete.
+ function unreadableMarkerSnapshot() {
fs.stat.mockResolvedValue({ isDirectory: () => true });
fs.access.mockImplementation((path) => String(path).endsWith('.failed')
? Promise.reject(Object.assign(new Error('missing'), { code: 'ENOENT' }))
: Promise.resolve()); // .in-progress present
+ fs.readFile.mockRejectedValue(Object.assign(new Error('unreadable'), { code: 'EIO' }));
}
const ended = (stream) => new Promise((resolveEnd) => stream.once('end', resolveEnd));
@@ -777,10 +803,8 @@ describe('openSnapshotStream', () => {
}
});
- it('refuses a snapshot whose .in-progress marker survived a crash', async () => {
- // activeSnapshotId is module state and resets on restart; the marker is what
- // still says "this tree was never finished" after PM2 restarts mid-backup.
- crashedSnapshot();
+ it('refuses a snapshot whose .in-progress marker owner cannot be read', async () => {
+ unreadableMarkerSnapshot();
await expect(openSnapshotStream('/dest', 'snap-1'))
.rejects.toMatchObject({ status: 409, code: 'SNAPSHOT_INCOMPLETE' });
@@ -4164,8 +4188,10 @@ describe('runBackup lifecycle', () => {
expect(await pending).toBeInstanceOf(Error);
if (state !== 'reservation') expect(await fsp.readFile(joinPath(oldDir, 'portos-db.sql'), 'utf8')).toBe('old dump');
if (state === 'failed') expect(await fsp.readFile(joinPath(oldDir, '.failed'), 'utf8')).toBe('old failure');
- if (state === 'interrupted' || state === 'reservation') expect(await fsp.readFile(guard, 'utf8')).toBe('other reservation');
+ if (state === 'reservation') expect(await fsp.readFile(guard, 'utf8')).toBe('other reservation');
else await expect(fsp.access(guard)).rejects.toThrow();
+ // The new run converged the dead run's snapshot to a durable failure.
+ if (state === 'interrupted') expect(await fsp.readFile(joinPath(oldDir, '.failed'), 'utf8')).toBe('interrupted by restart');
} finally {
vi.useRealTimers();
}
@@ -4617,6 +4643,9 @@ describe('runBackup lifecycle', () => {
.rejects.toMatchObject({ status: 409, code: 'SNAPSHOT_INCOMPLETE' });
await expect(restore(destRoot, snapshotId, { dryRun: true }))
.rejects.toMatchObject({ status: 409, code: 'SNAPSHOT_INCOMPLETE' });
+ const { deleteSnapshot: remove } = await import('./backup.js');
+ await expect(remove(destRoot, snapshotId))
+ .rejects.toMatchObject({ status: 409, code: 'SNAPSHOT_INCOMPLETE' });
// Only the rsync — neither guard reached tar or a second rsync.
expect(spawn).toHaveBeenCalledTimes(1);
@@ -4748,11 +4777,12 @@ describe('runBackup lifecycle', () => {
expect(io.emit.mock.calls.some(([event]) => event === 'backup:completed')).toBe(false);
await expect(fsp.access(joinPath(destRoot, 'snapshots', machineHost, `.${snapshotId}.in-progress`)))
.resolves.toBeUndefined();
- // The on-disk guard, not process state, must block restore after restart.
+ // The on-disk guard, not process state, must block restore after restart,
+ // where the dead process's token makes the snapshot interrupted.
vi.resetModules();
const { restoreSnapshot: restoreAfterRestart } = await import('./backup.js');
await expect(restoreAfterRestart(destRoot, snapshotId, { dryRun: true }))
- .rejects.toMatchObject({ code: 'SNAPSHOT_INCOMPLETE' });
+ .rejects.toMatchObject({ code: 'SNAPSHOT_INTERRUPTED' });
});
it('retains incomplete guards when the failed marker cannot be written while releasing the process lock', async () => {
@@ -4979,29 +5009,53 @@ describe('runBackup lifecycle', () => {
// -----------------------------------------------------------------------
// Retention pruning (#8334) — after a dump-successful run, the oldest COMPLETED
- // snapshots on THIS machine beyond retentionCount are deleted. Failed,
- // in-progress, and other-source snapshots must never be touched, and
- // retentionCount null/undefined must never prune anything (an existing
- // install with no stored value keeps everything — see backupConfig.js).
+ // snapshots on THIS machine beyond retentionCount are deleted. Failed and
+ // other-source snapshots must never be touched, and retentionCount
+ // null/undefined must never prune a completed snapshot (an existing install
+ // with no stored value keeps everything — see backupConfig.js). Interrupted
+ // snapshots older than the newest completed one are always pruned (#12618).
// -----------------------------------------------------------------------
describe('retention pruning', () => {
- async function seedSnapshot(source, id, { generatedAt, marker } = {}) {
+ async function seedSnapshot(source, id, { generatedAt, marker, markerContent = '' } = {}) {
const fsp = await actualFs();
const dir = joinPath(destRoot, 'snapshots', source, id);
await fsp.mkdir(joinPath(dir, 'data'), { recursive: true });
if (generatedAt) {
await fsp.writeFile(joinPath(dir, 'manifest.json'), JSON.stringify({ generatedAt, fileCount: 0, files: {} }));
}
- if (marker) await fsp.writeFile(joinPath(dir, marker), '');
+ if (marker) await fsp.writeFile(joinPath(dir, marker), markerContent);
return dir;
}
- it('prunes only the oldest completed snapshots in the current machine namespace, keeping failed/in-progress/other-source snapshots', async () => {
+ async function completeRun(options) {
+ const rsync = fakeProc();
+ spawn.mockReturnValue(rsync);
+ const pending = runBackup(destRoot, null, options);
+ await waitFor(() => spawn.mock.calls.length === 1, 'rsync spawn');
+ rsync.emit('close', 0);
+ return pending;
+ }
+
+ it('prunes interrupted snapshots older than the new completed one even under unlimited retention', async () => {
+ const fsp = await actualFs();
+ const completed = await seedSnapshot(machineHost, '2020-01-01T00-00-00', { generatedAt: '2020-01-01T00:00:00.000Z' });
+ const interrupted = await seedSnapshot(machineHost, '2020-01-02T00-00-00', { marker: '.in-progress', markerContent: 'previous-process-token' });
+ const otherMachine = await seedSnapshot('previous-machine', '2020-01-02T00-00-00', { marker: '.in-progress' });
+
+ const result = await completeRun({ retentionCount: null });
+
+ expect(result).toMatchObject({ status: 'ok', prunedSnapshots: 1 });
+ await expect(fsp.stat(interrupted)).rejects.toThrow();
+ await expect(fsp.stat(completed)).resolves.toBeDefined();
+ await expect(fsp.stat(otherMachine)).resolves.toBeDefined();
+ });
+
+ it('prunes only the oldest completed snapshots in the current machine namespace, keeping failed/other-source snapshots', async () => {
const fsp = await actualFs();
const old1 = await seedSnapshot(machineHost, '2020-01-01T00-00-00', { generatedAt: '2020-01-01T00:00:00.000Z' });
const old2 = await seedSnapshot(machineHost, '2020-01-02T00-00-00', { generatedAt: '2020-01-02T00:00:00.000Z' });
const old3 = await seedSnapshot(machineHost, '2020-01-03T00-00-00', { generatedAt: '2020-01-03T00:00:00.000Z' });
- const incomplete = await seedSnapshot(machineHost, '2020-01-04T00-00-00', { marker: '.in-progress' });
+ const interrupted = await seedSnapshot(machineHost, '2020-01-04T00-00-00', { marker: '.in-progress' });
const failed = await seedSnapshot(machineHost, '2020-01-05T00-00-00', { generatedAt: '2020-01-05T00:00:00.000Z', marker: '.failed' });
const otherMachine = await seedSnapshot('previous-machine', '2020-01-01T00-00-00', { generatedAt: '2020-01-01T00:00:00.000Z' });
@@ -5019,18 +5073,19 @@ describe('runBackup lifecycle', () => {
await fsp.writeFile(dumpPath, 'CREATE TABLE example (id integer);');
pgDump.emit('close', 0);
const result = await pending;
- expect(result).toMatchObject({ status: 'ok', pgBackup: { status: 'ok' }, prunedSnapshots: 2 });
+ expect(result).toMatchObject({ status: 'ok', pgBackup: { status: 'ok' }, prunedSnapshots: 3 });
// Keeps the newest 2 completed snapshots on this machine: the run just
- // created and old3. old1 and old2 are pruned.
+ // created and old3. old1 and old2 are pruned, and the interrupted one
+ // goes without spending the budget.
await expect(fsp.stat(old1)).rejects.toThrow();
await expect(fsp.stat(old2)).rejects.toThrow();
await expect(fsp.stat(old3)).resolves.toBeDefined();
const newSnapshotDir = joinPath(destRoot, 'snapshots', machineHost, result.snapshotId);
await expect(fsp.stat(newSnapshotDir)).resolves.toBeDefined();
- // Never touched: still-being-written, failed, and another machine's namespace.
- await expect(fsp.stat(incomplete)).resolves.toBeDefined();
+ await expect(fsp.stat(interrupted)).rejects.toThrow();
+ // Never touched: failed, and another machine's namespace.
await expect(fsp.stat(failed)).resolves.toBeDefined();
await expect(fsp.stat(otherMachine)).resolves.toBeDefined();
});