Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

131 KB
Raw Blame
/**
 * Backup Service
 *
 * Rsync-based incremental backup from ./data/ to an external drive.
 * Generates SHA-256 manifests for integrity verification.
 * Integrates with eventScheduler for daily cron scheduling.
 */

import { dashboardEvents } from './dashboardEvents.js';
import { spawn } from '../lib/childProcess.js';
import { killWithEscalation } from '../lib/killWithEscalation.js';
import { access, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, stat, unlink, writeFile } from 'fs/promises';
import { randomUUID } from 'node:crypto';
import { PassThrough } from 'node:stream';
import { hostname, tmpdir } from 'os';
import { basename, dirname, join, resolve, relative, isAbsolute } from 'path';
import { PATHS, ensureDir, readJSONFile, readJSONFileStrict, atomicWrite, sha256File } from '../lib/fileUtils.js';
import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
import { acquireBackupSnapshotCut } from '../lib/backupSnapshotBoundary.js';
import { backupAssetConsistency } from '../lib/backupAssetOwners.js';
import { assertDatabaseAdmission } from '../lib/databaseMaintenanceJournal.js';
import { createLineReader } from '../lib/streamLines.js';
import { getEvent } from './eventScheduler.js';
import { POOL_CONFIG, checkHealth, databaseRestoreRecovery, getServerMajorVersion, query, withDatabaseMaintenance } from '../lib/db.js';
import { withPgToolEnv, resolvePgDumpBinary } from '../lib/pgTools.js';
import { inspectDatabaseDump, prepareDatabaseReplay } from './backupDatabaseDump.js';
import {
  captureSyncFeedPositions, pendingRecoveryResult, repairCommittedRestore, restoreApplicationName, restoreReceiptSql,
  restoreRecoveryRefusal, settleReplayOutcome,
} from './backupRestoreRecovery.js';
import { getBackendName } from './memoryBackend.js';
import { APPLIED_LEDGER_RELPATH, LEGACY_APPLIED_LEDGER_RELPATH } from '../../scripts/lib/migrationLedger.js';
import { emitErrorEvent, ServerError } from '../lib/errorHandler.js';
import { isSafeSnapshotSource, isSafeSubdirFilter, anchorUserExcludes } from '../lib/sharedSchemas.js';
import { reloadSettings, withLiveSettingsRestore } from './settings.js';
import { invalidateAllCaches as invalidateBrainCaches } from './brainStorage.js';
import { convertRestoredLegacySplitStores, describeLegacySplitStores, snapshotLegacySplitStores } from './backupLegacyStores.js';
import { noteSystemActivity } from './systemActivityNotify.js';

// Module-level state
let isRunning = false;
let failedStateProjection = null;

// The in-process lock is the activity signal the updater reads. Note the edge
// in the same assignment so a client cannot keep a stale "backup running"
// verdict after the lock drops, or miss one that just started. There is no
// cancel path: a failed run releases the lock as `failure`.
function setBackupRunning(running, phase) {
  isRunning = running;
  noteSystemActivity('backup', phase);
}

// Backups and restores can legitimately run for hours on large or remote
// volumes, so a short elapsed-time cap would turn healthy work into failure.
// Treat ten minutes with no observable progress as a stall, while retaining a
// generous hard ceiling for a process that stays noisy forever.
const BACKUP_PROCESS_IDLE_TIMEOUT_MS = 10 * 60 * 1000;
const BACKUP_PROCESS_WALL_TIMEOUT_MS = 4 * 60 * 60 * 1000;
const BACKUP_PROCESS_PROGRESS_POLL_MS = 30 * 1000;

// `-ii` heartbeats once per file rsync finishes deciding about, so a single
// file whose `--checksum` digest outlasts the idle deadline still emits
// nothing for its whole duration (#7302). The digest reads BOTH copies — the
// snapshot side and the live side — so a restore floors its idle timeout at
// 2x the largest in-scope file over this throughput. Sized near the bottom of
// a healthy network/iCloud destination (~25 MB/s observed), the floor keeps
// the tight default for small trees and relaxes only where a long silence is
// explainable.
const RESTORE_DIGEST_WORST_CASE_BPS = 10 * 1024 * 1024;

const STATE_PATH = join(PATHS.data, 'backup', 'state.json');
// A snapshot mid-assembly is a truncated tree that looks like a finished backup.
// Two signals guard it, because neither alone is sufficient:
//   - `activeSnapshotId` catches the in-process case with no I/O.
//   - the `.in-progress` marker survives a hard crash or PM2 restart, which
//     resets module state while the partial directory stays on the drive.
// A finished failure replaces these guards with `.failed`: downloads remain
// available for manual salvage, while restore paths can distinguish the partial
// tree from both completed and legacy snapshots after a restart.
// Each in-progress marker carries the writing process's start id. Only the
// server process runs `runBackup`, so in this host's namespace a marker with
// any other token (or the legacy empty marker) belongs to a killed run: that
// snapshot is `interrupted` — never restorable or downloadable, but deletable,
// pruned once a newer complete snapshot exists, and converged to `.failed`
// with INTERRUPTED_FAILURE_REASON by the next run (#12618).
const SNAPSHOT_IN_PROGRESS_MARKER = '.in-progress';
const SNAPSHOT_FAILED_MARKER = '.failed';
const BACKUP_PROCESS_START_ID = randomUUID();
const INTERRUPTED_FAILURE_REASON = 'interrupted by restart';
const MARKER_UNREADABLE = Symbol('marker-unreadable');
let activeSnapshotId = null;

const markerPath = (snapshotDir) => join(snapshotDir, SNAPSHOT_IN_PROGRESS_MARKER);
const failedMarkerPath = (snapshotDir) => join(snapshotDir, SNAPSHOT_FAILED_MARKER);
const parentMarkerPath = (snapshotDir, snapshotId) =>
  join(resolve(snapshotDir, '..'), `.${snapshotId}${SNAPSHOT_IN_PROGRESS_MARKER}`);
const markerExistsAt = (path) =>
  access(path).then(
    () => true,
    (err) => err?.code === 'ENOENT' || err?.code === 'ENOTDIR' ? false : true,
  );
// null when absent, the marker's text when readable, MARKER_UNREADABLE when it
// exists (or cannot be proven absent) but its owner cannot be read.
const readMarkerAt = async (path) => {
  if (!await markerExistsAt(path)) return null;
  return readFile(path, 'utf8').catch(() => MARKER_UNREADABLE);
};

async function snapshotState(snapshotDir, snapshotId, currentSource = true) {
  const [snapshotMarker, parentMarker, failedMarker] = await Promise.all([
    readMarkerAt(markerPath(snapshotDir)),
    snapshotId ? readMarkerAt(parentMarkerPath(snapshotDir, snapshotId)) : null,
    readMarkerAt(failedMarkerPath(snapshotDir)),
  ]);
  const inProgressMarkers = [snapshotMarker, parentMarker].filter(marker => marker !== null);
  const failed = failedMarker !== null;
  const active = currentSource && snapshotId === activeSnapshotId;
  // Another machine's namespace stays `incomplete`: its run may still be
  // live. An unreadable owner, or one this process wrote (the reservation
  // window before `activeSnapshotId` is set), is never presumed dead.
  const abandoned = currentSource && !active && !failed && inProgressMarkers.length > 0
    && inProgressMarkers.every(marker => marker !== MARKER_UNREADABLE && marker !== BACKUP_PROCESS_START_ID);
  return {
    failed,
    // Once `.failed` exists the run is finished even if marker cleanup was
    // interrupted. The durable failed state still blocks every restore.
    incomplete: active || (inProgressMarkers.length > 0 && !failed && !abandoned),
    interrupted: abandoned || (!active && failedMarker === INTERRUPTED_FAILURE_REASON),
  };
}

const snapshotInterruptedError = (snapshotId) =>
  new ServerError(`Snapshot backup was interrupted by a restart: ${snapshotId}. Delete it or choose a completed backup.`, {
    status: 409,
    code: 'SNAPSHOT_INTERRUPTED',
  });

/** Reject a snapshot that is still being written, or was abandoned mid-write, before any consumer reads it. */
async function assertSnapshotComplete(snapshotDir, snapshotId, currentSource) {
  const { incomplete, interrupted } = await snapshotState(snapshotDir, snapshotId, currentSource);
  if (incomplete) {
    throw new ServerError(`Snapshot is still being written: ${snapshotId}`, {
      status: 409,
      code: 'SNAPSHOT_INCOMPLETE',
    });
  }
  if (interrupted) throw snapshotInterruptedError(snapshotId);
}

/** Reject snapshots whose backup run finished unsuccessfully before restoring. */
async function assertSnapshotRestorable(snapshotDir, snapshotId, currentSource) {
  const { incomplete, failed, interrupted } = await snapshotState(snapshotDir, snapshotId, currentSource);
  if (incomplete) {
    throw new ServerError(`Snapshot is still being written: ${snapshotId}`, {
      status: 409,
      code: 'SNAPSHOT_INCOMPLETE',
    });
  }
  if (interrupted) throw snapshotInterruptedError(snapshotId);
  if (failed) {
    throw new ServerError(`Snapshot backup failed: ${snapshotId}. Choose a completed backup to restore.`, {
      status: 409,
      code: 'SNAPSHOT_FAILED',
    });
  }
}

// Snapshot source lifetime (#10898). Verifying snapshot bytes proves nothing if
// the snapshot can be deleted while a consumer is still reading it, so every
// consumer holds a READ lease from before its preflight until its last read
// settles (file restore/preview through rsync's close and reconciliation,
// database restore through dump admission, a download through tar's close),
// and every deletion — explicit or retention — needs the snapshot EXCLUSIVELY.
// Readers share; a deletion refuses while any reader holds it, and a reader
// refuses while a deletion holds it. Keyed by the canonical path (realpath
// folds the omitted-vs-explicit current source, destination symlinks and, on a
// case-insensitive volume, letter case into one owner); a path that does not
// exist keeps its resolved form, which every alias of it shares too. This
// coordinates this server's own callers — it is not a cross-machine lock.
const snapshotLeases = new Map();

const SNAPSHOT_LEASE_REFUSALS = Object.freeze({
  SNAPSHOT_DELETING: (snapshotId) => `Snapshot is being deleted: ${snapshotId}`,
  SNAPSHOT_IN_USE: (snapshotId) => `Snapshot is in use by a restore or download: ${snapshotId}. Retry when it finishes.`,
});

/**
 * Claim `snapshotDir` for `mode` ('read' | 'delete'). Resolves `{ release }`,
 * or `{ refusal }` naming the conflicting owner; never throws. The check and
 * the claim run in one synchronous step after the key is resolved, so two
 * claimants cannot both win.
 */
async function claimSnapshotLease(snapshotDir, mode) {
  // snapshotDir comes from resolveSnapshotPath, so it is already resolved.
  const key = await realpath(snapshotDir).catch(() => snapshotDir);
  const lease = snapshotLeases.get(key) ?? { readers: 0, deleting: false };
  if (lease.deleting) return { refusal: 'SNAPSHOT_DELETING' };
  if (mode === 'delete' && lease.readers > 0) return { refusal: 'SNAPSHOT_IN_USE' };
  if (mode === 'delete') lease.deleting = true;
  else lease.readers += 1;
  snapshotLeases.set(key, lease);
  let held = true;
  return {
    release: () => {
      if (!held) return;
      held = false;
      if (mode === 'delete') lease.deleting = false;
      else lease.readers -= 1;
      if (!lease.deleting && lease.readers === 0) snapshotLeases.delete(key);
    },
  };
}

/** Claim a snapshot lease or throw the structured 409 naming the conflict. */
async function acquireSnapshotLease(snapshotDir, snapshotId, mode) {
  const { release, refusal } = await claimSnapshotLease(snapshotDir, mode);
  if (refusal) {
    throw new ServerError(SNAPSHOT_LEASE_REFUSALS[refusal](snapshotId), {
      status: 409,
      code: refusal,
      context: { snapshotId },
    });
  }
  return release;
}

/** Hold a read lease on `snapshotDir` for exactly as long as `operation` runs. */
async function withSnapshotRead(snapshotDir, snapshotId, operation) {
  const release = await acquireSnapshotLease(snapshotDir, snapshotId, 'read');
  try {
    return await operation();
  } finally {
    release();
  }
}

// Serialize state read-merge-write so two saveState() calls (e.g. a run
// completing while the scheduler stamps a status) can't each read the same
// pre-image and clobber the other's fields. Single tail per shared state file.
const queueStateWrite = createFileWriteQueue();

// Paths under data/ that are skipped by default on top of user-configured excludes.
// Two classes live here: (1) ephemeral/cache data the user almost never wants in a
// snapshot (browser profile, agent worktrees), and (2) large re-downloadable assets
// (LoRA model files, cloned repos, browser downloads) that would bloat the backup
// target — typically iCloud or an external drive with limited capacity. Entries
// tagged `overridable: true` can be re-enabled from the Backup settings UI via
// `disabledDefaultExcludes`; non-overridable entries hold no irreplaceable user data
// and stay off unconditionally. When adding a new entry, ensure the path glob covers
// *every* on-disk location for that class of data — e.g. agent worktrees live under
// both cos/worktrees/ and cos/feature-agents/*/worktree/; cross-reference
// worktreeManager.js and agentLifecycle.js if introducing new worktree paths.
//
// All paths are anchored with a leading `/` (rsync filter syntax for "relative to
// the transfer root"). Without the anchor, a pattern like `loras/*.safetensors`
// matches any `loras/` directory anywhere under data/ (e.g. a user's
// brain/.../loras/ collection), which would silently exclude unrelated user data.
export const DEFAULT_EXCLUDES = [
  { path: '/backup-admission/', reason: 'Machine-local publication ownership and snapshot fences', overridable: false },
  { path: '/image-thumbnails/', reason: 'Regenerable image grid previews', overridable: false },
  { path: '/python/laya-mlx/', reason: 'Rebuildable Laya-MLX experiment runtime and pinned model weights', overridable: false },
  { path: '/browser-profile/', reason: 'Browser CDP profile — cache/cookies, can be several GB', overridable: false },
  { path: '/cos/worktrees/', reason: 'Ephemeral agent git worktrees — recreated on demand', overridable: false },
  { path: '/autofixer/worktrees/', reason: 'Disposable Autofixer repair worktrees — recreated for each repair', overridable: false },
  { path: '/cos/slashdo-resolved/', reason: 'Resolved slashdo command bodies staged for agent prompts — derived from the bundled submodule, regenerated on demand', overridable: false },
  { path: '/cos/feature-agents/*/worktree/', reason: 'Per-feature-agent git worktrees — recreated on demand', overridable: false },
  // `**` (not `*`) so both engines' checkpoint dirs match: the torch trainer
  // writes training-runs/<id>/checkpoints/, mflux writes
  // training-runs/<id>/mflux/checkpoints/.
  { path: '/training-runs/**/checkpoints/', reason: 'LoRA training checkpoints — large intermediate adapter state, resumable-but-regenerable. Deployed adapters in data/loras/, including promoted checkpoints, ARE backed up along with run samples + configs.', overridable: true },
  { path: '/training-runs/*/cache/', reason: 'Precomputed latent/text-embedding training cache — regenerated from the dataset on the next run', overridable: false },
  { path: '/training-runs/*/data/.mflux_cache/', reason: 'mflux low_ram disk-backed encode cache (written inside the staged training data dir) — regenerable', overridable: false },
  { path: '/repos/', reason: 'Cloned git repositories — large, re-cloneable from origin', overridable: true },
  { path: '/cos/reference-repos/', reason: 'Reference upstream repos used by agents — re-cloneable', overridable: true },
  { path: '/browser-downloads/', reason: 'Browser downloads cache — large, re-downloadable', overridable: true },
  { path: '/composition-proofs/', reason: 'HTML-composition contact-sheet proofs — review stills, re-rendered from the composition source', overridable: true },
  { path: '/code-animation-workspaces/', reason: 'In-flight sandboxed Code Animation worker workspaces — removed when each run ends', overridable: false },
  { path: '/code-animation-exports/', reason: 'Code Animation frame-exact export staging — the stored HTML plus render shim, staged once per export', overridable: true },
  { path: '/music-video-compositions/', reason: 'In-flight music-video typography overlay scratch — removed when its render ends and swept at boot', overridable: false },
  { path: '/music-video-film-look/', reason: 'In-flight music-video film look pass (a staged link to the finished render) — removed when the pass ends', overridable: false },
  { path: '/music-video-stats-outro/', reason: 'In-flight music-video outro page (fonts and floor stills) — removed when its render ends and swept at boot', overridable: false },
  { path: '/music-video-segments/', reason: 'Cached music-video shot segments — the final render re-renders any that are missing', overridable: true },
  { path: '/music-video-song-renders/', reason: 'In-flight music-video composition renders (a staged copy of the document plus scene media) — removed when the render ends and swept at boot', overridable: false },
  { path: '/launch-videos/*/*/proofs/', reason: 'Launch-video critique-loop contact sheets — re-rendered from the run composition', overridable: true },
  { path: '/cache/', reason: 'Remote-API metadata and licensed reading caches — regenerable on demand, and stale on restore anyway', overridable: false },
  // Anchored with a leading `/`, like every entry here. The manifest describes
  // which model weights are on THIS machine's disks; restoring it onto another
  // would claim gigabytes of models that machine does not have, and offer delete
  // buttons for them. It is fully re-derivable by a rescan from Models → Status.
  // Anchored, like every entry here. The restore recovery journal (#9725) is an
  // admission fence for THIS machine's in-flight database restore; a restored
  // copy would re-fence the database for an operation that no longer exists.
  { path: '/database-restore-recovery.json', reason: 'In-flight database restore recovery journal — a machine-local admission fence for one restore operation, never data to restore', overridable: false },
  { path: '/.database-restore-recovery-*.pending', reason: 'Unpublished database restore recovery journal bytes — scratch from an interrupted journal write', overridable: false },
  { path: '/model-manifest.json', reason: 'Tracked downloaded-model inventory — machine-local and re-derivable by rescanning the model stores; a restored copy would describe another machine\'s disks', overridable: false },
  // Anchored, like every entry here. Readiness evidence is bound to THIS
  // machine's local image id (a restored copy would read as stale anyway) and
  // the rest is in-flight render scratch; setup re-derives all of it.
  { path: '/supercollider/', reason: 'SuperCollider runtime readiness evidence, in-flight render scratch and 24-hour render previews — machine-local, re-derived by npm run setup:supercollider', overridable: false },
  // Sprite animation-run raw intermediates: 30–96 ffmpeg-extracted PNGs per
  // run, byte-for-byte regenerable from the archived source video by the
  // deterministic postprocess (walkPostprocess.js). The source video, packaged
  // frames, strips, manifests, and runtime atlases ARE backed up. `runs/` is
  // the live (vendor-neutral) layout; `grok/` covers pre-migration-202 runs.
  { path: '/sprites/*/grok/*/generated/raw/', reason: 'Sprite walk-run raw extracted frames — regenerable from the archived source video', overridable: true },
  { path: '/sprites/*/runs/*/generated/raw/', reason: 'Imported sprite-run raw extracted frames — regenerable from the archived source video', overridable: true },
  // Anchored with a leading `/` like every entry here — an unanchored `model.obj`
  // would match at any depth and silently drop unrelated user data.
  //
  // TRELLIS.2's `generate.py` writes this full-resolution OBJ next to the GLB it
  // exports: the decoder's mesh before bake-time decimation, measured at 930 MB /
  // 22.7M faces for one 1024_cascade render. It is regenerable by re-rendering the
  // same source at the same seed, it is not what the 3D page loads (that is
  // model.glb, which IS backed up), and at ~1 GB per render it would otherwise
  // dominate every snapshot. Overridable, because it is the only copy of the
  // discarded detail and someone archiving finished work may want it.
  // Anchored, like every entry here. Capability-test sandboxes are throwaway
  // copies of a fixture that ships in the repo — restoring one would restore a
  // half-finished agent edit, which is worse than not having it.
  { path: '/model-tests/sandboxes/', reason: 'Capability-test agent sandboxes — throwaway working copies of a repo fixture, recreated per run', overridable: false },
  // Anchored, like every entry here. Scratch for ONE in-flight auto-skin run: the
  // worker's staging copy of the rigged GLB plus its job/report files, deleted the
  // moment the run publishes or fails (services/rigging/autoSkin.js). A snapshot that
  // caught it mid-run would restore a half-written mesh next to the published pair.
  // The PUBLISHED rig (rig/<rigId>/) is deliberately NOT excluded: re-deriving it needs
  // a provisioned Blender runtime the restore target may not have, and a rigged GLB is
  // megabytes, not the gigabyte-per-render the model.obj sidecar below costs.
  { path: '/image-to-3d/*/rig/.staging/', reason: 'In-flight auto-skin staging directory — scratch for one rigging run, removed when it publishes or fails. The published rig/<rigId>/ pair IS backed up.', overridable: false },
  // Anchored, like every entry here. Same contract one step later: scratch for ONE
  // in-flight retarget run (services/rigging/retarget.js), including the probe pass's
  // job/report files. The PUBLISHED animation (retarget/<retargetId>/) is NOT excluded,
  // for the same reason the rig is not — re-deriving it needs both a Blender runtime and
  // the clip file the run used.
  { path: '/image-to-3d/*/retarget/.staging/', reason: 'In-flight retarget staging directory — scratch for one retarget run, removed when it publishes or fails. The published retarget/<retargetId>/ pair IS backed up.', overridable: false },
  { path: '/image-to-3d/*/model.obj', reason: 'TRELLIS.2 full-resolution mesh sidecar — ~1 GB per render, regenerable by re-rendering at the same seed. The exported model.glb and keyed source ARE backed up.', overridable: true },
  // Anchored, like every entry here. Not overridable: these are another
  // machine's conditioning bytes, staged for one federated render and swept on
  // a TTL measured in hours. Nothing here is this install's data to keep, and a
  // restored inbox entry is either already expired or already rendered.
  { path: '/federated-media-inbox/', reason: 'Conditioning images an allowlisted peer uploaded for one federated render — TTL-swept, and another machine\'s data rather than this install\'s', overridable: false },
  // Anchored, like every entry here. The Beeper attachment mirror (#37) is a
  // lazy CACHE of bytes Beeper Desktop can re-supply, re-fetched on first view
  // and rendered as a labelled reference when it cannot — so a snapshot that
  // skips it loses no record, only a re-download. It is also by far the largest
  // thing the Comms feature puts on disk, which is exactly the kind of
  // directory that turns a nightly snapshot into an hour.
  // Overridable, because an archive of a conversation is more useful with its
  // photos in it, and someone keeping one may well want to pay for them.
  { path: '/beeper/attachments/', reason: 'Beeper attachment byte mirror — a lazy cache re-fetchable from Beeper Desktop; the message bodies and attachment metadata live in Postgres and ARE backed up', overridable: true },
  // Anchored with a leading `/`, like every entry here — an unanchored
  // `corpora/` would match at any depth and silently drop unrelated user data.
  //
  // Two of the three `data/jev/` directories are excluded; `heads/` deliberately
  // is NOT, because a trained head is the one artifact there that does not
  // re-derive. The full three-way tier argument lives in docs/BACKUP.md
  // ("jev project heads — excluded bulk, retained artifact"), which is where a
  // tier change belongs.
  { path: '/jev/corpora/', reason: 'jev training corpora — rebuildable from the forge by scripts/jev-corpus.js. Trained heads in data/jev/heads/ are NOT excluded: a head is not regenerable once its corpus is stale.', overridable: true },
  { path: '/jev/embeddings/', reason: 'Cached frozen-encoder outputs for jev head training — keyed by (pair, model revision) and byte-identical on re-encode', overridable: false }
  // NOTE: legacy file→Postgres migration artifacts (`.imported` / `.bak-NNN`)
  // are intentionally NOT excluded here. They are deleted on disk by the
  // boot-time prune (pruneImportedLegacyFiles.js) the same boot the migration
  // runs — but ONLY once the DB is provably authoritative. While the prune is
  // *blocked* (a wiped/partial-restore DB short of the migration markers) those
  // parked files are the only recovery source, and `pg_dump` is capturing the
  // incomplete DB — so excluding them from snapshots would mean a backup taken
  // in that window restores neither the missing rows nor the source to rebuild
  // them. Letting rsync copy them is the safe default; once the prune removes
  // them from disk they leave subsequent snapshots naturally.
];

// Snapshots live under snapshots/<hostname>/<snapshotId> so a single shared
// destination (e.g. iCloud) can host backups from multiple machines without
// their snapshot IDs colliding.
const MACHINE_HOST = hostname().toLowerCase().replace(/[^\w.\-]/g, '_') || 'unknown';
const LEGACY_SNAPSHOT_SOURCE = '@legacy';
const SNAPSHOT_ID_PATTERN = /^[\w\-.:T]+$/;
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
const MANIFEST_ABSENT = Symbol('manifest-absent');

const DEFAULT_STATE = {
  lastRun: null,
  status: 'never',
  lastSnapshotId: null,
  filesChanged: 0,
  pgBackup: null,
  error: null
};

/**
 * Map a dumpPostgres result to the overall backup status. Only a *failed*
 * dump (PG configured but the dump errored) degrades the backup; a *skipped*
 * dump (no PG — file mode) is benign and stays 'ok'.
 * @param {{status: string}} pgResult
 * @returns {'ok'|'degraded'}
 */
export function backupStatusForPg(pgResult) {
  return pgResult?.status === 'failed' ? 'degraded' : 'ok';
}

// =============================================================================
// INTERNAL HELPERS
// =============================================================================

// Whole minutes for realistic deadlines ("10 minutes"), seconds below one
// minute, so the timeout message stays truthful under an override either way.
const describeIdleDuration = (ms) => {
  if (ms < 60_000) return `${Math.max(1, Math.round(ms / 1000))} seconds`;
  const minutes = Math.round(ms / 60_000);
  return `${minutes} minute${minutes === 1 ? '' : 's'}`;
};

class BackupProcessTimeoutError extends Error {
  constructor(label, timeoutKind, idleTimeoutMs = BACKUP_PROCESS_IDLE_TIMEOUT_MS) {
    const duration = timeoutKind === 'idle' ? `${describeIdleDuration(idleTimeoutMs)} without progress` : '4 hours';
    super(`${label} timed out after ${duration}`);
    this.name = 'BackupProcessTimeoutError';
    this.code = 'BACKUP_PROCESS_TIMEOUT';
    this.timeoutKind = timeoutKind;
  }
}

/**
 * Watch a backup subprocess for both stalled progress and runaway wall time.
 * The watchdog only starts termination; callers still settle from `close`, so
 * the backup lock and snapshot markers cannot clear while the child may write.
 *
 * pg_dump writes directly to `progressPath` and is normally silent. Polling its
 * growing output file lets a healthy large dump reset the idle timer without
 * adding verbose flags or buffering its SQL in memory.
 *
 * `idleTimeoutMs` overrides the default idle deadline for a caller whose
 * process can legitimately stay silent longer — restoreSnapshot derives one
 * from the largest in-scope file because a single file's `--checksum` digest
 * emits no output until it finishes (#7302). Callers that can produce regular
 * output leave the tight default so a genuinely wedged child is still caught
 * quickly.
 */
function watchBackupProcess(proc, { label, progressPath = null, idleTimeoutMs } = {}) {
  // Internal callers only, but a non-positive or non-numeric override would
  // silently disarm (0, NaN) or tighten (-1) the deadline — fall back instead.
  const idleDeadlineMs = Number.isFinite(idleTimeoutMs) && idleTimeoutMs > 0
    ? idleTimeoutMs
    : BACKUP_PROCESS_IDLE_TIMEOUT_MS;
  let finished = false;
  let timeoutError = null;
  let idleTimer = null;
  let escalationTimer = null;
  let progressPoll = null;
  let progressPollInFlight = false;
  let lastProgressSize = 0;

  const startTermination = (timeoutKind) => {
    if (finished || timeoutError || proc.exitCode !== null || proc.signalCode !== null) return;
    timeoutError = new BackupProcessTimeoutError(label, timeoutKind, idleDeadlineMs);
    console.warn(`⚠️ ${timeoutError.message} — terminating child process`);
    try {
      escalationTimer = killWithEscalation(proc, {
        label,
        stillRunning: () => !finished,
      });
    } catch (err) {
      // Keep the caller pending if termination itself fails. Releasing a backup
      // lock while its child may still mutate files or the database is unsafe.
      console.error(`❌ ${label} timeout termination failed: ${err.message}`);
    }
  };

  const armIdleTimer = () => {
    if (idleTimer) clearTimeout(idleTimer);
    idleTimer = setTimeout(() => startTermination('idle'), idleDeadlineMs);
    idleTimer.unref?.();
  };

  const markActivity = () => {
    if (!finished && !timeoutError) armIdleTimer();
  };

  armIdleTimer();
  const wallTimer = setTimeout(() => startTermination('wall'), BACKUP_PROCESS_WALL_TIMEOUT_MS);
  wallTimer.unref?.();

  if (progressPath) {
    progressPoll = setInterval(() => {
      if (finished || timeoutError || progressPollInFlight) return;
      progressPollInFlight = true;
      stat(progressPath)
        .then((info) => {
          if (info.size > lastProgressSize) {
            lastProgressSize = info.size;
            markActivity();
          }
        })
        // A missing or temporarily unreadable file is no progress. The idle
        // deadline remains authoritative and will terminate the child.
        .catch(() => {})
        .finally(() => { progressPollInFlight = false; });
    }, BACKUP_PROCESS_PROGRESS_POLL_MS);
    progressPoll.unref?.();
  }

  return {
    markActivity,
    getTimeoutError: () => timeoutError,
    finish: () => {
      if (finished) return false;
      finished = true;
      clearTimeout(idleTimer);
      clearTimeout(wallTimer);
      if (progressPoll) clearInterval(progressPoll);
      if (escalationTimer) clearTimeout(escalationTimer);
      return true;
    },
  };
}

/**
 * Run rsync from srcDir to destDir with optional flags.
 * Resolves with array of changed file lines. Rejects on non-zero exit (24 only
 * when the caller opts in — see `allowVanishedSources`).
 */
export function resolveRsyncBinary(env = process.env) {
  const override = typeof env.PORTOS_RSYNC === 'string' ? env.PORTOS_RSYNC.trim() : '';
  // A bare command lets spawn resolve rsync through PATH on macOS, Linux,
  // Windows/MSYS, and non-standard Unix layouts. PORTOS_RSYNC remains the
  // explicit escape hatch for bundled or custom installations.
  return override || 'rsync';
}

// `allowVanishedSources` is the per-operation exit policy (#10898). A backup
// reads the LIVE data tree, where a file deleted mid-scan (exit 24) is normal
// and the snapshot is still a usable point-in-time copy. A restore reads a
// snapshot that must not change under it: exit 24 means part of its source
// disappeared, so live data may now mix restored and old files. Restore keeps
// the strict default and treats 24 like every other nonzero exit.
function runRsync(srcDir, destDir, flags = [], { idleTimeoutMs, allowVanishedSources = false } = {}) {
  return new Promise((resolve, reject) => {
    // `--itemize-changes` emits only after each file finishes. `--progress` is
    // also supported by macOS's bundled rsync 2.6.9 and emits within a large
    // file, giving the idle watchdog evidence that a slow transfer is healthy.
    // Both report only TRANSFERRING files, so a caller that adds `--checksum`
    // must also pass `-ii` or its scan of an unchanged tree is silent for the
    // whole digest and the watchdog kills it — see restoreSnapshot. Even `-ii`
    // is a per-file heartbeat, so a caller expecting one very large file may
    // widen the idle deadline via `idleTimeoutMs` (see restoreSnapshot).
    const args = ['--archive', '--itemize-changes', '--progress', ...flags, srcDir + '/', destDir];
    const proc = spawn(resolveRsyncBinary(), args, { shell: false });

    const changed = [];
    let stderr = '';
    const watchdog = watchBackupProcess(proc, { label: 'backup rsync', idleTimeoutMs });

    const stdoutReader = createLineReader((line) => {
      if (line.startsWith('>') || line.startsWith('<')) {
        changed.push(line);
      }
    });
    proc.stdout.on('data', (chunk) => {
      watchdog.markActivity();
      stdoutReader.push(chunk);
    });

    proc.stderr.on('data', (chunk) => {
      watchdog.markActivity();
      stderr += chunk.toString();
    });

    proc.on('close', (code) => {
      if (!watchdog.finish()) return;
      const timeoutError = watchdog.getTimeoutError();
      if (timeoutError) {
        reject(timeoutError);
        return;
      }
      // Exit code 24 = some source files vanished mid-transfer.
      if (code === 0 || (code === 24 && allowVanishedSources)) {
        stdoutReader.flush();
        resolve(changed);
      } else {
        const detail = code === 24 ? ' (source files vanished during transfer)' : '';
        reject(new Error(`rsync exited with code ${code}${detail}: ${stderr.trim()}`));
      }
    });

    proc.on('error', (err) => {
      // A timeout does not settle until `close`: the child may still be alive
      // after an error from a failed termination attempt.
      if (watchdog.getTimeoutError() || !watchdog.finish()) return;
      reject(new Error(`rsync spawn error: ${err.message}`));
    });
  });
}

// =============================================================================
// EXPORTS
// =============================================================================

/**
 * Compute the effective rsync --exclude list for a backup run. Pure function
 * extracted so the Array.isArray guards + override allow-list can be unit
 * tested without spawning rsync.
 *
 * - Non-overridable defaults stay on regardless of `disabledDefaultExcludes` so
 *   ephemeral/cache paths can never be backed up by mistake (e.g. via a
 *   hand-edited settings.json).
 * - Array.isArray guards: settings can be hand-edited or sent by a stale
 *   client, so a non-array value here would otherwise throw inside .filter
 *   and abort the backup before the defensive allow-list has a chance to apply.
 * - User patterns are ANCHORED here (`anchorUserExcludes`), not in storage: a
 *   bare `cache/` is rsync for "every cache/ at any depth", which silently drops
 *   the per-run caches nested under training runs too. Normalizing on read leaves
 *   the stored value exactly as typed, so there is no migration and nothing is
 *   rewritten under the user. A `*`/`**`-led pattern stays as-is — that is the
 *   deliberate way to ask for any-depth matching.
 */
export function computeEffectiveExcludes({ excludePaths, disabledDefaultExcludes } = {}) {
  const overridablePaths = new Set(DEFAULT_EXCLUDES.filter(e => e.overridable).map(e => e.path));
  const disabledList = Array.isArray(disabledDefaultExcludes) ? disabledDefaultExcludes : [];
  const disabledSet = new Set(disabledList.filter(p => overridablePaths.has(p)));
  const activeDefaults = DEFAULT_EXCLUDES.filter(e => !disabledSet.has(e.path)).map(e => e.path);
  const userExcludes = anchorUserExcludes(excludePaths);
  return [...new Set([...activeDefaults, ...userExcludes])];
}

/**
 * Run a full backup snapshot from PATHS.data to destPath.
 * @param {string} destPath - Path to external drive backup root
 * @param {object|null} io - Socket.IO instance for real-time events (optional)
 */
export async function runBackup(destPath, io = null, { excludePaths = [], disabledDefaultExcludes = [], retentionCount = null } = {}) {
  if (isRunning) {
    console.log('💾 Backup already running — skipping');
    return { skipped: true };
  }

  if (!destPath) {
    throw new Error('Backup destination not configured');
  }

  setBackupRunning(true, 'start');
  let snapshotId = null;
  let snapshotDir;
  let parentMarker;

  const effectiveExcludes = computeEffectiveExcludes({ excludePaths, disabledDefaultExcludes });

  let changedFiles = [];
  let manifest;
  let previousSnapshotId;

  const removeMarker = (path, marker) => unlink(path).catch(cause => {
    if (cause.code === 'ENOENT') return;
    // Raw filesystem errors can contain the private backup destination.
    const code = ['EACCES', 'EPERM', 'EIO', 'ENOSPC', 'EROFS', 'ENOTDIR', 'EMFILE', 'ENFILE', 'EBUSY'].includes(cause.code) ? cause.code : 'UNKNOWN';
    console.error(`❌ Backup snapshot finalization failed: snapshot=${snapshotId} marker=${marker} code=${code}`);
    throw new ServerError(`Backup snapshot finalization failed (${marker} marker, ${code})`, {
      code: 'BACKUP_SNAPSHOT_FINALIZATION_FAILED',
    });
  });

  const clearInProgressMarkers = async () => {
    if (snapshotDir) await removeMarker(markerPath(snapshotDir), 'snapshot');
    if (parentMarker) await removeMarker(parentMarker, 'reservation');
  };

  const releaseActiveSnapshot = () => {
    activeSnapshotId = null;
  };

  const complete = (result) => {
    releaseActiveSnapshot();
    setBackupRunning(false, 'completion');
    return result;
  };

  const fail = async (err) => {
    // Persist failure BEFORE removing the incomplete guards. If the marker
    // cannot be written, retain those guards so a partial tree never becomes a
    // restore source. The process lock is independent and is always released.
    const failureRecorded = snapshotDir
      ? await writeFile(failedMarkerPath(snapshotDir), '').then(() => true, () => false)
      : false;
    // Cleanup errors are already logged by removeMarker. The durable failure
    // marker blocks restore even if cleanup fails; always release the run lock.
    if (failureRecorded) await clearInProgressMarkers().catch(() => {});
    releaseActiveSnapshot();
    setBackupRunning(false, 'failure');
    const failureState = {
      lastRun: new Date().toISOString(), status: 'error', error: err.message, pgBackup: null,
      ...(previousSnapshotId !== undefined ? { lastSnapshotId: previousSnapshotId } : {}),
    };
    await saveState(failureState, (cause) => {
      const code = ['EACCES', 'EPERM', 'EIO', 'ENOSPC', 'EROFS', 'ENOENT', 'EMFILE', 'ENFILE'].includes(cause.code) ? cause.code : 'UNKNOWN';
      failedStateProjection = { ...failureState, error: `Backup failed; status persistence failed (${code}). See server logs.` };
      console.error(`❌ Backup status persistence failed: transition=error snapshot=${snapshotId ?? 'none'} code=${code}`);
    });
    dashboardEvents.emit('backup:changed');
    if (io) io.emit('backup:failed', { snapshotId, error: err.message });
    throw err;
  };

  try {
    // Refuse before reserving a snapshot while database maintenance is fenced;
    // the cut below repeats this check after it drains.
    assertDatabaseAdmission();
    await access(destPath).catch((cause) => {
      // Never expose the filesystem message: it can include private paths.
      const code = ['ENOENT', 'EACCES', 'EPERM', 'EIO'].includes(cause.code) ? cause.code : 'UNKNOWN';
      const message = code === 'ENOENT' ? 'Backup destination not found' : 'Backup destination inaccessible';
      const error = new ServerError(`${message} (${code})`, { code: `BACKUP_DESTINATION_${code}` });
      error.cause = cause;
      throw error;
    });
    const timestamp = new Date().toISOString().replace(/:/g, '-').replace(/\..+/, '');
    const snapshotsRoot = join(destPath, 'snapshots', MACHINE_HOST);
    await ensureDir(snapshotsRoot);
    // Inside the single-run guard, so no run of this process can own a marker
    // the sweep is converging.
    await markInterruptedSnapshotsFailed(snapshotsRoot).catch(err => {
      console.error(`❌ Backup interrupted-snapshot sweep failed: code=${err?.code ?? 'UNKNOWN'}`);
    });
    // Own the durable guard before exposing a directory. Never adopt an
    // existing tree, even when its timestamp matches this run's clock.
    for (let attempt = 0; attempt < 100; attempt += 1) {
      const candidateId = attempt === 0 ? timestamp : `${timestamp}-${randomUUID()}`;
      const candidateDir = join(snapshotsRoot, candidateId);
      const candidateMarker = parentMarkerPath(candidateDir, candidateId);
      const exists = await lstat(candidateDir).then(() => true, err => {
        if (err.code === 'ENOENT') return false;
        throw err;
      });
      if (exists) continue;
      const reserved = await writeFile(candidateMarker, BACKUP_PROCESS_START_ID, { flag: 'wx' }).then(() => true, err => {
        if (err.code === 'EEXIST') return false;
        throw err;
      });
      if (!reserved) continue;
      const created = await mkdir(candidateDir).then(() => true, async err => {
        // Only our exclusive reservation can be released here. The directory
        // may belong to a competing writer and must remain untouched.
        await unlink(candidateMarker);
        if (err.code === 'EEXIST') return false;
        throw err;
      });
      if (!created) continue;
      snapshotId = candidateId;
      snapshotDir = candidateDir;
      parentMarker = candidateMarker;
      break;
    }
    if (!snapshotDir) throw new Error('Unable to reserve a fresh backup snapshot');
    const dataDestDir = join(snapshotDir, 'data');

    console.log(`💾 Backup starting: snapshot ${snapshotId} (excluding ${effectiveExcludes.length} paths)`);
    if (io) io.emit('backup:started', { snapshotId });
    await ensureDir(dataDestDir);
    activeSnapshotId = snapshotId;
    await writeFile(markerPath(snapshotDir), BACKUP_PROCESS_START_ID);
    dashboardEvents.emit('backup:changed');

    // Freeze admitted file-plus-row publications across both stores. A writer
    // that started first drains completely; new writers wait until the dump
    // and manifest are published. Rendering/provider work is outside admission.
    const releaseSnapshotCut = await acquireBackupSnapshotCut();
    // The snapshot records how far that guarantee reaches: `global` only once
    // no durable asset owner remains outside admission.
    const assetConsistency = backupAssetConsistency();
    let pgResult;
    try {
      const excludeFlags = effectiveExcludes.flatMap(p => ['--exclude', p]);
      changedFiles = await runRsync(PATHS.data, dataDestDir, excludeFlags, { allowVanishedSources: true });
      console.log(`💾 Backup rsync complete: ${changedFiles.length} files changed (exit 0)`);

      // A configured-but-failed dump degrades the backup and alerts the user.
      const pgDumpPath = join(snapshotDir, 'portos-db.sql');
      pgResult = await dumpPostgres(pgDumpPath);
      manifest = await generateManifest(dataDestDir, join(snapshotDir, 'manifest.json'), pgDumpPath, {
        allowMissingDump: pgResult.status === 'skipped' || pgResult.status === 'failed',
        assetConsistency,
      });
    } finally {
      releaseSnapshotCut();
    }

    // A backup is not complete while its durable guards still block restore.
    // Keep the reservation guard until success is persisted: if both the
    // status write and failure-marker write fail, it must survive a restart.
    await removeMarker(failedMarkerPath(snapshotDir), 'failed');
    await removeMarker(markerPath(snapshotDir), 'snapshot');

    const status = backupStatusForPg(pgResult);
    const lastRun = new Date().toISOString();
    previousSnapshotId = (await getState()).lastSnapshotId;
    await saveState({
      lastRun,
      lastSnapshotId: snapshotId,
      status,
      filesChanged: changedFiles.length,
      pgBackup: pgResult,
      assetConsistency,
      error: pgResult.status === 'failed' ? `DB dump ${pgResult.reason}` : null
    });

    await removeMarker(parentMarker, 'reservation');

    if (io) io.emit('backup:completed', { snapshotId, filesChanged: changedFiles.length, status, pgBackup: pgResult });

    // Loud-on-failure: surface a degraded DB dump as a warning toast, even on
    // unattended scheduled runs (which pass io=null) via the module-level io.
    if (pgResult.status === 'failed') {
      // Keep socket/auth initialization out of backup metadata reads and tests.
      const errIo = io || (await import('./socket.js')).getIo();
      if (errIo) {
        emitErrorEvent(errIo, new ServerError(
          `Backup DB dump failed: ${pgResult.reason}`,
          { status: 500, code: 'BACKUP_DB_DUMP_FAILED', severity: 'warning' }
        ));
      }
    }

    const result = complete({ snapshotId, filesChanged: changedFiles.length, status, lastRun, manifest, pgBackup: pgResult, assetConsistency });

    // Prune only after the snapshot markers are cleared and `complete()` has
    // released `activeSnapshotId` — pruning any
    // earlier would make the run's OWN fresh snapshot read as incomplete
    // (via `snapshotState`) and drop out of the retention count entirely,
    // silently keeping one snapshot MORE than configured. A run that throws
    // before this point takes the `fail()` path below and never reaches
    // here, so this run's own snapshot is never pruned as failed or in-progress.
    // Retention failures must not fail an otherwise-successful backup.
    let pruned = { pruned: 0 };
    if (pgResult.status === 'failed') {
      console.warn(`⚠️ Backup retention skipped: DB dump ${pgResult.reason} — keeping older snapshots`);
    } else {
      pruned = await pruneOldSnapshots(destPath, retentionCount).catch(err => {
        console.error(`❌ Backup retention prune failed: ${err.message}`);
        return { pruned: 0 };
      });
    }

    dashboardEvents.emit('backup:changed');
    return { ...result, prunedSnapshots: pruned.pruned };
  } catch (err) {
    return fail(err);
  }
}

/**
 * The explicit file escape hatch (MEMORY_BACKEND=file, or a backend that
 * resolved to 'file'): the one state in which PostgreSQL is not required. The
 * backend initializes lazily (ensureBackend), so the env var is consulted too —
 * an uninitialized backend under the hatch must not count as "Postgres required".
 * Shared by the dump, the restore compatibility check and restore reconciliation
 * so the three cannot disagree about one install.
 */
const usesFileEscapeHatch = () => process.env.MEMORY_BACKEND === 'file' || getBackendName() === 'file';

/**
 * Run pg_dump to create a PostgreSQL backup alongside the rsync snapshot.
 * Returns an explicit status so the caller can distinguish the benign file
 * escape hatch from "PG required but dump failed" (data at risk):
 *   { status: 'ok', sizeBytes, tableCount }
 *   { status: 'skipped', reason: 'not_configured' }   (explicit file escape hatch only)
 *   { status: 'failed', reason: 'pg_unreachable'|'pg_dump_missing'|'version_mismatch'|'dump_error'|'empty_dump'|'inspect_error'|'timeout', error }
 *     (pg_unreachable fires whenever Postgres is required — i.e. not the file
 *      escape hatch — but the DB is down at backup time; version_mismatch means
 *      no installed pg_dump is new enough for the running server)
 * @param {string} outputPath - Path to write the SQL dump file
 */
export async function dumpPostgres(outputPath) {
  const health = await checkHealth();
  if (!health.connected || !health.hasSchema) {
    // PG unreachable or uninitialized. Since PostgreSQL is now a mandatory
    // dependency, the ONLY benign "no PG to back up" case is the explicit file
    // escape hatch (MEMORY_BACKEND=file, or the backend resolved to 'file' in
    // test/dev mode). Every other state — including a default install whose
    // memory backend simply hasn't initialized yet (getBackendName() === null)
    // — means Postgres is required, so an unreachable DB is a real backup
    // failure (data that lives only in PG won't be captured). Degrade and alert
    // rather than silently skip; gating on getBackendName() === 'postgres'
    // alone would let an outage-before-first-memory-access read as a green
    // "not configured" run.
    if (!usesFileEscapeHatch()) {
      return { status: 'failed', reason: 'pg_unreachable', error: health.error || 'PostgreSQL is required but is unreachable or uninitialized' };
    }
    return { status: 'skipped', reason: 'not_configured' };
  }

  const { host: pgHost, port, database: pgDb, user: pgUser } = POOL_CONFIG;
  const pgPort = String(port);

  // pg_dump must be >= the server's major version or it aborts on a "server
  // version mismatch". On machines with multiple Postgres installs (the common
  // Homebrew case: an old postgresql@NN keg shadowing a newer running server in
  // PATH) the bare `pg_dump` is often the wrong one, so select a matching binary
  // instead of trusting PATH order.
  const serverMajor = await getServerMajorVersion();
  // Shared resolver (server/lib/pgTools.js): PORTOS_PGDUMP override wins, else
  // auto-select a binary whose major is >= the server when we know the version,
  // else fall back to bare `pg_dump` off PATH.
  const { binary: pgDumpBin, satisfies } = await resolvePgDumpBinary(serverMajor);
  if (!satisfies) {
    console.warn(`⚠️ No installed pg_dump satisfies server major ${serverMajor} (using ${pgDumpBin})`);
  }

  return withPgToolEnv(POOL_CONFIG, pgEnv => new Promise((resolvePromise) => {
    // --clean --if-exists: the dump DROPs each object before recreating it, so it
    // replays cleanly into the live, already-initialized PortOS database (the
    // common Restore-DB target) instead of erroring "relation already exists" on
    // the first CREATE. Without it the dump is only restorable into an empty DB.
    const proc = spawn(pgDumpBin, [
      '-h', pgHost,
      '-p', pgPort,
      '-U', pgUser,
      '-d', pgDb,
      '--no-owner',
      '--no-acl',
      '--no-comments',
      '--clean',
      '--if-exists',
      // Machine-local replay receipts (#9725) describe this install's past
      // restores, not application data; the table definition is still dumped.
      '--exclude-table-data=restore_receipts',
      // The asynchronous media mirror can lag its authoritative files. Rebuild
      // it after restore rather than snapshotting stale file references.
      '--exclude-table-data=media_assets',
      '-f', outputPath
    ], {
      shell: false,
      env: pgEnv
    });

    let stderr = '';
    const watchdog = watchBackupProcess(proc, { label: 'PostgreSQL dump', progressPath: outputPath });
    proc.stderr.on('data', (chunk) => {
      watchdog.markActivity();
      stderr += chunk.toString();
    });

    proc.on('close', async (code) => {
      if (!watchdog.finish()) return;
      const timeoutError = watchdog.getTimeoutError();
      if (timeoutError) {
        await unlink(outputPath).catch(() => {});
        resolvePromise({ status: 'failed', reason: 'timeout', error: timeoutError.message });
        return;
      }
      if (code !== 0) {
        console.warn(`⚠️ pg_dump failed (code ${code}): ${stderr.trim()}`);
        // pg_dump can exit non-zero after writing a partial file (e.g. mid-dump
        // connection loss). Remove it so a later restore can't trust a truncated
        // dump on size alone — a failed dump must leave no restorable artifact.
        await unlink(outputPath).catch(() => {});
        // A version mismatch is a distinct, actionable failure (install a newer
        // pg_dump) — classify it so the UI can point at the fix instead of the
        // generic "is pg_dump installed / is PG reachable" hint.
        const isMismatch = !satisfies || /server version mismatch|aborting because of server version/i.test(stderr);
        resolvePromise({ status: 'failed', reason: isMismatch ? 'version_mismatch' : 'dump_error', error: stderr.trim() });
        return;
      }
      // Verify: a dump that exits 0 but is empty/truncated is still a failure.
      const info = await stat(outputPath).catch(() => null);
      if (!info || info.size === 0) {
        console.warn('⚠️ pg_dump produced an empty dump file');
        resolvePromise({ status: 'failed', reason: 'empty_dump', error: 'dump file missing or 0 bytes' });
        return;
      }
      // Stream the dump for its table count — reading it whole would scale
      // server heap with database size. Own the rejection here: this is an
      // event callback, so a throw would leave the outer Promise unsettled.
      const inspected = await inspectDatabaseDump(outputPath).catch((err) => {
        console.warn(`⚠️ pg_dump inspection failed: ${err.code || err.message}`);
        return null;
      });
      if (!inspected) {
        resolvePromise({ status: 'failed', reason: 'inspect_error', error: 'dump could not be read for verification' });
        return;
      }
      const tableCount = inspected.tableCount;
      console.log(`💾 pg_dump complete: ${Math.round(info.size / 1024)}KB, ${tableCount} tables`);
      // Don't return the absolute dump path: this result is persisted into
      // state.pgBackup and surfaced to the client via GET /api/backup/status,
      // the backup:completed socket event, and the /run response. No client
      // reads it (restorePostgres recomputes its own sqlPath), so leaking an
      // internal FS path serves no purpose.
      resolvePromise({ status: 'ok', sizeBytes: info.size, tableCount });
    });

    proc.on('error', (err) => {
      if (watchdog.getTimeoutError() || !watchdog.finish()) return;
      // pg_dump not installed — a configured-but-unbacked-up DB is at risk,
      // so this is a failure, not a silent skip.
      console.warn(`⚠️ pg_dump not available: ${err.message}`);
      resolvePromise({ status: 'failed', reason: 'pg_dump_missing', error: err.message });
    });
  }));
}

// Filesystem messages contain private paths; only expose the operation and errno.
function manifestReadFailure(operation, err) {
  const code = /^E[A-Z0-9]+$/.test(err?.code) ? err.code : 'UNKNOWN';
  return new Error(`Backup manifest ${operation} failed (${code})`);
}

/**
 * Generate a SHA-256 manifest for all files in snapshotDataDir, plus the
 * sibling pg dump (which lives outside the data/ tree). Hashing the dump means
 * a truncated/corrupt portos-db.sql is detectable, not silently trusted.
 * @param {string} snapshotDataDir - Directory to hash
 * @param {string} manifestPath - Path to write manifest.json
 * @param {string|null} [pgDumpPath=null] - Sibling SQL dump to also hash
 * @param {object} [options] - Dump inventory expectations
 * @param {boolean} [options.allowMissingDump=false] - Only for skipped/failed dumps
 * @param {object} [options.assetConsistency] - The file-and-row consistency this snapshot claims
 */
export async function generateManifest(snapshotDataDir, manifestPath, pgDumpPath = null, { allowMissingDump = false, assetConsistency } = {}) {
  const entries = await readdir(snapshotDataDir, { recursive: true })
    .catch(err => { throw manifestReadFailure('data readdir', err); });
  const files = {};

  for (const entry of entries) {
    const filePath = join(snapshotDataDir, entry);
    const info = await stat(filePath).catch(async err => {
      // rsync --archive preserves links even when their targets are absent or
      // excluded. Preserve that compatibility, but never skip a missing entry.
      if (err.code === 'ENOENT') {
        const entryInfo = await lstat(filePath)
          .catch(entryErr => { throw manifestReadFailure('data lstat', entryErr); });
        if (entryInfo.isSymbolicLink()) return null;
      }
      throw manifestReadFailure('data stat', err);
    });
    if (!info || !info.isFile() || isOsMetadataFile(basename(entry))) continue;
    files[entry] = await sha256File(filePath)
      .catch(err => { throw manifestReadFailure('data hash', err); });
  }

  if (pgDumpPath) {
    const dumpInfo = await stat(pgDumpPath).catch(err => {
      if (allowMissingDump && err.code === 'ENOENT') return null;
      throw manifestReadFailure('dump stat', err);
    });
    if (dumpInfo && !dumpInfo.isFile()) {
      throw new Error('Backup manifest dump stat failed (not a regular file)');
    }
    if (dumpInfo?.isFile()) {
      // Parent-relative key: the dump lives one level ABOVE snapshotDataDir
      // (alongside it, not inside it). A future manifest-verify must not assume
      // every key resolves under snapshotDataDir.
      files['../portos-db.sql'] = await sha256File(pgDumpPath)
        .catch(err => { throw manifestReadFailure('dump hash', err); });
    }
  }

  const manifest = {
    generatedAt: new Date().toISOString(),
    fileCount: Object.keys(files).length,
    files,
    ...(assetConsistency ? { assetConsistency } : {}),
  };

  await atomicWrite(manifestPath, manifest);
  console.log(`💾 Backup manifest: ${manifest.fileCount} files`);
  return manifest;
}

// Keep filesystem diagnostics bounded: native errors include private paths.
function inventoryReadError(cause, operation) {
  const code = ['ENOENT', 'EACCES', 'EPERM', 'EIO', 'ENOTDIR', 'EMFILE', 'ENFILE', 'ESTALE']
    .includes(cause?.code) ? cause.code : 'UNKNOWN';
  return new ServerError(`Backup inventory unavailable: ${operation} (${code})`, {
    code: 'BACKUP_INVENTORY_UNAVAILABLE',
    context: { operation, filesystemCode: code },
  });
}

/**
 * List all snapshots in the backup destination.
 * @param {string} destPath - Path to external drive backup root
 * @returns {Array<{ id, source, selectionKey, createdAt, fileCount, incomplete, failed, interrupted }>} sorted newest-first
 */
export async function listSnapshots(destPath) {
  if (!destPath) return [];

  const snapshotsRoot = join(destPath, 'snapshots');
  // withFileTypes so we can skip non-directory entries: the backup target is
  // commonly an iCloud/Finder folder, where macOS drops a `.DS_Store` FILE into
  // every directory. Treating it as a snapshot id and reading
  // `<.DS_Store>/manifest.json` throws ENOTDIR. Also skip dotfile-named dirs so
  // nothing hidden can masquerade as a snapshot (real ids are timestamps).
  const rootEntries = await readdir(snapshotsRoot, { withFileTypes: true }).catch(async cause => {
    if (cause?.code !== 'ENOENT') throw inventoryReadError(cause, 'read-snapshots-root');
    // Absence is empty only when the configured destination itself is readable.
    await readdir(destPath).catch(error => { throw inventoryReadError(error, 'read-destination'); });
    return [];
  });
  const directories = rootEntries.filter(entry => entry.isDirectory() && !entry.name.startsWith('.'));
  const descriptors = (await Promise.all(directories.map(async (entry) => {
    const rootEntryPath = join(snapshotsRoot, entry.name);
    const contents = await readdir(rootEntryPath, { withFileTypes: true }).catch(cause => {
      if (cause?.code === 'ENOENT') return [];
      throw inventoryReadError(cause, 'read-namespace');
    });
    const isLegacySnapshot = SNAPSHOT_ID_PATTERN.test(entry.name) && contents.some(child =>
      (child.name === 'data' && child.isDirectory())
      || child.name === 'manifest.json'
      || child.name === 'portos-db.sql'
      || child.name === SNAPSHOT_IN_PROGRESS_MARKER
      || child.name === SNAPSHOT_FAILED_MARKER);

    if (isLegacySnapshot) return [{ id: entry.name, source: LEGACY_SNAPSHOT_SOURCE }];
    if (!isSafeSnapshotSource(entry.name) || entry.name === LEGACY_SNAPSHOT_SOURCE) return [];

    return contents
      .filter(child => child.isDirectory() && !child.name.startsWith('.') && SNAPSHOT_ID_PATTERN.test(child.name))
      .map(child => ({ id: child.name, source: entry.name }));
  }))).flat();

  const snapshots = await Promise.all(
    descriptors.map(async ({ id, source }) => {
      const { snapshotDir, currentSource } = resolveSnapshotPath(destPath, id, source);
      const manifestPath = join(snapshotDir, 'manifest.json');
      // logError:false — a snapshot taken before manifests existed legitimately
      // has none; the null is handled below, so it isn't worth a warning per list.
      // Read-only listing metadata; generateManifest rebuilds from snapshot bytes.
      const manifest = await readJSONFile(manifestPath, null, { logError: false });
      // Report a still-being-written snapshot rather than hiding it: the row is
      // real and the user should see the run in flight, but download and restore
      // must not be offered for it. Mirrors assertSnapshotComplete's two signals.
      const { incomplete, failed, interrupted } = await snapshotState(snapshotDir, id, currentSource);
      return {
        id,
        source,
        sourceLabel: source === LEGACY_SNAPSHOT_SOURCE
          ? 'Legacy (pre-namespace)'
          : source === MACHINE_HOST ? `${source} (current machine)` : source,
        selectionKey: `${source}/${id}`,
        currentMachine: source === MACHINE_HOST,
        createdAt: typeof manifest?.generatedAt === 'string' && !Number.isNaN(Date.parse(manifest.generatedAt))
          ? manifest.generatedAt : null,
        fileCount: manifest?.fileCount ?? 0,
        incomplete,
        failed,
        interrupted,
      };
    })
  );

  return snapshots.sort((a, b) => {
    if (a.createdAt && b.createdAt) return b.createdAt.localeCompare(a.createdAt)
      || a.selectionKey.localeCompare(b.selectionKey);
    if (a.createdAt) return -1;
    if (b.createdAt) return 1;
    return b.id.localeCompare(a.id) || a.source.localeCompare(b.source);
  });
}

/**
 * Converge every interrupted snapshot in this host's namespace to a durable
 * `.failed` (reason INTERRUPTED_FAILURE_REASON), then drop its in-progress
 * markers — the same order `fail()` uses, so a failed write leaves the
 * snapshot guarded. Holds the exclusive lease so a concurrent delete or
 * reader cannot interleave with the marker writes.
 */
async function markInterruptedSnapshotsFailed(snapshotsRoot) {
  const entries = await readdir(snapshotsRoot, { withFileTypes: true });
  let marked = 0;
  for (const entry of entries) {
    if (!entry.isDirectory() || entry.name.startsWith('.') || !SNAPSHOT_ID_PATTERN.test(entry.name)) continue;
    const id = entry.name;
    const snapshotDir = join(snapshotsRoot, id);
    const isCandidate = ({ interrupted, failed }) => interrupted && !failed;
    // Lease only candidates, so a download starting now is not refused
    // SNAPSHOT_DELETING by a healthy snapshot's sweep.
    if (!isCandidate(await snapshotState(snapshotDir, id, true))) continue;
    // Own the snapshot before the final state check, as deleteSnapshot does.
    const { release, refusal } = await claimSnapshotLease(snapshotDir, 'delete');
    if (refusal) continue;
    try {
      const exists = await stat(snapshotDir).then(info => info.isDirectory(), () => false);
      if (!exists || !isCandidate(await snapshotState(snapshotDir, id, true))) continue;
      const removeIfPresent = (path) => unlink(path).catch(err => { if (err.code !== 'ENOENT') throw err; });
      await writeFile(failedMarkerPath(snapshotDir), INTERRUPTED_FAILURE_REASON)
        .then(() => removeIfPresent(markerPath(snapshotDir)))
        .then(() => removeIfPresent(parentMarkerPath(snapshotDir, id)))
        .then(
          () => { marked += 1; },
          err => console.error(`❌ Backup: failed to mark interrupted snapshot ${id} failed: code=${err?.code ?? 'UNKNOWN'}`),
        );
    } finally {
      release();
    }
  }
  if (marked) console.log(`💾 Backup: marked ${marked} interrupted snapshot(s) failed`);
}

/**
 * Delete the oldest COMPLETED snapshots in the CURRENT machine's source
 * namespace beyond `retentionCount`, plus every interrupted snapshot older
 * than the newest healthy completed one. Interrupted snapshots never count
 * toward the budget and are pruned even under unlimited retention: they can
 * never be restored or downloaded, so they only fill the drive.
 * Called only from `runBackup()` after a run reaches a completed snapshot
 * with a successful or explicitly skipped database dump — a run that fails
 * earlier never reaches this call, so an in-progress snapshot is never a
 * candidate; state is re-checked here anyway via `snapshotState` in case an
 * unrelated run left one behind. Ordinary `.failed` snapshots are kept for
 * salvage. Only ever walks `snapshots/<MACHINE_HOST>/` — never another
 * machine's namespace and never the legacy pre-namespace root, both of which
 * this install has no authority to prune.
 * `retentionCount` of `null`/`undefined` means unlimited completed snapshots.
 * @param {string} destPath
 * @param {number|null} retentionCount
 * @returns {Promise<{ pruned: number }>}
 */
async function pruneOldSnapshots(destPath, retentionCount) {
  const unlimited = retentionCount === null || retentionCount === undefined;
  const sourceRoot = join(destPath, 'snapshots', MACHINE_HOST);
  // A completed snapshot was just written here; even ENOENT is a failed
  // inventory, not proof that there is nothing to prune. Keep paths private.
  const entries = await readdir(sourceRoot, { withFileTypes: true }).catch(cause => {
    throw inventoryReadError(cause, 'read-retention-namespace');
  });
  const candidateIds = entries
    .filter(entry => entry.isDirectory() && !entry.name.startsWith('.') && SNAPSHOT_ID_PATTERN.test(entry.name))
    .map(entry => entry.name);

  const descriptors = (await Promise.all(candidateIds.map(async (id) => {
    const snapshotDir = join(sourceRoot, id);
    const { incomplete, failed, interrupted } = await snapshotState(snapshotDir, id, true);
    if (interrupted) return { id, snapshotDir, interrupted: true };
    if (incomplete || failed) return null;
    // Unlimited retention prunes only interrupted snapshots; completed ones are
    // never candidates, so their manifests need no admission check.
    if (unlimited) return { id, snapshotDir };
    // Share restore's structural admission, without hashing old payloads.
    // Only confirmed absence is legacy; JSON null is an invalid manifest.
    const manifestRead = await readJSONFileStrict(join(snapshotDir, 'manifest.json'), MANIFEST_ABSENT, { logError: false });
    const valid = manifestRead.ok && (manifestRead.value === MANIFEST_ABSENT
      || manifestDataEntries(manifestRead.value, join(snapshotDir, 'data'), null) !== null);
    return { id, snapshotDir, protectedFromPrune: !valid };
  }))).filter(Boolean);

  // Newest-first by snapshot id (an ISO-style timestamp that never depends on
  // reading the manifest). IDs are unique within this source namespace.
  descriptors.sort((a, b) => b.id.localeCompare(a.id));

  // Keep protected snapshots in addition to the readable retention budget,
  // even when a bad clock placed a protected snapshot ahead of healthy ones.
  const healthy = descriptors.filter(d => {
    if (d.interrupted) return false;
    if (d.protectedFromPrune) {
      console.warn(`⚠️ Backup retention: kept snapshot ${d.id} (manifest unreadable or invalid)`);
      return false;
    }
    return true;
  });
  // An interrupted snapshot is only dropped once a newer healthy one exists.
  const newestHealthyId = healthy[0]?.id;
  const toDelete = [
    ...(newestHealthyId ? descriptors.filter(d => d.interrupted && d.id.localeCompare(newestHealthyId) < 0) : []),
    ...(unlimited ? [] : healthy.slice(retentionCount)),
  ];
  let pruned = 0;
  for (const { snapshotDir, id } of toDelete) {
    // A snapshot a restore or download is reading stays; a later run prunes it
    // once the reader finishes (#10898).
    const { release, refusal } = await claimSnapshotLease(snapshotDir, 'delete');
    if (refusal) {
      console.warn(`⚠️ Backup retention: skipped snapshot ${id} (${refusal})`);
      continue;
    }
    await rm(snapshotDir, { recursive: true, force: true }).then(
      () => { pruned += 1; },
      err => console.error(`❌ Backup retention: failed to remove snapshot ${id}: ${err.message}`),
    );
    release();
  }
  if (pruned) {
    console.log(`💾 Backup retention: pruned ${pruned} snapshot(s), keeping ${retentionCount ?? 'all'} completed`);
  }
  return { pruned };
}

function resolveSnapshotPath(destPath, snapshotId, source) {
  if (!snapshotId || !SNAPSHOT_ID_PATTERN.test(snapshotId)) {
    throw new ServerError(`Invalid snapshotId: ${snapshotId}`, {
      status: 400,
      code: 'VALIDATION_ERROR',
    });
  }

  const resolvedSource = source ?? MACHINE_HOST;
  if (!isSafeSnapshotSource(resolvedSource)) {
    throw new ServerError(`Invalid snapshot source: ${resolvedSource}`, {
      status: 400,
      code: 'VALIDATION_ERROR',
    });
  }

  const snapshotsRoot = resolve(join(destPath, 'snapshots'));
  const sourceRoot = resolvedSource === LEGACY_SNAPSHOT_SOURCE
    ? snapshotsRoot
    : resolve(join(snapshotsRoot, resolvedSource));
  const sourceRel = relative(snapshotsRoot, sourceRoot);
  if (resolvedSource !== LEGACY_SNAPSHOT_SOURCE
      && (!sourceRel || sourceRel.startsWith('..') || isAbsolute(sourceRel))) {
    throw new ServerError(`Path traversal detected for snapshot source: ${resolvedSource}`, {
      status: 400,
      code: 'VALIDATION_ERROR',
    });
  }

  const snapshotDir = resolve(join(sourceRoot, snapshotId));
  const rel = relative(sourceRoot, snapshotDir);
  if (!rel || rel.startsWith('..') || isAbsolute(rel)) {
    throw new ServerError(`Path traversal detected for snapshotId: ${snapshotId}`, {
      status: 400,
      code: 'VALIDATION_ERROR',
    });
  }

  return {
    snapshotsRoot: sourceRoot,
    snapshotsBase: snapshotsRoot,
    snapshotDir,
    resolvedSource,
    currentSource: resolvedSource === MACHINE_HOST,
    explicitSource: source !== undefined,
  };
}

async function assertExplicitSnapshotSourceSafe({
  snapshotsBase,
  snapshotsRoot,
  snapshotDir,
  explicitSource,
}, snapshotId) {
  if (!explicitSource) return;

  const [baseInfo, sourceInfo, snapshotInfo] = await Promise.all([
    lstat(snapshotsBase).catch(() => null),
    lstat(snapshotsRoot).catch(() => null),
    lstat(snapshotDir).catch(() => null),
  ]);
  if (baseInfo?.isSymbolicLink?.()
      || sourceInfo?.isSymbolicLink?.()
      || snapshotInfo?.isSymbolicLink?.()) {
    throw new ServerError(`Snapshot source escapes through a symbolic link: ${snapshotId}`, {
      status: 400,
      code: 'VALIDATION_ERROR',
    });
  }
}

/**
 * Permanently delete one snapshot, identified by its (source, id) pair.
 * Immediate — there is no undo — so this is reserved for an explicit operator
 * action, never automatic retention (see `pruneOldSnapshots`, which shares the
 * traversal/symlink guards through `resolveSnapshotPath` but is driven by
 * `runBackup()` instead). Refuses a snapshot that is still being written
 * (`incomplete`), matching the guard that blocks restore and download,
 * because the partial directory may still be owned by an in-flight
 * `runBackup()`. A `.failed` or interrupted snapshot IS deletable — unlike
 * restore, deletion has no reason to require a successful backup. A snapshot
 * a restore or download is still reading is refused with `SNAPSHOT_IN_USE`.
 * @param {string} destPath - Path to external drive backup root
 * @param {string} snapshotId - Snapshot ID to delete
 * @param {{ source?: string }} [options]
 * @returns {Promise<{ deleted: true, snapshotId: string, source: string }>}
 */
export async function deleteSnapshot(destPath, snapshotId, { source } = {}) {
  const resolved = resolveSnapshotPath(destPath, snapshotId, source);
  const { snapshotDir, currentSource, resolvedSource } = resolved;
  await assertExplicitSnapshotSourceSafe(resolved, snapshotId);
  // Own the snapshot BEFORE the final existence and in-progress checks: an
  // active reader refuses this deletion (SNAPSHOT_IN_USE), and once admitted no
  // new reader can start until the directory is gone (#10898).
  const release = await acquireSnapshotLease(snapshotDir, snapshotId, 'delete');
  try {
    const info = await stat(snapshotDir).catch(() => null);
    if (!info?.isDirectory?.()) {
      throw new ServerError(`Snapshot not found: ${snapshotId}`, { status: 404, code: 'NOT_FOUND' });
    }
    const { incomplete } = await snapshotState(snapshotDir, snapshotId, currentSource);
    if (incomplete) {
      throw new ServerError(`Snapshot is still being written: ${snapshotId}`, {
        status: 409,
        code: 'SNAPSHOT_INCOMPLETE',
      });
    }
    await rm(snapshotDir, { recursive: true, force: true });
  } finally {
    release();
  }
  dashboardEvents.emit('backup:changed');
  console.log(`💾 Backup snapshot deleted: ${resolvedSource}/${snapshotId}`);
  return { deleted: true, snapshotId, source: resolvedSource };
}

/**
 * Open a gzip tar stream for one complete snapshot.
 * @param {string} destPath - Path to external drive backup root
 * @param {string} snapshotId - Snapshot ID to archive
 * @returns {Promise<import('stream').Readable>}
 */
export async function openSnapshotStream(destPath, snapshotId, { source } = {}) {
  const resolved = resolveSnapshotPath(destPath, snapshotId, source);
  const { snapshotsRoot, snapshotDir, currentSource } = resolved;
  await assertExplicitSnapshotSourceSafe(resolved, snapshotId);
  // The read lease outlives this call: tar reads the snapshot until its child
  // closes, however the download ends (#10898).
  const releaseRead = await acquireSnapshotLease(snapshotDir, snapshotId, 'read');
  let proc;
  try {
    const info = await stat(snapshotDir).catch(() => null);
    if (!info?.isDirectory?.()) {
      throw new ServerError(`Snapshot not found: ${snapshotId}`, { status: 404, code: 'NOT_FOUND' });
    }
    await assertSnapshotComplete(snapshotDir, snapshotId, currentSource);

    // tar's stderr is a pipe (spawn's default) and MUST be drained: left unread
    // it fills its ~64KB buffer on a tree that warns a lot — files changing under
    // the archiver, unreadable modes — and tar then blocks on the write forever,
    // hanging the download with the process still alive. Keep the tail so a
    // non-zero exit can say why rather than just reporting the code.
    proc = spawn('tar', ['-czf', '-', '-C', snapshotsRoot, snapshotId], { shell: false });
  } catch (err) {
    releaseRead();
    throw err;
  }
  proc.on('close', releaseRead);
  // A child that never spawned has no pid and nothing reading the snapshot.
  // Any other error (a failed kill) leaves the child alive until `close`.
  proc.on('error', () => { if (proc.pid === undefined) releaseRead(); });
  const archive = new PassThrough();
  let stderrTail = '';
  proc.stderr?.on('data', (chunk) => { stderrTail = (stderrTail + chunk).slice(-500); });

  let finished = false;
  const finish = (error = null) => {
    if (finished) return;
    finished = true;
    if (error) archive.destroy(error);
    else archive.end();
  };

  // Keep the response open until tar's close event. stdout can end before tar
  // reports a read/permission failure; delaying EOF lets the route turn that
  // non-zero exit into a failed download instead of a false 200 success.
  proc.stdout.on('error', finish);
  proc.stdout.pipe(archive, { end: false });
  proc.on('error', finish);
  proc.on('close', (code, signal) => {
    if (code === 0) {
      finish();
      return;
    }
    const detail = code == null ? `signal ${signal || 'unknown'}` : `code ${code}`;
    finish(new Error(`tar exited with ${detail}${stderrTail ? `: ${stderrTail.trim()}` : ''}`));
  });
  archive.abort = () => {
    // Escalate like every other spawn-based job: the backup destination is an
    // external/network mount, so a tar wedged on stalled I/O is the exact case
    // SIGTERM alone does not clear.
    if (proc.exitCode === null && proc.signalCode === null) {
      // stillRunning is `true` on purpose: the helper already skips SIGKILL once
      // the child has exited, and this proc is captured in the closure so there is
      // no handle that could be swapped out from under us. Gating on the stream's
      // own state instead would read as false by the time the timer fires (finish()
      // runs on the next line) and silently disable the escalation.
      killWithEscalation(proc, { label: `snapshot download ${snapshotId}`, stillRunning: () => true });
    }
    finish(new Error(`Snapshot download aborted: ${snapshotId}`));
  };
  return archive;
}

// pg rejects an unreachable database with an AggregateError whose own message
// is empty; the useful text (e.g. ECONNREFUSED) lives on the nested errors.
function describeReconcileFailure(reason) {
  if (reason?.message) return reason.message;
  if (Array.isArray(reason?.errors)) {
    const parts = [...new Set(reason.errors.map(err => err?.code || err?.message).filter(Boolean))];
    if (parts.length > 0) return parts.join(', ');
  }
  return reason?.code || String(reason);
}

async function reconcileLiveFileRestore(subdirFilter) {
  const refreshes = [
    ...(!subdirFilter || subdirFilter === 'brain' || subdirFilter.startsWith('brain/')
      ? [{ label: 'Brain cache invalidation', run: invalidateBrainCaches }]
      : []),
    { label: 'settings reload', run: reloadSettings },
    ...((!subdirFilter || ['images', 'videos', 'video-thumbnails', 'video-history.json']
      .some(path => subdirFilter === path || subdirFilter.startsWith(`${path}/`)))
      && !usesFileEscapeHatch()
      ? [{ label: 'media index rebuild', run: async () => {
        const { reconcileMediaAssets } = await import('./mediaAssetIndex/db.js');
        await reconcileMediaAssets({ rebuild: true });
      } }] : []),
    // Migration 359 converts this plan once, at boot, so a plan restored from a
    // pre-359 snapshot would otherwise stay legacy and unrunnable: a folder
    // restore keeps the live ledger (359 never re-runs), and a full restore's
    // ledger only re-runs it on the next boot.
    ...(!subdirFilter || subdirFilter === 'cos' || subdirFilter === 'cos/quota-burn.json'
      ? [{ label: 'quota-burn plan conversion', run: async () => {
        const { reconcileRestoredQuotaBurnPlan } = await import('./quotaBurnConversion.js');
        await reconcileRestoredQuotaBurnPlan();
      } }] : []),
  ];
  const results = await Promise.allSettled(
    refreshes.map(({ run }) => Promise.resolve().then(run)),
  );
  const failures = results.flatMap((result, index) => result.status === 'rejected'
    ? [`${refreshes[index].label}: ${describeReconcileFailure(result.reason)}`]
    : []);

  if (failures.length > 0) {
    throw new Error(
      `Live restore cache reconciliation failed (${failures.join('; ')}). Restart PortOS before relying on restored settings, Brain data or media.`,
      { cause: results.find(result => result.status === 'rejected').reason },
    );
  }
}

function manifestDataEntries(manifest, srcDir, subdirFilter, isSkippedPath = isRestorePreservedPath) {
  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)
      || !manifest.files || typeof manifest.files !== 'object' || Array.isArray(manifest.files)
      || !Number.isInteger(manifest.fileCount) || manifest.fileCount < 0
      || typeof manifest.generatedAt !== 'string' || Number.isNaN(Date.parse(manifest.generatedAt))) {
    return null;
  }

  const entries = Object.entries(manifest.files);
  if (manifest.fileCount !== entries.length) return null;

  const selected = [];
  const dataPaths = new Set();
  for (const [entry, expectedHash] of entries) {
    if (typeof expectedHash !== 'string' || !SHA256_PATTERN.test(expectedHash)) return null;
    if (entry === '../portos-db.sql') continue;

    // Manifests are portable across supported platforms, so normalize both
    // separator forms before validating or comparing a literal filter. Validate
    // before resolving: resolve() would otherwise erase evidence of `..`.
    const normalized = entry.replaceAll('\\', '/');
    const segments = normalized.split('/');
    if (!normalized || normalized.startsWith('/') || /^[A-Za-z]:\//.test(normalized)
        || segments.some(segment => !segment || segment === '.' || segment === '..')) {
      return null;
    }

    const filePath = resolve(srcDir, ...segments);
    const rel = relative(resolve(srcDir), filePath);
    if (!rel || rel.startsWith('..') || isAbsolute(rel)) return null;
    if (dataPaths.has(normalized)) return null;
    dataPaths.add(normalized);

    // Still recorded in dataPaths (inventory consistency), but never hashed:
    // Finder may rewrite a hashed metadata file that restore never transfers.
    if (isSkippedPath(normalized) || isOsMetadataFile(segments.at(-1))) continue;
    if (!subdirFilter || normalized === subdirFilter || normalized.startsWith(`${subdirFilter}/`)) {
      selected.push({ filePath, expectedHash });
    }
  }

  return { selected, dataPaths };
}

/**
 * Filesystem metadata the OS writes into a directory on its own, after the
 * manifest was sealed. The backup destination is commonly an iCloud/Finder
 * folder (see `listSnapshots`), so simply BROWSING a snapshot in Finder drops a
 * `.DS_Store` beside the data — which the unmanifested-file check below would
 * then read as tampering and refuse to restore, permanently, on a snapshot that
 * is byte-for-byte intact. These names are skipped by the inventory AND
 * excluded from the restore transfer, so "everything transferred was verified"
 * still holds.
 */
const OS_METADATA_FILES = new Set(['.DS_Store', '.localized', 'Thumbs.db', 'desktop.ini']);
const isOsMetadataFile = (name) => OS_METADATA_FILES.has(name) || name.startsWith('._');

/**
 * rsync filter form of `OS_METADATA_FILES`. Deliberately UNANCHORED, unlike
 * every path exclude in `DEFAULT_EXCLUDES`: these are basenames the OS writes
 * into any directory, so matching at every depth is the point, not a bug.
 */
const OS_METADATA_RSYNC_EXCLUDES = [...OS_METADATA_FILES, '._*'].map(name => `--exclude=${name}`);

/**
 * Data-root files a restore never installs, though a snapshot keeps them as
 * recovery evidence. `database-authority.json` names which database backend a
 * completed cutover retired ON THE MACHINE THAT RAN IT. Installing another
 * machine's record (or an older snapshot's, after a reverse cutover) would fence
 * a healthy local backend as `DATABASE_RETIRED_BACKEND`, including across
 * restarts. The destination keeps whatever authority it already has,
 * byte-for-byte: absent stays absent, damaged stays damaged. Root-anchored
 * relative names, like `DEFAULT_EXCLUDES`; this one list feeds the rsync
 * filter, the manifest selection and the scope inventory so preview,
 * verification and execution cannot disagree.
 */
const RESTORE_PRESERVED_FILES = Object.freeze([
  'database-authority.json', 'peer-execution', 'peer-execution-catalog.json', 'peer-execution-grants.json', 'peer-execution-authority.json', 'peer-execution-recovery.jsonl', 'workflow-maintenance', 'backup-admission',
]);
// Matched case-insensitively (rsync has no such flag, so each letter becomes a
// `[xX]` class): on a case-insensitive volume a `Database-Authority.json` entry
// would otherwise overwrite the destination's lowercase record.
const caseInsensitiveGlob = (name) => name.replace(/[a-z]/gi, ch => `[${ch.toLowerCase()}${ch.toUpperCase()}]`);
const RESTORE_PRESERVED_RSYNC_EXCLUDES = RESTORE_PRESERVED_FILES.map(name => `--exclude=/${caseInsensitiveGlob(name)}`);
const isRestorePreservedPath = (relativePath) => {
  if (typeof relativePath !== 'string') return false;
  const normalized = relativePath.toLowerCase();
  return RESTORE_PRESERVED_FILES.some(path => normalized === path || normalized.startsWith(`${path}/`));
};

// A filter naming a preserved file would otherwise be a silent no-op restore.
// Compared case-insensitively after dropping empty/`.` segments, so
// `./Database-Authority.json/` cannot slip past on a case-insensitive volume.
const restoreScopeIsPreservedFile = (subdirFilter) => {
  const normalized = subdirFilter?.split('/').filter(part => part && part !== '.').join('/').toLowerCase();
  return isRestorePreservedPath(normalized);
};

/**
 * `instances.json` holds this install's federation identity (`self.instanceId`),
 * its peers and their pair secrets. A shared destination offers every machine's
 * snapshots, so installing another machine's file would give two live installs
 * one identity: each claims the other's CoS tasks and signs pushes with the
 * other's pair credentials (#12596). A restore therefore installs it only when
 * the identities match, the live install has none, or the operator adopts the
 * snapshot's identity (replacing a retired machine). Otherwise it is preserved
 * through the same three places as RESTORE_PRESERVED_FILES.
 */
const IDENTITY_FILE = 'instances.json';
const IDENTITY_RSYNC_EXCLUDE = `--exclude=/${caseInsensitiveGlob(IDENTITY_FILE)}`;
const isIdentityPath = relativePath => typeof relativePath === 'string' && relativePath.toLowerCase() === IDENTITY_FILE;
const IDENTITY_ABSENT = Symbol('identity-absent');

async function readIdentityFile(path) {
  const { ok, value } = await readJSONFileStrict(path, IDENTITY_ABSENT, { logError: false });
  if (ok && value === IDENTITY_ABSENT) return { present: false, readable: true, instanceId: null };
  const instanceId = ok ? value?.self?.instanceId : null;
  return { present: true, readable: ok, instanceId: typeof instanceId === 'string' && instanceId ? instanceId : null };
}

/**
 * Whether this restore installs the snapshot's instances.json. Decided before
 * the integrity preflight so preview and execution exclude the same bytes. An
 * unreadable live file counts as an identity (fail closed): only adoption
 * replaces it, since its owner cannot be known.
 */
async function resolveIdentityRestore(srcDir, subdirFilter, adoptIdentity) {
  const notInstalled = { installs: false, preserved: null, adopted: null };
  if (subdirFilter && !isIdentityPath(subdirFilter)) return notInstalled;
  const snapshot = await readIdentityFile(join(srcDir, IDENTITY_FILE));
  if (!snapshot.present) return notInstalled;
  const live = await readIdentityFile(join(PATHS.data, IDENTITY_FILE));
  const liveHasIdentity = live.present && (live.instanceId !== null || !live.readable);
  if (!liveHasIdentity || (live.instanceId !== null && live.instanceId === snapshot.instanceId)) {
    return { installs: true, preserved: null, adopted: null };
  }
  const ids = { liveInstanceId: live.instanceId, snapshotInstanceId: snapshot.instanceId };
  return adoptIdentity
    ? { installs: true, preserved: null, adopted: ids }
    : { ...notInstalled, preserved: ids };
}

// `beforeTransfer`: nothing was written, so the caller must not report a partial restore.
const identityChangedError = () => Object.assign(new ServerError(
  'This machine\'s federation identity (instances.json) changed after the restore was checked. Nothing was restored; preview the restore again.',
  { status: 409, code: 'BACKUP_RESTORE_IDENTITY_CHANGED' },
), { beforeTransfer: true });

const snapshotFileIntegrityError = (snapshotId, unmanifestedPath = null) => new ServerError(
  // Name the offending entry: without it the operator is told their only backup
  // failed integrity and given nothing to act on.
  `Snapshot file integrity check failed: ${snapshotId}${unmanifestedPath ? ` (unmanifested file: ${unmanifestedPath})` : ''}`,
  { status: 409, code: 'BACKUP_FILE_INTEGRITY_FAILED' },
);

/**
 * Rebuild the in-scope regular-file inventory the manifest describes.
 * `paths` feed the unmanifested-file check; `largestFileBytes` is free here —
 * the same `stat`s produce it — and sizes the restore's idle-deadline floor
 * (rsync `--checksum` digests only regular files in scope, so the largest one
 * bounds the longest explainable silence between `-ii` heartbeats).
 */
async function snapshotScopeInventory(srcDir, subdirFilter, isSkippedPath = isRestorePreservedPath) {
  const scopePath = subdirFilter
    ? resolve(srcDir, ...subdirFilter.split('/'))
    : srcDir;
  const scopeInfo = await lstat(scopePath).catch(err => {
    if (err?.code === 'ENOENT') return null;
    throw err;
  });
  if (!scopeInfo) return { paths: [], largestFileBytes: 0 };

  const candidates = scopeInfo.isDirectory()
    ? (await readdir(scopePath, { recursive: true })).map(entry => join(scopePath, entry))
    : [scopePath];
  const paths = [];
  let largestFileBytes = 0;

  for (const filePath of candidates) {
    const info = await stat(filePath).catch(async err => {
      // Match generateManifest(): dangling links are intentionally outside the
      // manifest, while readable links to regular files are hashed and tracked.
      if (err?.code === 'ENOENT') {
        const entryInfo = await lstat(filePath);
        if (entryInfo.isSymbolicLink()) return null;
      }
      throw err;
    });
    if (!info?.isFile()) continue;
    // OS metadata never reaches rsync (OS_METADATA_RSYNC_EXCLUDES), so it must
    // not inflate the digest floor either.
    if (isOsMetadataFile(basename(filePath))) continue;
    const scopedPath = relative(srcDir, filePath).replaceAll('\\', '/');
    if (isSkippedPath(scopedPath)) continue;
    if (info.size > largestFileBytes) largestFileBytes = info.size;

    const normalized = relative(srcDir, filePath).replaceAll('\\', '/');
    if (!normalized || normalized.startsWith('../') || isAbsolute(normalized)) {
      throw new Error('Snapshot entry escaped the data directory');
    }
    paths.push(normalized);
  }

  return { paths, largestFileBytes };
}

// These exact metadata shapes belonged to six marker-gated file→DB imports.
// Sibling assets and parked .imported evidence remain file-primary recovery data.
const LEGACY_DB_METADATA = [
  ['universes', /^universes\/[^/]+\/index\.json$/],
  ['pipeline-issues', /^pipeline-issues\/iss-[A-Za-z0-9-]+\/index\.json$/],
  ['pipeline-series', /^pipeline-series\/ser-[A-Za-z0-9-]+\/index\.json$/],
  ['story-builder', /^story-builder\/stb-[A-Za-z0-9-]+\/index\.json$/],
  ['writers-room', /^writers-room\/(?:(?:folders|exercises)\.json|works\/[^/]+\/manifest\.json)$/],
  ['creative-director', /^creative-director-projects\.json$/],
];

async function assertRestoreStorageCompatible(srcDir, inventory, snapshotId) {
  // Match dumpPostgres: an uninitialized backend still requires PostgreSQL.
  if (usesFileEscapeHatch()) return;
  if (!inventory) throw snapshotFileIntegrityError(snapshotId);
  const domains = new Set(LEGACY_DB_METADATA
    .filter(([, pattern]) => inventory.paths.some(path => pattern.test(path)))
    .map(([domain]) => domain));
  if (inventory.paths.includes('universes/index.json')) {
    const index = await readJSONFileStrict(join(srcDir, 'universes', 'index.json'), null, { logError: false });
    if (!index.ok || index.value?.config?.runs !== undefined) domains.add('universes');
  }
  if (domains.size) {
    throw new ServerError(
      `Historical file metadata for ${[...domains].join(', ')} cannot restore database-primary records. Convert this backup in a disposable offline upgrade install, verify its records, then restore a current-format file/database backup (docs/BACKUP.md).`,
      { status: 409, code: 'BACKUP_LEGACY_DB_METADATA', context: { domains: [...domains] } },
    );
  }
}

async function verifySnapshotFiles(snapshotDir, srcDir, snapshotId, subdirFilter, { preserveIdentity = false } = {}) {
  const isSkippedPath = preserveIdentity
    ? path => isRestorePreservedPath(path) || isIdentityPath(path)
    : isRestorePreservedPath;
  // A unique fallback distinguishes a genuinely absent legacy manifest from an
  // existing file whose parsed value is JSON null.
  const manifestRead = await readJSONFileStrict(
    join(snapshotDir, 'manifest.json'),
    MANIFEST_ABSENT,
    { logError: false },
  );
  if (!manifestRead.ok) {
    throw new ServerError(`Snapshot integrity manifest is unreadable: ${snapshotId}`, {
      status: 409,
      code: 'BACKUP_MANIFEST_UNREADABLE',
    });
  }
  if (manifestRead.value === MANIFEST_ABSENT) {
    // No manifest to verify against, but the restore's idle-deadline floor
    // still needs the largest in-scope file. PostgreSQL also needs a complete
    // inventory to reject ignored legacy metadata; only the file escape hatch
    // can retain the old best-effort behavior when inventory fails.
    const legacyInventory = await snapshotScopeInventory(srcDir, subdirFilter, isSkippedPath).catch(() => null);
    await assertRestoreStorageCompatible(srcDir, legacyInventory, snapshotId);
    return {
      status: 'unverified',
      reason: 'manifest_absent',
      checkedFiles: 0,
      largestFileBytes: legacyInventory?.largestFileBytes ?? 0,
    };
  }

  const manifestEntries = manifestDataEntries(manifestRead.value, srcDir, subdirFilter, isSkippedPath);
  if (!manifestEntries) {
    throw new ServerError(`Snapshot integrity manifest is invalid: ${snapshotId}`, {
      status: 409,
      code: 'BACKUP_MANIFEST_INVALID',
    });
  }
  const { selected, dataPaths } = manifestEntries;

  // Hash agreement is insufficient if rsync can also copy files the manifest
  // never recorded. Rebuild the same regular-file inventory in the selected
  // scope and reject additions before rsync reads or overwrites anything.
  const inventory = await snapshotScopeInventory(srcDir, subdirFilter, isSkippedPath)
    .catch(() => null);
  if (!inventory) throw snapshotFileIntegrityError(snapshotId);
  await assertRestoreStorageCompatible(srcDir, inventory, snapshotId);
  const { paths: snapshotPaths, largestFileBytes } = inventory;
  const unmanifested = snapshotPaths.find(entry => !dataPaths.has(entry));
  if (unmanifested) throw snapshotFileIntegrityError(snapshotId, unmanifested);

  for (const { filePath, expectedHash } of selected) {
    const info = await lstat(filePath).catch(() => null);
    if (!info || (!info.isFile() && !info.isSymbolicLink())) {
      throw snapshotFileIntegrityError(snapshotId);
    }
    const actualHash = await sha256File(filePath).catch(() => null);
    if (!actualHash || actualHash !== expectedHash) {
      throw snapshotFileIntegrityError(snapshotId);
    }
  }

  return { status: 'verified', checkedFiles: selected.length, largestFileBytes };
}

/**
 * Idle deadline for a restore's rsync. `--checksum` digests BOTH copies of a
 * same-size file before rsync reports the entry, so the largest in-scope file
 * bounds the longest silence a healthy scan can produce: floor the deadline at
 * both copies over a conservative worst-case throughput. A scope holding only
 * small files keeps the default — the relaxed deadline is spent only where a
 * long silence is explainable (#7302).
 */
function restoreIdleTimeoutMs(largestFileBytes = 0) {
  const digestMs = (2 * largestFileBytes * 1000) / RESTORE_DIGEST_WORST_CASE_BPS;
  return Math.max(BACKUP_PROCESS_IDLE_TIMEOUT_MS, Math.ceil(digestMs));
}

/**
 * Restore a snapshot back to PATHS.data using rsync.
 * @param {string} destPath - Path to external drive backup root
 * @param {string} snapshotId - Snapshot ID to restore
 * @param {object} options
 * @param {boolean} [options.dryRun=true] - If true, do not write any files
 * @param {string|null} [options.subdirFilter=null] - Limit restore to a subdirectory
 * @param {boolean} [options.adoptIdentity=false] - Install a snapshot's different
 *   federation identity (instances.json) over the live one; only for replacing a
 *   retired machine. Without it the live identity is preserved (`identityPreserved`).
 */
export async function restoreSnapshot(destPath, snapshotId, { dryRun = true, subdirFilter: requestedFilter = null, source, adoptIdentity = false } = {}) {
  // Canonicalize once: manifest selection, the inventory and the rsync include
  // chain each spell the filter differently, so `brain/` or `./brain` would
  // select nothing yet verify clean. An all-dot/empty filter means no filter.
  const subdirFilter = typeof requestedFilter === 'string'
    ? requestedFilter.split('/').filter(part => part && part !== '.').join('/') || null
    : requestedFilter;
  const resolved = resolveSnapshotPath(destPath, snapshotId, source);
  await assertExplicitSnapshotSourceSafe(resolved, snapshotId);

  // Defense-in-depth for non-route callers (the route already validates via
  // subdirFilterSchema). subdirFilter is interpolated into an rsync include arg,
  // so a `*` would override the filter chain (restoring everything) and `..`
  // would traverse out of the snapshot subdir. Reuse the same predicate the
  // schema does so the two can't drift — see issue #1822.
  if (requestedFilter != null && (String(requestedFilter).startsWith('/') || (subdirFilter != null && !isSafeSubdirFilter(subdirFilter)))) {
    throw new Error(`Invalid subdirFilter: ${requestedFilter}`);
  }
  // Refused for previews too, before any verification or transfer, so a preview
  // never promises a restore that execution would silently skip.
  if (restoreScopeIsPreservedFile(subdirFilter)) {
    throw new ServerError(
      'These machine-local authority and recovery records (database-authority.json, peer-execution-authority.json, peer-execution-recovery.jsonl, workflow-maintenance and backup-admission) are never restored from a snapshot. Restore application records with a data or database restore; existing local authority and unresolved owners must be reconciled on this machine (see docs/STORAGE.md).',
      { status: 400, code: 'BACKUP_RESTORE_MACHINE_LOCAL' },
    );
  }

  // Hold the source from before the restorability and integrity preflight
  // until rsync has closed (and, for execution, caches are reconciled), so
  // neither explicit deletion nor retention can remove verified bytes while
  // rsync still reads them (#10898).
  return withSnapshotRead(resolved.snapshotDir, snapshotId,
    () => restoreHeldSnapshot(resolved, snapshotId, { dryRun, subdirFilter, adoptIdentity: adoptIdentity === true }));
}

async function restoreHeldSnapshot({ snapshotDir, currentSource }, snapshotId, { dryRun, subdirFilter, adoptIdentity }) {
  await assertSnapshotRestorable(snapshotDir, snapshotId, currentSource);
  const srcDir = join(snapshotDir, 'data');
  const identity = await resolveIdentityRestore(srcDir, subdirFilter, adoptIdentity);
  // Keyed on the approval, not on a mismatch: a snapshot instances.json that
  // appears after the check must not reach the live file unfenced.
  const preserveIdentity = !identity.installs;

  // Run the preflight independently for preview and execution. A preview is an
  // aid to confirmation, not an integrity lease: snapshot bytes may change
  // between requests, especially on removable or network-backed destinations.
  const verification = await verifySnapshotFiles(snapshotDir, srcDir, snapshotId, subdirFilter, { preserveIdentity });
  // A full restore installs the snapshot's ledger too, so the next boot re-runs
  // whatever it lacks; only a folder restore can leave restored files behind
  // migrations the live ledger already records.
  const ledgerGap = subdirFilter ? await migrationLedgerGap(srcDir) : undefined;
  // Split migrations gate on their output, so neither ledger re-runs them over
  // a restored monolith; restore converts those stores itself (#12626). A full
  // restore installs the snapshot's ledger, so the migrations it lacks before a
  // split are replayed with that split, in order (#12813).
  const legacyStores = await snapshotLegacySplitStores(srcDir, subdirFilter, {
    ledger: subdirFilter ? null : await readSnapshotMigrationLedger(srcDir),
  });
  const legacyLedger = await snapshotLegacyLedger(srcDir, subdirFilter);
  const restoreNotes = (restoredLegacy, restoredLegacyLedger = legacyLedger) => ({
    ...(ledgerGap !== undefined && { ledgerGap }),
    ...(restoredLegacy.stores.length > 0 && { restoredLegacyStores: restoredLegacy.stores }),
    ...(restoredLegacy.migrations && { restoredLegacyMigrations: restoredLegacy.migrations }),
    ...(restoredLegacyLedger && { restoredLegacyLedger }),
    ...(identity.preserved && { identityPreserved: identity.preserved }),
    ...(identity.adopted && { identityAdopted: identity.adopted }),
  });

  // Restore must compare destination bytes even when size and mtime match.
  // Rsync's default quick-check would otherwise report a successful no-op for
  // equal-length edits that retain the snapshot timestamp.
  //
  // `-ii` is what keeps that affordable. `--checksum` makes rsync digest BOTH
  // copies of every file in scope, and a file that turns out to match emits
  // nothing under plain `--itemize-changes`/`--progress` — those report only
  // transferring files. A preview of a large unchanged tree therefore runs
  // silent for as long as the digest takes, and runRsync's idle watchdog
  // (BACKUP_PROCESS_IDLE_TIMEOUT_MS) SIGKILLs a perfectly healthy restore.
  // Doubling itemize makes rsync report unchanged entries too (`.f <path>`),
  // one line per file digested, which is what keeps the watchdog fed. The
  // changed-file count runRsync returns is unaffected: the added lines start
  // with `.`, and runRsync collects only `>`/`<` transfer lines.
  //
  // It must be the SHORT `-ii`, and it is deliberately not `--info=progress2`:
  // macOS ships openrsync, where the long `--itemize-changes` does NOT stack
  // when repeated and `--info=progress2` is rejected outright as an unknown
  // option (it arrived in rsync 3.1). `-ii` behaves identically on openrsync
  // and rsync 3.x, so no version probe is needed. See issue #7299.
  //
  // Before the include chain below: rsync takes the FIRST matching rule, so an
  // exclude placed after `--include=/<filter>/***` would never be consulted.
  // These are the files `snapshotScopeInventory` skips — keeping the two in
  // step is what preserves "everything transferred was verified".
  // Same placement rule: excluded before the include chain, and kept out of the
  // manifest selection and inventory above, so what rsync may write is exactly
  // what the preflight verified.
  const flags = ['-ii', '--checksum', ...OS_METADATA_RSYNC_EXCLUDES, ...RESTORE_PRESERVED_RSYNC_EXCLUDES,
    ...(preserveIdentity ? [IDENTITY_RSYNC_EXCLUDE] : [])];
  if (dryRun) flags.push('--dry-run');
  if (subdirFilter) {
    // Anchored with a leading `/` — rsync matches an unanchored pattern against
    // the END of every path, so a bare `youtube/***` would also restore
    // `data/brain/youtube/**` over live files the user never selected. The
    // integrity preflight above scopes itself to `data/<filter>/**` only, so an
    // unanchored transfer overwrites bytes it never verified.
    // Exact files need their own rule: rsync 3 excludes a file from the directory-only /*** pattern.
    flags.push(`--include=/${subdirFilter}`);
    flags.push(`--include=/${subdirFilter}/***`);
    flags.push('--include=*/');
    flags.push('--exclude=*');
  }

  // The preflight just walked the scope, so its largest file is free — one
  // file whose digest outlasts the default deadline emits no `-ii` heartbeat
  // until it finishes (#7302), so this restore's idle floor scales to it.
  const idleTimeoutMs = restoreIdleTimeoutMs(verification.largestFileBytes);
  const transferFiles = () => runRsync(srcDir, PATHS.data, flags, { idleTimeoutMs });
  // A preview holds no owner, fences no identity and reconciles no cache.
  if (dryRun) {
    const changedFiles = await transferFiles();
    return { dryRun, snapshotId, subdirFilter, changedFiles, verification, ...restoreNotes(describeLegacySplitStores(legacyStores)) };
  }

  const scope = subdirFilter;
  console.log(`💾 Restore starting: snapshot ${snapshotId}${scope ? ` (filter ${scope})` : ''}`);
  let restoredLegacy = { stores: [], migrations: null };
  let restoredLegacyLedger = legacyLedger;
  // Conversion parks live stores, so it runs only after a successful transfer,
  // and before reconciliation so the Brain caches drop the parked records. It
  // follows ledger adoption so a replay reads the ledger the boot would read.
  const changedFiles = await withLiveFileRestoreOwners(scope, srcDir, () => transferAndReconcileLiveFiles({
    subdirFilter,
    transfer: async () => {
      const changed = await transferWithIdentityFence({
        installsIdentity: identity.installs,
        assertIdentityUnchanged: async () => {
          const current = await resolveIdentityRestore(srcDir, subdirFilter, adoptIdentity);
          if (JSON.stringify(current) !== JSON.stringify(identity)) throw identityChangedError();
        },
        transferFiles,
      });
      if (legacyLedger) restoredLegacyLedger = await adoptRestoredLegacyLedger(legacyLedger);
      restoredLegacy = await convertRestoredLegacySplitStores(legacyStores);
      return changed;
    },
  }));
  console.log(`💾 Restore complete: snapshot ${snapshotId}, ${changedFiles.length} files changed`);
  return { dryRun, snapshotId, subdirFilter, changedFiles, verification, ...restoreNotes(restoredLegacy, restoredLegacyLedger) };
}

async function readMigrationLedger(path) {
  const { ok, value } = await readJSONFileStrict(path, null, { logError: false });
  return ok && Array.isArray(value) ? value.filter(name => typeof name === 'string') : null;
}

const ledgerPaths = (dataDir) => ({ current: join(dataDir, APPLIED_LEDGER_RELPATH), legacy: join(dataDir, LEGACY_APPLIED_LEDGER_RELPATH) });
const fileExists = (path) => stat(path).then(info => info.isFile(), error => { if (error.code === 'ENOENT') return false; throw error; });

/**
 * The snapshot's migration ledger: `data/migrations.applied.json`, or the
 * `data/migrations/.applied.json` a snapshot from before the May 2026 move of
 * migrations out of `data/` still carries. Null when neither is readable.
 */
async function readSnapshotMigrationLedger(srcDir) {
  const { current, legacy } = ledgerPaths(srcDir);
  return (await readMigrationLedger(current)) ?? (await readMigrationLedger(legacy));
}

/**
 * Whether a full restore of this snapshot must adopt its ledger by hand: the
 * snapshot carries the ledger only at the legacy path. rsync lands that file
 * beside the live `migrations.applied.json` instead of over it, and the runner
 * then keeps the live ledger (a current ledger is authoritative over a legacy
 * one), so nothing would re-run over the restored records (#12355 assumed the
 * two paths coincide). Null when the snapshot carries a current ledger or none.
 * A legacy ledger that does not parse as a list is reported `unreadable` and
 * never adopted: the live ledger is worth more than a broken one, which the
 * runner would rename aside and rebuild from scratch.
 */
async function snapshotLegacyLedger(srcDir, subdirFilter) {
  if (subdirFilter) return null;
  const { current, legacy } = ledgerPaths(srcDir);
  if (await fileExists(current) || !await fileExists(legacy)) return null;
  return { status: await readMigrationLedger(legacy) ? 'pending' : 'unreadable', parkedAs: null };
}

/**
 * Install the restored legacy ledger as the live one, after the transfer and
 * before cache reconciliation: the live ledger is parked beside it (evidence,
 * like a parked split store), and the next migration run starts from the
 * snapshot's list, exactly as a full restore with a current-path ledger does.
 */
async function adoptRestoredLegacyLedger(legacyLedger) {
  if (legacyLedger.status !== 'pending') {
    console.warn(`⚠️ Restore left the live migration ledger in place: the snapshot's ${LEGACY_APPLIED_LEDGER_RELPATH} is unreadable`);
    return legacyLedger;
  }
  const { current, legacy } = ledgerPaths(PATHS.data);
  // Confirmed on the transferred copy before the live ledger moves: parking it
  // and then failing would leave no ledger (or a broken one), which the next
  // boot reads as a fresh install: every migration re-runs and purge migrations
  // are disarmed.
  if (!await readMigrationLedger(legacy)) {
    throw new Error(`Restore transferred no readable ${LEGACY_APPLIED_LEDGER_RELPATH}; the live migration ledger was left unchanged`);
  }
  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
  const parkedAs = `${basename(current)}.pre-restore-${stamp}`;
  const parked = await rename(current, join(dirname(current), parkedAs)).then(
    () => true,
    error => { if (error.code === 'ENOENT') return false; throw error; },
  );
  await rename(legacy, current);
  console.log(`📦 Restore adopted the snapshot's pre-May-2026 migration ledger${parked ? `; previous ledger parked as ${parkedAs}` : ''}`);
  return { status: 'adopted', parkedAs: parked ? parkedAs : null };
}

/**
 * Migrations the live install has applied that the snapshot's ledger lacks:
 * files a folder restore installs from that snapshot may predate them, and no
 * runner revisits a migration the live ledger already records. `null` means
 * unknown — the snapshot (or the live install) has no readable ledger.
 */
async function migrationLedgerGap(srcDir) {
  const snapshotLedger = await readSnapshotMigrationLedger(srcDir);
  const liveLedger = snapshotLedger && await readMigrationLedger(ledgerPaths(PATHS.data).current);
  if (!snapshotLedger || !liveLedger) return null;
  const recorded = new Set(snapshotLedger);
  const names = liveLedger.filter(name => !recorded.has(name));
  return { count: names.length, names };
}

const COS_RESTORE_SCOPES = ['cos', 'cos/config.json', 'cos/state.json', 'cos/agents'];

/**
 * The CoS config a restore of `scope` installs from a snapshot taken before the
 * #6182 split: such a snapshot has no cos/config.json and keeps the settings in
 * the `config` slice of cos/state.json. Null when the snapshot carries its own
 * config.json or the scope leaves cos/state.json alone.
 */
async function snapshotLegacyCosConfig(srcDir, scope) {
  if (scope && scope !== 'cos' && scope !== 'cos/state.json') return null;
  const carriesConfigFile = await stat(join(srcDir, 'cos', 'config.json')).then(
    info => info.isFile(),
    error => { if (error.code === 'ENOENT') return false; throw error; },
  );
  if (carriesConfigFile) return null;
  const { ok, value } = await readJSONFileStrict(join(srcDir, 'cos', 'state.json'), null, { logError: false });
  const config = ok ? value?.config : null;
  return config && typeof config === 'object' && !Array.isArray(config) ? config : null;
}

/**
 * Owners a live file restore holds, OUTERMOST FIRST — this array order IS the
 * acquisition order, and each owner is released in reverse after everything
 * inside it (transfer plus partial-failure reconciliation) has settled.
 * `appliesTo` receives the normalized scope (`''`/undefined = full restore);
 * `hold(inner, { srcDir, scope })` acquires the owner, runs `inner`, and
 * releases on every exit; `srcDir` is the verified snapshot's data directory. Domain
 * modules are imported only once every outer owner is held.
 *
 * - The snapshot cut drains in-flight asset publications BEFORE any domain
 *   queue they may need to finish, and keeps admission closed throughout.
 * - Schedule mutations can await CoS state, so their queue precedes CoS.
 * - The settings queue stays held through CoS reconciliation.
 * - CoS acquires config then runtime internally (`withLiveCosRestore`) and
 *   refuses a busy daemon/mind/agent with `COS_RESTORE_BUSY`.
 * - The media registry refuses edits for the whole hold and is acquired after
 *   CoS, so its write fence never outlives a refused CoS restore.
 * - Apple Health admission is acquired last, for the same reason: it closes
 *   import/archive admission and drains admitted day cycles (which wait on no
 *   other owner) before the transfer, then reopens after cache invalidation.
 */
const LIVE_FILE_RESTORE_OWNERS = Object.freeze([
  {
    name: 'backup snapshot cut',
    appliesTo: () => true,
    hold: async (inner) => {
      const releaseSnapshotCut = await acquireBackupSnapshotCut();
      try {
        return await inner();
      } finally {
        releaseSnapshotCut();
      }
    },
  },
  {
    name: 'task schedule',
    appliesTo: scope => !scope || scope === 'cos' || scope === 'cos/task-schedule.json',
    hold: async (inner) => {
      const { withLiveTaskScheduleRestore } = await import('./taskScheduleStore.js');
      return withLiveTaskScheduleRestore(inner);
    },
  },
  {
    name: 'settings',
    appliesTo: scope => !scope || scope === 'settings.json',
    hold: inner => withLiveSettingsRestore(inner),
  },
  {
    name: 'CoS config and runtime',
    appliesTo: scope => !scope || COS_RESTORE_SCOPES.includes(scope) || scope.startsWith('cos/agents/'),
    hold: async (inner, { srcDir, scope }) => {
      const snapshotLegacyConfig = await snapshotLegacyCosConfig(srcDir, scope);
      const { withLiveCosRestore } = await import('./cosState.js');
      return withLiveCosRestore(inner, { snapshotLegacyConfig });
    },
  },
  {
    name: 'media model registry',
    appliesTo: scope => !scope || scope === 'media-models.json',
    hold: async (inner) => {
      const { withLiveMediaModelsRestore } = await import('../lib/mediaModels.js');
      return withLiveMediaModelsRestore(inner);
    },
  },
  {
    name: 'Apple Health day files',
    appliesTo: scope => !scope || scope === 'health' || scope.startsWith('health/'),
    hold: async (inner) => {
      const { withLiveHealthRestore } = await import('./appleHealthIngest.js');
      return withLiveHealthRestore(inner);
    },
  },
]);

/** Run `operation` inside every live-restore owner that applies to `scope`, in table order. */
function withLiveFileRestoreOwners(scope, srcDir, operation) {
  const nested = LIVE_FILE_RESTORE_OWNERS.reduceRight(
    (inner, owner) => (owner.appliesTo(scope) ? () => owner.hold(inner, { srcDir, scope }) : inner),
    operation,
  );
  return nested();
}

/**
 * Peer-execution identity fence: shorter-lived than the owners above. It wraps
 * ONLY the rsync transfer, never reconciliation, and only when this restore
 * installs instances.json (resolveIdentityRestore). The decision was made
 * before verification, so it is re-checked here under the identity writer lock:
 * a live identity written since then (first pairing, ensureSelf) must not be
 * overwritten without adoption. The memoized instance id is dropped even after
 * a partial transfer, so the process never stamps an id the file no longer holds.
 */
async function transferWithIdentityFence({ installsIdentity, assertIdentityUnchanged, transferFiles }) {
  if (!installsIdentity) return transferFiles();
  const [{ withPeerExecutionIdentityRestore }, { resetInstanceIdCache }, { resetSelfInstanceIdCache }] = await Promise.all([
    import('./peerExecutionRuntime.js'),
    import('./instanceIdentity.js'),
    import('../lib/peerHttpClient.js'),
  ]);
  return withPeerExecutionIdentityRestore(async () => {
    await assertIdentityUnchanged();
    try {
      return await transferFiles();
    } finally {
      resetInstanceIdCache();
      resetSelfInstanceIdCache();
    }
  });
}

/**
 * Rsync may overwrite live files before reporting failure, so settle the
 * transfer and reconcile caches while the owners are still held. A transfer
 * failure stays the primary error (keeping its code); a reconciliation failure
 * is appended to it, or thrown on its own after a successful transfer.
 */
async function transferAndReconcileLiveFiles({ subdirFilter, transfer }) {
  const [result] = await Promise.allSettled([transfer()]);
  const reconciliationError = await reconcileLiveFileRestore(subdirFilter).then(() => null, error => error);
  if (result.status === 'rejected') {
    if (result.reason?.beforeTransfer) throw result.reason;
    const partialRestoreError = new Error(
      `${result.reason.message}. Some files may already have been overwritten because file restore is not transactional.${reconciliationError ? ` ${reconciliationError.message}` : ''}`,
      { cause: result.reason },
    );
    if (result.reason?.code) partialRestoreError.code = result.reason.code;
    throw partialRestoreError;
  }
  if (reconciliationError) throw reconciliationError;
  return result.value;
}

const DUMP_UNREADABLE = Object.freeze({ status: 'failed', reason: 'dump_unreadable', error: 'The snapshot database dump could not be read or staged for restore. Restore was refused without changing data.' });

/**
 * Restore the PostgreSQL dump from a snapshot. Dry-run by default — mirrors
 * restoreSnapshot's safety default. Both modes first admit the snapshot's
 * portos-db.sql (complete pg_dump envelope, manifest hash when recorded); a
 * real restore then replays the admitted private copy into psql. The dump was
 * written with --no-owner --no-acl so it replays cleanly.
 *   { status: 'ok', dryRun, sizeBytes, tableCount }   (dry-run or applied)
 *   { status: 'skipped', reason: 'no_dump' }           (no sql file in snapshot)
 *   { status: 'skipped', reason: 'not_configured' }    (real restore, PG unreachable)
 *   { status: 'failed', reason: 'manifest_unreadable'|'manifest_mismatch'|'dump_unreadable'|'dump_incomplete'|'restore_compatibility'|'restore_preflight'|'restore_journal'|'backup_snapshot_busy'|'restore_error'|'timeout', error? }
 *     (nothing changed; a failed replay is reported only once proven rolled back)
 *   { status: 'failed', reason: 'restore_recovery_pending'|'restore_commit_unknown'|'restore_schema_reconciliation'|'restore_catalog_reconciliation'|'restore_sync_resync'|'restore_recovery_release', error, recovery }
 *     (a restore awaits recovery: ordinary database work stays fenced until
 *     resumeDatabaseRestore finishes it — see backupRestoreRecovery.js, #9725)
 * A successful real restore also carries `syncCursorsRewound` (peer count).
 * @param {string} destPath - Backup destination root
 * @param {string} snapshotId
 * @param {{dryRun?: boolean}} [options]
 */
export async function restorePostgres(destPath, snapshotId, { dryRun = true, source } = {}) {
  const resolved = resolveSnapshotPath(destPath, snapshotId, source);
  await assertExplicitSnapshotSourceSafe(resolved, snapshotId);
  // Held for the whole call, which spans every read of snapshot bytes: the
  // restorability check, the manifest and the dump admission/spool read. (The
  // replay itself reads only the private spool copy.) Neither deletion nor
  // retention can remove the dump while it is being admitted (#10898).
  return withSnapshotRead(resolved.snapshotDir, snapshotId,
    () => restoreHeldDatabase(resolved, snapshotId, { dryRun }));
}

async function restoreHeldDatabase({ snapshotDir, currentSource }, snapshotId, { dryRun }) {
  await assertSnapshotRestorable(snapshotDir, snapshotId, currentSource);
  // A committed restore awaiting repair fences the database; neither a preview
  // nor another replay may start until it is resolved (#9725).
  const pendingRecovery = restoreRecoveryRefusal();
  if (pendingRecovery) return pendingRecovery;
  const sqlPath = join(snapshotDir, 'portos-db.sql');

  const info = await stat(sqlPath).catch(cause => {
    if (cause?.code === 'ENOENT') return null;
    const { filesystemCode } = inventoryReadError(cause, 'stat-database-dump').context;
    console.error(`❌ restore: dump metadata unreadable for snapshot ${snapshotId} (${filesystemCode})`);
    return false;
  });
  if (info === false) return DUMP_UNREADABLE;
  // An empty/0-byte dump is as good as absent — restoring it is a silent no-op.
  // Mirror dumpPostgres's empty-dump guard so a truncated snapshot can't read
  // as a successful "0 tables" restore.
  if (!info || !info.isFile?.() || info.size === 0) {
    return { status: 'skipped', reason: 'no_dump' };
  }
  // Verify the dump against the manifest's stored SHA-256 before trusting it.
  // The dump is hashed in generateManifest under the parent-relative key
  // '../portos-db.sql' (it lives ALONGSIDE the snapshot data/ dir, not inside
  // it). Backward-compat: snapshots taken before manifests existed — or missing
  // the dump key — have nothing to verify against, so we SKIP verification and
  // proceed rather than hard-failing. An existing manifest refuses the restore
  // when it cannot be read or when its recorded dump hash does not match.
  const manifestPath = join(snapshotDir, 'manifest.json');
  // Read-only verification metadata. A confirmed ENOENT remains the legacy
  // no-manifest case, while corrupt bytes and every other read failure mean the
  // dump cannot be trusted. This path never writes the manifest back.
  const manifestRead = await readJSONFileStrict(manifestPath, null);
  if (!manifestRead.ok) {
    console.error(`❌ restore: integrity manifest unreadable for snapshot ${snapshotId}`);
    return { status: 'failed', reason: 'manifest_unreadable' };
  }
  const expectedHash = manifestRead.value?.files?.['../portos-db.sql'];
  // Execution replays a private copy written by the same read that admits the
  // dump, so replay derives only from the checked bytes even if the
  // snapshot changes afterwards (#8782). Preview only inspects.
  const spoolDir = dryRun ? null : await mkdtemp(join(tmpdir(), 'portos-restore-')).catch((err) => {
    console.error(`❌ restore: cannot stage dump for snapshot ${snapshotId}: ${err.message}`);
    return false;
  });
  if (spoolDir === false) return DUMP_UNREADABLE;
  try {
    return await restoreAdmittedDump({ sqlPath, spoolPath: spoolDir && join(spoolDir, 'dump.sql'), expectedHash, snapshotId, dryRun, sizeBytes: info.size });
  } finally {
    if (spoolDir) {
      await rm(spoolDir, { recursive: true, force: true })
        .catch(err => console.error(`❌ restore: failed to remove dump spool ${spoolDir}: ${err.message}`));
    }
  }
}

async function restoreAdmittedDump({ sqlPath, spoolPath, expectedHash, snapshotId, dryRun, sizeBytes }) {
  // One streamed read supplies the checksum and the completeness proof. A read
  // failure is refused here — never mistaken for an empty dump.
  const targetServerMajor = await getServerMajorVersion().catch(() => null);
  const dump = await inspectDatabaseDump(sqlPath, { spoolTo: spoolPath, targetServerMajor }).catch((err) => {
    console.error(`❌ restore: dump unreadable for snapshot ${snapshotId}: ${err.message}`);
    return null;
  });
  if (!dump) {
    return DUMP_UNREADABLE;
  }
  if (expectedHash && dump.sha256 !== expectedHash) {
    console.error(`❌ restore: manifest hash mismatch for snapshot ${snapshotId} (expected ${expectedHash}, got ${dump.sha256})`);
    return { status: 'failed', reason: 'manifest_mismatch' };
  }
  // The restore resets every application table first, so a dump that parses
  // but stops early (header-only, or truncated between statements) would
  // commit an empty database. ON_ERROR_STOP cannot see that; the envelope can.
  if (!dump.complete || dump.missingTables.length) {
    const missing = [
      ...(dump.complete ? [] : ['the pg_dump completion marker']),
      ...dump.missingTables.map(name => `table ${name}`),
    ].join(', ');
    console.error(`❌ restore: incomplete dump for snapshot ${snapshotId} (missing ${missing})`);
    return { status: 'failed', reason: 'dump_incomplete', error: `The snapshot database dump is incomplete (missing ${missing}). Restore was refused without changing data.` };
  }
  const { tableCount } = dump;
  const health = await checkHealth();
  if (!health.connected) return { status: 'skipped', reason: 'not_configured' };
  if (!Number.isInteger(targetServerMajor) || targetServerMajor <= 0) {
    return {
      status: 'failed',
      reason: 'restore_compatibility',
      error: 'Cannot establish the target PostgreSQL major version. Check database connectivity and permission to SHOW server_version_num, then retry preview before restoring. No data was changed.',
    };
  }
  // Admission checks above always cover the original bytes. Only the private
  // replay copy omits extension comments and clean-dump extension drops,
  // whose ownership may belong to a provisioning role, plus the exact PG17
  // transaction_timeout header when replaying to an older target.
  const replayPath = dryRun ? null : await prepareDatabaseReplay(spoolPath, dump.extensionMetadata).catch(err => {
    console.error(`❌ restore: cannot normalize admitted dump for snapshot ${snapshotId}: ${err.message}`);
    return null;
  });
  if (!dryRun && !replayPath) return DUMP_UNREADABLE;

  // Preview remains read-only. Recheck inside the replay transaction as well,
  // so a changed catalog cannot turn a previously safe preview into a cascade.
  const { getDatabaseResetPlan } = await import('./backupDatabaseReset.js');
  const { preflight, reset } = await getDatabaseResetPlan();
  const preflightError = await query(preflight).then(() => null, error => error);
  if (preflightError) {
    return { status: 'failed', reason: 'restore_preflight', error: 'Database contains unexpected objects, ownership, or dependencies. Restore was refused without changing data.' };
  }
  if (dryRun) return { status: 'ok', dryRun: true, sizeBytes, tableCount };

  // The replay replaces every row. Take the backup boundary first so no admitted
  // file-plus-row publication is half done underneath it and no backup cut is
  // capturing the database mid-replay. Only this restore's own cut is released.
  let releaseSnapshotCut;
  try {
    releaseSnapshotCut = await acquireBackupSnapshotCut();
  } catch (err) {
    if (err.code !== 'BACKUP_SNAPSHOT_BUSY') throw err;
    console.warn(`⚠️ restore: refused for snapshot ${snapshotId}: ${err.message}`);
    return {
      status: 'failed',
      reason: 'backup_snapshot_busy',
      error: 'A backup is capturing the database or an asset publication did not finish draining. Restore was refused without changing data; retry when it finishes.',
    };
  }
  try {
    return await replayAdmittedDump({ tableCount, sizeBytes, snapshotId, dump, reset, replayPath });
  } finally {
    releaseSnapshotCut();
  }
}

async function replayAdmittedDump({ tableCount, sizeBytes, snapshotId, dump, reset, replayPath }) {
  return withDatabaseMaintenance(async ({ adoptRestoreRecovery }) => {
    // Read before the replay rewinds them, then make them durable together
    // with this operation's identity BEFORE anything destructive runs: a
    // committed replay whose repair fails (or a restart) must floor sequences
    // at THESE positions, never at values re-read after the dump reset them.
    const feedPositions = await captureSyncFeedPositions();
    let record;
    try {
      record = databaseRestoreRecovery.begin({ snapshotId, dumpSha256: dump.sha256, feedPositions });
      adoptRestoreRecovery(record.id);
    } catch (err) {
      if (record) throw err;
      console.error(`❌ restore: recovery journal could not be written for snapshot ${snapshotId}: ${err.message}`);
      return {
        status: 'failed',
        reason: 'restore_journal',
        error: 'The restore recovery journal could not be written. Restore was refused without changing data.',
      };
    }
    try {
      const { preparePeerExecutionRestore } = await import('./peerExecutionRestore.js');
      await preparePeerExecutionRestore(record.id);
    } catch (err) {
      console.error(`❌ restore: execution consumption capture failed: ${err.message}`);
      return pendingRecoveryResult('restore_execution_reconciliation', record);
    }
    const { host: pgHost, port, database: pgDb, user: pgUser } = POOL_CONFIG;
    const pgPort = String(port);

    const replay = await withPgToolEnv(POOL_CONFIG, pgEnv => new Promise((resolveP) => {
      // ON_ERROR_STOP=1 aborts on the first failed statement; --single-transaction
      // wraps the whole replay in one transaction so that abort ROLLs BACK every
      // prior statement. Together they make the restore atomic: it either fully
      // applies or leaves the live DB untouched — never a mixed snapshot/current
      // state. (The dump is written with --clean --if-exists, so the DROPs and
      // recreates all commit or roll back as one unit.) psql reads the admitted
      // spool copy, never the snapshot path. The trailing receipt commits with
      // the dump or not at all, and the application name identifies this
      // replay's session to recovery (#9725).
      const proc = spawn('psql', [
        '-X', '-v', 'ON_ERROR_STOP=1',
        '--single-transaction',
        '--echo-all',
        '-h', pgHost, '-p', pgPort, '-U', pgUser, '-d', pgDb, '-c', reset, '-f', replayPath,
        '-c', restoreReceiptSql(record),
      ], { shell: false, stdio: ['ignore', 'pipe', 'pipe'], env: { ...pgEnv, PGAPPNAME: restoreApplicationName(record.id) } });

      let stderr = '';
      const watchdog = watchBackupProcess(proc, { label: 'PostgreSQL restore' });
      // `--echo-all` echoes input as psql consumes it, including rows within a
      // long COPY. The output is never retained, but it must be drained: unread
      // piped output can backpressure and deadlock a verbose restore. Each chunk
      // is also the progress signal that keeps an active long restore alive.
      proc.stdout.on('data', watchdog.markActivity);
      proc.stderr.on('data', (chunk) => {
        watchdog.markActivity();
        stderr += chunk.toString();
      });

      proc.on('close', (code) => {
        if (!watchdog.finish()) return;
        const timeoutError = watchdog.getTimeoutError();
        if (timeoutError) {
          resolveP({ status: 'failed', reason: 'timeout', error: timeoutError.message });
          return;
        }
        if (code === 0) {
          console.log(`💾 psql restore complete from snapshot ${snapshotId}: ${tableCount} tables`);
          resolveP({ status: 'ok', dryRun: false, sizeBytes, tableCount });
        } else {
          console.warn(`⚠️ psql restore failed (code ${code}): ${stderr.trim()}`);
          resolveP({ status: 'failed', reason: 'restore_error', error: stderr.trim() });
        }
      });
      proc.on('error', (err) => {
        if (watchdog.getTimeoutError() || !watchdog.finish()) return;
        console.warn(`⚠️ psql not available: ${err.message}`);
        resolveP({ status: 'failed', reason: 'restore_error', error: err.message });
      });
    })).catch((err) => ({ status: 'failed', reason: 'restore_error', error: err.message }));

    if (replay.status === 'ok') {
      // psql exit 0 under --single-transaction means COMMIT succeeded.
      record = databaseRestoreRecovery.markCommitted(record.id);
    } else {
      // A failed/killed/lost psql may still have committed (a crash or lost
      // response at COMMIT). Only the receipt can say; an unknown outcome stays
      // fenced instead of reopening writes or replaying again.
      const settled = await settleReplayOutcome(record);
      if (settled.outcome === 'rolled_back') return replay;
      if (settled.outcome === 'uncertain') return pendingRecoveryResult(settled.reason ?? 'restore_commit_unknown', record);
      record = settled.record;
    }

    // Replay has committed. Still inside maintenance and still fenced, so no
    // sync apply or feed write can interleave with the repair, and a failed
    // step leaves ordinary work closed with the original positions recorded.
    const repaired = await repairCommittedRestore(record);
    if (repaired.status !== 'ok') return repaired;
    return { status: 'ok', dryRun: false, sizeBytes, tableCount, syncCursorsRewound: repaired.syncCursorsRewound };
  });
}

/**
 * Get backup state with process-local failure and running projections.
 */
export async function getState() {
  const state = await readJSONFile(STATE_PATH, DEFAULT_STATE, { strict: true });
  const projected = failedStateProjection ? { ...state, ...failedStateProjection } : state;
  return isRunning ? { ...projected, status: 'running' } : projected;
}

/**
 * Merge patch into current backup state and persist.
 * @param {object} patch - Fields to merge into state
 * @param {Function} [onFailure] - Handle a rejected write inside the serialized queue
 */
export async function saveState(patch, onFailure) {
  return queueStateWrite(() => (async () => {
    await ensureDir(join(PATHS.data, 'backup'));
    // Status projections are read-only and must never become durable state.
    const current = await readJSONFile(STATE_PATH, DEFAULT_STATE, { strict: true });
    const updated = { ...current, ...patch };
    await atomicWrite(STATE_PATH, updated);
    failedStateProjection = null;
    return updated;
  })().catch((error) => {
    // Handle the failed transition inside the queue: a later successful write
    // must clear it, never race with an out-of-queue failure handler.
    if (onFailure) return onFailure(error);
    throw error;
  }));
}

/**
 * Get the next scheduled backup run time from eventScheduler.
 * @returns {string|null} ISO timestamp of next run, or null
 */
export function getNextRunTime() {
  const event = getEvent('backup-daily');
  return event?.nextRunAt ? new Date(event.nextRunAt).toISOString() : null;
}

/**
 * Whether THIS process has a backup snapshot in flight right now, for the
 * system-idle gate. Deliberately reads the in-process `isRunning` flag rather
 * than scanning for an on-disk `.in-progress` marker: that marker is designed
 * to OUTLIVE a crash/PM2 restart so a restore never mistakes a partial
 * snapshot for a complete one, and only the next run converges an orphaned
 * one — gating the updater on its mere existence would block it until then
 * after a single interrupted backup on a drive that might not even be
 * mounted. The gate only needs to know whether restarting would kill a live
 * rsync/pg_dump in this server instance, and `isRunning` answers exactly
 * that, with no I/O and nothing that can fail to read.
 */
export function isBackupInProgress() {
  return isRunning;
}