Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: restore a pre-081 backup with its inbox and journals upgraded in order (#12813)
Adam Eivydc3f4e19388d
diff --git a/client/src/components/BackupWidget.jsx b/client/src/components/BackupWidget.jsxindex 5608c48991..9839e31751 100644--- a/client/src/components/BackupWidget.jsx+++ b/client/src/components/BackupWidget.jsx@@ -159,10 +159,13 @@ function RestorePanel({ snapshot, onClose, restoring, onRestoreStateChange }) {const suffix = result.verification?.status === 'unverified' ? ' (unverified legacy snapshot)' : '';const parked = (result.restoredLegacyStores ?? []).filter(store => store.parkedAs).map(store => store.parkedAs);const conversion = parked.length > 0 ? `. Converted older-format data; the replaced stores were kept as ${parked.join(', ')}` : '';+ const replayed = result.restoredLegacyMigrations?.status === 'applied'+ ? `. Ran the ${pluralize(result.restoredLegacyMigrations.names.length, 'data migration')} the snapshot predates through ${result.restoredLegacyMigrations.through}`+ : '';const ledger = result.restoredLegacyLedger?.status === 'adopted'? `. Adopted the snapshot's migration record${result.restoredLegacyLedger.parkedAs ? ` (previous record kept as ${result.restoredLegacyLedger.parkedAs})` : ''}; restart PortOS to re-run the migrations it predates`: '';- toast.success(`File restore complete — ${result.changedFiles?.length ?? 0} file(s) restored${suffix}${conversion}${ledger}`);+ toast.success(`File restore complete — ${result.changedFiles?.length ?? 0} file(s) restored${suffix}${conversion}${replayed}${ledger}`);onClose();}}, [acceptedPreview, onClose, onRestoreStateChange, previewMatchesCurrentRequest, restoring, snapshot]);@@ -292,6 +295,14 @@ function RestorePanel({ snapshot, onClose, restoring, onRestoreStateChange }) {</span></p>)}+ {preview.restoredLegacyMigrations && (+ <p className="mb-2 flex items-start gap-1.5 rounded border border-port-warning/40 bg-port-warning/10 p-2 text-xs text-port-warning" role="alert">+ <AlertTriangle size={13} className="mt-0.5 shrink-0" />+ <span className="min-w-0 break-words">+ This snapshot predates {pluralize(preview.restoredLegacyMigrations.names.length, 'data migration')} up to <code>{preview.restoredLegacyMigrations.through}</code>. Restoring runs them right away, in order, so its records convert exactly as an upgrade would; later migrations run on the next start.+ </span>+ </p>+ )}<p className="text-xs text-gray-500 mb-1">{preview.changedFiles?.length ?? 0} file(s) would change:</p>diff --git a/client/src/components/BackupWidget.test.jsx b/client/src/components/BackupWidget.test.jsxindex b805eb409e..c96fcf165c 100644--- a/client/src/components/BackupWidget.test.jsx+++ b/client/src/components/BackupWidget.test.jsx@@ -380,6 +380,22 @@ describe('BackupWidget snapshots', () => {));});+ it('announces the migrations a pre-split restore runs through the split and reports them afterwards', async () => {+ const migrations = { through: '200-split-example.js', names: ['081-example.js', '200-split-example.js'] };+ mockRestoreBackup+ .mockResolvedValueOnce({ dryRun: true, changedFiles: ['brain/journals.json'], restoredLegacyMigrations: { status: 'pending', ...migrations } })+ .mockResolvedValueOnce({ changedFiles: ['brain/journals.json'], restoredLegacyMigrations: { status: 'applied', ...migrations } });+ renderWidget();+ await openRestorePanel();+ fireEvent.click(screen.getByRole('button', { name: 'Preview changes' }));++ expect(await screen.findByRole('alert')).toHaveTextContent(/predates 2 data migrations up to 200-split-example\.js.*runs them right away, in order/);+ fireEvent.click(screen.getByRole('button', { name: 'Restore 1 file(s)' }));+ await waitFor(() => expect(mockToast.success).toHaveBeenCalledWith(+ 'File restore complete — 1 file(s) restored. Ran the 2 data migrations the snapshot predates through 200-split-example.js',+ ));+ });+it('announces a pre-May-2026 ledger adoption and asks for a restart afterwards', async () => {mockRestoreBackup.mockResolvedValueOnce({diff --git a/docs/BACKUP.md b/docs/BACKUP.mdindex 3149accb3f..8fac11a288 100644--- a/docs/BACKUP.md+++ b/docs/BACKUP.md@@ -517,7 +517,7 @@ A snapshot taken before the agent records moved into PostgreSQL (#12738) also caKeep the original backup intact. Copy it into a disposable offline install running the compatible historical version, upgrade that isolated install through the file-to-database transition, and verify the imported records and linked files there. Make a current-format file and PostgreSQL backup from the converted install, then use the normal verified restore workflow. Do not convert against the live database or delete its markers. Current file-primary siblings (such as manuscript reviews, draft Markdown and media assets) and parked `.imported` recovery evidence remain eligible for file restore. Selecting one of those siblings is allowed even if another part of the snapshot contains legacy metadata.-A full restore also installs the snapshot's `migrations.applied.json`, so the next start re-runs every file migration the snapshot predates. A snapshot taken before migrations moved out of `data/` (May 2026) carries that ledger at `migrations/.applied.json` instead; the transfer lands it beside the live `migrations.applied.json`, which the runner would then keep, so a full restore adopts it by hand after the transfer: the live ledger is parked as `migrations.applied.json.pre-restore-<timestamp>` and the restored one takes its place. Preview and result carry `restoredLegacyLedger` (`{ status, parkedAs }`, `pending` in a preview and `adopted` after execution; `unreadable` when the snapshot's legacy ledger does not parse as a list, in which case the live ledger is kept); the widget warns before restore and asks for a restart afterwards. A snapshot that also carries a current-path ledger is left to the transfer. That re-run only upgrades restored files for migrations gated on their input. The store-split migrations (059 for media collections, 200 for the Brain stores) gate on their output, a type index already at schema v1, and file restore leaves the live index in place, so their re-run is a no-op over a restored monolith. Restore therefore converts those stores itself (#12626): when the scope covers a store (full restore, `brain`, `brain/<type>.json`, `media-collections` or `media-collections.json`) and the snapshot carries its monolith (`brain/<type>.json`, `media-collections.json`) without `<store>/index.json`, a successful transfer moves the live `<store>/` directory to `<store>.pre-restore-<timestamp>/` (and a live `.bak-200`/`.bak-059` backup beside it), runs the frozen split migration against the install, and invalidates the Brain caches. The restored records, tombstones included, then load immediately, for folder restores too. A snapshot that has the split layout plus a stray monolith (setup-data re-seeds those after an upgrade) is current and is not converted. A monolith the split would refuse (unparseable, or a malformed records container) is restored as-is and never parks the live store. Preview and result list these stores in `restoredLegacyStores` (`{ store, legacyFile, status, parkedAs }`, with `status` `pending` in a preview and `converted` or `unreadable` after execution); the widget warns before restore and names the parked directory afterwards. A folder restore (`subdirFilter`) keeps the live ledger, so restored files can come back in a shape older than migrations no runner will revisit. Its preview and result therefore carry `ledgerGap`: `{ count, names }` listing the migrations the live ledger records and the snapshot's lacks, or `null` when either ledger is missing or unreadable. A full restore omits the field. The widget warns on a non-empty gap and recommends a full restore, but does not refuse: a folder no migration touched restores correctly. The quota-burn plan is converted on the spot: a restore covering `cos/quota-burn.json` (full, `cos`, or that file) runs any legacy `jobType` steps through the same conversion as migration 359 (`reconcileRestoredQuotaBurnPlan`), reusing each step's deterministic `job-burn-<family>-<step>` custom job, so a repeated restore creates no duplicates.+A full restore also installs the snapshot's `migrations.applied.json`, so the next start re-runs every file migration the snapshot predates. A snapshot taken before migrations moved out of `data/` (May 2026) carries that ledger at `migrations/.applied.json` instead; the transfer lands it beside the live `migrations.applied.json`, which the runner would then keep, so a full restore adopts it by hand after the transfer: the live ledger is parked as `migrations.applied.json.pre-restore-<timestamp>` and the restored one takes its place. Preview and result carry `restoredLegacyLedger` (`{ status, parkedAs }`, `pending` in a preview and `adopted` after execution; `unreadable` when the snapshot's legacy ledger does not parse as a list, in which case the live ledger is kept); the widget warns before restore and asks for a restart afterwards. A snapshot that also carries a current-path ledger is left to the transfer. That re-run only upgrades restored files for migrations gated on their input. The store-split migrations (059 for media collections, 200 for the Brain stores) gate on their output, a type index already at schema v1, and file restore leaves the live index in place, so their re-run is a no-op over a restored monolith. Restore therefore converts those stores itself (#12626): when the scope covers a store (full restore, `brain`, `brain/<type>.json`, `media-collections` or `media-collections.json`) and the snapshot carries its monolith (`brain/<type>.json`, `media-collections.json`) without `<store>/index.json`, a successful transfer moves the live `<store>/` directory to `<store>.pre-restore-<timestamp>/` (and a live `.bak-200`/`.bak-059` backup beside it), runs the frozen split migration against the install, and invalidates the Brain caches. The restored records, tombstones included, then load immediately, for folder restores too. A full restore whose snapshot ledger lacks the split migration also lacks the migrations an upgrade ran before it (080/081/190 for the Brain stores, 021/038 for media collections), so it does not run the split alone: after adopting the restored ledger it runs that ledger's pending migrations through the split with the real runner (`runMigrations({ through })`), in order (#12813). It also parks every live Brain type directory the snapshot has no `<type>/index.json` for, because an earlier migration can create the monolith (081 builds `brain/inbox.json` from `inbox_log.jsonl`). The result matches an in-order upgrade of the snapshot, and migrations after the split run on the next start. Preview and result carry `restoredLegacyMigrations` (`{ status, through, names }`, `pending` in a preview and `applied` after execution). Folder restores never replay. A snapshot that has the split layout plus a stray monolith (setup-data re-seeds those after an upgrade) is current and is not converted. A monolith the split would refuse (unparseable, or a malformed records container) is restored as-is and never parks the live store. Preview and result list these stores in `restoredLegacyStores` (`{ store, legacyFile, status, parkedAs }`, with `status` `pending` in a preview and `converted` or `unreadable` after execution); the widget warns before restore and names the parked directory afterwards. A folder restore (`subdirFilter`) keeps the live ledger, so restored files can come back in a shape older than migrations no runner will revisit. Its preview and result therefore carry `ledgerGap`: `{ count, names }` listing the migrations the live ledger records and the snapshot's lacks, or `null` when either ledger is missing or unreadable. A full restore omits the field. The widget warns on a non-empty gap and recommends a full restore, but does not refuse: a folder no migration touched restores correctly. The quota-burn plan is converted on the spot: a restore covering `cos/quota-burn.json` (full, `cos`, or that file) runs any legacy `jobType` steps through the same conversion as migration 359 (`reconcileRestoredQuotaBurnPlan`), reusing each step's deterministic `job-burn-<family>-<step>` custom job, so a repeated restore creates no duplicates.**Every nonzero rsync exit fails a restore, including 24.** A backup tolerates exit 24 ("some source files vanished") because it reads the live data tree, where a file deleted mid-scan is normal. A restore reads a snapshot that must not change under it, so a vanished source file means live data may now mix restored and older files: preview and execution both reject it. An execution failure is reported as a partial restore — the error says some files may already have been overwritten because file restore is not transactional — and caches (settings, Brain, media index) are reconciled before the error is returned, exactly as for any other failed transfer.diff --git a/scripts/run-migrations.js b/scripts/run-migrations.jsindex 40735246d1..d30d4338ae 100644--- a/scripts/run-migrations.js+++ b/scripts/run-migrations.js@@ -196,6 +196,10 @@ export async function runMigrations({migrationsDir = DEFAULT_MIGRATIONS_DIR,lockWaitMs = LOCK_WAIT_MS,lockPollMs = LOCK_POLL_MS,+ // Stop after this migration file, leaving later ones pending. A restore+ // replays a pre-split snapshot's pending migrations through its split this+ // way, so the split sees what an in-order upgrade would have given it (#12813).+ through = null,} = {}) {// Backstop for #1947: a process booted from inside a CoS agent git worktree// (data/cos/worktrees/agent-*) has no gitignored data/ runtime tree, so@@ -217,13 +221,13 @@ export async function runMigrations({// the read is only trustworthy while no other runner can append to it.const releaseLock = await acquireMigrationLock(lockDirPath(rootDir), { waitMs: lockWaitMs, pollMs: lockPollMs });try {- return await runPendingMigrations({ rootDir, migrationsDir, appliedFile });+ return await runPendingMigrations({ rootDir, migrationsDir, appliedFile, through });} finally {await releaseLock();}}-async function runPendingMigrations({ rootDir, migrationsDir, appliedFile }) {+async function runPendingMigrations({ rootDir, migrationsDir, appliedFile, through }) {if (await adoptLegacyAppliedList(rootDir)) {console.log('🧹 Moved data/migrations/.applied.json → data/migrations.applied.json');}@@ -260,6 +264,9 @@ async function runPendingMigrations({ rootDir, migrationsDir, appliedFile }) {const ledgerStartedEmpty = applied.length === 0;const files = await scanMigrationFiles(migrationsDir);+ if (through !== null && !files.includes(through)) {+ throw new Error(`Cannot run migrations through "${through}": no such migration in ${migrationsDir}`);+ }// The migration helper also carries prompt-drift and provider-seed// machinery. Boot/status callers import this runner widely, but only an@@ -301,7 +308,7 @@ async function runPendingMigrations({ rootDir, migrationsDir, appliedFile }) {if (disarmed > 0) await persistAppliedList();let ran = 0;- for (const file of files) {+ for (const file of through === null ? files : files.slice(0, files.indexOf(through) + 1)) {if (applied.includes(file)) continue;const mod = await import(pathToFileURL(join(migrationsDir, file)).href);diff --git a/scripts/run-migrations.test.js b/scripts/run-migrations.test.jsindex 6ef6d24d8b..6e8e029dfd 100644--- a/scripts/run-migrations.test.js+++ b/scripts/run-migrations.test.js@@ -248,6 +248,18 @@ export default {expect(JSON.parse(readFileSync(appliedFile, 'utf-8'))).toEqual(['001-fixture.js']);await expect(listPendingMigrations({ rootDir, migrationsDir })).resolves.toEqual(['002-invalid.js']);});++ it('stops after the migration named by `through`, leaving later ones pending (#12813)', async () => {+ writeFileSync(join(migrationsDir, '002-bounded.js'), 'export default { up() {} };');+ writeFileSync(join(migrationsDir, '003-later.js'), 'throw new Error("A migration after the bound was imported");');++ await expect(runMigrations({ rootDir, migrationsDir, through: '002-bounded.js' })).resolves.toBe(2);++ expect(JSON.parse(readFileSync(appliedFile, 'utf-8'))).toEqual(['001-fixture.js', '002-bounded.js']);+ await expect(listPendingMigrations({ rootDir, migrationsDir })).resolves.toEqual(['003-later.js']);+ await expect(runMigrations({ rootDir, migrationsDir, through: '009-missing.js' }))+ .rejects.toThrow('Cannot run migrations through "009-missing.js"');+ });});describe('runMigrations purge-migration guard (#2770)', () => {diff --git a/server/services/backup.js b/server/services/backup.jsindex df0c0b4008..a46ac689f3 100644--- a/server/services/backup.js+++ b/server/services/backup.js@@ -1903,12 +1903,17 @@ async function restoreHeldSnapshot({ snapshotDir, currentSource }, snapshotId, {// migrations the live ledger already records.const ledgerGap = subdirFilter ? await migrationLedgerGap(srcDir) : undefined;// Split migrations gate on their output, so neither ledger re-runs them over- // a restored monolith; restore converts those stores itself (#12626).- const legacyStores = await snapshotLegacySplitStores(srcDir, subdirFilter);+ // a restored monolith; restore converts those stores itself (#12626). A full+ // restore installs the snapshot's ledger, so the migrations it lacks before a+ // split are replayed with that split, in order (#12813).+ const legacyStores = await snapshotLegacySplitStores(srcDir, subdirFilter, {+ ledger: subdirFilter ? null : await readSnapshotMigrationLedger(srcDir),+ });const legacyLedger = await snapshotLegacyLedger(srcDir, subdirFilter);- const restoreNotes = (restoredLegacyStores, restoredLegacyLedger = legacyLedger) => ({+ const restoreNotes = (restoredLegacy, restoredLegacyLedger = legacyLedger) => ({...(ledgerGap !== undefined && { ledgerGap }),- ...(restoredLegacyStores.length > 0 && { restoredLegacyStores }),+ ...(restoredLegacy.stores.length > 0 && { restoredLegacyStores: restoredLegacy.stores }),+ ...(restoredLegacy.migrations && { restoredLegacyMigrations: restoredLegacy.migrations }),...(restoredLegacyLedger && { restoredLegacyLedger }),...(identity.preserved && { identityPreserved: identity.preserved }),...(identity.adopted && { identityAdopted: identity.adopted }),@@ -1970,10 +1975,11 @@ async function restoreHeldSnapshot({ snapshotDir, currentSource }, snapshotId, {}const scope = subdirFilter;- let restoredLegacyStores = [];+ let restoredLegacy = { stores: [], migrations: null };let restoredLegacyLedger = legacyLedger;// Conversion parks live stores, so it runs only after a successful transfer,- // and before reconciliation so the Brain caches drop the parked records.+ // and before reconciliation so the Brain caches drop the parked records. It+ // follows ledger adoption so a replay reads the ledger the boot would read.const changedFiles = await withLiveFileRestoreOwners(scope, srcDir, () => transferAndReconcileLiveFiles({subdirFilter,transfer: async () => {@@ -1985,12 +1991,12 @@ async function restoreHeldSnapshot({ snapshotDir, currentSource }, snapshotId, {},transferFiles,});- restoredLegacyStores = await convertRestoredLegacySplitStores(legacyStores);if (legacyLedger) restoredLegacyLedger = await adoptRestoredLegacyLedger(legacyLedger);+ restoredLegacy = await convertRestoredLegacySplitStores(legacyStores);return changed;},}));- return { dryRun, snapshotId, subdirFilter, changedFiles, verification, ...restoreNotes(restoredLegacyStores, restoredLegacyLedger) };+ return { dryRun, snapshotId, subdirFilter, changedFiles, verification, ...restoreNotes(restoredLegacy, restoredLegacyLedger) };}async function readMigrationLedger(path) {diff --git a/server/services/backup.test.js b/server/services/backup.test.jsindex 969ccaa86d..31bf29a48a 100644--- a/server/services/backup.test.js+++ b/server/services/backup.test.js@@ -83,6 +83,10 @@ vi.mock('../lib/databaseMaintenanceJournal.js', () => ({vi.mock('../scripts/run-db-migrations.js', () => ({runDbMigrations: vi.fn().mockResolvedValue(0),}));+// The file-migration runner a pre-split restore replays through (#12813);+// backupLegacyStores.test.js runs the real one against the real migrations.+const runFileMigrations = vi.hoisted(() => vi.fn());+vi.mock('../../scripts/run-migrations.js', () => ({ runMigrations: runFileMigrations }));const catalogRepair = vi.hoisted(() => ({migrate: vi.fn().mockResolvedValue({ skipped: false }),tags: vi.fn().mockResolvedValue({ skipped: false }),@@ -2509,6 +2513,45 @@ describe('restoreSnapshot manifest verification', () => {await realFs.rm(live('cos', 'example.json'), { force: true });});+ it('replays the adopted ledger through the split and reports it in preview and result (#12813)', async () => {+ const ledgerPath = live('migrations.applied.json');+ const legacyLedgerPath = live('migrations', '.applied.json');+ const split = '200-split-brain-stores-to-per-record.js';+ const journals = JSON.stringify({ records: { '2025-03-01': { id: 'jrnl-example', date: '2025-03-01' } } });+ await seedSnapshot({ 'brain/journals.json': journals, 'migrations/.applied.json': JSON.stringify(['001-example.js']) });+ await realFs.writeFile(ledgerPath, JSON.stringify(['001-example.js', split]));+ try {+ const preview = await finishRestore({ dryRun: true });+ expect(preview.restoredLegacyMigrations).toMatchObject({ status: 'pending', through: split });+ expect(preview.restoredLegacyMigrations.names).toContain('081-brain-daily-log-inbox-sync.js');+ expect(preview.restoredLegacyMigrations.names.at(-1)).toBe(split);+ expect(preview.restoredLegacyStores.map(entry => entry.store)).toEqual(['brain/people', 'brain/journals']);++ // rsync is faked, so stand in for its transfer of the snapshot files.+ await realFs.writeFile(live('brain', 'journals.json'), journals);+ await realFs.mkdir(live('migrations'), { recursive: true });+ await realFs.writeFile(legacyLedgerPath, JSON.stringify(['001-example.js']));+ runFileMigrations.mockImplementationOnce(async ({ through }) => {+ // The replay must read the snapshot's ledger, so adoption runs first.+ expect(JSON.parse(await realFs.readFile(ledgerPath, 'utf8'))).toEqual(['001-example.js']);+ await realFs.writeFile(ledgerPath, JSON.stringify(['001-example.js', through]));+ });+ spawn.mockReset();+ const result = await finishRestore({ dryRun: false });++ expect(runFileMigrations).toHaveBeenCalledWith(expect.objectContaining({ through: split }));+ expect(result.restoredLegacyLedger).toMatchObject({ status: 'adopted' });+ expect(result.restoredLegacyMigrations).toEqual({ ...preview.restoredLegacyMigrations, status: 'applied' });+ expect(existsSync(live('brain', 'people', 'live-1'))).toBe(false);+ } finally {+ runFileMigrations.mockReset();+ await realFs.rm(live('migrations'), { recursive: true, force: true });+ for (const name of await realFs.readdir(PATHS.data)) {+ if (name.startsWith('migrations.applied.json')) await realFs.rm(live(name), { force: true });+ }+ }+ });+it('never parks the live store for a monolith the split would refuse', async () => {const malformed = JSON.stringify({ records: [{ id: 'p-1' }] });await seedSnapshot({ 'brain/people.json': malformed });diff --git a/server/services/backupLegacyStores.js b/server/services/backupLegacyStores.jsindex 5c6195ef56..d2b58c3171 100644--- a/server/services/backupLegacyStores.js+++ b/server/services/backupLegacyStores.js@@ -14,10 +14,19 @@* install immediately, splitting the restored monolith exactly as an upgrade* would have. The migrations are invoked, never re-implemented, so a restore* converts with the same semantics the install crossed when it upgraded.+ *+ * A full restore whose ledger lacks the split also lacks the migrations an+ * upgrade ran BEFORE it (081 builds `brain/inbox.json` from `inbox_log.jsonl`+ * and normalizes journals). Running the split alone and leaving those to the+ * next boot would run them after it, writing monoliths nothing reads (#12813).+ * So that restore replays the restored ledger's pending migrations through the+ * split with the real runner, and parks every live store of that split the+ * snapshot carries no split index for, so the result is an in-order upgrade.*/import { rename, stat } from 'fs/promises';import { basename, dirname, join } from 'path';import { PATHS, readJSONFileStrict } from '../lib/fileUtils.js';+import { DEFAULT_MIGRATIONS_DIR, appliedFilePath, readAppliedList, scanMigrationFiles } from '../../scripts/lib/migrationLedger.js';// Frozen with the migrations they invoke: migration 200's type list as of 200.const BRAIN_SPLIT_TYPES = [@@ -25,6 +34,8 @@ const BRAIN_SPLIT_TYPES = ['journals', 'inbox', 'songs',];+const BRAIN_SPLIT_MIGRATION = '200-split-brain-stores-to-per-record.js';+const MEDIA_COLLECTIONS_SPLIT_MIGRATION = '059-split-media-collections.js';const loadBrainSplit = () => import('../../scripts/migrations/200-split-brain-stores-to-per-record.js');const loadMediaCollectionsSplit = () => import('../../scripts/migrations/059-split-media-collections.js');@@ -34,7 +45,11 @@ const LEGACY_SPLIT_STORES = Object.freeze([scopes: ['brain', `brain/${type}.json`],legacyFile: `brain/${type}.json`,backupSuffix: '.bak-200',+ migration: BRAIN_SPLIT_MIGRATION,loadMigration: loadBrainSplit,+ // Earlier migrations create some Brain monoliths (081 the inbox, 190 the+ // songs), so a replay converts a type the snapshot has no monolith for.+ replayRoot: 'brain',// Migration 200 throws on a records container that is present but not a map.readable: doc => doc.records == null || (typeof doc.records === 'object' && !Array.isArray(doc.records)),})),@@ -43,7 +58,9 @@ const LEGACY_SPLIT_STORES = Object.freeze([scopes: ['media-collections', 'media-collections.json'],legacyFile: 'media-collections.json',backupSuffix: '.bak-059',+ migration: MEDIA_COLLECTIONS_SPLIT_MIGRATION,loadMigration: loadMediaCollectionsSplit,+ replayRoot: null,readable: () => true,},]);@@ -59,40 +76,73 @@ const exists = path => stat(path).then(* both carries a stray monolith setup-data re-seeded after the split, which is* not legacy. An unreadable monolith is reported but never converted, so its* live store is not parked for a split that would throw.+ *+ * `ledger` is the snapshot's migration ledger, passed only for a full restore+ * (which installs it). When it lacks a split migration, the restore replays the+ * ledger's pending migrations through that split (`migrations`), and every live+ * store of that split the snapshot has no split index for is converted too.*/-export async function snapshotLegacySplitStores(srcDir, scope) {- const found = [];+export async function snapshotLegacySplitStores(srcDir, scope, { ledger = null, migrationsDir = DEFAULT_MIGRATIONS_DIR } = {}) {+ const recorded = !scope && Array.isArray(ledger) ? new Set(ledger) : null;+ const replays = entry => recorded !== null && !recorded.has(entry.migration);+ const stores = [];for (const entry of LEGACY_SPLIT_STORES) {if (scope && !entry.scopes.includes(scope)) continue;- if (!await exists(join(srcDir, entry.legacyFile))) continue;if (await exists(join(srcDir, entry.store, 'index.json'))) continue;+ const carriesMonolith = await exists(join(srcDir, entry.legacyFile));+ if (!carriesMonolith) {+ // Only a live store needs parking for a replay to rebuild it.+ const rebuilt = replays(entry) && entry.replayRoot+ && await exists(join(srcDir, entry.replayRoot)) && await exists(join(PATHS.data, entry.store));+ if (!rebuilt) continue;+ stores.push({ ...entry, carriesMonolith, readable: true });+ continue;+ }const { ok, value } = await readJSONFileStrict(join(srcDir, entry.legacyFile), null, { logError: false });const readable = ok && value !== null && typeof value === 'object' && !Array.isArray(value) && entry.readable(value);- found.push({ ...entry, readable });+ stores.push({ ...entry, carriesMonolith, readable });}// Migrations take the install root and join `data/` themselves. Refuse before// any transfer, so a store that cannot be converted is never restored over.- if (found.some(entry => entry.readable) && basename(PATHS.data) !== 'data') {+ if (stores.some(entry => entry.readable) && basename(PATHS.data) !== 'data') {throw new Error(`Cannot convert restored legacy stores: data directory ${PATHS.data} is not <install>/data`);}- return found;+ const through = stores.filter(entry => entry.readable && replays(entry)).map(entry => entry.migration).sort().at(-1) ?? null;+ const migrations = through ? { through, names: pendingThrough(await scanMigrationFiles(migrationsDir), recorded, through) } : null;+ return { stores, migrations };}+const pendingThrough = (files, applied, through) => files.slice(0, files.indexOf(through) + 1).filter(file => !applied.has(file));+/** The result-facing projection of `snapshotLegacySplitStores` for a preview. */-export const describeLegacySplitStores = stores => stores.map(({ store, legacyFile, readable }) => ({- store, legacyFile, status: readable ? 'pending' : 'unreadable', parkedAs: null,-}));+export const describeLegacySplitStores = ({ stores, migrations }) => ({+ stores: stores.map(({ store, legacyFile, readable }) => ({+ store, legacyFile, status: readable ? 'pending' : 'unreadable', parkedAs: null,+ })),+ migrations: migrations && { status: 'pending', ...migrations },+});/*** Park each convertible store's live directory (and a live migration backup the- * split's own rename would overwrite), then run the frozen split migrations.- * Call only after the transfer succeeded, inside the live-restore owners.+ * split's own rename would overwrite), then run the frozen split migrations:+ * through the runner when the restored ledger lacks the split, alone otherwise.+ * Call only after the transfer and ledger adoption succeeded, inside the+ * live-restore owners.*/-export async function convertRestoredLegacySplitStores(stores) {- if (stores.length === 0) return [];+export async function convertRestoredLegacySplitStores({ stores, migrations }, { migrationsDir = DEFAULT_MIGRATIONS_DIR } = {}) {+ if (stores.length === 0) return { stores: [], migrations: null };+ const rootDir = dirname(PATHS.data);+ // Decided again from the ledger the transfer installed: replaying through a+ // split the live ledger already records would park stores nothing rebuilds.+ const applied = migrations ? new Set(await readAppliedList(appliedFilePath(rootDir))) : new Set();+ const through = migrations && !applied.has(migrations.through) ? migrations.through : null;+ const replays = entry => through !== null && !applied.has(entry.migration);+const stamp = new Date().toISOString().replace(/[:.]/g, '-');const results = [];+ const direct = new Set();for (const entry of stores) {+ if (!entry.carriesMonolith && !replays(entry)) continue;if (!entry.readable) {console.warn(`⚠️ Restore left ${entry.legacyFile} in its pre-split layout: the snapshot copy is unreadable`);results.push({ store: entry.store, legacyFile: entry.legacyFile, status: 'unreadable', parkedAs: null });@@ -107,21 +157,33 @@ export async function convertRestoredLegacySplitStores(stores) {const liveBackup = join(PATHS.data, `${entry.legacyFile}${entry.backupSuffix}`);if (await exists(liveBackup)) await rename(liveBackup, `${liveBackup}.pre-restore-${stamp}`);results.push({ store: entry.store, legacyFile: entry.legacyFile, status: 'converted', parkedAs: parked ? parkedAs : null });+ if (!replays(entry)) direct.add(entry.loadMigration);}+ const failed = (error) => {+ const parked = results.filter(result => result.parkedAs).map(result => result.parkedAs);+ return new Error(`Converting restored legacy stores failed: ${error.message}${parked.length ? `. Previous stores are parked as ${parked.join(', ')}` : ''}`, { cause: error });+ };+ let replayed = null;+ if (through !== null) {+ const names = pendingThrough(await scanMigrationFiles(migrationsDir), applied, through);+ const { runMigrations } = await import('../../scripts/run-migrations.js');+ await runMigrations({ rootDir, migrationsDir, through }).catch((error) => { throw failed(error); });+ // The runner skips silently for some roots (a worktree checkout); a split+ // it never recorded left the parked stores unrebuilt.+ if (!(await readAppliedList(appliedFilePath(rootDir))).includes(through)) {+ throw failed(new Error(`the migration run did not apply ${through}`));+ }+ replayed = { status: 'applied', through, names };+ }// One run per migration: 200 splits every Brain type, and a type whose live// index was not parked is a gate-1 no-op.- const loaders = [...new Set(stores.filter(entry => entry.readable).map(entry => entry.loadMigration))];- const rootDir = dirname(PATHS.data);- for (const load of loaders) {+ for (const load of direct) {const { default: migration } = await load();- await migration.up({ rootDir }).catch((error) => {- const parked = results.filter(result => result.parkedAs).map(result => result.parkedAs);- throw new Error(`Converting restored legacy stores failed: ${error.message}${parked.length ? `. Previous stores are parked as ${parked.join(', ')}` : ''}`, { cause: error });- });+ await migration.up({ rootDir }).catch((error) => { throw failed(error); });}for (const result of results.filter(result => result.status === 'converted')) {console.log(`📦 Restore converted ${result.legacyFile} to per-record files${result.parkedAs ? `; previous store parked as ${result.parkedAs}` : ''}`);}- return results;+ return { stores: results, migrations: replayed };}diff --git a/server/services/backupLegacyStores.test.js b/server/services/backupLegacyStores.test.jsnew file mode 100644index 0000000000..08771e7072--- /dev/null+++ b/server/services/backupLegacyStores.test.js@@ -0,0 +1,167 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';+import { join } from 'path';+import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';+import { runMigrations, listPendingMigrations } from '../../scripts/run-migrations.js';++const state = vi.hoisted(() => ({ liveData: null }));+vi.mock('../lib/fileUtils.js', async (importOriginal) => makePathsProxy(await importOriginal(), { dataRoot: () => state.liveData }));++const { snapshotLegacySplitStores, describeLegacySplitStores, convertRestoredLegacySplitStores } = await import('./backupLegacyStores.js');++// The real, frozen migrations whose order #12813 is about, re-exported from a+// scratch directory so the run stays bounded to them.+const REAL_MIGRATIONS = [+ '021-link-orphan-universe-collections.js',+ '038-canonicalize-linked-collection-ids.js',+ '059-split-media-collections.js',+ '080-brain-tombstone-and-synclog-cleanup.js',+ '081-brain-daily-log-inbox-sync.js',+ '190-seed-songbook-songs.js',+ '200-split-brain-stores-to-per-record.js',+];+const LATER = '999-later-example.js';+const migrationsRoot = new URL('../../scripts/migrations/', import.meta.url);++const writeJson = (path, value) => {+ mkdirSync(join(path, '..'), { recursive: true });+ writeFileSync(path, JSON.stringify(value));+};+const read = path => readFileSync(path, 'utf8');++describe('restore of a snapshot from before a store split replays earlier migrations in order (#12813)', () => {+ let base;+ let migrationsDir;+ let snapshotRoot;+ let liveRoot;+ let directRoot;++ beforeEach(() => {+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime(new Date('2026-03-01T12:00:00.000Z'));+ vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ base = createTempDataRoot('portos-legacy-stores-');+ migrationsDir = join(base, 'migrations');+ mkdirSync(migrationsDir);+ for (const file of REAL_MIGRATIONS) {+ writeFileSync(join(migrationsDir, file), `export { default } from ${JSON.stringify(new URL(file, migrationsRoot).href)};`);+ }+ writeFileSync(join(migrationsDir, LATER), `+import { writeFileSync } from 'fs';+import { join } from 'path';+export default { up({ rootDir }) { writeFileSync(join(rootDir, 'data', 'later-marker.txt'), 'ran'); } };+`);+ snapshotRoot = join(base, 'snapshot');+ liveRoot = join(base, 'live');+ directRoot = join(base, 'direct');+ state.liveData = join(liveRoot, 'data');++ // A current, fully migrated install with records of its own.+ const live = state.liveData;+ for (const type of ['inbox', 'journals', 'people']) {+ writeJson(join(live, 'brain', type, 'index.json'), { schemaVersion: 1, type, config: {} });+ }+ writeJson(join(live, 'brain', 'inbox', 'inb-live-1', 'index.json'), { text: 'Live capture' });+ writeJson(join(live, 'media-collections', 'index.json'), { schemaVersion: 1, type: 'mediaCollections', config: {} });+ writeJson(join(live, 'migrations.applied.json'), [...REAL_MIGRATIONS, LATER]);+ });++ afterEach(() => {+ vi.useRealTimers();+ vi.restoreAllMocks();+ rmSync(base, { recursive: true, force: true });+ });++ /** Restore `snapshotRoot` into the live install the way a full restore does. */+ async function restore() {+ const ledger = JSON.parse(read(join(snapshotRoot, 'data', 'migrations.applied.json')));+ const legacy = await snapshotLegacySplitStores(join(snapshotRoot, 'data'), undefined, { ledger, migrationsDir });+ const preview = describeLegacySplitStores(legacy);+ cpSync(join(snapshotRoot, 'data'), state.liveData, { recursive: true }); // rsync is additive+ const result = await convertRestoredLegacySplitStores(legacy, { migrationsDir });+ return { preview, result };+ }++ async function upgradeDirectly() {+ cpSync(snapshotRoot, directRoot, { recursive: true });+ await runMigrations({ rootDir: directRoot, migrationsDir });+ }++ it('builds and splits the inbox and normalizes journals exactly as an upgrade of the same root', async () => {+ const data = join(snapshotRoot, 'data');+ writeJson(join(data, 'migrations.applied.json'), REAL_MIGRATIONS.slice(0, 3));+ mkdirSync(join(data, 'brain'), { recursive: true });+ writeFileSync(join(data, 'brain', 'inbox_log.jsonl'), `${JSON.stringify({ id: 'inb-example-1', text: 'Example capture', capturedAt: '2025-03-01T09:00:00.000Z' })}\n`);+ writeJson(join(data, 'brain', 'journals.json'), { records: {+ '2025-03-01': { id: 'jrnl-example-uuid', date: '2025-03-01', content: 'Example day', obsidianPath: 'Daily/2025-03-01.md' },+ } });++ const { preview, result } = await restore();++ expect(preview.migrations).toEqual({+ status: 'pending',+ through: '200-split-brain-stores-to-per-record.js',+ names: REAL_MIGRATIONS.slice(3),+ });+ expect(result.migrations).toEqual({ ...preview.migrations, status: 'applied' });+ expect(result.stores.map(({ store, status }) => [store, status])).toEqual([+ ['brain/people', 'converted'], ['brain/journals', 'converted'], ['brain/inbox', 'converted'],+ ]);+ // The replay stops at the split; the next boot runs what follows.+ await expect(listPendingMigrations({ rootDir: liveRoot, migrationsDir })).resolves.toEqual([LATER]);+ expect(existsSync(join(state.liveData, 'later-marker.txt'))).toBe(false);+ await runMigrations({ rootDir: liveRoot, migrationsDir });+ await upgradeDirectly();++ for (const record of ['brain/inbox/inb-example-1/index.json', 'brain/journals/2025-03-01/index.json']) {+ expect(read(join(state.liveData, record))).toBe(read(join(directRoot, 'data', record)));+ }+ const journal = JSON.parse(read(join(state.liveData, 'brain/journals/2025-03-01/index.json')));+ expect(journal).not.toHaveProperty('id');+ expect(journal).toMatchObject({ createdAt: expect.any(String), updatedAt: expect.any(String) });+ expect(existsSync(join(state.liveData, 'brain', 'inbox', 'inb-live-1'))).toBe(false);+ const parkedInbox = result.stores.find(store => store.store === 'brain/inbox').parkedAs;+ expect(JSON.parse(read(join(state.liveData, parkedInbox, 'inb-live-1', 'index.json')))).toEqual({ text: 'Live capture' });+ });++ it('runs 021 and 038 before splitting a pre-059 media-collections store', async () => {+ const data = join(snapshotRoot, 'data');+ writeJson(join(data, 'migrations.applied.json'), []);+ writeJson(join(data, 'universe-builder.json'), { universes: [{ id: 'uni-example', name: 'Example' }] });+ writeJson(join(data, 'media-collections.json'), { collections: [{ id: 'col-example', name: 'universe: example', items: [] }] });++ const { result } = await restore();++ expect(result.migrations).toMatchObject({ status: 'applied', through: '059-split-media-collections.js', names: REAL_MIGRATIONS.slice(0, 3) });+ await runMigrations({ rootDir: liveRoot, migrationsDir });+ await upgradeDirectly();++ // 021 links the collection to its universe; 038 then re-keys it to the canonical id.+ const record = 'media-collections/uc-uni-example/index.json';+ expect(read(join(state.liveData, record))).toBe(read(join(directRoot, 'data', record)));+ expect(JSON.parse(read(join(state.liveData, record)))).toMatchObject({ universeId: 'uni-example', name: 'Universe: Example' });+ });++ it('converts with the split alone when the restored ledger already records it', async () => {+ const data = join(snapshotRoot, 'data');+ writeJson(join(data, 'migrations.applied.json'), REAL_MIGRATIONS);+ writeJson(join(data, 'brain', 'people.json'), { records: { 'p-example': { name: 'Example Person' } } });++ const { preview, result } = await restore();++ expect(preview.migrations).toBeNull();+ expect(result).toMatchObject({ migrations: null, stores: [{ store: 'brain/people', status: 'converted' }] });+ expect(JSON.parse(read(join(state.liveData, 'brain', 'people', 'p-example', 'index.json')))).toEqual({ name: 'Example Person' });+ // Stores the snapshot carries no monolith for are left alone without a replay.+ expect(existsSync(join(state.liveData, 'brain', 'inbox', 'inb-live-1'))).toBe(true);+ });++ it('keeps a folder restore to the snapshot monoliths and never replays', async () => {+ const data = join(snapshotRoot, 'data');+ writeJson(join(data, 'brain', 'journals.json'), { records: { '2025-03-01': { date: '2025-03-01' } } });+ const legacy = await snapshotLegacySplitStores(data, 'brain', { ledger: [], migrationsDir });+ expect(legacy.migrations).toBeNull();+ expect(legacy.stores.map(entry => entry.store)).toEqual(['brain/journals']);+ });+});