Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: restore a pre-081 backup with its inbox and journals upgraded in order (#12813)

Adam Eivydc3f4e19388d
diff --git a/client/src/components/BackupWidget.jsx b/client/src/components/BackupWidget.jsx
index 5608c48991..9839e31751 100644
--- a/client/src/components/BackupWidget.jsx
+++ b/client/src/components/BackupWidget.jsx
@@ -159,10 +159,13 @@ function RestorePanel({ snapshot, onClose, restoring, onRestoreStateChange }) {
const suffix = result.verification?.status === 'unverified' ? ' (unverified legacy snapshot)' : '';
const parked = (result.restoredLegacyStores ?? []).filter(store => store.parkedAs).map(store => store.parkedAs);
const conversion = parked.length > 0 ? `. Converted older-format data; the replaced stores were kept as ${parked.join(', ')}` : '';
+ const replayed = result.restoredLegacyMigrations?.status === 'applied'
+ ? `. Ran the ${pluralize(result.restoredLegacyMigrations.names.length, 'data migration')} the snapshot predates through ${result.restoredLegacyMigrations.through}`
+ : '';
const ledger = result.restoredLegacyLedger?.status === 'adopted'
? `. Adopted the snapshot's migration record${result.restoredLegacyLedger.parkedAs ? ` (previous record kept as ${result.restoredLegacyLedger.parkedAs})` : ''}; restart PortOS to re-run the migrations it predates`
: '';
- toast.success(`File restore complete — ${result.changedFiles?.length ?? 0} file(s) restored${suffix}${conversion}${ledger}`);
+ toast.success(`File restore complete — ${result.changedFiles?.length ?? 0} file(s) restored${suffix}${conversion}${replayed}${ledger}`);
onClose();
}
}, [acceptedPreview, onClose, onRestoreStateChange, previewMatchesCurrentRequest, restoring, snapshot]);
@@ -292,6 +295,14 @@ function RestorePanel({ snapshot, onClose, restoring, onRestoreStateChange }) {
</span>
</p>
)}
+ {preview.restoredLegacyMigrations && (
+ <p className="mb-2 flex items-start gap-1.5 rounded border border-port-warning/40 bg-port-warning/10 p-2 text-xs text-port-warning" role="alert">
+ <AlertTriangle size={13} className="mt-0.5 shrink-0" />
+ <span className="min-w-0 break-words">
+ This snapshot predates {pluralize(preview.restoredLegacyMigrations.names.length, 'data migration')} up to <code>{preview.restoredLegacyMigrations.through}</code>. Restoring runs them right away, in order, so its records convert exactly as an upgrade would; later migrations run on the next start.
+ </span>
+ </p>
+ )}
<p className="text-xs text-gray-500 mb-1">
{preview.changedFiles?.length ?? 0} file(s) would change:
</p>
diff --git a/client/src/components/BackupWidget.test.jsx b/client/src/components/BackupWidget.test.jsx
index b805eb409e..c96fcf165c 100644
--- a/client/src/components/BackupWidget.test.jsx
+++ b/client/src/components/BackupWidget.test.jsx
@@ -380,6 +380,22 @@ describe('BackupWidget snapshots', () => {
));
});
+ it('announces the migrations a pre-split restore runs through the split and reports them afterwards', async () => {
+ const migrations = { through: '200-split-example.js', names: ['081-example.js', '200-split-example.js'] };
+ mockRestoreBackup
+ .mockResolvedValueOnce({ dryRun: true, changedFiles: ['brain/journals.json'], restoredLegacyMigrations: { status: 'pending', ...migrations } })
+ .mockResolvedValueOnce({ changedFiles: ['brain/journals.json'], restoredLegacyMigrations: { status: 'applied', ...migrations } });
+ renderWidget();
+ await openRestorePanel();
+ fireEvent.click(screen.getByRole('button', { name: 'Preview changes' }));
+
+ expect(await screen.findByRole('alert')).toHaveTextContent(/predates 2 data migrations up to 200-split-example\.js.*runs them right away, in order/);
+ fireEvent.click(screen.getByRole('button', { name: 'Restore 1 file(s)' }));
+ await waitFor(() => expect(mockToast.success).toHaveBeenCalledWith(
+ 'File restore complete — 1 file(s) restored. Ran the 2 data migrations the snapshot predates through 200-split-example.js',
+ ));
+ });
+
it('announces a pre-May-2026 ledger adoption and asks for a restart afterwards', async () => {
mockRestoreBackup
.mockResolvedValueOnce({
diff --git a/docs/BACKUP.md b/docs/BACKUP.md
index 3149accb3f..8fac11a288 100644
--- a/docs/BACKUP.md
+++ b/docs/BACKUP.md
@@ -517,7 +517,7 @@ A snapshot taken before the agent records moved into PostgreSQL (#12738) also ca
Keep the original backup intact. Copy it into a disposable offline install running the compatible historical version, upgrade that isolated install through the file-to-database transition, and verify the imported records and linked files there. Make a current-format file and PostgreSQL backup from the converted install, then use the normal verified restore workflow. Do not convert against the live database or delete its markers. Current file-primary siblings (such as manuscript reviews, draft Markdown and media assets) and parked `.imported` recovery evidence remain eligible for file restore. Selecting one of those siblings is allowed even if another part of the snapshot contains legacy metadata.
-A full restore also installs the snapshot's `migrations.applied.json`, so the next start re-runs every file migration the snapshot predates. A snapshot taken before migrations moved out of `data/` (May 2026) carries that ledger at `migrations/.applied.json` instead; the transfer lands it beside the live `migrations.applied.json`, which the runner would then keep, so a full restore adopts it by hand after the transfer: the live ledger is parked as `migrations.applied.json.pre-restore-<timestamp>` and the restored one takes its place. Preview and result carry `restoredLegacyLedger` (`{ status, parkedAs }`, `pending` in a preview and `adopted` after execution; `unreadable` when the snapshot's legacy ledger does not parse as a list, in which case the live ledger is kept); the widget warns before restore and asks for a restart afterwards. A snapshot that also carries a current-path ledger is left to the transfer. That re-run only upgrades restored files for migrations gated on their input. The store-split migrations (059 for media collections, 200 for the Brain stores) gate on their output, a type index already at schema v1, and file restore leaves the live index in place, so their re-run is a no-op over a restored monolith. Restore therefore converts those stores itself (#12626): when the scope covers a store (full restore, `brain`, `brain/<type>.json`, `media-collections` or `media-collections.json`) and the snapshot carries its monolith (`brain/<type>.json`, `media-collections.json`) without `<store>/index.json`, a successful transfer moves the live `<store>/` directory to `<store>.pre-restore-<timestamp>/` (and a live `.bak-200`/`.bak-059` backup beside it), runs the frozen split migration against the install, and invalidates the Brain caches. The restored records, tombstones included, then load immediately, for folder restores too. A snapshot that has the split layout plus a stray monolith (setup-data re-seeds those after an upgrade) is current and is not converted. A monolith the split would refuse (unparseable, or a malformed records container) is restored as-is and never parks the live store. Preview and result list these stores in `restoredLegacyStores` (`{ store, legacyFile, status, parkedAs }`, with `status` `pending` in a preview and `converted` or `unreadable` after execution); the widget warns before restore and names the parked directory afterwards. A folder restore (`subdirFilter`) keeps the live ledger, so restored files can come back in a shape older than migrations no runner will revisit. Its preview and result therefore carry `ledgerGap`: `{ count, names }` listing the migrations the live ledger records and the snapshot's lacks, or `null` when either ledger is missing or unreadable. A full restore omits the field. The widget warns on a non-empty gap and recommends a full restore, but does not refuse: a folder no migration touched restores correctly. The quota-burn plan is converted on the spot: a restore covering `cos/quota-burn.json` (full, `cos`, or that file) runs any legacy `jobType` steps through the same conversion as migration 359 (`reconcileRestoredQuotaBurnPlan`), reusing each step's deterministic `job-burn-<family>-<step>` custom job, so a repeated restore creates no duplicates.
+A full restore also installs the snapshot's `migrations.applied.json`, so the next start re-runs every file migration the snapshot predates. A snapshot taken before migrations moved out of `data/` (May 2026) carries that ledger at `migrations/.applied.json` instead; the transfer lands it beside the live `migrations.applied.json`, which the runner would then keep, so a full restore adopts it by hand after the transfer: the live ledger is parked as `migrations.applied.json.pre-restore-<timestamp>` and the restored one takes its place. Preview and result carry `restoredLegacyLedger` (`{ status, parkedAs }`, `pending` in a preview and `adopted` after execution; `unreadable` when the snapshot's legacy ledger does not parse as a list, in which case the live ledger is kept); the widget warns before restore and asks for a restart afterwards. A snapshot that also carries a current-path ledger is left to the transfer. That re-run only upgrades restored files for migrations gated on their input. The store-split migrations (059 for media collections, 200 for the Brain stores) gate on their output, a type index already at schema v1, and file restore leaves the live index in place, so their re-run is a no-op over a restored monolith. Restore therefore converts those stores itself (#12626): when the scope covers a store (full restore, `brain`, `brain/<type>.json`, `media-collections` or `media-collections.json`) and the snapshot carries its monolith (`brain/<type>.json`, `media-collections.json`) without `<store>/index.json`, a successful transfer moves the live `<store>/` directory to `<store>.pre-restore-<timestamp>/` (and a live `.bak-200`/`.bak-059` backup beside it), runs the frozen split migration against the install, and invalidates the Brain caches. The restored records, tombstones included, then load immediately, for folder restores too. A full restore whose snapshot ledger lacks the split migration also lacks the migrations an upgrade ran before it (080/081/190 for the Brain stores, 021/038 for media collections), so it does not run the split alone: after adopting the restored ledger it runs that ledger's pending migrations through the split with the real runner (`runMigrations({ through })`), in order (#12813). It also parks every live Brain type directory the snapshot has no `<type>/index.json` for, because an earlier migration can create the monolith (081 builds `brain/inbox.json` from `inbox_log.jsonl`). The result matches an in-order upgrade of the snapshot, and migrations after the split run on the next start. Preview and result carry `restoredLegacyMigrations` (`{ status, through, names }`, `pending` in a preview and `applied` after execution). Folder restores never replay. A snapshot that has the split layout plus a stray monolith (setup-data re-seeds those after an upgrade) is current and is not converted. A monolith the split would refuse (unparseable, or a malformed records container) is restored as-is and never parks the live store. Preview and result list these stores in `restoredLegacyStores` (`{ store, legacyFile, status, parkedAs }`, with `status` `pending` in a preview and `converted` or `unreadable` after execution); the widget warns before restore and names the parked directory afterwards. A folder restore (`subdirFilter`) keeps the live ledger, so restored files can come back in a shape older than migrations no runner will revisit. Its preview and result therefore carry `ledgerGap`: `{ count, names }` listing the migrations the live ledger records and the snapshot's lacks, or `null` when either ledger is missing or unreadable. A full restore omits the field. The widget warns on a non-empty gap and recommends a full restore, but does not refuse: a folder no migration touched restores correctly. The quota-burn plan is converted on the spot: a restore covering `cos/quota-burn.json` (full, `cos`, or that file) runs any legacy `jobType` steps through the same conversion as migration 359 (`reconcileRestoredQuotaBurnPlan`), reusing each step's deterministic `job-burn-<family>-<step>` custom job, so a repeated restore creates no duplicates.
**Every nonzero rsync exit fails a restore, including 24.** A backup tolerates exit 24 ("some source files vanished") because it reads the live data tree, where a file deleted mid-scan is normal. A restore reads a snapshot that must not change under it, so a vanished source file means live data may now mix restored and older files: preview and execution both reject it. An execution failure is reported as a partial restore — the error says some files may already have been overwritten because file restore is not transactional — and caches (settings, Brain, media index) are reconciled before the error is returned, exactly as for any other failed transfer.
diff --git a/scripts/run-migrations.js b/scripts/run-migrations.js
index 40735246d1..d30d4338ae 100644
--- a/scripts/run-migrations.js
+++ b/scripts/run-migrations.js
@@ -196,6 +196,10 @@ export async function runMigrations({
migrationsDir = DEFAULT_MIGRATIONS_DIR,
lockWaitMs = LOCK_WAIT_MS,
lockPollMs = LOCK_POLL_MS,
+ // Stop after this migration file, leaving later ones pending. A restore
+ // replays a pre-split snapshot's pending migrations through its split this
+ // way, so the split sees what an in-order upgrade would have given it (#12813).
+ through = null,
} = {}) {
// Backstop for #1947: a process booted from inside a CoS agent git worktree
// (data/cos/worktrees/agent-*) has no gitignored data/ runtime tree, so
@@ -217,13 +221,13 @@ export async function runMigrations({
// the read is only trustworthy while no other runner can append to it.
const releaseLock = await acquireMigrationLock(lockDirPath(rootDir), { waitMs: lockWaitMs, pollMs: lockPollMs });
try {
- return await runPendingMigrations({ rootDir, migrationsDir, appliedFile });
+ return await runPendingMigrations({ rootDir, migrationsDir, appliedFile, through });
} finally {
await releaseLock();
}
}
-async function runPendingMigrations({ rootDir, migrationsDir, appliedFile }) {
+async function runPendingMigrations({ rootDir, migrationsDir, appliedFile, through }) {
if (await adoptLegacyAppliedList(rootDir)) {
console.log('🧹 Moved data/migrations/.applied.json → data/migrations.applied.json');
}
@@ -260,6 +264,9 @@ async function runPendingMigrations({ rootDir, migrationsDir, appliedFile }) {
const ledgerStartedEmpty = applied.length === 0;
const files = await scanMigrationFiles(migrationsDir);
+ if (through !== null && !files.includes(through)) {
+ throw new Error(`Cannot run migrations through "${through}": no such migration in ${migrationsDir}`);
+ }
// The migration helper also carries prompt-drift and provider-seed
// machinery. Boot/status callers import this runner widely, but only an
@@ -301,7 +308,7 @@ async function runPendingMigrations({ rootDir, migrationsDir, appliedFile }) {
if (disarmed > 0) await persistAppliedList();
let ran = 0;
- for (const file of files) {
+ for (const file of through === null ? files : files.slice(0, files.indexOf(through) + 1)) {
if (applied.includes(file)) continue;
const mod = await import(pathToFileURL(join(migrationsDir, file)).href);
diff --git a/scripts/run-migrations.test.js b/scripts/run-migrations.test.js
index 6ef6d24d8b..6e8e029dfd 100644
--- a/scripts/run-migrations.test.js
+++ b/scripts/run-migrations.test.js
@@ -248,6 +248,18 @@ export default {
expect(JSON.parse(readFileSync(appliedFile, 'utf-8'))).toEqual(['001-fixture.js']);
await expect(listPendingMigrations({ rootDir, migrationsDir })).resolves.toEqual(['002-invalid.js']);
});
+
+ it('stops after the migration named by `through`, leaving later ones pending (#12813)', async () => {
+ writeFileSync(join(migrationsDir, '002-bounded.js'), 'export default { up() {} };');
+ writeFileSync(join(migrationsDir, '003-later.js'), 'throw new Error("A migration after the bound was imported");');
+
+ await expect(runMigrations({ rootDir, migrationsDir, through: '002-bounded.js' })).resolves.toBe(2);
+
+ expect(JSON.parse(readFileSync(appliedFile, 'utf-8'))).toEqual(['001-fixture.js', '002-bounded.js']);
+ await expect(listPendingMigrations({ rootDir, migrationsDir })).resolves.toEqual(['003-later.js']);
+ await expect(runMigrations({ rootDir, migrationsDir, through: '009-missing.js' }))
+ .rejects.toThrow('Cannot run migrations through "009-missing.js"');
+ });
});
describe('runMigrations purge-migration guard (#2770)', () => {
diff --git a/server/services/backup.js b/server/services/backup.js
index df0c0b4008..a46ac689f3 100644
--- a/server/services/backup.js
+++ b/server/services/backup.js
@@ -1903,12 +1903,17 @@ async function restoreHeldSnapshot({ snapshotDir, currentSource }, snapshotId, {
// migrations the live ledger already records.
const ledgerGap = subdirFilter ? await migrationLedgerGap(srcDir) : undefined;
// Split migrations gate on their output, so neither ledger re-runs them over
- // a restored monolith; restore converts those stores itself (#12626).
- const legacyStores = await snapshotLegacySplitStores(srcDir, subdirFilter);
+ // a restored monolith; restore converts those stores itself (#12626). A full
+ // restore installs the snapshot's ledger, so the migrations it lacks before a
+ // split are replayed with that split, in order (#12813).
+ const legacyStores = await snapshotLegacySplitStores(srcDir, subdirFilter, {
+ ledger: subdirFilter ? null : await readSnapshotMigrationLedger(srcDir),
+ });
const legacyLedger = await snapshotLegacyLedger(srcDir, subdirFilter);
- const restoreNotes = (restoredLegacyStores, restoredLegacyLedger = legacyLedger) => ({
+ const restoreNotes = (restoredLegacy, restoredLegacyLedger = legacyLedger) => ({
...(ledgerGap !== undefined && { ledgerGap }),
- ...(restoredLegacyStores.length > 0 && { restoredLegacyStores }),
+ ...(restoredLegacy.stores.length > 0 && { restoredLegacyStores: restoredLegacy.stores }),
+ ...(restoredLegacy.migrations && { restoredLegacyMigrations: restoredLegacy.migrations }),
...(restoredLegacyLedger && { restoredLegacyLedger }),
...(identity.preserved && { identityPreserved: identity.preserved }),
...(identity.adopted && { identityAdopted: identity.adopted }),
@@ -1970,10 +1975,11 @@ async function restoreHeldSnapshot({ snapshotDir, currentSource }, snapshotId, {
}
const scope = subdirFilter;
- let restoredLegacyStores = [];
+ let restoredLegacy = { stores: [], migrations: null };
let restoredLegacyLedger = legacyLedger;
// Conversion parks live stores, so it runs only after a successful transfer,
- // and before reconciliation so the Brain caches drop the parked records.
+ // and before reconciliation so the Brain caches drop the parked records. It
+ // follows ledger adoption so a replay reads the ledger the boot would read.
const changedFiles = await withLiveFileRestoreOwners(scope, srcDir, () => transferAndReconcileLiveFiles({
subdirFilter,
transfer: async () => {
@@ -1985,12 +1991,12 @@ async function restoreHeldSnapshot({ snapshotDir, currentSource }, snapshotId, {
},
transferFiles,
});
- restoredLegacyStores = await convertRestoredLegacySplitStores(legacyStores);
if (legacyLedger) restoredLegacyLedger = await adoptRestoredLegacyLedger(legacyLedger);
+ restoredLegacy = await convertRestoredLegacySplitStores(legacyStores);
return changed;
},
}));
- return { dryRun, snapshotId, subdirFilter, changedFiles, verification, ...restoreNotes(restoredLegacyStores, restoredLegacyLedger) };
+ return { dryRun, snapshotId, subdirFilter, changedFiles, verification, ...restoreNotes(restoredLegacy, restoredLegacyLedger) };
}
async function readMigrationLedger(path) {
diff --git a/server/services/backup.test.js b/server/services/backup.test.js
index 969ccaa86d..31bf29a48a 100644
--- a/server/services/backup.test.js
+++ b/server/services/backup.test.js
@@ -83,6 +83,10 @@ vi.mock('../lib/databaseMaintenanceJournal.js', () => ({
vi.mock('../scripts/run-db-migrations.js', () => ({
runDbMigrations: vi.fn().mockResolvedValue(0),
}));
+// The file-migration runner a pre-split restore replays through (#12813);
+// backupLegacyStores.test.js runs the real one against the real migrations.
+const runFileMigrations = vi.hoisted(() => vi.fn());
+vi.mock('../../scripts/run-migrations.js', () => ({ runMigrations: runFileMigrations }));
const catalogRepair = vi.hoisted(() => ({
migrate: vi.fn().mockResolvedValue({ skipped: false }),
tags: vi.fn().mockResolvedValue({ skipped: false }),
@@ -2509,6 +2513,45 @@ describe('restoreSnapshot manifest verification', () => {
await realFs.rm(live('cos', 'example.json'), { force: true });
});
+ it('replays the adopted ledger through the split and reports it in preview and result (#12813)', async () => {
+ const ledgerPath = live('migrations.applied.json');
+ const legacyLedgerPath = live('migrations', '.applied.json');
+ const split = '200-split-brain-stores-to-per-record.js';
+ const journals = JSON.stringify({ records: { '2025-03-01': { id: 'jrnl-example', date: '2025-03-01' } } });
+ await seedSnapshot({ 'brain/journals.json': journals, 'migrations/.applied.json': JSON.stringify(['001-example.js']) });
+ await realFs.writeFile(ledgerPath, JSON.stringify(['001-example.js', split]));
+ try {
+ const preview = await finishRestore({ dryRun: true });
+ expect(preview.restoredLegacyMigrations).toMatchObject({ status: 'pending', through: split });
+ expect(preview.restoredLegacyMigrations.names).toContain('081-brain-daily-log-inbox-sync.js');
+ expect(preview.restoredLegacyMigrations.names.at(-1)).toBe(split);
+ expect(preview.restoredLegacyStores.map(entry => entry.store)).toEqual(['brain/people', 'brain/journals']);
+
+ // rsync is faked, so stand in for its transfer of the snapshot files.
+ await realFs.writeFile(live('brain', 'journals.json'), journals);
+ await realFs.mkdir(live('migrations'), { recursive: true });
+ await realFs.writeFile(legacyLedgerPath, JSON.stringify(['001-example.js']));
+ runFileMigrations.mockImplementationOnce(async ({ through }) => {
+ // The replay must read the snapshot's ledger, so adoption runs first.
+ expect(JSON.parse(await realFs.readFile(ledgerPath, 'utf8'))).toEqual(['001-example.js']);
+ await realFs.writeFile(ledgerPath, JSON.stringify(['001-example.js', through]));
+ });
+ spawn.mockReset();
+ const result = await finishRestore({ dryRun: false });
+
+ expect(runFileMigrations).toHaveBeenCalledWith(expect.objectContaining({ through: split }));
+ expect(result.restoredLegacyLedger).toMatchObject({ status: 'adopted' });
+ expect(result.restoredLegacyMigrations).toEqual({ ...preview.restoredLegacyMigrations, status: 'applied' });
+ expect(existsSync(live('brain', 'people', 'live-1'))).toBe(false);
+ } finally {
+ runFileMigrations.mockReset();
+ await realFs.rm(live('migrations'), { recursive: true, force: true });
+ for (const name of await realFs.readdir(PATHS.data)) {
+ if (name.startsWith('migrations.applied.json')) await realFs.rm(live(name), { force: true });
+ }
+ }
+ });
+
it('never parks the live store for a monolith the split would refuse', async () => {
const malformed = JSON.stringify({ records: [{ id: 'p-1' }] });
await seedSnapshot({ 'brain/people.json': malformed });
diff --git a/server/services/backupLegacyStores.js b/server/services/backupLegacyStores.js
index 5c6195ef56..d2b58c3171 100644
--- a/server/services/backupLegacyStores.js
+++ b/server/services/backupLegacyStores.js
@@ -14,10 +14,19 @@
* install immediately, splitting the restored monolith exactly as an upgrade
* would have. The migrations are invoked, never re-implemented, so a restore
* converts with the same semantics the install crossed when it upgraded.
+ *
+ * A full restore whose ledger lacks the split also lacks the migrations an
+ * upgrade ran BEFORE it (081 builds `brain/inbox.json` from `inbox_log.jsonl`
+ * and normalizes journals). Running the split alone and leaving those to the
+ * next boot would run them after it, writing monoliths nothing reads (#12813).
+ * So that restore replays the restored ledger's pending migrations through the
+ * split with the real runner, and parks every live store of that split the
+ * snapshot carries no split index for, so the result is an in-order upgrade.
*/
import { rename, stat } from 'fs/promises';
import { basename, dirname, join } from 'path';
import { PATHS, readJSONFileStrict } from '../lib/fileUtils.js';
+import { DEFAULT_MIGRATIONS_DIR, appliedFilePath, readAppliedList, scanMigrationFiles } from '../../scripts/lib/migrationLedger.js';
// Frozen with the migrations they invoke: migration 200's type list as of 200.
const BRAIN_SPLIT_TYPES = [
@@ -25,6 +34,8 @@ const BRAIN_SPLIT_TYPES = [
'journals', 'inbox', 'songs',
];
+const BRAIN_SPLIT_MIGRATION = '200-split-brain-stores-to-per-record.js';
+const MEDIA_COLLECTIONS_SPLIT_MIGRATION = '059-split-media-collections.js';
const loadBrainSplit = () => import('../../scripts/migrations/200-split-brain-stores-to-per-record.js');
const loadMediaCollectionsSplit = () => import('../../scripts/migrations/059-split-media-collections.js');
@@ -34,7 +45,11 @@ const LEGACY_SPLIT_STORES = Object.freeze([
scopes: ['brain', `brain/${type}.json`],
legacyFile: `brain/${type}.json`,
backupSuffix: '.bak-200',
+ migration: BRAIN_SPLIT_MIGRATION,
loadMigration: loadBrainSplit,
+ // Earlier migrations create some Brain monoliths (081 the inbox, 190 the
+ // songs), so a replay converts a type the snapshot has no monolith for.
+ replayRoot: 'brain',
// Migration 200 throws on a records container that is present but not a map.
readable: doc => doc.records == null || (typeof doc.records === 'object' && !Array.isArray(doc.records)),
})),
@@ -43,7 +58,9 @@ const LEGACY_SPLIT_STORES = Object.freeze([
scopes: ['media-collections', 'media-collections.json'],
legacyFile: 'media-collections.json',
backupSuffix: '.bak-059',
+ migration: MEDIA_COLLECTIONS_SPLIT_MIGRATION,
loadMigration: loadMediaCollectionsSplit,
+ replayRoot: null,
readable: () => true,
},
]);
@@ -59,40 +76,73 @@ const exists = path => stat(path).then(
* both carries a stray monolith setup-data re-seeded after the split, which is
* not legacy. An unreadable monolith is reported but never converted, so its
* live store is not parked for a split that would throw.
+ *
+ * `ledger` is the snapshot's migration ledger, passed only for a full restore
+ * (which installs it). When it lacks a split migration, the restore replays the
+ * ledger's pending migrations through that split (`migrations`), and every live
+ * store of that split the snapshot has no split index for is converted too.
*/
-export async function snapshotLegacySplitStores(srcDir, scope) {
- const found = [];
+export async function snapshotLegacySplitStores(srcDir, scope, { ledger = null, migrationsDir = DEFAULT_MIGRATIONS_DIR } = {}) {
+ const recorded = !scope && Array.isArray(ledger) ? new Set(ledger) : null;
+ const replays = entry => recorded !== null && !recorded.has(entry.migration);
+ const stores = [];
for (const entry of LEGACY_SPLIT_STORES) {
if (scope && !entry.scopes.includes(scope)) continue;
- if (!await exists(join(srcDir, entry.legacyFile))) continue;
if (await exists(join(srcDir, entry.store, 'index.json'))) continue;
+ const carriesMonolith = await exists(join(srcDir, entry.legacyFile));
+ if (!carriesMonolith) {
+ // Only a live store needs parking for a replay to rebuild it.
+ const rebuilt = replays(entry) && entry.replayRoot
+ && await exists(join(srcDir, entry.replayRoot)) && await exists(join(PATHS.data, entry.store));
+ if (!rebuilt) continue;
+ stores.push({ ...entry, carriesMonolith, readable: true });
+ continue;
+ }
const { ok, value } = await readJSONFileStrict(join(srcDir, entry.legacyFile), null, { logError: false });
const readable = ok && value !== null && typeof value === 'object' && !Array.isArray(value) && entry.readable(value);
- found.push({ ...entry, readable });
+ stores.push({ ...entry, carriesMonolith, readable });
}
// Migrations take the install root and join `data/` themselves. Refuse before
// any transfer, so a store that cannot be converted is never restored over.
- if (found.some(entry => entry.readable) && basename(PATHS.data) !== 'data') {
+ if (stores.some(entry => entry.readable) && basename(PATHS.data) !== 'data') {
throw new Error(`Cannot convert restored legacy stores: data directory ${PATHS.data} is not <install>/data`);
}
- return found;
+ const through = stores.filter(entry => entry.readable && replays(entry)).map(entry => entry.migration).sort().at(-1) ?? null;
+ const migrations = through ? { through, names: pendingThrough(await scanMigrationFiles(migrationsDir), recorded, through) } : null;
+ return { stores, migrations };
}
+const pendingThrough = (files, applied, through) => files.slice(0, files.indexOf(through) + 1).filter(file => !applied.has(file));
+
/** The result-facing projection of `snapshotLegacySplitStores` for a preview. */
-export const describeLegacySplitStores = stores => stores.map(({ store, legacyFile, readable }) => ({
- store, legacyFile, status: readable ? 'pending' : 'unreadable', parkedAs: null,
-}));
+export const describeLegacySplitStores = ({ stores, migrations }) => ({
+ stores: stores.map(({ store, legacyFile, readable }) => ({
+ store, legacyFile, status: readable ? 'pending' : 'unreadable', parkedAs: null,
+ })),
+ migrations: migrations && { status: 'pending', ...migrations },
+});
/**
* Park each convertible store's live directory (and a live migration backup the
- * split's own rename would overwrite), then run the frozen split migrations.
- * Call only after the transfer succeeded, inside the live-restore owners.
+ * split's own rename would overwrite), then run the frozen split migrations:
+ * through the runner when the restored ledger lacks the split, alone otherwise.
+ * Call only after the transfer and ledger adoption succeeded, inside the
+ * live-restore owners.
*/
-export async function convertRestoredLegacySplitStores(stores) {
- if (stores.length === 0) return [];
+export async function convertRestoredLegacySplitStores({ stores, migrations }, { migrationsDir = DEFAULT_MIGRATIONS_DIR } = {}) {
+ if (stores.length === 0) return { stores: [], migrations: null };
+ const rootDir = dirname(PATHS.data);
+ // Decided again from the ledger the transfer installed: replaying through a
+ // split the live ledger already records would park stores nothing rebuilds.
+ const applied = migrations ? new Set(await readAppliedList(appliedFilePath(rootDir))) : new Set();
+ const through = migrations && !applied.has(migrations.through) ? migrations.through : null;
+ const replays = entry => through !== null && !applied.has(entry.migration);
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
const results = [];
+ const direct = new Set();
for (const entry of stores) {
+ if (!entry.carriesMonolith && !replays(entry)) continue;
if (!entry.readable) {
console.warn(`⚠️ Restore left ${entry.legacyFile} in its pre-split layout: the snapshot copy is unreadable`);
results.push({ store: entry.store, legacyFile: entry.legacyFile, status: 'unreadable', parkedAs: null });
@@ -107,21 +157,33 @@ export async function convertRestoredLegacySplitStores(stores) {
const liveBackup = join(PATHS.data, `${entry.legacyFile}${entry.backupSuffix}`);
if (await exists(liveBackup)) await rename(liveBackup, `${liveBackup}.pre-restore-${stamp}`);
results.push({ store: entry.store, legacyFile: entry.legacyFile, status: 'converted', parkedAs: parked ? parkedAs : null });
+ if (!replays(entry)) direct.add(entry.loadMigration);
}
+ const failed = (error) => {
+ const parked = results.filter(result => result.parkedAs).map(result => result.parkedAs);
+ return new Error(`Converting restored legacy stores failed: ${error.message}${parked.length ? `. Previous stores are parked as ${parked.join(', ')}` : ''}`, { cause: error });
+ };
+ let replayed = null;
+ if (through !== null) {
+ const names = pendingThrough(await scanMigrationFiles(migrationsDir), applied, through);
+ const { runMigrations } = await import('../../scripts/run-migrations.js');
+ await runMigrations({ rootDir, migrationsDir, through }).catch((error) => { throw failed(error); });
+ // The runner skips silently for some roots (a worktree checkout); a split
+ // it never recorded left the parked stores unrebuilt.
+ if (!(await readAppliedList(appliedFilePath(rootDir))).includes(through)) {
+ throw failed(new Error(`the migration run did not apply ${through}`));
+ }
+ replayed = { status: 'applied', through, names };
+ }
// One run per migration: 200 splits every Brain type, and a type whose live
// index was not parked is a gate-1 no-op.
- const loaders = [...new Set(stores.filter(entry => entry.readable).map(entry => entry.loadMigration))];
- const rootDir = dirname(PATHS.data);
- for (const load of loaders) {
+ for (const load of direct) {
const { default: migration } = await load();
- await migration.up({ rootDir }).catch((error) => {
- const parked = results.filter(result => result.parkedAs).map(result => result.parkedAs);
- throw new Error(`Converting restored legacy stores failed: ${error.message}${parked.length ? `. Previous stores are parked as ${parked.join(', ')}` : ''}`, { cause: error });
- });
+ await migration.up({ rootDir }).catch((error) => { throw failed(error); });
}
for (const result of results.filter(result => result.status === 'converted')) {
console.log(`📦 Restore converted ${result.legacyFile} to per-record files${result.parkedAs ? `; previous store parked as ${result.parkedAs}` : ''}`);
}
- return results;
+ return { stores: results, migrations: replayed };
}
diff --git a/server/services/backupLegacyStores.test.js b/server/services/backupLegacyStores.test.js
new file mode 100644
index 0000000000..08771e7072
--- /dev/null
+++ b/server/services/backupLegacyStores.test.js
@@ -0,0 +1,167 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';
+import { runMigrations, listPendingMigrations } from '../../scripts/run-migrations.js';
+
+const state = vi.hoisted(() => ({ liveData: null }));
+vi.mock('../lib/fileUtils.js', async (importOriginal) => makePathsProxy(await importOriginal(), { dataRoot: () => state.liveData }));
+
+const { snapshotLegacySplitStores, describeLegacySplitStores, convertRestoredLegacySplitStores } = await import('./backupLegacyStores.js');
+
+// The real, frozen migrations whose order #12813 is about, re-exported from a
+// scratch directory so the run stays bounded to them.
+const REAL_MIGRATIONS = [
+ '021-link-orphan-universe-collections.js',
+ '038-canonicalize-linked-collection-ids.js',
+ '059-split-media-collections.js',
+ '080-brain-tombstone-and-synclog-cleanup.js',
+ '081-brain-daily-log-inbox-sync.js',
+ '190-seed-songbook-songs.js',
+ '200-split-brain-stores-to-per-record.js',
+];
+const LATER = '999-later-example.js';
+const migrationsRoot = new URL('../../scripts/migrations/', import.meta.url);
+
+const writeJson = (path, value) => {
+ mkdirSync(join(path, '..'), { recursive: true });
+ writeFileSync(path, JSON.stringify(value));
+};
+const read = path => readFileSync(path, 'utf8');
+
+describe('restore of a snapshot from before a store split replays earlier migrations in order (#12813)', () => {
+ let base;
+ let migrationsDir;
+ let snapshotRoot;
+ let liveRoot;
+ let directRoot;
+
+ beforeEach(() => {
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime(new Date('2026-03-01T12:00:00.000Z'));
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ base = createTempDataRoot('portos-legacy-stores-');
+ migrationsDir = join(base, 'migrations');
+ mkdirSync(migrationsDir);
+ for (const file of REAL_MIGRATIONS) {
+ writeFileSync(join(migrationsDir, file), `export { default } from ${JSON.stringify(new URL(file, migrationsRoot).href)};`);
+ }
+ writeFileSync(join(migrationsDir, LATER), `
+import { writeFileSync } from 'fs';
+import { join } from 'path';
+export default { up({ rootDir }) { writeFileSync(join(rootDir, 'data', 'later-marker.txt'), 'ran'); } };
+`);
+ snapshotRoot = join(base, 'snapshot');
+ liveRoot = join(base, 'live');
+ directRoot = join(base, 'direct');
+ state.liveData = join(liveRoot, 'data');
+
+ // A current, fully migrated install with records of its own.
+ const live = state.liveData;
+ for (const type of ['inbox', 'journals', 'people']) {
+ writeJson(join(live, 'brain', type, 'index.json'), { schemaVersion: 1, type, config: {} });
+ }
+ writeJson(join(live, 'brain', 'inbox', 'inb-live-1', 'index.json'), { text: 'Live capture' });
+ writeJson(join(live, 'media-collections', 'index.json'), { schemaVersion: 1, type: 'mediaCollections', config: {} });
+ writeJson(join(live, 'migrations.applied.json'), [...REAL_MIGRATIONS, LATER]);
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ rmSync(base, { recursive: true, force: true });
+ });
+
+ /** Restore `snapshotRoot` into the live install the way a full restore does. */
+ async function restore() {
+ const ledger = JSON.parse(read(join(snapshotRoot, 'data', 'migrations.applied.json')));
+ const legacy = await snapshotLegacySplitStores(join(snapshotRoot, 'data'), undefined, { ledger, migrationsDir });
+ const preview = describeLegacySplitStores(legacy);
+ cpSync(join(snapshotRoot, 'data'), state.liveData, { recursive: true }); // rsync is additive
+ const result = await convertRestoredLegacySplitStores(legacy, { migrationsDir });
+ return { preview, result };
+ }
+
+ async function upgradeDirectly() {
+ cpSync(snapshotRoot, directRoot, { recursive: true });
+ await runMigrations({ rootDir: directRoot, migrationsDir });
+ }
+
+ it('builds and splits the inbox and normalizes journals exactly as an upgrade of the same root', async () => {
+ const data = join(snapshotRoot, 'data');
+ writeJson(join(data, 'migrations.applied.json'), REAL_MIGRATIONS.slice(0, 3));
+ mkdirSync(join(data, 'brain'), { recursive: true });
+ writeFileSync(join(data, 'brain', 'inbox_log.jsonl'), `${JSON.stringify({ id: 'inb-example-1', text: 'Example capture', capturedAt: '2025-03-01T09:00:00.000Z' })}\n`);
+ writeJson(join(data, 'brain', 'journals.json'), { records: {
+ '2025-03-01': { id: 'jrnl-example-uuid', date: '2025-03-01', content: 'Example day', obsidianPath: 'Daily/2025-03-01.md' },
+ } });
+
+ const { preview, result } = await restore();
+
+ expect(preview.migrations).toEqual({
+ status: 'pending',
+ through: '200-split-brain-stores-to-per-record.js',
+ names: REAL_MIGRATIONS.slice(3),
+ });
+ expect(result.migrations).toEqual({ ...preview.migrations, status: 'applied' });
+ expect(result.stores.map(({ store, status }) => [store, status])).toEqual([
+ ['brain/people', 'converted'], ['brain/journals', 'converted'], ['brain/inbox', 'converted'],
+ ]);
+ // The replay stops at the split; the next boot runs what follows.
+ await expect(listPendingMigrations({ rootDir: liveRoot, migrationsDir })).resolves.toEqual([LATER]);
+ expect(existsSync(join(state.liveData, 'later-marker.txt'))).toBe(false);
+ await runMigrations({ rootDir: liveRoot, migrationsDir });
+ await upgradeDirectly();
+
+ for (const record of ['brain/inbox/inb-example-1/index.json', 'brain/journals/2025-03-01/index.json']) {
+ expect(read(join(state.liveData, record))).toBe(read(join(directRoot, 'data', record)));
+ }
+ const journal = JSON.parse(read(join(state.liveData, 'brain/journals/2025-03-01/index.json')));
+ expect(journal).not.toHaveProperty('id');
+ expect(journal).toMatchObject({ createdAt: expect.any(String), updatedAt: expect.any(String) });
+ expect(existsSync(join(state.liveData, 'brain', 'inbox', 'inb-live-1'))).toBe(false);
+ const parkedInbox = result.stores.find(store => store.store === 'brain/inbox').parkedAs;
+ expect(JSON.parse(read(join(state.liveData, parkedInbox, 'inb-live-1', 'index.json')))).toEqual({ text: 'Live capture' });
+ });
+
+ it('runs 021 and 038 before splitting a pre-059 media-collections store', async () => {
+ const data = join(snapshotRoot, 'data');
+ writeJson(join(data, 'migrations.applied.json'), []);
+ writeJson(join(data, 'universe-builder.json'), { universes: [{ id: 'uni-example', name: 'Example' }] });
+ writeJson(join(data, 'media-collections.json'), { collections: [{ id: 'col-example', name: 'universe: example', items: [] }] });
+
+ const { result } = await restore();
+
+ expect(result.migrations).toMatchObject({ status: 'applied', through: '059-split-media-collections.js', names: REAL_MIGRATIONS.slice(0, 3) });
+ await runMigrations({ rootDir: liveRoot, migrationsDir });
+ await upgradeDirectly();
+
+ // 021 links the collection to its universe; 038 then re-keys it to the canonical id.
+ const record = 'media-collections/uc-uni-example/index.json';
+ expect(read(join(state.liveData, record))).toBe(read(join(directRoot, 'data', record)));
+ expect(JSON.parse(read(join(state.liveData, record)))).toMatchObject({ universeId: 'uni-example', name: 'Universe: Example' });
+ });
+
+ it('converts with the split alone when the restored ledger already records it', async () => {
+ const data = join(snapshotRoot, 'data');
+ writeJson(join(data, 'migrations.applied.json'), REAL_MIGRATIONS);
+ writeJson(join(data, 'brain', 'people.json'), { records: { 'p-example': { name: 'Example Person' } } });
+
+ const { preview, result } = await restore();
+
+ expect(preview.migrations).toBeNull();
+ expect(result).toMatchObject({ migrations: null, stores: [{ store: 'brain/people', status: 'converted' }] });
+ expect(JSON.parse(read(join(state.liveData, 'brain', 'people', 'p-example', 'index.json')))).toEqual({ name: 'Example Person' });
+ // Stores the snapshot carries no monolith for are left alone without a replay.
+ expect(existsSync(join(state.liveData, 'brain', 'inbox', 'inb-live-1'))).toBe(true);
+ });
+
+ it('keeps a folder restore to the snapshot monoliths and never replays', async () => {
+ const data = join(snapshotRoot, 'data');
+ writeJson(join(data, 'brain', 'journals.json'), { records: { '2025-03-01': { date: '2025-03-01' } } });
+ const legacy = await snapshotLegacySplitStores(data, 'brain', { ledger: [], migrationsDir });
+ expect(legacy.migrations).toBeNull();
+ expect(legacy.stores.map(entry => entry.store)).toEqual(['brain/journals']);
+ });
+});