Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: reconcile catalog payloads and universe tags after database restore (#10940)

Adam Eivy0896fdea752b
diff --git a/docs/BACKUP.md b/docs/BACKUP.md
index cb33431d91..cebd701b2f 100644
--- a/docs/BACKUP.md
+++ b/docs/BACKUP.md
@@ -528,6 +528,7 @@ Replays the snapshot's `portos-db.sql` into the live database via `psql -v ON_ER
| `{ status: 'failed', reason: 'restore_error' \| 'timeout', error }` | `psql` replay failed (stderr captured) and was **proven rolled back** from its receipt (below) |
| `{ status: 'failed', reason: 'restore_commit_unknown', error, recovery }` | `psql` did not report success and the receipt could not settle whether the replay committed — recovery pending |
| `{ status: 'failed', reason: 'restore_schema_reconciliation', error, recovery }` | The dump committed, but current schema recovery failed; **not rolled back**, recovery pending |
+| `{ status: 'failed', reason: 'restore_catalog_reconciliation', error, recovery }` | The dump committed and the schema recovered, but forced catalog payload migration or universe-tag repair failed — recovery pending |
| `{ status: 'failed', reason: 'restore_sync_resync', error, recovery }` | The dump committed and the schema recovered, but peer sync could not be repaired (below) — recovery pending |
| `{ status: 'failed', reason: 'restore_recovery_release', error, recovery }` | Repair finished but the journal could not be cleared — recovery pending |
| `{ status: 'failed', reason: 'restore_recovery_pending', error, recovery }` | An earlier restore is still awaiting recovery; preview and replay are refused |
diff --git a/server/services/backup.js b/server/services/backup.js
index fe4ebe0a00..f017bab272 100644
--- a/server/services/backup.js
+++ b/server/services/backup.js
@@ -1846,7 +1846,7 @@ const DUMP_UNREADABLE = Object.freeze({ status: 'failed', reason: 'dump_unreadab
* { status: 'skipped', reason: 'not_configured' } (real restore, PG unreachable)
* { status: 'failed', reason: 'manifest_unreadable'|'manifest_mismatch'|'dump_unreadable'|'dump_incomplete'|'restore_compatibility'|'restore_preflight'|'restore_journal'|'backup_snapshot_busy'|'restore_error'|'timeout', error? }
* (nothing changed; a failed replay is reported only once proven rolled back)
- * { status: 'failed', reason: 'restore_recovery_pending'|'restore_commit_unknown'|'restore_schema_reconciliation'|'restore_sync_resync'|'restore_recovery_release', error, recovery }
+ * { status: 'failed', reason: 'restore_recovery_pending'|'restore_commit_unknown'|'restore_schema_reconciliation'|'restore_catalog_reconciliation'|'restore_sync_resync'|'restore_recovery_release', error, recovery }
* (a restore awaits recovery: ordinary database work stays fenced until
* resumeDatabaseRestore finishes it — see backupRestoreRecovery.js, #9725)
* A successful real restore also carries `syncCursorsRewound` (peer count).
diff --git a/server/services/backup.test.js b/server/services/backup.test.js
index a5965b671a..e09ef4e9eb 100644
--- a/server/services/backup.test.js
+++ b/server/services/backup.test.js
@@ -73,6 +73,12 @@ vi.mock('../lib/databaseMaintenanceJournal.js', () => ({
vi.mock('../scripts/run-db-migrations.js', () => ({
runDbMigrations: vi.fn().mockResolvedValue(0),
}));
+const catalogRepair = vi.hoisted(() => ({
+ migrate: vi.fn().mockResolvedValue({ skipped: false }),
+ tags: vi.fn().mockResolvedValue({ skipped: false }),
+}));
+vi.mock('../scripts/migrateCatalogPayload.js', () => ({ migrateCatalogPayload: catalogRepair.migrate }));
+vi.mock('../scripts/repairUniverseTags.js', () => ({ repairUniverseTags: catalogRepair.tags }));
const reconcileMediaAssets = vi.hoisted(() => vi.fn().mockResolvedValue({ ok: true }));
vi.mock('./mediaAssetIndex/db.js', () => ({ reconcileMediaAssets }));
const executionRestore = vi.hoisted(() => ({ prepare: vi.fn(), finish: vi.fn() }));
@@ -1601,6 +1607,14 @@ describe('restorePostgres', () => {
expect(readRecoveryJournal()).toBeNull();
});
+ it('forces catalog payload migration and universe-tag repair past stale disk markers', async () => {
+ catalogRepair.migrate.mockClear();
+ catalogRepair.tags.mockClear();
+ expect(await runRestore()).toMatchObject({ status: 'ok' });
+ expect(catalogRepair.migrate).toHaveBeenCalledWith({ force: true });
+ expect(catalogRepair.tags).toHaveBeenCalledWith({ force: true });
+ });
+
it('retains the generic fence when rollback execution reconciliation fails', async () => {
executionRestore.finish.mockRejectedValueOnce(new Error('fixture rollback evidence unavailable'));
const result = await runRestore({ exitCode: 1 });
@@ -1613,6 +1627,8 @@ describe('restorePostgres', () => {
it.each([
['schema upgrade', () => ensureSchema.mockRejectedValueOnce(new Error('ddl failed')), 'restore_schema_reconciliation'],
['ordered migration', () => runDbMigrations.mockRejectedValueOnce(new Error('migration failed')), 'restore_schema_reconciliation'],
+ ['catalog payload migration', () => catalogRepair.migrate.mockRejectedValueOnce(new Error('payload walk failed')), 'restore_catalog_reconciliation'],
+ ['universe tag repair', () => catalogRepair.tags.mockRejectedValueOnce(new Error('tag walk failed')), 'restore_catalog_reconciliation'],
['media rebuild', () => reconcileMediaAssets.mockRejectedValueOnce(new Error('media read failed')), 'restore_media_reconciliation'],
['cursor write', () => rewindPostgresSyncCursors.mockRejectedValueOnce(new Error('disk full')), 'restore_sync_resync'],
])('a %s fault after commit stays fenced, then recovery repairs from the original floors without replaying', async (_case, inject, reason) => {
diff --git a/server/services/backupRestoreRecovery.js b/server/services/backupRestoreRecovery.js
index 6af1cd27d0..179721da52 100644
--- a/server/services/backupRestoreRecovery.js
+++ b/server/services/backupRestoreRecovery.js
@@ -33,6 +33,7 @@ const MESSAGES = {
restore_commit_unknown: 'The restore could not confirm whether the database dump was committed. Ordinary database work stays paused and the dump will NOT be replayed; retry recovery from Settings > Backup once PostgreSQL is reachable.',
restore_schema_reconciliation: 'The database dump was applied, but schema recovery is incomplete. It was not rolled back and will not be replayed. Ordinary database work stays paused: retry recovery from Settings > Backup, or restart PortOS to retry automatically. If it keeps failing, check the server logs.',
restore_execution_reconciliation: 'Execution request consumption and unresolved ownership are awaiting reconciliation. Database work stays paused; the dump will not be replayed. Recovery retries automatically at startup and can also be retried from Settings > Backup.',
+ restore_catalog_reconciliation: 'The database dump was applied, but catalog payload or universe-tag repair is incomplete. It will not be replayed. Ordinary database work stays paused: retry recovery from Settings > Backup, or restart PortOS to retry automatically.',
restore_media_reconciliation: 'The database dump was applied, but the media index could not be rebuilt from local files. It will not be replayed. Ordinary database work stays paused: retry recovery from Settings > Backup, or restart PortOS.',
restore_sync_resync: 'The database dump was applied, but peer sync could not be reset yet. It will not be replayed. Ordinary database work stays paused: retry recovery from Settings > Backup, or restart PortOS to retry automatically.',
restore_recovery_release: 'The database dump was applied and repaired, but the recovery journal could not be cleared. Ordinary database work stays paused: retry recovery from Settings > Backup.',
@@ -171,6 +172,19 @@ export async function repairCommittedRestore(record) {
console.error(`❌ DB restore ${record.id}: schema reconciliation failed: ${reconciliationError.message}`);
return pendingRecoveryResult('restore_schema_reconciliation', record);
}
+ // The on-disk markers describe the pre-restore database, so an older snapshot
+ // would otherwise look already migrated on every later boot. Force both walks
+ // (each also re-stamps its marker) so catalog rows match the restored state.
+ const catalogError = await (async () => {
+ const { migrateCatalogPayload } = await import('../scripts/migrateCatalogPayload.js');
+ const { repairUniverseTags } = await import('../scripts/repairUniverseTags.js');
+ await migrateCatalogPayload({ force: true });
+ await repairUniverseTags({ force: true });
+ })().then(() => null, (err) => err);
+ if (catalogError) {
+ console.error(`❌ DB restore ${record.id}: catalog reconciliation failed: ${catalogError.message}`);
+ return pendingRecoveryResult('restore_catalog_reconciliation', record);
+ }
// Old snapshots may contain stale mirror rows; new snapshots omit them.
// Either way disk is authoritative, including for a DB-only restore.
const mediaError = await (async () => {