Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: reconcile catalog payloads and universe tags after database restore (#10940)
Adam Eivy0896fdea752b
diff --git a/docs/BACKUP.md b/docs/BACKUP.mdindex cb33431d91..cebd701b2f 100644--- a/docs/BACKUP.md+++ b/docs/BACKUP.md@@ -528,6 +528,7 @@ Replays the snapshot's `portos-db.sql` into the live database via `psql -v ON_ER| `{ status: 'failed', reason: 'restore_error' \| 'timeout', error }` | `psql` replay failed (stderr captured) and was **proven rolled back** from its receipt (below) || `{ status: 'failed', reason: 'restore_commit_unknown', error, recovery }` | `psql` did not report success and the receipt could not settle whether the replay committed — recovery pending || `{ status: 'failed', reason: 'restore_schema_reconciliation', error, recovery }` | The dump committed, but current schema recovery failed; **not rolled back**, recovery pending |+| `{ status: 'failed', reason: 'restore_catalog_reconciliation', error, recovery }` | The dump committed and the schema recovered, but forced catalog payload migration or universe-tag repair failed — recovery pending || `{ status: 'failed', reason: 'restore_sync_resync', error, recovery }` | The dump committed and the schema recovered, but peer sync could not be repaired (below) — recovery pending || `{ status: 'failed', reason: 'restore_recovery_release', error, recovery }` | Repair finished but the journal could not be cleared — recovery pending || `{ status: 'failed', reason: 'restore_recovery_pending', error, recovery }` | An earlier restore is still awaiting recovery; preview and replay are refused |diff --git a/server/services/backup.js b/server/services/backup.jsindex fe4ebe0a00..f017bab272 100644--- a/server/services/backup.js+++ b/server/services/backup.js@@ -1846,7 +1846,7 @@ const DUMP_UNREADABLE = Object.freeze({ status: 'failed', reason: 'dump_unreadab* { status: 'skipped', reason: 'not_configured' } (real restore, PG unreachable)* { status: 'failed', reason: 'manifest_unreadable'|'manifest_mismatch'|'dump_unreadable'|'dump_incomplete'|'restore_compatibility'|'restore_preflight'|'restore_journal'|'backup_snapshot_busy'|'restore_error'|'timeout', error? }* (nothing changed; a failed replay is reported only once proven rolled back)- * { status: 'failed', reason: 'restore_recovery_pending'|'restore_commit_unknown'|'restore_schema_reconciliation'|'restore_sync_resync'|'restore_recovery_release', error, recovery }+ * { status: 'failed', reason: 'restore_recovery_pending'|'restore_commit_unknown'|'restore_schema_reconciliation'|'restore_catalog_reconciliation'|'restore_sync_resync'|'restore_recovery_release', error, recovery }* (a restore awaits recovery: ordinary database work stays fenced until* resumeDatabaseRestore finishes it — see backupRestoreRecovery.js, #9725)* A successful real restore also carries `syncCursorsRewound` (peer count).diff --git a/server/services/backup.test.js b/server/services/backup.test.jsindex a5965b671a..e09ef4e9eb 100644--- a/server/services/backup.test.js+++ b/server/services/backup.test.js@@ -73,6 +73,12 @@ vi.mock('../lib/databaseMaintenanceJournal.js', () => ({vi.mock('../scripts/run-db-migrations.js', () => ({runDbMigrations: vi.fn().mockResolvedValue(0),}));+const catalogRepair = vi.hoisted(() => ({+ migrate: vi.fn().mockResolvedValue({ skipped: false }),+ tags: vi.fn().mockResolvedValue({ skipped: false }),+}));+vi.mock('../scripts/migrateCatalogPayload.js', () => ({ migrateCatalogPayload: catalogRepair.migrate }));+vi.mock('../scripts/repairUniverseTags.js', () => ({ repairUniverseTags: catalogRepair.tags }));const reconcileMediaAssets = vi.hoisted(() => vi.fn().mockResolvedValue({ ok: true }));vi.mock('./mediaAssetIndex/db.js', () => ({ reconcileMediaAssets }));const executionRestore = vi.hoisted(() => ({ prepare: vi.fn(), finish: vi.fn() }));@@ -1601,6 +1607,14 @@ describe('restorePostgres', () => {expect(readRecoveryJournal()).toBeNull();});+ it('forces catalog payload migration and universe-tag repair past stale disk markers', async () => {+ catalogRepair.migrate.mockClear();+ catalogRepair.tags.mockClear();+ expect(await runRestore()).toMatchObject({ status: 'ok' });+ expect(catalogRepair.migrate).toHaveBeenCalledWith({ force: true });+ expect(catalogRepair.tags).toHaveBeenCalledWith({ force: true });+ });+it('retains the generic fence when rollback execution reconciliation fails', async () => {executionRestore.finish.mockRejectedValueOnce(new Error('fixture rollback evidence unavailable'));const result = await runRestore({ exitCode: 1 });@@ -1613,6 +1627,8 @@ describe('restorePostgres', () => {it.each([['schema upgrade', () => ensureSchema.mockRejectedValueOnce(new Error('ddl failed')), 'restore_schema_reconciliation'],['ordered migration', () => runDbMigrations.mockRejectedValueOnce(new Error('migration failed')), 'restore_schema_reconciliation'],+ ['catalog payload migration', () => catalogRepair.migrate.mockRejectedValueOnce(new Error('payload walk failed')), 'restore_catalog_reconciliation'],+ ['universe tag repair', () => catalogRepair.tags.mockRejectedValueOnce(new Error('tag walk failed')), 'restore_catalog_reconciliation'],['media rebuild', () => reconcileMediaAssets.mockRejectedValueOnce(new Error('media read failed')), 'restore_media_reconciliation'],['cursor write', () => rewindPostgresSyncCursors.mockRejectedValueOnce(new Error('disk full')), 'restore_sync_resync'],])('a %s fault after commit stays fenced, then recovery repairs from the original floors without replaying', async (_case, inject, reason) => {diff --git a/server/services/backupRestoreRecovery.js b/server/services/backupRestoreRecovery.jsindex 6af1cd27d0..179721da52 100644--- a/server/services/backupRestoreRecovery.js+++ b/server/services/backupRestoreRecovery.js@@ -33,6 +33,7 @@ const MESSAGES = {restore_commit_unknown: 'The restore could not confirm whether the database dump was committed. Ordinary database work stays paused and the dump will NOT be replayed; retry recovery from Settings > Backup once PostgreSQL is reachable.',restore_schema_reconciliation: 'The database dump was applied, but schema recovery is incomplete. It was not rolled back and will not be replayed. Ordinary database work stays paused: retry recovery from Settings > Backup, or restart PortOS to retry automatically. If it keeps failing, check the server logs.',restore_execution_reconciliation: 'Execution request consumption and unresolved ownership are awaiting reconciliation. Database work stays paused; the dump will not be replayed. Recovery retries automatically at startup and can also be retried from Settings > Backup.',+ restore_catalog_reconciliation: 'The database dump was applied, but catalog payload or universe-tag repair is incomplete. It will not be replayed. Ordinary database work stays paused: retry recovery from Settings > Backup, or restart PortOS to retry automatically.',restore_media_reconciliation: 'The database dump was applied, but the media index could not be rebuilt from local files. It will not be replayed. Ordinary database work stays paused: retry recovery from Settings > Backup, or restart PortOS.',restore_sync_resync: 'The database dump was applied, but peer sync could not be reset yet. It will not be replayed. Ordinary database work stays paused: retry recovery from Settings > Backup, or restart PortOS to retry automatically.',restore_recovery_release: 'The database dump was applied and repaired, but the recovery journal could not be cleared. Ordinary database work stays paused: retry recovery from Settings > Backup.',@@ -171,6 +172,19 @@ export async function repairCommittedRestore(record) {console.error(`❌ DB restore ${record.id}: schema reconciliation failed: ${reconciliationError.message}`);return pendingRecoveryResult('restore_schema_reconciliation', record);}+ // The on-disk markers describe the pre-restore database, so an older snapshot+ // would otherwise look already migrated on every later boot. Force both walks+ // (each also re-stamps its marker) so catalog rows match the restored state.+ const catalogError = await (async () => {+ const { migrateCatalogPayload } = await import('../scripts/migrateCatalogPayload.js');+ const { repairUniverseTags } = await import('../scripts/repairUniverseTags.js');+ await migrateCatalogPayload({ force: true });+ await repairUniverseTags({ force: true });+ })().then(() => null, (err) => err);+ if (catalogError) {+ console.error(`❌ DB restore ${record.id}: catalog reconciliation failed: ${catalogError.message}`);+ return pendingRecoveryResult('restore_catalog_reconciliation', record);+ }// Old snapshots may contain stale mirror rows; new snapshots omit them.// Either way disk is authoritative, including for a DB-only restore.const mediaError = await (async () => {