PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Keep the WAL segment that carries a FULL's start_lsn so restores after cleanup are not refused with a wal gap
Rostislav Duginffe353e4fa62
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.goindex 8b11e5f9..003ad500 100644--- a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go@@ -20,6 +20,11 @@ import ("databasus-backend/internal/util/logger")+const (+ firstRecordOffset = physical_testing.FirstRecordOffset+ fullLSNSpan = physical_testing.FullLSNSpan+)+// Test_RunStartupSlotCleanup_DropsOrphanSlot is the end-to-end proof of the// "Databasus crashed mid-backup" path: the source keeps the per-backup slot and// the in-flight claim survives the crash. In single-instance mode the backup that@@ -261,6 +266,47 @@ func Test_CleanOrphanWalForDatabase_WhenWalCoveredByChain_KeepsIt(t *testing.T)assert.True(t, walExists(t, covered.ID), "chain-covered WAL must never be caught by the orphan pass")}+// A FULL taken right after a segment switch reports a start_lsn at+// firstRecordOffset. The segment's own bounds come from its filename and are+// file-aligned, so it starts below the FULL and must still be kept: it carries the+// FULL's own start and stop positions.+func Test_CleanOrphanWalForDatabase_WhenFullStartsMidSegment_KeepsBoundarySegment(t *testing.T) {+ prereqs := seedBackupPrereqs(t)++ physical_testing.CreateTestFullBackup(t, physical_testing.NewTestCompletedFullBackup(+ prereqs.DB.ID, prereqs.Storage.ID, 1,+ testLSN(1)+firstRecordOffset, testLSN(1)+firstRecordOffset+fullLSNSpan))++ boundarySegment := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(+ prereqs.DB.ID, prereqs.Storage.ID, 1, "000000010000000000000001", testLSN(1), testLSN(2)))++ cleaner := CreateTestPhysicalCleaner()+ cleaner.cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), prereqs.DB.ID)++ assert.True(t, walExists(t, boundarySegment.ID),+ "the segment holding the FULL's start_lsn and stop_lsn must survive the orphan sweep")+}++// The counterpart of the boundary case: a segment that ends at or before the+// earliest COMPLETED FULL's start_lsn holds nothing any retained backup replays,+// so the sweep must still reclaim it.+func Test_CleanOrphanWalForDatabase_WhenWalEndsBeforeEarliestFull_DeletesOrphan(t *testing.T) {+ prereqs := seedBackupPrereqs(t)++ physical_testing.CreateTestFullBackup(t, physical_testing.NewTestCompletedFullBackup(+ prereqs.DB.ID, prereqs.Storage.ID, 1,+ testLSN(4)+firstRecordOffset, testLSN(4)+firstRecordOffset+fullLSNSpan))++ staleSegment := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(+ prereqs.DB.ID, prereqs.Storage.ID, 1, "000000010000000000000001", testLSN(1), testLSN(2)))++ cleaner := CreateTestPhysicalCleaner()+ cleaner.cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), prereqs.DB.ID)++ assert.False(t, walExists(t, staleSegment.ID),+ "WAL that ends below the earliest retained FULL's start is still reclaimable")+}+func Test_ReapAbandonedWalClaims_WhenClaimOlderThanGrace_DeletesIt(t *testing.T) {prereqs := seedBackupPrereqs(t)diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.goindex 5e24e436..845c10a7 100644--- a/backend/internal/features/backups/backups/backuping/physical/testing.go+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go@@ -211,3 +211,15 @@ func StartPhysicalWalStreamSupervisorForTest(t *testing.T) context.CancelFunc {return nil}++// RunOrphanWalCleanupForTest sweeps one database, not every enabled one: the test+// suite shares a metadata database across parallel packages, so a global pass+// would reclaim WAL belonging to another package's fixtures. Run cannot serve+// either — it is an infinite ticker that panics when called a second time in a+// process, and waiting out its tick would tie the assertion to timing rather than+// to sequence.+func RunOrphanWalCleanupForTest(t *testing.T, databaseID uuid.UUID) {+ t.Helper()++ CreateTestPhysicalCleaner().cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), databaseID)+}diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go b/backend/internal/features/backups/backups/core/physical/chain_view/service_test.goindex 4eb3b79a..2d44e305 100644--- a/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go+++ b/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go@@ -26,6 +26,11 @@ import (const segmentBytes = 16 * 1024 * 1024+const (+ firstRecordOffset = physical_testing.FirstRecordOffset+ fullLSNSpan = physical_testing.FullLSNSpan+)+type chainViewTestPrereqs struct {user *users_dto.SignInResponseDTOworkspace *workspaces_models.Workspace@@ -373,6 +378,37 @@ func Test_FindWalOrphansByDatabase_WhenWalOutsideAllChainSpans_ReturnsOrphan(t *assert.Equal(t, orphanSeg.ID, orphans[0].WalSegment.ID)}+// A FULL whose start_lsn lands exactly on a segment boundary is the one alignment+// where comparing it against a segment's start_lsn happens to be right. Real FULLs+// start at firstRecordOffset into the segment, so the segment that physically+// holds the FULL's start and stop positions begins below it and is not an orphan.+func Test_FindWalOrphansByDatabase_WhenFullStartsMidSegment_BoundarySegmentIsNotOrphan(t *testing.T) {+ t.Parallel()++ prereqs := createChainViewTestPrereqs(t)+ databaseID := prereqs.database.ID++ physical_testing.CreateTestFullBackup(t,+ physical_testing.NewTestCompletedFullBackup(+ databaseID, prereqs.storage.ID, 1,+ walmath.LSN(4*segmentBytes+firstRecordOffset),+ walmath.LSN(4*segmentBytes+firstRecordOffset+fullLSNSpan),+ ))++ physical_testing.CreateTestWalSegment(t,+ physical_testing.NewTestWalSegment(+ databaseID, prereqs.storage.ID, 1, "000000010000000000000004",+ walmath.LSN(4*segmentBytes), walmath.LSN(5*segmentBytes),+ ))++ service := chain_view.GetChainViewService()++ orphans, err := service.FindWalOrphansByDatabase(databaseID)+ require.NoError(t, err)+ assert.Empty(t, orphans,+ "the segment carrying the FULL's own start_lsn is referenced by that FULL")+}+func Test_FindWalGapsInChain_WhenMultipleGapsExist_ReturnsAllGaps(t *testing.T) {t.Parallel()diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.goindex 38dda3e0..e583b85f 100644--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go@@ -80,6 +80,12 @@ func (r *PhysicalWalSegmentRepository) FindByChainSpan(// Anti-join not covered by idx_physical_wal_segments_database_id_received_at;// expect a seq scan on physical_full_backups when invoked in bulk.+//+// The comparison anchors on the segment's end_lsn because a FULL's start_lsn is+// a real recovery position a short way inside the segment that holds it, while a+// segment's own bounds are derived from its filename and are file-aligned.+// Comparing against start_lsn would classify that boundary segment as an orphan+// and delete the WAL the FULL itself starts and stops in.func (r *PhysicalWalSegmentRepository) FindOrphans(databaseID uuid.UUID,) ([]*physical_models.PhysicalWalSegment, error) {@@ -97,7 +103,7 @@ func (r *PhysicalWalSegmentRepository) FindOrphans(WHERE f.database_id = w.database_idAND f.timeline_id = w.timeline_idAND f.start_lsn IS NOT NULL- AND f.start_lsn <= w.start_lsn+ AND f.start_lsn < w.end_lsnAND f.status = ?)ORDER BY w.start_lsn ASCdiff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.goindex 9814a8c5..558ccb40 100644--- a/backend/internal/features/backups/backups/core/physical/service/service.go+++ b/backend/internal/features/backups/backups/core/physical/service/service.go@@ -580,8 +580,11 @@ func (s *PhysicalBackupService) cascadeDelete(return summary, nil}-// deleteWalInSpanBudgeted removes WAL rows in the span oldest-LSN first, batched,-// stopping when the byte budget is reached or the span is drained. Storage+// deleteWalInSpanBudgeted removes only WAL rows lying fully inside the span. The+// upper bound is the segment's end_lsn, not its start_lsn: a span ends at the+// next chain's start_lsn, which sits inside a segment, and that straddling+// segment carries the bytes the next chain replays from. Bounding on start_lsn+// would delete it along with the chain being pruned. Storage// deletes are fail-closed: a transient DeleteFile error aborts the batch (the// rows survive and retry) rather than orphaning an object. lockRows takes a// row-level FOR UPDATE on each batch (orphan path, no anchor FULL to serialize@@ -601,7 +604,7 @@ func (s *PhysicalBackupService) deleteWalInSpanBudgeted(batchQuery := tx.Where(- "database_id = ? AND timeline_id = ? AND start_lsn >= ?::pg_lsn AND start_lsn < ?::pg_lsn",+ "database_id = ? AND timeline_id = ? AND start_lsn >= ?::pg_lsn AND end_lsn <= ?::pg_lsn",databaseID, timelineID, span.Start.String(), span.End.String(),).Order("start_lsn ASC").diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.goindex f5707410..315ba9a1 100644--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go@@ -28,6 +28,11 @@ import (const segmentMB = 16+const (+ firstRecordOffset = physical_testing.FirstRecordOffset+ fullLSNSpan = physical_testing.FullLSNSpan+)+type serviceTestPrereqs struct {storage *storages.Storagedatabase *databases.Database@@ -172,6 +177,43 @@ func Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects(t *testing.T)assert.NotNil(t, survivingSuccessor, "successor chain on TL2 must be untouched")}+// Pruning a chain deletes the WAL it anchored, up to where the next chain begins.+// The next FULL starts at firstRecordOffset inside a segment, so that segment+// begins below the boundary and holds the bytes the surviving chain replays from.+// It belongs to the successor, not to the chain being pruned.+func Test_DeleteFull_WhenSuccessorStartsMidSegment_KeepsSuccessorBoundarySegment(t *testing.T) {+ prereqs := createServiceTestPrereqs(t)+ databaseID := prereqs.database.ID+ storageID := prereqs.storage.ID+ service := physical_service.GetPhysicalBackupService()++ prunedFull := physical_testing.CreateTestFullBackup(t,+ physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(0), lsn(1)))+ physical_testing.CreateTestFullBackup(t,+ physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1,+ lsn(4)+firstRecordOffset, lsn(4)+firstRecordOffset+fullLSNSpan))++ anchoredSegment := physical_testing.CreateTestWalSegment(t,+ physical_testing.NewTestWalSegment(databaseID, storageID, 1, "000000010000000000000001", lsn(1), lsn(2)))+ successorBoundarySegment := physical_testing.CreateTestWalSegment(t,+ physical_testing.NewTestWalSegment(databaseID, storageID, 1, "000000010000000000000004", lsn(4), lsn(5)))++ saveObject(t, prereqs.storage, *anchoredSegment.FileName)+ saveObject(t, prereqs.storage, *successorBoundarySegment.FileName)++ _, err := service.DeleteFull(t.Context(), prunedFull.ID, 1_000_000)+ require.NoError(t, err)++ reloadedAnchoredSegment, _ := physical_repositories.GetWalSegmentRepository().FindByID(anchoredSegment.ID)+ assert.Nil(t, reloadedAnchoredSegment, "WAL the pruned chain anchored goes with it")++ reloadedBoundarySegment, _ := physical_repositories.GetWalSegmentRepository().FindByID(successorBoundarySegment.ID)+ assert.NotNil(t, reloadedBoundarySegment,+ "the segment carrying the surviving FULL's start_lsn must outlive the pruned chain")+ assert.True(t, objectExists(t, prereqs.storage, *successorBoundarySegment.FileName),+ "its stored object must survive too")+}+func Test_GetDependentsSummary_ReturnsCountsWithoutDeleting(t *testing.T) {prereqs := createServiceTestPrereqs(t)databaseID := prereqs.database.IDdiff --git a/backend/internal/features/backups/backups/core/physical/testing/testing.go b/backend/internal/features/backups/backups/core/physical/testing/testing.goindex 02094a4f..1c504ccf 100644--- a/backend/internal/features/backups/backups/core/physical/testing/testing.go+++ b/backend/internal/features/backups/backups/core/physical/testing/testing.go@@ -13,6 +13,14 @@ import ("databasus-backend/internal/util/walmath")+// Past the segment's long page header. A FULL's start_lsn carries this offset,+// while a segment's own bounds are file-aligned.+const FirstRecordOffset = 0x28++// Wide enough to keep a test FULL's start_lsn and stop_lsn apart, narrow enough+// that both stay inside the segment holding the start.+const FullLSNSpan = 0x100+func NewTestCompletedFullBackup(databaseID, storageID uuid.UUID,timelineID int,diff --git a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.goindex a9265023..464fb882 100644--- a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go+++ b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go@@ -27,6 +27,10 @@ func Test_PhysicalRestore_WhenWalGapBeforeTarget_TokenRequestReturns422(t *testiphysicaltesting.RunWhenWalGapBeforeTargetTokenRequestReturns422(t, pgVersion, pgImage)}+func Test_PhysicalRestore_WhenOrphanSweepRunsAfterFull_TargetPastFullStaysRestorable(t *testing.T) {+ physicaltesting.RunWhenOrphanSweepRunsAfterFullTargetPastFullStaysRestorable(t, pgVersion, pgImage)+}+func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t *testing.T) {physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)}diff --git a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.goindex 87c2910f..97ed40f7 100644--- a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go+++ b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go@@ -27,6 +27,10 @@ func Test_PhysicalRestore_WhenWalGapBeforeTarget_TokenRequestReturns422(t *testiphysicaltesting.RunWhenWalGapBeforeTargetTokenRequestReturns422(t, pgVersion, pgImage)}+func Test_PhysicalRestore_WhenOrphanSweepRunsAfterFull_TargetPastFullStaysRestorable(t *testing.T) {+ physicaltesting.RunWhenOrphanSweepRunsAfterFullTargetPastFullStaysRestorable(t, pgVersion, pgImage)+}+func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t *testing.T) {physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)}diff --git a/backend/internal/features/tests/physical/postgresql/shared/bodies.go b/backend/internal/features/tests/physical/postgresql/shared/bodies.goindex e14f14d8..2eff353c 100644--- a/backend/internal/features/tests/physical/postgresql/shared/bodies.go+++ b/backend/internal/features/tests/physical/postgresql/shared/bodies.go@@ -12,6 +12,7 @@ import ("github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ backuping_physical "databasus-backend/internal/features/backups/backups/backuping/physical"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"@@ -302,6 +303,61 @@ func RunWhenWalGapBeforeTargetTokenRequestReturns422(t *testing.T, version, imag"the gap must be reported so the user never burns a token on an unreachable target")}+// A FULL's start_lsn sits inside the segment that carries it, so that segment's file-aligned start+// is lower than the FULL's and the retention sweep must still keep it: it holds the bytes replay+// begins from. No incremental is taken on purpose. An incremental moves the replay window past the+// FULL's own boundary segment and hides whether the sweep kept it.+func RunWhenOrphanSweepRunsAfterFullTargetPastFullStaysRestorable(t *testing.T, version, image string) {+ if testing.Short() {+ t.Skip("streams WAL and runs the retention sweep against a real cluster; skipped in -short")+ }++ router, fixture := setupReplicationOnlyFixture(+ t, version, image, postgresql_physical.BackupTypeFullIncrementalAndWalStream)++ sourceConn := openSourceTestDBConn(t, fixture)++ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)+ defer cancel()++ createMarkerTable(t, ctx, sourceConn)+ insertMarker(t, ctx, sourceConn, "before-full", "row-in-base-backup")++ enablePhysicalBackupsViaAPI(t, router, fixture, true)++ // Wait for the slot before writing anything: WAL produced before the supervisor claims the+ // database is never streamed, and that hole at the head of the window would break the restore+ // on its own, hiding whatever the sweep does.+ waitForSlotPresent(t, postgresql_executor.OpenAdminConn(t, fixture),+ fixture.DB.PostgresqlPhysical.ReplicationSlotName, 60*time.Second)++ chain := waitForChainBackups(t, router, fixture, 0, 3*time.Minute)++ insertMarker(t, ctx, sourceConn, "after-full", "row-replayed-up-to-target")+ streamPostFullSegments(t, ctx, router, sourceConn, fixture, chainTipStopLSN(t, chain), 2, 90*time.Second)++ // A margin wider than the whole-second recovery_target_time truncation on either side keeps the+ // target unambiguously inside the streamed range.+ time.Sleep(2 * time.Second)+ targetTime := time.Now().UTC()+ time.Sleep(2 * time.Second)++ insertMarker(t, ctx, sourceConn, "after-target", "row-after-target")++ // Natural WAL rather than a forced switch: the segment covering the target has to fill and+ // archive before the resolver counts it, and a switch would leave it partial.+ _, err := postgresql_executor.GenerateWalActivity(ctx, sourceConn, 64*1024*1024)+ require.NoError(t, err)++ waitForTargetTimeReplayable(t, fixture.DB.ID, targetTime, 90*time.Second)++ backuping_physical.RunOrphanWalCleanupForTest(t, fixture.DB.ID)++ token := requestRestoreTokenViaAPI(t, router, fixture, &targetTime)+ assert.NotEmpty(t, token,+ "a target covered by archived WAL must stay restorable after the orphan sweep has run")+}+// RunWalSlotAppearsWhenBackupingStartsRemovedWhenDatabaseDeleted proves the WAL replication-slot// lifecycle end to end and purely over the API: no slot exists until WAL-stream backups are enabled,// enabling them makes the supervisor create the persistent slot, and deleting the database (DELETEdiff --git a/backend/internal/features/tests/physical/postgresql/shared/helpers.go b/backend/internal/features/tests/physical/postgresql/shared/helpers.goindex b7ec7802..d9833999 100644--- a/backend/internal/features/tests/physical/postgresql/shared/helpers.go+++ b/backend/internal/features/tests/physical/postgresql/shared/helpers.go@@ -779,6 +779,28 @@ func waitForReplayableThroughLSN(t *testing.T, databaseID uuid.UUID, throughLSNthroughLSN.String(), timeout, lastReachable.String())}+// Reaching the target's LSN is not enough on its own: the resolver compares the+// target against the receive time of the last segment in the contiguous run, so a+// run that already extends past the target by LSN can still end at a segment+// archived before it.+func waitForTargetTimeReplayable(t *testing.T, databaseID uuid.UUID, targetTime time.Time, timeout time.Duration) {+ t.Helper()++ deadline := time.Now().UTC().Add(timeout)++ var lastErr error+ for time.Now().UTC().Before(deadline) {+ _, lastErr = chain_view.GetChainViewService().ResolveRestoreSet(databaseID, &targetTime)+ if lastErr == nil {+ return+ }++ time.Sleep(250 * time.Millisecond)+ }++ t.Fatalf("target time %s never became restorable within %s: %v", targetTime, timeout, lastErr)+}+// seedChainAndStreamPastTarget seeds the marker table, builds FULL -> INCR -> INCR over the API, then// streams WAL past a captured target time (with a post-target row that must be dropped). It returns// the PITR target and the phases that must survive recovery to it - shared by the in-container and the