Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Keep the WAL segment that carries a FULL's start_lsn so restores after cleanup are not refused with a wal gap

Rostislav Duginffe353e4fa62
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
index 8b11e5f9..003ad500 100644
--- a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
@@ -20,6 +20,11 @@ import (
"databasus-backend/internal/util/logger"
)
+const (
+ firstRecordOffset = physical_testing.FirstRecordOffset
+ fullLSNSpan = physical_testing.FullLSNSpan
+)
+
// Test_RunStartupSlotCleanup_DropsOrphanSlot is the end-to-end proof of the
// "Databasus crashed mid-backup" path: the source keeps the per-backup slot and
// the in-flight claim survives the crash. In single-instance mode the backup that
@@ -261,6 +266,47 @@ func Test_CleanOrphanWalForDatabase_WhenWalCoveredByChain_KeepsIt(t *testing.T)
assert.True(t, walExists(t, covered.ID), "chain-covered WAL must never be caught by the orphan pass")
}
+// A FULL taken right after a segment switch reports a start_lsn at
+// firstRecordOffset. The segment's own bounds come from its filename and are
+// file-aligned, so it starts below the FULL and must still be kept: it carries the
+// FULL's own start and stop positions.
+func Test_CleanOrphanWalForDatabase_WhenFullStartsMidSegment_KeepsBoundarySegment(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+
+ physical_testing.CreateTestFullBackup(t, physical_testing.NewTestCompletedFullBackup(
+ prereqs.DB.ID, prereqs.Storage.ID, 1,
+ testLSN(1)+firstRecordOffset, testLSN(1)+firstRecordOffset+fullLSNSpan))
+
+ boundarySegment := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(
+ prereqs.DB.ID, prereqs.Storage.ID, 1, "000000010000000000000001", testLSN(1), testLSN(2)))
+
+ cleaner := CreateTestPhysicalCleaner()
+ cleaner.cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), prereqs.DB.ID)
+
+ assert.True(t, walExists(t, boundarySegment.ID),
+ "the segment holding the FULL's start_lsn and stop_lsn must survive the orphan sweep")
+}
+
+// The counterpart of the boundary case: a segment that ends at or before the
+// earliest COMPLETED FULL's start_lsn holds nothing any retained backup replays,
+// so the sweep must still reclaim it.
+func Test_CleanOrphanWalForDatabase_WhenWalEndsBeforeEarliestFull_DeletesOrphan(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+
+ physical_testing.CreateTestFullBackup(t, physical_testing.NewTestCompletedFullBackup(
+ prereqs.DB.ID, prereqs.Storage.ID, 1,
+ testLSN(4)+firstRecordOffset, testLSN(4)+firstRecordOffset+fullLSNSpan))
+
+ staleSegment := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(
+ prereqs.DB.ID, prereqs.Storage.ID, 1, "000000010000000000000001", testLSN(1), testLSN(2)))
+
+ cleaner := CreateTestPhysicalCleaner()
+ cleaner.cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), prereqs.DB.ID)
+
+ assert.False(t, walExists(t, staleSegment.ID),
+ "WAL that ends below the earliest retained FULL's start is still reclaimable")
+}
+
func Test_ReapAbandonedWalClaims_WhenClaimOlderThanGrace_DeletesIt(t *testing.T) {
prereqs := seedBackupPrereqs(t)
diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.go
index 5e24e436..845c10a7 100644
--- a/backend/internal/features/backups/backups/backuping/physical/testing.go
+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go
@@ -211,3 +211,15 @@ func StartPhysicalWalStreamSupervisorForTest(t *testing.T) context.CancelFunc {
return nil
}
+
+// RunOrphanWalCleanupForTest sweeps one database, not every enabled one: the test
+// suite shares a metadata database across parallel packages, so a global pass
+// would reclaim WAL belonging to another package's fixtures. Run cannot serve
+// either — it is an infinite ticker that panics when called a second time in a
+// process, and waiting out its tick would tie the assertion to timing rather than
+// to sequence.
+func RunOrphanWalCleanupForTest(t *testing.T, databaseID uuid.UUID) {
+ t.Helper()
+
+ CreateTestPhysicalCleaner().cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), databaseID)
+}
diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go b/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go
index 4eb3b79a..2d44e305 100644
--- a/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go
+++ b/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go
@@ -26,6 +26,11 @@ import (
const segmentBytes = 16 * 1024 * 1024
+const (
+ firstRecordOffset = physical_testing.FirstRecordOffset
+ fullLSNSpan = physical_testing.FullLSNSpan
+)
+
type chainViewTestPrereqs struct {
user *users_dto.SignInResponseDTO
workspace *workspaces_models.Workspace
@@ -373,6 +378,37 @@ func Test_FindWalOrphansByDatabase_WhenWalOutsideAllChainSpans_ReturnsOrphan(t *
assert.Equal(t, orphanSeg.ID, orphans[0].WalSegment.ID)
}
+// A FULL whose start_lsn lands exactly on a segment boundary is the one alignment
+// where comparing it against a segment's start_lsn happens to be right. Real FULLs
+// start at firstRecordOffset into the segment, so the segment that physically
+// holds the FULL's start and stop positions begins below it and is not an orphan.
+func Test_FindWalOrphansByDatabase_WhenFullStartsMidSegment_BoundarySegmentIsNotOrphan(t *testing.T) {
+ t.Parallel()
+
+ prereqs := createChainViewTestPrereqs(t)
+ databaseID := prereqs.database.ID
+
+ physical_testing.CreateTestFullBackup(t,
+ physical_testing.NewTestCompletedFullBackup(
+ databaseID, prereqs.storage.ID, 1,
+ walmath.LSN(4*segmentBytes+firstRecordOffset),
+ walmath.LSN(4*segmentBytes+firstRecordOffset+fullLSNSpan),
+ ))
+
+ physical_testing.CreateTestWalSegment(t,
+ physical_testing.NewTestWalSegment(
+ databaseID, prereqs.storage.ID, 1, "000000010000000000000004",
+ walmath.LSN(4*segmentBytes), walmath.LSN(5*segmentBytes),
+ ))
+
+ service := chain_view.GetChainViewService()
+
+ orphans, err := service.FindWalOrphansByDatabase(databaseID)
+ require.NoError(t, err)
+ assert.Empty(t, orphans,
+ "the segment carrying the FULL's own start_lsn is referenced by that FULL")
+}
+
func Test_FindWalGapsInChain_WhenMultipleGapsExist_ReturnsAllGaps(t *testing.T) {
t.Parallel()
diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
index 38dda3e0..e583b85f 100644
--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
@@ -80,6 +80,12 @@ func (r *PhysicalWalSegmentRepository) FindByChainSpan(
// Anti-join not covered by idx_physical_wal_segments_database_id_received_at;
// expect a seq scan on physical_full_backups when invoked in bulk.
+//
+// The comparison anchors on the segment's end_lsn because a FULL's start_lsn is
+// a real recovery position a short way inside the segment that holds it, while a
+// segment's own bounds are derived from its filename and are file-aligned.
+// Comparing against start_lsn would classify that boundary segment as an orphan
+// and delete the WAL the FULL itself starts and stops in.
func (r *PhysicalWalSegmentRepository) FindOrphans(
databaseID uuid.UUID,
) ([]*physical_models.PhysicalWalSegment, error) {
@@ -97,7 +103,7 @@ func (r *PhysicalWalSegmentRepository) FindOrphans(
WHERE f.database_id = w.database_id
AND f.timeline_id = w.timeline_id
AND f.start_lsn IS NOT NULL
- AND f.start_lsn <= w.start_lsn
+ AND f.start_lsn < w.end_lsn
AND f.status = ?
)
ORDER BY w.start_lsn ASC
diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.go
index 9814a8c5..558ccb40 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service.go
@@ -580,8 +580,11 @@ func (s *PhysicalBackupService) cascadeDelete(
return summary, nil
}
-// deleteWalInSpanBudgeted removes WAL rows in the span oldest-LSN first, batched,
-// stopping when the byte budget is reached or the span is drained. Storage
+// deleteWalInSpanBudgeted removes only WAL rows lying fully inside the span. The
+// upper bound is the segment's end_lsn, not its start_lsn: a span ends at the
+// next chain's start_lsn, which sits inside a segment, and that straddling
+// segment carries the bytes the next chain replays from. Bounding on start_lsn
+// would delete it along with the chain being pruned. Storage
// deletes are fail-closed: a transient DeleteFile error aborts the batch (the
// rows survive and retry) rather than orphaning an object. lockRows takes a
// row-level FOR UPDATE on each batch (orphan path, no anchor FULL to serialize
@@ -601,7 +604,7 @@ func (s *PhysicalBackupService) deleteWalInSpanBudgeted(
batchQuery := tx.
Where(
- "database_id = ? AND timeline_id = ? AND start_lsn >= ?::pg_lsn AND start_lsn < ?::pg_lsn",
+ "database_id = ? AND timeline_id = ? AND start_lsn >= ?::pg_lsn AND end_lsn <= ?::pg_lsn",
databaseID, timelineID, span.Start.String(), span.End.String(),
).
Order("start_lsn ASC").
diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.go
index f5707410..315ba9a1 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go
@@ -28,6 +28,11 @@ import (
const segmentMB = 16
+const (
+ firstRecordOffset = physical_testing.FirstRecordOffset
+ fullLSNSpan = physical_testing.FullLSNSpan
+)
+
type serviceTestPrereqs struct {
storage *storages.Storage
database *databases.Database
@@ -172,6 +177,43 @@ func Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects(t *testing.T)
assert.NotNil(t, survivingSuccessor, "successor chain on TL2 must be untouched")
}
+// Pruning a chain deletes the WAL it anchored, up to where the next chain begins.
+// The next FULL starts at firstRecordOffset inside a segment, so that segment
+// begins below the boundary and holds the bytes the surviving chain replays from.
+// It belongs to the successor, not to the chain being pruned.
+func Test_DeleteFull_WhenSuccessorStartsMidSegment_KeepsSuccessorBoundarySegment(t *testing.T) {
+ prereqs := createServiceTestPrereqs(t)
+ databaseID := prereqs.database.ID
+ storageID := prereqs.storage.ID
+ service := physical_service.GetPhysicalBackupService()
+
+ prunedFull := physical_testing.CreateTestFullBackup(t,
+ physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(0), lsn(1)))
+ physical_testing.CreateTestFullBackup(t,
+ physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1,
+ lsn(4)+firstRecordOffset, lsn(4)+firstRecordOffset+fullLSNSpan))
+
+ anchoredSegment := physical_testing.CreateTestWalSegment(t,
+ physical_testing.NewTestWalSegment(databaseID, storageID, 1, "000000010000000000000001", lsn(1), lsn(2)))
+ successorBoundarySegment := physical_testing.CreateTestWalSegment(t,
+ physical_testing.NewTestWalSegment(databaseID, storageID, 1, "000000010000000000000004", lsn(4), lsn(5)))
+
+ saveObject(t, prereqs.storage, *anchoredSegment.FileName)
+ saveObject(t, prereqs.storage, *successorBoundarySegment.FileName)
+
+ _, err := service.DeleteFull(t.Context(), prunedFull.ID, 1_000_000)
+ require.NoError(t, err)
+
+ reloadedAnchoredSegment, _ := physical_repositories.GetWalSegmentRepository().FindByID(anchoredSegment.ID)
+ assert.Nil(t, reloadedAnchoredSegment, "WAL the pruned chain anchored goes with it")
+
+ reloadedBoundarySegment, _ := physical_repositories.GetWalSegmentRepository().FindByID(successorBoundarySegment.ID)
+ assert.NotNil(t, reloadedBoundarySegment,
+ "the segment carrying the surviving FULL's start_lsn must outlive the pruned chain")
+ assert.True(t, objectExists(t, prereqs.storage, *successorBoundarySegment.FileName),
+ "its stored object must survive too")
+}
+
func Test_GetDependentsSummary_ReturnsCountsWithoutDeleting(t *testing.T) {
prereqs := createServiceTestPrereqs(t)
databaseID := prereqs.database.ID
diff --git a/backend/internal/features/backups/backups/core/physical/testing/testing.go b/backend/internal/features/backups/backups/core/physical/testing/testing.go
index 02094a4f..1c504ccf 100644
--- a/backend/internal/features/backups/backups/core/physical/testing/testing.go
+++ b/backend/internal/features/backups/backups/core/physical/testing/testing.go
@@ -13,6 +13,14 @@ import (
"databasus-backend/internal/util/walmath"
)
+// Past the segment's long page header. A FULL's start_lsn carries this offset,
+// while a segment's own bounds are file-aligned.
+const FirstRecordOffset = 0x28
+
+// Wide enough to keep a test FULL's start_lsn and stop_lsn apart, narrow enough
+// that both stay inside the segment holding the start.
+const FullLSNSpan = 0x100
+
func NewTestCompletedFullBackup(
databaseID, storageID uuid.UUID,
timelineID int,
diff --git a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
index a9265023..464fb882 100644
--- a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
+++ b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
@@ -27,6 +27,10 @@ func Test_PhysicalRestore_WhenWalGapBeforeTarget_TokenRequestReturns422(t *testi
physicaltesting.RunWhenWalGapBeforeTargetTokenRequestReturns422(t, pgVersion, pgImage)
}
+func Test_PhysicalRestore_WhenOrphanSweepRunsAfterFull_TargetPastFullStaysRestorable(t *testing.T) {
+ physicaltesting.RunWhenOrphanSweepRunsAfterFullTargetPastFullStaysRestorable(t, pgVersion, pgImage)
+}
+
func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t *testing.T) {
physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)
}
diff --git a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
index 87c2910f..97ed40f7 100644
--- a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
+++ b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
@@ -27,6 +27,10 @@ func Test_PhysicalRestore_WhenWalGapBeforeTarget_TokenRequestReturns422(t *testi
physicaltesting.RunWhenWalGapBeforeTargetTokenRequestReturns422(t, pgVersion, pgImage)
}
+func Test_PhysicalRestore_WhenOrphanSweepRunsAfterFull_TargetPastFullStaysRestorable(t *testing.T) {
+ physicaltesting.RunWhenOrphanSweepRunsAfterFullTargetPastFullStaysRestorable(t, pgVersion, pgImage)
+}
+
func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t *testing.T) {
physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)
}
diff --git a/backend/internal/features/tests/physical/postgresql/shared/bodies.go b/backend/internal/features/tests/physical/postgresql/shared/bodies.go
index e14f14d8..2eff353c 100644
--- a/backend/internal/features/tests/physical/postgresql/shared/bodies.go
+++ b/backend/internal/features/tests/physical/postgresql/shared/bodies.go
@@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ backuping_physical "databasus-backend/internal/features/backups/backups/backuping/physical"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
@@ -302,6 +303,61 @@ func RunWhenWalGapBeforeTargetTokenRequestReturns422(t *testing.T, version, imag
"the gap must be reported so the user never burns a token on an unreachable target")
}
+// A FULL's start_lsn sits inside the segment that carries it, so that segment's file-aligned start
+// is lower than the FULL's and the retention sweep must still keep it: it holds the bytes replay
+// begins from. No incremental is taken on purpose. An incremental moves the replay window past the
+// FULL's own boundary segment and hides whether the sweep kept it.
+func RunWhenOrphanSweepRunsAfterFullTargetPastFullStaysRestorable(t *testing.T, version, image string) {
+ if testing.Short() {
+ t.Skip("streams WAL and runs the retention sweep against a real cluster; skipped in -short")
+ }
+
+ router, fixture := setupReplicationOnlyFixture(
+ t, version, image, postgresql_physical.BackupTypeFullIncrementalAndWalStream)
+
+ sourceConn := openSourceTestDBConn(t, fixture)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
+ defer cancel()
+
+ createMarkerTable(t, ctx, sourceConn)
+ insertMarker(t, ctx, sourceConn, "before-full", "row-in-base-backup")
+
+ enablePhysicalBackupsViaAPI(t, router, fixture, true)
+
+ // Wait for the slot before writing anything: WAL produced before the supervisor claims the
+ // database is never streamed, and that hole at the head of the window would break the restore
+ // on its own, hiding whatever the sweep does.
+ waitForSlotPresent(t, postgresql_executor.OpenAdminConn(t, fixture),
+ fixture.DB.PostgresqlPhysical.ReplicationSlotName, 60*time.Second)
+
+ chain := waitForChainBackups(t, router, fixture, 0, 3*time.Minute)
+
+ insertMarker(t, ctx, sourceConn, "after-full", "row-replayed-up-to-target")
+ streamPostFullSegments(t, ctx, router, sourceConn, fixture, chainTipStopLSN(t, chain), 2, 90*time.Second)
+
+ // A margin wider than the whole-second recovery_target_time truncation on either side keeps the
+ // target unambiguously inside the streamed range.
+ time.Sleep(2 * time.Second)
+ targetTime := time.Now().UTC()
+ time.Sleep(2 * time.Second)
+
+ insertMarker(t, ctx, sourceConn, "after-target", "row-after-target")
+
+ // Natural WAL rather than a forced switch: the segment covering the target has to fill and
+ // archive before the resolver counts it, and a switch would leave it partial.
+ _, err := postgresql_executor.GenerateWalActivity(ctx, sourceConn, 64*1024*1024)
+ require.NoError(t, err)
+
+ waitForTargetTimeReplayable(t, fixture.DB.ID, targetTime, 90*time.Second)
+
+ backuping_physical.RunOrphanWalCleanupForTest(t, fixture.DB.ID)
+
+ token := requestRestoreTokenViaAPI(t, router, fixture, &targetTime)
+ assert.NotEmpty(t, token,
+ "a target covered by archived WAL must stay restorable after the orphan sweep has run")
+}
+
// RunWalSlotAppearsWhenBackupingStartsRemovedWhenDatabaseDeleted proves the WAL replication-slot
// lifecycle end to end and purely over the API: no slot exists until WAL-stream backups are enabled,
// enabling them makes the supervisor create the persistent slot, and deleting the database (DELETE
diff --git a/backend/internal/features/tests/physical/postgresql/shared/helpers.go b/backend/internal/features/tests/physical/postgresql/shared/helpers.go
index b7ec7802..d9833999 100644
--- a/backend/internal/features/tests/physical/postgresql/shared/helpers.go
+++ b/backend/internal/features/tests/physical/postgresql/shared/helpers.go
@@ -779,6 +779,28 @@ func waitForReplayableThroughLSN(t *testing.T, databaseID uuid.UUID, throughLSN
throughLSN.String(), timeout, lastReachable.String())
}
+// Reaching the target's LSN is not enough on its own: the resolver compares the
+// target against the receive time of the last segment in the contiguous run, so a
+// run that already extends past the target by LSN can still end at a segment
+// archived before it.
+func waitForTargetTimeReplayable(t *testing.T, databaseID uuid.UUID, targetTime time.Time, timeout time.Duration) {
+ t.Helper()
+
+ deadline := time.Now().UTC().Add(timeout)
+
+ var lastErr error
+ for time.Now().UTC().Before(deadline) {
+ _, lastErr = chain_view.GetChainViewService().ResolveRestoreSet(databaseID, &targetTime)
+ if lastErr == nil {
+ return
+ }
+
+ time.Sleep(250 * time.Millisecond)
+ }
+
+ t.Fatalf("target time %s never became restorable within %s: %v", targetTime, timeout, lastErr)
+}
+
// seedChainAndStreamPastTarget seeds the marker table, builds FULL -> INCR -> INCR over the API, then
// streams WAL past a captured target time (with a post-target row that must be dropped). It returns
// the PITR target and the phases that must survive recovery to it - shared by the in-container and the