PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Preserve PITR WAL during full backup transitions
Rostislav Dugind520edd26523
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.goindex e322ad55..a8339763 100644--- a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go@@ -297,6 +297,29 @@ func Test_CleanOrphanWalForDatabase_WhenFirstFullInProgress_KeepsWalArchivedDuri"WAL archived while the first full is running must remain available when the full completes")}+func Test_CleanOrphanWalForDatabase_WhenInProgressFullHasNoClaim_DeletesOrphanWal(t *testing.T) {+ prereqs := seedBackupPrereqs(t)++ physical_testing.CreateTestFullBackup(t, physical_testing.NewTestInProgressFullBackup(+ prereqs.DB.ID,+ prereqs.Storage.ID,+ 1,+ ))+ orphanWal := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(+ prereqs.DB.ID,+ prereqs.Storage.ID,+ 1,+ "000000010000000000000001",+ testLSN(1),+ testLSN(2),+ ))++ cleaner := CreateTestPhysicalCleaner()+ cleaner.cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), prereqs.DB.ID)++ assert.False(t, walExists(t, orphanWal.ID))+}+// A FULL taken right after a segment switch reports a start_lsn at// firstRecordOffset. The segment's own bounds come from its filename and are// file-aligned, so it starts below the FULL and must still be kept: it carries thediff --git a/backend/internal/features/backups/backups/controllers/physical/controller_test.go b/backend/internal/features/backups/backups/controllers/physical/controller_test.goindex d18d1c7c..09b84a1e 100644--- a/backend/internal/features/backups/backups/controllers/physical/controller_test.go+++ b/backend/internal/features/backups/backups/controllers/physical/controller_test.go@@ -445,6 +445,58 @@ func Test_GenerateRestoreToken_WhenTargetReachable_ReturnsToken(t *testing.T) {assert.NotEmpty(t, response.Token)}+func Test_GenerateRestoreToken_WhenTargetDuringNewerFull_ReturnsToken(t *testing.T) {+ prereqs := createPhysicalControllerPrereqs(t)+ base := time.Now().UTC().Add(-2 * time.Hour)++ olderFull := physical_testing.NewTestCompletedFullBackup(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ walmath.LSN(0),+ walmath.LSN(segmentBytes),+ )+ olderFull.CreatedAt = base.Add(-time.Minute)+ olderFull.CompletedAt = ptrTime(base)+ physical_testing.CreateTestFullBackup(t, olderFull)++ newerFull := physical_testing.NewTestCompletedFullBackup(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ walmath.LSN(2*segmentBytes),+ walmath.LSN(2*segmentBytes+physical_testing.FullLSNSpan),+ )+ newerFull.CreatedAt = base.Add(5 * time.Minute)+ newerFull.CompletedAt = ptrTime(base.Add(10 * time.Minute))+ physical_testing.CreateTestFullBackup(t, newerFull)++ for walIndex := 1; walIndex <= 3; walIndex++ {+ walSegment := physical_testing.NewTestWalSegment(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ "00000001000000000000000"+string(rune('0'+walIndex)),+ walmath.LSN(walIndex*segmentBytes),+ walmath.LSN((walIndex+1)*segmentBytes),+ )+ walSegment.ReceivedAt = base.Add(time.Duration(3+walIndex*2) * time.Minute)+ physical_testing.CreateTestWalSegment(t, walSegment)+ }++ targetTime := base.Add(7 * time.Minute)+ var response backups_dto_physical.GenerateRestoreTokenResponse+ test_utils.MakePostRequestAndUnmarshal(t, prereqs.router,+ "/api/v1/backups/physical/database/"+prereqs.database.ID.String()+"/restore-token",+ "Bearer "+prereqs.user.Token,+ backups_dto_physical.GenerateRestoreTokenRequest{TargetTime: &targetTime},+ http.StatusOK,+ &response,+ )++ assert.NotEmpty(t, response.Token)+}+// Test_GenerateRestoreToken_WhenTargetPastWalGap_Returns422 verifies the gap is// caught at token-issue time (the restore set is resolved before a token is// minted), so the user never burns a token on an unreachable target.diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.goindex d3c456fb..3e8b310f 100644--- a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go+++ b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go@@ -173,7 +173,7 @@ func Test_ResolveRestoreSet_WhenTargetBetweenIncrementals_StopsAtEarlierIncremenassertWalFilenames(t, set, w2.WalFilename, w3.WalFilename, w4.WalFilename)}-func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTarget(t *testing.T) {+func Test_ResolveRestoreSet_WhenTargetAfterNewerFullStopBeforeCompletion_UsesAllOlderChainWal(t *testing.T) {t.Parallel()prereqs := createChainViewTestPrereqs(t)@@ -195,7 +195,7 @@ func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTaprereqs.storage.ID,1,lsnAt(2)+physical_testing.FirstRecordOffset,- lsnAt(3)+physical_testing.FirstRecordOffset,+ lsnAt(2)+physical_testing.FirstRecordOffset+physical_testing.FullLSNSpan,)newerFull.CreatedAt = clock.at(5)newerFull.CompletedAt = new(clock.at(10))@@ -225,6 +225,14 @@ func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTalsnAt(4),clock.at(11),)+ walAfterTarget := seedWalSegment(+ t,+ prereqs,+ "000000010000000000000004",+ lsnAt(4),+ lsnAt(5),+ clock.at(12),+ )target := clock.at(7)@@ -238,6 +246,7 @@ func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTawalBeforeNewerFull.WalFilename,walAtNewerFullStart.WalFilename,walCoveringTarget.WalFilename,+ walAfterTarget.WalFilename,)}diff --git a/backend/internal/features/backups/backups/core/physical/service/di.go b/backend/internal/features/backups/backups/core/physical/service/di.goindex 88611061..58f397d8 100644--- a/backend/internal/features/backups/backups/core/physical/service/di.go+++ b/backend/internal/features/backups/backups/core/physical/service/di.go@@ -1,7 +1,6 @@package physical_serviceimport (- "databasus-backend/internal/features/backups/backups/core/physical/chain_view"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories""databasus-backend/internal/features/storages""databasus-backend/internal/util/encryption"@@ -10,10 +9,7 @@ import (var physicalBackupService = &PhysicalBackupService{physical_repositories.GetFullBackupRepository(),- physical_repositories.GetIncrementalBackupRepository(),physical_repositories.GetWalSegmentRepository(),- physical_repositories.GetWalHistoryRepository(),- chain_view.GetChainViewService(),storages.GetStorageService(),encryption.GetFieldEncryptor(),logger.GetLogger(),diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.goindex d1c98550..febb8e01 100644--- a/backend/internal/features/backups/backups/core/physical/service/service.go+++ b/backend/internal/features/backups/backups/core/physical/service/service.go@@ -43,14 +43,11 @@ const (// the single seam through which the cleaner mutates catalog rows — the cleaner// never issues raw DELETEs.type PhysicalBackupService struct {- fullBackupRepository *physical_repositories.PhysicalFullBackupRepository- incrementalBackupRepository *physical_repositories.PhysicalIncrementalBackupRepository- walSegmentRepository *physical_repositories.PhysicalWalSegmentRepository- walHistoryRepository *physical_repositories.PhysicalWalHistoryRepository- chainViewService *chain_view.ChainViewService- storageService *storages.StorageService- fieldEncryptor util_encryption.FieldEncryptor- logger *slog.Logger+ fullBackupRepository *physical_repositories.PhysicalFullBackupRepository+ walSegmentRepository *physical_repositories.PhysicalWalSegmentRepository+ storageService *storages.StorageService+ fieldEncryptor util_encryption.FieldEncryptor+ logger *slog.Logger}// GetLatestCompletedFullBackup returns the most recent completed FULL backup for@@ -301,32 +298,74 @@ func (s *PhysicalBackupService) DeleteWalSegment(// GetDependentsSummary counts a chain's dependents and total size without// deleting anything. Idempotent and read-only.func (s *PhysicalBackupService) GetDependentsSummary(rootFullBackupID uuid.UUID) (DependentsSummary, error) {- full, err := s.fullBackupRepository.FindByID(rootFullBackupID)+ targetFull, err := s.fullBackupRepository.FindByID(rootFullBackupID)if err != nil {return DependentsSummary{}, err}- if full == nil {+ if targetFull == nil {return DependentsSummary{}, ErrFullNotFound}- span, err := s.chainViewService.GetChainSpan(rootFullBackupID)- if err != nil {- return DependentsSummary{}, err- }+ var segments []*physical_models.PhysicalWalSegment+ var incrementals []*physical_models.PhysicalIncrementalBackup+ var historyFiles []*physical_models.PhysicalWalHistoryFile+ var lockedFull physical_models.PhysicalFullBackup- segments, err := s.chainViewService.FindWalSegmentsInSpan(full.DatabaseID, full.TimelineID, span.Start, span.End)- if err != nil {- return DependentsSummary{}, err- }+ transactionErr := storage.GetDb().Transaction(func(tx *gorm.DB) error {+ if err := physical_repositories.AcquireBackupAndOrphanCleanupLock(tx, targetFull.DatabaseID); err != nil {+ return err+ }- incrementals, err := s.incrementalBackupRepository.FindAllByRootFull(rootFullBackupID)- if err != nil {- return DependentsSummary{}, err- }+ lockErr := tx.+ Clauses(clause.Locking{Strength: "UPDATE"}).+ Where("id = ?", rootFullBackupID).+ First(&lockedFull).Error+ if errors.Is(lockErr, gorm.ErrRecordNotFound) {+ return ErrFullNotFound+ }+ if lockErr != nil {+ return lockErr+ }- historyFiles, err := s.walHistoryRepository.FindAllByDatabase(full.DatabaseID)- if err != nil {- return DependentsSummary{}, err+ walPlan, err := s.getFullRemovalWalPlan(tx, &lockedFull)+ if err != nil {+ return err+ }+ if walPlan.hasDeletableWal {+ if err := selectWalSegmentsFullyContainedInSpan(+ tx,+ lockedFull.DatabaseID,+ lockedFull.TimelineID,+ walPlan.span,+ ).Find(&segments).Error; err != nil {+ return err+ }+ }++ if err := tx.+ Where("root_full_backup_id = ?", rootFullBackupID).+ Find(&incrementals).Error; err != nil {+ return err+ }++ if walPlan.hasInFlightFull {+ return nil+ }++ hasSurvivingFull, err := hasSurvivingCompletedFullOnTimeline(tx, &lockedFull)+ if err != nil {+ return err+ }+ if hasSurvivingFull {+ return nil+ }++ return tx.+ Where("database_id = ? AND timeline_id = ?", lockedFull.DatabaseID, lockedFull.TimelineID).+ Find(&historyFiles).Error+ })+ if transactionErr != nil {+ return DependentsSummary{}, transactionErr}summary := DependentsSummary{@@ -335,8 +374,8 @@ func (s *PhysicalBackupService) GetDependentsSummary(rootFullBackupID uuid.UUID)Incrementals: len(incrementals),}- if full.BackupSizeMb != nil {- summary.TotalSizeMB += *full.BackupSizeMb+ if lockedFull.BackupSizeMb != nil {+ summary.TotalSizeMB += *lockedFull.BackupSizeMb}for _, segment := range segments {@@ -350,10 +389,8 @@ func (s *PhysicalBackupService) GetDependentsSummary(rootFullBackupID uuid.UUID)}for _, historyFile := range historyFiles {- if historyFile.TimelineID == full.TimelineID {- summary.HistoryFiles++- summary.TotalSizeMB += historyFile.CompressedSizeMb- }+ summary.HistoryFiles+++ summary.TotalSizeMB += historyFile.CompressedSizeMb}return summary, nil@@ -565,15 +602,30 @@ func (s *PhysicalBackupService) cascadeDelete() (DeletedSummary, error) {summary := DeletedSummary{RootFullBackupID: rootFullBackupID}+ targetFull, err := s.fullBackupRepository.FindByID(rootFullBackupID)+ if err != nil {+ return DeletedSummary{}, err+ }+ if targetFull == nil {+ return summary, nil+ }+txErr := storage.GetDb().WithContext(ctx).Transaction(func(tx *gorm.DB) error {- var full physical_models.PhysicalFullBackup+ if err := physical_repositories.AcquireBackupAndOrphanCleanupLock(+ tx,+ targetFull.DatabaseID,+ ); err != nil {+ return err+ }++ var lockedFull physical_models.PhysicalFullBackup// FOR UPDATE serializes with concurrent cleaner instances and the WAL// uploader (which locks the same anchor FULL before its INSERT).lockErr := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", rootFullBackupID).- First(&full).Error+ First(&lockedFull).Errorif errors.Is(lockErr, gorm.ErrRecordNotFound) {s.logger.DebugContext(ctx, "chain already deleted by peer", "root_full_backup_id", rootFullBackupID)@@ -583,24 +635,48 @@ func (s *PhysicalBackupService) cascadeDelete(return lockErr}- span, err := s.chainViewService.GetChainSpan(rootFullBackupID)- if err != nil {- return err- }+ var span chain_view.LSNRange+ var hasDeletableWal bool+ var err error- walRows, walBytes, budgetHit, err := s.deleteWalInSpanBudgeted(ctx,- tx, full.DatabaseID, full.TimelineID, span, walByteBudgetMB, false,- )- summary.WalSegments = walRows- summary.BytesDeletedMB = walBytes+ var hasInFlightFull bool+ if keepFull {+ hasInFlightFull, err = physical_repositories.HasInFlightFullBackup(tx, lockedFull.DatabaseID)+ if err != nil {+ return err+ }+ if !hasInFlightFull {+ span, err = s.getWalOwnershipSpan(tx, &lockedFull)+ hasDeletableWal = true+ }+ } else {+ walPlan, planErr := s.getFullRemovalWalPlan(tx, &lockedFull)+ if planErr != nil {+ return planErr+ }+ span = walPlan.span+ hasDeletableWal = walPlan.hasDeletableWal+ hasInFlightFull = walPlan.hasInFlightFull+ }if err != nil {return err}- if budgetHit {- // WAL budget exhausted before the chain emptied — commit the partial- // deletion; the next tick re-queries the now-oldest rows.- return nil+ if hasDeletableWal {+ walRows, walBytes, budgetHit, err := s.deleteWalInSpanBudgeted(ctx,+ tx, lockedFull.DatabaseID, lockedFull.TimelineID, span, walByteBudgetMB, false,+ )+ summary.WalSegments = walRows+ summary.BytesDeletedMB = walBytes+ if err != nil {+ return err+ }++ if budgetHit {+ // WAL budget exhausted before the chain emptied — commit the partial+ // deletion; the next tick re-queries the now-oldest rows.+ return nil+ }}incrementalCount, err := s.deleteIncrementals(ctx, tx, rootFullBackupID)@@ -613,13 +689,15 @@ func (s *PhysicalBackupService) cascadeDelete(return nil}- historyCount, err := s.deleteOrphanedHistoryFiles(ctx, tx, &full)- summary.HistoryFiles = historyCount- if err != nil {- return err+ if !hasInFlightFull {+ historyCount, err := s.deleteOrphanedHistoryFiles(ctx, tx, &lockedFull)+ summary.HistoryFiles = historyCount+ if err != nil {+ return err+ }}- if err := s.deleteFullArtifactAndRow(ctx, tx, &full); err != nil {+ if err := s.deleteFullArtifactAndRow(ctx, tx, &lockedFull); err != nil {return err}@@ -634,6 +712,120 @@ func (s *PhysicalBackupService) cascadeDelete(return summary, nil}+type fullRemovalWalPlan struct {+ span chain_view.LSNRange+ hasDeletableWal bool+ hasInFlightFull bool+}++func (s *PhysicalBackupService) getFullRemovalWalPlan(+ database *gorm.DB,+ full *physical_models.PhysicalFullBackup,+) (fullRemovalWalPlan, error) {+ hasInFlightFull, err := physical_repositories.HasInFlightFullBackup(database, full.DatabaseID)+ if err != nil {+ return fullRemovalWalPlan{}, err+ }+ if hasInFlightFull {+ return fullRemovalWalPlan{hasInFlightFull: true}, nil+ }++ if full.StartLSN == nil {+ return fullRemovalWalPlan{}, errors.New("cannot determine deletable WAL without FULL start_lsn")+ }++ var predecessorCount int64+ if err := database.Model(&physical_models.PhysicalFullBackup{}).+ Where(+ "database_id = ? AND timeline_id = ? AND status = ? AND start_lsn IS NOT NULL AND start_lsn < ?::pg_lsn",+ full.DatabaseID,+ full.TimelineID,+ physical_enums.PhysicalBackupStatusCompleted,+ full.StartLSN.String(),+ ).+ Count(&predecessorCount).Error; err != nil {+ return fullRemovalWalPlan{}, err+ }+ if predecessorCount > 0 {+ return fullRemovalWalPlan{}, nil+ }++ span, err := s.getWalOwnershipSpan(database, full)+ if err != nil {+ return fullRemovalWalPlan{}, err+ }++ return fullRemovalWalPlan{span: span, hasDeletableWal: true}, nil+}++func hasSurvivingCompletedFullOnTimeline(+ database *gorm.DB,+ full *physical_models.PhysicalFullBackup,+) (bool, error) {+ var survivingFullCount int64++ if err := database.+ Model(&physical_models.PhysicalFullBackup{}).+ Where(+ "database_id = ? AND timeline_id = ? AND status = ? AND id != ?",+ full.DatabaseID,+ full.TimelineID,+ physical_enums.PhysicalBackupStatusCompleted,+ full.ID,+ ).+ Count(&survivingFullCount).Error; err != nil {+ return false, err+ }++ return survivingFullCount > 0, nil+}++func (s *PhysicalBackupService) getWalOwnershipSpan(+ database *gorm.DB,+ full *physical_models.PhysicalFullBackup,+) (chain_view.LSNRange, error) {+ if full.StartLSN == nil {+ return chain_view.LSNRange{}, errors.New("cannot determine WAL ownership without FULL start_lsn")+ }++ var successor physical_models.PhysicalFullBackup+ findSuccessorErr := database.+ Where(+ "database_id = ? AND timeline_id = ? AND status = ? AND start_lsn > ?::pg_lsn",+ full.DatabaseID,+ full.TimelineID,+ physical_enums.PhysicalBackupStatusCompleted,+ full.StartLSN.String(),+ ).+ Order("start_lsn ASC").+ First(&successor).Error+ if findSuccessorErr != nil && !errors.Is(findSuccessorErr, gorm.ErrRecordNotFound) {+ return chain_view.LSNRange{}, findSuccessorErr+ }++ spanEnd := chain_view.LSNMax+ if findSuccessorErr == nil && successor.StartLSN != nil {+ spanEnd = *successor.StartLSN+ }++ return chain_view.LSNRange{Start: *full.StartLSN, End: spanEnd}, nil+}++func selectWalSegmentsFullyContainedInSpan(+ database *gorm.DB,+ databaseID uuid.UUID,+ timelineID int,+ span chain_view.LSNRange,+) *gorm.DB {+ return database.Where(+ "database_id = ? AND timeline_id = ? AND start_lsn >= ?::pg_lsn AND end_lsn <= ?::pg_lsn",+ databaseID,+ timelineID,+ span.Start.String(),+ span.End.String(),+ )+}+// deleteWalInSpanBudgeted removes only WAL rows lying fully inside the span. The// upper bound is the segment's end_lsn, not its start_lsn: a span ends at the// next chain's start_lsn, which sits inside a segment, and that straddling@@ -656,11 +848,7 @@ func (s *PhysicalBackupService) deleteWalInSpanBudgeted(for {var segments []*physical_models.PhysicalWalSegment- batchQuery := tx.- Where(- "database_id = ? AND timeline_id = ? AND start_lsn >= ?::pg_lsn AND end_lsn <= ?::pg_lsn",- databaseID, timelineID, span.Start.String(), span.End.String(),- ).+ batchQuery := selectWalSegmentsFullyContainedInSpan(tx, databaseID, timelineID, span).Order("start_lsn ASC").Limit(maxWalDeleteBatchRows)@@ -814,17 +1002,12 @@ func (s *PhysicalBackupService) deleteOrphanedHistoryFiles(tx *gorm.DB,full *physical_models.PhysicalFullBackup,) (int, error) {- var survivingFulls int64-- if err := tx.- Model(&physical_models.PhysicalFullBackup{}).- Where("database_id = ? AND timeline_id = ? AND status = ? AND id != ?",- full.DatabaseID, full.TimelineID, physical_enums.PhysicalBackupStatusCompleted, full.ID).- Count(&survivingFulls).Error; err != nil {+ hasSurvivingFull, err := hasSurvivingCompletedFullOnTimeline(tx, full)+ if err != nil {return 0, err}- if survivingFulls > 0 {+ if hasSurvivingFull {return 0, nil}diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.goindex 315ba9a1..01a3a756 100644--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go@@ -9,6 +9,7 @@ import ("github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "databasus-backend/internal/features/backups/backups/core/physical/chain_view"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"@@ -21,6 +22,7 @@ import (users_testing "databasus-backend/internal/features/users/testing"workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"workspaces_testing "databasus-backend/internal/features/workspaces/testing"+ "databasus-backend/internal/storage""databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger""databasus-backend/internal/util/walmath"@@ -214,6 +216,272 @@ func Test_DeleteFull_WhenSuccessorStartsMidSegment_KeepsSuccessorBoundarySegment"its stored object must survive too")}+func Test_DeleteFull_WhenPredecessorSurvives_KeepsSharedWalRestorable(t *testing.T) {+ prereqs := createServiceTestPrereqs(t)+ databaseID := prereqs.database.ID+ storageID := prereqs.storage.ID+ service := physical_service.GetPhysicalBackupService()+ base := time.Now().UTC().Add(-time.Hour)++ predecessorModel := physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(0), lsn(1))+ predecessorModel.CreatedAt = base+ predecessorModel.CompletedAt = new(base.Add(time.Minute))+ predecessor := physical_testing.CreateTestFullBackup(t, predecessorModel)++ successorModel := physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(2), lsn(3))+ successorModel.CreatedAt = base.Add(5 * time.Minute)+ successorModel.CompletedAt = new(base.Add(10 * time.Minute))+ successor := physical_testing.CreateTestFullBackup(t, successorModel)+ saveObject(t, prereqs.storage, *successor.FileName)++ walSegments := []*physical_models.PhysicalWalSegment{+ physical_testing.NewTestWalSegment(+ databaseID, storageID, 1, "000000010000000000000001", lsn(1), lsn(2)),+ physical_testing.NewTestWalSegment(+ databaseID, storageID, 1, "000000010000000000000002", lsn(2), lsn(3)),+ physical_testing.NewTestWalSegment(+ databaseID, storageID, 1, "000000010000000000000003", lsn(3), lsn(4)),+ }+ for walIndex, walSegment := range walSegments {+ walSegment.ReceivedAt = base.Add(time.Duration(4+walIndex*3) * time.Minute)+ physical_testing.CreateTestWalSegment(t, walSegment)+ saveObject(t, prereqs.storage, *walSegment.FileName)+ }++ preview, err := service.GetDependentsSummary(successor.ID)+ require.NoError(t, err)+ assert.Zero(t, preview.WalSegments)++ deleted, err := service.DeleteFull(t.Context(), successor.ID, 1_000_000)+ require.NoError(t, err)+ assert.True(t, deleted.ChainFullyDeleted)+ assert.Zero(t, deleted.WalSegments)++ for _, walSegment := range walSegments {+ reloadedSegment, reloadErr := physical_repositories.GetWalSegmentRepository().FindByID(walSegment.ID)+ require.NoError(t, reloadErr)+ assert.NotNil(t, reloadedSegment)+ assert.True(t, objectExists(t, prereqs.storage, *walSegment.FileName))+ }++ targetTime := base.Add(7 * time.Minute)+ restoreSet, err := chain_view.GetChainViewService().ResolveRestoreSet(databaseID, &targetTime)+ require.NoError(t, err)+ assert.Equal(t, predecessor.ID, restoreSet.RootFull.ID)+ require.Len(t, restoreSet.WalSegments, len(walSegments))+}++func Test_CascadeDelete_WhenFullClaimCommitsConcurrently_KeepsWal(t *testing.T) {+ testCases := []struct {+ name string+ keepFull bool+ }{+ {name: "delete full", keepFull: false},+ {name: "keep full", keepFull: true},+ }++ for _, testCase := range testCases {+ t.Run(testCase.name, func(t *testing.T) {+ prereqs := createServiceTestPrereqs(t)+ physicalBackupService := physical_service.GetPhysicalBackupService()+ rootFull := physical_testing.CreateTestFullBackup(t,+ physical_testing.NewTestCompletedFullBackup(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ lsn(0),+ lsn(1),+ ))+ walSegment := physical_testing.CreateTestWalSegment(t,+ physical_testing.NewTestWalSegment(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ "000000010000000000000001",+ lsn(1),+ lsn(2),+ ))+ saveObject(t, prereqs.storage, *walSegment.FileName)++ claimTransaction := storage.GetDb().Begin()+ require.NoError(t, claimTransaction.Error)+ t.Cleanup(func() { _ = claimTransaction.Rollback().Error })+ require.NoError(t, physical_repositories.AcquireBackupAndOrphanCleanupLock(+ claimTransaction,+ prereqs.database.ID,+ ))++ type cascadeDeletionOutcome struct {+ summary physical_service.DeletedSummary+ err error+ }+ deletionOutcomes := make(chan cascadeDeletionOutcome, 1)+ go func() {+ var summary physical_service.DeletedSummary+ var err error+ if testCase.keepFull {+ summary, err = physicalBackupService.DeleteChainDependentsKeepFull(+ t.Context(),+ rootFull.ID,+ 1_000_000,+ )+ } else {+ summary, err = physicalBackupService.DeleteFull(+ t.Context(),+ rootFull.ID,+ 1_000_000,+ )+ }+ deletionOutcomes <- cascadeDeletionOutcome{summary: summary, err: err}+ }()++ select {+ case <-deletionOutcomes:+ t.Fatal("cascade deletion must wait while FULL claim creation holds the database lock")+ case <-time.After(100 * time.Millisecond):+ }++ activeFull := physical_testing.NewTestCompletedFullBackup(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ lsn(4),+ lsn(5),+ )+ activeFull.Status = physical_enums.PhysicalBackupStatusInProgress+ activeFull.StartLSN = nil+ activeFull.StopLSN = nil+ activeFull.FileName = nil+ activeFull.CompletedAt = nil+ require.NoError(t, claimTransaction.Create(activeFull).Error)++ isClaimed, err := physical_repositories.GetInFlightBackupRepository().Claim(+ claimTransaction,+ physical_repositories.ClaimSpec{+ DatabaseID: prereqs.database.ID,+ BackupType: physical_enums.PhysicalBackupTypeFull,+ BackupID: activeFull.ID,+ },+ )+ require.NoError(t, err)+ require.True(t, isClaimed)+ require.NoError(t, claimTransaction.Commit().Error)++ deletionOutcome := <-deletionOutcomes+ require.NoError(t, deletionOutcome.err)+ assert.Zero(t, deletionOutcome.summary.WalSegments)++ reloadedWalSegment, err := physical_repositories.GetWalSegmentRepository().FindByID(walSegment.ID)+ require.NoError(t, err)+ assert.NotNil(t, reloadedWalSegment)+ assert.True(t, objectExists(t, prereqs.storage, *walSegment.FileName))++ reloadedFull, err := physical_repositories.GetFullBackupRepository().FindByID(rootFull.ID)+ require.NoError(t, err)+ if testCase.keepFull {+ assert.NotNil(t, reloadedFull)+ } else {+ assert.Nil(t, reloadedFull)+ }+ })+ }+}++func Test_GetDependentsSummary_WhenFullClaimIsActive_MatchesDeleteSet(t *testing.T) {+ prereqs := createServiceTestPrereqs(t)+ physicalBackupService := physical_service.GetPhysicalBackupService()+ rootFull := physical_testing.CreateTestFullBackup(t,+ physical_testing.NewTestCompletedFullBackup(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ lsn(0),+ lsn(1),+ ))+ walSegment := physical_testing.CreateTestWalSegment(t,+ physical_testing.NewTestWalSegment(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ "000000010000000000000001",+ lsn(1),+ lsn(2),+ ))+ historyFile := physical_testing.CreateTestWalHistoryFile(t,+ physical_testing.NewTestWalHistoryFile(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ ))+ physical_testing.CreateTestInFlightClaim(+ t,+ prereqs.database.ID,+ uuid.New(),+ physical_enums.PhysicalBackupTypeFull,+ )++ deletionPreview, err := physicalBackupService.GetDependentsSummary(rootFull.ID)+ require.NoError(t, err)+ assert.Zero(t, deletionPreview.WalSegments)+ assert.Zero(t, deletionPreview.HistoryFiles)++ deletionSummary, err := physicalBackupService.DeleteFull(t.Context(), rootFull.ID, 1_000_000)+ require.NoError(t, err)+ assert.Zero(t, deletionSummary.WalSegments)+ assert.Zero(t, deletionSummary.HistoryFiles)+ assert.Equal(t, deletionPreview.WalSegments, deletionSummary.WalSegments)+ assert.Equal(t, deletionPreview.HistoryFiles, deletionSummary.HistoryFiles)++ reloadedWalSegment, err := physical_repositories.GetWalSegmentRepository().FindByID(walSegment.ID)+ require.NoError(t, err)+ assert.NotNil(t, reloadedWalSegment)+ reloadedHistoryFile, err := physical_repositories.GetWalHistoryRepository().FindByDatabaseTimeline(+ prereqs.database.ID,+ historyFile.TimelineID,+ )+ require.NoError(t, err)+ assert.NotNil(t, reloadedHistoryFile)+}++func Test_GetDependentsSummary_WhenUpperSegmentCrossesOwnershipBoundary_MatchesDeleteSet(t *testing.T) {+ prereqs := createServiceTestPrereqs(t)+ databaseID := prereqs.database.ID+ storageID := prereqs.storage.ID+ service := physical_service.GetPhysicalBackupService()++ fullModel := physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(0), lsn(1))+ fullModel.BackupSizeMb = new(50.0)+ full := physical_testing.CreateTestFullBackup(t, fullModel)+ physical_testing.CreateTestFullBackup(t, physical_testing.NewTestCompletedFullBackup(+ databaseID,+ storageID,+ 1,+ lsn(4)+firstRecordOffset,+ lsn(4)+firstRecordOffset+fullLSNSpan,+ ))++ ownedSegment := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(+ databaseID, storageID, 1, "000000010000000000000001", lsn(1), lsn(2)))+ boundarySegment := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(+ databaseID, storageID, 1, "000000010000000000000004", lsn(4), lsn(5)))+ saveObject(t, prereqs.storage, *full.FileName)+ saveObject(t, prereqs.storage, *ownedSegment.FileName)+ saveObject(t, prereqs.storage, *boundarySegment.FileName)++ preview, err := service.GetDependentsSummary(full.ID)+ require.NoError(t, err)+ assert.Equal(t, 1, preview.WalSegments)+ assert.Equal(t, 66.0, preview.TotalSizeMB)++ deleted, err := service.DeleteFull(t.Context(), full.ID, 1_000_000)+ require.NoError(t, err)+ assert.Equal(t, preview.WalSegments, deleted.WalSegments)++ remainingBoundary, err := physical_repositories.GetWalSegmentRepository().FindByID(boundarySegment.ID)+ require.NoError(t, err)+ assert.NotNil(t, remainingBoundary)+}+func Test_GetDependentsSummary_ReturnsCountsWithoutDeleting(t *testing.T) {prereqs := createServiceTestPrereqs(t)databaseID := prereqs.database.ID