Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Preserve PITR WAL during full backup transitions

Rostislav Dugind520edd26523
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
index e322ad55..a8339763 100644
--- a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
@@ -297,6 +297,29 @@ func Test_CleanOrphanWalForDatabase_WhenFirstFullInProgress_KeepsWalArchivedDuri
"WAL archived while the first full is running must remain available when the full completes")
}
+func Test_CleanOrphanWalForDatabase_WhenInProgressFullHasNoClaim_DeletesOrphanWal(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+
+ physical_testing.CreateTestFullBackup(t, physical_testing.NewTestInProgressFullBackup(
+ prereqs.DB.ID,
+ prereqs.Storage.ID,
+ 1,
+ ))
+ orphanWal := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(
+ prereqs.DB.ID,
+ prereqs.Storage.ID,
+ 1,
+ "000000010000000000000001",
+ testLSN(1),
+ testLSN(2),
+ ))
+
+ cleaner := CreateTestPhysicalCleaner()
+ cleaner.cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), prereqs.DB.ID)
+
+ assert.False(t, walExists(t, orphanWal.ID))
+}
+
// A FULL taken right after a segment switch reports a start_lsn at
// firstRecordOffset. The segment's own bounds come from its filename and are
// file-aligned, so it starts below the FULL and must still be kept: it carries the
diff --git a/backend/internal/features/backups/backups/controllers/physical/controller_test.go b/backend/internal/features/backups/backups/controllers/physical/controller_test.go
index d18d1c7c..09b84a1e 100644
--- a/backend/internal/features/backups/backups/controllers/physical/controller_test.go
+++ b/backend/internal/features/backups/backups/controllers/physical/controller_test.go
@@ -445,6 +445,58 @@ func Test_GenerateRestoreToken_WhenTargetReachable_ReturnsToken(t *testing.T) {
assert.NotEmpty(t, response.Token)
}
+func Test_GenerateRestoreToken_WhenTargetDuringNewerFull_ReturnsToken(t *testing.T) {
+ prereqs := createPhysicalControllerPrereqs(t)
+ base := time.Now().UTC().Add(-2 * time.Hour)
+
+ olderFull := physical_testing.NewTestCompletedFullBackup(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ walmath.LSN(0),
+ walmath.LSN(segmentBytes),
+ )
+ olderFull.CreatedAt = base.Add(-time.Minute)
+ olderFull.CompletedAt = ptrTime(base)
+ physical_testing.CreateTestFullBackup(t, olderFull)
+
+ newerFull := physical_testing.NewTestCompletedFullBackup(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ walmath.LSN(2*segmentBytes),
+ walmath.LSN(2*segmentBytes+physical_testing.FullLSNSpan),
+ )
+ newerFull.CreatedAt = base.Add(5 * time.Minute)
+ newerFull.CompletedAt = ptrTime(base.Add(10 * time.Minute))
+ physical_testing.CreateTestFullBackup(t, newerFull)
+
+ for walIndex := 1; walIndex <= 3; walIndex++ {
+ walSegment := physical_testing.NewTestWalSegment(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ "00000001000000000000000"+string(rune('0'+walIndex)),
+ walmath.LSN(walIndex*segmentBytes),
+ walmath.LSN((walIndex+1)*segmentBytes),
+ )
+ walSegment.ReceivedAt = base.Add(time.Duration(3+walIndex*2) * time.Minute)
+ physical_testing.CreateTestWalSegment(t, walSegment)
+ }
+
+ targetTime := base.Add(7 * time.Minute)
+ var response backups_dto_physical.GenerateRestoreTokenResponse
+ test_utils.MakePostRequestAndUnmarshal(t, prereqs.router,
+ "/api/v1/backups/physical/database/"+prereqs.database.ID.String()+"/restore-token",
+ "Bearer "+prereqs.user.Token,
+ backups_dto_physical.GenerateRestoreTokenRequest{TargetTime: &targetTime},
+ http.StatusOK,
+ &response,
+ )
+
+ assert.NotEmpty(t, response.Token)
+}
+
// Test_GenerateRestoreToken_WhenTargetPastWalGap_Returns422 verifies the gap is
// caught at token-issue time (the restore set is resolved before a token is
// minted), so the user never burns a token on an unreachable target.
diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go
index d3c456fb..3e8b310f 100644
--- a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go
+++ b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go
@@ -173,7 +173,7 @@ func Test_ResolveRestoreSet_WhenTargetBetweenIncrementals_StopsAtEarlierIncremen
assertWalFilenames(t, set, w2.WalFilename, w3.WalFilename, w4.WalFilename)
}
-func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTarget(t *testing.T) {
+func Test_ResolveRestoreSet_WhenTargetAfterNewerFullStopBeforeCompletion_UsesAllOlderChainWal(t *testing.T) {
t.Parallel()
prereqs := createChainViewTestPrereqs(t)
@@ -195,7 +195,7 @@ func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTa
prereqs.storage.ID,
1,
lsnAt(2)+physical_testing.FirstRecordOffset,
- lsnAt(3)+physical_testing.FirstRecordOffset,
+ lsnAt(2)+physical_testing.FirstRecordOffset+physical_testing.FullLSNSpan,
)
newerFull.CreatedAt = clock.at(5)
newerFull.CompletedAt = new(clock.at(10))
@@ -225,6 +225,14 @@ func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTa
lsnAt(4),
clock.at(11),
)
+ walAfterTarget := seedWalSegment(
+ t,
+ prereqs,
+ "000000010000000000000004",
+ lsnAt(4),
+ lsnAt(5),
+ clock.at(12),
+ )
target := clock.at(7)
@@ -238,6 +246,7 @@ func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTa
walBeforeNewerFull.WalFilename,
walAtNewerFullStart.WalFilename,
walCoveringTarget.WalFilename,
+ walAfterTarget.WalFilename,
)
}
diff --git a/backend/internal/features/backups/backups/core/physical/service/di.go b/backend/internal/features/backups/backups/core/physical/service/di.go
index 88611061..58f397d8 100644
--- a/backend/internal/features/backups/backups/core/physical/service/di.go
+++ b/backend/internal/features/backups/backups/core/physical/service/di.go
@@ -1,7 +1,6 @@
package physical_service
import (
- "databasus-backend/internal/features/backups/backups/core/physical/chain_view"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/encryption"
@@ -10,10 +9,7 @@ import (
var physicalBackupService = &PhysicalBackupService{
physical_repositories.GetFullBackupRepository(),
- physical_repositories.GetIncrementalBackupRepository(),
physical_repositories.GetWalSegmentRepository(),
- physical_repositories.GetWalHistoryRepository(),
- chain_view.GetChainViewService(),
storages.GetStorageService(),
encryption.GetFieldEncryptor(),
logger.GetLogger(),
diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.go
index d1c98550..febb8e01 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service.go
@@ -43,14 +43,11 @@ const (
// the single seam through which the cleaner mutates catalog rows — the cleaner
// never issues raw DELETEs.
type PhysicalBackupService struct {
- fullBackupRepository *physical_repositories.PhysicalFullBackupRepository
- incrementalBackupRepository *physical_repositories.PhysicalIncrementalBackupRepository
- walSegmentRepository *physical_repositories.PhysicalWalSegmentRepository
- walHistoryRepository *physical_repositories.PhysicalWalHistoryRepository
- chainViewService *chain_view.ChainViewService
- storageService *storages.StorageService
- fieldEncryptor util_encryption.FieldEncryptor
- logger *slog.Logger
+ fullBackupRepository *physical_repositories.PhysicalFullBackupRepository
+ walSegmentRepository *physical_repositories.PhysicalWalSegmentRepository
+ storageService *storages.StorageService
+ fieldEncryptor util_encryption.FieldEncryptor
+ logger *slog.Logger
}
// GetLatestCompletedFullBackup returns the most recent completed FULL backup for
@@ -301,32 +298,74 @@ func (s *PhysicalBackupService) DeleteWalSegment(
// GetDependentsSummary counts a chain's dependents and total size without
// deleting anything. Idempotent and read-only.
func (s *PhysicalBackupService) GetDependentsSummary(rootFullBackupID uuid.UUID) (DependentsSummary, error) {
- full, err := s.fullBackupRepository.FindByID(rootFullBackupID)
+ targetFull, err := s.fullBackupRepository.FindByID(rootFullBackupID)
if err != nil {
return DependentsSummary{}, err
}
- if full == nil {
+ if targetFull == nil {
return DependentsSummary{}, ErrFullNotFound
}
- span, err := s.chainViewService.GetChainSpan(rootFullBackupID)
- if err != nil {
- return DependentsSummary{}, err
- }
+ var segments []*physical_models.PhysicalWalSegment
+ var incrementals []*physical_models.PhysicalIncrementalBackup
+ var historyFiles []*physical_models.PhysicalWalHistoryFile
+ var lockedFull physical_models.PhysicalFullBackup
- segments, err := s.chainViewService.FindWalSegmentsInSpan(full.DatabaseID, full.TimelineID, span.Start, span.End)
- if err != nil {
- return DependentsSummary{}, err
- }
+ transactionErr := storage.GetDb().Transaction(func(tx *gorm.DB) error {
+ if err := physical_repositories.AcquireBackupAndOrphanCleanupLock(tx, targetFull.DatabaseID); err != nil {
+ return err
+ }
- incrementals, err := s.incrementalBackupRepository.FindAllByRootFull(rootFullBackupID)
- if err != nil {
- return DependentsSummary{}, err
- }
+ lockErr := tx.
+ Clauses(clause.Locking{Strength: "UPDATE"}).
+ Where("id = ?", rootFullBackupID).
+ First(&lockedFull).Error
+ if errors.Is(lockErr, gorm.ErrRecordNotFound) {
+ return ErrFullNotFound
+ }
+ if lockErr != nil {
+ return lockErr
+ }
- historyFiles, err := s.walHistoryRepository.FindAllByDatabase(full.DatabaseID)
- if err != nil {
- return DependentsSummary{}, err
+ walPlan, err := s.getFullRemovalWalPlan(tx, &lockedFull)
+ if err != nil {
+ return err
+ }
+ if walPlan.hasDeletableWal {
+ if err := selectWalSegmentsFullyContainedInSpan(
+ tx,
+ lockedFull.DatabaseID,
+ lockedFull.TimelineID,
+ walPlan.span,
+ ).Find(&segments).Error; err != nil {
+ return err
+ }
+ }
+
+ if err := tx.
+ Where("root_full_backup_id = ?", rootFullBackupID).
+ Find(&incrementals).Error; err != nil {
+ return err
+ }
+
+ if walPlan.hasInFlightFull {
+ return nil
+ }
+
+ hasSurvivingFull, err := hasSurvivingCompletedFullOnTimeline(tx, &lockedFull)
+ if err != nil {
+ return err
+ }
+ if hasSurvivingFull {
+ return nil
+ }
+
+ return tx.
+ Where("database_id = ? AND timeline_id = ?", lockedFull.DatabaseID, lockedFull.TimelineID).
+ Find(&historyFiles).Error
+ })
+ if transactionErr != nil {
+ return DependentsSummary{}, transactionErr
}
summary := DependentsSummary{
@@ -335,8 +374,8 @@ func (s *PhysicalBackupService) GetDependentsSummary(rootFullBackupID uuid.UUID)
Incrementals: len(incrementals),
}
- if full.BackupSizeMb != nil {
- summary.TotalSizeMB += *full.BackupSizeMb
+ if lockedFull.BackupSizeMb != nil {
+ summary.TotalSizeMB += *lockedFull.BackupSizeMb
}
for _, segment := range segments {
@@ -350,10 +389,8 @@ func (s *PhysicalBackupService) GetDependentsSummary(rootFullBackupID uuid.UUID)
}
for _, historyFile := range historyFiles {
- if historyFile.TimelineID == full.TimelineID {
- summary.HistoryFiles++
- summary.TotalSizeMB += historyFile.CompressedSizeMb
- }
+ summary.HistoryFiles++
+ summary.TotalSizeMB += historyFile.CompressedSizeMb
}
return summary, nil
@@ -565,15 +602,30 @@ func (s *PhysicalBackupService) cascadeDelete(
) (DeletedSummary, error) {
summary := DeletedSummary{RootFullBackupID: rootFullBackupID}
+ targetFull, err := s.fullBackupRepository.FindByID(rootFullBackupID)
+ if err != nil {
+ return DeletedSummary{}, err
+ }
+ if targetFull == nil {
+ return summary, nil
+ }
+
txErr := storage.GetDb().WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- var full physical_models.PhysicalFullBackup
+ if err := physical_repositories.AcquireBackupAndOrphanCleanupLock(
+ tx,
+ targetFull.DatabaseID,
+ ); err != nil {
+ return err
+ }
+
+ var lockedFull physical_models.PhysicalFullBackup
// FOR UPDATE serializes with concurrent cleaner instances and the WAL
// uploader (which locks the same anchor FULL before its INSERT).
lockErr := tx.
Clauses(clause.Locking{Strength: "UPDATE"}).
Where("id = ?", rootFullBackupID).
- First(&full).Error
+ First(&lockedFull).Error
if errors.Is(lockErr, gorm.ErrRecordNotFound) {
s.logger.DebugContext(ctx, "chain already deleted by peer", "root_full_backup_id", rootFullBackupID)
@@ -583,24 +635,48 @@ func (s *PhysicalBackupService) cascadeDelete(
return lockErr
}
- span, err := s.chainViewService.GetChainSpan(rootFullBackupID)
- if err != nil {
- return err
- }
+ var span chain_view.LSNRange
+ var hasDeletableWal bool
+ var err error
- walRows, walBytes, budgetHit, err := s.deleteWalInSpanBudgeted(ctx,
- tx, full.DatabaseID, full.TimelineID, span, walByteBudgetMB, false,
- )
- summary.WalSegments = walRows
- summary.BytesDeletedMB = walBytes
+ var hasInFlightFull bool
+ if keepFull {
+ hasInFlightFull, err = physical_repositories.HasInFlightFullBackup(tx, lockedFull.DatabaseID)
+ if err != nil {
+ return err
+ }
+ if !hasInFlightFull {
+ span, err = s.getWalOwnershipSpan(tx, &lockedFull)
+ hasDeletableWal = true
+ }
+ } else {
+ walPlan, planErr := s.getFullRemovalWalPlan(tx, &lockedFull)
+ if planErr != nil {
+ return planErr
+ }
+ span = walPlan.span
+ hasDeletableWal = walPlan.hasDeletableWal
+ hasInFlightFull = walPlan.hasInFlightFull
+ }
if err != nil {
return err
}
- if budgetHit {
- // WAL budget exhausted before the chain emptied — commit the partial
- // deletion; the next tick re-queries the now-oldest rows.
- return nil
+ if hasDeletableWal {
+ walRows, walBytes, budgetHit, err := s.deleteWalInSpanBudgeted(ctx,
+ tx, lockedFull.DatabaseID, lockedFull.TimelineID, span, walByteBudgetMB, false,
+ )
+ summary.WalSegments = walRows
+ summary.BytesDeletedMB = walBytes
+ if err != nil {
+ return err
+ }
+
+ if budgetHit {
+ // WAL budget exhausted before the chain emptied — commit the partial
+ // deletion; the next tick re-queries the now-oldest rows.
+ return nil
+ }
}
incrementalCount, err := s.deleteIncrementals(ctx, tx, rootFullBackupID)
@@ -613,13 +689,15 @@ func (s *PhysicalBackupService) cascadeDelete(
return nil
}
- historyCount, err := s.deleteOrphanedHistoryFiles(ctx, tx, &full)
- summary.HistoryFiles = historyCount
- if err != nil {
- return err
+ if !hasInFlightFull {
+ historyCount, err := s.deleteOrphanedHistoryFiles(ctx, tx, &lockedFull)
+ summary.HistoryFiles = historyCount
+ if err != nil {
+ return err
+ }
}
- if err := s.deleteFullArtifactAndRow(ctx, tx, &full); err != nil {
+ if err := s.deleteFullArtifactAndRow(ctx, tx, &lockedFull); err != nil {
return err
}
@@ -634,6 +712,120 @@ func (s *PhysicalBackupService) cascadeDelete(
return summary, nil
}
+type fullRemovalWalPlan struct {
+ span chain_view.LSNRange
+ hasDeletableWal bool
+ hasInFlightFull bool
+}
+
+func (s *PhysicalBackupService) getFullRemovalWalPlan(
+ database *gorm.DB,
+ full *physical_models.PhysicalFullBackup,
+) (fullRemovalWalPlan, error) {
+ hasInFlightFull, err := physical_repositories.HasInFlightFullBackup(database, full.DatabaseID)
+ if err != nil {
+ return fullRemovalWalPlan{}, err
+ }
+ if hasInFlightFull {
+ return fullRemovalWalPlan{hasInFlightFull: true}, nil
+ }
+
+ if full.StartLSN == nil {
+ return fullRemovalWalPlan{}, errors.New("cannot determine deletable WAL without FULL start_lsn")
+ }
+
+ var predecessorCount int64
+ if err := database.Model(&physical_models.PhysicalFullBackup{}).
+ Where(
+ "database_id = ? AND timeline_id = ? AND status = ? AND start_lsn IS NOT NULL AND start_lsn < ?::pg_lsn",
+ full.DatabaseID,
+ full.TimelineID,
+ physical_enums.PhysicalBackupStatusCompleted,
+ full.StartLSN.String(),
+ ).
+ Count(&predecessorCount).Error; err != nil {
+ return fullRemovalWalPlan{}, err
+ }
+ if predecessorCount > 0 {
+ return fullRemovalWalPlan{}, nil
+ }
+
+ span, err := s.getWalOwnershipSpan(database, full)
+ if err != nil {
+ return fullRemovalWalPlan{}, err
+ }
+
+ return fullRemovalWalPlan{span: span, hasDeletableWal: true}, nil
+}
+
+func hasSurvivingCompletedFullOnTimeline(
+ database *gorm.DB,
+ full *physical_models.PhysicalFullBackup,
+) (bool, error) {
+ var survivingFullCount int64
+
+ if err := database.
+ Model(&physical_models.PhysicalFullBackup{}).
+ Where(
+ "database_id = ? AND timeline_id = ? AND status = ? AND id != ?",
+ full.DatabaseID,
+ full.TimelineID,
+ physical_enums.PhysicalBackupStatusCompleted,
+ full.ID,
+ ).
+ Count(&survivingFullCount).Error; err != nil {
+ return false, err
+ }
+
+ return survivingFullCount > 0, nil
+}
+
+func (s *PhysicalBackupService) getWalOwnershipSpan(
+ database *gorm.DB,
+ full *physical_models.PhysicalFullBackup,
+) (chain_view.LSNRange, error) {
+ if full.StartLSN == nil {
+ return chain_view.LSNRange{}, errors.New("cannot determine WAL ownership without FULL start_lsn")
+ }
+
+ var successor physical_models.PhysicalFullBackup
+ findSuccessorErr := database.
+ Where(
+ "database_id = ? AND timeline_id = ? AND status = ? AND start_lsn > ?::pg_lsn",
+ full.DatabaseID,
+ full.TimelineID,
+ physical_enums.PhysicalBackupStatusCompleted,
+ full.StartLSN.String(),
+ ).
+ Order("start_lsn ASC").
+ First(&successor).Error
+ if findSuccessorErr != nil && !errors.Is(findSuccessorErr, gorm.ErrRecordNotFound) {
+ return chain_view.LSNRange{}, findSuccessorErr
+ }
+
+ spanEnd := chain_view.LSNMax
+ if findSuccessorErr == nil && successor.StartLSN != nil {
+ spanEnd = *successor.StartLSN
+ }
+
+ return chain_view.LSNRange{Start: *full.StartLSN, End: spanEnd}, nil
+}
+
+func selectWalSegmentsFullyContainedInSpan(
+ database *gorm.DB,
+ databaseID uuid.UUID,
+ timelineID int,
+ span chain_view.LSNRange,
+) *gorm.DB {
+ return database.Where(
+ "database_id = ? AND timeline_id = ? AND start_lsn >= ?::pg_lsn AND end_lsn <= ?::pg_lsn",
+ databaseID,
+ timelineID,
+ span.Start.String(),
+ span.End.String(),
+ )
+}
+
// deleteWalInSpanBudgeted removes only WAL rows lying fully inside the span. The
// upper bound is the segment's end_lsn, not its start_lsn: a span ends at the
// next chain's start_lsn, which sits inside a segment, and that straddling
@@ -656,11 +848,7 @@ func (s *PhysicalBackupService) deleteWalInSpanBudgeted(
for {
var segments []*physical_models.PhysicalWalSegment
- batchQuery := tx.
- Where(
- "database_id = ? AND timeline_id = ? AND start_lsn >= ?::pg_lsn AND end_lsn <= ?::pg_lsn",
- databaseID, timelineID, span.Start.String(), span.End.String(),
- ).
+ batchQuery := selectWalSegmentsFullyContainedInSpan(tx, databaseID, timelineID, span).
Order("start_lsn ASC").
Limit(maxWalDeleteBatchRows)
@@ -814,17 +1002,12 @@ func (s *PhysicalBackupService) deleteOrphanedHistoryFiles(
tx *gorm.DB,
full *physical_models.PhysicalFullBackup,
) (int, error) {
- var survivingFulls int64
-
- if err := tx.
- Model(&physical_models.PhysicalFullBackup{}).
- Where("database_id = ? AND timeline_id = ? AND status = ? AND id != ?",
- full.DatabaseID, full.TimelineID, physical_enums.PhysicalBackupStatusCompleted, full.ID).
- Count(&survivingFulls).Error; err != nil {
+ hasSurvivingFull, err := hasSurvivingCompletedFullOnTimeline(tx, full)
+ if err != nil {
return 0, err
}
- if survivingFulls > 0 {
+ if hasSurvivingFull {
return 0, nil
}
diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.go
index 315ba9a1..01a3a756 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go
@@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "databasus-backend/internal/features/backups/backups/core/physical/chain_view"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
@@ -21,6 +22,7 @@ import (
users_testing "databasus-backend/internal/features/users/testing"
workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
+ "databasus-backend/internal/storage"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/walmath"
@@ -214,6 +216,272 @@ func Test_DeleteFull_WhenSuccessorStartsMidSegment_KeepsSuccessorBoundarySegment
"its stored object must survive too")
}
+func Test_DeleteFull_WhenPredecessorSurvives_KeepsSharedWalRestorable(t *testing.T) {
+ prereqs := createServiceTestPrereqs(t)
+ databaseID := prereqs.database.ID
+ storageID := prereqs.storage.ID
+ service := physical_service.GetPhysicalBackupService()
+ base := time.Now().UTC().Add(-time.Hour)
+
+ predecessorModel := physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(0), lsn(1))
+ predecessorModel.CreatedAt = base
+ predecessorModel.CompletedAt = new(base.Add(time.Minute))
+ predecessor := physical_testing.CreateTestFullBackup(t, predecessorModel)
+
+ successorModel := physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(2), lsn(3))
+ successorModel.CreatedAt = base.Add(5 * time.Minute)
+ successorModel.CompletedAt = new(base.Add(10 * time.Minute))
+ successor := physical_testing.CreateTestFullBackup(t, successorModel)
+ saveObject(t, prereqs.storage, *successor.FileName)
+
+ walSegments := []*physical_models.PhysicalWalSegment{
+ physical_testing.NewTestWalSegment(
+ databaseID, storageID, 1, "000000010000000000000001", lsn(1), lsn(2)),
+ physical_testing.NewTestWalSegment(
+ databaseID, storageID, 1, "000000010000000000000002", lsn(2), lsn(3)),
+ physical_testing.NewTestWalSegment(
+ databaseID, storageID, 1, "000000010000000000000003", lsn(3), lsn(4)),
+ }
+ for walIndex, walSegment := range walSegments {
+ walSegment.ReceivedAt = base.Add(time.Duration(4+walIndex*3) * time.Minute)
+ physical_testing.CreateTestWalSegment(t, walSegment)
+ saveObject(t, prereqs.storage, *walSegment.FileName)
+ }
+
+ preview, err := service.GetDependentsSummary(successor.ID)
+ require.NoError(t, err)
+ assert.Zero(t, preview.WalSegments)
+
+ deleted, err := service.DeleteFull(t.Context(), successor.ID, 1_000_000)
+ require.NoError(t, err)
+ assert.True(t, deleted.ChainFullyDeleted)
+ assert.Zero(t, deleted.WalSegments)
+
+ for _, walSegment := range walSegments {
+ reloadedSegment, reloadErr := physical_repositories.GetWalSegmentRepository().FindByID(walSegment.ID)
+ require.NoError(t, reloadErr)
+ assert.NotNil(t, reloadedSegment)
+ assert.True(t, objectExists(t, prereqs.storage, *walSegment.FileName))
+ }
+
+ targetTime := base.Add(7 * time.Minute)
+ restoreSet, err := chain_view.GetChainViewService().ResolveRestoreSet(databaseID, &targetTime)
+ require.NoError(t, err)
+ assert.Equal(t, predecessor.ID, restoreSet.RootFull.ID)
+ require.Len(t, restoreSet.WalSegments, len(walSegments))
+}
+
+func Test_CascadeDelete_WhenFullClaimCommitsConcurrently_KeepsWal(t *testing.T) {
+ testCases := []struct {
+ name string
+ keepFull bool
+ }{
+ {name: "delete full", keepFull: false},
+ {name: "keep full", keepFull: true},
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ prereqs := createServiceTestPrereqs(t)
+ physicalBackupService := physical_service.GetPhysicalBackupService()
+ rootFull := physical_testing.CreateTestFullBackup(t,
+ physical_testing.NewTestCompletedFullBackup(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ lsn(0),
+ lsn(1),
+ ))
+ walSegment := physical_testing.CreateTestWalSegment(t,
+ physical_testing.NewTestWalSegment(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ "000000010000000000000001",
+ lsn(1),
+ lsn(2),
+ ))
+ saveObject(t, prereqs.storage, *walSegment.FileName)
+
+ claimTransaction := storage.GetDb().Begin()
+ require.NoError(t, claimTransaction.Error)
+ t.Cleanup(func() { _ = claimTransaction.Rollback().Error })
+ require.NoError(t, physical_repositories.AcquireBackupAndOrphanCleanupLock(
+ claimTransaction,
+ prereqs.database.ID,
+ ))
+
+ type cascadeDeletionOutcome struct {
+ summary physical_service.DeletedSummary
+ err error
+ }
+ deletionOutcomes := make(chan cascadeDeletionOutcome, 1)
+ go func() {
+ var summary physical_service.DeletedSummary
+ var err error
+ if testCase.keepFull {
+ summary, err = physicalBackupService.DeleteChainDependentsKeepFull(
+ t.Context(),
+ rootFull.ID,
+ 1_000_000,
+ )
+ } else {
+ summary, err = physicalBackupService.DeleteFull(
+ t.Context(),
+ rootFull.ID,
+ 1_000_000,
+ )
+ }
+ deletionOutcomes <- cascadeDeletionOutcome{summary: summary, err: err}
+ }()
+
+ select {
+ case <-deletionOutcomes:
+ t.Fatal("cascade deletion must wait while FULL claim creation holds the database lock")
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ activeFull := physical_testing.NewTestCompletedFullBackup(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ lsn(4),
+ lsn(5),
+ )
+ activeFull.Status = physical_enums.PhysicalBackupStatusInProgress
+ activeFull.StartLSN = nil
+ activeFull.StopLSN = nil
+ activeFull.FileName = nil
+ activeFull.CompletedAt = nil
+ require.NoError(t, claimTransaction.Create(activeFull).Error)
+
+ isClaimed, err := physical_repositories.GetInFlightBackupRepository().Claim(
+ claimTransaction,
+ physical_repositories.ClaimSpec{
+ DatabaseID: prereqs.database.ID,
+ BackupType: physical_enums.PhysicalBackupTypeFull,
+ BackupID: activeFull.ID,
+ },
+ )
+ require.NoError(t, err)
+ require.True(t, isClaimed)
+ require.NoError(t, claimTransaction.Commit().Error)
+
+ deletionOutcome := <-deletionOutcomes
+ require.NoError(t, deletionOutcome.err)
+ assert.Zero(t, deletionOutcome.summary.WalSegments)
+
+ reloadedWalSegment, err := physical_repositories.GetWalSegmentRepository().FindByID(walSegment.ID)
+ require.NoError(t, err)
+ assert.NotNil(t, reloadedWalSegment)
+ assert.True(t, objectExists(t, prereqs.storage, *walSegment.FileName))
+
+ reloadedFull, err := physical_repositories.GetFullBackupRepository().FindByID(rootFull.ID)
+ require.NoError(t, err)
+ if testCase.keepFull {
+ assert.NotNil(t, reloadedFull)
+ } else {
+ assert.Nil(t, reloadedFull)
+ }
+ })
+ }
+}
+
+func Test_GetDependentsSummary_WhenFullClaimIsActive_MatchesDeleteSet(t *testing.T) {
+ prereqs := createServiceTestPrereqs(t)
+ physicalBackupService := physical_service.GetPhysicalBackupService()
+ rootFull := physical_testing.CreateTestFullBackup(t,
+ physical_testing.NewTestCompletedFullBackup(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ lsn(0),
+ lsn(1),
+ ))
+ walSegment := physical_testing.CreateTestWalSegment(t,
+ physical_testing.NewTestWalSegment(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ "000000010000000000000001",
+ lsn(1),
+ lsn(2),
+ ))
+ historyFile := physical_testing.CreateTestWalHistoryFile(t,
+ physical_testing.NewTestWalHistoryFile(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ ))
+ physical_testing.CreateTestInFlightClaim(
+ t,
+ prereqs.database.ID,
+ uuid.New(),
+ physical_enums.PhysicalBackupTypeFull,
+ )
+
+ deletionPreview, err := physicalBackupService.GetDependentsSummary(rootFull.ID)
+ require.NoError(t, err)
+ assert.Zero(t, deletionPreview.WalSegments)
+ assert.Zero(t, deletionPreview.HistoryFiles)
+
+ deletionSummary, err := physicalBackupService.DeleteFull(t.Context(), rootFull.ID, 1_000_000)
+ require.NoError(t, err)
+ assert.Zero(t, deletionSummary.WalSegments)
+ assert.Zero(t, deletionSummary.HistoryFiles)
+ assert.Equal(t, deletionPreview.WalSegments, deletionSummary.WalSegments)
+ assert.Equal(t, deletionPreview.HistoryFiles, deletionSummary.HistoryFiles)
+
+ reloadedWalSegment, err := physical_repositories.GetWalSegmentRepository().FindByID(walSegment.ID)
+ require.NoError(t, err)
+ assert.NotNil(t, reloadedWalSegment)
+ reloadedHistoryFile, err := physical_repositories.GetWalHistoryRepository().FindByDatabaseTimeline(
+ prereqs.database.ID,
+ historyFile.TimelineID,
+ )
+ require.NoError(t, err)
+ assert.NotNil(t, reloadedHistoryFile)
+}
+
+func Test_GetDependentsSummary_WhenUpperSegmentCrossesOwnershipBoundary_MatchesDeleteSet(t *testing.T) {
+ prereqs := createServiceTestPrereqs(t)
+ databaseID := prereqs.database.ID
+ storageID := prereqs.storage.ID
+ service := physical_service.GetPhysicalBackupService()
+
+ fullModel := physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(0), lsn(1))
+ fullModel.BackupSizeMb = new(50.0)
+ full := physical_testing.CreateTestFullBackup(t, fullModel)
+ physical_testing.CreateTestFullBackup(t, physical_testing.NewTestCompletedFullBackup(
+ databaseID,
+ storageID,
+ 1,
+ lsn(4)+firstRecordOffset,
+ lsn(4)+firstRecordOffset+fullLSNSpan,
+ ))
+
+ ownedSegment := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(
+ databaseID, storageID, 1, "000000010000000000000001", lsn(1), lsn(2)))
+ boundarySegment := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(
+ databaseID, storageID, 1, "000000010000000000000004", lsn(4), lsn(5)))
+ saveObject(t, prereqs.storage, *full.FileName)
+ saveObject(t, prereqs.storage, *ownedSegment.FileName)
+ saveObject(t, prereqs.storage, *boundarySegment.FileName)
+
+ preview, err := service.GetDependentsSummary(full.ID)
+ require.NoError(t, err)
+ assert.Equal(t, 1, preview.WalSegments)
+ assert.Equal(t, 66.0, preview.TotalSizeMB)
+
+ deleted, err := service.DeleteFull(t.Context(), full.ID, 1_000_000)
+ require.NoError(t, err)
+ assert.Equal(t, preview.WalSegments, deleted.WalSegments)
+
+ remainingBoundary, err := physical_repositories.GetWalSegmentRepository().FindByID(boundarySegment.ID)
+ require.NoError(t, err)
+ assert.NotNil(t, remainingBoundary)
+}
+
func Test_GetDependentsSummary_ReturnsCountsWithoutDeleting(t *testing.T) {
prereqs := createServiceTestPrereqs(t)
databaseID := prereqs.database.ID