Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Re-anchor PostgreSQL physical backups automatically after failover

Rostislav Dugin9a2b269248fb
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper.go b/backend/internal/features/backups/backups/backuping/physical/backuper.go
index 7e23da18..376b916e 100644
--- a/backend/internal/features/backups/backups/backuping/physical/backuper.go
+++ b/backend/internal/features/backups/backups/backuping/physical/backuper.go
@@ -434,13 +434,13 @@ func (b *PhysicalBackuper) persistFullResult(
fullBackup.ManifestEncryptionSalt = nilOrPtr(backupResult.ManifestEncryptionSalt)
fullBackup.ManifestEncryptionIV = nilOrPtr(backupResult.ManifestEncryptionIV)
- completed := backupResult.CompletedAt
- if completed.IsZero() {
- completed = time.Now().UTC()
- }
+ }
- fullBackup.CompletedAt = &completed
+ completedAt := backupResult.CompletedAt
+ if completedAt.IsZero() {
+ completedAt = time.Now().UTC()
}
+ fullBackup.CompletedAt = &completedAt
return b.saveTerminalResultIfInProgress(
fullBackup.DatabaseID,
@@ -484,13 +484,13 @@ func (b *PhysicalBackuper) persistIncrResult(
incrBackup.ManifestEncryptionSalt = nilOrPtr(backupResult.ManifestEncryptionSalt)
incrBackup.ManifestEncryptionIV = nilOrPtr(backupResult.ManifestEncryptionIV)
- completed := backupResult.CompletedAt
- if completed.IsZero() {
- completed = time.Now().UTC()
- }
+ }
- incrBackup.CompletedAt = &completed
+ completedAt := backupResult.CompletedAt
+ if completedAt.IsZero() {
+ completedAt = time.Now().UTC()
}
+ incrBackup.CompletedAt = &completedAt
return b.saveTerminalResultIfInProgress(
incrBackup.DatabaseID,
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go b/backend/internal/features/backups/backups/backuping/physical/backuper_test.go
index 116d5f0d..77de91fc 100644
--- a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/backuper_test.go
@@ -208,7 +208,7 @@ func Test_PersistFullResult_WhenClusterSizeUnmeasured_LeavesRawSizeMbNil(t *test
assert.Nil(t, persisted.RawSizeMb)
}
-func Test_PersistFullResult_WhenErrorStatus_SkipsCompletionFieldsAndReleasesInFlight(t *testing.T) {
+func Test_PersistFullResult_WhenErrorStatus_RecordsTerminalTimeAndReleasesInFlight(t *testing.T) {
prereqs := seedBackupPrereqs(t)
backuper := CreateTestPhysicalBackuper(nil)
@@ -231,11 +231,29 @@ func Test_PersistFullResult_WhenErrorStatus_SkipsCompletionFieldsAndReleasesInFl
require.NotNil(t, persisted.ErrorReason)
assert.Equal(t, physical_enums.PhysicalBackupErrorPgBasebackupFailed, *persisted.ErrorReason)
assert.Nil(t, persisted.ManifestFileName, "completion-only fields must not be copied on a non-COMPLETED result")
- assert.Nil(t, persisted.CompletedAt)
+ require.NotNil(t, persisted.CompletedAt)
assertInFlightReleased(t, prereqs.DB.ID)
}
+func Test_PersistIncrementalResult_WhenErrorStatus_RecordsTerminalTime(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ backuper := CreateTestPhysicalBackuper(nil)
+ rootFull := seedCompletedRootFull(t, prereqs)
+ incremental := seedInProgressIncr(t, prereqs, rootFull.ID)
+ claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeIncremental, incremental.ID)
+
+ result := postgresql_executor.PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusChainBroken,
+ ErrorReason: new(physical_enums.PhysicalBackupErrorTimelineSwitchDetected),
+ }
+ require.NoError(t, backuper.persistIncrResult(incremental, result))
+
+ persistedIncremental, err := physical_repositories.GetIncrementalBackupRepository().FindByID(incremental.ID)
+ require.NoError(t, err)
+ require.NotNil(t, persistedIncremental.CompletedAt)
+}
+
func Test_PersistIncrementalResult_WhenCompleted_CopiesFieldsAndReleasesInFlight(t *testing.T) {
prereqs := seedBackupPrereqs(t)
backuper := CreateTestPhysicalBackuper(nil)
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner.go b/backend/internal/features/backups/backups/backuping/physical/cleaner.go
index 679d6ade..039cb249 100644
--- a/backend/internal/features/backups/backups/backuping/physical/cleaner.go
+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner.go
@@ -526,8 +526,14 @@ func (c *PhysicalBackupCleaner) cleanOrphanWalForDatabase(
// never overlap, so no chain-covered WAL is ever caught.
span := chain_view.LSNRange{Start: segment.StartLSN, End: segment.EndLSN}
- rows, mb, err := c.physicalBackupService.DeleteWalSegmentsInSpan(
- ctx, databaseID, segment.TimelineID, span, budget,
+ rows, mb, err := c.physicalBackupService.DeleteOrphanWalSegmentsInSpan(
+ ctx,
+ physical_service.DeleteOrphanWalSegmentsSpec{
+ DatabaseID: databaseID,
+ TimelineID: segment.TimelineID,
+ Span: span,
+ WalByteBudgetMB: budget,
+ },
)
if err != nil {
logger.ErrorContext(ctx, "failed to delete orphan wal segment",
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
index 003ad500..e322ad55 100644
--- a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go
@@ -266,6 +266,37 @@ func Test_CleanOrphanWalForDatabase_WhenWalCoveredByChain_KeepsIt(t *testing.T)
assert.True(t, walExists(t, covered.ID), "chain-covered WAL must never be caught by the orphan pass")
}
+func Test_CleanOrphanWalForDatabase_WhenFirstFullInProgress_KeepsWalArchivedDuringBackup(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+
+ inProgressFull := physical_testing.CreateTestFullBackup(t, physical_testing.NewTestInProgressFullBackup(
+ prereqs.DB.ID,
+ prereqs.Storage.ID,
+ 1,
+ ))
+ physical_testing.CreateTestInFlightClaim(
+ t,
+ prereqs.DB.ID,
+ inProgressFull.ID,
+ physical_enums.PhysicalBackupTypeFull,
+ )
+
+ walArchivedDuringBackup := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(
+ prereqs.DB.ID,
+ prereqs.Storage.ID,
+ 1,
+ "000000010000000000000001",
+ testLSN(1),
+ testLSN(2),
+ ))
+
+ cleaner := CreateTestPhysicalCleaner()
+ cleaner.cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), prereqs.DB.ID)
+
+ assert.True(t, walExists(t, walArchivedDuringBackup.ID),
+ "WAL archived while the first full is running must remain available when the full completes")
+}
+
// A FULL taken right after a segment switch reports a start_lsn at
// firstRecordOffset. The segment's own bounds come from its filename and are
// file-aligned, so it starts below the FULL and must still be kept: it carries the
diff --git a/backend/internal/features/backups/backups/backuping/physical/errors.go b/backend/internal/features/backups/backups/backuping/physical/errors.go
new file mode 100644
index 00000000..07e409cb
--- /dev/null
+++ b/backend/internal/features/backups/backups/backuping/physical/errors.go
@@ -0,0 +1,5 @@
+package backuping_physical
+
+import "errors"
+
+var errBackupDecisionSuperseded = errors.New("physical backup decision superseded")
diff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler.go b/backend/internal/features/backups/backups/backuping/physical/scheduler.go
index b901f6ff..3dc669e8 100644
--- a/backend/internal/features/backups/backups/backuping/physical/scheduler.go
+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler.go
@@ -2,6 +2,7 @@ package backuping_physical
import (
"context"
+ "errors"
"fmt"
"log/slog"
"slices"
@@ -156,6 +157,12 @@ type backupDecision struct {
incrParentIncrID *uuid.UUID
forceFullRequestedAt *time.Time
forceIncrementalRequestedAt *time.Time
+ immediateTrigger *backupTrigger
+}
+
+type backupTrigger struct {
+ kind physical_enums.PhysicalBackupType
+ backupID uuid.UUID
}
// decideBackupKind picks FULL or INCR purely from catalog state and cadence, with
@@ -181,10 +188,16 @@ func (s *PhysicalBackupsScheduler) decideBackupKind(
}, true
}
- if backupConfig.ForceIncrementalRequestedAt != nil {
- if decision, ok := s.decideForcedIncremental(logger, backupConfig); ok {
- return decision, true
- }
+ hasUnresolvedMismatch, err := hasUnresolvedSystemIdentifierMismatch(storage.GetDb(), backupConfig.DatabaseID)
+ if err != nil {
+ logger.Error("failed to inspect system identifier mismatch state", "error", err)
+
+ return backupDecision{}, false
+ }
+ if hasUnresolvedMismatch {
+ logger.Debug("automatic scheduling suppressed after system identifier mismatch")
+
+ return backupDecision{}, false
}
lastIncr, err := s.incrRepo.FindLastByDatabase(backupConfig.DatabaseID)
@@ -194,6 +207,16 @@ func (s *PhysicalBackupsScheduler) decideBackupKind(
return backupDecision{}, false
}
+ if decision, ok := decideImmediateReplacementFull(lastFull, lastIncr); ok {
+ return decision, true
+ }
+
+ if backupConfig.ForceIncrementalRequestedAt != nil {
+ if decision, ok := s.decideForcedIncremental(logger, backupConfig); ok {
+ return decision, true
+ }
+ }
+
if backupConfig.FullBackupInterval.ShouldTriggerBackup(now, createdAtOrNil(lastFull)) {
return backupDecision{
kind: physical_enums.PhysicalBackupTypeFull,
@@ -372,6 +395,10 @@ func (s *PhysicalBackupsScheduler) claimAndInsert(
claimed := false
txErr := storage.GetDb().Transaction(func(tx *gorm.DB) error {
+ if err := physical_repositories.AcquireBackupAndOrphanCleanupLock(tx, backupConfig.DatabaseID); err != nil {
+ return err
+ }
+
ok, claimErr := s.inFlightRepo.Claim(tx, physical_repositories.ClaimSpec{
DatabaseID: backupConfig.DatabaseID,
BackupType: decision.kind,
@@ -385,6 +412,30 @@ func (s *PhysicalBackupsScheduler) claimAndInsert(
return nil
}
+ if decision.forceFullRequestedAt == nil {
+ hasUnresolvedMismatch, err := hasUnresolvedSystemIdentifierMismatch(tx, backupConfig.DatabaseID)
+ if err != nil {
+ return err
+ }
+ if hasUnresolvedMismatch {
+ return errBackupDecisionSuperseded
+ }
+ }
+
+ if decision.immediateTrigger != nil {
+ isCurrent, err := isImmediateReplacementTriggerCurrent(
+ tx,
+ backupConfig.DatabaseID,
+ *decision.immediateTrigger,
+ )
+ if err != nil {
+ return err
+ }
+ if !isCurrent {
+ return errBackupDecisionSuperseded
+ }
+ }
+
claimed = true
if decision.kind == physical_enums.PhysicalBackupTypeFull {
@@ -407,6 +458,9 @@ func (s *PhysicalBackupsScheduler) claimAndInsert(
CreatedAt: now,
}).Error
})
+ if errors.Is(txErr, errBackupDecisionSuperseded) {
+ return false, nil
+ }
if txErr != nil {
return false, txErr
}
@@ -414,6 +468,131 @@ func (s *PhysicalBackupsScheduler) claimAndInsert(
return claimed, nil
}
+func decideImmediateReplacementFull(
+ lastFull *physical_models.PhysicalFullBackup,
+ lastIncr *physical_models.PhysicalIncrementalBackup,
+) (backupDecision, bool) {
+ if lastFull != nil && lastFull.Status == physical_enums.PhysicalBackupStatusError &&
+ hasFullBackupErrorReason(lastFull, physical_enums.PhysicalBackupErrorFailoverDuringBackup) {
+ return backupDecision{
+ kind: physical_enums.PhysicalBackupTypeFull,
+ reason: "replacing FULL interrupted by failover",
+ immediateTrigger: &backupTrigger{
+ kind: physical_enums.PhysicalBackupTypeFull,
+ backupID: lastFull.ID,
+ },
+ }, true
+ }
+
+ if !hasIncrementalBackupErrorReason(lastIncr, physical_enums.PhysicalBackupErrorTimelineSwitchDetected) ||
+ lastIncr.Status != physical_enums.PhysicalBackupStatusChainBroken ||
+ lastIncr.CompletedAt == nil {
+ return backupDecision{}, false
+ }
+
+ if lastFull != nil && lastFull.CreatedAt.After(*lastIncr.CompletedAt) {
+ return backupDecision{}, false
+ }
+
+ return backupDecision{
+ kind: physical_enums.PhysicalBackupTypeFull,
+ reason: "re-anchoring after timeline switch",
+ immediateTrigger: &backupTrigger{
+ kind: physical_enums.PhysicalBackupTypeIncremental,
+ backupID: lastIncr.ID,
+ },
+ }, true
+}
+
+func hasFullBackupErrorReason(
+ full *physical_models.PhysicalFullBackup,
+ reason physical_enums.PhysicalBackupErrorReason,
+) bool {
+ return full != nil && full.ErrorReason != nil && *full.ErrorReason == reason
+}
+
+func hasIncrementalBackupErrorReason(
+ incremental *physical_models.PhysicalIncrementalBackup,
+ reason physical_enums.PhysicalBackupErrorReason,
+) bool {
+ return incremental != nil && incremental.ErrorReason != nil && *incremental.ErrorReason == reason
+}
+
+func hasUnresolvedSystemIdentifierMismatch(db *gorm.DB, databaseID uuid.UUID) (bool, error) {
+ var state struct {
+ LatestMismatchAt *time.Time
+ LatestCompletedFullAt *time.Time
+ }
+
+ err := db.Raw(`
+ SELECT
+ (
+ SELECT MAX(terminal_at)
+ FROM (
+ SELECT COALESCE(completed_at, created_at) AS terminal_at FROM physical_full_backups
+ WHERE database_id = ? AND error_reason = ?
+ UNION ALL
+ SELECT COALESCE(completed_at, created_at) AS terminal_at FROM physical_incremental_backups
+ WHERE database_id = ? AND error_reason = ?
+ ) mismatches
+ ) AS latest_mismatch_at,
+ (
+ SELECT MAX(completed_at) FROM physical_full_backups
+ WHERE database_id = ? AND status = ?
+ ) AS latest_completed_full_at
+ `,
+ databaseID, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch,
+ databaseID, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch,
+ databaseID, physical_enums.PhysicalBackupStatusCompleted,
+ ).Scan(&state).Error
+ if err != nil {
+ return false, err
+ }
+
+ return state.LatestMismatchAt != nil &&
+ (state.LatestCompletedFullAt == nil || !state.LatestCompletedFullAt.After(*state.LatestMismatchAt)), nil
+}
+
+func isImmediateReplacementTriggerCurrent(
+ tx *gorm.DB,
+ databaseID uuid.UUID,
+ trigger backupTrigger,
+) (bool, error) {
+ hasUnresolvedMismatch, err := hasUnresolvedSystemIdentifierMismatch(tx, databaseID)
+ if err != nil || hasUnresolvedMismatch {
+ return false, err
+ }
+
+ var lastFull physical_models.PhysicalFullBackup
+ fullErr := tx.Where("database_id = ?", databaseID).Order("created_at DESC").First(&lastFull).Error
+ if fullErr != nil && !errors.Is(fullErr, gorm.ErrRecordNotFound) {
+ return false, fullErr
+ }
+
+ if trigger.kind == physical_enums.PhysicalBackupTypeFull {
+ return fullErr == nil && lastFull.ID == trigger.backupID &&
+ lastFull.Status == physical_enums.PhysicalBackupStatusError &&
+ hasFullBackupErrorReason(&lastFull, physical_enums.PhysicalBackupErrorFailoverDuringBackup), nil
+ }
+
+ var lastIncr physical_models.PhysicalIncrementalBackup
+ if err := tx.Where("database_id = ?", databaseID).Order("created_at DESC").First(&lastIncr).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return false, nil
+ }
+
+ return false, err
+ }
+
+ if lastIncr.ID != trigger.backupID || lastIncr.CompletedAt == nil ||
+ lastIncr.Status != physical_enums.PhysicalBackupStatusChainBroken ||
+ !hasIncrementalBackupErrorReason(&lastIncr, physical_enums.PhysicalBackupErrorTimelineSwitchDetected) {
+ return false, nil
+ }
+
+ return errors.Is(fullErr, gorm.ErrRecordNotFound) || !lastFull.CreatedAt.After(*lastIncr.CompletedAt), nil
+}
+
// recoverInFlightBackupsOnRestart reconciles the IN_PROGRESS backups the DB still
// holds when this instance (re)started. In single-instance mode the goroutine that
// was running each backup died with the previous process, so every in-flight claim
diff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go b/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go
index cad6efe2..d1f385dc 100644
--- a/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go
@@ -37,8 +37,7 @@ func makeIncrementalDue(prereqs *backupPrereqs) {
}
// seedFullWithStatusAndAge persists a FULL of the given status, aged by ageHours,
-// so the cadence / chain-state decision tests are deterministic. Non-COMPLETED
-// fulls clear CompletedAt — only COMPLETED fulls anchor an extendable chain.
+// so the cadence and chain-state decision tests are deterministic.
func seedFullWithStatusAndAge(
t *testing.T,
prereqs *backupPrereqs,
@@ -53,10 +52,6 @@ func seedFullWithStatusAndAge(
full.CreatedAt = time.Now().UTC().Add(-time.Duration(ageHours) * time.Hour)
full.Status = status
- if status != physical_enums.PhysicalBackupStatusCompleted {
- full.CompletedAt = nil
- }
-
return physical_testing.CreateTestFullBackup(t, full)
}
@@ -78,11 +73,244 @@ func seedIncrWithStatusAndAge(
incr.CreatedAt = time.Now().UTC().Add(-time.Duration(ageHours) * time.Hour)
incr.Status = status
- if status != physical_enums.PhysicalBackupStatusCompleted {
- incr.CompletedAt = nil
+ return physical_testing.CreateTestIncrementalBackup(t, incr)
+}
+
+func setFullErrorReason(
+ t *testing.T,
+ full *physical_models.PhysicalFullBackup,
+ reason physical_enums.PhysicalBackupErrorReason,
+) {
+ t.Helper()
+
+ full.ErrorReason = &reason
+ require.NoError(t, storage.GetDb().Save(full).Error)
+}
+
+func setIncrementalErrorReason(
+ t *testing.T,
+ incremental *physical_models.PhysicalIncrementalBackup,
+ reason physical_enums.PhysicalBackupErrorReason,
+) {
+ t.Helper()
+
+ incremental.ErrorReason = &reason
+ require.NoError(t, storage.GetDb().Save(incremental).Error)
+}
+
+func Test_DecideBackupKind_WhenIncrementalDetectsTimelineSwitch_SchedulesImmediateFull(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ makeIncrementalDue(prereqs)
+ rootFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusCompleted, 1, 1)
+ brokenIncremental := seedIncrWithStatusAndAge(
+ t, prereqs, rootFull.ID, physical_enums.PhysicalBackupStatusChainBroken, 2, 0,
+ )
+ setIncrementalErrorReason(t, brokenIncremental, physical_enums.PhysicalBackupErrorTimelineSwitchDetected)
+
+ decision, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(
+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,
+ )
+
+ require.True(t, canSchedule)
+ require.Equal(t, physical_enums.PhysicalBackupTypeFull, decision.kind)
+ require.NotNil(t, decision.immediateTrigger)
+ require.Equal(t, brokenIncremental.ID, decision.immediateTrigger.backupID)
+}
+
+func Test_DecideBackupKind_WhenFullFailsDuringFailover_SchedulesImmediateFull(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ prereqs.Config.FullBackupInterval = intervals.Interval{Type: intervals.IntervalWeekly}
+ failedFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusError, 1, 0)
+ setFullErrorReason(t, failedFull, physical_enums.PhysicalBackupErrorFailoverDuringBackup)
+
+ decision, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(
+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,
+ )
+
+ require.True(t, canSchedule)
+ require.NotNil(t, decision.immediateTrigger)
+ require.Equal(t, failedFull.ID, decision.immediateTrigger.backupID)
+}
+
+func Test_ClaimAndInsert_WhenImmediateTriggerWasReplaced_RejectsStaleDecision(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ makeIncrementalDue(prereqs)
+ rootFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusCompleted, 1, 2)
+ brokenIncremental := seedIncrWithStatusAndAge(
+ t, prereqs, rootFull.ID, physical_enums.PhysicalBackupStatusChainBroken, 2, 1,
+ )
+ setIncrementalErrorReason(t, brokenIncremental, physical_enums.PhysicalBackupErrorTimelineSwitchDetected)
+
+ scheduler := CreateTestPhysicalScheduler()
+ staleDecision, canSchedule := scheduler.decideBackupKind(logger.GetLogger(), time.Now().UTC(), prereqs.Config)
+ require.True(t, canSchedule)
+
+ replacementFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusError, 3, 0)
+ replacementFull.CreatedAt = brokenIncremental.CompletedAt.Add(time.Second)
+ require.NoError(t, storage.GetDb().Save(replacementFull).Error)
+
+ claimed, err := scheduler.claimAndInsert(prereqs.Config, uuid.New(), staleDecision)
+
+ require.NoError(t, err)
+ require.False(t, claimed)
+}
+
+func Test_ClaimAndInsert_WhenMismatchAppearsAfterAutomaticDecision_RejectsStaleDecision(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ scheduler := CreateTestPhysicalScheduler()
+ automaticDecision, canSchedule := scheduler.decideBackupKind(
+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,
+ )
+ require.True(t, canSchedule)
+
+ mismatchedFull := seedFullWithStatusAndAge(
+ t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 0,
+ )
+ setFullErrorReason(t, mismatchedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)
+
+ claimed, err := scheduler.claimAndInsert(prereqs.Config, uuid.New(), automaticDecision)
+
+ require.NoError(t, err)
+ require.False(t, claimed)
+}
+
+func Test_ClaimAndInsert_WhenMismatchCommitsWhileClaimWaits_RejectsStaleAutomaticDecision(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ scheduler := CreateTestPhysicalScheduler()
+ automaticDecision, canSchedule := scheduler.decideBackupKind(
+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,
+ )
+ require.True(t, canSchedule)
+
+ terminalTransaction := storage.GetDb().Begin()
+ require.NoError(t, terminalTransaction.Error)
+ t.Cleanup(func() { _ = terminalTransaction.Rollback().Error })
+
+ mismatchedFull := physical_testing.NewTestCompletedFullBackup(
+ prereqs.DB.ID,
+ prereqs.Storage.ID,
+ 1,
+ testLSN(1),
+ testLSN(2),
+ )
+ mismatchReason := physical_enums.PhysicalBackupErrorSystemIdentifierMismatch
+ mismatchedFull.Status = physical_enums.PhysicalBackupStatusChainBroken
+ mismatchedFull.ErrorReason = &mismatchReason
+ require.NoError(t, terminalTransaction.Create(mismatchedFull).Error)
+
+ isOccupied, err := scheduler.inFlightRepo.Claim(terminalTransaction, physical_repositories.ClaimSpec{
+ DatabaseID: prereqs.DB.ID,
+ BackupType: physical_enums.PhysicalBackupTypeFull,
+ BackupID: uuid.New(),
+ })
+ require.NoError(t, err)
+ require.True(t, isOccupied)
+
+ type claimResult struct {
+ isClaimed bool
+ err error
}
+ claimResults := make(chan claimResult, 1)
+ go func() {
+ isClaimed, claimErr := scheduler.claimAndInsert(
+ prereqs.Config,
+ uuid.New(),
+ automaticDecision,
+ )
+ claimResults <- claimResult{isClaimed: isClaimed, err: claimErr}
+ }()
- return physical_testing.CreateTestIncrementalBackup(t, incr)
+ select {
+ case <-claimResults:
+ t.Fatal("automatic claim must wait for the pending terminal transaction")
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ require.NoError(t, terminalTransaction.Delete(
+ &physical_models.PhysicalInFlightBackup{},
+ "database_id = ?",
+ prereqs.DB.ID,
+ ).Error)
+ require.NoError(t, terminalTransaction.Commit().Error)
+
+ result := <-claimResults
+ require.NoError(t, result.err)
+ require.False(t, result.isClaimed)
+
+ inFlightBackup, err := scheduler.inFlightRepo.FindByDatabaseID(prereqs.DB.ID)
+ require.NoError(t, err)
+ require.Nil(t, inFlightBackup)
+
+ inProgressFulls, err := scheduler.fullRepo.FindAllInProgress()
+ require.NoError(t, err)
+ for _, full := range inProgressFulls {
+ require.NotEqual(t, prereqs.DB.ID, full.DatabaseID)
+ }
+}
+
+func Test_DecideBackupKind_WhenSystemIdentifierMismatchUnresolved_SuppressesAutomaticBackup(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ failedFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 1)
+ setFullErrorReason(t, failedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)
+
+ _, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(
+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,
+ )
+
+ require.False(t, canSchedule)
+}
+
+func Test_DecideBackupKind_WhenSystemIdentifierMismatchUnresolved_AllowsForcedFull(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ failedFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 1)
+ setFullErrorReason(t, failedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)
+ requestedAt := time.Now().UTC()
+ prereqs.Config.ForceFullRequestedAt = &requestedAt
+
+ decision, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(
+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,
+ )
+
+ require.True(t, canSchedule)
+ require.Equal(t, physical_enums.PhysicalBackupTypeFull, decision.kind)
+ require.NotNil(t, decision.forceFullRequestedAt)
+}
+
+func Test_DecideBackupKind_WhenCompletedFullFollowsMismatch_ResumesAutomaticScheduling(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ prereqs.Config.FullBackupInterval = intervals.Interval{Type: intervals.IntervalHourly}
+ failedFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 3)
+ setFullErrorReason(t, failedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)
+ seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusCompleted, 2, 2)
+
+ decision, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(
+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,
+ )
+
+ require.True(t, canSchedule)
+ require.Equal(t, physical_enums.PhysicalBackupTypeFull, decision.kind)
+}
+
+func Test_DecideBackupKind_WhenUnsuccessfulFullFollowsMismatch_KeepsAutomaticSchedulingSuppressed(t *testing.T) {
+ for _, terminalStatus := range []physical_enums.PhysicalBackupStatus{
+ physical_enums.PhysicalBackupStatusError,
+ physical_enums.PhysicalBackupStatusCanceled,
+ } {
+ t.Run(string(terminalStatus), func(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ mismatchedFull := seedFullWithStatusAndAge(
+ t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 3,
+ )
+ setFullErrorReason(t, mismatchedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)
+ seedFullWithStatusAndAge(t, prereqs, terminalStatus, 2, 2)
+
+ _, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(
+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,
+ )
+
+ require.False(t, canSchedule)
+ })
+ }
}
func Test_DecideBackupKind_WhenRecentlyCompletedFullAndIncrNotDue_SchedulesNothing(t *testing.T) {
@@ -408,6 +636,40 @@ func Test_ClaimAndInsert_WhenConcurrentClaimSameDatabase_OnlyOneSucceeds(t *test
assert.Equal(t, 1, inProgressForDB, "only the winning claim inserts a typed row")
}
+func Test_ClaimAndInsert_WhenOrphanCleanupLockIsHeld_WaitsForTransaction(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ scheduler := CreateTestPhysicalScheduler()
+ cleanupTransaction := storage.GetDb().Begin()
+ require.NoError(t, cleanupTransaction.Error)
+ t.Cleanup(func() { _ = cleanupTransaction.Rollback().Error })
+ require.NoError(t, physical_repositories.AcquireBackupAndOrphanCleanupLock(cleanupTransaction, prereqs.DB.ID))
+
+ type claimResult struct {
+ isClaimed bool
+ err error
+ }
+ claimResults := make(chan claimResult, 1)
+ go func() {
+ isClaimed, err := scheduler.claimAndInsert(
+ prereqs.Config,
+ uuid.New(),
+ backupDecision{kind: physical_enums.PhysicalBackupTypeFull},
+ )
+ claimResults <- claimResult{isClaimed: isClaimed, err: err}
+ }()
+
+ select {
+ case <-claimResults:
+ t.Fatal("FULL claim must wait while orphan cleanup holds the database lock")
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ require.NoError(t, cleanupTransaction.Commit().Error)
+ result := <-claimResults
+ require.NoError(t, result.err)
+ require.True(t, result.isClaimed)
+}
+
func Test_EvaluateConfig_WhenInFlightClaimExists_SkipsWithoutNewRow(t *testing.T) {
prereqs := seedBackupPrereqs(t)
scheduler := CreateTestPhysicalScheduler()
diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set.go b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set.go
index 5fcbe723..7cde1ad2 100644
--- a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set.go
+++ b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set.go
@@ -58,7 +58,17 @@ func (s *ChainViewService) ResolveRestoreSet(
incrementals, lastStopLSN, lastBackupTime := includedIncrementals(chain, targetTime)
- walRun, reachableLSN, reachableTime := contiguousWalRun(chain.WalSegments, lastStopLSN, lastBackupTime)
+ restoreWalSegments, err := s.walSegmentRepository.FindByChainSpan(
+ databaseID,
+ chain.RootFull.TimelineID,
+ chain.Span.Start,
+ LSNMax,
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ walRun, reachableLSN, reachableTime := contiguousWalRun(restoreWalSegments, lastStopLSN, lastBackupTime)
// A target later than the furthest point we can actually replay to means a
// WAL gap (or simply missing WAL) sits before it. Refuse loudly; the caller
diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go
index be5e6207..d3c456fb 100644
--- a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go
+++ b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go
@@ -173,6 +173,74 @@ func Test_ResolveRestoreSet_WhenTargetBetweenIncrementals_StopsAtEarlierIncremen
assertWalFilenames(t, set, w2.WalFilename, w3.WalFilename, w4.WalFilename)
}
+func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTarget(t *testing.T) {
+ t.Parallel()
+
+ prereqs := createChainViewTestPrereqs(t)
+ clock := newRestoreClock()
+
+ olderFull := physical_testing.NewTestCompletedFullBackup(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ lsnAt(0)+physical_testing.FirstRecordOffset,
+ lsnAt(1)+physical_testing.FirstRecordOffset,
+ )
+ olderFull.CreatedAt = clock.at(-5)
+ olderFull.CompletedAt = new(clock.at(0))
+ physical_testing.CreateTestFullBackup(t, olderFull)
+
+ newerFull := physical_testing.NewTestCompletedFullBackup(
+ prereqs.database.ID,
+ prereqs.storage.ID,
+ 1,
+ lsnAt(2)+physical_testing.FirstRecordOffset,
+ lsnAt(3)+physical_testing.FirstRecordOffset,
+ )
+ newerFull.CreatedAt = clock.at(5)
+ newerFull.CompletedAt = new(clock.at(10))
+ physical_testing.CreateTestFullBackup(t, newerFull)
+
+ walBeforeNewerFull := seedWalSegment(
+ t,
+ prereqs,
+ "000000010000000000000001",
+ lsnAt(1),
+ lsnAt(2),
+ clock.at(4),
+ )
+ walAtNewerFullStart := seedWalSegment(
+ t,
+ prereqs,
+ "000000010000000000000002",
+ lsnAt(2),
+ lsnAt(3),
+ clock.at(6),
+ )
+ walCoveringTarget := seedWalSegment(
+ t,
+ prereqs,
+ "000000010000000000000003",
+ lsnAt(3),
+ lsnAt(4),
+ clock.at(11),
+ )
+
+ target := clock.at(7)
+
+ set, err := chain_view.GetChainViewService().ResolveRestoreSet(prereqs.database.ID, &target)
+ require.NoError(t, err)
+
+ assert.Equal(t, olderFull.ID, set.RootFull.ID)
+ assertWalFilenames(
+ t,
+ set,
+ walBeforeNewerFull.WalFilename,
+ walAtNewerFullStart.WalFilename,
+ walCoveringTarget.WalFilename,
+ )
+}
+
func Test_ResolveRestoreSet_WhenWalGapBeforeTarget_ReturnsGapError(t *testing.T) {
t.Parallel()
diff --git a/backend/internal/features/backups/backups/core/physical/enums/enums.go b/backend/internal/features/backups/backups/core/physical/enums/enums.go
index f537a4af..7ddc61ea 100644
--- a/backend/internal/features/backups/backups/core/physical/enums/enums.go
+++ b/backend/internal/features/backups/backups/core/physical/enums/enums.go
@@ -38,6 +38,8 @@ const (
PhysicalBackupErrorApplicationRestart PhysicalBackupErrorReason = "APPLICATION_RESTART"
PhysicalBackupErrorSystemIdentifierMismatch PhysicalBackupErrorReason = "SYSTEM_IDENTIFIER_MISMATCH"
PhysicalBackupErrorTimelineRegression PhysicalBackupErrorReason = "TIMELINE_REGRESSION"
+ PhysicalBackupErrorTimelineSwitchDetected PhysicalBackupErrorReason = "TIMELINE_SWITCH_DETECTED"
+ PhysicalBackupErrorFailoverDuringBackup PhysicalBackupErrorReason = "FAILOVER_DURING_BACKUP"
PhysicalBackupErrorManifestCorrupted PhysicalBackupErrorReason = "MANIFEST_CORRUPTED"
PhysicalBackupErrorStartLsnOutsideTimeline PhysicalBackupErrorReason = "START_LSN_OUTSIDE_TIMELINE_RANGE"
diff --git a/backend/internal/features/backups/backups/core/physical/enums/enums_test.go b/backend/internal/features/backups/backups/core/physical/enums/enums_test.go
new file mode 100644
index 00000000..3ea0e35f
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/enums/enums_test.go
@@ -0,0 +1,19 @@
+package physical_enums
+
+import (
+ "encoding/json"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func Test_PhysicalBackupFailoverErrorReasons_SerializeAsStableValues(t *testing.T) {
+ reasons := []PhysicalBackupErrorReason{
+ PhysicalBackupErrorTimelineSwitchDetected,
+ PhysicalBackupErrorFailoverDuringBackup,
+ }
+
+ serializedReasons, err := json.Marshal(reasons)
+ require.NoError(t, err)
+ require.JSONEq(t, `["TIMELINE_SWITCH_DETECTED","FAILOVER_DURING_BACKUP"]`, string(serializedReasons))
+}
diff --git a/backend/internal/features/backups/backups/core/physical/repositories/coordination.go b/backend/internal/features/backups/backups/core/physical/repositories/coordination.go
new file mode 100644
index 00000000..78f77df6
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/repositories/coordination.go
@@ -0,0 +1,59 @@
+package physical_repositories
+
+import (
+ "errors"
+
+ "github.com/google/uuid"
+ "gorm.io/gorm"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ "databasus-backend/internal/util/walmath"
+)
+
+func AcquireBackupAndOrphanCleanupLock(tx *gorm.DB, databaseID uuid.UUID) error {
+ return tx.Exec(
+ `SELECT pg_advisory_xact_lock(hashtextextended('physical-backup:' || ?::text, 0))`,
+ databaseID,
+ ).Error
+}
+
+func HasInFlightFullBackup(tx *gorm.DB, databaseID uuid.UUID) (bool, error) {
+ var claim physical_models.PhysicalInFlightBackup
+
+ err := tx.
+ Where("database_id = ? AND backup_type = ?", databaseID, physical_enums.PhysicalBackupTypeFull).
+ First(&claim).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return false, nil
+ }
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+}
+
+func HasCompletedFullBackupCoveringWal(
+ tx *gorm.DB,
+ databaseID uuid.UUID,
+ timelineID int,
+ walEndLSN walmath.LSN,
+) (bool, error) {
+ var fullCount int64
+
+ err := tx.Model(&physical_models.PhysicalFullBackup{}).
+ Where(
+ "database_id = ? AND timeline_id = ? AND status = ? AND start_lsn IS NOT NULL AND start_lsn < ?::pg_lsn",
+ databaseID,
+ timelineID,
+ physical_enums.PhysicalBackupStatusCompleted,
+ walEndLSN.String(),
+ ).
+ Count(&fullCount).Error
+ if err != nil {
+ return false, err
+ }
+
+ return fullCount > 0, nil
+}
diff --git a/backend/internal/features/backups/backups/core/physical/service/dto.go b/backend/internal/features/backups/backups/core/physical/service/dto.go
index 7a710e74..847a5a53 100644
--- a/backend/internal/features/backups/backups/core/physical/service/dto.go
+++ b/backend/internal/features/backups/backups/core/physical/service/dto.go
@@ -5,8 +5,17 @@ import (
"time"
"github.com/google/uuid"
+
+ "databasus-backend/internal/features/backups/backups/core/physical/chain_view"
)
+type DeleteOrphanWalSegmentsSpec struct {
+ DatabaseID uuid.UUID
+ TimelineID int
+ Span chain_view.LSNRange
+ WalByteBudgetMB float64
+}
+
// PhysicalBackupListRow is one row of the merged, paginated backup list across
// the FULL, incremental and committed-WAL tables. Type-only columns are NULL for
// the other types (scanned into the pointer fields); LSNs arrive as text. It is
diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.go
index 558ccb40..d1c98550 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service.go
@@ -156,6 +156,60 @@ func (s *PhysicalBackupService) DeleteWalSegmentsInSpan(
return deletedRows, deletedMB, nil
}
+func (s *PhysicalBackupService) DeleteOrphanWalSegmentsInSpan(
+ ctx context.Context,
+ spec DeleteOrphanWalSegmentsSpec,
+) (int, float64, error) {
+ var deletedRows int
+ var deletedMB float64
+
+ txErr := storage.GetDb().WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := physical_repositories.AcquireBackupAndOrphanCleanupLock(tx, spec.DatabaseID); err != nil {
+ return err
+ }
+
+ hasInFlightFull, err := physical_repositories.HasInFlightFullBackup(tx, spec.DatabaseID)
+ if err != nil {
+ return err
+ }
+ if hasInFlightFull {
+ return nil
+ }
+
+ hasAnchoringFull, err := physical_repositories.HasCompletedFullBackupCoveringWal(
+ tx,
+ spec.DatabaseID,
+ spec.TimelineID,
+ spec.Span.End,
+ )
+ if err != nil {
+ return err
+ }
+ if hasAnchoringFull {
+ return nil
+ }
+
+ rows, mb, _, err := s.deleteWalInSpanBudgeted(
+ ctx,
+ tx,
+ spec.DatabaseID,
+ spec.TimelineID,
+ spec.Span,
+ spec.WalByteBudgetMB,
+ true,
+ )
+ deletedRows = rows
+ deletedMB = mb
+
+ return err
+ })
+ if txErr != nil {
+ return 0, 0, txErr
+ }
+
+ return deletedRows, deletedMB, nil
+}
+
// DeleteIncrementalCascade removes one incremental and every incremental that
// descends from it (its children, their children, …), leaves first so the
// RESTRICT FK on parent_incremental_backup_id is never violated. The chain's
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
index efa17602..1b578cb6 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
@@ -32,9 +32,7 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS
}
defer creds.Remove()
- // nil onFailover: a detected failover does not refuse a FULL — it proceeds on
- // the live TL, and the scheduler observes the bumped TL across two FULL rows.
- refusalResult, canProceed := verifyTimelineCompatibility(ctx, spec.CommonBackupSpec, nil)
+ refusalResult, preflightTimelineID, canProceed := verifyFullTimelineCompatibility(ctx, spec.CommonBackupSpec)
if !canProceed {
return refusalResult, nil
}
@@ -67,7 +65,7 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS
}
if streamResult.Status != physical_enums.PhysicalBackupStatusCompleted {
- result = streamResult
+ result = recheckFullStreamFailure(ctx, spec.CommonBackupSpec, preflightTimelineID, streamResult)
return nil
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
index 2848b943..9b7db5c9 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
@@ -43,7 +43,7 @@ func (uc *CreateIncrementalBackupUsecase) Execute(
}
defer creds.Remove()
- refusalResult, canProceed := verifyIncrTimelineCompatibility(ctx, spec)
+ refusalResult, rootFullTimelineID, canProceed := verifyIncrTimelineCompatibility(ctx, spec)
if !canProceed {
return refusalResult, nil
}
@@ -102,7 +102,12 @@ func (uc *CreateIncrementalBackupUsecase) Execute(
}
if streamResult.Status != physical_enums.PhysicalBackupStatusCompleted {
- result = streamResult
+ result = recheckIncrementalStreamFailure(
+ ctx,
+ spec.CommonBackupSpec,
+ rootFullTimelineID,
+ streamResult,
+ )
return nil
}
@@ -130,28 +135,42 @@ func (uc *CreateIncrementalBackupUsecase) Execute(
return result, nil
}
-// verifyIncrTimelineCompatibility gates an INCR on timeline/cluster
-// compatibility. Unlike a FULL, an INCR cannot extend across a timeline switch —
-// a detected failover is chain-killing, and the chain must re-anchor on a fresh
-// FULL on the new TL.
func verifyIncrTimelineCompatibility(
ctx context.Context,
spec IncrementalBackupSpec,
-) (PhysicalBackupResult, bool) {
- return verifyTimelineCompatibility(ctx, spec.CommonBackupSpec,
- func(decision *TimelineDecision) (PhysicalBackupResult, bool) {
- reason := physical_enums.PhysicalBackupErrorTimelineRegression
-
- return PhysicalBackupResult{
- Status: physical_enums.PhysicalBackupStatusChainBroken,
- ErrorReason: &reason,
- ErrorMessage: fmt.Sprintf(
- "timeline switch detected (live TL %d > known TL): incremental refused, new FULL required",
- decision.NewTLI,
- ),
- CompletedAt: time.Now().UTC(),
- }, false
- })
+) (PhysicalBackupResult, int, bool) {
+ rootFull, err := spec.FullRepo.FindByID(spec.Backup.RootFullBackupID)
+ if err != nil {
+ return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure, "load root FULL", err), 0, false
+ }
+
+ if rootFull == nil || rootFull.Status != physical_enums.PhysicalBackupStatusCompleted {
+ reason := physical_enums.PhysicalBackupErrorParentManifestMissing
+
+ return PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusChainBroken,
+ ErrorReason: &reason,
+ ErrorMessage: "root FULL is missing or incomplete",
+ CompletedAt: time.Now().UTC(),
+ }, 0, false
+ }
+
+ conn, err := spec.SourceDB.OpenInspectionConn(ctx, spec.FieldEncryptor)
+ if err != nil {
+ return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure,
+ "open inspection connection", err), 0, false
+ }
+ defer func() { _ = conn.Close(ctx) }()
+
+ decision, err := CheckIncrementalTimelineCompatibility(ctx, conn, spec.SourceDB, rootFull.TimelineID)
+ if err != nil {
+ return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure,
+ "timeline compatibility check", err), 0, false
+ }
+
+ refusalResult, canProceed := timelineRefusalResult(decision, true)
+
+ return refusalResult, rootFull.TimelineID, canProceed
}
// runSummarizerPreCheck opens an inspection connection and resolves the
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
index 00e2b8ee..d29a876c 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
@@ -1,6 +1,7 @@
package usecases_physical_postgresql
import (
+ "context"
"log/slog"
"time"
@@ -16,6 +17,8 @@ import (
"databasus-backend/internal/util/walmath"
)
+type timelineIdentityProbe func(context.Context, int) (*TimelineDecision, error)
+
type CommonBackupSpec struct {
SourceDB *postgresql_physical.PostgresqlPhysicalDatabase
DatabaseName string
@@ -27,6 +30,7 @@ type CommonBackupSpec struct {
FullRepo *physical_repositories.PhysicalFullBackupRepository
HistoryRepo *physical_repositories.PhysicalWalHistoryRepository
Logger *slog.Logger
+ timelineProbe timelineIdentityProbe
// ProgressListener, when set, receives the streamed size and elapsed time
// periodically during the backup so the row's size grows in near-real-time.
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
index 3b23269b..b6263013 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
@@ -163,39 +163,49 @@ func buildObjectName(
)
}
-// verifyTimelineCompatibility gates a backup on the live timeline/cluster
-// verdict. FailoverDetected is the one kind FULL and INCR treat differently — a
-// FULL proceeds on the live TL while an INCR must re-anchor on a fresh FULL — so
-// the caller supplies onFailover; a nil onFailover treats failover as
-// non-breaking (FULL). canProceed=false means the caller returns refusal and stops.
-func verifyTimelineCompatibility(
+func verifyFullTimelineCompatibility(
ctx context.Context,
common CommonBackupSpec,
- onFailover func(decision *TimelineDecision) (refusal PhysicalBackupResult, canProceed bool),
-) (refusal PhysicalBackupResult, canProceed bool) {
+) (refusal PhysicalBackupResult, liveTimelineID int, canProceed bool) {
conn, err := common.SourceDB.OpenInspectionConn(ctx, common.FieldEncryptor)
if err != nil {
return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure,
- "open inspection connection", err), false
+ "open inspection connection", err), 0, false
}
defer func() { _ = conn.Close(ctx) }()
- decision, err := CheckTimelineCompatibility(ctx, conn, common.SourceDB, common.FullRepo, common.HistoryRepo)
+ decision, err := CheckFullTimelineCompatibility(ctx, conn, common.SourceDB, common.FullRepo, common.HistoryRepo)
if err != nil {
return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure,
- "timeline compatibility check", err), false
+ "timeline compatibility check", err), 0, false
}
+ refusal, canProceed = timelineRefusalResult(decision, false)
+
+ return refusal, decision.LiveTimelineID, canProceed
+}
+
+func timelineRefusalResult(decision *TimelineDecision, refuseNewerTimeline bool) (PhysicalBackupResult, bool) {
switch decision.Kind {
case TimelineContinue:
return PhysicalBackupResult{}, true
case TimelineFailoverDetected:
- if onFailover == nil {
+ if !refuseNewerTimeline {
return PhysicalBackupResult{}, true
}
- return onFailover(decision)
+ reason := physical_enums.PhysicalBackupErrorTimelineSwitchDetected
+
+ return PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusChainBroken,
+ ErrorReason: &reason,
+ ErrorMessage: fmt.Sprintf(
+ "timeline switch detected: root FULL TL %d, live TL %d",
+ decision.ExpectedTimelineID, decision.LiveTimelineID,
+ ),
+ CompletedAt: time.Now().UTC(),
+ }, false
case TimelineRegression:
reason := physical_enums.PhysicalBackupErrorTimelineRegression
@@ -205,7 +215,7 @@ func verifyTimelineCompatibility(
ErrorReason: &reason,
ErrorMessage: fmt.Sprintf(
"timeline regression: expected TL %d, live TL %d",
- decision.ExpectedTLI, decision.ActualTLI,
+ decision.ExpectedTimelineID, decision.LiveTimelineID,
),
CompletedAt: time.Now().UTC(),
}, false
@@ -218,7 +228,7 @@ func verifyTimelineCompatibility(
ErrorReason: &reason,
ErrorMessage: fmt.Sprintf(
"system_identifier mismatch: catalog %s, live %s",
- decision.ExpectedSysID, decision.ActualSysID,
+ decision.ExpectedSystemIdentifier, decision.LiveSystemIdentifier,
),
CompletedAt: time.Now().UTC(),
}, false
@@ -226,3 +236,101 @@ func verifyTimelineCompatibility(
return PhysicalBackupResult{}, true
}
+
+func recheckFullStreamFailure(
+ ctx context.Context,
+ common CommonBackupSpec,
+ preflightTimelineID int,
+ streamResult PhysicalBackupResult,
+) PhysicalBackupResult {
+ if ctx.Err() != nil || streamResult.Status == physical_enums.PhysicalBackupStatusCanceled {
+ return streamResult
+ }
+
+ decision, err := inspectTimelineAgainstExpected(ctx, common, preflightTimelineID)
+ if err != nil {
+ common.Logger.WarnContext(ctx, "cluster identity recheck failed after FULL stream failure", "error", err)
+
+ return streamResult
+ }
+
+ return classifyFullStreamFailureAfterIdentity(streamResult, preflightTimelineID, decision)
+}
+
+func classifyFullStreamFailureAfterIdentity(
+ streamResult PhysicalBackupResult,
+ preflightTimelineID int,
+ decision *TimelineDecision,
+) PhysicalBackupResult {
+ if decision.Kind == TimelineFailoverDetected {
+ reason := physical_enums.PhysicalBackupErrorFailoverDuringBackup
+
+ return PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusError,
+ ErrorReason: &reason,
+ ErrorMessage: fmt.Sprintf(
+ "timeline advanced during FULL: preflight TL %d, live TL %d",
+ preflightTimelineID,
+ decision.LiveTimelineID,
+ ),
+ CompletedAt: time.Now().UTC(),
+ }
+ }
+
+ refusal, canProceed := timelineRefusalResult(decision, false)
+ if !canProceed {
+ return refusal
+ }
+
+ return streamResult
+}
+
+func recheckIncrementalStreamFailure(
+ ctx context.Context,
+ common CommonBackupSpec,
+ rootFullTimelineID int,
+ streamResult PhysicalBackupResult,
+) PhysicalBackupResult {
+ if ctx.Err() != nil || streamResult.Status == physical_enums.PhysicalBackupStatusCanceled {
+ return streamResult
+ }
+
+ decision, err := inspectTimelineAgainstExpected(ctx, common, rootFullTimelineID)
+ if err != nil {
+ common.Logger.WarnContext(ctx, "cluster identity recheck failed after incremental stream failure", "error", err)
+
+ return streamResult
+ }
+
+ return classifyIncrementalStreamFailureAfterIdentity(streamResult, decision)
+}
+
+func classifyIncrementalStreamFailureAfterIdentity(
+ streamResult PhysicalBackupResult,
+ decision *TimelineDecision,
+) PhysicalBackupResult {
+ refusal, canProceed := timelineRefusalResult(decision, true)
+ if !canProceed {
+ return refusal
+ }
+
+ return streamResult
+}
+
+func inspectTimelineAgainstExpected(
+ ctx context.Context,
+ common CommonBackupSpec,
+ expectedTimelineID int,
+) (*TimelineDecision, error) {
+ if common.timelineProbe != nil {
+ return common.timelineProbe(ctx, expectedTimelineID)
+ }
+
+ conn, err := common.SourceDB.OpenInspectionConn(ctx, common.FieldEncryptor)
+ if err != nil {
+ return nil, fmt.Errorf("open inspection connection: %w", err)
+ }
+ defer func() { _ = conn.Close(context.Background()) }()
+
+ return CheckIncrementalTimelineCompatibility(ctx, conn, common.SourceDB, expectedTimelineID)
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
index 913c6290..6f4c9263 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
@@ -27,8 +27,6 @@ import (
"databasus-backend/internal/util/walmath"
)
-// TimelineDecisionKind discriminates the outcome of CheckTimelineCompatibility.
-// Continue is the green path; the other three are refusal branches.
type TimelineDecisionKind int
const (
@@ -38,76 +36,107 @@ const (
TimelineDifferentCluster
)
-// TimelineDecision is a discriminated result. Only the fields belonging to
-// Kind are meaningful; the rest are zero.
type TimelineDecision struct {
- Kind TimelineDecisionKind
-
- NewTLI int // TimelineFailoverDetected — promote-detected target TL
-
- ExpectedTLI int // TimelineRegression
- ActualTLI int // TimelineRegression
+ Kind TimelineDecisionKind
+ ExpectedTimelineID int
+ LiveTimelineID int
+ ExpectedSystemIdentifier string
+ LiveSystemIdentifier string
+}
- ExpectedSysID string // TimelineDifferentCluster — db.SystemIdentifier
- ActualSysID string // TimelineDifferentCluster — live cluster system_identifier
+type timelineComparison struct {
+ ExpectedSystemIdentifier *string
+ LiveTimelineID int
+ LiveSystemIdentifier string
+ ExpectedTimelineID int
}
-// CheckTimelineCompatibility validates the source cluster against the
-// catalog before spawning a FULL or INCR. Three refusal branches:
-// system_identifier mismatch (different cluster pointed at the same DB row),
-// timeline regression (live TL < newest known TL — never legitimate), and
-// failover detected (live TL > newest known TL — the new FULL must anchor a
-// fresh chain on the new TL). Continue means proceed.
-//
-// "newest known TL" is derived from GREATEST(MAX(full.timeline_id),
-// MAX(history.timeline_id)) per BACKUPING-SCHEDULER-PLAN.md §B step 1.
-// First-tick (no fulls, no history) returns Continue — the live TL seeds
-// the first FULL's timeline_id.
-func CheckTimelineCompatibility(
+func CheckFullTimelineCompatibility(
ctx context.Context,
conn *pgx.Conn,
db *postgresql_physical.PostgresqlPhysicalDatabase,
fullRepo *physical_repositories.PhysicalFullBackupRepository,
historyRepo *physical_repositories.PhysicalWalHistoryRepository,
) (*TimelineDecision, error) {
- liveTLI, liveSysID, err := readClusterIdentity(ctx, conn)
+ liveTimelineID, liveSystemIdentifier, err := readClusterIdentity(ctx, conn)
if err != nil {
return nil, err
}
- if db.SystemIdentifier != nil && *db.SystemIdentifier != liveSysID {
- return &TimelineDecision{
- Kind: TimelineDifferentCluster,
- ExpectedSysID: *db.SystemIdentifier,
- ActualSysID: liveSysID,
- }, nil
+ decision := decideTimelineCompatibility(timelineComparison{
+ ExpectedSystemIdentifier: db.SystemIdentifier,
+ LiveTimelineID: liveTimelineID,
+ LiveSystemIdentifier: liveSystemIdentifier,
+ })
+ if decision.Kind == TimelineDifferentCluster {
+ return decision, nil
}
- knownTLI, err := newestKnownTimeline(db.ParentDatabaseID(), fullRepo, historyRepo)
+ knownTimelineID, err := newestKnownTimeline(db.ParentDatabaseID(), fullRepo, historyRepo)
+ if err != nil {
+ return nil, err
+ }
+
+ return decideTimelineCompatibility(timelineComparison{
+ ExpectedSystemIdentifier: db.SystemIdentifier,
+ LiveTimelineID: liveTimelineID,
+ LiveSystemIdentifier: liveSystemIdentifier,
+ ExpectedTimelineID: knownTimelineID,
+ }), nil
+}
+
+func CheckIncrementalTimelineCompatibility(
+ ctx context.Context,
+ conn *pgx.Conn,
+ db *postgresql_physical.PostgresqlPhysicalDatabase,
+ rootFullTimelineID int,
+) (*TimelineDecision, error) {
+ liveTimelineID, liveSystemIdentifier, err := readClusterIdentity(ctx, conn)
if err != nil {
return nil, err
}
- if knownTLI == 0 {
- return &TimelineDecision{Kind: TimelineContinue}, nil
+ return decideTimelineCompatibility(timelineComparison{
+ ExpectedSystemIdentifier: db.SystemIdentifier,
+ LiveTimelineID: liveTimelineID,
+ LiveSystemIdentifier: liveSystemIdentifier,
+ ExpectedTimelineID: rootFullTimelineID,
+ }), nil
+}
+
+func decideTimelineCompatibility(comparison timelineComparison) *TimelineDecision {
+ decision := &TimelineDecision{
+ Kind: TimelineContinue,
+ ExpectedTimelineID: comparison.ExpectedTimelineID,
+ LiveTimelineID: comparison.LiveTimelineID,
+ LiveSystemIdentifier: comparison.LiveSystemIdentifier,
+ ExpectedSystemIdentifier: "",
}
- if liveTLI < knownTLI {
- return &TimelineDecision{
- Kind: TimelineRegression,
- ExpectedTLI: knownTLI,
- ActualTLI: liveTLI,
- }, nil
+ if comparison.ExpectedSystemIdentifier != nil {
+ decision.ExpectedSystemIdentifier = *comparison.ExpectedSystemIdentifier
+ if *comparison.ExpectedSystemIdentifier != comparison.LiveSystemIdentifier {
+ decision.Kind = TimelineDifferentCluster
+
+ return decision
+ }
}
- if liveTLI > knownTLI {
- return &TimelineDecision{
- Kind: TimelineFailoverDetected,
- NewTLI: liveTLI,
- }, nil
+ if comparison.ExpectedTimelineID == 0 {
+ return decision
+ }
+
+ if comparison.LiveTimelineID < comparison.ExpectedTimelineID {
+ decision.Kind = TimelineRegression
+
+ return decision
+ }
+
+ if comparison.LiveTimelineID > comparison.ExpectedTimelineID {
+ decision.Kind = TimelineFailoverDetected
}
- return &TimelineDecision{Kind: TimelineContinue}, nil
+ return decision
}
// ValidateStartLsnAgainstHistory checks that startLSN falls inside the LSN
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go
new file mode 100644
index 00000000..e9b9aa49
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go
@@ -0,0 +1,221 @@
+package usecases_physical_postgresql
+
+import (
+ "context"
+ "errors"
+ "io"
+ "log/slog"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+)
+
+func Test_DecideTimelineCompatibility_WithMatchingAndConflictingIdentity_ReturnsExpectedDecision(t *testing.T) {
+ expectedSystemIdentifier := "123"
+
+ tests := []struct {
+ name string
+ liveTimelineID int
+ liveSystemIdentifier string
+ expectedTimelineID int
+ decisionKind TimelineDecisionKind
+ }{
+ {
+ name: "matching timeline",
+ liveTimelineID: 3,
+ liveSystemIdentifier: "123",
+ expectedTimelineID: 3,
+ decisionKind: TimelineContinue,
+ },
+ {
+ name: "newer timeline",
+ liveTimelineID: 4,
+ liveSystemIdentifier: "123",
+ expectedTimelineID: 3,
+ decisionKind: TimelineFailoverDetected,
+ },
+ {
+ name: "older timeline",
+ liveTimelineID: 2,
+ liveSystemIdentifier: "123",
+ expectedTimelineID: 3,
+ decisionKind: TimelineRegression,
+ },
+ {
+ name: "different cluster",
+ liveTimelineID: 3,
+ liveSystemIdentifier: "456",
+ expectedTimelineID: 3,
+ decisionKind: TimelineDifferentCluster,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ decision := decideTimelineCompatibility(timelineComparison{
+ ExpectedSystemIdentifier: &expectedSystemIdentifier,
+ LiveTimelineID: test.liveTimelineID,
+ LiveSystemIdentifier: test.liveSystemIdentifier,
+ ExpectedTimelineID: test.expectedTimelineID,
+ })
+
+ require.Equal(t, test.decisionKind, decision.Kind)
+ require.Equal(t, test.expectedTimelineID, decision.ExpectedTimelineID)
+ require.Equal(t, test.liveTimelineID, decision.LiveTimelineID)
+ })
+ }
+}
+
+func Test_ClassifyIncrementalStreamFailureAfterIdentity_WhenTimelineAdvanced_BreaksChain(t *testing.T) {
+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed
+ originalResult := PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusError,
+ ErrorReason: &originalReason,
+ }
+ decision := &TimelineDecision{
+ Kind: TimelineFailoverDetected,
+ ExpectedTimelineID: 3,
+ LiveTimelineID: 4,
+ }
+
+ result := classifyIncrementalStreamFailureAfterIdentity(originalResult, decision)
+
+ require.Equal(t, physical_enums.PhysicalBackupStatusChainBroken, result.Status)
+ require.Equal(t, physical_enums.PhysicalBackupErrorTimelineSwitchDetected, *result.ErrorReason)
+}
+
+func Test_ClassifyFullStreamFailureAfterIdentity_WhenTimelineAdvanced_ClassifiesFailover(t *testing.T) {
+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed
+ originalResult := PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusError,
+ ErrorReason: &originalReason,
+ }
+ decision := &TimelineDecision{
+ Kind: TimelineFailoverDetected,
+ ExpectedTimelineID: 3,
+ LiveTimelineID: 4,
+ }
+
+ result := classifyFullStreamFailureAfterIdentity(originalResult, 3, decision)
+
+ require.Equal(t, physical_enums.PhysicalBackupStatusError, result.Status)
+ require.Equal(t, physical_enums.PhysicalBackupErrorFailoverDuringBackup, *result.ErrorReason)
+}
+
+func Test_RecheckFullStreamFailure_WhenCanceled_PreservesResultWithoutProbe(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ reason := physical_enums.PhysicalBackupErrorCanceledByUser
+ originalResult := PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusCanceled,
+ ErrorReason: &reason,
+ }
+
+ probeCalls := 0
+ common := CommonBackupSpec{
+ timelineProbe: func(context.Context, int) (*TimelineDecision, error) {
+ probeCalls++
+
+ return nil, errors.New("unexpected identity probe")
+ },
+ }
+
+ result := recheckFullStreamFailure(ctx, common, 3, originalResult)
+
+ require.Equal(t, originalResult, result)
+ require.Zero(t, probeCalls)
+}
+
+func Test_RecheckIncrementalStreamFailure_WhenCanceled_PreservesResultWithoutProbe(t *testing.T) {
+ reason := physical_enums.PhysicalBackupErrorCanceledByUser
+ originalResult := PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusCanceled,
+ ErrorReason: &reason,
+ }
+ probeCalls := 0
+ common := CommonBackupSpec{
+ timelineProbe: func(context.Context, int) (*TimelineDecision, error) {
+ probeCalls++
+
+ return nil, errors.New("unexpected identity probe")
+ },
+ }
+
+ result := recheckIncrementalStreamFailure(context.Background(), common, 3, originalResult)
+
+ require.Equal(t, originalResult, result)
+ require.Zero(t, probeCalls)
+}
+
+func Test_RecheckFullStreamFailure_WhenTimelineAdvanced_ClassifiesFailover(t *testing.T) {
+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed
+ originalResult := PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusError,
+ ErrorReason: &originalReason,
+ }
+ common := CommonBackupSpec{
+ Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
+ timelineProbe: func(_ context.Context, expectedTimelineID int) (*TimelineDecision, error) {
+ require.Equal(t, 3, expectedTimelineID)
+
+ return &TimelineDecision{
+ Kind: TimelineFailoverDetected,
+ ExpectedTimelineID: expectedTimelineID,
+ LiveTimelineID: 4,
+ }, nil
+ },
+ }
+
+ result := recheckFullStreamFailure(context.Background(), common, 3, originalResult)
+
+ require.Equal(t, physical_enums.PhysicalBackupStatusError, result.Status)
+ require.Equal(t, physical_enums.PhysicalBackupErrorFailoverDuringBackup, *result.ErrorReason)
+}
+
+func Test_RecheckIncrementalStreamFailure_WhenTimelineAdvanced_BreaksChain(t *testing.T) {
+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed
+ originalResult := PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusError,
+ ErrorReason: &originalReason,
+ }
+ common := CommonBackupSpec{
+ Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
+ timelineProbe: func(_ context.Context, expectedTimelineID int) (*TimelineDecision, error) {
+ require.Equal(t, 3, expectedTimelineID)
+
+ return &TimelineDecision{
+ Kind: TimelineFailoverDetected,
+ ExpectedTimelineID: expectedTimelineID,
+ LiveTimelineID: 4,
+ }, nil
+ },
+ }
+
+ result := recheckIncrementalStreamFailure(context.Background(), common, 3, originalResult)
+
+ require.Equal(t, physical_enums.PhysicalBackupStatusChainBroken, result.Status)
+ require.Equal(t, physical_enums.PhysicalBackupErrorTimelineSwitchDetected, *result.ErrorReason)
+}
+
+func Test_RecheckStreamFailure_WhenIdentityProbeFails_PreservesOriginalResult(t *testing.T) {
+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed
+ originalResult := PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusError,
+ ErrorReason: &originalReason,
+ ErrorMessage: "original stream failure",
+ }
+ common := CommonBackupSpec{
+ Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
+ timelineProbe: func(context.Context, int) (*TimelineDecision, error) {
+ return nil, errors.New("identity probe failed")
+ },
+ }
+
+ fullResult := recheckFullStreamFailure(context.Background(), common, 3, originalResult)
+ incrementalResult := recheckIncrementalStreamFailure(context.Background(), common, 3, originalResult)
+
+ require.Equal(t, originalResult, fullResult)
+ require.Equal(t, originalResult, incrementalResult)
+}
diff --git a/backend/internal/features/databases/databases/postgresql/physical/model.go b/backend/internal/features/databases/databases/postgresql/physical/model.go
index 0c16453a..495878b2 100644
--- a/backend/internal/features/databases/databases/postgresql/physical/model.go
+++ b/backend/internal/features/databases/databases/postgresql/physical/model.go
@@ -810,10 +810,8 @@ func (p *PostgresqlPhysicalDatabase) CreateReplicationOnlyUser(
return nil, errors.New("failed to generate unique username after 3 attempts")
}
-// OpenInspectionConn opens a regular (non-replication) connection to the
-// source cluster. Used by the FULL/INCR executors for pre-flight checks
-// (timeline.CheckTimelineCompatibility), .history file reads, and
-// post-stream LSN validation.
+// Replication connections cannot run the SQL identity checks or read timeline
+// history files, so inspection uses PostgreSQL's regular protocol.
func (p *PostgresqlPhysicalDatabase) OpenInspectionConn(
ctx context.Context,
encryptor encryption.FieldEncryptor,
diff --git a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
index 464fb882..ade29433 100644
--- a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
+++ b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
@@ -35,6 +35,10 @@ func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t
physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)
}
+func Test_PhysicalBackup_AfterPromotion_ReanchorsIncrementalChain(t *testing.T) {
+ physicaltesting.RunPromotionReanchorsIncrementalChain(t, pgVersion, pgImage)
+}
+
func Test_PhysicalRestore_BootViaEntrypointVolumeMount_RecoversBaseRows(t *testing.T) {
physicaltesting.RunBootViaEntrypointVolumeMountRecoversBaseRows(t, pgVersion, pgImage)
}
diff --git a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
index 97ed40f7..62c91f00 100644
--- a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
+++ b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
@@ -35,6 +35,10 @@ func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t
physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)
}
+func Test_PhysicalBackup_AfterPromotion_ReanchorsIncrementalChain(t *testing.T) {
+ physicaltesting.RunPromotionReanchorsIncrementalChain(t, pgVersion, pgImage)
+}
+
func Test_PhysicalRestore_BootViaEntrypointVolumeMount_RecoversBaseRows(t *testing.T) {
physicaltesting.RunBootViaEntrypointVolumeMountRecoversBaseRows(t, pgVersion, pgImage)
}
diff --git a/backend/internal/features/tests/physical/postgresql/shared/bodies.go b/backend/internal/features/tests/physical/postgresql/shared/bodies.go
index 2eff353c..49e60e9d 100644
--- a/backend/internal/features/tests/physical/postgresql/shared/bodies.go
+++ b/backend/internal/features/tests/physical/postgresql/shared/bodies.go
@@ -6,13 +6,17 @@ import (
"net/http"
"os"
"path/filepath"
+ "slices"
"testing"
"time"
+ "github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
backuping_physical "databasus-backend/internal/features/backups/backups/backuping/physical"
+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
@@ -179,6 +183,78 @@ func RunWalStreamCatalogsHistoryOnTimelineSwitch(t *testing.T, version, image st
waitForTimelineHistoryOnParent(t, fixture, 2, 60*time.Second)
}
+func RunPromotionReanchorsIncrementalChain(t *testing.T, version, image string) {
+ if testing.Short() {
+ t.Skip("clones a standby, promotes it, and restores the replacement chain; skipped in -short")
+ }
+
+ primary, standby := containers.StartPhysicalPrimaryWithStandby(t, image)
+ router := newPhysicalTestRouter()
+ fixture := postgresql_executor.SetupPhysicalDBForScheduledBackupVersion(
+ t, standby.Host, standby.Port, version, postgresql_physical.BackupTypeFullIncrementalAndWalStream,
+ )
+ target := prepareRestoreTarget(t, image)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
+ defer cancel()
+
+ primaryConn := openConnAt(t, primary.Host, primary.Port)
+ standbyConn := openSourceTestDBConn(t, fixture)
+ createMarkerTable(t, ctx, primaryConn)
+ insertMarker(t, ctx, primaryConn, "before-promotion", "row-before-promotion")
+ waitForMarkerOnStandby(t, ctx, standbyConn, "before-promotion", 60*time.Second)
+
+ enablePhysicalBackupsViaAPI(t, router, fixture, true)
+ initialChain := waitForChainBackups(t, router, fixture, 0, 3*time.Minute)
+ initialFullID := rootFullBackupID(t, initialChain)
+
+ PromoteStandby(t, ctx, standbyConn)
+ insertMarker(t, ctx, standbyConn, "after-promotion", "row-after-promotion")
+ _, err := postgresql_executor.GenerateWalActivity(ctx, standbyConn, 16*1024*1024)
+ require.NoError(t, err)
+ waitForTimelineHistoryOnParent(t, fixture, 2, 60*time.Second)
+ triggerIncrementalViaAPI(t, router, fixture)
+
+ deadline := time.Now().UTC().Add(4 * time.Minute)
+ var replacementFullID uuid.UUID
+ for time.Now().UTC().Before(deadline) {
+ incrementals, err := physical_repositories.GetIncrementalBackupRepository().FindAllByRootFull(initialFullID)
+ require.NoError(t, err)
+ fulls, err := physical_repositories.GetFullBackupRepository().FindCompletedNewestFirstByDatabase(fixture.DB.ID)
+ require.NoError(t, err)
+
+ hasTimelineRefusal := slices.ContainsFunc(
+ incrementals,
+ func(incremental *physical_models.PhysicalIncrementalBackup) bool {
+ return incremental.Status == physical_enums.PhysicalBackupStatusChainBroken &&
+ incremental.ErrorReason != nil &&
+ *incremental.ErrorReason == physical_enums.PhysicalBackupErrorTimelineSwitchDetected
+ },
+ )
+ if hasTimelineRefusal && len(fulls) >= 2 && fulls[0].TimelineID == 2 {
+ replacementFullID = fulls[0].ID
+
+ break
+ }
+
+ time.Sleep(500 * time.Millisecond)
+ }
+ require.NotEqual(
+ t,
+ uuid.Nil,
+ replacementFullID,
+ "promotion must refuse the stale incremental and complete a timeline-2 FULL",
+ )
+
+ backuping_physical.RunOrphanWalCleanupForTest(t, fixture.DB.ID)
+ bundle := downloadRestoreBundleViaAPI(t, router, fixture, nil)
+ reconstructCluster(t, target, router, image, bundle, nil)
+ startRestoredCluster(t, target, image)
+
+ restoredPhases := queryRestoredMarkerRows(t, target)
+ assert.ElementsMatch(t, []string{"before-promotion", "after-promotion"}, restoredPhases)
+}
+
// RunBootViaEntrypointVolumeMountRecoversBaseRows reproduces the user's docker-compose flow: restore on
// the host in --combine-image mode, then serve the result through the postgres image's own entrypoint
// with the output bind-mounted at the image VOLUME (the volume root on PG 18). The booted cluster must