PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Re-anchor PostgreSQL physical backups automatically after failover
Rostislav Dugin9a2b269248fb
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper.go b/backend/internal/features/backups/backups/backuping/physical/backuper.goindex 7e23da18..376b916e 100644--- a/backend/internal/features/backups/backups/backuping/physical/backuper.go+++ b/backend/internal/features/backups/backups/backuping/physical/backuper.go@@ -434,13 +434,13 @@ func (b *PhysicalBackuper) persistFullResult(fullBackup.ManifestEncryptionSalt = nilOrPtr(backupResult.ManifestEncryptionSalt)fullBackup.ManifestEncryptionIV = nilOrPtr(backupResult.ManifestEncryptionIV)- completed := backupResult.CompletedAt- if completed.IsZero() {- completed = time.Now().UTC()- }+ }- fullBackup.CompletedAt = &completed+ completedAt := backupResult.CompletedAt+ if completedAt.IsZero() {+ completedAt = time.Now().UTC()}+ fullBackup.CompletedAt = &completedAtreturn b.saveTerminalResultIfInProgress(fullBackup.DatabaseID,@@ -484,13 +484,13 @@ func (b *PhysicalBackuper) persistIncrResult(incrBackup.ManifestEncryptionSalt = nilOrPtr(backupResult.ManifestEncryptionSalt)incrBackup.ManifestEncryptionIV = nilOrPtr(backupResult.ManifestEncryptionIV)- completed := backupResult.CompletedAt- if completed.IsZero() {- completed = time.Now().UTC()- }+ }- incrBackup.CompletedAt = &completed+ completedAt := backupResult.CompletedAt+ if completedAt.IsZero() {+ completedAt = time.Now().UTC()}+ incrBackup.CompletedAt = &completedAtreturn b.saveTerminalResultIfInProgress(incrBackup.DatabaseID,diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go b/backend/internal/features/backups/backups/backuping/physical/backuper_test.goindex 116d5f0d..77de91fc 100644--- a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/backuper_test.go@@ -208,7 +208,7 @@ func Test_PersistFullResult_WhenClusterSizeUnmeasured_LeavesRawSizeMbNil(t *testassert.Nil(t, persisted.RawSizeMb)}-func Test_PersistFullResult_WhenErrorStatus_SkipsCompletionFieldsAndReleasesInFlight(t *testing.T) {+func Test_PersistFullResult_WhenErrorStatus_RecordsTerminalTimeAndReleasesInFlight(t *testing.T) {prereqs := seedBackupPrereqs(t)backuper := CreateTestPhysicalBackuper(nil)@@ -231,11 +231,29 @@ func Test_PersistFullResult_WhenErrorStatus_SkipsCompletionFieldsAndReleasesInFlrequire.NotNil(t, persisted.ErrorReason)assert.Equal(t, physical_enums.PhysicalBackupErrorPgBasebackupFailed, *persisted.ErrorReason)assert.Nil(t, persisted.ManifestFileName, "completion-only fields must not be copied on a non-COMPLETED result")- assert.Nil(t, persisted.CompletedAt)+ require.NotNil(t, persisted.CompletedAt)assertInFlightReleased(t, prereqs.DB.ID)}+func Test_PersistIncrementalResult_WhenErrorStatus_RecordsTerminalTime(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ backuper := CreateTestPhysicalBackuper(nil)+ rootFull := seedCompletedRootFull(t, prereqs)+ incremental := seedInProgressIncr(t, prereqs, rootFull.ID)+ claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeIncremental, incremental.ID)++ result := postgresql_executor.PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusChainBroken,+ ErrorReason: new(physical_enums.PhysicalBackupErrorTimelineSwitchDetected),+ }+ require.NoError(t, backuper.persistIncrResult(incremental, result))++ persistedIncremental, err := physical_repositories.GetIncrementalBackupRepository().FindByID(incremental.ID)+ require.NoError(t, err)+ require.NotNil(t, persistedIncremental.CompletedAt)+}+func Test_PersistIncrementalResult_WhenCompleted_CopiesFieldsAndReleasesInFlight(t *testing.T) {prereqs := seedBackupPrereqs(t)backuper := CreateTestPhysicalBackuper(nil)diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner.go b/backend/internal/features/backups/backups/backuping/physical/cleaner.goindex 679d6ade..039cb249 100644--- a/backend/internal/features/backups/backups/backuping/physical/cleaner.go+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner.go@@ -526,8 +526,14 @@ func (c *PhysicalBackupCleaner) cleanOrphanWalForDatabase(// never overlap, so no chain-covered WAL is ever caught.span := chain_view.LSNRange{Start: segment.StartLSN, End: segment.EndLSN}- rows, mb, err := c.physicalBackupService.DeleteWalSegmentsInSpan(- ctx, databaseID, segment.TimelineID, span, budget,+ rows, mb, err := c.physicalBackupService.DeleteOrphanWalSegmentsInSpan(+ ctx,+ physical_service.DeleteOrphanWalSegmentsSpec{+ DatabaseID: databaseID,+ TimelineID: segment.TimelineID,+ Span: span,+ WalByteBudgetMB: budget,+ },)if err != nil {logger.ErrorContext(ctx, "failed to delete orphan wal segment",diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.goindex 003ad500..e322ad55 100644--- a/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner_test.go@@ -266,6 +266,37 @@ func Test_CleanOrphanWalForDatabase_WhenWalCoveredByChain_KeepsIt(t *testing.T)assert.True(t, walExists(t, covered.ID), "chain-covered WAL must never be caught by the orphan pass")}+func Test_CleanOrphanWalForDatabase_WhenFirstFullInProgress_KeepsWalArchivedDuringBackup(t *testing.T) {+ prereqs := seedBackupPrereqs(t)++ inProgressFull := physical_testing.CreateTestFullBackup(t, physical_testing.NewTestInProgressFullBackup(+ prereqs.DB.ID,+ prereqs.Storage.ID,+ 1,+ ))+ physical_testing.CreateTestInFlightClaim(+ t,+ prereqs.DB.ID,+ inProgressFull.ID,+ physical_enums.PhysicalBackupTypeFull,+ )++ walArchivedDuringBackup := physical_testing.CreateTestWalSegment(t, physical_testing.NewTestWalSegment(+ prereqs.DB.ID,+ prereqs.Storage.ID,+ 1,+ "000000010000000000000001",+ testLSN(1),+ testLSN(2),+ ))++ cleaner := CreateTestPhysicalCleaner()+ cleaner.cleanOrphanWalForDatabase(t.Context(), logger.GetLogger(), prereqs.DB.ID)++ assert.True(t, walExists(t, walArchivedDuringBackup.ID),+ "WAL archived while the first full is running must remain available when the full completes")+}+// A FULL taken right after a segment switch reports a start_lsn at// firstRecordOffset. The segment's own bounds come from its filename and are// file-aligned, so it starts below the FULL and must still be kept: it carries thediff --git a/backend/internal/features/backups/backups/backuping/physical/errors.go b/backend/internal/features/backups/backups/backuping/physical/errors.gonew file mode 100644index 00000000..07e409cb--- /dev/null+++ b/backend/internal/features/backups/backups/backuping/physical/errors.go@@ -0,0 +1,5 @@+package backuping_physical++import "errors"++var errBackupDecisionSuperseded = errors.New("physical backup decision superseded")diff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler.go b/backend/internal/features/backups/backups/backuping/physical/scheduler.goindex b901f6ff..3dc669e8 100644--- a/backend/internal/features/backups/backups/backuping/physical/scheduler.go+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler.go@@ -2,6 +2,7 @@ package backuping_physicalimport ("context"+ "errors""fmt""log/slog""slices"@@ -156,6 +157,12 @@ type backupDecision struct {incrParentIncrID *uuid.UUIDforceFullRequestedAt *time.TimeforceIncrementalRequestedAt *time.Time+ immediateTrigger *backupTrigger+}++type backupTrigger struct {+ kind physical_enums.PhysicalBackupType+ backupID uuid.UUID}// decideBackupKind picks FULL or INCR purely from catalog state and cadence, with@@ -181,10 +188,16 @@ func (s *PhysicalBackupsScheduler) decideBackupKind(}, true}- if backupConfig.ForceIncrementalRequestedAt != nil {- if decision, ok := s.decideForcedIncremental(logger, backupConfig); ok {- return decision, true- }+ hasUnresolvedMismatch, err := hasUnresolvedSystemIdentifierMismatch(storage.GetDb(), backupConfig.DatabaseID)+ if err != nil {+ logger.Error("failed to inspect system identifier mismatch state", "error", err)++ return backupDecision{}, false+ }+ if hasUnresolvedMismatch {+ logger.Debug("automatic scheduling suppressed after system identifier mismatch")++ return backupDecision{}, false}lastIncr, err := s.incrRepo.FindLastByDatabase(backupConfig.DatabaseID)@@ -194,6 +207,16 @@ func (s *PhysicalBackupsScheduler) decideBackupKind(return backupDecision{}, false}+ if decision, ok := decideImmediateReplacementFull(lastFull, lastIncr); ok {+ return decision, true+ }++ if backupConfig.ForceIncrementalRequestedAt != nil {+ if decision, ok := s.decideForcedIncremental(logger, backupConfig); ok {+ return decision, true+ }+ }+if backupConfig.FullBackupInterval.ShouldTriggerBackup(now, createdAtOrNil(lastFull)) {return backupDecision{kind: physical_enums.PhysicalBackupTypeFull,@@ -372,6 +395,10 @@ func (s *PhysicalBackupsScheduler) claimAndInsert(claimed := falsetxErr := storage.GetDb().Transaction(func(tx *gorm.DB) error {+ if err := physical_repositories.AcquireBackupAndOrphanCleanupLock(tx, backupConfig.DatabaseID); err != nil {+ return err+ }+ok, claimErr := s.inFlightRepo.Claim(tx, physical_repositories.ClaimSpec{DatabaseID: backupConfig.DatabaseID,BackupType: decision.kind,@@ -385,6 +412,30 @@ func (s *PhysicalBackupsScheduler) claimAndInsert(return nil}+ if decision.forceFullRequestedAt == nil {+ hasUnresolvedMismatch, err := hasUnresolvedSystemIdentifierMismatch(tx, backupConfig.DatabaseID)+ if err != nil {+ return err+ }+ if hasUnresolvedMismatch {+ return errBackupDecisionSuperseded+ }+ }++ if decision.immediateTrigger != nil {+ isCurrent, err := isImmediateReplacementTriggerCurrent(+ tx,+ backupConfig.DatabaseID,+ *decision.immediateTrigger,+ )+ if err != nil {+ return err+ }+ if !isCurrent {+ return errBackupDecisionSuperseded+ }+ }+claimed = trueif decision.kind == physical_enums.PhysicalBackupTypeFull {@@ -407,6 +458,9 @@ func (s *PhysicalBackupsScheduler) claimAndInsert(CreatedAt: now,}).Error})+ if errors.Is(txErr, errBackupDecisionSuperseded) {+ return false, nil+ }if txErr != nil {return false, txErr}@@ -414,6 +468,131 @@ func (s *PhysicalBackupsScheduler) claimAndInsert(return claimed, nil}+func decideImmediateReplacementFull(+ lastFull *physical_models.PhysicalFullBackup,+ lastIncr *physical_models.PhysicalIncrementalBackup,+) (backupDecision, bool) {+ if lastFull != nil && lastFull.Status == physical_enums.PhysicalBackupStatusError &&+ hasFullBackupErrorReason(lastFull, physical_enums.PhysicalBackupErrorFailoverDuringBackup) {+ return backupDecision{+ kind: physical_enums.PhysicalBackupTypeFull,+ reason: "replacing FULL interrupted by failover",+ immediateTrigger: &backupTrigger{+ kind: physical_enums.PhysicalBackupTypeFull,+ backupID: lastFull.ID,+ },+ }, true+ }++ if !hasIncrementalBackupErrorReason(lastIncr, physical_enums.PhysicalBackupErrorTimelineSwitchDetected) ||+ lastIncr.Status != physical_enums.PhysicalBackupStatusChainBroken ||+ lastIncr.CompletedAt == nil {+ return backupDecision{}, false+ }++ if lastFull != nil && lastFull.CreatedAt.After(*lastIncr.CompletedAt) {+ return backupDecision{}, false+ }++ return backupDecision{+ kind: physical_enums.PhysicalBackupTypeFull,+ reason: "re-anchoring after timeline switch",+ immediateTrigger: &backupTrigger{+ kind: physical_enums.PhysicalBackupTypeIncremental,+ backupID: lastIncr.ID,+ },+ }, true+}++func hasFullBackupErrorReason(+ full *physical_models.PhysicalFullBackup,+ reason physical_enums.PhysicalBackupErrorReason,+) bool {+ return full != nil && full.ErrorReason != nil && *full.ErrorReason == reason+}++func hasIncrementalBackupErrorReason(+ incremental *physical_models.PhysicalIncrementalBackup,+ reason physical_enums.PhysicalBackupErrorReason,+) bool {+ return incremental != nil && incremental.ErrorReason != nil && *incremental.ErrorReason == reason+}++func hasUnresolvedSystemIdentifierMismatch(db *gorm.DB, databaseID uuid.UUID) (bool, error) {+ var state struct {+ LatestMismatchAt *time.Time+ LatestCompletedFullAt *time.Time+ }++ err := db.Raw(`+ SELECT+ (+ SELECT MAX(terminal_at)+ FROM (+ SELECT COALESCE(completed_at, created_at) AS terminal_at FROM physical_full_backups+ WHERE database_id = ? AND error_reason = ?+ UNION ALL+ SELECT COALESCE(completed_at, created_at) AS terminal_at FROM physical_incremental_backups+ WHERE database_id = ? AND error_reason = ?+ ) mismatches+ ) AS latest_mismatch_at,+ (+ SELECT MAX(completed_at) FROM physical_full_backups+ WHERE database_id = ? AND status = ?+ ) AS latest_completed_full_at+ `,+ databaseID, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch,+ databaseID, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch,+ databaseID, physical_enums.PhysicalBackupStatusCompleted,+ ).Scan(&state).Error+ if err != nil {+ return false, err+ }++ return state.LatestMismatchAt != nil &&+ (state.LatestCompletedFullAt == nil || !state.LatestCompletedFullAt.After(*state.LatestMismatchAt)), nil+}++func isImmediateReplacementTriggerCurrent(+ tx *gorm.DB,+ databaseID uuid.UUID,+ trigger backupTrigger,+) (bool, error) {+ hasUnresolvedMismatch, err := hasUnresolvedSystemIdentifierMismatch(tx, databaseID)+ if err != nil || hasUnresolvedMismatch {+ return false, err+ }++ var lastFull physical_models.PhysicalFullBackup+ fullErr := tx.Where("database_id = ?", databaseID).Order("created_at DESC").First(&lastFull).Error+ if fullErr != nil && !errors.Is(fullErr, gorm.ErrRecordNotFound) {+ return false, fullErr+ }++ if trigger.kind == physical_enums.PhysicalBackupTypeFull {+ return fullErr == nil && lastFull.ID == trigger.backupID &&+ lastFull.Status == physical_enums.PhysicalBackupStatusError &&+ hasFullBackupErrorReason(&lastFull, physical_enums.PhysicalBackupErrorFailoverDuringBackup), nil+ }++ var lastIncr physical_models.PhysicalIncrementalBackup+ if err := tx.Where("database_id = ?", databaseID).Order("created_at DESC").First(&lastIncr).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return false, nil+ }++ return false, err+ }++ if lastIncr.ID != trigger.backupID || lastIncr.CompletedAt == nil ||+ lastIncr.Status != physical_enums.PhysicalBackupStatusChainBroken ||+ !hasIncrementalBackupErrorReason(&lastIncr, physical_enums.PhysicalBackupErrorTimelineSwitchDetected) {+ return false, nil+ }++ return errors.Is(fullErr, gorm.ErrRecordNotFound) || !lastFull.CreatedAt.After(*lastIncr.CompletedAt), nil+}+// recoverInFlightBackupsOnRestart reconciles the IN_PROGRESS backups the DB still// holds when this instance (re)started. In single-instance mode the goroutine that// was running each backup died with the previous process, so every in-flight claimdiff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go b/backend/internal/features/backups/backups/backuping/physical/scheduler_test.goindex cad6efe2..d1f385dc 100644--- a/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go@@ -37,8 +37,7 @@ func makeIncrementalDue(prereqs *backupPrereqs) {}// seedFullWithStatusAndAge persists a FULL of the given status, aged by ageHours,-// so the cadence / chain-state decision tests are deterministic. Non-COMPLETED-// fulls clear CompletedAt — only COMPLETED fulls anchor an extendable chain.+// so the cadence and chain-state decision tests are deterministic.func seedFullWithStatusAndAge(t *testing.T,prereqs *backupPrereqs,@@ -53,10 +52,6 @@ func seedFullWithStatusAndAge(full.CreatedAt = time.Now().UTC().Add(-time.Duration(ageHours) * time.Hour)full.Status = status- if status != physical_enums.PhysicalBackupStatusCompleted {- full.CompletedAt = nil- }-return physical_testing.CreateTestFullBackup(t, full)}@@ -78,11 +73,244 @@ func seedIncrWithStatusAndAge(incr.CreatedAt = time.Now().UTC().Add(-time.Duration(ageHours) * time.Hour)incr.Status = status- if status != physical_enums.PhysicalBackupStatusCompleted {- incr.CompletedAt = nil+ return physical_testing.CreateTestIncrementalBackup(t, incr)+}++func setFullErrorReason(+ t *testing.T,+ full *physical_models.PhysicalFullBackup,+ reason physical_enums.PhysicalBackupErrorReason,+) {+ t.Helper()++ full.ErrorReason = &reason+ require.NoError(t, storage.GetDb().Save(full).Error)+}++func setIncrementalErrorReason(+ t *testing.T,+ incremental *physical_models.PhysicalIncrementalBackup,+ reason physical_enums.PhysicalBackupErrorReason,+) {+ t.Helper()++ incremental.ErrorReason = &reason+ require.NoError(t, storage.GetDb().Save(incremental).Error)+}++func Test_DecideBackupKind_WhenIncrementalDetectsTimelineSwitch_SchedulesImmediateFull(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ makeIncrementalDue(prereqs)+ rootFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusCompleted, 1, 1)+ brokenIncremental := seedIncrWithStatusAndAge(+ t, prereqs, rootFull.ID, physical_enums.PhysicalBackupStatusChainBroken, 2, 0,+ )+ setIncrementalErrorReason(t, brokenIncremental, physical_enums.PhysicalBackupErrorTimelineSwitchDetected)++ decision, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,+ )++ require.True(t, canSchedule)+ require.Equal(t, physical_enums.PhysicalBackupTypeFull, decision.kind)+ require.NotNil(t, decision.immediateTrigger)+ require.Equal(t, brokenIncremental.ID, decision.immediateTrigger.backupID)+}++func Test_DecideBackupKind_WhenFullFailsDuringFailover_SchedulesImmediateFull(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ prereqs.Config.FullBackupInterval = intervals.Interval{Type: intervals.IntervalWeekly}+ failedFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusError, 1, 0)+ setFullErrorReason(t, failedFull, physical_enums.PhysicalBackupErrorFailoverDuringBackup)++ decision, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,+ )++ require.True(t, canSchedule)+ require.NotNil(t, decision.immediateTrigger)+ require.Equal(t, failedFull.ID, decision.immediateTrigger.backupID)+}++func Test_ClaimAndInsert_WhenImmediateTriggerWasReplaced_RejectsStaleDecision(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ makeIncrementalDue(prereqs)+ rootFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusCompleted, 1, 2)+ brokenIncremental := seedIncrWithStatusAndAge(+ t, prereqs, rootFull.ID, physical_enums.PhysicalBackupStatusChainBroken, 2, 1,+ )+ setIncrementalErrorReason(t, brokenIncremental, physical_enums.PhysicalBackupErrorTimelineSwitchDetected)++ scheduler := CreateTestPhysicalScheduler()+ staleDecision, canSchedule := scheduler.decideBackupKind(logger.GetLogger(), time.Now().UTC(), prereqs.Config)+ require.True(t, canSchedule)++ replacementFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusError, 3, 0)+ replacementFull.CreatedAt = brokenIncremental.CompletedAt.Add(time.Second)+ require.NoError(t, storage.GetDb().Save(replacementFull).Error)++ claimed, err := scheduler.claimAndInsert(prereqs.Config, uuid.New(), staleDecision)++ require.NoError(t, err)+ require.False(t, claimed)+}++func Test_ClaimAndInsert_WhenMismatchAppearsAfterAutomaticDecision_RejectsStaleDecision(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ scheduler := CreateTestPhysicalScheduler()+ automaticDecision, canSchedule := scheduler.decideBackupKind(+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,+ )+ require.True(t, canSchedule)++ mismatchedFull := seedFullWithStatusAndAge(+ t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 0,+ )+ setFullErrorReason(t, mismatchedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)++ claimed, err := scheduler.claimAndInsert(prereqs.Config, uuid.New(), automaticDecision)++ require.NoError(t, err)+ require.False(t, claimed)+}++func Test_ClaimAndInsert_WhenMismatchCommitsWhileClaimWaits_RejectsStaleAutomaticDecision(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ scheduler := CreateTestPhysicalScheduler()+ automaticDecision, canSchedule := scheduler.decideBackupKind(+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,+ )+ require.True(t, canSchedule)++ terminalTransaction := storage.GetDb().Begin()+ require.NoError(t, terminalTransaction.Error)+ t.Cleanup(func() { _ = terminalTransaction.Rollback().Error })++ mismatchedFull := physical_testing.NewTestCompletedFullBackup(+ prereqs.DB.ID,+ prereqs.Storage.ID,+ 1,+ testLSN(1),+ testLSN(2),+ )+ mismatchReason := physical_enums.PhysicalBackupErrorSystemIdentifierMismatch+ mismatchedFull.Status = physical_enums.PhysicalBackupStatusChainBroken+ mismatchedFull.ErrorReason = &mismatchReason+ require.NoError(t, terminalTransaction.Create(mismatchedFull).Error)++ isOccupied, err := scheduler.inFlightRepo.Claim(terminalTransaction, physical_repositories.ClaimSpec{+ DatabaseID: prereqs.DB.ID,+ BackupType: physical_enums.PhysicalBackupTypeFull,+ BackupID: uuid.New(),+ })+ require.NoError(t, err)+ require.True(t, isOccupied)++ type claimResult struct {+ isClaimed bool+ err error}+ claimResults := make(chan claimResult, 1)+ go func() {+ isClaimed, claimErr := scheduler.claimAndInsert(+ prereqs.Config,+ uuid.New(),+ automaticDecision,+ )+ claimResults <- claimResult{isClaimed: isClaimed, err: claimErr}+ }()- return physical_testing.CreateTestIncrementalBackup(t, incr)+ select {+ case <-claimResults:+ t.Fatal("automatic claim must wait for the pending terminal transaction")+ case <-time.After(100 * time.Millisecond):+ }++ require.NoError(t, terminalTransaction.Delete(+ &physical_models.PhysicalInFlightBackup{},+ "database_id = ?",+ prereqs.DB.ID,+ ).Error)+ require.NoError(t, terminalTransaction.Commit().Error)++ result := <-claimResults+ require.NoError(t, result.err)+ require.False(t, result.isClaimed)++ inFlightBackup, err := scheduler.inFlightRepo.FindByDatabaseID(prereqs.DB.ID)+ require.NoError(t, err)+ require.Nil(t, inFlightBackup)++ inProgressFulls, err := scheduler.fullRepo.FindAllInProgress()+ require.NoError(t, err)+ for _, full := range inProgressFulls {+ require.NotEqual(t, prereqs.DB.ID, full.DatabaseID)+ }+}++func Test_DecideBackupKind_WhenSystemIdentifierMismatchUnresolved_SuppressesAutomaticBackup(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ failedFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 1)+ setFullErrorReason(t, failedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)++ _, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,+ )++ require.False(t, canSchedule)+}++func Test_DecideBackupKind_WhenSystemIdentifierMismatchUnresolved_AllowsForcedFull(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ failedFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 1)+ setFullErrorReason(t, failedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)+ requestedAt := time.Now().UTC()+ prereqs.Config.ForceFullRequestedAt = &requestedAt++ decision, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,+ )++ require.True(t, canSchedule)+ require.Equal(t, physical_enums.PhysicalBackupTypeFull, decision.kind)+ require.NotNil(t, decision.forceFullRequestedAt)+}++func Test_DecideBackupKind_WhenCompletedFullFollowsMismatch_ResumesAutomaticScheduling(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ prereqs.Config.FullBackupInterval = intervals.Interval{Type: intervals.IntervalHourly}+ failedFull := seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 3)+ setFullErrorReason(t, failedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)+ seedFullWithStatusAndAge(t, prereqs, physical_enums.PhysicalBackupStatusCompleted, 2, 2)++ decision, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,+ )++ require.True(t, canSchedule)+ require.Equal(t, physical_enums.PhysicalBackupTypeFull, decision.kind)+}++func Test_DecideBackupKind_WhenUnsuccessfulFullFollowsMismatch_KeepsAutomaticSchedulingSuppressed(t *testing.T) {+ for _, terminalStatus := range []physical_enums.PhysicalBackupStatus{+ physical_enums.PhysicalBackupStatusError,+ physical_enums.PhysicalBackupStatusCanceled,+ } {+ t.Run(string(terminalStatus), func(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ mismatchedFull := seedFullWithStatusAndAge(+ t, prereqs, physical_enums.PhysicalBackupStatusChainBroken, 1, 3,+ )+ setFullErrorReason(t, mismatchedFull, physical_enums.PhysicalBackupErrorSystemIdentifierMismatch)+ seedFullWithStatusAndAge(t, prereqs, terminalStatus, 2, 2)++ _, canSchedule := CreateTestPhysicalScheduler().decideBackupKind(+ logger.GetLogger(), time.Now().UTC(), prereqs.Config,+ )++ require.False(t, canSchedule)+ })+ }}func Test_DecideBackupKind_WhenRecentlyCompletedFullAndIncrNotDue_SchedulesNothing(t *testing.T) {@@ -408,6 +636,40 @@ func Test_ClaimAndInsert_WhenConcurrentClaimSameDatabase_OnlyOneSucceeds(t *testassert.Equal(t, 1, inProgressForDB, "only the winning claim inserts a typed row")}+func Test_ClaimAndInsert_WhenOrphanCleanupLockIsHeld_WaitsForTransaction(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ scheduler := CreateTestPhysicalScheduler()+ cleanupTransaction := storage.GetDb().Begin()+ require.NoError(t, cleanupTransaction.Error)+ t.Cleanup(func() { _ = cleanupTransaction.Rollback().Error })+ require.NoError(t, physical_repositories.AcquireBackupAndOrphanCleanupLock(cleanupTransaction, prereqs.DB.ID))++ type claimResult struct {+ isClaimed bool+ err error+ }+ claimResults := make(chan claimResult, 1)+ go func() {+ isClaimed, err := scheduler.claimAndInsert(+ prereqs.Config,+ uuid.New(),+ backupDecision{kind: physical_enums.PhysicalBackupTypeFull},+ )+ claimResults <- claimResult{isClaimed: isClaimed, err: err}+ }()++ select {+ case <-claimResults:+ t.Fatal("FULL claim must wait while orphan cleanup holds the database lock")+ case <-time.After(100 * time.Millisecond):+ }++ require.NoError(t, cleanupTransaction.Commit().Error)+ result := <-claimResults+ require.NoError(t, result.err)+ require.True(t, result.isClaimed)+}+func Test_EvaluateConfig_WhenInFlightClaimExists_SkipsWithoutNewRow(t *testing.T) {prereqs := seedBackupPrereqs(t)scheduler := CreateTestPhysicalScheduler()diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set.go b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set.goindex 5fcbe723..7cde1ad2 100644--- a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set.go+++ b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set.go@@ -58,7 +58,17 @@ func (s *ChainViewService) ResolveRestoreSet(incrementals, lastStopLSN, lastBackupTime := includedIncrementals(chain, targetTime)- walRun, reachableLSN, reachableTime := contiguousWalRun(chain.WalSegments, lastStopLSN, lastBackupTime)+ restoreWalSegments, err := s.walSegmentRepository.FindByChainSpan(+ databaseID,+ chain.RootFull.TimelineID,+ chain.Span.Start,+ LSNMax,+ )+ if err != nil {+ return nil, err+ }++ walRun, reachableLSN, reachableTime := contiguousWalRun(restoreWalSegments, lastStopLSN, lastBackupTime)// A target later than the furthest point we can actually replay to means a// WAL gap (or simply missing WAL) sits before it. Refuse loudly; the callerdiff --git a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.goindex be5e6207..d3c456fb 100644--- a/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go+++ b/backend/internal/features/backups/backups/core/physical/chain_view/restore_set_test.go@@ -173,6 +173,74 @@ func Test_ResolveRestoreSet_WhenTargetBetweenIncrementals_StopsAtEarlierIncremenassertWalFilenames(t, set, w2.WalFilename, w3.WalFilename, w4.WalFilename)}+func Test_ResolveRestoreSet_WhenTargetDuringNewerFull_UsesOlderChainWalThroughTarget(t *testing.T) {+ t.Parallel()++ prereqs := createChainViewTestPrereqs(t)+ clock := newRestoreClock()++ olderFull := physical_testing.NewTestCompletedFullBackup(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ lsnAt(0)+physical_testing.FirstRecordOffset,+ lsnAt(1)+physical_testing.FirstRecordOffset,+ )+ olderFull.CreatedAt = clock.at(-5)+ olderFull.CompletedAt = new(clock.at(0))+ physical_testing.CreateTestFullBackup(t, olderFull)++ newerFull := physical_testing.NewTestCompletedFullBackup(+ prereqs.database.ID,+ prereqs.storage.ID,+ 1,+ lsnAt(2)+physical_testing.FirstRecordOffset,+ lsnAt(3)+physical_testing.FirstRecordOffset,+ )+ newerFull.CreatedAt = clock.at(5)+ newerFull.CompletedAt = new(clock.at(10))+ physical_testing.CreateTestFullBackup(t, newerFull)++ walBeforeNewerFull := seedWalSegment(+ t,+ prereqs,+ "000000010000000000000001",+ lsnAt(1),+ lsnAt(2),+ clock.at(4),+ )+ walAtNewerFullStart := seedWalSegment(+ t,+ prereqs,+ "000000010000000000000002",+ lsnAt(2),+ lsnAt(3),+ clock.at(6),+ )+ walCoveringTarget := seedWalSegment(+ t,+ prereqs,+ "000000010000000000000003",+ lsnAt(3),+ lsnAt(4),+ clock.at(11),+ )++ target := clock.at(7)++ set, err := chain_view.GetChainViewService().ResolveRestoreSet(prereqs.database.ID, &target)+ require.NoError(t, err)++ assert.Equal(t, olderFull.ID, set.RootFull.ID)+ assertWalFilenames(+ t,+ set,+ walBeforeNewerFull.WalFilename,+ walAtNewerFullStart.WalFilename,+ walCoveringTarget.WalFilename,+ )+}+func Test_ResolveRestoreSet_WhenWalGapBeforeTarget_ReturnsGapError(t *testing.T) {t.Parallel()diff --git a/backend/internal/features/backups/backups/core/physical/enums/enums.go b/backend/internal/features/backups/backups/core/physical/enums/enums.goindex f537a4af..7ddc61ea 100644--- a/backend/internal/features/backups/backups/core/physical/enums/enums.go+++ b/backend/internal/features/backups/backups/core/physical/enums/enums.go@@ -38,6 +38,8 @@ const (PhysicalBackupErrorApplicationRestart PhysicalBackupErrorReason = "APPLICATION_RESTART"PhysicalBackupErrorSystemIdentifierMismatch PhysicalBackupErrorReason = "SYSTEM_IDENTIFIER_MISMATCH"PhysicalBackupErrorTimelineRegression PhysicalBackupErrorReason = "TIMELINE_REGRESSION"+ PhysicalBackupErrorTimelineSwitchDetected PhysicalBackupErrorReason = "TIMELINE_SWITCH_DETECTED"+ PhysicalBackupErrorFailoverDuringBackup PhysicalBackupErrorReason = "FAILOVER_DURING_BACKUP"PhysicalBackupErrorManifestCorrupted PhysicalBackupErrorReason = "MANIFEST_CORRUPTED"PhysicalBackupErrorStartLsnOutsideTimeline PhysicalBackupErrorReason = "START_LSN_OUTSIDE_TIMELINE_RANGE"diff --git a/backend/internal/features/backups/backups/core/physical/enums/enums_test.go b/backend/internal/features/backups/backups/core/physical/enums/enums_test.gonew file mode 100644index 00000000..3ea0e35f--- /dev/null+++ b/backend/internal/features/backups/backups/core/physical/enums/enums_test.go@@ -0,0 +1,19 @@+package physical_enums++import (+ "encoding/json"+ "testing"++ "github.com/stretchr/testify/require"+)++func Test_PhysicalBackupFailoverErrorReasons_SerializeAsStableValues(t *testing.T) {+ reasons := []PhysicalBackupErrorReason{+ PhysicalBackupErrorTimelineSwitchDetected,+ PhysicalBackupErrorFailoverDuringBackup,+ }++ serializedReasons, err := json.Marshal(reasons)+ require.NoError(t, err)+ require.JSONEq(t, `["TIMELINE_SWITCH_DETECTED","FAILOVER_DURING_BACKUP"]`, string(serializedReasons))+}diff --git a/backend/internal/features/backups/backups/core/physical/repositories/coordination.go b/backend/internal/features/backups/backups/core/physical/repositories/coordination.gonew file mode 100644index 00000000..78f77df6--- /dev/null+++ b/backend/internal/features/backups/backups/core/physical/repositories/coordination.go@@ -0,0 +1,59 @@+package physical_repositories++import (+ "errors"++ "github.com/google/uuid"+ "gorm.io/gorm"++ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"+ "databasus-backend/internal/util/walmath"+)++func AcquireBackupAndOrphanCleanupLock(tx *gorm.DB, databaseID uuid.UUID) error {+ return tx.Exec(+ `SELECT pg_advisory_xact_lock(hashtextextended('physical-backup:' || ?::text, 0))`,+ databaseID,+ ).Error+}++func HasInFlightFullBackup(tx *gorm.DB, databaseID uuid.UUID) (bool, error) {+ var claim physical_models.PhysicalInFlightBackup++ err := tx.+ Where("database_id = ? AND backup_type = ?", databaseID, physical_enums.PhysicalBackupTypeFull).+ First(&claim).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return false, nil+ }+ if err != nil {+ return false, err+ }++ return true, nil+}++func HasCompletedFullBackupCoveringWal(+ tx *gorm.DB,+ databaseID uuid.UUID,+ timelineID int,+ walEndLSN walmath.LSN,+) (bool, error) {+ var fullCount int64++ err := tx.Model(&physical_models.PhysicalFullBackup{}).+ Where(+ "database_id = ? AND timeline_id = ? AND status = ? AND start_lsn IS NOT NULL AND start_lsn < ?::pg_lsn",+ databaseID,+ timelineID,+ physical_enums.PhysicalBackupStatusCompleted,+ walEndLSN.String(),+ ).+ Count(&fullCount).Error+ if err != nil {+ return false, err+ }++ return fullCount > 0, nil+}diff --git a/backend/internal/features/backups/backups/core/physical/service/dto.go b/backend/internal/features/backups/backups/core/physical/service/dto.goindex 7a710e74..847a5a53 100644--- a/backend/internal/features/backups/backups/core/physical/service/dto.go+++ b/backend/internal/features/backups/backups/core/physical/service/dto.go@@ -5,8 +5,17 @@ import ("time""github.com/google/uuid"++ "databasus-backend/internal/features/backups/backups/core/physical/chain_view")+type DeleteOrphanWalSegmentsSpec struct {+ DatabaseID uuid.UUID+ TimelineID int+ Span chain_view.LSNRange+ WalByteBudgetMB float64+}+// PhysicalBackupListRow is one row of the merged, paginated backup list across// the FULL, incremental and committed-WAL tables. Type-only columns are NULL for// the other types (scanned into the pointer fields); LSNs arrive as text. It isdiff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.goindex 558ccb40..d1c98550 100644--- a/backend/internal/features/backups/backups/core/physical/service/service.go+++ b/backend/internal/features/backups/backups/core/physical/service/service.go@@ -156,6 +156,60 @@ func (s *PhysicalBackupService) DeleteWalSegmentsInSpan(return deletedRows, deletedMB, nil}+func (s *PhysicalBackupService) DeleteOrphanWalSegmentsInSpan(+ ctx context.Context,+ spec DeleteOrphanWalSegmentsSpec,+) (int, float64, error) {+ var deletedRows int+ var deletedMB float64++ txErr := storage.GetDb().WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := physical_repositories.AcquireBackupAndOrphanCleanupLock(tx, spec.DatabaseID); err != nil {+ return err+ }++ hasInFlightFull, err := physical_repositories.HasInFlightFullBackup(tx, spec.DatabaseID)+ if err != nil {+ return err+ }+ if hasInFlightFull {+ return nil+ }++ hasAnchoringFull, err := physical_repositories.HasCompletedFullBackupCoveringWal(+ tx,+ spec.DatabaseID,+ spec.TimelineID,+ spec.Span.End,+ )+ if err != nil {+ return err+ }+ if hasAnchoringFull {+ return nil+ }++ rows, mb, _, err := s.deleteWalInSpanBudgeted(+ ctx,+ tx,+ spec.DatabaseID,+ spec.TimelineID,+ spec.Span,+ spec.WalByteBudgetMB,+ true,+ )+ deletedRows = rows+ deletedMB = mb++ return err+ })+ if txErr != nil {+ return 0, 0, txErr+ }++ return deletedRows, deletedMB, nil+}+// DeleteIncrementalCascade removes one incremental and every incremental that// descends from it (its children, their children, …), leaves first so the// RESTRICT FK on parent_incremental_backup_id is never violated. The chain'sdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.goindex efa17602..1b578cb6 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go@@ -32,9 +32,7 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS}defer creds.Remove()- // nil onFailover: a detected failover does not refuse a FULL — it proceeds on- // the live TL, and the scheduler observes the bumped TL across two FULL rows.- refusalResult, canProceed := verifyTimelineCompatibility(ctx, spec.CommonBackupSpec, nil)+ refusalResult, preflightTimelineID, canProceed := verifyFullTimelineCompatibility(ctx, spec.CommonBackupSpec)if !canProceed {return refusalResult, nil}@@ -67,7 +65,7 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS}if streamResult.Status != physical_enums.PhysicalBackupStatusCompleted {- result = streamResult+ result = recheckFullStreamFailure(ctx, spec.CommonBackupSpec, preflightTimelineID, streamResult)return nil}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.goindex 2848b943..9b7db5c9 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go@@ -43,7 +43,7 @@ func (uc *CreateIncrementalBackupUsecase) Execute(}defer creds.Remove()- refusalResult, canProceed := verifyIncrTimelineCompatibility(ctx, spec)+ refusalResult, rootFullTimelineID, canProceed := verifyIncrTimelineCompatibility(ctx, spec)if !canProceed {return refusalResult, nil}@@ -102,7 +102,12 @@ func (uc *CreateIncrementalBackupUsecase) Execute(}if streamResult.Status != physical_enums.PhysicalBackupStatusCompleted {- result = streamResult+ result = recheckIncrementalStreamFailure(+ ctx,+ spec.CommonBackupSpec,+ rootFullTimelineID,+ streamResult,+ )return nil}@@ -130,28 +135,42 @@ func (uc *CreateIncrementalBackupUsecase) Execute(return result, nil}-// verifyIncrTimelineCompatibility gates an INCR on timeline/cluster-// compatibility. Unlike a FULL, an INCR cannot extend across a timeline switch —-// a detected failover is chain-killing, and the chain must re-anchor on a fresh-// FULL on the new TL.func verifyIncrTimelineCompatibility(ctx context.Context,spec IncrementalBackupSpec,-) (PhysicalBackupResult, bool) {- return verifyTimelineCompatibility(ctx, spec.CommonBackupSpec,- func(decision *TimelineDecision) (PhysicalBackupResult, bool) {- reason := physical_enums.PhysicalBackupErrorTimelineRegression-- return PhysicalBackupResult{- Status: physical_enums.PhysicalBackupStatusChainBroken,- ErrorReason: &reason,- ErrorMessage: fmt.Sprintf(- "timeline switch detected (live TL %d > known TL): incremental refused, new FULL required",- decision.NewTLI,- ),- CompletedAt: time.Now().UTC(),- }, false- })+) (PhysicalBackupResult, int, bool) {+ rootFull, err := spec.FullRepo.FindByID(spec.Backup.RootFullBackupID)+ if err != nil {+ return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure, "load root FULL", err), 0, false+ }++ if rootFull == nil || rootFull.Status != physical_enums.PhysicalBackupStatusCompleted {+ reason := physical_enums.PhysicalBackupErrorParentManifestMissing++ return PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusChainBroken,+ ErrorReason: &reason,+ ErrorMessage: "root FULL is missing or incomplete",+ CompletedAt: time.Now().UTC(),+ }, 0, false+ }++ conn, err := spec.SourceDB.OpenInspectionConn(ctx, spec.FieldEncryptor)+ if err != nil {+ return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure,+ "open inspection connection", err), 0, false+ }+ defer func() { _ = conn.Close(ctx) }()++ decision, err := CheckIncrementalTimelineCompatibility(ctx, conn, spec.SourceDB, rootFull.TimelineID)+ if err != nil {+ return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure,+ "timeline compatibility check", err), 0, false+ }++ refusalResult, canProceed := timelineRefusalResult(decision, true)++ return refusalResult, rootFull.TimelineID, canProceed}// runSummarizerPreCheck opens an inspection connection and resolves thediff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.goindex 00e2b8ee..d29a876c 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go@@ -1,6 +1,7 @@package usecases_physical_postgresqlimport (+ "context""log/slog""time"@@ -16,6 +17,8 @@ import ("databasus-backend/internal/util/walmath")+type timelineIdentityProbe func(context.Context, int) (*TimelineDecision, error)+type CommonBackupSpec struct {SourceDB *postgresql_physical.PostgresqlPhysicalDatabaseDatabaseName string@@ -27,6 +30,7 @@ type CommonBackupSpec struct {FullRepo *physical_repositories.PhysicalFullBackupRepositoryHistoryRepo *physical_repositories.PhysicalWalHistoryRepositoryLogger *slog.Logger+ timelineProbe timelineIdentityProbe// ProgressListener, when set, receives the streamed size and elapsed time// periodically during the backup so the row's size grows in near-real-time.diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.goindex 3b23269b..b6263013 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go@@ -163,39 +163,49 @@ func buildObjectName()}-// verifyTimelineCompatibility gates a backup on the live timeline/cluster-// verdict. FailoverDetected is the one kind FULL and INCR treat differently — a-// FULL proceeds on the live TL while an INCR must re-anchor on a fresh FULL — so-// the caller supplies onFailover; a nil onFailover treats failover as-// non-breaking (FULL). canProceed=false means the caller returns refusal and stops.-func verifyTimelineCompatibility(+func verifyFullTimelineCompatibility(ctx context.Context,common CommonBackupSpec,- onFailover func(decision *TimelineDecision) (refusal PhysicalBackupResult, canProceed bool),-) (refusal PhysicalBackupResult, canProceed bool) {+) (refusal PhysicalBackupResult, liveTimelineID int, canProceed bool) {conn, err := common.SourceDB.OpenInspectionConn(ctx, common.FieldEncryptor)if err != nil {return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure,- "open inspection connection", err), false+ "open inspection connection", err), 0, false}defer func() { _ = conn.Close(ctx) }()- decision, err := CheckTimelineCompatibility(ctx, conn, common.SourceDB, common.FullRepo, common.HistoryRepo)+ decision, err := CheckFullTimelineCompatibility(ctx, conn, common.SourceDB, common.FullRepo, common.HistoryRepo)if err != nil {return errorResult(physical_enums.PhysicalBackupErrorNetworkFailure,- "timeline compatibility check", err), false+ "timeline compatibility check", err), 0, false}+ refusal, canProceed = timelineRefusalResult(decision, false)++ return refusal, decision.LiveTimelineID, canProceed+}++func timelineRefusalResult(decision *TimelineDecision, refuseNewerTimeline bool) (PhysicalBackupResult, bool) {switch decision.Kind {case TimelineContinue:return PhysicalBackupResult{}, truecase TimelineFailoverDetected:- if onFailover == nil {+ if !refuseNewerTimeline {return PhysicalBackupResult{}, true}- return onFailover(decision)+ reason := physical_enums.PhysicalBackupErrorTimelineSwitchDetected++ return PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusChainBroken,+ ErrorReason: &reason,+ ErrorMessage: fmt.Sprintf(+ "timeline switch detected: root FULL TL %d, live TL %d",+ decision.ExpectedTimelineID, decision.LiveTimelineID,+ ),+ CompletedAt: time.Now().UTC(),+ }, falsecase TimelineRegression:reason := physical_enums.PhysicalBackupErrorTimelineRegression@@ -205,7 +215,7 @@ func verifyTimelineCompatibility(ErrorReason: &reason,ErrorMessage: fmt.Sprintf("timeline regression: expected TL %d, live TL %d",- decision.ExpectedTLI, decision.ActualTLI,+ decision.ExpectedTimelineID, decision.LiveTimelineID,),CompletedAt: time.Now().UTC(),}, false@@ -218,7 +228,7 @@ func verifyTimelineCompatibility(ErrorReason: &reason,ErrorMessage: fmt.Sprintf("system_identifier mismatch: catalog %s, live %s",- decision.ExpectedSysID, decision.ActualSysID,+ decision.ExpectedSystemIdentifier, decision.LiveSystemIdentifier,),CompletedAt: time.Now().UTC(),}, false@@ -226,3 +236,101 @@ func verifyTimelineCompatibility(return PhysicalBackupResult{}, true}++func recheckFullStreamFailure(+ ctx context.Context,+ common CommonBackupSpec,+ preflightTimelineID int,+ streamResult PhysicalBackupResult,+) PhysicalBackupResult {+ if ctx.Err() != nil || streamResult.Status == physical_enums.PhysicalBackupStatusCanceled {+ return streamResult+ }++ decision, err := inspectTimelineAgainstExpected(ctx, common, preflightTimelineID)+ if err != nil {+ common.Logger.WarnContext(ctx, "cluster identity recheck failed after FULL stream failure", "error", err)++ return streamResult+ }++ return classifyFullStreamFailureAfterIdentity(streamResult, preflightTimelineID, decision)+}++func classifyFullStreamFailureAfterIdentity(+ streamResult PhysicalBackupResult,+ preflightTimelineID int,+ decision *TimelineDecision,+) PhysicalBackupResult {+ if decision.Kind == TimelineFailoverDetected {+ reason := physical_enums.PhysicalBackupErrorFailoverDuringBackup++ return PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusError,+ ErrorReason: &reason,+ ErrorMessage: fmt.Sprintf(+ "timeline advanced during FULL: preflight TL %d, live TL %d",+ preflightTimelineID,+ decision.LiveTimelineID,+ ),+ CompletedAt: time.Now().UTC(),+ }+ }++ refusal, canProceed := timelineRefusalResult(decision, false)+ if !canProceed {+ return refusal+ }++ return streamResult+}++func recheckIncrementalStreamFailure(+ ctx context.Context,+ common CommonBackupSpec,+ rootFullTimelineID int,+ streamResult PhysicalBackupResult,+) PhysicalBackupResult {+ if ctx.Err() != nil || streamResult.Status == physical_enums.PhysicalBackupStatusCanceled {+ return streamResult+ }++ decision, err := inspectTimelineAgainstExpected(ctx, common, rootFullTimelineID)+ if err != nil {+ common.Logger.WarnContext(ctx, "cluster identity recheck failed after incremental stream failure", "error", err)++ return streamResult+ }++ return classifyIncrementalStreamFailureAfterIdentity(streamResult, decision)+}++func classifyIncrementalStreamFailureAfterIdentity(+ streamResult PhysicalBackupResult,+ decision *TimelineDecision,+) PhysicalBackupResult {+ refusal, canProceed := timelineRefusalResult(decision, true)+ if !canProceed {+ return refusal+ }++ return streamResult+}++func inspectTimelineAgainstExpected(+ ctx context.Context,+ common CommonBackupSpec,+ expectedTimelineID int,+) (*TimelineDecision, error) {+ if common.timelineProbe != nil {+ return common.timelineProbe(ctx, expectedTimelineID)+ }++ conn, err := common.SourceDB.OpenInspectionConn(ctx, common.FieldEncryptor)+ if err != nil {+ return nil, fmt.Errorf("open inspection connection: %w", err)+ }+ defer func() { _ = conn.Close(context.Background()) }()++ return CheckIncrementalTimelineCompatibility(ctx, conn, common.SourceDB, expectedTimelineID)+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.goindex 913c6290..6f4c9263 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go@@ -27,8 +27,6 @@ import ("databasus-backend/internal/util/walmath")-// TimelineDecisionKind discriminates the outcome of CheckTimelineCompatibility.-// Continue is the green path; the other three are refusal branches.type TimelineDecisionKind intconst (@@ -38,76 +36,107 @@ const (TimelineDifferentCluster)-// TimelineDecision is a discriminated result. Only the fields belonging to-// Kind are meaningful; the rest are zero.type TimelineDecision struct {- Kind TimelineDecisionKind-- NewTLI int // TimelineFailoverDetected — promote-detected target TL-- ExpectedTLI int // TimelineRegression- ActualTLI int // TimelineRegression+ Kind TimelineDecisionKind+ ExpectedTimelineID int+ LiveTimelineID int+ ExpectedSystemIdentifier string+ LiveSystemIdentifier string+}- ExpectedSysID string // TimelineDifferentCluster — db.SystemIdentifier- ActualSysID string // TimelineDifferentCluster — live cluster system_identifier+type timelineComparison struct {+ ExpectedSystemIdentifier *string+ LiveTimelineID int+ LiveSystemIdentifier string+ ExpectedTimelineID int}-// CheckTimelineCompatibility validates the source cluster against the-// catalog before spawning a FULL or INCR. Three refusal branches:-// system_identifier mismatch (different cluster pointed at the same DB row),-// timeline regression (live TL < newest known TL — never legitimate), and-// failover detected (live TL > newest known TL — the new FULL must anchor a-// fresh chain on the new TL). Continue means proceed.-//-// "newest known TL" is derived from GREATEST(MAX(full.timeline_id),-// MAX(history.timeline_id)) per BACKUPING-SCHEDULER-PLAN.md §B step 1.-// First-tick (no fulls, no history) returns Continue — the live TL seeds-// the first FULL's timeline_id.-func CheckTimelineCompatibility(+func CheckFullTimelineCompatibility(ctx context.Context,conn *pgx.Conn,db *postgresql_physical.PostgresqlPhysicalDatabase,fullRepo *physical_repositories.PhysicalFullBackupRepository,historyRepo *physical_repositories.PhysicalWalHistoryRepository,) (*TimelineDecision, error) {- liveTLI, liveSysID, err := readClusterIdentity(ctx, conn)+ liveTimelineID, liveSystemIdentifier, err := readClusterIdentity(ctx, conn)if err != nil {return nil, err}- if db.SystemIdentifier != nil && *db.SystemIdentifier != liveSysID {- return &TimelineDecision{- Kind: TimelineDifferentCluster,- ExpectedSysID: *db.SystemIdentifier,- ActualSysID: liveSysID,- }, nil+ decision := decideTimelineCompatibility(timelineComparison{+ ExpectedSystemIdentifier: db.SystemIdentifier,+ LiveTimelineID: liveTimelineID,+ LiveSystemIdentifier: liveSystemIdentifier,+ })+ if decision.Kind == TimelineDifferentCluster {+ return decision, nil}- knownTLI, err := newestKnownTimeline(db.ParentDatabaseID(), fullRepo, historyRepo)+ knownTimelineID, err := newestKnownTimeline(db.ParentDatabaseID(), fullRepo, historyRepo)+ if err != nil {+ return nil, err+ }++ return decideTimelineCompatibility(timelineComparison{+ ExpectedSystemIdentifier: db.SystemIdentifier,+ LiveTimelineID: liveTimelineID,+ LiveSystemIdentifier: liveSystemIdentifier,+ ExpectedTimelineID: knownTimelineID,+ }), nil+}++func CheckIncrementalTimelineCompatibility(+ ctx context.Context,+ conn *pgx.Conn,+ db *postgresql_physical.PostgresqlPhysicalDatabase,+ rootFullTimelineID int,+) (*TimelineDecision, error) {+ liveTimelineID, liveSystemIdentifier, err := readClusterIdentity(ctx, conn)if err != nil {return nil, err}- if knownTLI == 0 {- return &TimelineDecision{Kind: TimelineContinue}, nil+ return decideTimelineCompatibility(timelineComparison{+ ExpectedSystemIdentifier: db.SystemIdentifier,+ LiveTimelineID: liveTimelineID,+ LiveSystemIdentifier: liveSystemIdentifier,+ ExpectedTimelineID: rootFullTimelineID,+ }), nil+}++func decideTimelineCompatibility(comparison timelineComparison) *TimelineDecision {+ decision := &TimelineDecision{+ Kind: TimelineContinue,+ ExpectedTimelineID: comparison.ExpectedTimelineID,+ LiveTimelineID: comparison.LiveTimelineID,+ LiveSystemIdentifier: comparison.LiveSystemIdentifier,+ ExpectedSystemIdentifier: "",}- if liveTLI < knownTLI {- return &TimelineDecision{- Kind: TimelineRegression,- ExpectedTLI: knownTLI,- ActualTLI: liveTLI,- }, nil+ if comparison.ExpectedSystemIdentifier != nil {+ decision.ExpectedSystemIdentifier = *comparison.ExpectedSystemIdentifier+ if *comparison.ExpectedSystemIdentifier != comparison.LiveSystemIdentifier {+ decision.Kind = TimelineDifferentCluster++ return decision+ }}- if liveTLI > knownTLI {- return &TimelineDecision{- Kind: TimelineFailoverDetected,- NewTLI: liveTLI,- }, nil+ if comparison.ExpectedTimelineID == 0 {+ return decision+ }++ if comparison.LiveTimelineID < comparison.ExpectedTimelineID {+ decision.Kind = TimelineRegression++ return decision+ }++ if comparison.LiveTimelineID > comparison.ExpectedTimelineID {+ decision.Kind = TimelineFailoverDetected}- return &TimelineDecision{Kind: TimelineContinue}, nil+ return decision}// ValidateStartLsnAgainstHistory checks that startLSN falls inside the LSNdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.gonew file mode 100644index 00000000..e9b9aa49--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go@@ -0,0 +1,221 @@+package usecases_physical_postgresql++import (+ "context"+ "errors"+ "io"+ "log/slog"+ "testing"++ "github.com/stretchr/testify/require"++ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"+)++func Test_DecideTimelineCompatibility_WithMatchingAndConflictingIdentity_ReturnsExpectedDecision(t *testing.T) {+ expectedSystemIdentifier := "123"++ tests := []struct {+ name string+ liveTimelineID int+ liveSystemIdentifier string+ expectedTimelineID int+ decisionKind TimelineDecisionKind+ }{+ {+ name: "matching timeline",+ liveTimelineID: 3,+ liveSystemIdentifier: "123",+ expectedTimelineID: 3,+ decisionKind: TimelineContinue,+ },+ {+ name: "newer timeline",+ liveTimelineID: 4,+ liveSystemIdentifier: "123",+ expectedTimelineID: 3,+ decisionKind: TimelineFailoverDetected,+ },+ {+ name: "older timeline",+ liveTimelineID: 2,+ liveSystemIdentifier: "123",+ expectedTimelineID: 3,+ decisionKind: TimelineRegression,+ },+ {+ name: "different cluster",+ liveTimelineID: 3,+ liveSystemIdentifier: "456",+ expectedTimelineID: 3,+ decisionKind: TimelineDifferentCluster,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ decision := decideTimelineCompatibility(timelineComparison{+ ExpectedSystemIdentifier: &expectedSystemIdentifier,+ LiveTimelineID: test.liveTimelineID,+ LiveSystemIdentifier: test.liveSystemIdentifier,+ ExpectedTimelineID: test.expectedTimelineID,+ })++ require.Equal(t, test.decisionKind, decision.Kind)+ require.Equal(t, test.expectedTimelineID, decision.ExpectedTimelineID)+ require.Equal(t, test.liveTimelineID, decision.LiveTimelineID)+ })+ }+}++func Test_ClassifyIncrementalStreamFailureAfterIdentity_WhenTimelineAdvanced_BreaksChain(t *testing.T) {+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed+ originalResult := PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusError,+ ErrorReason: &originalReason,+ }+ decision := &TimelineDecision{+ Kind: TimelineFailoverDetected,+ ExpectedTimelineID: 3,+ LiveTimelineID: 4,+ }++ result := classifyIncrementalStreamFailureAfterIdentity(originalResult, decision)++ require.Equal(t, physical_enums.PhysicalBackupStatusChainBroken, result.Status)+ require.Equal(t, physical_enums.PhysicalBackupErrorTimelineSwitchDetected, *result.ErrorReason)+}++func Test_ClassifyFullStreamFailureAfterIdentity_WhenTimelineAdvanced_ClassifiesFailover(t *testing.T) {+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed+ originalResult := PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusError,+ ErrorReason: &originalReason,+ }+ decision := &TimelineDecision{+ Kind: TimelineFailoverDetected,+ ExpectedTimelineID: 3,+ LiveTimelineID: 4,+ }++ result := classifyFullStreamFailureAfterIdentity(originalResult, 3, decision)++ require.Equal(t, physical_enums.PhysicalBackupStatusError, result.Status)+ require.Equal(t, physical_enums.PhysicalBackupErrorFailoverDuringBackup, *result.ErrorReason)+}++func Test_RecheckFullStreamFailure_WhenCanceled_PreservesResultWithoutProbe(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ reason := physical_enums.PhysicalBackupErrorCanceledByUser+ originalResult := PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusCanceled,+ ErrorReason: &reason,+ }++ probeCalls := 0+ common := CommonBackupSpec{+ timelineProbe: func(context.Context, int) (*TimelineDecision, error) {+ probeCalls++++ return nil, errors.New("unexpected identity probe")+ },+ }++ result := recheckFullStreamFailure(ctx, common, 3, originalResult)++ require.Equal(t, originalResult, result)+ require.Zero(t, probeCalls)+}++func Test_RecheckIncrementalStreamFailure_WhenCanceled_PreservesResultWithoutProbe(t *testing.T) {+ reason := physical_enums.PhysicalBackupErrorCanceledByUser+ originalResult := PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusCanceled,+ ErrorReason: &reason,+ }+ probeCalls := 0+ common := CommonBackupSpec{+ timelineProbe: func(context.Context, int) (*TimelineDecision, error) {+ probeCalls++++ return nil, errors.New("unexpected identity probe")+ },+ }++ result := recheckIncrementalStreamFailure(context.Background(), common, 3, originalResult)++ require.Equal(t, originalResult, result)+ require.Zero(t, probeCalls)+}++func Test_RecheckFullStreamFailure_WhenTimelineAdvanced_ClassifiesFailover(t *testing.T) {+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed+ originalResult := PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusError,+ ErrorReason: &originalReason,+ }+ common := CommonBackupSpec{+ Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),+ timelineProbe: func(_ context.Context, expectedTimelineID int) (*TimelineDecision, error) {+ require.Equal(t, 3, expectedTimelineID)++ return &TimelineDecision{+ Kind: TimelineFailoverDetected,+ ExpectedTimelineID: expectedTimelineID,+ LiveTimelineID: 4,+ }, nil+ },+ }++ result := recheckFullStreamFailure(context.Background(), common, 3, originalResult)++ require.Equal(t, physical_enums.PhysicalBackupStatusError, result.Status)+ require.Equal(t, physical_enums.PhysicalBackupErrorFailoverDuringBackup, *result.ErrorReason)+}++func Test_RecheckIncrementalStreamFailure_WhenTimelineAdvanced_BreaksChain(t *testing.T) {+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed+ originalResult := PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusError,+ ErrorReason: &originalReason,+ }+ common := CommonBackupSpec{+ Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),+ timelineProbe: func(_ context.Context, expectedTimelineID int) (*TimelineDecision, error) {+ require.Equal(t, 3, expectedTimelineID)++ return &TimelineDecision{+ Kind: TimelineFailoverDetected,+ ExpectedTimelineID: expectedTimelineID,+ LiveTimelineID: 4,+ }, nil+ },+ }++ result := recheckIncrementalStreamFailure(context.Background(), common, 3, originalResult)++ require.Equal(t, physical_enums.PhysicalBackupStatusChainBroken, result.Status)+ require.Equal(t, physical_enums.PhysicalBackupErrorTimelineSwitchDetected, *result.ErrorReason)+}++func Test_RecheckStreamFailure_WhenIdentityProbeFails_PreservesOriginalResult(t *testing.T) {+ originalReason := physical_enums.PhysicalBackupErrorPgBasebackupFailed+ originalResult := PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusError,+ ErrorReason: &originalReason,+ ErrorMessage: "original stream failure",+ }+ common := CommonBackupSpec{+ Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),+ timelineProbe: func(context.Context, int) (*TimelineDecision, error) {+ return nil, errors.New("identity probe failed")+ },+ }++ fullResult := recheckFullStreamFailure(context.Background(), common, 3, originalResult)+ incrementalResult := recheckIncrementalStreamFailure(context.Background(), common, 3, originalResult)++ require.Equal(t, originalResult, fullResult)+ require.Equal(t, originalResult, incrementalResult)+}diff --git a/backend/internal/features/databases/databases/postgresql/physical/model.go b/backend/internal/features/databases/databases/postgresql/physical/model.goindex 0c16453a..495878b2 100644--- a/backend/internal/features/databases/databases/postgresql/physical/model.go+++ b/backend/internal/features/databases/databases/postgresql/physical/model.go@@ -810,10 +810,8 @@ func (p *PostgresqlPhysicalDatabase) CreateReplicationOnlyUser(return nil, errors.New("failed to generate unique username after 3 attempts")}-// OpenInspectionConn opens a regular (non-replication) connection to the-// source cluster. Used by the FULL/INCR executors for pre-flight checks-// (timeline.CheckTimelineCompatibility), .history file reads, and-// post-stream LSN validation.+// Replication connections cannot run the SQL identity checks or read timeline+// history files, so inspection uses PostgreSQL's regular protocol.func (p *PostgresqlPhysicalDatabase) OpenInspectionConn(ctx context.Context,encryptor encryption.FieldEncryptor,diff --git a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.goindex 464fb882..ade29433 100644--- a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go+++ b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go@@ -35,6 +35,10 @@ func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(tphysicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)}+func Test_PhysicalBackup_AfterPromotion_ReanchorsIncrementalChain(t *testing.T) {+ physicaltesting.RunPromotionReanchorsIncrementalChain(t, pgVersion, pgImage)+}+func Test_PhysicalRestore_BootViaEntrypointVolumeMount_RecoversBaseRows(t *testing.T) {physicaltesting.RunBootViaEntrypointVolumeMountRecoversBaseRows(t, pgVersion, pgImage)}diff --git a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.goindex 97ed40f7..62c91f00 100644--- a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go+++ b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go@@ -35,6 +35,10 @@ func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(tphysicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)}+func Test_PhysicalBackup_AfterPromotion_ReanchorsIncrementalChain(t *testing.T) {+ physicaltesting.RunPromotionReanchorsIncrementalChain(t, pgVersion, pgImage)+}+func Test_PhysicalRestore_BootViaEntrypointVolumeMount_RecoversBaseRows(t *testing.T) {physicaltesting.RunBootViaEntrypointVolumeMountRecoversBaseRows(t, pgVersion, pgImage)}diff --git a/backend/internal/features/tests/physical/postgresql/shared/bodies.go b/backend/internal/features/tests/physical/postgresql/shared/bodies.goindex 2eff353c..49e60e9d 100644--- a/backend/internal/features/tests/physical/postgresql/shared/bodies.go+++ b/backend/internal/features/tests/physical/postgresql/shared/bodies.go@@ -6,13 +6,17 @@ import ("net/http""os""path/filepath"+ "slices""testing""time"+ "github.com/google/uuid""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"backuping_physical "databasus-backend/internal/features/backups/backups/backuping/physical"+ physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"@@ -179,6 +183,78 @@ func RunWalStreamCatalogsHistoryOnTimelineSwitch(t *testing.T, version, image stwaitForTimelineHistoryOnParent(t, fixture, 2, 60*time.Second)}+func RunPromotionReanchorsIncrementalChain(t *testing.T, version, image string) {+ if testing.Short() {+ t.Skip("clones a standby, promotes it, and restores the replacement chain; skipped in -short")+ }++ primary, standby := containers.StartPhysicalPrimaryWithStandby(t, image)+ router := newPhysicalTestRouter()+ fixture := postgresql_executor.SetupPhysicalDBForScheduledBackupVersion(+ t, standby.Host, standby.Port, version, postgresql_physical.BackupTypeFullIncrementalAndWalStream,+ )+ target := prepareRestoreTarget(t, image)++ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)+ defer cancel()++ primaryConn := openConnAt(t, primary.Host, primary.Port)+ standbyConn := openSourceTestDBConn(t, fixture)+ createMarkerTable(t, ctx, primaryConn)+ insertMarker(t, ctx, primaryConn, "before-promotion", "row-before-promotion")+ waitForMarkerOnStandby(t, ctx, standbyConn, "before-promotion", 60*time.Second)++ enablePhysicalBackupsViaAPI(t, router, fixture, true)+ initialChain := waitForChainBackups(t, router, fixture, 0, 3*time.Minute)+ initialFullID := rootFullBackupID(t, initialChain)++ PromoteStandby(t, ctx, standbyConn)+ insertMarker(t, ctx, standbyConn, "after-promotion", "row-after-promotion")+ _, err := postgresql_executor.GenerateWalActivity(ctx, standbyConn, 16*1024*1024)+ require.NoError(t, err)+ waitForTimelineHistoryOnParent(t, fixture, 2, 60*time.Second)+ triggerIncrementalViaAPI(t, router, fixture)++ deadline := time.Now().UTC().Add(4 * time.Minute)+ var replacementFullID uuid.UUID+ for time.Now().UTC().Before(deadline) {+ incrementals, err := physical_repositories.GetIncrementalBackupRepository().FindAllByRootFull(initialFullID)+ require.NoError(t, err)+ fulls, err := physical_repositories.GetFullBackupRepository().FindCompletedNewestFirstByDatabase(fixture.DB.ID)+ require.NoError(t, err)++ hasTimelineRefusal := slices.ContainsFunc(+ incrementals,+ func(incremental *physical_models.PhysicalIncrementalBackup) bool {+ return incremental.Status == physical_enums.PhysicalBackupStatusChainBroken &&+ incremental.ErrorReason != nil &&+ *incremental.ErrorReason == physical_enums.PhysicalBackupErrorTimelineSwitchDetected+ },+ )+ if hasTimelineRefusal && len(fulls) >= 2 && fulls[0].TimelineID == 2 {+ replacementFullID = fulls[0].ID++ break+ }++ time.Sleep(500 * time.Millisecond)+ }+ require.NotEqual(+ t,+ uuid.Nil,+ replacementFullID,+ "promotion must refuse the stale incremental and complete a timeline-2 FULL",+ )++ backuping_physical.RunOrphanWalCleanupForTest(t, fixture.DB.ID)+ bundle := downloadRestoreBundleViaAPI(t, router, fixture, nil)+ reconstructCluster(t, target, router, image, bundle, nil)+ startRestoredCluster(t, target, image)++ restoredPhases := queryRestoredMarkerRows(t, target)+ assert.ElementsMatch(t, []string{"before-promotion", "after-promotion"}, restoredPhases)+}+// RunBootViaEntrypointVolumeMountRecoversBaseRows reproduces the user's docker-compose flow: restore on// the host in --combine-image mode, then serve the result through the postgres image's own entrypoint// with the output bind-mounted at the image VOLUME (the volume root on PG 18). The booted cluster must