PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (physical): Realign WAL resume point with the slot (issue #680)
Rostislav Dugine2fadbab1cd7
diff --git a/backend/internal/features/backups/backups/backuping/physical/constants.go b/backend/internal/features/backups/backups/backuping/physical/constants.goindex 3ec15f9b..9a8ca04d 100644--- a/backend/internal/features/backups/backups/backuping/physical/constants.go+++ b/backend/internal/features/backups/backups/backuping/physical/constants.go@@ -52,6 +52,9 @@ const (streamerStopTimeout = 30 * time.SecondwalStreamSupervisorJobName = "physical_wal_stream_supervisor"++ // One notification per incident, not one per reclaim cycle.+ chainAlertMinInterval = 15 * time.Minute)// Per-tick WAL deletion budget anchors to the latest FULL's size (a clusterdiff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.goindex e9d31324..0a8d0eee 100644--- a/backend/internal/features/backups/backups/backuping/physical/di.go+++ b/backend/internal/features/backups/backups/backuping/physical/di.go@@ -3,6 +3,7 @@ package backuping_physicalimport ("sync""sync/atomic"+ "time""github.com/google/uuid"@@ -84,6 +85,7 @@ var physicalWalStreamSupervisor = &PhysicalWalStreamSupervisor{sync.Mutex{},make(map[uuid.UUID]*runningStreamer),atomicTime{},+ make(map[chainAlertKey]time.Time),atomic.Bool{},atomic.Bool{},}diff --git a/backend/internal/features/backups/backups/backuping/physical/dto.go b/backend/internal/features/backups/backups/backuping/physical/dto.goindex be50f13e..84478b4d 100644--- a/backend/internal/features/backups/backups/backuping/physical/dto.go+++ b/backend/internal/features/backups/backups/backuping/physical/dto.go@@ -3,6 +3,8 @@ package backuping_physicalimport ("time"+ "github.com/google/uuid"+"databasus-backend/internal/features/backups/backups/core/physical/chain_view"backups_config_physical "databasus-backend/internal/features/backups/config/physical""databasus-backend/internal/features/databases"@@ -22,3 +24,25 @@ type chainCandidate struct {view *chain_view.ChainViewendTs time.Time}++// A WAL chain break is notified once per incident per database, so the throttle+// key carries the kind: a retention warning must not swallow the slot-rebuild+// alert that says a fresh full backup was requested.+type chainAlertKind string++const (+ chainAlertStreamerFailed chainAlertKind = "STREAMER_FAILED"+ chainAlertSlotRebuilt chainAlertKind = "SLOT_REBUILT"+ chainAlertChainAtRisk chainAlertKind = "CHAIN_AT_RISK"+)++type chainAlertKey struct {+ DatabaseID uuid.UUID+ Kind chainAlertKind+}++type chainAlert struct {+ Kind chainAlertKind+ Heading string+ Message string+}diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.goindex d4c2caaa..cfe21a8c 100644--- a/backend/internal/features/backups/backups/backuping/physical/testing.go+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go@@ -75,7 +75,7 @@ func CreateTestPhysicalScheduler() *PhysicalBackupsScheduler {// the production repos and services. A fresh instance (not a copy of the DI// singleton) keeps each test's hasRun/running state isolated and avoids copying// the embedded mutex.-func CreateTestWalStreamSupervisor() *PhysicalWalStreamSupervisor {+func CreateTestWalStreamSupervisor(notificationSender NotificationSender) *PhysicalWalStreamSupervisor {return &PhysicalWalStreamSupervisor{databases.GetDatabaseService(),backups_config_physical.GetBackupConfigService(),@@ -83,7 +83,7 @@ func CreateTestWalStreamSupervisor() *PhysicalWalStreamSupervisor {physical_repositories.GetWalSegmentRepository(),physical_repositories.GetWalHistoryRepository(),physical_repositories.GetWalStreamerRepository(),- notifiers.GetNotifierService(),+ notificationSender,tasks_cancellation.GetTaskCancelManager(),encryption_secrets.GetSecretKeyService(),encryption.GetFieldEncryptor(),@@ -91,6 +91,7 @@ func CreateTestWalStreamSupervisor() *PhysicalWalStreamSupervisor {sync.Mutex{},make(map[uuid.UUID]*runningStreamer),atomicTime{},+ make(map[chainAlertKey]time.Time),atomic.Bool{},atomic.Bool{},}@@ -162,7 +163,7 @@ func StartPhysicalSchedulerForTest(t *testing.T) context.CancelFunc {func StartPhysicalWalStreamSupervisorForTest(t *testing.T) context.CancelFunc {t.Helper()- supervisor := CreateTestWalStreamSupervisor()+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())ctx, cancel := context.WithCancel(context.Background())diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.goindex 6b06eb67..71dfe11a 100644--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go@@ -4,6 +4,7 @@ import ("context""fmt""log/slog"+ "maps""os""path/filepath""slices"@@ -49,6 +50,11 @@ type PhysicalWalStreamSupervisor struct {lastTickTime atomicTime+ // A FAILED streamer is reclaimable on the next tick, so an unfixable break+ // re-enters startStreamer every ~45 s. Without this the operator would get one+ // notification per cycle instead of one per incident.+ lastChainAlertAt map[chainAlertKey]time.Time+hasRun atomic.BoolisReady atomic.Bool}@@ -257,7 +263,8 @@ func (s *PhysicalWalStreamSupervisor) startStreamer(WatchDirRoot: config.GetEnv().DataFolder,WalLagThresholdBytes: backupConfig.WalLagThresholdBytes,OnGapDetected: s.gapNotifier(db, backupConfig),- OnSlotRebuilt: s.slotRebuildFullRequester(logger, backupConfig),+ OnSlotRebuilt: s.slotRebuildFullRequester(logger, db, backupConfig),+ OnChainAtRisk: s.chainRiskNotifier(db, backupConfig),Logger: s.logger,})@@ -275,6 +282,12 @@ func (s *PhysicalWalStreamSupervisor) startStreamer(if err := supervisor.Run(streamerCtx); err != nil {logger.Error("wal stream supervisor exited with error", "error", err)+ s.notifyChainBroken(db, backupConfig, chainAlert{+ Kind: chainAlertStreamerFailed,+ Heading: fmt.Sprintf("WAL streaming stopped for %q", db.Name),+ Message: fmt.Sprintf("database_id=%s error=%s", db.ID, err),+ })+// Mark FAILED so a later tick can reclaim it. A clean// ctx-cancel (shutdown / lifecycle stop) does not reach here with an// error, so the row is left for the cancelling path to handle.@@ -330,6 +343,7 @@ func (s *PhysicalWalStreamSupervisor) gapNotifier(func (s *PhysicalWalStreamSupervisor) slotRebuildFullRequester(logger *slog.Logger,+ db *databases.Database,backupConfig *backups_config_physical.PhysicalBackupConfig,) func(context.Context, string) error {return func(_ context.Context, reason string) error {@@ -339,10 +353,73 @@ func (s *PhysicalWalStreamSupervisor) slotRebuildFullRequester(logger.Warn("requested out-of-cadence full backup after wal slot rebuild", "reason", reason)+ // Dropping and recreating the slot always leaves a WAL gap, so the chain+ // this notifies about is already broken by the time we get here.+ s.notifyChainBroken(db, backupConfig, chainAlert{+ Kind: chainAlertSlotRebuilt,+ Heading: fmt.Sprintf("Physical WAL chain rebuilt for %q", db.Name),+ Message: fmt.Sprintf("database_id=%s reason=%s; a fresh full backup was requested to anchor the new chain",+ db.ID, reason),+ })+return nil}}+func (s *PhysicalWalStreamSupervisor) chainRiskNotifier(+ db *databases.Database,+ backupConfig *backups_config_physical.PhysicalBackupConfig,+) func(postgresql_executor.ChainRiskReport) {+ return func(report postgresql_executor.ChainRiskReport) {+ s.notifyChainBroken(db, backupConfig, chainAlert{+ Kind: chainAlertChainAtRisk,+ Heading: fmt.Sprintf("Physical WAL chain at risk for %q", db.Name),+ Message: fmt.Sprintf("database_id=%s reason=%s slot_wal_status=%s lag_bytes=%d",+ db.ID, report.Reason, report.SlotWalStatus, report.LagBytes),+ })+ }+}++func (s *PhysicalWalStreamSupervisor) notifyChainBroken(+ db *databases.Database,+ backupConfig *backups_config_physical.PhysicalBackupConfig,+ alert chainAlert,+) {+ if !slices.Contains(backupConfig.SendNotificationsOn, backups_config_physical.NotificationChainBroken) {+ return+ }++ if !s.recordChainAlertIfDue(chainAlertKey{DatabaseID: db.ID, Kind: alert.Kind}) {+ return+ }++ notification := notifier_models.Notification{+ Type: notifier_models.NotificationTypeBackupFailed,+ Heading: alert.Heading,+ Message: alert.Message,+ }++ for _, notifier := range db.Notifiers {+ s.notificationSender.SendNotification(¬ifier, notification)+ }+}++func (s *PhysicalWalStreamSupervisor) recordChainAlertIfDue(key chainAlertKey) bool {+ s.mu.Lock()+ defer s.mu.Unlock()++ now := time.Now().UTC()++ if sentAt, wasNotified := s.lastChainAlertAt[key]; wasNotified &&+ now.Sub(sentAt) < chainAlertMinInterval {+ return false+ }++ s.lastChainAlertAt[key] = now++ return true+}+func (s *PhysicalWalStreamSupervisor) stopNonCandidates(candidates map[uuid.UUID]bool) {s.mu.Lock()var toStop []uuid.UUID@@ -382,6 +459,11 @@ func (s *PhysicalWalStreamSupervisor) stopStreamer(databaseID uuid.UUID, shouldRs.mu.Lock()delete(s.running, databaseID)++ maps.DeleteFunc(s.lastChainAlertAt, func(key chainAlertKey, _ time.Time) bool {+ return key.DatabaseID == databaseID+ })+s.mu.Unlock()if err := s.walStreamerRepo.MarkFailed(databaseID); err != nil {diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.goindex e866fa07..22bab209 100644--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go@@ -11,6 +11,8 @@ import (physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"+ backups_config_physical "databasus-backend/internal/features/backups/config/physical"+ "databasus-backend/internal/features/notifiers""databasus-backend/internal/storage")@@ -131,7 +133,7 @@ func Test_ClaimIfClaimable_WhenConcurrentClaims_ExactlyOneWins(t *testing.T) {}func Test_PhysicalWalStreamSupervisorRun_WhenCalledTwice_Panics(t *testing.T) {- supervisor := CreateTestWalStreamSupervisor()+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())ctx := t.Context()@@ -154,7 +156,7 @@ func Test_PhysicalWalStreamSupervisorRun_WhenCalledTwice_Panics(t *testing.T) {func Test_StopStreamer_WhenOwnedStreamerStops_MarksRowFailed(t *testing.T) {prereqs := seedBackupPrereqs(t)- supervisor := CreateTestWalStreamSupervisor()+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())require.NoError(t, physical_repositories.GetWalStreamerRepository().Claim(prereqs.DB.ID))done := make(chan struct{})@@ -182,7 +184,7 @@ func Test_RecoverStreamersOnStartup_WhenRunningRowStale_MarksFailed(t *testing.Tphysical_enums.PhysicalWalStreamerStatusRunning,)- supervisor := CreateTestWalStreamSupervisor()+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())require.NoError(t, supervisor.recoverStreamersOnStartup())@@ -193,7 +195,7 @@ func Test_RecoverStreamersOnStartup_WhenRunningRowStale_MarksFailed(t *testing.T}func Test_RemoveWatchDirIfRequested_WhenCleanupRequested_RemovesQueue(t *testing.T) {- supervisor := CreateTestWalStreamSupervisor()+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())watchDir := filepath.Join(t.TempDir(), "wal-queue")require.NoError(t, os.MkdirAll(watchDir, 0o700))require.NoError(t, os.WriteFile(filepath.Join(watchDir, "segment"), []byte("wal"), 0o600))@@ -205,3 +207,73 @@ func Test_RemoveWatchDirIfRequested_WhenCleanupRequested_RemovesQueue(t *testingrequire.NoDirExists(t, watchDir)}++func Test_NotifyChainBroken_WhenSameIncidentRepeats_SendsOneNotificationPerWindow(t *testing.T) {+ prereqs := seedBackupPrereqs(t)++ sender := &recordingNotificationSender{}+ supervisor := CreateTestWalStreamSupervisor(sender)++ prereqs.Config.SendNotificationsOn = []backups_config_physical.BackupNotificationType{+ backups_config_physical.NotificationChainBroken,+ }++ // A FAILED streamer is reclaimable on the next tick, so the same incident+ // re-enters this path every cycle until the cause is fixed.+ for range 3 {+ supervisor.notifyChainBroken(prereqs.DB, prereqs.Config, chainAlert{+ Kind: chainAlertStreamerFailed,+ Heading: "WAL streaming stopped",+ Message: "database_id=x",+ })+ }++ require.Len(t, sender.sentNotifications, 1, "a reclaim loop must not turn into a notification loop")+ require.Equal(t, "WAL streaming stopped", sender.sentNotifications[0].Notification.Heading)+}++func Test_NotifyChainBroken_WhenConfigDoesNotOptIn_SendsNoNotification(t *testing.T) {+ prereqs := seedBackupPrereqs(t)++ sender := &recordingNotificationSender{}+ supervisor := CreateTestWalStreamSupervisor(sender)++ prereqs.Config.SendNotificationsOn = []backups_config_physical.BackupNotificationType{+ backups_config_physical.NotificationBackupSuccess,+ }++ supervisor.notifyChainBroken(prereqs.DB, prereqs.Config, chainAlert{+ Kind: chainAlertStreamerFailed,+ Heading: "WAL streaming stopped",+ Message: "database_id=x",+ })++ require.Empty(t, sender.sentNotifications)+}++func Test_NotifyChainBroken_WhenDifferentIncidentKinds_SendsEachOnce(t *testing.T) {+ prereqs := seedBackupPrereqs(t)++ sender := &recordingNotificationSender{}+ supervisor := CreateTestWalStreamSupervisor(sender)++ prereqs.Config.SendNotificationsOn = []backups_config_physical.BackupNotificationType{+ backups_config_physical.NotificationChainBroken,+ }++ supervisor.notifyChainBroken(prereqs.DB, prereqs.Config, chainAlert{+ Kind: chainAlertChainAtRisk,+ Heading: "chain at risk",+ Message: "database_id=x",+ })+ supervisor.notifyChainBroken(prereqs.DB, prereqs.Config, chainAlert{+ Kind: chainAlertSlotRebuilt,+ Heading: "chain rebuilt",+ Message: "database_id=x",+ })++ require.Len(t, sender.sentNotifications, 2,+ "a retention warning must not swallow the rebuild alert that reports the new chain anchor")+ require.Equal(t, "chain at risk", sender.sentNotifications[0].Notification.Heading)+ require.Equal(t, "chain rebuilt", sender.sentNotifications[1].Notification.Heading)+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.goindex 42626ef9..617d06df 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go@@ -90,3 +90,11 @@ type ParentManifestRef struct {IV stringStopLSN walmath.LSN}++// Payload of WalStreamSpec.OnChainAtRisk. Reason is a walBreakReason string:+// the type is unexported, and the wiring package cannot name it.+type ChainRiskReport struct {+ Reason string+ SlotWalStatus string+ LagBytes int64+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.goindex db4280b0..2fa20b36 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go@@ -2,6 +2,7 @@ package usecases_physical_postgresqlimport ("context"+ "fmt""log/slog""time")@@ -11,9 +12,9 @@ const (// indexed row).lagMonitorPollInterval = 30 * time.Second- // An 'extended' slot status must persist this long before we treat it as a- // real lag (vs a transient write burst).- extendedSlotStatusHoldPeriod = 5 * time.Minute+ // A warning wal_status must persist this long before we alert on it (vs a+ // transient write burst).+ warningSlotStatusHoldPeriod = 5 * time.Minute// Beyond this many rebuild ATTEMPTS in a sliding hour (counted regardless of// outcome), mechanical retry won't help (creds rotated, pg_hba changed,@@ -32,68 +33,46 @@ const (type walBreakReason stringconst (- breakReasonSlotLost walBreakReason = "SLOT_LOST"- breakReasonWalLag walBreakReason = "WAL_LAG_THRESHOLD"- breakReasonSlotStolen walBreakReason = "SLOT_STOLEN"+ breakReasonSlotLost walBreakReason = "SLOT_LOST"+ breakReasonWalLag walBreakReason = "WAL_LAG_THRESHOLD"+ breakReasonSlotStolen walBreakReason = "SLOT_STOLEN"+ breakReasonSlotRetention walBreakReason = "SLOT_WAL_RETENTION")-// Source-side slot state only; consumer-side liveness is the slot-LSN watcher's-// job (slot_lsn_watcher.go).-func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Logger) {- ticker := time.NewTicker(lagMonitorPollInterval)- defer ticker.Stop()-- var extendedSince time.Time-- for {- select {- case <-ctx.Done():- return-- case <-ticker.C:- reason, shouldRebuild := s.evaluateSlotForBreak(ctx, logger, &extendedSince)- if !shouldRebuild {- continue- }+type slotBreakAction int- logger.Warn("wal_stream_break_observed", "reason", string(reason), "slot", s.slotName)+const (+ slotBreakActionNone slotBreakAction = iota+ slotBreakActionAlert+ slotBreakActionRebuild+)- if err := s.rebuildSlot(ctx, logger, reason); err != nil {- logger.Error("slot rebuild failed", "reason", string(reason), "error", err)- }- }- }+type slotBreakSample struct {+ SlotState *SlotState+ WalLagThresholdBytes int64+ IsReceiverRunning bool+ ObservedAt time.Time}-// extendedSince tracks how long wal_status has been 'extended' across ticks.-func (s *WalStreamSupervisor) evaluateSlotForBreak(- ctx context.Context,- logger *slog.Logger,- extendedSince *time.Time,-) (walBreakReason, bool) {- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)- if err != nil {- logger.Debug("lag monitor: source unreachable this tick", "error", err)-- return "", false- }- defer func() { _ = conn.Close(context.Background()) }()-- state, err := InspectSlot(ctx, conn, s.slotName)- if err != nil || state == nil {- return "", false- }-- return classifySlotBreak(state, s.spec.WalLagThresholdBytes, extendedSince)+type slotBreakClassifier struct {+ warningStatusSince time.Time}-func classifySlotBreak(- state *SlotState,- walLagThresholdBytes int64,- extendedSince *time.Time,-) (walBreakReason, bool) {+// Only 'lost' and a stolen slot are unrecoverable enough to justify dropping the+// slot, which always costs a WAL gap and a fresh FULL. 'extended' merely means+// the slot retains WAL beyond max_wal_size — routine on a busy cluster, and+// guaranteed whenever back pressure pauses us — and 'unreserved' means PG will+// trim that WAL at the next checkpoint, which already protects the primary. Both+// alert instead. The operator's WalLagThresholdBytes stays the explicit+// "sacrifice the chain to protect the primary" knob, but never fires against lag+// we caused by holding the receiver down ourselves.+func (c *slotBreakClassifier) recordSampleAndClassifyBreak(+ sample slotBreakSample,+) (walBreakReason, slotBreakAction) {+ state := sample.SlotState+if state == nil {- return "", false+ return "", slotBreakActionNone}// A foreign backend holding our slot (active, but not one of our own@@ -102,34 +81,96 @@ func classifySlotBreak(// refuses to drop a slot held by a consumer we cannot attribute, so a genuine// third party trips loop-protection rather than getting force-dropped.if state.Active && !isOwnedReceiverBackend(state) {- return breakReasonSlotStolen, true+ return breakReasonSlotStolen, slotBreakActionRebuild}- switch state.WalStatus {- case "lost":- return breakReasonSlotLost, true+ if state.WalStatus == "lost" {+ c.warningStatusSince = time.Time{}- case "unreserved":- return breakReasonWalLag, true+ return breakReasonSlotLost, slotBreakActionRebuild+ }- case "extended":- if extendedSince.IsZero() {- *extendedSince = time.Now().UTC()+ if state.WalStatus == "extended" || state.WalStatus == "unreserved" {+ if c.warningStatusSince.IsZero() {+ c.warningStatusSince = sample.ObservedAt}- if time.Since(*extendedSince) > extendedSlotStatusHoldPeriod {- return breakReasonWalLag, true+ if sample.ObservedAt.Sub(c.warningStatusSince) > warningSlotStatusHoldPeriod {+ return breakReasonSlotRetention, slotBreakActionAlert}- return "", false+ return "", slotBreakActionNone+ }++ c.warningStatusSince = time.Time{}- default:- *extendedSince = time.Time{}+ if sample.WalLagThresholdBytes > 0 &&+ state.LagBytes > sample.WalLagThresholdBytes &&+ sample.IsReceiverRunning {+ return breakReasonWalLag, slotBreakActionRebuild}- if walLagThresholdBytes > 0 && state.LagBytes > walLagThresholdBytes {- return breakReasonWalLag, true+ return "", slotBreakActionNone+}++// Source-side slot state only; consumer-side liveness is the slot-LSN watcher's+// job (slot_lsn_watcher.go).+func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Logger) {+ ticker := time.NewTicker(lagMonitorPollInterval)+ defer ticker.Stop()++ var classifier slotBreakClassifier++ for {+ select {+ case <-ctx.Done():+ return++ case <-ticker.C:+ state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)+ if !isSourceReachable {+ continue+ }++ reason, action := classifier.recordSampleAndClassifyBreak(slotBreakSample{+ SlotState: state,+ WalLagThresholdBytes: s.spec.WalLagThresholdBytes,+ IsReceiverRunning: s.isReceiverRunning.Load(),+ ObservedAt: time.Now().UTC(),+ })++ switch action {+ case slotBreakActionNone:+ continue++ case slotBreakActionAlert:+ s.reportChainAtRisk(logger, reason, state)++ case slotBreakActionRebuild:+ logger.Warn("wal_stream_break_observed", "reason", string(reason), "slot", s.slotName)++ if err := s.rebuildSlot(ctx, logger, reason); err != nil {+ logger.Error("slot rebuild failed", "reason", string(reason), "error", err)+ }+ }+ }+ }+}++func (s *WalStreamSupervisor) reportChainAtRisk(logger *slog.Logger, reason walBreakReason, state *SlotState) {+ logger.Warn(+ fmt.Sprintf("wal chain at risk: slot wal_status=%s, %d bytes behind", state.WalStatus, state.LagBytes),+ "reason", string(reason),+ "slot", s.slotName,+ )++ if s.spec.OnChainAtRisk == nil {+ return}- return "", false+ s.spec.OnChainAtRisk(ChainRiskReport{+ Reason: string(reason),+ SlotWalStatus: state.WalStatus,+ LagBytes: state.LagBytes,+ })}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.goindex 4e85f8df..b04d36b7 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go@@ -7,81 +7,139 @@ import ("github.com/stretchr/testify/require")-func Test_ClassifySlotBreak_WhenWalStatusLost_RebuildsForSlotLost(t *testing.T) {- var extendedSince time.Time+func runningSlotBreakSample(state *SlotState, walLagThresholdBytes int64, observedAt time.Time) slotBreakSample {+ return slotBreakSample{+ SlotState: state,+ WalLagThresholdBytes: walLagThresholdBytes,+ IsReceiverRunning: true,+ ObservedAt: observedAt,+ }+}- reason, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "lost"}, 0, &extendedSince)+func Test_RecordSampleAndClassifyBreak_WhenWalStatusLost_Rebuilds(t *testing.T) {+ var classifier slotBreakClassifier- require.True(t, shouldRebuild)+ reason, action := classifier.recordSampleAndClassifyBreak(+ runningSlotBreakSample(&SlotState{WalStatus: "lost"}, 0, time.Now().UTC()),+ )++ require.Equal(t, slotBreakActionRebuild, action)require.Equal(t, breakReasonSlotLost, reason)}-func Test_ClassifySlotBreak_WhenWalStatusUnreserved_RebuildsForWalLag(t *testing.T) {- var extendedSince time.Time+func Test_RecordSampleAndClassifyBreak_WhenWalStatusUnreserved_AlertsWithoutRebuilding(t *testing.T) {+ var classifier slotBreakClassifier- reason, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "unreserved"}, 0, &extendedSince)+ firstSeenAt := time.Now().UTC()+ state := &SlotState{WalStatus: "unreserved"}- require.True(t, shouldRebuild)- require.Equal(t, breakReasonWalLag, reason)+ _, immediateAction := classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 0, firstSeenAt))+ require.Equal(t, slotBreakActionNone, immediateAction,+ "unreserved is a warning; it must hold before it is worth telling anyone about")++ reason, heldAction := classifier.recordSampleAndClassifyBreak(+ runningSlotBreakSample(state, 0, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second)),+ )++ require.Equal(t, slotBreakActionAlert, heldAction,+ "PG trims that WAL at the next checkpoint, so dropping the slot would only add a gap")+ require.Equal(t, breakReasonSlotRetention, reason)}-func Test_ClassifySlotBreak_WhenExtendedPersistsPastHold_RebuildsForWalLag(t *testing.T) {- extendedSince := time.Now().UTC().Add(-extendedSlotStatusHoldPeriod - time.Second)+func Test_RecordSampleAndClassifyBreak_WhenWalStatusExtendedPastHold_AlertsWithoutRebuilding(t *testing.T) {+ var classifier slotBreakClassifier- reason, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "extended"}, 0, &extendedSince)+ firstSeenAt := time.Now().UTC()+ state := &SlotState{WalStatus: "extended"}- require.True(t, shouldRebuild)- require.Equal(t, breakReasonWalLag, reason)+ classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 0, firstSeenAt))++ reason, action := classifier.recordSampleAndClassifyBreak(+ runningSlotBreakSample(state, 0, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second)),+ )++ require.Equal(t, slotBreakActionAlert, action,+ "retaining WAL beyond max_wal_size is routine on a busy cluster, not a reason to drop the slot")+ require.Equal(t, breakReasonSlotRetention, reason)}-func Test_ClassifySlotBreak_WhenLagExceedsThreshold_RebuildsForWalLag(t *testing.T) {- var extendedSince time.Time+func Test_RecordSampleAndClassifyBreak_WhenLagOverThresholdAndReceiverRunning_Rebuilds(t *testing.T) {+ var classifier slotBreakClassifier- reason, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "reserved", LagBytes: 101}, 100, &extendedSince)+ reason, action := classifier.recordSampleAndClassifyBreak(+ runningSlotBreakSample(&SlotState{WalStatus: "reserved", LagBytes: 101}, 100, time.Now().UTC()),+ )- require.True(t, shouldRebuild)+ require.Equal(t, slotBreakActionRebuild, action)require.Equal(t, breakReasonWalLag, reason)}-func Test_ClassifySlotBreak_WhenForeignConsumerHoldsSlot_RebuildsForSlotStolen(t *testing.T) {- var extendedSince time.Time+func Test_RecordSampleAndClassifyBreak_WhenLagOverThresholdAndReceiverStopped_DoesNotRebuild(t *testing.T) {+ var classifier slotBreakClassifier++ _, action := classifier.recordSampleAndClassifyBreak(slotBreakSample{+ SlotState: &SlotState{WalStatus: "reserved", LagBytes: 101},+ WalLagThresholdBytes: 100,+ IsReceiverRunning: false,+ ObservedAt: time.Now().UTC(),+ })++ require.Equal(t, slotBreakActionNone, action,+ "lag we caused by holding the receiver down must never cost the chain")+}++func Test_RecordSampleAndClassifyBreak_WhenForeignConsumerHoldsSlot_Rebuilds(t *testing.T) {+ var classifier slotBreakClassifier- foreignPID := 4242state := &SlotState{WalStatus: "reserved",Active: true,- ActivePID: &foreignPID,+ ActivePID: new(4242),ApplicationName: "some_other_consumer",}- reason, shouldRebuild := classifySlotBreak(state, 0, &extendedSince)+ reason, action := classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 0, time.Now().UTC()))- require.True(t, shouldRebuild)+ require.Equal(t, slotBreakActionRebuild, action)require.Equal(t, breakReasonSlotStolen, reason)}-func Test_ClassifySlotBreak_WhenOurReceiverActive_DoesNotRebuild(t *testing.T) {- var extendedSince time.Time+func Test_RecordSampleAndClassifyBreak_WhenOurReceiverActive_DoesNotRebuild(t *testing.T) {+ var classifier slotBreakClassifier- ourPID := 17state := &SlotState{WalStatus: "reserved",Active: true,- ActivePID: &ourPID,+ ActivePID: new(17),ApplicationName: receivewalApplicationNamePrefix + "db-1",LagBytes: 10,}- _, shouldRebuild := classifySlotBreak(state, 100, &extendedSince)+ _, action := classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 100, time.Now().UTC()))- require.False(t, shouldRebuild)+ require.Equal(t, slotBreakActionNone, action)}-func Test_ClassifySlotBreak_WhenSlotHealthy_DoesNotRebuildAndClearsExtendedSample(t *testing.T) {- extendedSince := time.Now().UTC().Add(-extendedSlotStatusHoldPeriod - time.Second)+func Test_RecordSampleAndClassifyBreak_WhenSlotHealthy_DoesNotRebuildAndClearsWarningClock(t *testing.T) {+ var classifier slotBreakClassifier++ firstSeenAt := time.Now().UTC()++ classifier.recordSampleAndClassifyBreak(+ runningSlotBreakSample(&SlotState{WalStatus: "extended"}, 100, firstSeenAt),+ )++ _, healthyAction := classifier.recordSampleAndClassifyBreak(+ runningSlotBreakSample(&SlotState{WalStatus: "reserved", LagBytes: 10}, 100, firstSeenAt.Add(time.Minute)),+ )+ require.Equal(t, slotBreakActionNone, healthyAction)- _, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "reserved", LagBytes: 10}, 100, &extendedSince)+ _, actionAfterRecovery := classifier.recordSampleAndClassifyBreak(+ runningSlotBreakSample(+ &SlotState{WalStatus: "extended"}, 100, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second),+ ),+ )- require.False(t, shouldRebuild)- require.True(t, extendedSince.IsZero())+ require.Equal(t, slotBreakActionNone, actionAfterRecovery,+ "a recovered slot restarts the hold window instead of inheriting the previous one")}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.goindex 35ca0335..ce59daf4 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go@@ -131,7 +131,7 @@ func Test_WalStreamOverMtls_StreamerArchivesSegments(t *testing.T) {store := newMockWalStorage()- stop := startStreamerForTest(t, fixture, store)+ stop := StartWalStreamerForTest(t, fixture, store, t.TempDir()).Stopt.Cleanup(stop)adminConn := OpenAdminConn(t, fixture)diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.goindex d7e3fff0..95ce3905 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go@@ -25,8 +25,15 @@ const (receiverInternalRestart // our own SIGTERM (back pressure / slot stall) — respawn promptlyreceiverRetryable // non-zero exit that a local respawn may fix (network)receiverFatal // non-retryable exit — escalate so the supervisor reclaims it on a later tick+ receiverResumeMismatch // asked for WAL the server no longer has — realign the resume point and retry)+type receiverRunResult struct {+ Exit receiverExit+ RanFor time.Duration+ FatalErr error+}+type receivewalCommandSpec struct {PgBin stringSourceDB *postgresql_physical.PostgresqlPhysicalDatabase@@ -39,12 +46,12 @@ func (s *WalStreamSupervisor) spawnAndSupervise(ctx context.Context,logger *slog.Logger,pgBin string,-) (receiverExit, time.Duration, error) {+) receiverRunResult {password, err := postgresql_shared.DecryptFieldIfNeeded(s.spec.SourceDB.Password, s.spec.FieldEncryptor)if err != nil {logger.Error("decrypt source password for pg_receivewal", "error", err)- return receiverRetryable, 0, nil+ return receiverRunResult{Exit: receiverRetryable}}creds, err := postgresql_shared.WriteCredentialFilesToTempDir(@@ -53,7 +60,7 @@ func (s *WalStreamSupervisor) spawnAndSupervise(if err != nil {logger.Error("write pg_receivewal credentials", "error", err)- return receiverRetryable, 0, nil+ return receiverRunResult{Exit: receiverRetryable}}defer creds.Remove()@@ -70,22 +77,25 @@ func (s *WalStreamSupervisor) spawnAndSupervise(if err != nil {logger.Error("build pg_receivewal command", "error", err)- return receiverRetryable, 0, nil+ return receiverRunResult{Exit: receiverRetryable}}stderrPipe, err := cmd.StderrPipe()if err != nil {logger.Error("pg_receivewal stderr pipe", "error", err)- return receiverRetryable, 0, nil+ return receiverRunResult{Exit: receiverRetryable}}if err := cmd.Start(); err != nil {logger.Error("start pg_receivewal", "error", err)- return receiverRetryable, 0, nil+ return receiverRunResult{Exit: receiverRetryable}}+ s.isReceiverRunning.Store(true)+ defer s.isReceiverRunning.Store(false)+stderr := newStderrCapture(stderrPipe)startedAt := time.Now().UTC()@@ -101,7 +111,7 @@ func (s *WalStreamSupervisor) spawnAndSupervise(<-exitedstderr.stop()- return receiverCtxCancelled, 0, nil+ return receiverRunResult{Exit: receiverCtxCancelled}case <-s.restartSignal:logger.Info("restarting pg_receivewal on internal signal (back pressure or slot stall)")@@ -109,31 +119,42 @@ func (s *WalStreamSupervisor) spawnAndSupervise(<-exitedstderr.stop()- return receiverInternalRestart, time.Since(startedAt), nil+ return receiverRunResult{Exit: receiverInternalRestart, RanFor: time.Since(startedAt)}case waitErr := <-exited:stderr.stop()ranFor := time.Since(startedAt)if waitErr == nil || procCtx.Err() != nil {- return receiverRetryable, ranFor, nil+ return receiverRunResult{Exit: receiverRetryable, RanFor: ranFor}}stderrText := stderr.contents()+ if isResumeMismatchError(stderrText) {+ logger.Warn("pg_receivewal asked for WAL the server has recycled; realigning the resume point",+ "error", waitErr, "stderr", truncateStderr(stderrText))++ return receiverRunResult{Exit: receiverResumeMismatch, RanFor: ranFor}+ }+if isFatalReceivewalError(stderrText) {logger.Error("pg_receivewal exited with a non-retryable error; marking streamer for reassignment","error", waitErr, "stderr", truncateStderr(stderrText))- return receiverFatal, ranFor, fmt.Errorf(- "pg_receivewal fatal error: %w; stderr: %s", waitErr, truncateStderr(stderrText),- )+ return receiverRunResult{+ Exit: receiverFatal,+ RanFor: ranFor,+ FatalErr: fmt.Errorf(+ "pg_receivewal fatal error: %w; stderr: %s", waitErr, truncateStderr(stderrText),+ ),+ }}logger.Warn("pg_receivewal exited; will respawn","error", waitErr, "stderr", truncateStderr(stderrText))- return receiverRetryable, ranFor, nil+ return receiverRunResult{Exit: receiverRetryable, RanFor: ranFor}}}@@ -161,6 +182,13 @@ func isFatalReceivewalError(stderr []byte) bool {return false}+// The resume point pg_receivewal derived from the local queue sits below what+// the server still has. A local respawn alone repeats it forever, so this is its+// own disposition: the supervision loop realigns the queue and tries again.+func isResumeMismatchError(stderr []byte) bool {+ return strings.Contains(strings.ToLower(string(stderr)), "has already been removed")+}+// WAL is left uncompressed locally (no --compress) because the uploader// re-compresses with zstd on upload; --no-loop makes the process exit on// connection loss so the supervision loop owns retry; --synchronous flushes eachdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.goindex 73dcf401..a888dd0f 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go@@ -49,12 +49,33 @@ func Test_IsFatalReceivewalError_ClassifiesNonRetryableStderr(t *testing.T) {require.True(t, isFatalReceivewalError([]byte(stderr)), stderr)}- transient := []string{+ transientStderrs := []string{"pg_receivewal: error: could not receive data from WAL stream: server closed the connection unexpectedly","pg_receivewal: error: connection to server failed: Connection refused","",}- for _, stderr := range transient {+ for _, stderr := range transientStderrs {require.False(t, isFatalReceivewalError([]byte(stderr)), stderr)}}++func Test_IsResumeMismatchError_WhenSegmentAlreadyRemoved_ReturnsTrue(t *testing.T) {+ stderr := "pg_receivewal: error: could not send replication command \"START_REPLICATION\": " ++ "ERROR: requested WAL segment 0000000100000000E000002F has already been removed"++ require.True(t, isResumeMismatchError([]byte(stderr)))+ require.False(t, isFatalReceivewalError([]byte(stderr)),+ "a realign fixes this, so it must not escalate the streamer to FAILED")+}++func Test_IsResumeMismatchError_WhenTransientStderr_ReturnsFalse(t *testing.T) {+ transientStderrs := []string{+ "pg_receivewal: error: could not receive data from WAL stream: server closed the connection unexpectedly",+ "pg_receivewal: error: connection to server failed: Connection refused",+ "",+ }++ for _, stderr := range transientStderrs {+ require.False(t, isResumeMismatchError([]byte(stderr)), stderr)+ }+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_cleanup_listener_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_cleanup_listener_test.goindex 604df2c1..d6969170 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_cleanup_listener_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_cleanup_listener_test.go@@ -99,7 +99,7 @@ func Test_DropWalSlot_WhenStreamerSlotActive_RefusesAndKeepsSlot(t *testing.T) {adminConn := postgresql_executor.OpenAdminConn(t, fixture)streamerSlot := fixture.DB.PostgresqlPhysical.ReplicationSlotName- stop := postgresql_executor.StartWalStreamerForTest(t, fixture, fixture.Storage)+ stop := postgresql_executor.StartWalStreamerForTest(t, fixture, fixture.Storage, t.TempDir()).Stopdefer stop()waitUntilSlotActive(t, adminConn, streamerSlot, 30*time.Second)diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.goindex 40c13c8b..0e599483 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go@@ -2,6 +2,7 @@ package usecases_physical_postgresqlimport ("context"+ "fmt""log/slog""time"@@ -13,31 +14,49 @@ const (// actually flushing.slotLsnWatcherPollInterval = 10 * time.Second- // restart_lsn unchanged this long on a healthy PG means pg_receivewal is+ // restart_lsn unchanged this long while WAL is waiting means pg_receivewal is// stuck (alive but flushing nothing); restart it locally.slotLsnStallTimeout = 60 * time.Second)+type slotLivenessSample struct {+ RestartLSN walmath.LSN+ LagBytes int64+ IsReceiverRunning bool+ ObservedAt time.Time+}+type stallTracker struct {lastRestartLSN walmath.LSNlastAdvanceAt time.TimehasSample bool}-// A changed LSN (or the very first sample) re-arms the advance clock. On a-// positive result the clock re-arms too, so the caller restarts at most once per-// stallTimeout window.-func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeout time.Duration) bool {- if !t.hasSample || restartLSN != t.lastRestartLSN {- t.lastRestartLSN = restartLSN- t.lastAdvanceAt = now+// A frozen restart_lsn is only a stall when the source has WAL we have not taken+// yet and a receiver is up to take it: an idle database legitimately parks+// restart_lsn, and so does a receiver we are deliberately holding down for a+// rebuild or back pressure. A positive result re-arms the clock, so the caller+// restarts at most once per stallTimeout window.+func (t *stallTracker) recordSampleAndDetectStall(+ sample slotLivenessSample,+ stallTimeout time.Duration,+) bool {+ if sample.LagBytes == 0 || !sample.IsReceiverRunning {+ t.reset()++ return false+ }++ if !t.hasSample || sample.RestartLSN != t.lastRestartLSN {+ t.lastRestartLSN = sample.RestartLSN+ t.lastAdvanceAt = sample.ObservedAtt.hasSample = truereturn false}- if now.Sub(t.lastAdvanceAt) > stallTimeout {- t.lastAdvanceAt = now+ if sample.ObservedAt.Sub(t.lastAdvanceAt) > stallTimeout {+ t.lastAdvanceAt = sample.ObservedAtreturn true}@@ -45,9 +64,40 @@ func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeoreturn false}-// A frozen restart_lsn on a reachable source is a stuck consumer on a healthy-// server. A stall with an unreachable server is left to the lag monitor (slot-// loss / network down).+func (t *stallTracker) reset() {+ t.hasSample = false+ t.lastRestartLSN = 0+ t.lastAdvanceAt = time.Time{}+}++// isSourceReachable=false means defer to the lag monitor.+func (s *WalStreamSupervisor) GetSlotStateIfReachable(+ ctx context.Context,+ logger *slog.Logger,+) (state *SlotState, isSourceReachable bool) {+ conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)+ if err != nil {+ logger.Debug("slot-lsn watcher: source unreachable, deferring to lag monitor", "error", err)++ return nil, false+ }+ defer func() { _ = conn.Close(context.Background()) }()++ state, err = InspectSlot(ctx, conn, s.slotName)+ if err != nil || state == nil {+ return nil, false+ }++ var alive int+ if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {+ return nil, false+ }++ return state, true+}++// A stall with an unreachable server is left to the lag monitor (slot loss /+// network down).func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slog.Logger) {ticker := time.NewTicker(slotLsnWatcherPollInterval)defer ticker.Stop()@@ -60,40 +110,27 @@ func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *sloreturncase <-ticker.C:- restartLSN, pgReachable := s.sampleSlotRestartLSN(ctx, logger)- if !pgReachable {+ state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)+ if !isSourceReachable {continue}- if tracker.observe(restartLSN, time.Now().UTC(), slotLsnStallTimeout) {- logger.Warn("slot restart_lsn stalled on a reachable source; restarting pg_receivewal",- "restart_lsn", restartLSN.String())+ sample := slotLivenessSample{+ RestartLSN: state.RestartLSN,+ LagBytes: state.LagBytes,+ IsReceiverRunning: s.isReceiverRunning.Load(),+ ObservedAt: time.Now().UTC(),+ }++ if tracker.recordSampleAndDetectStall(sample, slotLsnStallTimeout) {+ logger.Warn(+ fmt.Sprintf("slot restart_lsn stalled with %d bytes pending; restarting pg_receivewal",+ state.LagBytes),+ "restart_lsn", state.RestartLSN.String(),+ )s.signalRestart()}}}}--// pgReachable=false means defer to the lag monitor.-func (s *WalStreamSupervisor) sampleSlotRestartLSN(ctx context.Context, logger *slog.Logger) (walmath.LSN, bool) {- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)- if err != nil {- logger.Debug("slot-lsn watcher: source unreachable, deferring to lag monitor", "error", err)-- return 0, false- }- defer func() { _ = conn.Close(context.Background()) }()-- state, err := InspectSlot(ctx, conn, s.slotName)- if err != nil || state == nil {- return 0, false- }-- var alive int- if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {- return 0, false- }-- return state.RestartLSN, true-}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.goindex f017532e..dcaeef34 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go@@ -9,46 +9,109 @@ import ("databasus-backend/internal/util/walmath")-func Test_StallTracker_WhenFirstSample_DoesNotRestart(t *testing.T) {+func streamingSample(restartLSN walmath.LSN, observedAt time.Time) slotLivenessSample {+ return slotLivenessSample{+ RestartLSN: restartLSN,+ LagBytes: 16 * 1024 * 1024,+ IsReceiverRunning: true,+ ObservedAt: observedAt,+ }+}++func Test_RecordSampleAndDetectStall_WhenFirstSample_DoesNotDetectStall(t *testing.T) {var tracker stallTrackerbase := time.Now().UTC()- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute),+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute),"the first sample only arms the clock; it can never be a stall")}-func Test_StallTracker_WhenRestartLsnAdvances_ReArmsAndDoesNotRestart(t *testing.T) {+func Test_RecordSampleAndDetectStall_WhenRestartLsnAdvances_ReArmsAndDoesNotDetectStall(t *testing.T) {var tracker stallTrackerbase := time.Now().UTC()- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))- require.False(t, tracker.observe(walmath.LSN(200), base.Add(2*time.Minute), time.Minute),+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute))+ require.False(t,+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(200), base.Add(2*time.Minute)), time.Minute),"a changed restart_lsn means progress — the advance clock must reset")}-func Test_StallTracker_WhenFrozenWithinTimeout_DoesNotRestart(t *testing.T) {+func Test_RecordSampleAndDetectStall_WhenFrozenWithinTimeout_DoesNotDetectStall(t *testing.T) {var tracker stallTrackerbase := time.Now().UTC()- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))- require.False(t, tracker.observe(walmath.LSN(100), base.Add(30*time.Second), time.Minute),+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute))+ require.False(t,+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(30*time.Second)), time.Minute),"a frozen restart_lsn within the stall timeout is not yet a stall")}-func Test_StallTracker_WhenFrozenPastTimeout_RestartsThenReArms(t *testing.T) {+func Test_RecordSampleAndDetectStall_WhenFrozenPastTimeout_DetectsStallThenReArms(t *testing.T) {var tracker stallTrackerbase := time.Now().UTC()- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))- require.True(t, tracker.observe(walmath.LSN(100), base.Add(90*time.Second), time.Minute),+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute))+ require.True(t,+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(90*time.Second)), time.Minute),"a frozen restart_lsn past the stall timeout must trigger a restart")- require.False(t, tracker.observe(walmath.LSN(100), base.Add(2*time.Minute), time.Minute),+ require.False(t,+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(2*time.Minute)), time.Minute),"after firing, the clock re-arms so we restart at most once per window")- require.True(t, tracker.observe(walmath.LSN(100), base.Add(4*time.Minute), time.Minute),+ require.True(t,+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(4*time.Minute)), time.Minute),"a sustained stall fires again only after another full timeout window")}++func Test_RecordSampleAndDetectStall_WhenLsnFrozenAndNoLag_DoesNotDetectStall(t *testing.T) {+ var tracker stallTracker++ base := time.Now().UTC()++ idleSample := func(observedAt time.Time) slotLivenessSample {+ return slotLivenessSample{+ RestartLSN: walmath.LSN(100),+ LagBytes: 0,+ IsReceiverRunning: true,+ ObservedAt: observedAt,+ }+ }++ require.False(t, tracker.recordSampleAndDetectStall(idleSample(base), time.Minute))+ require.False(t, tracker.recordSampleAndDetectStall(idleSample(base.Add(10*time.Minute)), time.Minute),+ "an idle database parks restart_lsn legitimately — there is nothing left to flush")+}++func Test_RecordSampleAndDetectStall_WhenLsnFrozenWithLag_DetectsStall(t *testing.T) {+ var tracker stallTracker++ base := time.Now().UTC()++ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute))+ require.True(t,+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(90*time.Second)), time.Minute),+ "WAL is waiting and restart_lsn has not moved — the receiver is wedged")+}++func Test_RecordSampleAndDetectStall_WhenReceiverNotRunning_DoesNotDetectStall(t *testing.T) {+ var tracker stallTracker++ base := time.Now().UTC()++ stoppedSample := func(observedAt time.Time) slotLivenessSample {+ return slotLivenessSample{+ RestartLSN: walmath.LSN(100),+ LagBytes: 16 * 1024 * 1024,+ IsReceiverRunning: false,+ ObservedAt: observedAt,+ }+ }++ require.False(t, tracker.recordSampleAndDetectStall(stoppedSample(base), time.Minute))+ require.False(t, tracker.recordSampleAndDetectStall(stoppedSample(base.Add(10*time.Minute)), time.Minute),+ "a receiver we are deliberately holding down is not a stalled one")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.goindex fc22e93c..2afb2505 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go@@ -61,6 +61,11 @@ func (s *WalStreamSupervisor) rebuildSlot(ctx context.Context, logger *slog.Loggreturn fmt.Errorf("recreate slot: %w", err)}+ // The new slot reserves from the cluster's current WAL position, so the queue+ // we just stopped writing to now sits below it and would drag pg_receivewal's+ // resume point into recycled WAL.+ s.realignResumePath(ctx, logger)+if s.spec.OnSlotRebuilt != nil {if err := s.spec.OnSlotRebuilt(ctx, string(reason)); err != nil {return fmt.Errorf("request full after slot rebuild: %w", err)diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.goindex 9b4e43d3..507d2f0b 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go@@ -4,6 +4,7 @@ import ("context""errors""fmt"+ "path/filepath""strings""sync/atomic""testing"@@ -711,20 +712,27 @@ func CountCommittedWalSegments(t *testing.T, databaseID uuid.UUID) int64 {return count}-// StartWalStreamerForTest runs a WalStreamSupervisor against the fixture's-// source PG in a goroutine, archiving rotated segments into store, and returns a-// stop func that cancels and waits for a clean drain (so pg_receivewal releases-// the slot before the DB and its slot are torn down). Cross-package backup→-// restore tests pass the real fixture.Storage so archived segments can be read-// back and replayed.-func StartWalStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store storages.StorageFileSaver) func() {+// WalStreamerForTest exposes the watch dir so a test can restart a streamer+// against the queue an earlier run left behind.+type WalStreamerForTest struct {+ Supervisor *WalStreamSupervisor+ WatchDir string+ Stop func()+}++// A replication slot is meant to outlive a streamer restart (so WAL is never+// lost), so the supervisor deliberately does NOT drop it on shutdown. In tests+// that would leak one slot per run until max_replication_slots is exhausted, so+// drop this streamer's slot here. Registered before the caller's stop cleanup,+// it runs (LIFO) right after the streamer has stopped and before the conn closes.+func StartWalStreamerForTest(+ t *testing.T,+ fixture *PhysicalDBFixture,+ store storages.StorageFileSaver,+ watchDirRoot string,+) *WalStreamerForTest {t.Helper()- // A replication slot is meant to outlive a streamer restart (so WAL is never- // lost), so the supervisor deliberately does NOT drop it on shutdown. In tests- // that would leak one slot per run until max_replication_slots is exhausted, so- // drop this streamer's slot here. Registered before the caller's stop cleanup,- // it runs (LIFO) right after the streamer has stopped and before the conn closes.adminConn := OpenAdminConn(t, fixture)t.Cleanup(func() {DropReplicationSlotExternally(t, adminConn, fixture.DB.PostgresqlPhysical.ReplicationSlotName)@@ -739,7 +747,7 @@ func StartWalStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store stoFieldEncryptor: encryption.GetFieldEncryptor(),WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),HistoryRepo: physical_repositories.GetWalHistoryRepository(),- WatchDirRoot: t.TempDir(),+ WatchDirRoot: watchDirRoot,Logger: logger.GetLogger(),})@@ -752,14 +760,18 @@ func StartWalStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store sto_ = supervisor.Run(ctx)}()- return func() {- cancel()+ return &WalStreamerForTest{+ Supervisor: supervisor,+ WatchDir: filepath.Join(watchDirRoot, "wal-queue", fixture.DB.ID.String()),+ Stop: func() {+ cancel()- select {- case <-done:- case <-time.After(30 * time.Second):- t.Log("streamer did not stop within timeout")- }+ select {+ case <-done:+ case <-time.After(30 * time.Second):+ t.Log("streamer did not stop within timeout")+ }+ },}}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.goindex 0c0ae05c..1d8ab98b 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go@@ -15,7 +15,7 @@ const (// uploads drain below LOW. The 5x hysteresis prevents flapping on the// boundary. HIGH scales up for clusters with non-default wal_segment_size so// one segment does not stop the receiver.- walLocalMinHighWatermarkBytes int64 = 100 * 1024 * 1024+ walLocalMinHighWatermarkBytes int64 = 512 * 1024 * 1024backpressurePollInterval = 1 * time.Second)diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.goindex a6730189..ecf615e4 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go@@ -20,6 +20,6 @@ func Test_WalStream_BackpressureWatermarks_ScaleWithWalSegmentSize(t *testing.T)Logger: logger.GetLogger(),})- require.Equal(t, 4*customSegSize, supervisor.highWatermarkBytes)- require.Equal(t, 4*customSegSize/5, supervisor.lowWatermarkBytes)+ require.Equal(t, 8*customSegSize, supervisor.highWatermarkBytes)+ require.Equal(t, 8*customSegSize/5, supervisor.lowWatermarkBytes)}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.gonew file mode 100644index 00000000..eaef9124--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go@@ -0,0 +1,137 @@+package usecases_physical_postgresql++import (+ "context"+ "fmt"+ "log/slog"+ "os"+ "path/filepath"++ "databasus-backend/internal/util/walmath"+)++const pendingUploadDirName = "pending-upload"++// Mirrors pg_receivewal's FindStreamingStart: it starts at the segment after the+// highest complete file in --directory, and falls back to the slot's restart_lsn+// only when it finds none (PG 15+).+func GetResumeSegmentNo(+ watchDir string,+ segmentSizeBytes int64,+) (resumeSegmentNo walmath.WalSegmentNo, isResumeSegmentFound bool) {+ entries, err := os.ReadDir(watchDir)+ if err != nil {+ return 0, false+ }++ var highestSegmentNo uint64++ for _, entry := range entries {+ if entry.IsDir() {+ continue+ }++ _, segmentNo, err := walmath.ParseWALFilenameWithSize(entry.Name(), uint64(segmentSizeBytes))+ if err != nil {+ continue+ }++ // pg_receivewal ignores a short file: a torn segment is re-streamed, not resumed past.+ info, err := entry.Info()+ if err != nil || info.Size() != segmentSizeBytes {+ continue+ }++ if !isResumeSegmentFound || segmentNo > highestSegmentNo {+ highestSegmentNo = segmentNo+ isResumeSegmentFound = true+ }+ }++ if !isResumeSegmentFound {+ return 0, false+ }++ return walmath.WalSegmentNo(highestSegmentNo).Next(), true+}++// pg_receivewal reads only the top level of --directory, so a pending segment+// below the slot's restart_lsn drags the resume point into WAL the server has+// already recycled. Moving rather than deleting keeps those segments archivable:+// they are valid WAL of the older chain.+func movePendingUploadsOutOfResumePath(+ watchDir string,+ resumeFloor walmath.WalSegmentNo,+ segmentSizeBytes int64,+) (movedCount int, err error) {+ entries, err := os.ReadDir(watchDir)+ if err != nil {+ return 0, fmt.Errorf("read wal watch dir: %w", err)+ }++ pendingUploadDir := filepath.Join(watchDir, pendingUploadDirName)+ if err := os.MkdirAll(pendingUploadDir, 0o700); err != nil {+ return 0, fmt.Errorf("create pending upload dir: %w", err)+ }++ for _, entry := range entries {+ if entry.IsDir() {+ continue+ }++ _, segmentNo, err := walmath.ParseWALFilenameWithSize(entry.Name(), uint64(segmentSizeBytes))+ if err != nil || walmath.WalSegmentNo(segmentNo) >= resumeFloor {+ continue+ }++ moveErr := os.Rename(+ filepath.Join(watchDir, entry.Name()),+ filepath.Join(pendingUploadDir, entry.Name()),+ )+ if moveErr != nil && !os.IsNotExist(moveErr) {+ return movedCount, fmt.Errorf("move %s out of the resume path: %w", entry.Name(), moveErr)+ }++ movedCount+++ }++ return movedCount, nil+}++// walmath.NewWalSegmentNo divides by the package-global segment size, which is+// wrong for a cluster with a non-default wal_segment_size.+func segmentNoAtLSN(lsn walmath.LSN, segmentSizeBytes int64) walmath.WalSegmentNo {+ return walmath.WalSegmentNo(uint64(lsn) / uint64(segmentSizeBytes))+}++func (s *WalStreamSupervisor) realignResumePath(ctx context.Context, logger *slog.Logger) {+ state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)+ if !isSourceReachable {+ return+ }++ segmentSizeBytes := walSegmentSizeBytes(s.spec.SourceDB)++ resumeSegmentNo, isResumeSegmentFound := GetResumeSegmentNo(s.watchDir, segmentSizeBytes)+ if !isResumeSegmentFound {+ return+ }++ slotSegmentNo := segmentNoAtLSN(state.RestartLSN, segmentSizeBytes)+ if resumeSegmentNo >= slotSegmentNo {+ return+ }++ movedCount, err := movePendingUploadsOutOfResumePath(s.watchDir, slotSegmentNo, segmentSizeBytes)+ if err != nil {+ logger.Error("failed to realign wal resume point", "error", err)++ return+ }++ logger.Warn(+ fmt.Sprintf("realigned wal resume point, moved %d segments out of the resume path", movedCount),+ "slot_restart_lsn", state.RestartLSN.String(),+ "resume_segment_no", uint64(resumeSegmentNo),+ )+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume_test.gonew file mode 100644index 00000000..e0283c31--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume_test.go@@ -0,0 +1,91 @@+package usecases_physical_postgresql++import (+ "bytes"+ "os"+ "path/filepath"+ "testing"++ "github.com/stretchr/testify/require"++ "databasus-backend/internal/util/walmath"+)++func writeSegmentOfSize(t *testing.T, dir, name string, sizeBytes int64) {+ t.Helper()++ require.NoError(t, os.WriteFile(filepath.Join(dir, name), bytes.Repeat([]byte{0}, int(sizeBytes)), 0o600))+}++func Test_GetResumeSegmentNo_WithCompletePartialAndTornFiles_ReturnsHighestCompletePlusOne(t *testing.T) {+ watchDir := t.TempDir()++ writeSegmentOfSize(t, watchDir, walName(1, 40), testWalSegmentSize)+ writeSegmentOfSize(t, watchDir, walName(1, 41), testWalSegmentSize)+ writeSegmentOfSize(t, watchDir, walName(1, 42), testWalSegmentSize/2)+ writeSegmentOfSize(t, watchDir, walName(1, 43)+".partial", testWalSegmentSize)+ writeSegmentOfSize(t, watchDir, "00000002.history", 64)++ resumeSegmentNo, isResumeSegmentFound := GetResumeSegmentNo(watchDir, testWalSegmentSize)++ require.True(t, isResumeSegmentFound)+ require.Equal(t, walmath.WalSegmentNo(42), resumeSegmentNo,+ "a torn segment is re-streamed, so 42 is not a resume point — 41 is the highest complete one")+}++func Test_GetResumeSegmentNo_WhenWatchDirHasNoSegments_ReportsNotFound(t *testing.T) {+ watchDir := t.TempDir()++ writeSegmentOfSize(t, watchDir, "00000002.history", 64)+ require.NoError(t, os.MkdirAll(filepath.Join(watchDir, pendingUploadDirName), 0o700))+ writeSegmentOfSize(t, filepath.Join(watchDir, pendingUploadDirName), walName(1, 40), testWalSegmentSize)++ _, isResumeSegmentFound := GetResumeSegmentNo(watchDir, testWalSegmentSize)++ require.False(t, isResumeSegmentFound,+ "pg_receivewal reads only the top level, so a staged segment must not anchor the resume point")+}++func Test_MovePendingUploadsOutOfResumePath_MovesOnlySegmentsBelowFloor(t *testing.T) {+ watchDir := t.TempDir()++ for _, segmentNo := range []uint64{40, 41, 42, 43} {+ writeSegmentOfSize(t, watchDir, walName(1, segmentNo), testWalSegmentSize)+ }++ movedCount, err := movePendingUploadsOutOfResumePath(watchDir, walmath.WalSegmentNo(42), testWalSegmentSize)+ require.NoError(t, err)+ require.Equal(t, 2, movedCount)++ for _, movedSegment := range []uint64{40, 41} {+ require.NoFileExists(t, filepath.Join(watchDir, walName(1, movedSegment)))+ require.FileExists(t, filepath.Join(watchDir, pendingUploadDirName, walName(1, movedSegment)))+ }++ for _, keptSegment := range []uint64{42, 43} {+ require.FileExists(t, filepath.Join(watchDir, walName(1, keptSegment)))+ }+}++func Test_MovePendingUploadsOutOfResumePath_LeavesHistoryFilesAndSubdirectories(t *testing.T) {+ watchDir := t.TempDir()++ writeSegmentOfSize(t, watchDir, "00000002.history", 64)+ require.NoError(t, os.MkdirAll(filepath.Join(watchDir, "archive_status"), 0o700))++ movedCount, err := movePendingUploadsOutOfResumePath(watchDir, walmath.WalSegmentNo(99), testWalSegmentSize)+ require.NoError(t, err)+ require.Zero(t, movedCount)++ require.FileExists(t, filepath.Join(watchDir, "00000002.history"))+ require.DirExists(t, filepath.Join(watchDir, "archive_status"))+}++func Test_SegmentNoAtLsn_WithNonDefaultSegmentSize_UsesTheClusterSize(t *testing.T) {+ const customSegmentSize = int64(64 * 1024 * 1024)++ lsn := walmath.LSN(3 * uint64(customSegmentSize))++ require.Equal(t, walmath.WalSegmentNo(3), segmentNoAtLSN(lsn, customSegmentSize),+ "the package-global segment size would give 12 here")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.goindex f7949d85..cc4cc009 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go@@ -40,6 +40,11 @@ const (// a local respawn can never fix (ENOSPC, bad creds, a slot held by a thief).receivewalMaxRapidFailures = 5+ // A realign empties the resume path, so the very next spawn starts from the+ // slot. A second mismatch means the slot itself no longer covers what the+ // server retains — only a rebuild clears that.+ receivewalMaxResumeMismatches = 2+pausePollInterval = 1 * time.Second)@@ -70,6 +75,10 @@ type WalStreamSpec struct {// request a fresh base backup that anchors the new WAL chain.OnSlotRebuilt func(ctx context.Context, reason string) error+ // Fires while the chain is still intact but degrading (slot retention+ // warnings, a wedged receiver); nil disables notification.+ OnChainAtRisk func(report ChainRiskReport)+Logger *slog.Logger}@@ -95,6 +104,10 @@ type WalStreamSupervisor struct {// drop+recreate the slot without the receiver re-attaching.isPaused atomic.Bool+ // Lets the slot-LSN watcher and the lag monitor tell a real problem from lag+ // we are causing ourselves by holding the receiver down.+ isReceiverRunning atomic.Bool+// rebuildMu serializes slot rebuilds in this process; rebuildTimestamps powers// the per-hour loop-protection cap. One supervisor owns a DB at a time (the// physical_wal_streamers heartbeat claim), so this is the only guard needed.@@ -118,7 +131,7 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {OnGapDetected: spec.OnGapDetected,})- highWatermarkBytes := max(walLocalMinHighWatermarkBytes, 4*walSegmentSizeBytes(spec.SourceDB))+ highWatermarkBytes := max(walLocalMinHighWatermarkBytes, 8*walSegmentSizeBytes(spec.SourceDB))return &WalStreamSupervisor{spec: spec,@@ -141,6 +154,10 @@ func (s *WalStreamSupervisor) Run(ctx context.Context) error {return fmt.Errorf("create wal watch dir: %w", err)}+ if err := os.MkdirAll(filepath.Join(s.watchDir, pendingUploadDirName), 0o700); err != nil {+ return fmt.Errorf("create pending upload dir: %w", err)+ }+if err := s.spec.SourceDB.VerifyWalSlot(ctx, logger, s.spec.FieldEncryptor); err != nil {return fmt.Errorf("verify persistent replication slot: %w", err)}@@ -193,6 +210,7 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, loggpgBin := tools.GetPostgresqlExecutable(s.spec.SourceDB.Version, tools.PostgresqlExecutablePgReceivewal)respawnBackoff := receivewalRespawnBackoffrapidFailures := 0+ resumeMismatches := 0for {if ctx.Err() != nil {@@ -211,29 +229,50 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg// signal raised while no process was running.s.drainRestartSignal()s.removePartials(logger)+ s.realignResumePath(ctx, logger)- outcome, ranFor, fatalErr := s.spawnAndSupervise(ctx, logger, pgBin)+ receiverRun := s.spawnAndSupervise(ctx, logger, pgBin)- switch outcome {+ switch receiverRun.Exit {case receiverCtxCancelled:return nilcase receiverFatal:- return fatalErr+ return receiverRun.FatalErrcase receiverInternalRestart:// Our own SIGTERM (back pressure or slot stall): respawn promptly; the// top-of-loop backlog/pause gates already throttle the cause.respawnBackoff = receivewalRespawnBackoffrapidFailures = 0+ resumeMismatches = 0++ continue++ case receiverResumeMismatch:+ resumeMismatches++++ if resumeMismatches >= receivewalMaxResumeMismatches {+ resumeMismatches = 0++ if err := s.rebuildSlot(ctx, logger, breakReasonSlotLost); err != nil {+ return fmt.Errorf("rebuild slot after repeated wal resume mismatch: %w", err)+ }+ }++ if !sleepCtx(ctx, receivewalRespawnBackoff) {+ return nil+ }continue}+ resumeMismatches = 0+// receiverRetryable: a run that streamed for a healthy span is a transient// blip (network) — reset the crash-loop counter; a string of sub-uptime// exits is a crash loop a local respawn cannot fix, so escalate.- if ranFor >= receivewalMinHealthyUptime {+ if receiverRun.RanFor >= receivewalMinHealthyUptime {rapidFailures = 0respawnBackoff = receivewalRespawnBackoff} else {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.goindex 7549d9ac..0360035b 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go@@ -2,62 +2,24 @@ package usecases_physical_postgresqlimport ("context"+ "os"+ "path/filepath""testing""time""github.com/google/uuid"+ "github.com/jackc/pgx/v5""github.com/stretchr/testify/require"backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums""databasus-backend/internal/features/backups/backups/core/physical/chain_view"physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"- "databasus-backend/internal/features/storages""databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger""databasus-backend/internal/util/walmath")-// The stop func waits for full drain, so pg_receivewal has released the slot-// before the DB (and its slot) are torn down by later cleanups.-func startStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store storages.StorageFileSaver) func() {- t.Helper()-- spec := WalStreamSpec{- DatabaseID: fixture.DB.ID,- SourceDB: fixture.DB.PostgresqlPhysical,- StorageID: fixture.Storage.ID,- Storage: store,- Encryption: backups_core_enums.BackupEncryptionNone,- FieldEncryptor: encryption.GetFieldEncryptor(),- WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),- HistoryRepo: physical_repositories.GetWalHistoryRepository(),- WatchDirRoot: t.TempDir(),- Logger: logger.GetLogger(),- }-- supervisor := NewWalStreamSupervisor(spec)-- ctx, cancel := context.WithCancel(t.Context())- done := make(chan struct{})-- go func() {- defer close(done)-- _ = supervisor.Run(ctx)- }()-- return func() {- cancel()-- select {- case <-done:- case <-time.After(30 * time.Second):- t.Log("streamer did not stop within timeout")- }- }-}-func Test_WalStream_FullIncrementalAndWalStream_StreamerArchivesSegments(t *testing.T) {if testing.Short() {t.Skip("streamer integration test runs pg_receivewal; skipped in -short")@@ -70,7 +32,7 @@ func Test_WalStream_FullIncrementalAndWalStream_StreamerArchivesSegments(t *teststore := newMockWalStorage()- stop := startStreamerForTest(t, fixture, store)+ stop := StartWalStreamerForTest(t, fixture, store, t.TempDir()).Stopt.Cleanup(stop)adminConn := OpenAdminConn(t, fixture)@@ -167,8 +129,8 @@ func Test_WalStream_MultipleDbs_EachArchivesSegmentsIndependently(t *testing.T)storeA := newMockWalStorage()storeB := newMockWalStorage()- t.Cleanup(startStreamerForTest(t, fixtureA, storeA))- t.Cleanup(startStreamerForTest(t, fixtureB, storeB))+ t.Cleanup(StartWalStreamerForTest(t, fixtureA, storeA, t.TempDir()).Stop)+ t.Cleanup(StartWalStreamerForTest(t, fixtureB, storeB, t.TempDir()).Stop)connA := OpenAdminConn(t, fixtureA)@@ -204,7 +166,7 @@ func Test_WalStream_MissingSegmentInStreamedChain_SurfacesAsGapChainStaysExtendastore := newMockWalStorage()adminConn := OpenAdminConn(t, fixture)- t.Cleanup(startStreamerForTest(t, fixture, store))+ t.Cleanup(StartWalStreamerForTest(t, fixture, store, t.TempDir()).Stop)ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)defer cancel()@@ -266,7 +228,7 @@ func Test_WalStream_SlotLagGrowsWithoutConsumer_DrainsOnceStreaming(t *testing.TWaitUntilSlotLag(t, adminConn, slotName, lagTarget, 30*time.Second)// Once our streamer attaches, it consumes the backlog and the lag drains.- t.Cleanup(startStreamerForTest(t, fixture, newMockWalStorage()))+ t.Cleanup(StartWalStreamerForTest(t, fixture, newMockWalStorage(), t.TempDir()).Stop)deadline := time.Now().UTC().Add(60 * time.Second)for time.Now().UTC().Before(deadline) {@@ -351,3 +313,177 @@ func Test_Cleaner_AbandonedNullClaim_OlderThanGrace_DeletedYoungerSurvives(t *terequire.Nil(t, findWalSegment(t, fixture.DB.ID, 1, oldClaim.StartLSN), "aged claim must be gone")require.NotNil(t, findWalSegment(t, fixture.DB.ID, 1, youngClaim.StartLSN), "within-grace claim must survive")}++func Test_WalStream_ResumePointBelowSlotRestartLsn_RealignsAndKeepsStreaming(t *testing.T) {+ if testing.Short() {+ t.Skip("streamer integration test runs pg_receivewal; skipped in -short")+ }++ fixture := SetupPhysicalDBForBackup(t)+ t.Cleanup(func() {+ _ = physical_repositories.GetWalStreamerRepository().DeleteByDatabaseID(fixture.DB.ID)+ })++ adminConn := OpenAdminConn(t, fixture)+ slotName := fixture.DB.PostgresqlPhysical.ReplicationSlotName+ watchDirRoot := t.TempDir()++ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute)+ defer cancel()++ // A storage that refuses every save leaves finalized segments sitting in the+ // watch dir — the local queue this issue is about. It has to keep failing+ // across the restart, or startup recovery would drain the queue and there+ // would be nothing left to drag the resume point down.+ store := newMockWalStorage()+ store.startFailingSaves()++ firstRun := StartWalStreamerForTest(t, fixture, store, watchDirRoot)++ for range 3 {+ _, err := ForceWalRotation(ctx, adminConn)+ require.NoError(t, err)+ }++ waitForQueuedSegments(t, firstRun.WatchDir, 1, 60*time.Second)+ firstRun.Stop()++ queuedBeforeRebuild := queuedSegmentNames(t, firstRun.WatchDir)+ require.NotEmpty(t, queuedBeforeRebuild)++ // Model the incident: the slot is rebuilt while the queue still holds+ // pre-rebuild segments, so the recreated slot reserves from a position far+ // above them and pg_receivewal would otherwise resume below it.+ DropReplicationSlotExternally(t, adminConn, slotName)++ segmentSizeBytes := int64(walmath.WalSegmentSize)++ resumeSegmentNo, isResumeSegmentFound := GetResumeSegmentNo(firstRun.WatchDir, segmentSizeBytes)+ require.True(t, isResumeSegmentFound, "the local queue must hold at least one complete segment")++ burnWalPastSegment(t, ctx, adminConn, resumeSegmentNo+1)++ // pg_create_physical_replication_slot(..., true) reserves from the last+ // checkpoint's redo point, not from the current insert position, so without+ // this the recreated slot can still land below the queue.+ _, err := adminConn.Exec(ctx, "CHECKPOINT")+ require.NoError(t, err)++ require.NoError(t, fixture.DB.PostgresqlPhysical.VerifyWalSlot(+ ctx, logger.GetLogger(), encryption.GetFieldEncryptor(),+ ))++ requireQueueBelowSlot(t, ctx, adminConn, slotName, resumeSegmentNo)++ secondRun := StartWalStreamerForTest(t, fixture, store, watchDirRoot)+ t.Cleanup(secondRun.Stop)++ pendingUploadDir := filepath.Join(secondRun.WatchDir, pendingUploadDirName)++ for _, staleSegment := range queuedBeforeRebuild {+ require.Eventually(t, func() bool {+ _, err := os.Stat(filepath.Join(pendingUploadDir, staleSegment))++ return err == nil+ }, 60*time.Second, 250*time.Millisecond,+ "segment below the new restart_lsn must leave pg_receivewal's resume path: %s", staleSegment)++ require.NoFileExists(t, filepath.Join(secondRun.WatchDir, staleSegment))+ }++ // Storage recovers: the staged segments are valid WAL of the older chain, so+ // they must still reach storage, and the receiver must keep streaming the new+ // chain rather than crash-looping on the recycled WAL it used to ask for.+ store.stopFailingSaves()++ for range 3 {+ _, err := ForceWalRotation(ctx, adminConn)+ require.NoError(t, err)+ }++ for _, staleSegment := range queuedBeforeRebuild {+ require.Eventually(t, func() bool {+ return store.hasObject(walSegmentObjectName(fixture.DB.ID, 1, staleSegment))+ }, 60*time.Second, 250*time.Millisecond,+ "a segment moved out of the resume path must still reach storage: %s", staleSegment)+ }++ WaitForCommittedWalSegmentCount(t, fixture.DB.ID, 1, 90*time.Second)+}++func queuedSegmentNames(t *testing.T, watchDir string) []string {+ t.Helper()++ entries, err := os.ReadDir(watchDir)+ require.NoError(t, err)++ var queuedSegments []string++ for _, entry := range entries {+ if !entry.IsDir() && walmath.IsWalFilename(entry.Name()) {+ queuedSegments = append(queuedSegments, entry.Name())+ }+ }++ return queuedSegments+}++func waitForQueuedSegments(t *testing.T, watchDir string, minCount int, timeout time.Duration) {+ t.Helper()++ require.Eventually(t, func() bool {+ return len(queuedSegmentNames(t, watchDir)) >= minCount+ }, timeout, 250*time.Millisecond, "watch dir never accumulated %d finalized segments", minCount)+}++// pg_switch_wal is a no-op on an already-empty segment, so the cluster only+// moves past the queue if real WAL is written between switches.+func burnWalPastSegment(+ t *testing.T,+ ctx context.Context,+ adminConn *pgx.Conn,+ targetSegmentNo walmath.WalSegmentNo,+) {+ t.Helper()++ segmentSizeBytes := int64(walmath.WalSegmentSize)++ require.Eventually(t, func() bool {+ if _, err := GenerateWalActivity(ctx, adminConn, segmentSizeBytes); err != nil {+ return false+ }++ if _, err := ForceWalRotation(ctx, adminConn); err != nil {+ return false+ }++ var currentLSN walmath.LSN+ if err := adminConn.QueryRow(ctx, "SELECT pg_current_wal_lsn()::text").Scan(¤tLSN); err != nil {+ return false+ }++ return segmentNoAtLSN(currentLSN, segmentSizeBytes) > targetSegmentNo+ }, 2*time.Minute, 100*time.Millisecond, "cluster never wrote past segment %d", uint64(targetSegmentNo))+}++// The realign only has work to do when the recreated slot reserves above the+// queue pg_receivewal would otherwise resume from, so assert that rather than+// let the test pass without exercising anything.+func requireQueueBelowSlot(+ t *testing.T,+ ctx context.Context,+ adminConn *pgx.Conn,+ slotName string,+ resumeSegmentNo walmath.WalSegmentNo,+) {+ t.Helper()++ segmentSizeBytes := int64(walmath.WalSegmentSize)++ slotState, err := InspectSlot(ctx, adminConn, slotName)+ require.NoError(t, err)+ require.NotNil(t, slotState)++ require.Less(t, uint64(resumeSegmentNo), uint64(segmentNoAtLSN(slotState.RestartLSN, segmentSizeBytes)),+ "the queue must sit below the recreated slot for this test to mean anything")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.goindex 3ce9a62d..0eddc8db 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go@@ -39,6 +39,8 @@ type mockWalStorage struct {failSaveTimes int+ isFailingAllSaves atomic.Bool+// blockOn, when set, makes SaveFile for that exact object name signal started// and wait on release before returning.blockOn string@@ -62,6 +64,10 @@ func (m *mockWalStorage) SaveFile(<-m.release}+ if m.isFailingAllSaves.Load() {+ return fmt.Errorf("mock storage is failing every save for %s", fileName)+ }+m.mu.Lock()defer m.mu.Unlock()@@ -76,6 +82,14 @@ func (m *mockWalStorage) SaveFile(return nil}+func (m *mockWalStorage) startFailingSaves() {+ m.isFailingAllSaves.Store(true)+}++func (m *mockWalStorage) stopFailingSaves() {+ m.isFailingAllSaves.Store(false)+}+func (m *mockWalStorage) DeleteFile(_ encryption.FieldEncryptor, fileName string) error {m.mu.Lock()defer m.mu.Unlock()diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.goindex c27f42ca..b1e2eb2a 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go@@ -57,6 +57,8 @@ func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context,s.archiveTimelineHistoryFile(ctx, logger, name)}}++ s.sweepPendingUploads(ctx, logger, s.uploader.RecoverSegment)}func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Logger) {@@ -84,6 +86,34 @@ func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Los.archiveTimelineHistoryFile(ctx, logger, name)}}++ s.sweepPendingUploads(ctx, logger, s.uploader.ProcessSegment)+}++// Segments moved out of pg_receivewal's resume path (wal_resume.go) are valid+// WAL of the older chain, so they keep flowing to storage from here.+func (s *WalStreamSupervisor) sweepPendingUploads(+ ctx context.Context,+ logger *slog.Logger,+ upload func(ctx context.Context, localPath, walFilename string) error,+) {+ pendingUploadDir := filepath.Join(s.watchDir, pendingUploadDirName)++ entries, err := os.ReadDir(pendingUploadDir)+ if err != nil {+ return+ }++ for _, entry := range entries {+ if entry.IsDir() || !walmath.IsWalFilename(entry.Name()) {+ continue+ }++ if err := upload(ctx, filepath.Join(pendingUploadDir, entry.Name()), entry.Name()); err != nil {+ logger.Warn("pending wal segment upload failed; will retry next tick",+ "wal_filename", entry.Name(), "error", err)+ }+ }}// UploadHistoryFile reads the body from the source cluster and is idempotent on