Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (physical): Realign WAL resume point with the slot (issue #680)

Rostislav Dugine2fadbab1cd7
diff --git a/backend/internal/features/backups/backups/backuping/physical/constants.go b/backend/internal/features/backups/backups/backuping/physical/constants.go
index 3ec15f9b..9a8ca04d 100644
--- a/backend/internal/features/backups/backups/backuping/physical/constants.go
+++ b/backend/internal/features/backups/backups/backuping/physical/constants.go
@@ -52,6 +52,9 @@ const (
streamerStopTimeout = 30 * time.Second
walStreamSupervisorJobName = "physical_wal_stream_supervisor"
+
+ // One notification per incident, not one per reclaim cycle.
+ chainAlertMinInterval = 15 * time.Minute
)
// Per-tick WAL deletion budget anchors to the latest FULL's size (a cluster
diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.go
index e9d31324..0a8d0eee 100644
--- a/backend/internal/features/backups/backups/backuping/physical/di.go
+++ b/backend/internal/features/backups/backups/backuping/physical/di.go
@@ -3,6 +3,7 @@ package backuping_physical
import (
"sync"
"sync/atomic"
+ "time"
"github.com/google/uuid"
@@ -84,6 +85,7 @@ var physicalWalStreamSupervisor = &PhysicalWalStreamSupervisor{
sync.Mutex{},
make(map[uuid.UUID]*runningStreamer),
atomicTime{},
+ make(map[chainAlertKey]time.Time),
atomic.Bool{},
atomic.Bool{},
}
diff --git a/backend/internal/features/backups/backups/backuping/physical/dto.go b/backend/internal/features/backups/backups/backuping/physical/dto.go
index be50f13e..84478b4d 100644
--- a/backend/internal/features/backups/backups/backuping/physical/dto.go
+++ b/backend/internal/features/backups/backups/backuping/physical/dto.go
@@ -3,6 +3,8 @@ package backuping_physical
import (
"time"
+ "github.com/google/uuid"
+
"databasus-backend/internal/features/backups/backups/core/physical/chain_view"
backups_config_physical "databasus-backend/internal/features/backups/config/physical"
"databasus-backend/internal/features/databases"
@@ -22,3 +24,25 @@ type chainCandidate struct {
view *chain_view.ChainView
endTs time.Time
}
+
+// A WAL chain break is notified once per incident per database, so the throttle
+// key carries the kind: a retention warning must not swallow the slot-rebuild
+// alert that says a fresh full backup was requested.
+type chainAlertKind string
+
+const (
+ chainAlertStreamerFailed chainAlertKind = "STREAMER_FAILED"
+ chainAlertSlotRebuilt chainAlertKind = "SLOT_REBUILT"
+ chainAlertChainAtRisk chainAlertKind = "CHAIN_AT_RISK"
+)
+
+type chainAlertKey struct {
+ DatabaseID uuid.UUID
+ Kind chainAlertKind
+}
+
+type chainAlert struct {
+ Kind chainAlertKind
+ Heading string
+ Message string
+}
diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.go
index d4c2caaa..cfe21a8c 100644
--- a/backend/internal/features/backups/backups/backuping/physical/testing.go
+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go
@@ -75,7 +75,7 @@ func CreateTestPhysicalScheduler() *PhysicalBackupsScheduler {
// the production repos and services. A fresh instance (not a copy of the DI
// singleton) keeps each test's hasRun/running state isolated and avoids copying
// the embedded mutex.
-func CreateTestWalStreamSupervisor() *PhysicalWalStreamSupervisor {
+func CreateTestWalStreamSupervisor(notificationSender NotificationSender) *PhysicalWalStreamSupervisor {
return &PhysicalWalStreamSupervisor{
databases.GetDatabaseService(),
backups_config_physical.GetBackupConfigService(),
@@ -83,7 +83,7 @@ func CreateTestWalStreamSupervisor() *PhysicalWalStreamSupervisor {
physical_repositories.GetWalSegmentRepository(),
physical_repositories.GetWalHistoryRepository(),
physical_repositories.GetWalStreamerRepository(),
- notifiers.GetNotifierService(),
+ notificationSender,
tasks_cancellation.GetTaskCancelManager(),
encryption_secrets.GetSecretKeyService(),
encryption.GetFieldEncryptor(),
@@ -91,6 +91,7 @@ func CreateTestWalStreamSupervisor() *PhysicalWalStreamSupervisor {
sync.Mutex{},
make(map[uuid.UUID]*runningStreamer),
atomicTime{},
+ make(map[chainAlertKey]time.Time),
atomic.Bool{},
atomic.Bool{},
}
@@ -162,7 +163,7 @@ func StartPhysicalSchedulerForTest(t *testing.T) context.CancelFunc {
func StartPhysicalWalStreamSupervisorForTest(t *testing.T) context.CancelFunc {
t.Helper()
- supervisor := CreateTestWalStreamSupervisor()
+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())
ctx, cancel := context.WithCancel(context.Background())
diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
index 6b06eb67..71dfe11a 100644
--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
+ "maps"
"os"
"path/filepath"
"slices"
@@ -49,6 +50,11 @@ type PhysicalWalStreamSupervisor struct {
lastTickTime atomicTime
+ // A FAILED streamer is reclaimable on the next tick, so an unfixable break
+ // re-enters startStreamer every ~45 s. Without this the operator would get one
+ // notification per cycle instead of one per incident.
+ lastChainAlertAt map[chainAlertKey]time.Time
+
hasRun atomic.Bool
isReady atomic.Bool
}
@@ -257,7 +263,8 @@ func (s *PhysicalWalStreamSupervisor) startStreamer(
WatchDirRoot: config.GetEnv().DataFolder,
WalLagThresholdBytes: backupConfig.WalLagThresholdBytes,
OnGapDetected: s.gapNotifier(db, backupConfig),
- OnSlotRebuilt: s.slotRebuildFullRequester(logger, backupConfig),
+ OnSlotRebuilt: s.slotRebuildFullRequester(logger, db, backupConfig),
+ OnChainAtRisk: s.chainRiskNotifier(db, backupConfig),
Logger: s.logger,
})
@@ -275,6 +282,12 @@ func (s *PhysicalWalStreamSupervisor) startStreamer(
if err := supervisor.Run(streamerCtx); err != nil {
logger.Error("wal stream supervisor exited with error", "error", err)
+ s.notifyChainBroken(db, backupConfig, chainAlert{
+ Kind: chainAlertStreamerFailed,
+ Heading: fmt.Sprintf("WAL streaming stopped for %q", db.Name),
+ Message: fmt.Sprintf("database_id=%s error=%s", db.ID, err),
+ })
+
// Mark FAILED so a later tick can reclaim it. A clean
// ctx-cancel (shutdown / lifecycle stop) does not reach here with an
// error, so the row is left for the cancelling path to handle.
@@ -330,6 +343,7 @@ func (s *PhysicalWalStreamSupervisor) gapNotifier(
func (s *PhysicalWalStreamSupervisor) slotRebuildFullRequester(
logger *slog.Logger,
+ db *databases.Database,
backupConfig *backups_config_physical.PhysicalBackupConfig,
) func(context.Context, string) error {
return func(_ context.Context, reason string) error {
@@ -339,10 +353,73 @@ func (s *PhysicalWalStreamSupervisor) slotRebuildFullRequester(
logger.Warn("requested out-of-cadence full backup after wal slot rebuild", "reason", reason)
+ // Dropping and recreating the slot always leaves a WAL gap, so the chain
+ // this notifies about is already broken by the time we get here.
+ s.notifyChainBroken(db, backupConfig, chainAlert{
+ Kind: chainAlertSlotRebuilt,
+ Heading: fmt.Sprintf("Physical WAL chain rebuilt for %q", db.Name),
+ Message: fmt.Sprintf("database_id=%s reason=%s; a fresh full backup was requested to anchor the new chain",
+ db.ID, reason),
+ })
+
return nil
}
}
+func (s *PhysicalWalStreamSupervisor) chainRiskNotifier(
+ db *databases.Database,
+ backupConfig *backups_config_physical.PhysicalBackupConfig,
+) func(postgresql_executor.ChainRiskReport) {
+ return func(report postgresql_executor.ChainRiskReport) {
+ s.notifyChainBroken(db, backupConfig, chainAlert{
+ Kind: chainAlertChainAtRisk,
+ Heading: fmt.Sprintf("Physical WAL chain at risk for %q", db.Name),
+ Message: fmt.Sprintf("database_id=%s reason=%s slot_wal_status=%s lag_bytes=%d",
+ db.ID, report.Reason, report.SlotWalStatus, report.LagBytes),
+ })
+ }
+}
+
+func (s *PhysicalWalStreamSupervisor) notifyChainBroken(
+ db *databases.Database,
+ backupConfig *backups_config_physical.PhysicalBackupConfig,
+ alert chainAlert,
+) {
+ if !slices.Contains(backupConfig.SendNotificationsOn, backups_config_physical.NotificationChainBroken) {
+ return
+ }
+
+ if !s.recordChainAlertIfDue(chainAlertKey{DatabaseID: db.ID, Kind: alert.Kind}) {
+ return
+ }
+
+ notification := notifier_models.Notification{
+ Type: notifier_models.NotificationTypeBackupFailed,
+ Heading: alert.Heading,
+ Message: alert.Message,
+ }
+
+ for _, notifier := range db.Notifiers {
+ s.notificationSender.SendNotification(&notifier, notification)
+ }
+}
+
+func (s *PhysicalWalStreamSupervisor) recordChainAlertIfDue(key chainAlertKey) bool {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ now := time.Now().UTC()
+
+ if sentAt, wasNotified := s.lastChainAlertAt[key]; wasNotified &&
+ now.Sub(sentAt) < chainAlertMinInterval {
+ return false
+ }
+
+ s.lastChainAlertAt[key] = now
+
+ return true
+}
+
func (s *PhysicalWalStreamSupervisor) stopNonCandidates(candidates map[uuid.UUID]bool) {
s.mu.Lock()
var toStop []uuid.UUID
@@ -382,6 +459,11 @@ func (s *PhysicalWalStreamSupervisor) stopStreamer(databaseID uuid.UUID, shouldR
s.mu.Lock()
delete(s.running, databaseID)
+
+ maps.DeleteFunc(s.lastChainAlertAt, func(key chainAlertKey, _ time.Time) bool {
+ return key.DatabaseID == databaseID
+ })
+
s.mu.Unlock()
if err := s.walStreamerRepo.MarkFailed(databaseID); err != nil {
diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go
index e866fa07..22bab209 100644
--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor_test.go
@@ -11,6 +11,8 @@ import (
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
+ backups_config_physical "databasus-backend/internal/features/backups/config/physical"
+ "databasus-backend/internal/features/notifiers"
"databasus-backend/internal/storage"
)
@@ -131,7 +133,7 @@ func Test_ClaimIfClaimable_WhenConcurrentClaims_ExactlyOneWins(t *testing.T) {
}
func Test_PhysicalWalStreamSupervisorRun_WhenCalledTwice_Panics(t *testing.T) {
- supervisor := CreateTestWalStreamSupervisor()
+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())
ctx := t.Context()
@@ -154,7 +156,7 @@ func Test_PhysicalWalStreamSupervisorRun_WhenCalledTwice_Panics(t *testing.T) {
func Test_StopStreamer_WhenOwnedStreamerStops_MarksRowFailed(t *testing.T) {
prereqs := seedBackupPrereqs(t)
- supervisor := CreateTestWalStreamSupervisor()
+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())
require.NoError(t, physical_repositories.GetWalStreamerRepository().Claim(prereqs.DB.ID))
done := make(chan struct{})
@@ -182,7 +184,7 @@ func Test_RecoverStreamersOnStartup_WhenRunningRowStale_MarksFailed(t *testing.T
physical_enums.PhysicalWalStreamerStatusRunning,
)
- supervisor := CreateTestWalStreamSupervisor()
+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())
require.NoError(t, supervisor.recoverStreamersOnStartup())
@@ -193,7 +195,7 @@ func Test_RecoverStreamersOnStartup_WhenRunningRowStale_MarksFailed(t *testing.T
}
func Test_RemoveWatchDirIfRequested_WhenCleanupRequested_RemovesQueue(t *testing.T) {
- supervisor := CreateTestWalStreamSupervisor()
+ supervisor := CreateTestWalStreamSupervisor(notifiers.GetNotifierService())
watchDir := filepath.Join(t.TempDir(), "wal-queue")
require.NoError(t, os.MkdirAll(watchDir, 0o700))
require.NoError(t, os.WriteFile(filepath.Join(watchDir, "segment"), []byte("wal"), 0o600))
@@ -205,3 +207,73 @@ func Test_RemoveWatchDirIfRequested_WhenCleanupRequested_RemovesQueue(t *testing
require.NoDirExists(t, watchDir)
}
+
+func Test_NotifyChainBroken_WhenSameIncidentRepeats_SendsOneNotificationPerWindow(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+
+ sender := &recordingNotificationSender{}
+ supervisor := CreateTestWalStreamSupervisor(sender)
+
+ prereqs.Config.SendNotificationsOn = []backups_config_physical.BackupNotificationType{
+ backups_config_physical.NotificationChainBroken,
+ }
+
+ // A FAILED streamer is reclaimable on the next tick, so the same incident
+ // re-enters this path every cycle until the cause is fixed.
+ for range 3 {
+ supervisor.notifyChainBroken(prereqs.DB, prereqs.Config, chainAlert{
+ Kind: chainAlertStreamerFailed,
+ Heading: "WAL streaming stopped",
+ Message: "database_id=x",
+ })
+ }
+
+ require.Len(t, sender.sentNotifications, 1, "a reclaim loop must not turn into a notification loop")
+ require.Equal(t, "WAL streaming stopped", sender.sentNotifications[0].Notification.Heading)
+}
+
+func Test_NotifyChainBroken_WhenConfigDoesNotOptIn_SendsNoNotification(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+
+ sender := &recordingNotificationSender{}
+ supervisor := CreateTestWalStreamSupervisor(sender)
+
+ prereqs.Config.SendNotificationsOn = []backups_config_physical.BackupNotificationType{
+ backups_config_physical.NotificationBackupSuccess,
+ }
+
+ supervisor.notifyChainBroken(prereqs.DB, prereqs.Config, chainAlert{
+ Kind: chainAlertStreamerFailed,
+ Heading: "WAL streaming stopped",
+ Message: "database_id=x",
+ })
+
+ require.Empty(t, sender.sentNotifications)
+}
+
+func Test_NotifyChainBroken_WhenDifferentIncidentKinds_SendsEachOnce(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+
+ sender := &recordingNotificationSender{}
+ supervisor := CreateTestWalStreamSupervisor(sender)
+
+ prereqs.Config.SendNotificationsOn = []backups_config_physical.BackupNotificationType{
+ backups_config_physical.NotificationChainBroken,
+ }
+
+ supervisor.notifyChainBroken(prereqs.DB, prereqs.Config, chainAlert{
+ Kind: chainAlertChainAtRisk,
+ Heading: "chain at risk",
+ Message: "database_id=x",
+ })
+ supervisor.notifyChainBroken(prereqs.DB, prereqs.Config, chainAlert{
+ Kind: chainAlertSlotRebuilt,
+ Heading: "chain rebuilt",
+ Message: "database_id=x",
+ })
+
+ require.Len(t, sender.sentNotifications, 2,
+ "a retention warning must not swallow the rebuild alert that reports the new chain anchor")
+ require.Equal(t, "chain at risk", sender.sentNotifications[0].Notification.Heading)
+ require.Equal(t, "chain rebuilt", sender.sentNotifications[1].Notification.Heading)
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
index 42626ef9..617d06df 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
@@ -90,3 +90,11 @@ type ParentManifestRef struct {
IV string
StopLSN walmath.LSN
}
+
+// Payload of WalStreamSpec.OnChainAtRisk. Reason is a walBreakReason string:
+// the type is unexported, and the wiring package cannot name it.
+type ChainRiskReport struct {
+ Reason string
+ SlotWalStatus string
+ LagBytes int64
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
index db4280b0..2fa20b36 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
@@ -2,6 +2,7 @@ package usecases_physical_postgresql
import (
"context"
+ "fmt"
"log/slog"
"time"
)
@@ -11,9 +12,9 @@ const (
// indexed row).
lagMonitorPollInterval = 30 * time.Second
- // An 'extended' slot status must persist this long before we treat it as a
- // real lag (vs a transient write burst).
- extendedSlotStatusHoldPeriod = 5 * time.Minute
+ // A warning wal_status must persist this long before we alert on it (vs a
+ // transient write burst).
+ warningSlotStatusHoldPeriod = 5 * time.Minute
// Beyond this many rebuild ATTEMPTS in a sliding hour (counted regardless of
// outcome), mechanical retry won't help (creds rotated, pg_hba changed,
@@ -32,68 +33,46 @@ const (
type walBreakReason string
const (
- breakReasonSlotLost walBreakReason = "SLOT_LOST"
- breakReasonWalLag walBreakReason = "WAL_LAG_THRESHOLD"
- breakReasonSlotStolen walBreakReason = "SLOT_STOLEN"
+ breakReasonSlotLost walBreakReason = "SLOT_LOST"
+ breakReasonWalLag walBreakReason = "WAL_LAG_THRESHOLD"
+ breakReasonSlotStolen walBreakReason = "SLOT_STOLEN"
+ breakReasonSlotRetention walBreakReason = "SLOT_WAL_RETENTION"
)
-// Source-side slot state only; consumer-side liveness is the slot-LSN watcher's
-// job (slot_lsn_watcher.go).
-func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Logger) {
- ticker := time.NewTicker(lagMonitorPollInterval)
- defer ticker.Stop()
-
- var extendedSince time.Time
-
- for {
- select {
- case <-ctx.Done():
- return
-
- case <-ticker.C:
- reason, shouldRebuild := s.evaluateSlotForBreak(ctx, logger, &extendedSince)
- if !shouldRebuild {
- continue
- }
+type slotBreakAction int
- logger.Warn("wal_stream_break_observed", "reason", string(reason), "slot", s.slotName)
+const (
+ slotBreakActionNone slotBreakAction = iota
+ slotBreakActionAlert
+ slotBreakActionRebuild
+)
- if err := s.rebuildSlot(ctx, logger, reason); err != nil {
- logger.Error("slot rebuild failed", "reason", string(reason), "error", err)
- }
- }
- }
+type slotBreakSample struct {
+ SlotState *SlotState
+ WalLagThresholdBytes int64
+ IsReceiverRunning bool
+ ObservedAt time.Time
}
-// extendedSince tracks how long wal_status has been 'extended' across ticks.
-func (s *WalStreamSupervisor) evaluateSlotForBreak(
- ctx context.Context,
- logger *slog.Logger,
- extendedSince *time.Time,
-) (walBreakReason, bool) {
- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
- if err != nil {
- logger.Debug("lag monitor: source unreachable this tick", "error", err)
-
- return "", false
- }
- defer func() { _ = conn.Close(context.Background()) }()
-
- state, err := InspectSlot(ctx, conn, s.slotName)
- if err != nil || state == nil {
- return "", false
- }
-
- return classifySlotBreak(state, s.spec.WalLagThresholdBytes, extendedSince)
+type slotBreakClassifier struct {
+ warningStatusSince time.Time
}
-func classifySlotBreak(
- state *SlotState,
- walLagThresholdBytes int64,
- extendedSince *time.Time,
-) (walBreakReason, bool) {
+// Only 'lost' and a stolen slot are unrecoverable enough to justify dropping the
+// slot, which always costs a WAL gap and a fresh FULL. 'extended' merely means
+// the slot retains WAL beyond max_wal_size — routine on a busy cluster, and
+// guaranteed whenever back pressure pauses us — and 'unreserved' means PG will
+// trim that WAL at the next checkpoint, which already protects the primary. Both
+// alert instead. The operator's WalLagThresholdBytes stays the explicit
+// "sacrifice the chain to protect the primary" knob, but never fires against lag
+// we caused by holding the receiver down ourselves.
+func (c *slotBreakClassifier) recordSampleAndClassifyBreak(
+ sample slotBreakSample,
+) (walBreakReason, slotBreakAction) {
+ state := sample.SlotState
+
if state == nil {
- return "", false
+ return "", slotBreakActionNone
}
// A foreign backend holding our slot (active, but not one of our own
@@ -102,34 +81,96 @@ func classifySlotBreak(
// refuses to drop a slot held by a consumer we cannot attribute, so a genuine
// third party trips loop-protection rather than getting force-dropped.
if state.Active && !isOwnedReceiverBackend(state) {
- return breakReasonSlotStolen, true
+ return breakReasonSlotStolen, slotBreakActionRebuild
}
- switch state.WalStatus {
- case "lost":
- return breakReasonSlotLost, true
+ if state.WalStatus == "lost" {
+ c.warningStatusSince = time.Time{}
- case "unreserved":
- return breakReasonWalLag, true
+ return breakReasonSlotLost, slotBreakActionRebuild
+ }
- case "extended":
- if extendedSince.IsZero() {
- *extendedSince = time.Now().UTC()
+ if state.WalStatus == "extended" || state.WalStatus == "unreserved" {
+ if c.warningStatusSince.IsZero() {
+ c.warningStatusSince = sample.ObservedAt
}
- if time.Since(*extendedSince) > extendedSlotStatusHoldPeriod {
- return breakReasonWalLag, true
+ if sample.ObservedAt.Sub(c.warningStatusSince) > warningSlotStatusHoldPeriod {
+ return breakReasonSlotRetention, slotBreakActionAlert
}
- return "", false
+ return "", slotBreakActionNone
+ }
+
+ c.warningStatusSince = time.Time{}
- default:
- *extendedSince = time.Time{}
+ if sample.WalLagThresholdBytes > 0 &&
+ state.LagBytes > sample.WalLagThresholdBytes &&
+ sample.IsReceiverRunning {
+ return breakReasonWalLag, slotBreakActionRebuild
}
- if walLagThresholdBytes > 0 && state.LagBytes > walLagThresholdBytes {
- return breakReasonWalLag, true
+ return "", slotBreakActionNone
+}
+
+// Source-side slot state only; consumer-side liveness is the slot-LSN watcher's
+// job (slot_lsn_watcher.go).
+func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Logger) {
+ ticker := time.NewTicker(lagMonitorPollInterval)
+ defer ticker.Stop()
+
+ var classifier slotBreakClassifier
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+
+ case <-ticker.C:
+ state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)
+ if !isSourceReachable {
+ continue
+ }
+
+ reason, action := classifier.recordSampleAndClassifyBreak(slotBreakSample{
+ SlotState: state,
+ WalLagThresholdBytes: s.spec.WalLagThresholdBytes,
+ IsReceiverRunning: s.isReceiverRunning.Load(),
+ ObservedAt: time.Now().UTC(),
+ })
+
+ switch action {
+ case slotBreakActionNone:
+ continue
+
+ case slotBreakActionAlert:
+ s.reportChainAtRisk(logger, reason, state)
+
+ case slotBreakActionRebuild:
+ logger.Warn("wal_stream_break_observed", "reason", string(reason), "slot", s.slotName)
+
+ if err := s.rebuildSlot(ctx, logger, reason); err != nil {
+ logger.Error("slot rebuild failed", "reason", string(reason), "error", err)
+ }
+ }
+ }
+ }
+}
+
+func (s *WalStreamSupervisor) reportChainAtRisk(logger *slog.Logger, reason walBreakReason, state *SlotState) {
+ logger.Warn(
+ fmt.Sprintf("wal chain at risk: slot wal_status=%s, %d bytes behind", state.WalStatus, state.LagBytes),
+ "reason", string(reason),
+ "slot", s.slotName,
+ )
+
+ if s.spec.OnChainAtRisk == nil {
+ return
}
- return "", false
+ s.spec.OnChainAtRisk(ChainRiskReport{
+ Reason: string(reason),
+ SlotWalStatus: state.WalStatus,
+ LagBytes: state.LagBytes,
+ })
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go
index 4e85f8df..b04d36b7 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor_test.go
@@ -7,81 +7,139 @@ import (
"github.com/stretchr/testify/require"
)
-func Test_ClassifySlotBreak_WhenWalStatusLost_RebuildsForSlotLost(t *testing.T) {
- var extendedSince time.Time
+func runningSlotBreakSample(state *SlotState, walLagThresholdBytes int64, observedAt time.Time) slotBreakSample {
+ return slotBreakSample{
+ SlotState: state,
+ WalLagThresholdBytes: walLagThresholdBytes,
+ IsReceiverRunning: true,
+ ObservedAt: observedAt,
+ }
+}
- reason, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "lost"}, 0, &extendedSince)
+func Test_RecordSampleAndClassifyBreak_WhenWalStatusLost_Rebuilds(t *testing.T) {
+ var classifier slotBreakClassifier
- require.True(t, shouldRebuild)
+ reason, action := classifier.recordSampleAndClassifyBreak(
+ runningSlotBreakSample(&SlotState{WalStatus: "lost"}, 0, time.Now().UTC()),
+ )
+
+ require.Equal(t, slotBreakActionRebuild, action)
require.Equal(t, breakReasonSlotLost, reason)
}
-func Test_ClassifySlotBreak_WhenWalStatusUnreserved_RebuildsForWalLag(t *testing.T) {
- var extendedSince time.Time
+func Test_RecordSampleAndClassifyBreak_WhenWalStatusUnreserved_AlertsWithoutRebuilding(t *testing.T) {
+ var classifier slotBreakClassifier
- reason, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "unreserved"}, 0, &extendedSince)
+ firstSeenAt := time.Now().UTC()
+ state := &SlotState{WalStatus: "unreserved"}
- require.True(t, shouldRebuild)
- require.Equal(t, breakReasonWalLag, reason)
+ _, immediateAction := classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 0, firstSeenAt))
+ require.Equal(t, slotBreakActionNone, immediateAction,
+ "unreserved is a warning; it must hold before it is worth telling anyone about")
+
+ reason, heldAction := classifier.recordSampleAndClassifyBreak(
+ runningSlotBreakSample(state, 0, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second)),
+ )
+
+ require.Equal(t, slotBreakActionAlert, heldAction,
+ "PG trims that WAL at the next checkpoint, so dropping the slot would only add a gap")
+ require.Equal(t, breakReasonSlotRetention, reason)
}
-func Test_ClassifySlotBreak_WhenExtendedPersistsPastHold_RebuildsForWalLag(t *testing.T) {
- extendedSince := time.Now().UTC().Add(-extendedSlotStatusHoldPeriod - time.Second)
+func Test_RecordSampleAndClassifyBreak_WhenWalStatusExtendedPastHold_AlertsWithoutRebuilding(t *testing.T) {
+ var classifier slotBreakClassifier
- reason, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "extended"}, 0, &extendedSince)
+ firstSeenAt := time.Now().UTC()
+ state := &SlotState{WalStatus: "extended"}
- require.True(t, shouldRebuild)
- require.Equal(t, breakReasonWalLag, reason)
+ classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 0, firstSeenAt))
+
+ reason, action := classifier.recordSampleAndClassifyBreak(
+ runningSlotBreakSample(state, 0, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second)),
+ )
+
+ require.Equal(t, slotBreakActionAlert, action,
+ "retaining WAL beyond max_wal_size is routine on a busy cluster, not a reason to drop the slot")
+ require.Equal(t, breakReasonSlotRetention, reason)
}
-func Test_ClassifySlotBreak_WhenLagExceedsThreshold_RebuildsForWalLag(t *testing.T) {
- var extendedSince time.Time
+func Test_RecordSampleAndClassifyBreak_WhenLagOverThresholdAndReceiverRunning_Rebuilds(t *testing.T) {
+ var classifier slotBreakClassifier
- reason, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "reserved", LagBytes: 101}, 100, &extendedSince)
+ reason, action := classifier.recordSampleAndClassifyBreak(
+ runningSlotBreakSample(&SlotState{WalStatus: "reserved", LagBytes: 101}, 100, time.Now().UTC()),
+ )
- require.True(t, shouldRebuild)
+ require.Equal(t, slotBreakActionRebuild, action)
require.Equal(t, breakReasonWalLag, reason)
}
-func Test_ClassifySlotBreak_WhenForeignConsumerHoldsSlot_RebuildsForSlotStolen(t *testing.T) {
- var extendedSince time.Time
+func Test_RecordSampleAndClassifyBreak_WhenLagOverThresholdAndReceiverStopped_DoesNotRebuild(t *testing.T) {
+ var classifier slotBreakClassifier
+
+ _, action := classifier.recordSampleAndClassifyBreak(slotBreakSample{
+ SlotState: &SlotState{WalStatus: "reserved", LagBytes: 101},
+ WalLagThresholdBytes: 100,
+ IsReceiverRunning: false,
+ ObservedAt: time.Now().UTC(),
+ })
+
+ require.Equal(t, slotBreakActionNone, action,
+ "lag we caused by holding the receiver down must never cost the chain")
+}
+
+func Test_RecordSampleAndClassifyBreak_WhenForeignConsumerHoldsSlot_Rebuilds(t *testing.T) {
+ var classifier slotBreakClassifier
- foreignPID := 4242
state := &SlotState{
WalStatus: "reserved",
Active: true,
- ActivePID: &foreignPID,
+ ActivePID: new(4242),
ApplicationName: "some_other_consumer",
}
- reason, shouldRebuild := classifySlotBreak(state, 0, &extendedSince)
+ reason, action := classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 0, time.Now().UTC()))
- require.True(t, shouldRebuild)
+ require.Equal(t, slotBreakActionRebuild, action)
require.Equal(t, breakReasonSlotStolen, reason)
}
-func Test_ClassifySlotBreak_WhenOurReceiverActive_DoesNotRebuild(t *testing.T) {
- var extendedSince time.Time
+func Test_RecordSampleAndClassifyBreak_WhenOurReceiverActive_DoesNotRebuild(t *testing.T) {
+ var classifier slotBreakClassifier
- ourPID := 17
state := &SlotState{
WalStatus: "reserved",
Active: true,
- ActivePID: &ourPID,
+ ActivePID: new(17),
ApplicationName: receivewalApplicationNamePrefix + "db-1",
LagBytes: 10,
}
- _, shouldRebuild := classifySlotBreak(state, 100, &extendedSince)
+ _, action := classifier.recordSampleAndClassifyBreak(runningSlotBreakSample(state, 100, time.Now().UTC()))
- require.False(t, shouldRebuild)
+ require.Equal(t, slotBreakActionNone, action)
}
-func Test_ClassifySlotBreak_WhenSlotHealthy_DoesNotRebuildAndClearsExtendedSample(t *testing.T) {
- extendedSince := time.Now().UTC().Add(-extendedSlotStatusHoldPeriod - time.Second)
+func Test_RecordSampleAndClassifyBreak_WhenSlotHealthy_DoesNotRebuildAndClearsWarningClock(t *testing.T) {
+ var classifier slotBreakClassifier
+
+ firstSeenAt := time.Now().UTC()
+
+ classifier.recordSampleAndClassifyBreak(
+ runningSlotBreakSample(&SlotState{WalStatus: "extended"}, 100, firstSeenAt),
+ )
+
+ _, healthyAction := classifier.recordSampleAndClassifyBreak(
+ runningSlotBreakSample(&SlotState{WalStatus: "reserved", LagBytes: 10}, 100, firstSeenAt.Add(time.Minute)),
+ )
+ require.Equal(t, slotBreakActionNone, healthyAction)
- _, shouldRebuild := classifySlotBreak(&SlotState{WalStatus: "reserved", LagBytes: 10}, 100, &extendedSince)
+ _, actionAfterRecovery := classifier.recordSampleAndClassifyBreak(
+ runningSlotBreakSample(
+ &SlotState{WalStatus: "extended"}, 100, firstSeenAt.Add(warningSlotStatusHoldPeriod+time.Second),
+ ),
+ )
- require.False(t, shouldRebuild)
- require.True(t, extendedSince.IsZero())
+ require.Equal(t, slotBreakActionNone, actionAfterRecovery,
+ "a recovered slot restarts the hold window instead of inheriting the previous one")
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
index 35ca0335..ce59daf4 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
@@ -131,7 +131,7 @@ func Test_WalStreamOverMtls_StreamerArchivesSegments(t *testing.T) {
store := newMockWalStorage()
- stop := startStreamerForTest(t, fixture, store)
+ stop := StartWalStreamerForTest(t, fixture, store, t.TempDir()).Stop
t.Cleanup(stop)
adminConn := OpenAdminConn(t, fixture)
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
index d7e3fff0..95ce3905 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
@@ -25,8 +25,15 @@ const (
receiverInternalRestart // our own SIGTERM (back pressure / slot stall) — respawn promptly
receiverRetryable // non-zero exit that a local respawn may fix (network)
receiverFatal // non-retryable exit — escalate so the supervisor reclaims it on a later tick
+ receiverResumeMismatch // asked for WAL the server no longer has — realign the resume point and retry
)
+type receiverRunResult struct {
+ Exit receiverExit
+ RanFor time.Duration
+ FatalErr error
+}
+
type receivewalCommandSpec struct {
PgBin string
SourceDB *postgresql_physical.PostgresqlPhysicalDatabase
@@ -39,12 +46,12 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
ctx context.Context,
logger *slog.Logger,
pgBin string,
-) (receiverExit, time.Duration, error) {
+) receiverRunResult {
password, err := postgresql_shared.DecryptFieldIfNeeded(s.spec.SourceDB.Password, s.spec.FieldEncryptor)
if err != nil {
logger.Error("decrypt source password for pg_receivewal", "error", err)
- return receiverRetryable, 0, nil
+ return receiverRunResult{Exit: receiverRetryable}
}
creds, err := postgresql_shared.WriteCredentialFilesToTempDir(
@@ -53,7 +60,7 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
if err != nil {
logger.Error("write pg_receivewal credentials", "error", err)
- return receiverRetryable, 0, nil
+ return receiverRunResult{Exit: receiverRetryable}
}
defer creds.Remove()
@@ -70,22 +77,25 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
if err != nil {
logger.Error("build pg_receivewal command", "error", err)
- return receiverRetryable, 0, nil
+ return receiverRunResult{Exit: receiverRetryable}
}
stderrPipe, err := cmd.StderrPipe()
if err != nil {
logger.Error("pg_receivewal stderr pipe", "error", err)
- return receiverRetryable, 0, nil
+ return receiverRunResult{Exit: receiverRetryable}
}
if err := cmd.Start(); err != nil {
logger.Error("start pg_receivewal", "error", err)
- return receiverRetryable, 0, nil
+ return receiverRunResult{Exit: receiverRetryable}
}
+ s.isReceiverRunning.Store(true)
+ defer s.isReceiverRunning.Store(false)
+
stderr := newStderrCapture(stderrPipe)
startedAt := time.Now().UTC()
@@ -101,7 +111,7 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
<-exited
stderr.stop()
- return receiverCtxCancelled, 0, nil
+ return receiverRunResult{Exit: receiverCtxCancelled}
case <-s.restartSignal:
logger.Info("restarting pg_receivewal on internal signal (back pressure or slot stall)")
@@ -109,31 +119,42 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
<-exited
stderr.stop()
- return receiverInternalRestart, time.Since(startedAt), nil
+ return receiverRunResult{Exit: receiverInternalRestart, RanFor: time.Since(startedAt)}
case waitErr := <-exited:
stderr.stop()
ranFor := time.Since(startedAt)
if waitErr == nil || procCtx.Err() != nil {
- return receiverRetryable, ranFor, nil
+ return receiverRunResult{Exit: receiverRetryable, RanFor: ranFor}
}
stderrText := stderr.contents()
+ if isResumeMismatchError(stderrText) {
+ logger.Warn("pg_receivewal asked for WAL the server has recycled; realigning the resume point",
+ "error", waitErr, "stderr", truncateStderr(stderrText))
+
+ return receiverRunResult{Exit: receiverResumeMismatch, RanFor: ranFor}
+ }
+
if isFatalReceivewalError(stderrText) {
logger.Error("pg_receivewal exited with a non-retryable error; marking streamer for reassignment",
"error", waitErr, "stderr", truncateStderr(stderrText))
- return receiverFatal, ranFor, fmt.Errorf(
- "pg_receivewal fatal error: %w; stderr: %s", waitErr, truncateStderr(stderrText),
- )
+ return receiverRunResult{
+ Exit: receiverFatal,
+ RanFor: ranFor,
+ FatalErr: fmt.Errorf(
+ "pg_receivewal fatal error: %w; stderr: %s", waitErr, truncateStderr(stderrText),
+ ),
+ }
}
logger.Warn("pg_receivewal exited; will respawn",
"error", waitErr, "stderr", truncateStderr(stderrText))
- return receiverRetryable, ranFor, nil
+ return receiverRunResult{Exit: receiverRetryable, RanFor: ranFor}
}
}
@@ -161,6 +182,13 @@ func isFatalReceivewalError(stderr []byte) bool {
return false
}
+// The resume point pg_receivewal derived from the local queue sits below what
+// the server still has. A local respawn alone repeats it forever, so this is its
+// own disposition: the supervision loop realigns the queue and tries again.
+func isResumeMismatchError(stderr []byte) bool {
+ return strings.Contains(strings.ToLower(string(stderr)), "has already been removed")
+}
+
// WAL is left uncompressed locally (no --compress) because the uploader
// re-compresses with zstd on upload; --no-loop makes the process exit on
// connection loss so the supervision loop owns retry; --synchronous flushes each
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
index 73dcf401..a888dd0f 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
@@ -49,12 +49,33 @@ func Test_IsFatalReceivewalError_ClassifiesNonRetryableStderr(t *testing.T) {
require.True(t, isFatalReceivewalError([]byte(stderr)), stderr)
}
- transient := []string{
+ transientStderrs := []string{
"pg_receivewal: error: could not receive data from WAL stream: server closed the connection unexpectedly",
"pg_receivewal: error: connection to server failed: Connection refused",
"",
}
- for _, stderr := range transient {
+ for _, stderr := range transientStderrs {
require.False(t, isFatalReceivewalError([]byte(stderr)), stderr)
}
}
+
+func Test_IsResumeMismatchError_WhenSegmentAlreadyRemoved_ReturnsTrue(t *testing.T) {
+ stderr := "pg_receivewal: error: could not send replication command \"START_REPLICATION\": " +
+ "ERROR: requested WAL segment 0000000100000000E000002F has already been removed"
+
+ require.True(t, isResumeMismatchError([]byte(stderr)))
+ require.False(t, isFatalReceivewalError([]byte(stderr)),
+ "a realign fixes this, so it must not escalate the streamer to FAILED")
+}
+
+func Test_IsResumeMismatchError_WhenTransientStderr_ReturnsFalse(t *testing.T) {
+ transientStderrs := []string{
+ "pg_receivewal: error: could not receive data from WAL stream: server closed the connection unexpectedly",
+ "pg_receivewal: error: connection to server failed: Connection refused",
+ "",
+ }
+
+ for _, stderr := range transientStderrs {
+ require.False(t, isResumeMismatchError([]byte(stderr)), stderr)
+ }
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_cleanup_listener_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_cleanup_listener_test.go
index 604df2c1..d6969170 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_cleanup_listener_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_cleanup_listener_test.go
@@ -99,7 +99,7 @@ func Test_DropWalSlot_WhenStreamerSlotActive_RefusesAndKeepsSlot(t *testing.T) {
adminConn := postgresql_executor.OpenAdminConn(t, fixture)
streamerSlot := fixture.DB.PostgresqlPhysical.ReplicationSlotName
- stop := postgresql_executor.StartWalStreamerForTest(t, fixture, fixture.Storage)
+ stop := postgresql_executor.StartWalStreamerForTest(t, fixture, fixture.Storage, t.TempDir()).Stop
defer stop()
waitUntilSlotActive(t, adminConn, streamerSlot, 30*time.Second)
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
index 40c13c8b..0e599483 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
@@ -2,6 +2,7 @@ package usecases_physical_postgresql
import (
"context"
+ "fmt"
"log/slog"
"time"
@@ -13,31 +14,49 @@ const (
// actually flushing.
slotLsnWatcherPollInterval = 10 * time.Second
- // restart_lsn unchanged this long on a healthy PG means pg_receivewal is
+ // restart_lsn unchanged this long while WAL is waiting means pg_receivewal is
// stuck (alive but flushing nothing); restart it locally.
slotLsnStallTimeout = 60 * time.Second
)
+type slotLivenessSample struct {
+ RestartLSN walmath.LSN
+ LagBytes int64
+ IsReceiverRunning bool
+ ObservedAt time.Time
+}
+
type stallTracker struct {
lastRestartLSN walmath.LSN
lastAdvanceAt time.Time
hasSample bool
}
-// A changed LSN (or the very first sample) re-arms the advance clock. On a
-// positive result the clock re-arms too, so the caller restarts at most once per
-// stallTimeout window.
-func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeout time.Duration) bool {
- if !t.hasSample || restartLSN != t.lastRestartLSN {
- t.lastRestartLSN = restartLSN
- t.lastAdvanceAt = now
+// A frozen restart_lsn is only a stall when the source has WAL we have not taken
+// yet and a receiver is up to take it: an idle database legitimately parks
+// restart_lsn, and so does a receiver we are deliberately holding down for a
+// rebuild or back pressure. A positive result re-arms the clock, so the caller
+// restarts at most once per stallTimeout window.
+func (t *stallTracker) recordSampleAndDetectStall(
+ sample slotLivenessSample,
+ stallTimeout time.Duration,
+) bool {
+ if sample.LagBytes == 0 || !sample.IsReceiverRunning {
+ t.reset()
+
+ return false
+ }
+
+ if !t.hasSample || sample.RestartLSN != t.lastRestartLSN {
+ t.lastRestartLSN = sample.RestartLSN
+ t.lastAdvanceAt = sample.ObservedAt
t.hasSample = true
return false
}
- if now.Sub(t.lastAdvanceAt) > stallTimeout {
- t.lastAdvanceAt = now
+ if sample.ObservedAt.Sub(t.lastAdvanceAt) > stallTimeout {
+ t.lastAdvanceAt = sample.ObservedAt
return true
}
@@ -45,9 +64,40 @@ func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeo
return false
}
-// A frozen restart_lsn on a reachable source is a stuck consumer on a healthy
-// server. A stall with an unreachable server is left to the lag monitor (slot
-// loss / network down).
+func (t *stallTracker) reset() {
+ t.hasSample = false
+ t.lastRestartLSN = 0
+ t.lastAdvanceAt = time.Time{}
+}
+
+// isSourceReachable=false means defer to the lag monitor.
+func (s *WalStreamSupervisor) GetSlotStateIfReachable(
+ ctx context.Context,
+ logger *slog.Logger,
+) (state *SlotState, isSourceReachable bool) {
+ conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
+ if err != nil {
+ logger.Debug("slot-lsn watcher: source unreachable, deferring to lag monitor", "error", err)
+
+ return nil, false
+ }
+ defer func() { _ = conn.Close(context.Background()) }()
+
+ state, err = InspectSlot(ctx, conn, s.slotName)
+ if err != nil || state == nil {
+ return nil, false
+ }
+
+ var alive int
+ if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {
+ return nil, false
+ }
+
+ return state, true
+}
+
+// A stall with an unreachable server is left to the lag monitor (slot loss /
+// network down).
func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slog.Logger) {
ticker := time.NewTicker(slotLsnWatcherPollInterval)
defer ticker.Stop()
@@ -60,40 +110,27 @@ func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slo
return
case <-ticker.C:
- restartLSN, pgReachable := s.sampleSlotRestartLSN(ctx, logger)
- if !pgReachable {
+ state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)
+ if !isSourceReachable {
continue
}
- if tracker.observe(restartLSN, time.Now().UTC(), slotLsnStallTimeout) {
- logger.Warn("slot restart_lsn stalled on a reachable source; restarting pg_receivewal",
- "restart_lsn", restartLSN.String())
+ sample := slotLivenessSample{
+ RestartLSN: state.RestartLSN,
+ LagBytes: state.LagBytes,
+ IsReceiverRunning: s.isReceiverRunning.Load(),
+ ObservedAt: time.Now().UTC(),
+ }
+
+ if tracker.recordSampleAndDetectStall(sample, slotLsnStallTimeout) {
+ logger.Warn(
+ fmt.Sprintf("slot restart_lsn stalled with %d bytes pending; restarting pg_receivewal",
+ state.LagBytes),
+ "restart_lsn", state.RestartLSN.String(),
+ )
s.signalRestart()
}
}
}
}
-
-// pgReachable=false means defer to the lag monitor.
-func (s *WalStreamSupervisor) sampleSlotRestartLSN(ctx context.Context, logger *slog.Logger) (walmath.LSN, bool) {
- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
- if err != nil {
- logger.Debug("slot-lsn watcher: source unreachable, deferring to lag monitor", "error", err)
-
- return 0, false
- }
- defer func() { _ = conn.Close(context.Background()) }()
-
- state, err := InspectSlot(ctx, conn, s.slotName)
- if err != nil || state == nil {
- return 0, false
- }
-
- var alive int
- if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {
- return 0, false
- }
-
- return state.RestartLSN, true
-}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go
index f017532e..dcaeef34 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go
@@ -9,46 +9,109 @@ import (
"databasus-backend/internal/util/walmath"
)
-func Test_StallTracker_WhenFirstSample_DoesNotRestart(t *testing.T) {
+func streamingSample(restartLSN walmath.LSN, observedAt time.Time) slotLivenessSample {
+ return slotLivenessSample{
+ RestartLSN: restartLSN,
+ LagBytes: 16 * 1024 * 1024,
+ IsReceiverRunning: true,
+ ObservedAt: observedAt,
+ }
+}
+
+func Test_RecordSampleAndDetectStall_WhenFirstSample_DoesNotDetectStall(t *testing.T) {
var tracker stallTracker
base := time.Now().UTC()
- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute),
+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute),
"the first sample only arms the clock; it can never be a stall")
}
-func Test_StallTracker_WhenRestartLsnAdvances_ReArmsAndDoesNotRestart(t *testing.T) {
+func Test_RecordSampleAndDetectStall_WhenRestartLsnAdvances_ReArmsAndDoesNotDetectStall(t *testing.T) {
var tracker stallTracker
base := time.Now().UTC()
- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
- require.False(t, tracker.observe(walmath.LSN(200), base.Add(2*time.Minute), time.Minute),
+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute))
+ require.False(t,
+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(200), base.Add(2*time.Minute)), time.Minute),
"a changed restart_lsn means progress — the advance clock must reset")
}
-func Test_StallTracker_WhenFrozenWithinTimeout_DoesNotRestart(t *testing.T) {
+func Test_RecordSampleAndDetectStall_WhenFrozenWithinTimeout_DoesNotDetectStall(t *testing.T) {
var tracker stallTracker
base := time.Now().UTC()
- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
- require.False(t, tracker.observe(walmath.LSN(100), base.Add(30*time.Second), time.Minute),
+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute))
+ require.False(t,
+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(30*time.Second)), time.Minute),
"a frozen restart_lsn within the stall timeout is not yet a stall")
}
-func Test_StallTracker_WhenFrozenPastTimeout_RestartsThenReArms(t *testing.T) {
+func Test_RecordSampleAndDetectStall_WhenFrozenPastTimeout_DetectsStallThenReArms(t *testing.T) {
var tracker stallTracker
base := time.Now().UTC()
- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
- require.True(t, tracker.observe(walmath.LSN(100), base.Add(90*time.Second), time.Minute),
+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute))
+ require.True(t,
+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(90*time.Second)), time.Minute),
"a frozen restart_lsn past the stall timeout must trigger a restart")
- require.False(t, tracker.observe(walmath.LSN(100), base.Add(2*time.Minute), time.Minute),
+ require.False(t,
+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(2*time.Minute)), time.Minute),
"after firing, the clock re-arms so we restart at most once per window")
- require.True(t, tracker.observe(walmath.LSN(100), base.Add(4*time.Minute), time.Minute),
+ require.True(t,
+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(4*time.Minute)), time.Minute),
"a sustained stall fires again only after another full timeout window")
}
+
+func Test_RecordSampleAndDetectStall_WhenLsnFrozenAndNoLag_DoesNotDetectStall(t *testing.T) {
+ var tracker stallTracker
+
+ base := time.Now().UTC()
+
+ idleSample := func(observedAt time.Time) slotLivenessSample {
+ return slotLivenessSample{
+ RestartLSN: walmath.LSN(100),
+ LagBytes: 0,
+ IsReceiverRunning: true,
+ ObservedAt: observedAt,
+ }
+ }
+
+ require.False(t, tracker.recordSampleAndDetectStall(idleSample(base), time.Minute))
+ require.False(t, tracker.recordSampleAndDetectStall(idleSample(base.Add(10*time.Minute)), time.Minute),
+ "an idle database parks restart_lsn legitimately — there is nothing left to flush")
+}
+
+func Test_RecordSampleAndDetectStall_WhenLsnFrozenWithLag_DetectsStall(t *testing.T) {
+ var tracker stallTracker
+
+ base := time.Now().UTC()
+
+ require.False(t, tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base), time.Minute))
+ require.True(t,
+ tracker.recordSampleAndDetectStall(streamingSample(walmath.LSN(100), base.Add(90*time.Second)), time.Minute),
+ "WAL is waiting and restart_lsn has not moved — the receiver is wedged")
+}
+
+func Test_RecordSampleAndDetectStall_WhenReceiverNotRunning_DoesNotDetectStall(t *testing.T) {
+ var tracker stallTracker
+
+ base := time.Now().UTC()
+
+ stoppedSample := func(observedAt time.Time) slotLivenessSample {
+ return slotLivenessSample{
+ RestartLSN: walmath.LSN(100),
+ LagBytes: 16 * 1024 * 1024,
+ IsReceiverRunning: false,
+ ObservedAt: observedAt,
+ }
+ }
+
+ require.False(t, tracker.recordSampleAndDetectStall(stoppedSample(base), time.Minute))
+ require.False(t, tracker.recordSampleAndDetectStall(stoppedSample(base.Add(10*time.Minute)), time.Minute),
+ "a receiver we are deliberately holding down is not a stalled one")
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go
index fc22e93c..2afb2505 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go
@@ -61,6 +61,11 @@ func (s *WalStreamSupervisor) rebuildSlot(ctx context.Context, logger *slog.Logg
return fmt.Errorf("recreate slot: %w", err)
}
+ // The new slot reserves from the cluster's current WAL position, so the queue
+ // we just stopped writing to now sits below it and would drag pg_receivewal's
+ // resume point into recycled WAL.
+ s.realignResumePath(ctx, logger)
+
if s.spec.OnSlotRebuilt != nil {
if err := s.spec.OnSlotRebuilt(ctx, string(reason)); err != nil {
return fmt.Errorf("request full after slot rebuild: %w", err)
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
index 9b4e43d3..507d2f0b 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "path/filepath"
"strings"
"sync/atomic"
"testing"
@@ -711,20 +712,27 @@ func CountCommittedWalSegments(t *testing.T, databaseID uuid.UUID) int64 {
return count
}
-// StartWalStreamerForTest runs a WalStreamSupervisor against the fixture's
-// source PG in a goroutine, archiving rotated segments into store, and returns a
-// stop func that cancels and waits for a clean drain (so pg_receivewal releases
-// the slot before the DB and its slot are torn down). Cross-package backup→
-// restore tests pass the real fixture.Storage so archived segments can be read
-// back and replayed.
-func StartWalStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store storages.StorageFileSaver) func() {
+// WalStreamerForTest exposes the watch dir so a test can restart a streamer
+// against the queue an earlier run left behind.
+type WalStreamerForTest struct {
+ Supervisor *WalStreamSupervisor
+ WatchDir string
+ Stop func()
+}
+
+// A replication slot is meant to outlive a streamer restart (so WAL is never
+// lost), so the supervisor deliberately does NOT drop it on shutdown. In tests
+// that would leak one slot per run until max_replication_slots is exhausted, so
+// drop this streamer's slot here. Registered before the caller's stop cleanup,
+// it runs (LIFO) right after the streamer has stopped and before the conn closes.
+func StartWalStreamerForTest(
+ t *testing.T,
+ fixture *PhysicalDBFixture,
+ store storages.StorageFileSaver,
+ watchDirRoot string,
+) *WalStreamerForTest {
t.Helper()
- // A replication slot is meant to outlive a streamer restart (so WAL is never
- // lost), so the supervisor deliberately does NOT drop it on shutdown. In tests
- // that would leak one slot per run until max_replication_slots is exhausted, so
- // drop this streamer's slot here. Registered before the caller's stop cleanup,
- // it runs (LIFO) right after the streamer has stopped and before the conn closes.
adminConn := OpenAdminConn(t, fixture)
t.Cleanup(func() {
DropReplicationSlotExternally(t, adminConn, fixture.DB.PostgresqlPhysical.ReplicationSlotName)
@@ -739,7 +747,7 @@ func StartWalStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store sto
FieldEncryptor: encryption.GetFieldEncryptor(),
WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),
HistoryRepo: physical_repositories.GetWalHistoryRepository(),
- WatchDirRoot: t.TempDir(),
+ WatchDirRoot: watchDirRoot,
Logger: logger.GetLogger(),
})
@@ -752,14 +760,18 @@ func StartWalStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store sto
_ = supervisor.Run(ctx)
}()
- return func() {
- cancel()
+ return &WalStreamerForTest{
+ Supervisor: supervisor,
+ WatchDir: filepath.Join(watchDirRoot, "wal-queue", fixture.DB.ID.String()),
+ Stop: func() {
+ cancel()
- select {
- case <-done:
- case <-time.After(30 * time.Second):
- t.Log("streamer did not stop within timeout")
- }
+ select {
+ case <-done:
+ case <-time.After(30 * time.Second):
+ t.Log("streamer did not stop within timeout")
+ }
+ },
}
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
index 0c0ae05c..1d8ab98b 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
@@ -15,7 +15,7 @@ const (
// uploads drain below LOW. The 5x hysteresis prevents flapping on the
// boundary. HIGH scales up for clusters with non-default wal_segment_size so
// one segment does not stop the receiver.
- walLocalMinHighWatermarkBytes int64 = 100 * 1024 * 1024
+ walLocalMinHighWatermarkBytes int64 = 512 * 1024 * 1024
backpressurePollInterval = 1 * time.Second
)
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go
index a6730189..ecf615e4 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go
@@ -20,6 +20,6 @@ func Test_WalStream_BackpressureWatermarks_ScaleWithWalSegmentSize(t *testing.T)
Logger: logger.GetLogger(),
})
- require.Equal(t, 4*customSegSize, supervisor.highWatermarkBytes)
- require.Equal(t, 4*customSegSize/5, supervisor.lowWatermarkBytes)
+ require.Equal(t, 8*customSegSize, supervisor.highWatermarkBytes)
+ require.Equal(t, 8*customSegSize/5, supervisor.lowWatermarkBytes)
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go
new file mode 100644
index 00000000..eaef9124
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume.go
@@ -0,0 +1,137 @@
+package usecases_physical_postgresql
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+
+ "databasus-backend/internal/util/walmath"
+)
+
+const pendingUploadDirName = "pending-upload"
+
+// Mirrors pg_receivewal's FindStreamingStart: it starts at the segment after the
+// highest complete file in --directory, and falls back to the slot's restart_lsn
+// only when it finds none (PG 15+).
+func GetResumeSegmentNo(
+ watchDir string,
+ segmentSizeBytes int64,
+) (resumeSegmentNo walmath.WalSegmentNo, isResumeSegmentFound bool) {
+ entries, err := os.ReadDir(watchDir)
+ if err != nil {
+ return 0, false
+ }
+
+ var highestSegmentNo uint64
+
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+
+ _, segmentNo, err := walmath.ParseWALFilenameWithSize(entry.Name(), uint64(segmentSizeBytes))
+ if err != nil {
+ continue
+ }
+
+ // pg_receivewal ignores a short file: a torn segment is re-streamed, not resumed past.
+ info, err := entry.Info()
+ if err != nil || info.Size() != segmentSizeBytes {
+ continue
+ }
+
+ if !isResumeSegmentFound || segmentNo > highestSegmentNo {
+ highestSegmentNo = segmentNo
+ isResumeSegmentFound = true
+ }
+ }
+
+ if !isResumeSegmentFound {
+ return 0, false
+ }
+
+ return walmath.WalSegmentNo(highestSegmentNo).Next(), true
+}
+
+// pg_receivewal reads only the top level of --directory, so a pending segment
+// below the slot's restart_lsn drags the resume point into WAL the server has
+// already recycled. Moving rather than deleting keeps those segments archivable:
+// they are valid WAL of the older chain.
+func movePendingUploadsOutOfResumePath(
+ watchDir string,
+ resumeFloor walmath.WalSegmentNo,
+ segmentSizeBytes int64,
+) (movedCount int, err error) {
+ entries, err := os.ReadDir(watchDir)
+ if err != nil {
+ return 0, fmt.Errorf("read wal watch dir: %w", err)
+ }
+
+ pendingUploadDir := filepath.Join(watchDir, pendingUploadDirName)
+ if err := os.MkdirAll(pendingUploadDir, 0o700); err != nil {
+ return 0, fmt.Errorf("create pending upload dir: %w", err)
+ }
+
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+
+ _, segmentNo, err := walmath.ParseWALFilenameWithSize(entry.Name(), uint64(segmentSizeBytes))
+ if err != nil || walmath.WalSegmentNo(segmentNo) >= resumeFloor {
+ continue
+ }
+
+ moveErr := os.Rename(
+ filepath.Join(watchDir, entry.Name()),
+ filepath.Join(pendingUploadDir, entry.Name()),
+ )
+ if moveErr != nil && !os.IsNotExist(moveErr) {
+ return movedCount, fmt.Errorf("move %s out of the resume path: %w", entry.Name(), moveErr)
+ }
+
+ movedCount++
+ }
+
+ return movedCount, nil
+}
+
+// walmath.NewWalSegmentNo divides by the package-global segment size, which is
+// wrong for a cluster with a non-default wal_segment_size.
+func segmentNoAtLSN(lsn walmath.LSN, segmentSizeBytes int64) walmath.WalSegmentNo {
+ return walmath.WalSegmentNo(uint64(lsn) / uint64(segmentSizeBytes))
+}
+
+func (s *WalStreamSupervisor) realignResumePath(ctx context.Context, logger *slog.Logger) {
+ state, isSourceReachable := s.GetSlotStateIfReachable(ctx, logger)
+ if !isSourceReachable {
+ return
+ }
+
+ segmentSizeBytes := walSegmentSizeBytes(s.spec.SourceDB)
+
+ resumeSegmentNo, isResumeSegmentFound := GetResumeSegmentNo(s.watchDir, segmentSizeBytes)
+ if !isResumeSegmentFound {
+ return
+ }
+
+ slotSegmentNo := segmentNoAtLSN(state.RestartLSN, segmentSizeBytes)
+ if resumeSegmentNo >= slotSegmentNo {
+ return
+ }
+
+ movedCount, err := movePendingUploadsOutOfResumePath(s.watchDir, slotSegmentNo, segmentSizeBytes)
+ if err != nil {
+ logger.Error("failed to realign wal resume point", "error", err)
+
+ return
+ }
+
+ logger.Warn(
+ fmt.Sprintf("realigned wal resume point, moved %d segments out of the resume path", movedCount),
+ "slot_restart_lsn", state.RestartLSN.String(),
+ "resume_segment_no", uint64(resumeSegmentNo),
+ )
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume_test.go
new file mode 100644
index 00000000..e0283c31
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_resume_test.go
@@ -0,0 +1,91 @@
+package usecases_physical_postgresql
+
+import (
+ "bytes"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/util/walmath"
+)
+
+func writeSegmentOfSize(t *testing.T, dir, name string, sizeBytes int64) {
+ t.Helper()
+
+ require.NoError(t, os.WriteFile(filepath.Join(dir, name), bytes.Repeat([]byte{0}, int(sizeBytes)), 0o600))
+}
+
+func Test_GetResumeSegmentNo_WithCompletePartialAndTornFiles_ReturnsHighestCompletePlusOne(t *testing.T) {
+ watchDir := t.TempDir()
+
+ writeSegmentOfSize(t, watchDir, walName(1, 40), testWalSegmentSize)
+ writeSegmentOfSize(t, watchDir, walName(1, 41), testWalSegmentSize)
+ writeSegmentOfSize(t, watchDir, walName(1, 42), testWalSegmentSize/2)
+ writeSegmentOfSize(t, watchDir, walName(1, 43)+".partial", testWalSegmentSize)
+ writeSegmentOfSize(t, watchDir, "00000002.history", 64)
+
+ resumeSegmentNo, isResumeSegmentFound := GetResumeSegmentNo(watchDir, testWalSegmentSize)
+
+ require.True(t, isResumeSegmentFound)
+ require.Equal(t, walmath.WalSegmentNo(42), resumeSegmentNo,
+ "a torn segment is re-streamed, so 42 is not a resume point — 41 is the highest complete one")
+}
+
+func Test_GetResumeSegmentNo_WhenWatchDirHasNoSegments_ReportsNotFound(t *testing.T) {
+ watchDir := t.TempDir()
+
+ writeSegmentOfSize(t, watchDir, "00000002.history", 64)
+ require.NoError(t, os.MkdirAll(filepath.Join(watchDir, pendingUploadDirName), 0o700))
+ writeSegmentOfSize(t, filepath.Join(watchDir, pendingUploadDirName), walName(1, 40), testWalSegmentSize)
+
+ _, isResumeSegmentFound := GetResumeSegmentNo(watchDir, testWalSegmentSize)
+
+ require.False(t, isResumeSegmentFound,
+ "pg_receivewal reads only the top level, so a staged segment must not anchor the resume point")
+}
+
+func Test_MovePendingUploadsOutOfResumePath_MovesOnlySegmentsBelowFloor(t *testing.T) {
+ watchDir := t.TempDir()
+
+ for _, segmentNo := range []uint64{40, 41, 42, 43} {
+ writeSegmentOfSize(t, watchDir, walName(1, segmentNo), testWalSegmentSize)
+ }
+
+ movedCount, err := movePendingUploadsOutOfResumePath(watchDir, walmath.WalSegmentNo(42), testWalSegmentSize)
+ require.NoError(t, err)
+ require.Equal(t, 2, movedCount)
+
+ for _, movedSegment := range []uint64{40, 41} {
+ require.NoFileExists(t, filepath.Join(watchDir, walName(1, movedSegment)))
+ require.FileExists(t, filepath.Join(watchDir, pendingUploadDirName, walName(1, movedSegment)))
+ }
+
+ for _, keptSegment := range []uint64{42, 43} {
+ require.FileExists(t, filepath.Join(watchDir, walName(1, keptSegment)))
+ }
+}
+
+func Test_MovePendingUploadsOutOfResumePath_LeavesHistoryFilesAndSubdirectories(t *testing.T) {
+ watchDir := t.TempDir()
+
+ writeSegmentOfSize(t, watchDir, "00000002.history", 64)
+ require.NoError(t, os.MkdirAll(filepath.Join(watchDir, "archive_status"), 0o700))
+
+ movedCount, err := movePendingUploadsOutOfResumePath(watchDir, walmath.WalSegmentNo(99), testWalSegmentSize)
+ require.NoError(t, err)
+ require.Zero(t, movedCount)
+
+ require.FileExists(t, filepath.Join(watchDir, "00000002.history"))
+ require.DirExists(t, filepath.Join(watchDir, "archive_status"))
+}
+
+func Test_SegmentNoAtLsn_WithNonDefaultSegmentSize_UsesTheClusterSize(t *testing.T) {
+ const customSegmentSize = int64(64 * 1024 * 1024)
+
+ lsn := walmath.LSN(3 * uint64(customSegmentSize))
+
+ require.Equal(t, walmath.WalSegmentNo(3), segmentNoAtLSN(lsn, customSegmentSize),
+ "the package-global segment size would give 12 here")
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
index f7949d85..cc4cc009 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
@@ -40,6 +40,11 @@ const (
// a local respawn can never fix (ENOSPC, bad creds, a slot held by a thief).
receivewalMaxRapidFailures = 5
+ // A realign empties the resume path, so the very next spawn starts from the
+ // slot. A second mismatch means the slot itself no longer covers what the
+ // server retains — only a rebuild clears that.
+ receivewalMaxResumeMismatches = 2
+
pausePollInterval = 1 * time.Second
)
@@ -70,6 +75,10 @@ type WalStreamSpec struct {
// request a fresh base backup that anchors the new WAL chain.
OnSlotRebuilt func(ctx context.Context, reason string) error
+ // Fires while the chain is still intact but degrading (slot retention
+ // warnings, a wedged receiver); nil disables notification.
+ OnChainAtRisk func(report ChainRiskReport)
+
Logger *slog.Logger
}
@@ -95,6 +104,10 @@ type WalStreamSupervisor struct {
// drop+recreate the slot without the receiver re-attaching.
isPaused atomic.Bool
+ // Lets the slot-LSN watcher and the lag monitor tell a real problem from lag
+ // we are causing ourselves by holding the receiver down.
+ isReceiverRunning atomic.Bool
+
// rebuildMu serializes slot rebuilds in this process; rebuildTimestamps powers
// the per-hour loop-protection cap. One supervisor owns a DB at a time (the
// physical_wal_streamers heartbeat claim), so this is the only guard needed.
@@ -118,7 +131,7 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {
OnGapDetected: spec.OnGapDetected,
})
- highWatermarkBytes := max(walLocalMinHighWatermarkBytes, 4*walSegmentSizeBytes(spec.SourceDB))
+ highWatermarkBytes := max(walLocalMinHighWatermarkBytes, 8*walSegmentSizeBytes(spec.SourceDB))
return &WalStreamSupervisor{
spec: spec,
@@ -141,6 +154,10 @@ func (s *WalStreamSupervisor) Run(ctx context.Context) error {
return fmt.Errorf("create wal watch dir: %w", err)
}
+ if err := os.MkdirAll(filepath.Join(s.watchDir, pendingUploadDirName), 0o700); err != nil {
+ return fmt.Errorf("create pending upload dir: %w", err)
+ }
+
if err := s.spec.SourceDB.VerifyWalSlot(ctx, logger, s.spec.FieldEncryptor); err != nil {
return fmt.Errorf("verify persistent replication slot: %w", err)
}
@@ -193,6 +210,7 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg
pgBin := tools.GetPostgresqlExecutable(s.spec.SourceDB.Version, tools.PostgresqlExecutablePgReceivewal)
respawnBackoff := receivewalRespawnBackoff
rapidFailures := 0
+ resumeMismatches := 0
for {
if ctx.Err() != nil {
@@ -211,29 +229,50 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg
// signal raised while no process was running.
s.drainRestartSignal()
s.removePartials(logger)
+ s.realignResumePath(ctx, logger)
- outcome, ranFor, fatalErr := s.spawnAndSupervise(ctx, logger, pgBin)
+ receiverRun := s.spawnAndSupervise(ctx, logger, pgBin)
- switch outcome {
+ switch receiverRun.Exit {
case receiverCtxCancelled:
return nil
case receiverFatal:
- return fatalErr
+ return receiverRun.FatalErr
case receiverInternalRestart:
// Our own SIGTERM (back pressure or slot stall): respawn promptly; the
// top-of-loop backlog/pause gates already throttle the cause.
respawnBackoff = receivewalRespawnBackoff
rapidFailures = 0
+ resumeMismatches = 0
+
+ continue
+
+ case receiverResumeMismatch:
+ resumeMismatches++
+
+ if resumeMismatches >= receivewalMaxResumeMismatches {
+ resumeMismatches = 0
+
+ if err := s.rebuildSlot(ctx, logger, breakReasonSlotLost); err != nil {
+ return fmt.Errorf("rebuild slot after repeated wal resume mismatch: %w", err)
+ }
+ }
+
+ if !sleepCtx(ctx, receivewalRespawnBackoff) {
+ return nil
+ }
continue
}
+ resumeMismatches = 0
+
// receiverRetryable: a run that streamed for a healthy span is a transient
// blip (network) — reset the crash-loop counter; a string of sub-uptime
// exits is a crash loop a local respawn cannot fix, so escalate.
- if ranFor >= receivewalMinHealthyUptime {
+ if receiverRun.RanFor >= receivewalMinHealthyUptime {
rapidFailures = 0
respawnBackoff = receivewalRespawnBackoff
} else {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
index 7549d9ac..0360035b 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
@@ -2,62 +2,24 @@ package usecases_physical_postgresql
import (
"context"
+ "os"
+ "path/filepath"
"testing"
"time"
"github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
"github.com/stretchr/testify/require"
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
"databasus-backend/internal/features/backups/backups/core/physical/chain_view"
physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
- "databasus-backend/internal/features/storages"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/walmath"
)
-// The stop func waits for full drain, so pg_receivewal has released the slot
-// before the DB (and its slot) are torn down by later cleanups.
-func startStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store storages.StorageFileSaver) func() {
- t.Helper()
-
- spec := WalStreamSpec{
- DatabaseID: fixture.DB.ID,
- SourceDB: fixture.DB.PostgresqlPhysical,
- StorageID: fixture.Storage.ID,
- Storage: store,
- Encryption: backups_core_enums.BackupEncryptionNone,
- FieldEncryptor: encryption.GetFieldEncryptor(),
- WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),
- HistoryRepo: physical_repositories.GetWalHistoryRepository(),
- WatchDirRoot: t.TempDir(),
- Logger: logger.GetLogger(),
- }
-
- supervisor := NewWalStreamSupervisor(spec)
-
- ctx, cancel := context.WithCancel(t.Context())
- done := make(chan struct{})
-
- go func() {
- defer close(done)
-
- _ = supervisor.Run(ctx)
- }()
-
- return func() {
- cancel()
-
- select {
- case <-done:
- case <-time.After(30 * time.Second):
- t.Log("streamer did not stop within timeout")
- }
- }
-}
-
func Test_WalStream_FullIncrementalAndWalStream_StreamerArchivesSegments(t *testing.T) {
if testing.Short() {
t.Skip("streamer integration test runs pg_receivewal; skipped in -short")
@@ -70,7 +32,7 @@ func Test_WalStream_FullIncrementalAndWalStream_StreamerArchivesSegments(t *test
store := newMockWalStorage()
- stop := startStreamerForTest(t, fixture, store)
+ stop := StartWalStreamerForTest(t, fixture, store, t.TempDir()).Stop
t.Cleanup(stop)
adminConn := OpenAdminConn(t, fixture)
@@ -167,8 +129,8 @@ func Test_WalStream_MultipleDbs_EachArchivesSegmentsIndependently(t *testing.T)
storeA := newMockWalStorage()
storeB := newMockWalStorage()
- t.Cleanup(startStreamerForTest(t, fixtureA, storeA))
- t.Cleanup(startStreamerForTest(t, fixtureB, storeB))
+ t.Cleanup(StartWalStreamerForTest(t, fixtureA, storeA, t.TempDir()).Stop)
+ t.Cleanup(StartWalStreamerForTest(t, fixtureB, storeB, t.TempDir()).Stop)
connA := OpenAdminConn(t, fixtureA)
@@ -204,7 +166,7 @@ func Test_WalStream_MissingSegmentInStreamedChain_SurfacesAsGapChainStaysExtenda
store := newMockWalStorage()
adminConn := OpenAdminConn(t, fixture)
- t.Cleanup(startStreamerForTest(t, fixture, store))
+ t.Cleanup(StartWalStreamerForTest(t, fixture, store, t.TempDir()).Stop)
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
defer cancel()
@@ -266,7 +228,7 @@ func Test_WalStream_SlotLagGrowsWithoutConsumer_DrainsOnceStreaming(t *testing.T
WaitUntilSlotLag(t, adminConn, slotName, lagTarget, 30*time.Second)
// Once our streamer attaches, it consumes the backlog and the lag drains.
- t.Cleanup(startStreamerForTest(t, fixture, newMockWalStorage()))
+ t.Cleanup(StartWalStreamerForTest(t, fixture, newMockWalStorage(), t.TempDir()).Stop)
deadline := time.Now().UTC().Add(60 * time.Second)
for time.Now().UTC().Before(deadline) {
@@ -351,3 +313,177 @@ func Test_Cleaner_AbandonedNullClaim_OlderThanGrace_DeletedYoungerSurvives(t *te
require.Nil(t, findWalSegment(t, fixture.DB.ID, 1, oldClaim.StartLSN), "aged claim must be gone")
require.NotNil(t, findWalSegment(t, fixture.DB.ID, 1, youngClaim.StartLSN), "within-grace claim must survive")
}
+
+func Test_WalStream_ResumePointBelowSlotRestartLsn_RealignsAndKeepsStreaming(t *testing.T) {
+ if testing.Short() {
+ t.Skip("streamer integration test runs pg_receivewal; skipped in -short")
+ }
+
+ fixture := SetupPhysicalDBForBackup(t)
+ t.Cleanup(func() {
+ _ = physical_repositories.GetWalStreamerRepository().DeleteByDatabaseID(fixture.DB.ID)
+ })
+
+ adminConn := OpenAdminConn(t, fixture)
+ slotName := fixture.DB.PostgresqlPhysical.ReplicationSlotName
+ watchDirRoot := t.TempDir()
+
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute)
+ defer cancel()
+
+ // A storage that refuses every save leaves finalized segments sitting in the
+ // watch dir — the local queue this issue is about. It has to keep failing
+ // across the restart, or startup recovery would drain the queue and there
+ // would be nothing left to drag the resume point down.
+ store := newMockWalStorage()
+ store.startFailingSaves()
+
+ firstRun := StartWalStreamerForTest(t, fixture, store, watchDirRoot)
+
+ for range 3 {
+ _, err := ForceWalRotation(ctx, adminConn)
+ require.NoError(t, err)
+ }
+
+ waitForQueuedSegments(t, firstRun.WatchDir, 1, 60*time.Second)
+ firstRun.Stop()
+
+ queuedBeforeRebuild := queuedSegmentNames(t, firstRun.WatchDir)
+ require.NotEmpty(t, queuedBeforeRebuild)
+
+ // Model the incident: the slot is rebuilt while the queue still holds
+ // pre-rebuild segments, so the recreated slot reserves from a position far
+ // above them and pg_receivewal would otherwise resume below it.
+ DropReplicationSlotExternally(t, adminConn, slotName)
+
+ segmentSizeBytes := int64(walmath.WalSegmentSize)
+
+ resumeSegmentNo, isResumeSegmentFound := GetResumeSegmentNo(firstRun.WatchDir, segmentSizeBytes)
+ require.True(t, isResumeSegmentFound, "the local queue must hold at least one complete segment")
+
+ burnWalPastSegment(t, ctx, adminConn, resumeSegmentNo+1)
+
+ // pg_create_physical_replication_slot(..., true) reserves from the last
+ // checkpoint's redo point, not from the current insert position, so without
+ // this the recreated slot can still land below the queue.
+ _, err := adminConn.Exec(ctx, "CHECKPOINT")
+ require.NoError(t, err)
+
+ require.NoError(t, fixture.DB.PostgresqlPhysical.VerifyWalSlot(
+ ctx, logger.GetLogger(), encryption.GetFieldEncryptor(),
+ ))
+
+ requireQueueBelowSlot(t, ctx, adminConn, slotName, resumeSegmentNo)
+
+ secondRun := StartWalStreamerForTest(t, fixture, store, watchDirRoot)
+ t.Cleanup(secondRun.Stop)
+
+ pendingUploadDir := filepath.Join(secondRun.WatchDir, pendingUploadDirName)
+
+ for _, staleSegment := range queuedBeforeRebuild {
+ require.Eventually(t, func() bool {
+ _, err := os.Stat(filepath.Join(pendingUploadDir, staleSegment))
+
+ return err == nil
+ }, 60*time.Second, 250*time.Millisecond,
+ "segment below the new restart_lsn must leave pg_receivewal's resume path: %s", staleSegment)
+
+ require.NoFileExists(t, filepath.Join(secondRun.WatchDir, staleSegment))
+ }
+
+ // Storage recovers: the staged segments are valid WAL of the older chain, so
+ // they must still reach storage, and the receiver must keep streaming the new
+ // chain rather than crash-looping on the recycled WAL it used to ask for.
+ store.stopFailingSaves()
+
+ for range 3 {
+ _, err := ForceWalRotation(ctx, adminConn)
+ require.NoError(t, err)
+ }
+
+ for _, staleSegment := range queuedBeforeRebuild {
+ require.Eventually(t, func() bool {
+ return store.hasObject(walSegmentObjectName(fixture.DB.ID, 1, staleSegment))
+ }, 60*time.Second, 250*time.Millisecond,
+ "a segment moved out of the resume path must still reach storage: %s", staleSegment)
+ }
+
+ WaitForCommittedWalSegmentCount(t, fixture.DB.ID, 1, 90*time.Second)
+}
+
+func queuedSegmentNames(t *testing.T, watchDir string) []string {
+ t.Helper()
+
+ entries, err := os.ReadDir(watchDir)
+ require.NoError(t, err)
+
+ var queuedSegments []string
+
+ for _, entry := range entries {
+ if !entry.IsDir() && walmath.IsWalFilename(entry.Name()) {
+ queuedSegments = append(queuedSegments, entry.Name())
+ }
+ }
+
+ return queuedSegments
+}
+
+func waitForQueuedSegments(t *testing.T, watchDir string, minCount int, timeout time.Duration) {
+ t.Helper()
+
+ require.Eventually(t, func() bool {
+ return len(queuedSegmentNames(t, watchDir)) >= minCount
+ }, timeout, 250*time.Millisecond, "watch dir never accumulated %d finalized segments", minCount)
+}
+
+// pg_switch_wal is a no-op on an already-empty segment, so the cluster only
+// moves past the queue if real WAL is written between switches.
+func burnWalPastSegment(
+ t *testing.T,
+ ctx context.Context,
+ adminConn *pgx.Conn,
+ targetSegmentNo walmath.WalSegmentNo,
+) {
+ t.Helper()
+
+ segmentSizeBytes := int64(walmath.WalSegmentSize)
+
+ require.Eventually(t, func() bool {
+ if _, err := GenerateWalActivity(ctx, adminConn, segmentSizeBytes); err != nil {
+ return false
+ }
+
+ if _, err := ForceWalRotation(ctx, adminConn); err != nil {
+ return false
+ }
+
+ var currentLSN walmath.LSN
+ if err := adminConn.QueryRow(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&currentLSN); err != nil {
+ return false
+ }
+
+ return segmentNoAtLSN(currentLSN, segmentSizeBytes) > targetSegmentNo
+ }, 2*time.Minute, 100*time.Millisecond, "cluster never wrote past segment %d", uint64(targetSegmentNo))
+}
+
+// The realign only has work to do when the recreated slot reserves above the
+// queue pg_receivewal would otherwise resume from, so assert that rather than
+// let the test pass without exercising anything.
+func requireQueueBelowSlot(
+ t *testing.T,
+ ctx context.Context,
+ adminConn *pgx.Conn,
+ slotName string,
+ resumeSegmentNo walmath.WalSegmentNo,
+) {
+ t.Helper()
+
+ segmentSizeBytes := int64(walmath.WalSegmentSize)
+
+ slotState, err := InspectSlot(ctx, adminConn, slotName)
+ require.NoError(t, err)
+ require.NotNil(t, slotState)
+
+ require.Less(t, uint64(resumeSegmentNo), uint64(segmentNoAtLSN(slotState.RestartLSN, segmentSizeBytes)),
+ "the queue must sit below the recreated slot for this test to mean anything")
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
index 3ce9a62d..0eddc8db 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
@@ -39,6 +39,8 @@ type mockWalStorage struct {
failSaveTimes int
+ isFailingAllSaves atomic.Bool
+
// blockOn, when set, makes SaveFile for that exact object name signal started
// and wait on release before returning.
blockOn string
@@ -62,6 +64,10 @@ func (m *mockWalStorage) SaveFile(
<-m.release
}
+ if m.isFailingAllSaves.Load() {
+ return fmt.Errorf("mock storage is failing every save for %s", fileName)
+ }
+
m.mu.Lock()
defer m.mu.Unlock()
@@ -76,6 +82,14 @@ func (m *mockWalStorage) SaveFile(
return nil
}
+func (m *mockWalStorage) startFailingSaves() {
+ m.isFailingAllSaves.Store(true)
+}
+
+func (m *mockWalStorage) stopFailingSaves() {
+ m.isFailingAllSaves.Store(false)
+}
+
func (m *mockWalStorage) DeleteFile(_ encryption.FieldEncryptor, fileName string) error {
m.mu.Lock()
defer m.mu.Unlock()
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go
index c27f42ca..b1e2eb2a 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go
@@ -57,6 +57,8 @@ func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context,
s.archiveTimelineHistoryFile(ctx, logger, name)
}
}
+
+ s.sweepPendingUploads(ctx, logger, s.uploader.RecoverSegment)
}
func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Logger) {
@@ -84,6 +86,34 @@ func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Lo
s.archiveTimelineHistoryFile(ctx, logger, name)
}
}
+
+ s.sweepPendingUploads(ctx, logger, s.uploader.ProcessSegment)
+}
+
+// Segments moved out of pg_receivewal's resume path (wal_resume.go) are valid
+// WAL of the older chain, so they keep flowing to storage from here.
+func (s *WalStreamSupervisor) sweepPendingUploads(
+ ctx context.Context,
+ logger *slog.Logger,
+ upload func(ctx context.Context, localPath, walFilename string) error,
+) {
+ pendingUploadDir := filepath.Join(s.watchDir, pendingUploadDirName)
+
+ entries, err := os.ReadDir(pendingUploadDir)
+ if err != nil {
+ return
+ }
+
+ for _, entry := range entries {
+ if entry.IsDir() || !walmath.IsWalFilename(entry.Name()) {
+ continue
+ }
+
+ if err := upload(ctx, filepath.Join(pendingUploadDir, entry.Name()), entry.Name()); err != nil {
+ logger.Warn("pending wal segment upload failed; will retry next tick",
+ "wal_filename", entry.Name(), "error", err)
+ }
+ }
}
// UploadHistoryFile reads the body from the source cluster and is idempotent on