Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

REFACTOR (physical): Split wal_stream.go by responsibility

Rostislav Duginecb982a8e442
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
index 3a1afebd..0e2f5bf0 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
@@ -1,10 +1,230 @@
package usecases_physical_postgresql
import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
"os/exec"
+ "strconv"
+ "strings"
"syscall"
+ "time"
+
+ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
+ postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"
)
+const receivewalApplicationNamePrefix = "databasus_wal_receiver_"
+
+// receiverExit is the disposition of one pg_receivewal run, decided by
+// spawnAndSupervise and acted on by runReceivewalSupervision.
+type receiverExit int
+
+const (
+ receiverCtxCancelled receiverExit = iota // ctx cancelled — stop the loop, no error
+ receiverInternalRestart // our own SIGTERM (back pressure / slot stall) — respawn promptly
+ receiverRetryable // non-zero exit that a local respawn may fix (network)
+ receiverFatal // non-retryable exit — escalate so the supervisor reclaims it on a later tick
+)
+
+// spawnAndSupervise starts one pg_receivewal process and blocks until it exits,
+// the restart signal fires, or ctx is cancelled. It reports the run's
+// disposition, how long the process streamed, and (for receiverFatal) the error.
+func (s *WalStreamSupervisor) spawnAndSupervise(
+ ctx context.Context,
+ logger *slog.Logger,
+ pgBin string,
+) (receiverExit, time.Duration, error) {
+ password, err := postgresql_shared.DecryptFieldIfNeeded(s.spec.SourceDB.Password, s.spec.FieldEncryptor)
+ if err != nil {
+ logger.Error("decrypt source password for pg_receivewal", "error", err)
+
+ return receiverRetryable, 0, nil
+ }
+
+ creds, err := postgresql_shared.WriteCredentialFilesToTempDir(
+ s.spec.SourceDB.CredentialSpec(), password, s.spec.FieldEncryptor,
+ )
+ if err != nil {
+ logger.Error("write pg_receivewal credentials", "error", err)
+
+ return receiverRetryable, 0, nil
+ }
+ defer creds.Remove()
+
+ procCtx, procCancel := context.WithCancel(ctx)
+ defer procCancel()
+
+ cmd, err := newReceivewalCommand(procCtx, pgBin, s.spec.SourceDB, creds, s.watchDir, s.slotName)
+ if err != nil {
+ logger.Error("build pg_receivewal command", "error", err)
+
+ return receiverRetryable, 0, nil
+ }
+
+ stderrPipe, err := cmd.StderrPipe()
+ if err != nil {
+ logger.Error("pg_receivewal stderr pipe", "error", err)
+
+ return receiverRetryable, 0, nil
+ }
+
+ if err := cmd.Start(); err != nil {
+ logger.Error("start pg_receivewal", "error", err)
+
+ return receiverRetryable, 0, nil
+ }
+
+ stderr := newStderrCapture(stderrPipe)
+ startedAt := time.Now().UTC()
+
+ logger.Info("pg_receivewal started", "watch_dir", s.watchDir)
+
+ exited := make(chan error, 1)
+
+ go func() { exited <- cmd.Wait() }()
+
+ select {
+ case <-ctx.Done():
+ procCancel()
+ <-exited
+ stderr.stop()
+
+ return receiverCtxCancelled, 0, nil
+
+ case <-s.restartSignal:
+ logger.Info("restarting pg_receivewal on internal signal (back pressure or slot stall)")
+ procCancel()
+ <-exited
+ stderr.stop()
+
+ return receiverInternalRestart, time.Since(startedAt), nil
+
+ case waitErr := <-exited:
+ stderr.stop()
+ ranFor := time.Since(startedAt)
+
+ if waitErr == nil || procCtx.Err() != nil {
+ return receiverRetryable, ranFor, nil
+ }
+
+ stderrText := stderr.contents()
+
+ if isFatalReceivewalError(stderrText) {
+ logger.Error("pg_receivewal exited with a non-retryable error; marking streamer for reassignment",
+ "error", waitErr, "stderr", truncateStderr(stderrText))
+
+ return receiverFatal, ranFor, fmt.Errorf(
+ "pg_receivewal fatal error: %w; stderr: %s", waitErr, truncateStderr(stderrText),
+ )
+ }
+
+ logger.Warn("pg_receivewal exited; will respawn",
+ "error", waitErr, "stderr", truncateStderr(stderrText))
+
+ return receiverRetryable, ranFor, nil
+ }
+}
+
+// isFatalReceivewalError reports whether a pg_receivewal stderr indicates a
+// condition a local respawn can never clear — a full / unwritable local disk,
+// rejected credentials, or the replication slot being held by another consumer.
+// These escalate to streamer-FAILED so the supervisor can reclaim it on a later
+// tick instead of this process crash-looping on an unfixable cause.
+func isFatalReceivewalError(stderr []byte) bool {
+ // Lower-case both sides: OS errno strings vary in case ("Permission denied")
+ // while PG messages are lower-case, and we want to match either.
+ text := strings.ToLower(string(stderr))
+
+ for _, needle := range []string{
+ "no space left on device",
+ "could not write",
+ "authentication failed",
+ "no pg_hba.conf entry",
+ "permission denied",
+ "is active for", // replication slot "<name>" is active for PID <n>
+ } {
+ if strings.Contains(text, needle) {
+ return true
+ }
+ }
+
+ return false
+}
+
+// newReceivewalCommand builds the pg_receivewal invocation. WAL is left
+// uncompressed locally (no --compress) because the uploader re-compresses with
+// zstd on upload; --no-loop makes the process exit on connection loss so the
+// supervision loop owns retry; --synchronous flushes each segment promptly. SSL
+// is supplied through the same PGSSL* env path pg_basebackup uses, so mTLS needs
+// no extra handling here.
+func newReceivewalCommand(
+ ctx context.Context,
+ pgBin string,
+ sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,
+ creds *postgresql_shared.CredentialTempFiles,
+ watchDir string,
+ slotName string,
+) (*exec.Cmd, error) {
+ if _, err := exec.LookPath(pgBin); err != nil {
+ return nil, fmt.Errorf("pg_receivewal binary not found at %s: %w", pgBin, err)
+ }
+
+ args := []string{
+ "--directory=" + watchDir,
+ "--slot=" + slotName,
+ "--no-loop",
+ "--synchronous",
+ "--verbose",
+ "--no-password",
+ "-h", sourceDB.Host,
+ "-p", strconv.Itoa(sourceDB.Port),
+ "-U", sourceDB.Username,
+ }
+
+ cmd := exec.CommandContext(ctx, pgBin, args...)
+
+ cmd.Env = append(os.Environ(),
+ "PGPASSFILE="+creds.PgpassPath,
+ "PGAPPNAME="+receivewalApplicationName(sourceDB),
+ "PGCLIENTENCODING=UTF8",
+ "PGCONNECT_TIMEOUT=30",
+ "LC_ALL=C.UTF-8",
+ "LANG=C.UTF-8",
+ )
+
+ sslMode := sourceDB.SslMode
+ if sslMode == "" {
+ sslMode = postgresql_shared.PostgresSslModeDisable
+ }
+
+ cmd.Env = append(cmd.Env,
+ "PGSSLMODE="+string(sslMode),
+ "PGSSLCERT="+creds.ClientCertPath,
+ "PGSSLKEY="+creds.ClientKeyPath,
+ "PGSSLROOTCERT="+creds.RootCertPath,
+ "PGSSLCRL=",
+ )
+
+ cmd.Cancel = func() error {
+ return signalForGracefulCancel(cmd.Process)
+ }
+
+ cmd.WaitDelay = killAfterCancel
+ setReceivewalProcessAttributes(cmd)
+
+ return cmd, nil
+}
+
+func receivewalApplicationName(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) string {
+ if sourceDB.DatabaseID == nil {
+ return receivewalApplicationNamePrefix + sourceDB.ID.String()
+ }
+
+ return receivewalApplicationNamePrefix + sourceDB.DatabaseID.String()
+}
+
// setReceivewalProcessAttributes makes the kernel send pg_receivewal a SIGTERM
// if the Databasus process dies, so a crashed supervisor never leaks an orphaned
// receiver that keeps holding the replication slot. Pdeathsig is Linux-only;
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
new file mode 100644
index 00000000..e970fa75
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
@@ -0,0 +1,106 @@
+package usecases_physical_postgresql
+
+import (
+ "context"
+ "log/slog"
+ "time"
+
+ "databasus-backend/internal/util/walmath"
+)
+
+const (
+ // slotLsnWatcherPollInterval — consumer-side liveness poll. Tighter than the
+ // lag monitor because it tests whether OUR pg_receivewal is actually flushing.
+ slotLsnWatcherPollInterval = 10 * time.Second
+
+ // slotLsnStallTimeout — restart_lsn unchanged this long on a healthy PG means
+ // pg_receivewal is stuck (alive but flushing nothing); restart it locally.
+ slotLsnStallTimeout = 60 * time.Second
+)
+
+// stallTracker tracks slot restart_lsn advance across watcher ticks. A changed
+// LSN (or the very first sample) re-arms the advance clock; an LSN that stays
+// frozen past the stall timeout is the signal that pg_receivewal is alive but
+// flushing nothing on a reachable source.
+type stallTracker struct {
+ lastRestartLSN walmath.LSN
+ lastAdvanceAt time.Time
+ hasSample bool
+}
+
+// observe records a restart_lsn sample taken at now and reports whether the slot
+// has stalled longer than stallTimeout. On a positive result it re-arms the clock
+// so the caller restarts at most once per stallTimeout window.
+func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeout time.Duration) bool {
+ if !t.hasSample || restartLSN != t.lastRestartLSN {
+ t.lastRestartLSN = restartLSN
+ t.lastAdvanceAt = now
+ t.hasSample = true
+
+ return false
+ }
+
+ if now.Sub(t.lastAdvanceAt) > stallTimeout {
+ t.lastAdvanceAt = now
+
+ return true
+ }
+
+ return false
+}
+
+// runSlotLsnWatcher restarts pg_receivewal when the slot's restart_lsn has not
+// advanced for slotLsnStallTimeout while the source PG is still reachable — a
+// stuck consumer on a healthy server. A stall with an unreachable server is left
+// to the lag monitor (slot loss / network down).
+func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slog.Logger) {
+ ticker := time.NewTicker(slotLsnWatcherPollInterval)
+ defer ticker.Stop()
+
+ var tracker stallTracker
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+
+ case <-ticker.C:
+ restartLSN, pgReachable := s.sampleSlotRestartLSN(ctx, logger)
+ if !pgReachable {
+ // Connectivity / slot loss is the lag monitor's job, not ours.
+ continue
+ }
+
+ if tracker.observe(restartLSN, time.Now().UTC(), slotLsnStallTimeout) {
+ logger.Warn("slot restart_lsn stalled on a reachable source; restarting pg_receivewal",
+ "restart_lsn", restartLSN.String())
+
+ s.signalRestart()
+ }
+ }
+ }
+}
+
+// sampleSlotRestartLSN reads the slot's restart_lsn and confirms the source is
+// reachable with a keepalive. pgReachable=false means defer to the lag monitor.
+func (s *WalStreamSupervisor) sampleSlotRestartLSN(ctx context.Context, logger *slog.Logger) (walmath.LSN, bool) {
+ conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
+ if err != nil {
+ logger.Debug("slot-lsn watcher: source unreachable, deferring to lag monitor", "error", err)
+
+ return 0, false
+ }
+ defer func() { _ = conn.Close(context.Background()) }()
+
+ state, err := InspectSlot(ctx, conn, s.slotName)
+ if err != nil || state == nil {
+ return 0, false
+ }
+
+ var alive int
+ if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {
+ return 0, false
+ }
+
+ return state.RestartLSN, true
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go
new file mode 100644
index 00000000..f017532e
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go
@@ -0,0 +1,54 @@
+package usecases_physical_postgresql
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/util/walmath"
+)
+
+func Test_StallTracker_WhenFirstSample_DoesNotRestart(t *testing.T) {
+ var tracker stallTracker
+
+ base := time.Now().UTC()
+
+ require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute),
+ "the first sample only arms the clock; it can never be a stall")
+}
+
+func Test_StallTracker_WhenRestartLsnAdvances_ReArmsAndDoesNotRestart(t *testing.T) {
+ var tracker stallTracker
+
+ base := time.Now().UTC()
+
+ require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
+ require.False(t, tracker.observe(walmath.LSN(200), base.Add(2*time.Minute), time.Minute),
+ "a changed restart_lsn means progress — the advance clock must reset")
+}
+
+func Test_StallTracker_WhenFrozenWithinTimeout_DoesNotRestart(t *testing.T) {
+ var tracker stallTracker
+
+ base := time.Now().UTC()
+
+ require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
+ require.False(t, tracker.observe(walmath.LSN(100), base.Add(30*time.Second), time.Minute),
+ "a frozen restart_lsn within the stall timeout is not yet a stall")
+}
+
+func Test_StallTracker_WhenFrozenPastTimeout_RestartsThenReArms(t *testing.T) {
+ var tracker stallTracker
+
+ base := time.Now().UTC()
+
+ require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
+ require.True(t, tracker.observe(walmath.LSN(100), base.Add(90*time.Second), time.Minute),
+ "a frozen restart_lsn past the stall timeout must trigger a restart")
+
+ require.False(t, tracker.observe(walmath.LSN(100), base.Add(2*time.Minute), time.Minute),
+ "after firing, the clock re-arms so we restart at most once per window")
+ require.True(t, tracker.observe(walmath.LSN(100), base.Add(4*time.Minute), time.Minute),
+ "a sustained stall fires again only after another full timeout window")
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go
index 28f9c2f3..ab009377 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go
@@ -56,3 +56,12 @@ func Test_IsOwnedReceiverBackend_WhenActiveButEmptyApplicationName_ReturnsFalse(
require.False(t, isOwnedReceiverBackend(state))
}
+
+func Test_WalStream_RebuildAttemptCap_StopsFourthAttemptInHour(t *testing.T) {
+ supervisor := &WalStreamSupervisor{}
+
+ require.True(t, supervisor.recordRebuildAttemptWithinCap())
+ require.True(t, supervisor.recordRebuildAttemptWithinCap())
+ require.True(t, supervisor.recordRebuildAttemptWithinCap())
+ require.False(t, supervisor.recordRebuildAttemptWithinCap())
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
new file mode 100644
index 00000000..f46713a6
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
@@ -0,0 +1,96 @@
+package usecases_physical_postgresql
+
+import (
+ "context"
+ "log/slog"
+ "os"
+ "time"
+
+ "databasus-backend/internal/util/walmath"
+)
+
+const (
+ // walLocalMinHighWatermarkBytes — pg_receivewal has no
+ // kernel-pipe back pressure (it writes files), so the watch dir IS the buffer.
+ // Above HIGH we SIGTERM it; we resume only once uploads drain below LOW. The
+ // 5x hysteresis prevents flapping on the boundary. HIGH scales up for clusters
+ // with non-default wal_segment_size so one segment does not stop the receiver.
+ walLocalMinHighWatermarkBytes int64 = 100 * 1024 * 1024
+
+ backpressurePollInterval = 1 * time.Second
+)
+
+// runBackpressureMonitor SIGTERMs pg_receivewal (via the restart signal) when the
+// local watch-dir backlog crosses the high watermark; the supervision loop then
+// waits for the uploader to drain below the low watermark before respawning.
+func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slog.Logger) {
+ ticker := time.NewTicker(backpressurePollInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+
+ case <-ticker.C:
+ if s.backlogBytes() >= s.highWatermarkBytes {
+ s.signalRestart()
+ }
+ }
+ }
+}
+
+// waitForBacklogBelowLow blocks while the backlog is at/over the high watermark,
+// returning once it drains below the low watermark. Returns false if ctx is
+// cancelled while waiting.
+func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger *slog.Logger) bool {
+ if s.backlogBytes() < s.highWatermarkBytes {
+ return true
+ }
+
+ logger.Warn("wal backlog over high watermark; pausing pg_receivewal until it drains")
+
+ ticker := time.NewTicker(backpressurePollInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return false
+
+ case <-ticker.C:
+ if s.backlogBytes() < s.lowWatermarkBytes {
+ logger.Info("wal backlog drained below low watermark; resuming pg_receivewal")
+
+ return true
+ }
+ }
+ }
+}
+
+// backlogBytes sums the sizes of finalized (not-yet-uploaded) WAL segments in the
+// watch dir. Uploaded segments are removed by the uploader, so this is the local
+// queue depth; .partial and .history files are excluded.
+func (s *WalStreamSupervisor) backlogBytes() int64 {
+ entries, err := os.ReadDir(s.watchDir)
+ if err != nil {
+ return 0
+ }
+
+ var total int64
+
+ for _, entry := range entries {
+ if entry.IsDir() || !walmath.IsWalFilename(entry.Name()) {
+ continue
+ }
+
+ info, err := entry.Info()
+ if err != nil {
+ continue
+ }
+
+ total += info.Size()
+ }
+
+ return total
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go
new file mode 100644
index 00000000..a6730189
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go
@@ -0,0 +1,25 @@
+package usecases_physical_postgresql
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/util/logger"
+)
+
+func Test_WalStream_BackpressureWatermarks_ScaleWithWalSegmentSize(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+ customSegSize := int64(512 * 1024 * 1024)
+ fixture.DB.PostgresqlPhysical.WalSegmentSizeBytes = &customSegSize
+
+ supervisor := NewWalStreamSupervisor(WalStreamSpec{
+ DatabaseID: fixture.DB.ID,
+ SourceDB: fixture.DB.PostgresqlPhysical,
+ WatchDirRoot: t.TempDir(),
+ Logger: logger.GetLogger(),
+ })
+
+ require.Equal(t, 4*customSegSize, supervisor.highWatermarkBytes)
+ require.Equal(t, 4*customSegSize/5, supervisor.lowWatermarkBytes)
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
index 770b80af..f9a5541a 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
@@ -5,10 +5,7 @@ import (
"fmt"
"log/slog"
"os"
- "os/exec"
"path/filepath"
- "strconv"
- "strings"
"sync"
"sync/atomic"
"time"
@@ -18,7 +15,6 @@ import (
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
- postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"
"databasus-backend/internal/features/storages"
util_encryption "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
@@ -26,28 +22,6 @@ import (
)
const (
- receivewalApplicationNamePrefix = "databasus_wal_receiver_"
-
- // uploaderPollInterval — the uploader scans watch_dir this often for newly
- // rotated segments. Segments rotate at the source write rate; a tight loop
- // keeps local dwell time low without measurable CPU.
- uploaderPollInterval = 1 * time.Second
-
- // slotLsnWatcherPollInterval — consumer-side liveness poll. Tighter than the
- // lag monitor because it tests whether OUR pg_receivewal is actually flushing.
- slotLsnWatcherPollInterval = 10 * time.Second
-
- // slotLsnStallTimeout — restart_lsn unchanged this long on a healthy PG means
- // pg_receivewal is stuck (alive but flushing nothing); restart it locally.
- slotLsnStallTimeout = 60 * time.Second
-
- // walLocalMinHighWatermarkBytes — pg_receivewal has no
- // kernel-pipe back pressure (it writes files), so the watch dir IS the buffer.
- // Above HIGH we SIGTERM it; we resume only once uploads drain below LOW. The
- // 5x hysteresis prevents flapping on the boundary. HIGH scales up for clusters
- // with non-default wal_segment_size so one segment does not stop the receiver.
- walLocalMinHighWatermarkBytes int64 = 100 * 1024 * 1024
-
// receivewalRespawnBackoff — initial pause between a pg_receivewal exit and respawn so
// a hard-failing source (auth, pg_hba) is not hammered. --no-loop makes
// pg_receivewal exit on connection loss; this loop is its supervision.
@@ -66,17 +40,8 @@ const (
// forever on a condition a local respawn can never fix (ENOSPC, bad creds, a
// slot held by a thief).
receivewalMaxRapidFailures = 5
-)
-
-// receiverExit is the disposition of one pg_receivewal run, decided by
-// spawnAndSupervise and acted on by runReceivewalSupervision.
-type receiverExit int
-const (
- receiverCtxCancelled receiverExit = iota // ctx cancelled — stop the loop, no error
- receiverInternalRestart // our own SIGTERM (back pressure / slot stall) — respawn promptly
- receiverRetryable // non-zero exit that a local respawn may fix (network)
- receiverFatal // non-retryable exit — escalate so the supervisor reclaims it on a later tick
+ pausePollInterval = 1 * time.Second
)
// WalStreamSpec is the immutable configuration of one database's WAL streamer.
@@ -303,418 +268,6 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg
}
}
-// spawnAndSupervise starts one pg_receivewal process and blocks until it exits,
-// the restart signal fires, or ctx is cancelled. It reports the run's
-// disposition, how long the process streamed, and (for receiverFatal) the error.
-func (s *WalStreamSupervisor) spawnAndSupervise(
- ctx context.Context,
- logger *slog.Logger,
- pgBin string,
-) (receiverExit, time.Duration, error) {
- password, err := postgresql_shared.DecryptFieldIfNeeded(s.spec.SourceDB.Password, s.spec.FieldEncryptor)
- if err != nil {
- logger.Error("decrypt source password for pg_receivewal", "error", err)
-
- return receiverRetryable, 0, nil
- }
-
- creds, err := postgresql_shared.WriteCredentialFilesToTempDir(
- s.spec.SourceDB.CredentialSpec(), password, s.spec.FieldEncryptor,
- )
- if err != nil {
- logger.Error("write pg_receivewal credentials", "error", err)
-
- return receiverRetryable, 0, nil
- }
- defer creds.Remove()
-
- procCtx, procCancel := context.WithCancel(ctx)
- defer procCancel()
-
- cmd, err := newReceivewalCommand(procCtx, pgBin, s.spec.SourceDB, creds, s.watchDir, s.slotName)
- if err != nil {
- logger.Error("build pg_receivewal command", "error", err)
-
- return receiverRetryable, 0, nil
- }
-
- stderrPipe, err := cmd.StderrPipe()
- if err != nil {
- logger.Error("pg_receivewal stderr pipe", "error", err)
-
- return receiverRetryable, 0, nil
- }
-
- if err := cmd.Start(); err != nil {
- logger.Error("start pg_receivewal", "error", err)
-
- return receiverRetryable, 0, nil
- }
-
- stderr := newStderrCapture(stderrPipe)
- startedAt := time.Now().UTC()
-
- logger.Info("pg_receivewal started", "watch_dir", s.watchDir)
-
- exited := make(chan error, 1)
-
- go func() { exited <- cmd.Wait() }()
-
- select {
- case <-ctx.Done():
- procCancel()
- <-exited
- stderr.stop()
-
- return receiverCtxCancelled, 0, nil
-
- case <-s.restartSignal:
- logger.Info("restarting pg_receivewal on internal signal (back pressure or slot stall)")
- procCancel()
- <-exited
- stderr.stop()
-
- return receiverInternalRestart, time.Since(startedAt), nil
-
- case waitErr := <-exited:
- stderr.stop()
- ranFor := time.Since(startedAt)
-
- if waitErr == nil || procCtx.Err() != nil {
- return receiverRetryable, ranFor, nil
- }
-
- stderrText := stderr.contents()
-
- if isFatalReceivewalError(stderrText) {
- logger.Error("pg_receivewal exited with a non-retryable error; marking streamer for reassignment",
- "error", waitErr, "stderr", truncateStderr(stderrText))
-
- return receiverFatal, ranFor, fmt.Errorf(
- "pg_receivewal fatal error: %w; stderr: %s", waitErr, truncateStderr(stderrText),
- )
- }
-
- logger.Warn("pg_receivewal exited; will respawn",
- "error", waitErr, "stderr", truncateStderr(stderrText))
-
- return receiverRetryable, ranFor, nil
- }
-}
-
-// isFatalReceivewalError reports whether a pg_receivewal stderr indicates a
-// condition a local respawn can never clear — a full / unwritable local disk,
-// rejected credentials, or the replication slot being held by another consumer.
-// These escalate to streamer-FAILED so the supervisor can reclaim it on a later
-// tick instead of this process crash-looping on an unfixable cause.
-func isFatalReceivewalError(stderr []byte) bool {
- // Lower-case both sides: OS errno strings vary in case ("Permission denied")
- // while PG messages are lower-case, and we want to match either.
- text := strings.ToLower(string(stderr))
-
- for _, needle := range []string{
- "no space left on device",
- "could not write",
- "authentication failed",
- "no pg_hba.conf entry",
- "permission denied",
- "is active for", // replication slot "<name>" is active for PID <n>
- } {
- if strings.Contains(text, needle) {
- return true
- }
- }
-
- return false
-}
-
-// runUploaderLoop scans the watch dir on a tight interval and hands each
-// finalized segment / .history file to the appropriate handler.
-func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.Logger) {
- ticker := time.NewTicker(uploaderPollInterval)
- defer ticker.Stop()
-
- for {
- select {
- case <-ctx.Done():
- return
-
- case <-ticker.C:
- s.scanAndUpload(ctx, logger)
- }
- }
-}
-
-// recoverLocalSegmentsOnStartup sweeps watch_dir once at startup, taking over any
-// finalized segment / .history left by a crash. Uses the uploader's takeover path
-// so a segment whose pre-crash claim row is still file_name NULL gets finished
-// rather than left for the cleaner.
-func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context, logger *slog.Logger) {
- entries, err := os.ReadDir(s.watchDir)
- if err != nil {
- return
- }
-
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
-
- name := entry.Name()
-
- switch {
- case walmath.IsWalFilename(name):
- if err := s.uploader.RecoverSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {
- logger.Warn("startup wal recovery failed; live loop will retry", "wal_filename", name, "error", err)
- }
-
- case strings.HasSuffix(name, ".history"):
- s.handleHistoryFile(ctx, logger, name)
- }
- }
-}
-
-func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Logger) {
- entries, err := os.ReadDir(s.watchDir)
- if err != nil {
- logger.Error("read wal watch dir", "error", err)
-
- return
- }
-
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
-
- name := entry.Name()
-
- switch {
- case walmath.IsWalFilename(name):
- if err := s.uploader.ProcessSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {
- logger.Warn("wal segment upload failed; will retry next tick", "wal_filename", name, "error", err)
- }
-
- case strings.HasSuffix(name, ".history"):
- s.handleHistoryFile(ctx, logger, name)
- }
- }
-}
-
-// handleHistoryFile uploads a .history file the receiver dropped into watch_dir
-// (reusing UploadHistoryFile, which reads the body from the source cluster and is
-// idempotent on (database_id, timeline_id)), then removes the local copy.
-func (s *WalStreamSupervisor) handleHistoryFile(ctx context.Context, logger *slog.Logger, name string) {
- timelineID, err := parseHistoryTimeline(name)
- if err != nil {
- logger.Warn("skip unparseable history file", "name", name, "error", err)
-
- return
- }
-
- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
- if err != nil {
- logger.Warn("could not open connection to upload history file; will retry", "error", err)
-
- return
- }
- defer func() { _ = conn.Close(context.Background()) }()
-
- if _, err := UploadHistoryFile(
- ctx, conn, timelineID, s.spec.Storage, s.spec.SourceDB, s.spec.StorageID,
- s.spec.HistoryRepo, s.spec.Encryption, s.spec.MasterKey, s.spec.FieldEncryptor, logger,
- ); err != nil {
- logger.Warn("history upload failed; will retry next tick", "timeline_id", timelineID, "error", err)
-
- return
- }
-
- logger.Info("timeline switch observed via .history", "timeline_id", timelineID)
-
- if err := os.Remove(filepath.Join(s.watchDir, name)); err != nil && !os.IsNotExist(err) {
- logger.Warn("failed to remove uploaded history file", "name", name, "error", err)
- }
-}
-
-// stallTracker tracks slot restart_lsn advance across watcher ticks. A changed
-// LSN (or the very first sample) re-arms the advance clock; an LSN that stays
-// frozen past the stall timeout is the signal that pg_receivewal is alive but
-// flushing nothing on a reachable source.
-type stallTracker struct {
- lastRestartLSN walmath.LSN
- lastAdvanceAt time.Time
- hasSample bool
-}
-
-// observe records a restart_lsn sample taken at now and reports whether the slot
-// has stalled longer than stallTimeout. On a positive result it re-arms the clock
-// so the caller restarts at most once per stallTimeout window.
-func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeout time.Duration) bool {
- if !t.hasSample || restartLSN != t.lastRestartLSN {
- t.lastRestartLSN = restartLSN
- t.lastAdvanceAt = now
- t.hasSample = true
-
- return false
- }
-
- if now.Sub(t.lastAdvanceAt) > stallTimeout {
- t.lastAdvanceAt = now
-
- return true
- }
-
- return false
-}
-
-// runSlotLsnWatcher restarts pg_receivewal when the slot's restart_lsn has not
-// advanced for slotLsnStallTimeout while the source PG is still reachable — a
-// stuck consumer on a healthy server. A stall with an unreachable server is left
-// to the lag monitor (slot loss / network down).
-func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slog.Logger) {
- ticker := time.NewTicker(slotLsnWatcherPollInterval)
- defer ticker.Stop()
-
- var tracker stallTracker
-
- for {
- select {
- case <-ctx.Done():
- return
-
- case <-ticker.C:
- restartLSN, pgReachable := s.sampleSlotRestartLSN(ctx, logger)
- if !pgReachable {
- // Connectivity / slot loss is the lag monitor's job, not ours.
- continue
- }
-
- if tracker.observe(restartLSN, time.Now().UTC(), slotLsnStallTimeout) {
- logger.Warn("slot restart_lsn stalled on a reachable source; restarting pg_receivewal",
- "restart_lsn", restartLSN.String())
-
- s.signalRestart()
- }
- }
- }
-}
-
-// sampleSlotRestartLSN reads the slot's restart_lsn and confirms the source is
-// reachable with a keepalive. pgReachable=false means defer to the lag monitor.
-func (s *WalStreamSupervisor) sampleSlotRestartLSN(ctx context.Context, logger *slog.Logger) (walmath.LSN, bool) {
- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
- if err != nil {
- logger.Debug("slot-lsn watcher: source unreachable, deferring to lag monitor", "error", err)
-
- return 0, false
- }
- defer func() { _ = conn.Close(context.Background()) }()
-
- state, err := InspectSlot(ctx, conn, s.slotName)
- if err != nil || state == nil {
- return 0, false
- }
-
- var alive int
- if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {
- return 0, false
- }
-
- return state.RestartLSN, true
-}
-
-// runBackpressureMonitor SIGTERMs pg_receivewal (via the restart signal) when the
-// local watch-dir backlog crosses the high watermark; the supervision loop then
-// waits for the uploader to drain below the low watermark before respawning.
-func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slog.Logger) {
- ticker := time.NewTicker(uploaderPollInterval)
- defer ticker.Stop()
-
- for {
- select {
- case <-ctx.Done():
- return
-
- case <-ticker.C:
- if s.backlogBytes() >= s.highWatermarkBytes {
- s.signalRestart()
- }
- }
- }
-}
-
-// waitForBacklogBelowLow blocks while the backlog is at/over the high watermark,
-// returning once it drains below the low watermark. Returns false if ctx is
-// cancelled while waiting.
-func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger *slog.Logger) bool {
- if s.backlogBytes() < s.highWatermarkBytes {
- return true
- }
-
- logger.Warn("wal backlog over high watermark; pausing pg_receivewal until it drains")
-
- ticker := time.NewTicker(uploaderPollInterval)
- defer ticker.Stop()
-
- for {
- select {
- case <-ctx.Done():
- return false
-
- case <-ticker.C:
- if s.backlogBytes() < s.lowWatermarkBytes {
- logger.Info("wal backlog drained below low watermark; resuming pg_receivewal")
-
- return true
- }
- }
- }
-}
-
-// backlogBytes sums the sizes of finalized (not-yet-uploaded) WAL segments in the
-// watch dir. Uploaded segments are removed by the uploader, so this is the local
-// queue depth; .partial and .history files are excluded.
-func (s *WalStreamSupervisor) backlogBytes() int64 {
- entries, err := os.ReadDir(s.watchDir)
- if err != nil {
- return 0
- }
-
- var total int64
-
- for _, entry := range entries {
- if entry.IsDir() || !walmath.IsWalFilename(entry.Name()) {
- continue
- }
-
- info, err := entry.Info()
- if err != nil {
- continue
- }
-
- total += info.Size()
- }
-
- return total
-}
-
-func (s *WalStreamSupervisor) removePartials(logger *slog.Logger) {
- entries, err := os.ReadDir(s.watchDir)
- if err != nil {
- return
- }
-
- for _, entry := range entries {
- if !strings.HasSuffix(entry.Name(), ".partial") {
- continue
- }
-
- if err := os.Remove(filepath.Join(s.watchDir, entry.Name())); err != nil && !os.IsNotExist(err) {
- logger.Warn("failed to remove partial wal file", "name", entry.Name(), "error", err)
- }
- }
-}
-
// waitWhilePaused blocks the supervision loop while a slot rebuild holds the
// receiver down. Returns false if ctx is cancelled while paused.
func (s *WalStreamSupervisor) waitWhilePaused(ctx context.Context) bool {
@@ -722,7 +275,7 @@ func (s *WalStreamSupervisor) waitWhilePaused(ctx context.Context) bool {
return true
}
- ticker := time.NewTicker(uploaderPollInterval)
+ ticker := time.NewTicker(pausePollInterval)
defer ticker.Stop()
for {
@@ -752,78 +305,6 @@ func (s *WalStreamSupervisor) drainRestartSignal() {
}
}
-// newReceivewalCommand builds the pg_receivewal invocation. WAL is left
-// uncompressed locally (no --compress) because the uploader re-compresses with
-// zstd on upload; --no-loop makes the process exit on connection loss so the
-// supervision loop owns retry; --synchronous flushes each segment promptly. SSL
-// is supplied through the same PGSSL* env path pg_basebackup uses, so mTLS needs
-// no extra handling here.
-func newReceivewalCommand(
- ctx context.Context,
- pgBin string,
- sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,
- creds *postgresql_shared.CredentialTempFiles,
- watchDir string,
- slotName string,
-) (*exec.Cmd, error) {
- if _, err := exec.LookPath(pgBin); err != nil {
- return nil, fmt.Errorf("pg_receivewal binary not found at %s: %w", pgBin, err)
- }
-
- args := []string{
- "--directory=" + watchDir,
- "--slot=" + slotName,
- "--no-loop",
- "--synchronous",
- "--verbose",
- "--no-password",
- "-h", sourceDB.Host,
- "-p", strconv.Itoa(sourceDB.Port),
- "-U", sourceDB.Username,
- }
-
- cmd := exec.CommandContext(ctx, pgBin, args...)
-
- cmd.Env = append(os.Environ(),
- "PGPASSFILE="+creds.PgpassPath,
- "PGAPPNAME="+receivewalApplicationName(sourceDB),
- "PGCLIENTENCODING=UTF8",
- "PGCONNECT_TIMEOUT=30",
- "LC_ALL=C.UTF-8",
- "LANG=C.UTF-8",
- )
-
- sslMode := sourceDB.SslMode
- if sslMode == "" {
- sslMode = postgresql_shared.PostgresSslModeDisable
- }
-
- cmd.Env = append(cmd.Env,
- "PGSSLMODE="+string(sslMode),
- "PGSSLCERT="+creds.ClientCertPath,
- "PGSSLKEY="+creds.ClientKeyPath,
- "PGSSLROOTCERT="+creds.RootCertPath,
- "PGSSLCRL=",
- )
-
- cmd.Cancel = func() error {
- return signalForGracefulCancel(cmd.Process)
- }
-
- cmd.WaitDelay = killAfterCancel
- setReceivewalProcessAttributes(cmd)
-
- return cmd, nil
-}
-
-func receivewalApplicationName(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) string {
- if sourceDB.DatabaseID == nil {
- return receivewalApplicationNamePrefix + sourceDB.ID.String()
- }
-
- return receivewalApplicationNamePrefix + sourceDB.DatabaseID.String()
-}
-
// walSegmentSizeBytes returns the source cluster's captured wal_segment_size, or
// the 16 MB default when it has not been captured yet.
func walSegmentSizeBytes(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) int64 {
@@ -834,21 +315,6 @@ func walSegmentSizeBytes(sourceDB *postgresql_physical.PostgresqlPhysicalDatabas
return int64(walmath.WalSegmentSize)
}
-// parseHistoryTimeline extracts the timeline id from a "%08X.history" filename.
-func parseHistoryTimeline(name string) (int, error) {
- trimmed := strings.TrimSuffix(name, ".history")
- if trimmed == name {
- return 0, fmt.Errorf("not a history filename: %q", name)
- }
-
- timelineID, err := strconv.ParseUint(trimmed, 16, 32)
- if err != nil {
- return 0, fmt.Errorf("parse timeline from %q: %w", name, err)
- }
-
- return int(timelineID), nil
-}
-
func sleepCtx(ctx context.Context, d time.Duration) bool {
timer := time.NewTimer(d)
defer timer.Stop()
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
index fb95ada7..8687b8b9 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
@@ -286,31 +286,6 @@ func Test_WalStream_SlotLagGrowsWithoutConsumer_DrainsOnceStreaming(t *testing.T
t.Fatalf("slot lag did not drain below %d within 60s after streaming started", lagTarget)
}
-func Test_WalStream_BackpressureWatermarks_ScaleWithWalSegmentSize(t *testing.T) {
- fixture := SetupPhysicalDBForBackup(t)
- customSegSize := int64(512 * 1024 * 1024)
- fixture.DB.PostgresqlPhysical.WalSegmentSizeBytes = &customSegSize
-
- supervisor := NewWalStreamSupervisor(WalStreamSpec{
- DatabaseID: fixture.DB.ID,
- SourceDB: fixture.DB.PostgresqlPhysical,
- WatchDirRoot: t.TempDir(),
- Logger: logger.GetLogger(),
- })
-
- require.Equal(t, 4*customSegSize, supervisor.highWatermarkBytes)
- require.Equal(t, 4*customSegSize/5, supervisor.lowWatermarkBytes)
-}
-
-func Test_WalStream_RebuildAttemptCap_StopsFourthAttemptInHour(t *testing.T) {
- supervisor := &WalStreamSupervisor{}
-
- require.True(t, supervisor.recordRebuildAttemptWithinCap())
- require.True(t, supervisor.recordRebuildAttemptWithinCap())
- require.True(t, supervisor.recordRebuildAttemptWithinCap())
- require.False(t, supervisor.recordRebuildAttemptWithinCap())
-}
-
func Test_WalStream_CustomWalSegmentSize_LsnMathCorrect(t *testing.T) {
fixture := SetupPhysicalDBForBackup(t)
@@ -342,56 +317,6 @@ func Test_WalStream_CustomWalSegmentSize_LsnMathCorrect(t *testing.T) {
require.Equal(t, wantStart+walmath.LSN(customSegSize), row.EndLSN)
}
-func Test_WalUpload_ConcurrentClaimSameSegment_OnlyWinnerInserts(t *testing.T) {
- fixture := SetupPhysicalDBForBackup(t)
-
- repo := physical_repositories.GetWalSegmentRepository()
- startLSN := walmath.LSN(40 * uint64(testWalSegmentSize))
- endLSN := startLSN + walmath.LSN(testWalSegmentSize)
-
- const racers = 6
-
- type claimOutcome struct {
- inserted bool
- err error
- }
-
- results := make(chan claimOutcome, racers)
- start := make(chan struct{})
-
- for range racers {
- go func() {
- <-start
-
- // Don't call require.* off the test goroutine — collect and assert below.
- inserted, err := repo.ClaimInsert(&physical_models.PhysicalWalSegment{
- DatabaseID: fixture.DB.ID,
- StorageID: fixture.Storage.ID,
- TimelineID: 1,
- WalFilename: walName(1, 40),
- StartLSN: startLSN,
- EndLSN: endLSN,
- Encryption: backups_core_enums.BackupEncryptionNone,
- })
- results <- claimOutcome{inserted: inserted, err: err}
- }()
- }
-
- close(start)
-
- winners := 0
- for range racers {
- outcome := <-results
- require.NoError(t, outcome.err)
-
- if outcome.inserted {
- winners++
- }
- }
-
- require.Equal(t, 1, winners, "exactly one concurrent claim may win the (db, tl, start_lsn) slot")
-}
-
func Test_Cleaner_AbandonedNullClaim_OlderThanGrace_DeletedYoungerSurvives(t *testing.T) {
fixture := SetupPhysicalDBForBackup(t)
@@ -432,47 +357,3 @@ func Test_Cleaner_AbandonedNullClaim_OlderThanGrace_DeletedYoungerSurvives(t *te
require.Nil(t, findWalSegment(t, fixture.DB.ID, 1, oldClaim.StartLSN), "aged claim must be gone")
require.NotNil(t, findWalSegment(t, fixture.DB.ID, 1, youngClaim.StartLSN), "within-grace claim must survive")
}
-
-func Test_StallTracker_WhenFirstSample_DoesNotRestart(t *testing.T) {
- var tracker stallTracker
-
- base := time.Now().UTC()
-
- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute),
- "the first sample only arms the clock; it can never be a stall")
-}
-
-func Test_StallTracker_WhenRestartLsnAdvances_ReArmsAndDoesNotRestart(t *testing.T) {
- var tracker stallTracker
-
- base := time.Now().UTC()
-
- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
- require.False(t, tracker.observe(walmath.LSN(200), base.Add(2*time.Minute), time.Minute),
- "a changed restart_lsn means progress — the advance clock must reset")
-}
-
-func Test_StallTracker_WhenFrozenWithinTimeout_DoesNotRestart(t *testing.T) {
- var tracker stallTracker
-
- base := time.Now().UTC()
-
- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
- require.False(t, tracker.observe(walmath.LSN(100), base.Add(30*time.Second), time.Minute),
- "a frozen restart_lsn within the stall timeout is not yet a stall")
-}
-
-func Test_StallTracker_WhenFrozenPastTimeout_RestartsThenReArms(t *testing.T) {
- var tracker stallTracker
-
- base := time.Now().UTC()
-
- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))
- require.True(t, tracker.observe(walmath.LSN(100), base.Add(90*time.Second), time.Minute),
- "a frozen restart_lsn past the stall timeout must trigger a restart")
-
- require.False(t, tracker.observe(walmath.LSN(100), base.Add(2*time.Minute), time.Minute),
- "after firing, the clock re-arms so we restart at most once per window")
- require.True(t, tracker.observe(walmath.LSN(100), base.Add(4*time.Minute), time.Minute),
- "a sustained stall fires again only after another full timeout window")
-}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
index 300bbdca..08fd6a89 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
@@ -460,3 +460,53 @@ func Test_BuildWalSegmentArtifactReader_WhenLargeSegment_StreamsAndCountsArtifac
require.Equal(t, int64(len(body)), compressedSizeBytes)
require.NotEmpty(t, body)
}
+
+func Test_WalUpload_ConcurrentClaimSameSegment_OnlyWinnerInserts(t *testing.T) {
+ fixture := SetupPhysicalDBForBackup(t)
+
+ repo := physical_repositories.GetWalSegmentRepository()
+ startLSN := walmath.LSN(40 * uint64(testWalSegmentSize))
+ endLSN := startLSN + walmath.LSN(testWalSegmentSize)
+
+ const racers = 6
+
+ type claimOutcome struct {
+ inserted bool
+ err error
+ }
+
+ results := make(chan claimOutcome, racers)
+ start := make(chan struct{})
+
+ for range racers {
+ go func() {
+ <-start
+
+ // Don't call require.* off the test goroutine — collect and assert below.
+ inserted, err := repo.ClaimInsert(&physical_models.PhysicalWalSegment{
+ DatabaseID: fixture.DB.ID,
+ StorageID: fixture.Storage.ID,
+ TimelineID: 1,
+ WalFilename: walName(1, 40),
+ StartLSN: startLSN,
+ EndLSN: endLSN,
+ Encryption: backups_core_enums.BackupEncryptionNone,
+ })
+ results <- claimOutcome{inserted: inserted, err: err}
+ }()
+ }
+
+ close(start)
+
+ winners := 0
+ for range racers {
+ outcome := <-results
+ require.NoError(t, outcome.err)
+
+ if outcome.inserted {
+ winners++
+ }
+ }
+
+ require.Equal(t, 1, winners, "exactly one concurrent claim may win the (db, tl, start_lsn) slot")
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go
new file mode 100644
index 00000000..a746879a
--- /dev/null
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go
@@ -0,0 +1,159 @@
+package usecases_physical_postgresql
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "databasus-backend/internal/util/walmath"
+)
+
+// uploaderPollInterval — the uploader scans watch_dir this often for newly
+// rotated segments. Segments rotate at the source write rate; a tight loop
+// keeps local dwell time low without measurable CPU.
+const uploaderPollInterval = 1 * time.Second
+
+// runUploaderLoop scans the watch dir on a tight interval and hands each
+// finalized segment / .history file to the appropriate handler.
+func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.Logger) {
+ ticker := time.NewTicker(uploaderPollInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+
+ case <-ticker.C:
+ s.scanAndUpload(ctx, logger)
+ }
+ }
+}
+
+// recoverLocalSegmentsOnStartup sweeps watch_dir once at startup, taking over any
+// finalized segment / .history left by a crash. Uses the uploader's takeover path
+// so a segment whose pre-crash claim row is still file_name NULL gets finished
+// rather than left for the cleaner.
+func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context, logger *slog.Logger) {
+ entries, err := os.ReadDir(s.watchDir)
+ if err != nil {
+ return
+ }
+
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+
+ name := entry.Name()
+
+ switch {
+ case walmath.IsWalFilename(name):
+ if err := s.uploader.RecoverSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {
+ logger.Warn("startup wal recovery failed; live loop will retry", "wal_filename", name, "error", err)
+ }
+
+ case strings.HasSuffix(name, ".history"):
+ s.handleHistoryFile(ctx, logger, name)
+ }
+ }
+}
+
+func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Logger) {
+ entries, err := os.ReadDir(s.watchDir)
+ if err != nil {
+ logger.Error("read wal watch dir", "error", err)
+
+ return
+ }
+
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+
+ name := entry.Name()
+
+ switch {
+ case walmath.IsWalFilename(name):
+ if err := s.uploader.ProcessSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {
+ logger.Warn("wal segment upload failed; will retry next tick", "wal_filename", name, "error", err)
+ }
+
+ case strings.HasSuffix(name, ".history"):
+ s.handleHistoryFile(ctx, logger, name)
+ }
+ }
+}
+
+// handleHistoryFile uploads a .history file the receiver dropped into watch_dir
+// (reusing UploadHistoryFile, which reads the body from the source cluster and is
+// idempotent on (database_id, timeline_id)), then removes the local copy.
+func (s *WalStreamSupervisor) handleHistoryFile(ctx context.Context, logger *slog.Logger, name string) {
+ timelineID, err := parseHistoryTimeline(name)
+ if err != nil {
+ logger.Warn("skip unparseable history file", "name", name, "error", err)
+
+ return
+ }
+
+ conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
+ if err != nil {
+ logger.Warn("could not open connection to upload history file; will retry", "error", err)
+
+ return
+ }
+ defer func() { _ = conn.Close(context.Background()) }()
+
+ if _, err := UploadHistoryFile(
+ ctx, conn, timelineID, s.spec.Storage, s.spec.SourceDB, s.spec.StorageID,
+ s.spec.HistoryRepo, s.spec.Encryption, s.spec.MasterKey, s.spec.FieldEncryptor, logger,
+ ); err != nil {
+ logger.Warn("history upload failed; will retry next tick", "timeline_id", timelineID, "error", err)
+
+ return
+ }
+
+ logger.Info("timeline switch observed via .history", "timeline_id", timelineID)
+
+ if err := os.Remove(filepath.Join(s.watchDir, name)); err != nil && !os.IsNotExist(err) {
+ logger.Warn("failed to remove uploaded history file", "name", name, "error", err)
+ }
+}
+
+func (s *WalStreamSupervisor) removePartials(logger *slog.Logger) {
+ entries, err := os.ReadDir(s.watchDir)
+ if err != nil {
+ return
+ }
+
+ for _, entry := range entries {
+ if !strings.HasSuffix(entry.Name(), ".partial") {
+ continue
+ }
+
+ if err := os.Remove(filepath.Join(s.watchDir, entry.Name())); err != nil && !os.IsNotExist(err) {
+ logger.Warn("failed to remove partial wal file", "name", entry.Name(), "error", err)
+ }
+ }
+}
+
+// parseHistoryTimeline extracts the timeline id from a "%08X.history" filename.
+func parseHistoryTimeline(name string) (int, error) {
+ trimmed := strings.TrimSuffix(name, ".history")
+ if trimmed == name {
+ return 0, fmt.Errorf("not a history filename: %q", name)
+ }
+
+ timelineID, err := strconv.ParseUint(trimmed, 16, 32)
+ if err != nil {
+ return 0, fmt.Errorf("parse timeline from %q: %w", name, err)
+ }
+
+ return int(timelineID), nil
+}