PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
REFACTOR (physical): Split wal_stream.go by responsibility
Rostislav Duginecb982a8e442
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.goindex 3a1afebd..0e2f5bf0 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go@@ -1,10 +1,230 @@package usecases_physical_postgresqlimport (+ "context"+ "fmt"+ "log/slog"+ "os""os/exec"+ "strconv"+ "strings""syscall"+ "time"++ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"+ postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared")+const receivewalApplicationNamePrefix = "databasus_wal_receiver_"++// receiverExit is the disposition of one pg_receivewal run, decided by+// spawnAndSupervise and acted on by runReceivewalSupervision.+type receiverExit int++const (+ receiverCtxCancelled receiverExit = iota // ctx cancelled — stop the loop, no error+ receiverInternalRestart // our own SIGTERM (back pressure / slot stall) — respawn promptly+ receiverRetryable // non-zero exit that a local respawn may fix (network)+ receiverFatal // non-retryable exit — escalate so the supervisor reclaims it on a later tick+)++// spawnAndSupervise starts one pg_receivewal process and blocks until it exits,+// the restart signal fires, or ctx is cancelled. It reports the run's+// disposition, how long the process streamed, and (for receiverFatal) the error.+func (s *WalStreamSupervisor) spawnAndSupervise(+ ctx context.Context,+ logger *slog.Logger,+ pgBin string,+) (receiverExit, time.Duration, error) {+ password, err := postgresql_shared.DecryptFieldIfNeeded(s.spec.SourceDB.Password, s.spec.FieldEncryptor)+ if err != nil {+ logger.Error("decrypt source password for pg_receivewal", "error", err)++ return receiverRetryable, 0, nil+ }++ creds, err := postgresql_shared.WriteCredentialFilesToTempDir(+ s.spec.SourceDB.CredentialSpec(), password, s.spec.FieldEncryptor,+ )+ if err != nil {+ logger.Error("write pg_receivewal credentials", "error", err)++ return receiverRetryable, 0, nil+ }+ defer creds.Remove()++ procCtx, procCancel := context.WithCancel(ctx)+ defer procCancel()++ cmd, err := newReceivewalCommand(procCtx, pgBin, s.spec.SourceDB, creds, s.watchDir, s.slotName)+ if err != nil {+ logger.Error("build pg_receivewal command", "error", err)++ return receiverRetryable, 0, nil+ }++ stderrPipe, err := cmd.StderrPipe()+ if err != nil {+ logger.Error("pg_receivewal stderr pipe", "error", err)++ return receiverRetryable, 0, nil+ }++ if err := cmd.Start(); err != nil {+ logger.Error("start pg_receivewal", "error", err)++ return receiverRetryable, 0, nil+ }++ stderr := newStderrCapture(stderrPipe)+ startedAt := time.Now().UTC()++ logger.Info("pg_receivewal started", "watch_dir", s.watchDir)++ exited := make(chan error, 1)++ go func() { exited <- cmd.Wait() }()++ select {+ case <-ctx.Done():+ procCancel()+ <-exited+ stderr.stop()++ return receiverCtxCancelled, 0, nil++ case <-s.restartSignal:+ logger.Info("restarting pg_receivewal on internal signal (back pressure or slot stall)")+ procCancel()+ <-exited+ stderr.stop()++ return receiverInternalRestart, time.Since(startedAt), nil++ case waitErr := <-exited:+ stderr.stop()+ ranFor := time.Since(startedAt)++ if waitErr == nil || procCtx.Err() != nil {+ return receiverRetryable, ranFor, nil+ }++ stderrText := stderr.contents()++ if isFatalReceivewalError(stderrText) {+ logger.Error("pg_receivewal exited with a non-retryable error; marking streamer for reassignment",+ "error", waitErr, "stderr", truncateStderr(stderrText))++ return receiverFatal, ranFor, fmt.Errorf(+ "pg_receivewal fatal error: %w; stderr: %s", waitErr, truncateStderr(stderrText),+ )+ }++ logger.Warn("pg_receivewal exited; will respawn",+ "error", waitErr, "stderr", truncateStderr(stderrText))++ return receiverRetryable, ranFor, nil+ }+}++// isFatalReceivewalError reports whether a pg_receivewal stderr indicates a+// condition a local respawn can never clear — a full / unwritable local disk,+// rejected credentials, or the replication slot being held by another consumer.+// These escalate to streamer-FAILED so the supervisor can reclaim it on a later+// tick instead of this process crash-looping on an unfixable cause.+func isFatalReceivewalError(stderr []byte) bool {+ // Lower-case both sides: OS errno strings vary in case ("Permission denied")+ // while PG messages are lower-case, and we want to match either.+ text := strings.ToLower(string(stderr))++ for _, needle := range []string{+ "no space left on device",+ "could not write",+ "authentication failed",+ "no pg_hba.conf entry",+ "permission denied",+ "is active for", // replication slot "<name>" is active for PID <n>+ } {+ if strings.Contains(text, needle) {+ return true+ }+ }++ return false+}++// newReceivewalCommand builds the pg_receivewal invocation. WAL is left+// uncompressed locally (no --compress) because the uploader re-compresses with+// zstd on upload; --no-loop makes the process exit on connection loss so the+// supervision loop owns retry; --synchronous flushes each segment promptly. SSL+// is supplied through the same PGSSL* env path pg_basebackup uses, so mTLS needs+// no extra handling here.+func newReceivewalCommand(+ ctx context.Context,+ pgBin string,+ sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,+ creds *postgresql_shared.CredentialTempFiles,+ watchDir string,+ slotName string,+) (*exec.Cmd, error) {+ if _, err := exec.LookPath(pgBin); err != nil {+ return nil, fmt.Errorf("pg_receivewal binary not found at %s: %w", pgBin, err)+ }++ args := []string{+ "--directory=" + watchDir,+ "--slot=" + slotName,+ "--no-loop",+ "--synchronous",+ "--verbose",+ "--no-password",+ "-h", sourceDB.Host,+ "-p", strconv.Itoa(sourceDB.Port),+ "-U", sourceDB.Username,+ }++ cmd := exec.CommandContext(ctx, pgBin, args...)++ cmd.Env = append(os.Environ(),+ "PGPASSFILE="+creds.PgpassPath,+ "PGAPPNAME="+receivewalApplicationName(sourceDB),+ "PGCLIENTENCODING=UTF8",+ "PGCONNECT_TIMEOUT=30",+ "LC_ALL=C.UTF-8",+ "LANG=C.UTF-8",+ )++ sslMode := sourceDB.SslMode+ if sslMode == "" {+ sslMode = postgresql_shared.PostgresSslModeDisable+ }++ cmd.Env = append(cmd.Env,+ "PGSSLMODE="+string(sslMode),+ "PGSSLCERT="+creds.ClientCertPath,+ "PGSSLKEY="+creds.ClientKeyPath,+ "PGSSLROOTCERT="+creds.RootCertPath,+ "PGSSLCRL=",+ )++ cmd.Cancel = func() error {+ return signalForGracefulCancel(cmd.Process)+ }++ cmd.WaitDelay = killAfterCancel+ setReceivewalProcessAttributes(cmd)++ return cmd, nil+}++func receivewalApplicationName(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) string {+ if sourceDB.DatabaseID == nil {+ return receivewalApplicationNamePrefix + sourceDB.ID.String()+ }++ return receivewalApplicationNamePrefix + sourceDB.DatabaseID.String()+}+// setReceivewalProcessAttributes makes the kernel send pg_receivewal a SIGTERM// if the Databasus process dies, so a crashed supervisor never leaks an orphaned// receiver that keeps holding the replication slot. Pdeathsig is Linux-only;diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.gonew file mode 100644index 00000000..e970fa75--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go@@ -0,0 +1,106 @@+package usecases_physical_postgresql++import (+ "context"+ "log/slog"+ "time"++ "databasus-backend/internal/util/walmath"+)++const (+ // slotLsnWatcherPollInterval — consumer-side liveness poll. Tighter than the+ // lag monitor because it tests whether OUR pg_receivewal is actually flushing.+ slotLsnWatcherPollInterval = 10 * time.Second++ // slotLsnStallTimeout — restart_lsn unchanged this long on a healthy PG means+ // pg_receivewal is stuck (alive but flushing nothing); restart it locally.+ slotLsnStallTimeout = 60 * time.Second+)++// stallTracker tracks slot restart_lsn advance across watcher ticks. A changed+// LSN (or the very first sample) re-arms the advance clock; an LSN that stays+// frozen past the stall timeout is the signal that pg_receivewal is alive but+// flushing nothing on a reachable source.+type stallTracker struct {+ lastRestartLSN walmath.LSN+ lastAdvanceAt time.Time+ hasSample bool+}++// observe records a restart_lsn sample taken at now and reports whether the slot+// has stalled longer than stallTimeout. On a positive result it re-arms the clock+// so the caller restarts at most once per stallTimeout window.+func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeout time.Duration) bool {+ if !t.hasSample || restartLSN != t.lastRestartLSN {+ t.lastRestartLSN = restartLSN+ t.lastAdvanceAt = now+ t.hasSample = true++ return false+ }++ if now.Sub(t.lastAdvanceAt) > stallTimeout {+ t.lastAdvanceAt = now++ return true+ }++ return false+}++// runSlotLsnWatcher restarts pg_receivewal when the slot's restart_lsn has not+// advanced for slotLsnStallTimeout while the source PG is still reachable — a+// stuck consumer on a healthy server. A stall with an unreachable server is left+// to the lag monitor (slot loss / network down).+func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slog.Logger) {+ ticker := time.NewTicker(slotLsnWatcherPollInterval)+ defer ticker.Stop()++ var tracker stallTracker++ for {+ select {+ case <-ctx.Done():+ return++ case <-ticker.C:+ restartLSN, pgReachable := s.sampleSlotRestartLSN(ctx, logger)+ if !pgReachable {+ // Connectivity / slot loss is the lag monitor's job, not ours.+ continue+ }++ if tracker.observe(restartLSN, time.Now().UTC(), slotLsnStallTimeout) {+ logger.Warn("slot restart_lsn stalled on a reachable source; restarting pg_receivewal",+ "restart_lsn", restartLSN.String())++ s.signalRestart()+ }+ }+ }+}++// sampleSlotRestartLSN reads the slot's restart_lsn and confirms the source is+// reachable with a keepalive. pgReachable=false means defer to the lag monitor.+func (s *WalStreamSupervisor) sampleSlotRestartLSN(ctx context.Context, logger *slog.Logger) (walmath.LSN, bool) {+ conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)+ if err != nil {+ logger.Debug("slot-lsn watcher: source unreachable, deferring to lag monitor", "error", err)++ return 0, false+ }+ defer func() { _ = conn.Close(context.Background()) }()++ state, err := InspectSlot(ctx, conn, s.slotName)+ if err != nil || state == nil {+ return 0, false+ }++ var alive int+ if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {+ return 0, false+ }++ return state.RestartLSN, true+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.gonew file mode 100644index 00000000..f017532e--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher_test.go@@ -0,0 +1,54 @@+package usecases_physical_postgresql++import (+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "databasus-backend/internal/util/walmath"+)++func Test_StallTracker_WhenFirstSample_DoesNotRestart(t *testing.T) {+ var tracker stallTracker++ base := time.Now().UTC()++ require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute),+ "the first sample only arms the clock; it can never be a stall")+}++func Test_StallTracker_WhenRestartLsnAdvances_ReArmsAndDoesNotRestart(t *testing.T) {+ var tracker stallTracker++ base := time.Now().UTC()++ require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))+ require.False(t, tracker.observe(walmath.LSN(200), base.Add(2*time.Minute), time.Minute),+ "a changed restart_lsn means progress — the advance clock must reset")+}++func Test_StallTracker_WhenFrozenWithinTimeout_DoesNotRestart(t *testing.T) {+ var tracker stallTracker++ base := time.Now().UTC()++ require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))+ require.False(t, tracker.observe(walmath.LSN(100), base.Add(30*time.Second), time.Minute),+ "a frozen restart_lsn within the stall timeout is not yet a stall")+}++func Test_StallTracker_WhenFrozenPastTimeout_RestartsThenReArms(t *testing.T) {+ var tracker stallTracker++ base := time.Now().UTC()++ require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))+ require.True(t, tracker.observe(walmath.LSN(100), base.Add(90*time.Second), time.Minute),+ "a frozen restart_lsn past the stall timeout must trigger a restart")++ require.False(t, tracker.observe(walmath.LSN(100), base.Add(2*time.Minute), time.Minute),+ "after firing, the clock re-arms so we restart at most once per window")+ require.True(t, tracker.observe(walmath.LSN(100), base.Add(4*time.Minute), time.Minute),+ "a sustained stall fires again only after another full timeout window")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.goindex 28f9c2f3..ab009377 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go@@ -56,3 +56,12 @@ func Test_IsOwnedReceiverBackend_WhenActiveButEmptyApplicationName_ReturnsFalse(require.False(t, isOwnedReceiverBackend(state))}++func Test_WalStream_RebuildAttemptCap_StopsFourthAttemptInHour(t *testing.T) {+ supervisor := &WalStreamSupervisor{}++ require.True(t, supervisor.recordRebuildAttemptWithinCap())+ require.True(t, supervisor.recordRebuildAttemptWithinCap())+ require.True(t, supervisor.recordRebuildAttemptWithinCap())+ require.False(t, supervisor.recordRebuildAttemptWithinCap())+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.gonew file mode 100644index 00000000..f46713a6--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go@@ -0,0 +1,96 @@+package usecases_physical_postgresql++import (+ "context"+ "log/slog"+ "os"+ "time"++ "databasus-backend/internal/util/walmath"+)++const (+ // walLocalMinHighWatermarkBytes — pg_receivewal has no+ // kernel-pipe back pressure (it writes files), so the watch dir IS the buffer.+ // Above HIGH we SIGTERM it; we resume only once uploads drain below LOW. The+ // 5x hysteresis prevents flapping on the boundary. HIGH scales up for clusters+ // with non-default wal_segment_size so one segment does not stop the receiver.+ walLocalMinHighWatermarkBytes int64 = 100 * 1024 * 1024++ backpressurePollInterval = 1 * time.Second+)++// runBackpressureMonitor SIGTERMs pg_receivewal (via the restart signal) when the+// local watch-dir backlog crosses the high watermark; the supervision loop then+// waits for the uploader to drain below the low watermark before respawning.+func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slog.Logger) {+ ticker := time.NewTicker(backpressurePollInterval)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return++ case <-ticker.C:+ if s.backlogBytes() >= s.highWatermarkBytes {+ s.signalRestart()+ }+ }+ }+}++// waitForBacklogBelowLow blocks while the backlog is at/over the high watermark,+// returning once it drains below the low watermark. Returns false if ctx is+// cancelled while waiting.+func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger *slog.Logger) bool {+ if s.backlogBytes() < s.highWatermarkBytes {+ return true+ }++ logger.Warn("wal backlog over high watermark; pausing pg_receivewal until it drains")++ ticker := time.NewTicker(backpressurePollInterval)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return false++ case <-ticker.C:+ if s.backlogBytes() < s.lowWatermarkBytes {+ logger.Info("wal backlog drained below low watermark; resuming pg_receivewal")++ return true+ }+ }+ }+}++// backlogBytes sums the sizes of finalized (not-yet-uploaded) WAL segments in the+// watch dir. Uploaded segments are removed by the uploader, so this is the local+// queue depth; .partial and .history files are excluded.+func (s *WalStreamSupervisor) backlogBytes() int64 {+ entries, err := os.ReadDir(s.watchDir)+ if err != nil {+ return 0+ }++ var total int64++ for _, entry := range entries {+ if entry.IsDir() || !walmath.IsWalFilename(entry.Name()) {+ continue+ }++ info, err := entry.Info()+ if err != nil {+ continue+ }++ total += info.Size()+ }++ return total+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.gonew file mode 100644index 00000000..a6730189--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure_test.go@@ -0,0 +1,25 @@+package usecases_physical_postgresql++import (+ "testing"++ "github.com/stretchr/testify/require"++ "databasus-backend/internal/util/logger"+)++func Test_WalStream_BackpressureWatermarks_ScaleWithWalSegmentSize(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)+ customSegSize := int64(512 * 1024 * 1024)+ fixture.DB.PostgresqlPhysical.WalSegmentSizeBytes = &customSegSize++ supervisor := NewWalStreamSupervisor(WalStreamSpec{+ DatabaseID: fixture.DB.ID,+ SourceDB: fixture.DB.PostgresqlPhysical,+ WatchDirRoot: t.TempDir(),+ Logger: logger.GetLogger(),+ })++ require.Equal(t, 4*customSegSize, supervisor.highWatermarkBytes)+ require.Equal(t, 4*customSegSize/5, supervisor.lowWatermarkBytes)+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.goindex 770b80af..f9a5541a 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go@@ -5,10 +5,7 @@ import ("fmt""log/slog""os"- "os/exec""path/filepath"- "strconv"- "strings""sync""sync/atomic""time"@@ -18,7 +15,6 @@ import (backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"- postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared""databasus-backend/internal/features/storages"util_encryption "databasus-backend/internal/util/encryption""databasus-backend/internal/util/tools"@@ -26,28 +22,6 @@ import ()const (- receivewalApplicationNamePrefix = "databasus_wal_receiver_"-- // uploaderPollInterval — the uploader scans watch_dir this often for newly- // rotated segments. Segments rotate at the source write rate; a tight loop- // keeps local dwell time low without measurable CPU.- uploaderPollInterval = 1 * time.Second-- // slotLsnWatcherPollInterval — consumer-side liveness poll. Tighter than the- // lag monitor because it tests whether OUR pg_receivewal is actually flushing.- slotLsnWatcherPollInterval = 10 * time.Second-- // slotLsnStallTimeout — restart_lsn unchanged this long on a healthy PG means- // pg_receivewal is stuck (alive but flushing nothing); restart it locally.- slotLsnStallTimeout = 60 * time.Second-- // walLocalMinHighWatermarkBytes — pg_receivewal has no- // kernel-pipe back pressure (it writes files), so the watch dir IS the buffer.- // Above HIGH we SIGTERM it; we resume only once uploads drain below LOW. The- // 5x hysteresis prevents flapping on the boundary. HIGH scales up for clusters- // with non-default wal_segment_size so one segment does not stop the receiver.- walLocalMinHighWatermarkBytes int64 = 100 * 1024 * 1024-// receivewalRespawnBackoff — initial pause between a pg_receivewal exit and respawn so// a hard-failing source (auth, pg_hba) is not hammered. --no-loop makes// pg_receivewal exit on connection loss; this loop is its supervision.@@ -66,17 +40,8 @@ const (// forever on a condition a local respawn can never fix (ENOSPC, bad creds, a// slot held by a thief).receivewalMaxRapidFailures = 5-)--// receiverExit is the disposition of one pg_receivewal run, decided by-// spawnAndSupervise and acted on by runReceivewalSupervision.-type receiverExit int-const (- receiverCtxCancelled receiverExit = iota // ctx cancelled — stop the loop, no error- receiverInternalRestart // our own SIGTERM (back pressure / slot stall) — respawn promptly- receiverRetryable // non-zero exit that a local respawn may fix (network)- receiverFatal // non-retryable exit — escalate so the supervisor reclaims it on a later tick+ pausePollInterval = 1 * time.Second)// WalStreamSpec is the immutable configuration of one database's WAL streamer.@@ -303,418 +268,6 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg}}-// spawnAndSupervise starts one pg_receivewal process and blocks until it exits,-// the restart signal fires, or ctx is cancelled. It reports the run's-// disposition, how long the process streamed, and (for receiverFatal) the error.-func (s *WalStreamSupervisor) spawnAndSupervise(- ctx context.Context,- logger *slog.Logger,- pgBin string,-) (receiverExit, time.Duration, error) {- password, err := postgresql_shared.DecryptFieldIfNeeded(s.spec.SourceDB.Password, s.spec.FieldEncryptor)- if err != nil {- logger.Error("decrypt source password for pg_receivewal", "error", err)-- return receiverRetryable, 0, nil- }-- creds, err := postgresql_shared.WriteCredentialFilesToTempDir(- s.spec.SourceDB.CredentialSpec(), password, s.spec.FieldEncryptor,- )- if err != nil {- logger.Error("write pg_receivewal credentials", "error", err)-- return receiverRetryable, 0, nil- }- defer creds.Remove()-- procCtx, procCancel := context.WithCancel(ctx)- defer procCancel()-- cmd, err := newReceivewalCommand(procCtx, pgBin, s.spec.SourceDB, creds, s.watchDir, s.slotName)- if err != nil {- logger.Error("build pg_receivewal command", "error", err)-- return receiverRetryable, 0, nil- }-- stderrPipe, err := cmd.StderrPipe()- if err != nil {- logger.Error("pg_receivewal stderr pipe", "error", err)-- return receiverRetryable, 0, nil- }-- if err := cmd.Start(); err != nil {- logger.Error("start pg_receivewal", "error", err)-- return receiverRetryable, 0, nil- }-- stderr := newStderrCapture(stderrPipe)- startedAt := time.Now().UTC()-- logger.Info("pg_receivewal started", "watch_dir", s.watchDir)-- exited := make(chan error, 1)-- go func() { exited <- cmd.Wait() }()-- select {- case <-ctx.Done():- procCancel()- <-exited- stderr.stop()-- return receiverCtxCancelled, 0, nil-- case <-s.restartSignal:- logger.Info("restarting pg_receivewal on internal signal (back pressure or slot stall)")- procCancel()- <-exited- stderr.stop()-- return receiverInternalRestart, time.Since(startedAt), nil-- case waitErr := <-exited:- stderr.stop()- ranFor := time.Since(startedAt)-- if waitErr == nil || procCtx.Err() != nil {- return receiverRetryable, ranFor, nil- }-- stderrText := stderr.contents()-- if isFatalReceivewalError(stderrText) {- logger.Error("pg_receivewal exited with a non-retryable error; marking streamer for reassignment",- "error", waitErr, "stderr", truncateStderr(stderrText))-- return receiverFatal, ranFor, fmt.Errorf(- "pg_receivewal fatal error: %w; stderr: %s", waitErr, truncateStderr(stderrText),- )- }-- logger.Warn("pg_receivewal exited; will respawn",- "error", waitErr, "stderr", truncateStderr(stderrText))-- return receiverRetryable, ranFor, nil- }-}--// isFatalReceivewalError reports whether a pg_receivewal stderr indicates a-// condition a local respawn can never clear — a full / unwritable local disk,-// rejected credentials, or the replication slot being held by another consumer.-// These escalate to streamer-FAILED so the supervisor can reclaim it on a later-// tick instead of this process crash-looping on an unfixable cause.-func isFatalReceivewalError(stderr []byte) bool {- // Lower-case both sides: OS errno strings vary in case ("Permission denied")- // while PG messages are lower-case, and we want to match either.- text := strings.ToLower(string(stderr))-- for _, needle := range []string{- "no space left on device",- "could not write",- "authentication failed",- "no pg_hba.conf entry",- "permission denied",- "is active for", // replication slot "<name>" is active for PID <n>- } {- if strings.Contains(text, needle) {- return true- }- }-- return false-}--// runUploaderLoop scans the watch dir on a tight interval and hands each-// finalized segment / .history file to the appropriate handler.-func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.Logger) {- ticker := time.NewTicker(uploaderPollInterval)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return-- case <-ticker.C:- s.scanAndUpload(ctx, logger)- }- }-}--// recoverLocalSegmentsOnStartup sweeps watch_dir once at startup, taking over any-// finalized segment / .history left by a crash. Uses the uploader's takeover path-// so a segment whose pre-crash claim row is still file_name NULL gets finished-// rather than left for the cleaner.-func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context, logger *slog.Logger) {- entries, err := os.ReadDir(s.watchDir)- if err != nil {- return- }-- for _, entry := range entries {- if entry.IsDir() {- continue- }-- name := entry.Name()-- switch {- case walmath.IsWalFilename(name):- if err := s.uploader.RecoverSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {- logger.Warn("startup wal recovery failed; live loop will retry", "wal_filename", name, "error", err)- }-- case strings.HasSuffix(name, ".history"):- s.handleHistoryFile(ctx, logger, name)- }- }-}--func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Logger) {- entries, err := os.ReadDir(s.watchDir)- if err != nil {- logger.Error("read wal watch dir", "error", err)-- return- }-- for _, entry := range entries {- if entry.IsDir() {- continue- }-- name := entry.Name()-- switch {- case walmath.IsWalFilename(name):- if err := s.uploader.ProcessSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {- logger.Warn("wal segment upload failed; will retry next tick", "wal_filename", name, "error", err)- }-- case strings.HasSuffix(name, ".history"):- s.handleHistoryFile(ctx, logger, name)- }- }-}--// handleHistoryFile uploads a .history file the receiver dropped into watch_dir-// (reusing UploadHistoryFile, which reads the body from the source cluster and is-// idempotent on (database_id, timeline_id)), then removes the local copy.-func (s *WalStreamSupervisor) handleHistoryFile(ctx context.Context, logger *slog.Logger, name string) {- timelineID, err := parseHistoryTimeline(name)- if err != nil {- logger.Warn("skip unparseable history file", "name", name, "error", err)-- return- }-- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)- if err != nil {- logger.Warn("could not open connection to upload history file; will retry", "error", err)-- return- }- defer func() { _ = conn.Close(context.Background()) }()-- if _, err := UploadHistoryFile(- ctx, conn, timelineID, s.spec.Storage, s.spec.SourceDB, s.spec.StorageID,- s.spec.HistoryRepo, s.spec.Encryption, s.spec.MasterKey, s.spec.FieldEncryptor, logger,- ); err != nil {- logger.Warn("history upload failed; will retry next tick", "timeline_id", timelineID, "error", err)-- return- }-- logger.Info("timeline switch observed via .history", "timeline_id", timelineID)-- if err := os.Remove(filepath.Join(s.watchDir, name)); err != nil && !os.IsNotExist(err) {- logger.Warn("failed to remove uploaded history file", "name", name, "error", err)- }-}--// stallTracker tracks slot restart_lsn advance across watcher ticks. A changed-// LSN (or the very first sample) re-arms the advance clock; an LSN that stays-// frozen past the stall timeout is the signal that pg_receivewal is alive but-// flushing nothing on a reachable source.-type stallTracker struct {- lastRestartLSN walmath.LSN- lastAdvanceAt time.Time- hasSample bool-}--// observe records a restart_lsn sample taken at now and reports whether the slot-// has stalled longer than stallTimeout. On a positive result it re-arms the clock-// so the caller restarts at most once per stallTimeout window.-func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeout time.Duration) bool {- if !t.hasSample || restartLSN != t.lastRestartLSN {- t.lastRestartLSN = restartLSN- t.lastAdvanceAt = now- t.hasSample = true-- return false- }-- if now.Sub(t.lastAdvanceAt) > stallTimeout {- t.lastAdvanceAt = now-- return true- }-- return false-}--// runSlotLsnWatcher restarts pg_receivewal when the slot's restart_lsn has not-// advanced for slotLsnStallTimeout while the source PG is still reachable — a-// stuck consumer on a healthy server. A stall with an unreachable server is left-// to the lag monitor (slot loss / network down).-func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slog.Logger) {- ticker := time.NewTicker(slotLsnWatcherPollInterval)- defer ticker.Stop()-- var tracker stallTracker-- for {- select {- case <-ctx.Done():- return-- case <-ticker.C:- restartLSN, pgReachable := s.sampleSlotRestartLSN(ctx, logger)- if !pgReachable {- // Connectivity / slot loss is the lag monitor's job, not ours.- continue- }-- if tracker.observe(restartLSN, time.Now().UTC(), slotLsnStallTimeout) {- logger.Warn("slot restart_lsn stalled on a reachable source; restarting pg_receivewal",- "restart_lsn", restartLSN.String())-- s.signalRestart()- }- }- }-}--// sampleSlotRestartLSN reads the slot's restart_lsn and confirms the source is-// reachable with a keepalive. pgReachable=false means defer to the lag monitor.-func (s *WalStreamSupervisor) sampleSlotRestartLSN(ctx context.Context, logger *slog.Logger) (walmath.LSN, bool) {- conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)- if err != nil {- logger.Debug("slot-lsn watcher: source unreachable, deferring to lag monitor", "error", err)-- return 0, false- }- defer func() { _ = conn.Close(context.Background()) }()-- state, err := InspectSlot(ctx, conn, s.slotName)- if err != nil || state == nil {- return 0, false- }-- var alive int- if err := conn.QueryRow(ctx, "SELECT 1").Scan(&alive); err != nil {- return 0, false- }-- return state.RestartLSN, true-}--// runBackpressureMonitor SIGTERMs pg_receivewal (via the restart signal) when the-// local watch-dir backlog crosses the high watermark; the supervision loop then-// waits for the uploader to drain below the low watermark before respawning.-func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slog.Logger) {- ticker := time.NewTicker(uploaderPollInterval)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return-- case <-ticker.C:- if s.backlogBytes() >= s.highWatermarkBytes {- s.signalRestart()- }- }- }-}--// waitForBacklogBelowLow blocks while the backlog is at/over the high watermark,-// returning once it drains below the low watermark. Returns false if ctx is-// cancelled while waiting.-func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger *slog.Logger) bool {- if s.backlogBytes() < s.highWatermarkBytes {- return true- }-- logger.Warn("wal backlog over high watermark; pausing pg_receivewal until it drains")-- ticker := time.NewTicker(uploaderPollInterval)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return false-- case <-ticker.C:- if s.backlogBytes() < s.lowWatermarkBytes {- logger.Info("wal backlog drained below low watermark; resuming pg_receivewal")-- return true- }- }- }-}--// backlogBytes sums the sizes of finalized (not-yet-uploaded) WAL segments in the-// watch dir. Uploaded segments are removed by the uploader, so this is the local-// queue depth; .partial and .history files are excluded.-func (s *WalStreamSupervisor) backlogBytes() int64 {- entries, err := os.ReadDir(s.watchDir)- if err != nil {- return 0- }-- var total int64-- for _, entry := range entries {- if entry.IsDir() || !walmath.IsWalFilename(entry.Name()) {- continue- }-- info, err := entry.Info()- if err != nil {- continue- }-- total += info.Size()- }-- return total-}--func (s *WalStreamSupervisor) removePartials(logger *slog.Logger) {- entries, err := os.ReadDir(s.watchDir)- if err != nil {- return- }-- for _, entry := range entries {- if !strings.HasSuffix(entry.Name(), ".partial") {- continue- }-- if err := os.Remove(filepath.Join(s.watchDir, entry.Name())); err != nil && !os.IsNotExist(err) {- logger.Warn("failed to remove partial wal file", "name", entry.Name(), "error", err)- }- }-}-// waitWhilePaused blocks the supervision loop while a slot rebuild holds the// receiver down. Returns false if ctx is cancelled while paused.func (s *WalStreamSupervisor) waitWhilePaused(ctx context.Context) bool {@@ -722,7 +275,7 @@ func (s *WalStreamSupervisor) waitWhilePaused(ctx context.Context) bool {return true}- ticker := time.NewTicker(uploaderPollInterval)+ ticker := time.NewTicker(pausePollInterval)defer ticker.Stop()for {@@ -752,78 +305,6 @@ func (s *WalStreamSupervisor) drainRestartSignal() {}}-// newReceivewalCommand builds the pg_receivewal invocation. WAL is left-// uncompressed locally (no --compress) because the uploader re-compresses with-// zstd on upload; --no-loop makes the process exit on connection loss so the-// supervision loop owns retry; --synchronous flushes each segment promptly. SSL-// is supplied through the same PGSSL* env path pg_basebackup uses, so mTLS needs-// no extra handling here.-func newReceivewalCommand(- ctx context.Context,- pgBin string,- sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,- creds *postgresql_shared.CredentialTempFiles,- watchDir string,- slotName string,-) (*exec.Cmd, error) {- if _, err := exec.LookPath(pgBin); err != nil {- return nil, fmt.Errorf("pg_receivewal binary not found at %s: %w", pgBin, err)- }-- args := []string{- "--directory=" + watchDir,- "--slot=" + slotName,- "--no-loop",- "--synchronous",- "--verbose",- "--no-password",- "-h", sourceDB.Host,- "-p", strconv.Itoa(sourceDB.Port),- "-U", sourceDB.Username,- }-- cmd := exec.CommandContext(ctx, pgBin, args...)-- cmd.Env = append(os.Environ(),- "PGPASSFILE="+creds.PgpassPath,- "PGAPPNAME="+receivewalApplicationName(sourceDB),- "PGCLIENTENCODING=UTF8",- "PGCONNECT_TIMEOUT=30",- "LC_ALL=C.UTF-8",- "LANG=C.UTF-8",- )-- sslMode := sourceDB.SslMode- if sslMode == "" {- sslMode = postgresql_shared.PostgresSslModeDisable- }-- cmd.Env = append(cmd.Env,- "PGSSLMODE="+string(sslMode),- "PGSSLCERT="+creds.ClientCertPath,- "PGSSLKEY="+creds.ClientKeyPath,- "PGSSLROOTCERT="+creds.RootCertPath,- "PGSSLCRL=",- )-- cmd.Cancel = func() error {- return signalForGracefulCancel(cmd.Process)- }-- cmd.WaitDelay = killAfterCancel- setReceivewalProcessAttributes(cmd)-- return cmd, nil-}--func receivewalApplicationName(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) string {- if sourceDB.DatabaseID == nil {- return receivewalApplicationNamePrefix + sourceDB.ID.String()- }-- return receivewalApplicationNamePrefix + sourceDB.DatabaseID.String()-}-// walSegmentSizeBytes returns the source cluster's captured wal_segment_size, or// the 16 MB default when it has not been captured yet.func walSegmentSizeBytes(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) int64 {@@ -834,21 +315,6 @@ func walSegmentSizeBytes(sourceDB *postgresql_physical.PostgresqlPhysicalDatabasreturn int64(walmath.WalSegmentSize)}-// parseHistoryTimeline extracts the timeline id from a "%08X.history" filename.-func parseHistoryTimeline(name string) (int, error) {- trimmed := strings.TrimSuffix(name, ".history")- if trimmed == name {- return 0, fmt.Errorf("not a history filename: %q", name)- }-- timelineID, err := strconv.ParseUint(trimmed, 16, 32)- if err != nil {- return 0, fmt.Errorf("parse timeline from %q: %w", name, err)- }-- return int(timelineID), nil-}-func sleepCtx(ctx context.Context, d time.Duration) bool {timer := time.NewTimer(d)defer timer.Stop()diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.goindex fb95ada7..8687b8b9 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go@@ -286,31 +286,6 @@ func Test_WalStream_SlotLagGrowsWithoutConsumer_DrainsOnceStreaming(t *testing.Tt.Fatalf("slot lag did not drain below %d within 60s after streaming started", lagTarget)}-func Test_WalStream_BackpressureWatermarks_ScaleWithWalSegmentSize(t *testing.T) {- fixture := SetupPhysicalDBForBackup(t)- customSegSize := int64(512 * 1024 * 1024)- fixture.DB.PostgresqlPhysical.WalSegmentSizeBytes = &customSegSize-- supervisor := NewWalStreamSupervisor(WalStreamSpec{- DatabaseID: fixture.DB.ID,- SourceDB: fixture.DB.PostgresqlPhysical,- WatchDirRoot: t.TempDir(),- Logger: logger.GetLogger(),- })-- require.Equal(t, 4*customSegSize, supervisor.highWatermarkBytes)- require.Equal(t, 4*customSegSize/5, supervisor.lowWatermarkBytes)-}--func Test_WalStream_RebuildAttemptCap_StopsFourthAttemptInHour(t *testing.T) {- supervisor := &WalStreamSupervisor{}-- require.True(t, supervisor.recordRebuildAttemptWithinCap())- require.True(t, supervisor.recordRebuildAttemptWithinCap())- require.True(t, supervisor.recordRebuildAttemptWithinCap())- require.False(t, supervisor.recordRebuildAttemptWithinCap())-}-func Test_WalStream_CustomWalSegmentSize_LsnMathCorrect(t *testing.T) {fixture := SetupPhysicalDBForBackup(t)@@ -342,56 +317,6 @@ func Test_WalStream_CustomWalSegmentSize_LsnMathCorrect(t *testing.T) {require.Equal(t, wantStart+walmath.LSN(customSegSize), row.EndLSN)}-func Test_WalUpload_ConcurrentClaimSameSegment_OnlyWinnerInserts(t *testing.T) {- fixture := SetupPhysicalDBForBackup(t)-- repo := physical_repositories.GetWalSegmentRepository()- startLSN := walmath.LSN(40 * uint64(testWalSegmentSize))- endLSN := startLSN + walmath.LSN(testWalSegmentSize)-- const racers = 6-- type claimOutcome struct {- inserted bool- err error- }-- results := make(chan claimOutcome, racers)- start := make(chan struct{})-- for range racers {- go func() {- <-start-- // Don't call require.* off the test goroutine — collect and assert below.- inserted, err := repo.ClaimInsert(&physical_models.PhysicalWalSegment{- DatabaseID: fixture.DB.ID,- StorageID: fixture.Storage.ID,- TimelineID: 1,- WalFilename: walName(1, 40),- StartLSN: startLSN,- EndLSN: endLSN,- Encryption: backups_core_enums.BackupEncryptionNone,- })- results <- claimOutcome{inserted: inserted, err: err}- }()- }-- close(start)-- winners := 0- for range racers {- outcome := <-results- require.NoError(t, outcome.err)-- if outcome.inserted {- winners++- }- }-- require.Equal(t, 1, winners, "exactly one concurrent claim may win the (db, tl, start_lsn) slot")-}-func Test_Cleaner_AbandonedNullClaim_OlderThanGrace_DeletedYoungerSurvives(t *testing.T) {fixture := SetupPhysicalDBForBackup(t)@@ -432,47 +357,3 @@ func Test_Cleaner_AbandonedNullClaim_OlderThanGrace_DeletedYoungerSurvives(t *terequire.Nil(t, findWalSegment(t, fixture.DB.ID, 1, oldClaim.StartLSN), "aged claim must be gone")require.NotNil(t, findWalSegment(t, fixture.DB.ID, 1, youngClaim.StartLSN), "within-grace claim must survive")}--func Test_StallTracker_WhenFirstSample_DoesNotRestart(t *testing.T) {- var tracker stallTracker-- base := time.Now().UTC()-- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute),- "the first sample only arms the clock; it can never be a stall")-}--func Test_StallTracker_WhenRestartLsnAdvances_ReArmsAndDoesNotRestart(t *testing.T) {- var tracker stallTracker-- base := time.Now().UTC()-- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))- require.False(t, tracker.observe(walmath.LSN(200), base.Add(2*time.Minute), time.Minute),- "a changed restart_lsn means progress — the advance clock must reset")-}--func Test_StallTracker_WhenFrozenWithinTimeout_DoesNotRestart(t *testing.T) {- var tracker stallTracker-- base := time.Now().UTC()-- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))- require.False(t, tracker.observe(walmath.LSN(100), base.Add(30*time.Second), time.Minute),- "a frozen restart_lsn within the stall timeout is not yet a stall")-}--func Test_StallTracker_WhenFrozenPastTimeout_RestartsThenReArms(t *testing.T) {- var tracker stallTracker-- base := time.Now().UTC()-- require.False(t, tracker.observe(walmath.LSN(100), base, time.Minute))- require.True(t, tracker.observe(walmath.LSN(100), base.Add(90*time.Second), time.Minute),- "a frozen restart_lsn past the stall timeout must trigger a restart")-- require.False(t, tracker.observe(walmath.LSN(100), base.Add(2*time.Minute), time.Minute),- "after firing, the clock re-arms so we restart at most once per window")- require.True(t, tracker.observe(walmath.LSN(100), base.Add(4*time.Minute), time.Minute),- "a sustained stall fires again only after another full timeout window")-}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.goindex 300bbdca..08fd6a89 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go@@ -460,3 +460,53 @@ func Test_BuildWalSegmentArtifactReader_WhenLargeSegment_StreamsAndCountsArtifacrequire.Equal(t, int64(len(body)), compressedSizeBytes)require.NotEmpty(t, body)}++func Test_WalUpload_ConcurrentClaimSameSegment_OnlyWinnerInserts(t *testing.T) {+ fixture := SetupPhysicalDBForBackup(t)++ repo := physical_repositories.GetWalSegmentRepository()+ startLSN := walmath.LSN(40 * uint64(testWalSegmentSize))+ endLSN := startLSN + walmath.LSN(testWalSegmentSize)++ const racers = 6++ type claimOutcome struct {+ inserted bool+ err error+ }++ results := make(chan claimOutcome, racers)+ start := make(chan struct{})++ for range racers {+ go func() {+ <-start++ // Don't call require.* off the test goroutine — collect and assert below.+ inserted, err := repo.ClaimInsert(&physical_models.PhysicalWalSegment{+ DatabaseID: fixture.DB.ID,+ StorageID: fixture.Storage.ID,+ TimelineID: 1,+ WalFilename: walName(1, 40),+ StartLSN: startLSN,+ EndLSN: endLSN,+ Encryption: backups_core_enums.BackupEncryptionNone,+ })+ results <- claimOutcome{inserted: inserted, err: err}+ }()+ }++ close(start)++ winners := 0+ for range racers {+ outcome := <-results+ require.NoError(t, outcome.err)++ if outcome.inserted {+ winners+++ }+ }++ require.Equal(t, 1, winners, "exactly one concurrent claim may win the (db, tl, start_lsn) slot")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.gonew file mode 100644index 00000000..a746879a--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go@@ -0,0 +1,159 @@+package usecases_physical_postgresql++import (+ "context"+ "fmt"+ "log/slog"+ "os"+ "path/filepath"+ "strconv"+ "strings"+ "time"++ "databasus-backend/internal/util/walmath"+)++// uploaderPollInterval — the uploader scans watch_dir this often for newly+// rotated segments. Segments rotate at the source write rate; a tight loop+// keeps local dwell time low without measurable CPU.+const uploaderPollInterval = 1 * time.Second++// runUploaderLoop scans the watch dir on a tight interval and hands each+// finalized segment / .history file to the appropriate handler.+func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.Logger) {+ ticker := time.NewTicker(uploaderPollInterval)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return++ case <-ticker.C:+ s.scanAndUpload(ctx, logger)+ }+ }+}++// recoverLocalSegmentsOnStartup sweeps watch_dir once at startup, taking over any+// finalized segment / .history left by a crash. Uses the uploader's takeover path+// so a segment whose pre-crash claim row is still file_name NULL gets finished+// rather than left for the cleaner.+func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context, logger *slog.Logger) {+ entries, err := os.ReadDir(s.watchDir)+ if err != nil {+ return+ }++ for _, entry := range entries {+ if entry.IsDir() {+ continue+ }++ name := entry.Name()++ switch {+ case walmath.IsWalFilename(name):+ if err := s.uploader.RecoverSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {+ logger.Warn("startup wal recovery failed; live loop will retry", "wal_filename", name, "error", err)+ }++ case strings.HasSuffix(name, ".history"):+ s.handleHistoryFile(ctx, logger, name)+ }+ }+}++func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Logger) {+ entries, err := os.ReadDir(s.watchDir)+ if err != nil {+ logger.Error("read wal watch dir", "error", err)++ return+ }++ for _, entry := range entries {+ if entry.IsDir() {+ continue+ }++ name := entry.Name()++ switch {+ case walmath.IsWalFilename(name):+ if err := s.uploader.ProcessSegment(ctx, filepath.Join(s.watchDir, name), name); err != nil {+ logger.Warn("wal segment upload failed; will retry next tick", "wal_filename", name, "error", err)+ }++ case strings.HasSuffix(name, ".history"):+ s.handleHistoryFile(ctx, logger, name)+ }+ }+}++// handleHistoryFile uploads a .history file the receiver dropped into watch_dir+// (reusing UploadHistoryFile, which reads the body from the source cluster and is+// idempotent on (database_id, timeline_id)), then removes the local copy.+func (s *WalStreamSupervisor) handleHistoryFile(ctx context.Context, logger *slog.Logger, name string) {+ timelineID, err := parseHistoryTimeline(name)+ if err != nil {+ logger.Warn("skip unparseable history file", "name", name, "error", err)++ return+ }++ conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)+ if err != nil {+ logger.Warn("could not open connection to upload history file; will retry", "error", err)++ return+ }+ defer func() { _ = conn.Close(context.Background()) }()++ if _, err := UploadHistoryFile(+ ctx, conn, timelineID, s.spec.Storage, s.spec.SourceDB, s.spec.StorageID,+ s.spec.HistoryRepo, s.spec.Encryption, s.spec.MasterKey, s.spec.FieldEncryptor, logger,+ ); err != nil {+ logger.Warn("history upload failed; will retry next tick", "timeline_id", timelineID, "error", err)++ return+ }++ logger.Info("timeline switch observed via .history", "timeline_id", timelineID)++ if err := os.Remove(filepath.Join(s.watchDir, name)); err != nil && !os.IsNotExist(err) {+ logger.Warn("failed to remove uploaded history file", "name", name, "error", err)+ }+}++func (s *WalStreamSupervisor) removePartials(logger *slog.Logger) {+ entries, err := os.ReadDir(s.watchDir)+ if err != nil {+ return+ }++ for _, entry := range entries {+ if !strings.HasSuffix(entry.Name(), ".partial") {+ continue+ }++ if err := os.Remove(filepath.Join(s.watchDir, entry.Name())); err != nil && !os.IsNotExist(err) {+ logger.Warn("failed to remove partial wal file", "name", entry.Name(), "error", err)+ }+ }+}++// parseHistoryTimeline extracts the timeline id from a "%08X.history" filename.+func parseHistoryTimeline(name string) (int, error) {+ trimmed := strings.TrimSuffix(name, ".history")+ if trimmed == name {+ return 0, fmt.Errorf("not a history filename: %q", name)+ }++ timelineID, err := strconv.ParseUint(trimmed, 16, 32)+ if err != nil {+ return 0, fmt.Errorf("parse timeline from %q: %w", name, err)+ }++ return int(timelineID), nil+}