PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
REFACTOR (physical): Clean up wal streaming names and comments
Rostislav Dugin2e4e17f241de
diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.goindex 43cb034e..6b06eb67 100644--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go@@ -28,11 +28,9 @@ import ("databasus-backend/internal/util/walmath")-// PhysicalWalStreamSupervisor is the background service that owns long-running-// pg_receivewal streamers. Ownership and restart recovery run through the-// physical_wal_streamers heartbeat table: every tick it discovers WAL_STREAM-// databases, CAS-claims any that are unclaimed / FAILED / stale, runs one-// WalStreamSupervisor goroutine per owned DB, and heartbeats the rows it owns.+// Ownership and restart recovery run through the physical_wal_streamers+// heartbeat table: every tick CAS-claims the databases that are unclaimed /+// FAILED / stale, so exactly one process streams a given database at a time.type PhysicalWalStreamSupervisor struct {databaseService *databases.DatabaseServicebackupConfigService *backups_config_physical.BackupConfigService@@ -55,7 +53,6 @@ type PhysicalWalStreamSupervisor struct {isReady atomic.Bool}-// runningStreamer is the handle to one locally-owned streamer goroutine.type runningStreamer struct {cancel context.CancelFuncdone chan struct{}@@ -133,9 +130,6 @@ func (s *PhysicalWalStreamSupervisor) recoverStreamersOnStartup() error {return nil}-// reconcile starts streamers for newly-claimable WAL_STREAM databases, stops-// local streamers whose database is no longer a candidate (disabled or demoted),-// and heartbeats the rows we own.func (s *PhysicalWalStreamSupervisor) reconcile(ctx context.Context) {configs, err := s.backupConfigService.GetBackupConfigsWithEnabledBackups()if err != nil {@@ -168,8 +162,6 @@ func (s *PhysicalWalStreamSupervisor) reconcile(ctx context.Context) {s.heartbeatOwnedStreamers()}-// isWalStreamCandidate reports whether a config should have a running streamer:-// WAL_STREAM backup type with storage configured.func isWalStreamCandidate(backupConfig *backups_config_physical.PhysicalBackupConfig) bool {if backupConfig.PostgresqlPhysical == nil ||backupConfig.PostgresqlPhysical.BackupType != postgresql_physical.BackupTypeFullIncrementalAndWalStream {@@ -179,8 +171,6 @@ func isWalStreamCandidate(backupConfig *backups_config_physical.PhysicalBackupCoreturn backupConfig.StorageID != nil}-// ensureStreamerRunning starts a streamer for backupConfig when we don't already-// run one locally and we can win the heartbeat-table claim.func (s *PhysicalWalStreamSupervisor) ensureStreamerRunning(ctx context.Context,logger *slog.Logger,@@ -317,8 +307,6 @@ func (s *PhysicalWalStreamSupervisor) resolveMasterKey(return key, true}-// gapNotifier builds the post-upload gap-probe callback: it sends a BackupFailed-// notification (when the config opts in) to each of the database's notifiers.func (s *PhysicalWalStreamSupervisor) gapNotifier(db *databases.Database,backupConfig *backups_config_physical.PhysicalBackupConfig,@@ -445,8 +433,7 @@ func (s *PhysicalWalStreamSupervisor) removeWatchDirIfRequested(logger *slog.Log}}-// releaseClaim marks a just-won streamer row FAILED when we could not actually-// start the streamer, so the slot is immediately reclaimable rather than looking+// Marking FAILED makes the slot immediately reclaimable rather than looking// alive until the heartbeat goes stale.func (s *PhysicalWalStreamSupervisor) releaseClaim(logger *slog.Logger, databaseID uuid.UUID) {if err := s.walStreamerRepo.MarkFailed(databaseID); err != nil {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.goindex ab87f592..db4280b0 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go@@ -7,27 +7,26 @@ import ()const (- // lagMonitorPollInterval — cadence for reading pg_replication_slots. Balances- // detection latency against query load on the source (one cheap indexed row).+ // Balances detection latency against query load on the source (one cheap+ // indexed row).lagMonitorPollInterval = 30 * time.Second- // extendedSlotStatusHoldPeriod — an 'extended' slot status must persist this- // long before we treat it as a real lag (vs a transient write burst).+ // An 'extended' slot status must persist this long before we treat it as a+ // real lag (vs a transient write burst).extendedSlotStatusHoldPeriod = 5 * time.Minute- // slotRebuildMaxAttemptsPerHour — beyond this many rebuild ATTEMPTS in a- // sliding hour (counted regardless of outcome), mechanical retry won't help- // (creds rotated, pg_hba changed, source dead); stop and surface the- // condition instead of dropping+recreating in a loop.+ // Beyond this many rebuild ATTEMPTS in a sliding hour (counted regardless of+ // outcome), mechanical retry won't help (creds rotated, pg_hba changed,+ // source dead); stop and surface the condition instead of dropping+recreating+ // in a loop.slotRebuildMaxAttemptsPerHour = 3- // rebuildReceiverStopTimeout — how long to wait for our own pg_receivewal to- // release the slot during a rebuild before concluding another consumer holds it.+ // How long to wait for our own pg_receivewal to release the slot during a+ // rebuild before concluding another consumer holds it.rebuildReceiverStopTimeout = 30 * time.SecondrebuildReceiverStopPoll = 1 * time.Second)-// walBreakReason is a structured-log value for an observed stream break. It is// NOT a catalog enum: WAL chain breaks are derived from LSN gaps between segment// rows, never stored — the log carries the human-readable "why".type walBreakReason string@@ -38,9 +37,8 @@ const (breakReasonSlotStolen walBreakReason = "SLOT_STOLEN")-// runLagMonitor polls the source slot and triggers a rebuild on slot loss /-// unreserved / sustained-extended / lag over threshold. Source-side slot state-// only; consumer-side liveness is the slot-LSN watcher's job (wal_stream.go).+// Source-side slot state only; consumer-side liveness is the slot-LSN watcher's+// job (slot_lsn_watcher.go).func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Logger) {ticker := time.NewTicker(lagMonitorPollInterval)defer ticker.Stop()@@ -67,7 +65,6 @@ func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Lo}}-// evaluateSlotForBreak inspects the slot and decides whether a rebuild is due.// extendedSince tracks how long wal_status has been 'extended' across ticks.func (s *WalStreamSupervisor) evaluateSlotForBreak(ctx context.Context,diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.goindex 0e2f5bf0..d7e3fff0 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go@@ -17,8 +17,7 @@ import (const receivewalApplicationNamePrefix = "databasus_wal_receiver_"-// receiverExit is the disposition of one pg_receivewal run, decided by-// spawnAndSupervise and acted on by runReceivewalSupervision.+// receiverExit is decided by spawnAndSupervise and acted on by runReceivewalSupervision.type receiverExit intconst (@@ -28,9 +27,14 @@ const (receiverFatal // non-retryable exit — escalate so the supervisor reclaims it on a later tick)-// spawnAndSupervise starts one pg_receivewal process and blocks until it exits,-// the restart signal fires, or ctx is cancelled. It reports the run's-// disposition, how long the process streamed, and (for receiverFatal) the error.+type receivewalCommandSpec struct {+ PgBin string+ SourceDB *postgresql_physical.PostgresqlPhysicalDatabase+ Creds *postgresql_shared.CredentialTempFiles+ WatchDir string+ SlotName string+}+func (s *WalStreamSupervisor) spawnAndSupervise(ctx context.Context,logger *slog.Logger,@@ -56,7 +60,13 @@ func (s *WalStreamSupervisor) spawnAndSupervise(procCtx, procCancel := context.WithCancel(ctx)defer procCancel()- cmd, err := newReceivewalCommand(procCtx, pgBin, s.spec.SourceDB, creds, s.watchDir, s.slotName)+ cmd, err := newReceivewalCommand(procCtx, receivewalCommandSpec{+ PgBin: pgBin,+ SourceDB: s.spec.SourceDB,+ Creds: creds,+ WatchDir: s.watchDir,+ SlotName: s.slotName,+ })if err != nil {logger.Error("build pg_receivewal command", "error", err)@@ -127,11 +137,9 @@ func (s *WalStreamSupervisor) spawnAndSupervise(}}-// isFatalReceivewalError reports whether a pg_receivewal stderr indicates a-// condition a local respawn can never clear — a full / unwritable local disk,-// rejected credentials, or the replication slot being held by another consumer.-// These escalate to streamer-FAILED so the supervisor can reclaim it on a later-// tick instead of this process crash-looping on an unfixable cause.+// A full / unwritable local disk, rejected credentials or a slot held by another+// consumer escalate to streamer-FAILED so the supervisor can reclaim it on a+// later tick, instead of this process crash-looping on an unfixable cause.func isFatalReceivewalError(stderr []byte) bool {// Lower-case both sides: OS errno strings vary in case ("Permission denied")// while PG messages are lower-case, and we want to match either.@@ -153,57 +161,49 @@ func isFatalReceivewalError(stderr []byte) bool {return false}-// newReceivewalCommand builds the pg_receivewal invocation. WAL is left-// uncompressed locally (no --compress) because the uploader re-compresses with-// zstd on upload; --no-loop makes the process exit on connection loss so the-// supervision loop owns retry; --synchronous flushes each segment promptly. SSL-// is supplied through the same PGSSL* env path pg_basebackup uses, so mTLS needs-// no extra handling here.-func newReceivewalCommand(- ctx context.Context,- pgBin string,- sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,- creds *postgresql_shared.CredentialTempFiles,- watchDir string,- slotName string,-) (*exec.Cmd, error) {- if _, err := exec.LookPath(pgBin); err != nil {- return nil, fmt.Errorf("pg_receivewal binary not found at %s: %w", pgBin, err)+// WAL is left uncompressed locally (no --compress) because the uploader+// re-compresses with zstd on upload; --no-loop makes the process exit on+// connection loss so the supervision loop owns retry; --synchronous flushes each+// segment promptly. SSL is supplied through the same PGSSL* env path+// pg_basebackup uses, so mTLS needs no extra handling here.+func newReceivewalCommand(ctx context.Context, spec receivewalCommandSpec) (*exec.Cmd, error) {+ if _, err := exec.LookPath(spec.PgBin); err != nil {+ return nil, fmt.Errorf("pg_receivewal binary not found at %s: %w", spec.PgBin, err)}args := []string{- "--directory=" + watchDir,- "--slot=" + slotName,+ "--directory=" + spec.WatchDir,+ "--slot=" + spec.SlotName,"--no-loop","--synchronous","--verbose","--no-password",- "-h", sourceDB.Host,- "-p", strconv.Itoa(sourceDB.Port),- "-U", sourceDB.Username,+ "-h", spec.SourceDB.Host,+ "-p", strconv.Itoa(spec.SourceDB.Port),+ "-U", spec.SourceDB.Username,}- cmd := exec.CommandContext(ctx, pgBin, args...)+ cmd := exec.CommandContext(ctx, spec.PgBin, args...)cmd.Env = append(os.Environ(),- "PGPASSFILE="+creds.PgpassPath,- "PGAPPNAME="+receivewalApplicationName(sourceDB),+ "PGPASSFILE="+spec.Creds.PgpassPath,+ "PGAPPNAME="+receivewalApplicationName(spec.SourceDB),"PGCLIENTENCODING=UTF8","PGCONNECT_TIMEOUT=30","LC_ALL=C.UTF-8","LANG=C.UTF-8",)- sslMode := sourceDB.SslMode+ sslMode := spec.SourceDB.SslModeif sslMode == "" {sslMode = postgresql_shared.PostgresSslModeDisable}cmd.Env = append(cmd.Env,"PGSSLMODE="+string(sslMode),- "PGSSLCERT="+creds.ClientCertPath,- "PGSSLKEY="+creds.ClientKeyPath,- "PGSSLROOTCERT="+creds.RootCertPath,+ "PGSSLCERT="+spec.Creds.ClientCertPath,+ "PGSSLKEY="+spec.Creds.ClientKeyPath,+ "PGSSLROOTCERT="+spec.Creds.RootCertPath,"PGSSLCRL=",)@@ -225,10 +225,10 @@ func receivewalApplicationName(sourceDB *postgresql_physical.PostgresqlPhysicalDreturn receivewalApplicationNamePrefix + sourceDB.DatabaseID.String()}-// setReceivewalProcessAttributes makes the kernel send pg_receivewal a SIGTERM-// if the Databasus process dies, so a crashed supervisor never leaks an orphaned-// receiver that keeps holding the replication slot. Pdeathsig is Linux-only;-// Databasus ships Linux containers exclusively.+// Pdeathsig makes the kernel SIGTERM pg_receivewal if the Databasus process+// dies, so a crashed supervisor never leaks an orphaned receiver that keeps+// holding the replication slot. Linux-only; Databasus ships Linux containers+// exclusively.func setReceivewalProcessAttributes(cmd *exec.Cmd) {cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGTERM}}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.goindex 8ca52637..73dcf401 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go@@ -22,14 +22,13 @@ func Test_NewReceivewalCommand_SetsParentDeathSignalAndApplicationName(t *testinReplicationSlotName: "slot",}- cmd, err := newReceivewalCommand(- t.Context(),- "sh",- sourceDB,- &postgresql_shared.CredentialTempFiles{PgpassPath: "/tmp/pgpass"},- t.TempDir(),- "slot",- )+ cmd, err := newReceivewalCommand(t.Context(), receivewalCommandSpec{+ PgBin: "sh",+ SourceDB: sourceDB,+ Creds: &postgresql_shared.CredentialTempFiles{PgpassPath: "/tmp/pgpass"},+ WatchDir: t.TempDir(),+ SlotName: "slot",+ })require.NoError(t, err)require.NotNil(t, cmd.SysProcAttr)require.Equal(t, syscall.SIGTERM, cmd.SysProcAttr.Pdeathsig)diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.goindex e970fa75..40c13c8b 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go@@ -9,28 +9,24 @@ import ()const (- // slotLsnWatcherPollInterval — consumer-side liveness poll. Tighter than the- // lag monitor because it tests whether OUR pg_receivewal is actually flushing.+ // Tighter than the lag monitor because it tests whether OUR pg_receivewal is+ // actually flushing.slotLsnWatcherPollInterval = 10 * time.Second- // slotLsnStallTimeout — restart_lsn unchanged this long on a healthy PG means- // pg_receivewal is stuck (alive but flushing nothing); restart it locally.+ // restart_lsn unchanged this long on a healthy PG means pg_receivewal is+ // stuck (alive but flushing nothing); restart it locally.slotLsnStallTimeout = 60 * time.Second)-// stallTracker tracks slot restart_lsn advance across watcher ticks. A changed-// LSN (or the very first sample) re-arms the advance clock; an LSN that stays-// frozen past the stall timeout is the signal that pg_receivewal is alive but-// flushing nothing on a reachable source.type stallTracker struct {lastRestartLSN walmath.LSNlastAdvanceAt time.TimehasSample bool}-// observe records a restart_lsn sample taken at now and reports whether the slot-// has stalled longer than stallTimeout. On a positive result it re-arms the clock-// so the caller restarts at most once per stallTimeout window.+// A changed LSN (or the very first sample) re-arms the advance clock. On a+// positive result the clock re-arms too, so the caller restarts at most once per+// stallTimeout window.func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeout time.Duration) bool {if !t.hasSample || restartLSN != t.lastRestartLSN {t.lastRestartLSN = restartLSN@@ -49,10 +45,9 @@ func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeoreturn false}-// runSlotLsnWatcher restarts pg_receivewal when the slot's restart_lsn has not-// advanced for slotLsnStallTimeout while the source PG is still reachable — a-// stuck consumer on a healthy server. A stall with an unreachable server is left-// to the lag monitor (slot loss / network down).+// A frozen restart_lsn on a reachable source is a stuck consumer on a healthy+// server. A stall with an unreachable server is left to the lag monitor (slot+// loss / network down).func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slog.Logger) {ticker := time.NewTicker(slotLsnWatcherPollInterval)defer ticker.Stop()@@ -67,7 +62,6 @@ func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slocase <-ticker.C:restartLSN, pgReachable := s.sampleSlotRestartLSN(ctx, logger)if !pgReachable {- // Connectivity / slot loss is the lag monitor's job, not ours.continue}@@ -81,8 +75,7 @@ func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slo}}-// sampleSlotRestartLSN reads the slot's restart_lsn and confirms the source is-// reachable with a keepalive. pgReachable=false means defer to the lag monitor.+// pgReachable=false means defer to the lag monitor.func (s *WalStreamSupervisor) sampleSlotRestartLSN(ctx context.Context, logger *slog.Logger) (walmath.LSN, bool) {conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)if err != nil {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.goindex fdbe3c57..fc22e93c 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go@@ -72,8 +72,6 @@ func (s *WalStreamSupervisor) rebuildSlot(ctx context.Context, logger *slog.Loggreturn nil}-// isOwnedReceiverBackend reports whether an active slot is held by one of our own-// pg_receivewal processes — its PGAPPNAME carries receivewalApplicationNamePrefix.// A slot held by anything else (a third-party consumer, or a backend we cannot// attribute) must never be force-terminated or dropped during a rebuild.func isOwnedReceiverBackend(state *SlotState) bool {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.goindex ab009377..4e331d15 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go@@ -6,9 +6,6 @@ import ("github.com/stretchr/testify/require")-// Test_IsOwnedReceiverBackend_WhenOurReceiverHoldsSlot_ReturnsTrue confirms the-// rebuild attribution accepts a slot held by our own pg_receivewal (active PID +-// our application-name prefix) — the only case where force-termination is safe.func Test_IsOwnedReceiverBackend_WhenOurReceiverHoldsSlot_ReturnsTrue(t *testing.T) {state := &SlotState{Active: true,@@ -19,9 +16,6 @@ func Test_IsOwnedReceiverBackend_WhenOurReceiverHoldsSlot_ReturnsTrue(t *testingrequire.True(t, isOwnedReceiverBackend(state))}-// Test_IsOwnedReceiverBackend_WhenForeignConsumerHoldsSlot_ReturnsFalse is the-// slot-stolen guard: a slot held by a consumer whose application name is not ours-// must not be attributed to us, so rebuildSlot refuses to terminate or drop it.func Test_IsOwnedReceiverBackend_WhenForeignConsumerHoldsSlot_ReturnsFalse(t *testing.T) {state := &SlotState{Active: true,@@ -32,8 +26,6 @@ func Test_IsOwnedReceiverBackend_WhenForeignConsumerHoldsSlot_ReturnsFalse(t *terequire.False(t, isOwnedReceiverBackend(state))}-// Test_IsOwnedReceiverBackend_WhenNoActiveBackend_ReturnsFalse: a slot with no-// active PID has no walsender to terminate — there is nothing to attribute.func Test_IsOwnedReceiverBackend_WhenNoActiveBackend_ReturnsFalse(t *testing.T) {state := &SlotState{Active: false,@@ -44,9 +36,6 @@ func Test_IsOwnedReceiverBackend_WhenNoActiveBackend_ReturnsFalse(t *testing.T)require.False(t, isOwnedReceiverBackend(state))}-// Test_IsOwnedReceiverBackend_WhenActiveButEmptyApplicationName_ReturnsFalse:-// an active backend we cannot name (no pg_stat_replication join, empty app name)-// is treated as foreign — we never drop a slot we cannot prove is ours.func Test_IsOwnedReceiverBackend_WhenActiveButEmptyApplicationName_ReturnsFalse(t *testing.T) {state := &SlotState{Active: true,diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.goindex 7470bf7f..9b4e43d3 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go@@ -763,10 +763,8 @@ func StartWalStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store sto}}-// MarkFullCompleted promotes the fixture's IN_PROGRESS FULL to COMPLETED with the-// given LSN bounds, so chain_view treats it as a real chain anchor whose span-// (GetChainSpan / FindWalGapsInChain) covers the streamed WAL segments. Streamer-// tests that assert on chain shape need a COMPLETED FULL to anchor against.+// chain_view only treats a COMPLETED FULL as a chain anchor, so streamer tests+// that assert on chain shape need one to anchor against.func MarkFullCompleted(t *testing.T, fullID uuid.UUID, timelineID int, startLSN, stopLSN walmath.LSN) {t.Helper()diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.goindex f46713a6..0c0ae05c 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go@@ -10,19 +10,16 @@ import ()const (- // walLocalMinHighWatermarkBytes — pg_receivewal has no- // kernel-pipe back pressure (it writes files), so the watch dir IS the buffer.- // Above HIGH we SIGTERM it; we resume only once uploads drain below LOW. The- // 5x hysteresis prevents flapping on the boundary. HIGH scales up for clusters- // with non-default wal_segment_size so one segment does not stop the receiver.+ // pg_receivewal has no kernel-pipe back pressure (it writes files), so the+ // watch dir IS the buffer. Above HIGH we SIGTERM it; we resume only once+ // uploads drain below LOW. The 5x hysteresis prevents flapping on the+ // boundary. HIGH scales up for clusters with non-default wal_segment_size so+ // one segment does not stop the receiver.walLocalMinHighWatermarkBytes int64 = 100 * 1024 * 1024backpressurePollInterval = 1 * time.Second)-// runBackpressureMonitor SIGTERMs pg_receivewal (via the restart signal) when the-// local watch-dir backlog crosses the high watermark; the supervision loop then-// waits for the uploader to drain below the low watermark before respawning.func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slog.Logger) {ticker := time.NewTicker(backpressurePollInterval)defer ticker.Stop()@@ -40,9 +37,6 @@ func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slo}}-// waitForBacklogBelowLow blocks while the backlog is at/over the high watermark,-// returning once it drains below the low watermark. Returns false if ctx is-// cancelled while waiting.func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger *slog.Logger) bool {if s.backlogBytes() < s.highWatermarkBytes {return true@@ -68,9 +62,8 @@ func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger}}-// backlogBytes sums the sizes of finalized (not-yet-uploaded) WAL segments in the-// watch dir. Uploaded segments are removed by the uploader, so this is the local-// queue depth; .partial and .history files are excluded.+// Uploaded segments are removed by the uploader, so summing what is left is the+// local queue depth; .partial and .history files are excluded.func (s *WalStreamSupervisor) backlogBytes() int64 {entries, err := os.ReadDir(s.watchDir)if err != nil {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.goindex f9a5541a..f7949d85 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go@@ -22,29 +22,27 @@ import ()const (- // receivewalRespawnBackoff — initial pause between a pg_receivewal exit and respawn so- // a hard-failing source (auth, pg_hba) is not hammered. --no-loop makes- // pg_receivewal exit on connection loss; this loop is its supervision.+ // Initial pause between a pg_receivewal exit and respawn so a hard-failing+ // source (auth, pg_hba) is not hammered. --no-loop makes pg_receivewal exit+ // on connection loss; this loop is its supervision.receivewalRespawnBackoff = 2 * time.SecondreceivewalRespawnMaxBackoff = 30 * time.Minute- // receivewalMinHealthyUptime — a receiver that streamed at least this long- // before exiting counts as a transient blip (network drop, slot resend) and- // resets the crash-loop counter; a shorter run counts toward escalation.+ // A receiver that streamed at least this long before exiting counts as a+ // transient blip (network drop, slot resend) and resets the crash-loop+ // counter; a shorter run counts toward escalation.receivewalMinHealthyUptime = 15 * time.Second- // receivewalMaxRapidFailures — this many back-to-back sub-uptime exits escalate- // to a fatal supervisor error so the streamer row is marked FAILED and the- // supervisor can reclaim it on a later tick, instead of crash-looping locally- // forever on a condition a local respawn can never fix (ENOSPC, bad creds, a- // slot held by a thief).+ // This many back-to-back sub-uptime exits escalate to a fatal supervisor+ // error so the streamer row is marked FAILED and the supervisor can reclaim+ // it on a later tick, instead of crash-looping locally forever on a condition+ // a local respawn can never fix (ENOSPC, bad creds, a slot held by a thief).receivewalMaxRapidFailures = 5pausePollInterval = 1 * time.Second)-// WalStreamSpec is the immutable configuration of one database's WAL streamer.type WalStreamSpec struct {DatabaseID uuid.UUIDSourceDB *postgresql_physical.PostgresqlPhysicalDatabase@@ -56,49 +54,45 @@ type WalStreamSpec struct {WalSegmentRepo *physical_repositories.PhysicalWalSegmentRepositoryHistoryRepo *physical_repositories.PhysicalWalHistoryRepository- // WatchDirRoot is config.DataFolder; the per-DB queue lives under+ // config.DataFolder; the per-DB queue lives under// <root>/wal-queue/<database_id>/. It must survive a process restart so crash// recovery can re-process finalized-but-not-uploaded segments.WatchDirRoot string- // WalLagThresholdBytes drives the lag monitor (lag_monitor.go): a slot lag over- // this many bytes triggers a slot rebuild.+ // A slot lag over this many bytes triggers a rebuild (lag_monitor.go).WalLagThresholdBytes int64- // OnGapDetected fires once per newly-observed WAL gap (see WalUploader); nil- // disables notification.+ // Fires once per newly-observed WAL gap (see WalUploader); nil disables+ // notification.OnGapDetected func(gapStart, gapEnd walmath.LSN)- // OnSlotRebuilt fires after the persistent slot has been recreated. Callers use- // it to request a fresh base backup that anchors the new WAL chain.+ // Fires after the persistent slot has been recreated. Callers use it to+ // request a fresh base backup that anchors the new WAL chain.OnSlotRebuilt func(ctx context.Context, reason string) errorLogger *slog.Logger}-// WalStreamSupervisor runs and supervises one pg_receivewal process per database:-// it spawns the receiver, archives every fully-rotated segment via the-// insert-first WalUploader, applies disk back pressure, restarts a stalled-// receiver, forwards .history files, and (lag_monitor.go) rebuilds the slot on-// lag/loss. Run blocks until ctx is cancelled.+// One supervisor owns one database's pg_receivewal process. Run blocks until ctx+// is cancelled.type WalStreamSupervisor struct {spec WalStreamSpecuploader *WalUploaderwatchDir stringslotName string- // Back-pressure watermarks derived once from the source's (immutable)- // wal_segment_size; recomputing them on every poll tick would be wasted work.+ // Derived once from the source's (immutable) wal_segment_size; recomputing+ // them on every poll tick would be wasted work.highWatermarkBytes int64lowWatermarkBytes int64- // restartSignal asks the supervision loop to SIGTERM the current- // pg_receivewal and respawn (sent by the back-pressure monitor and the- // slot-LSN watcher). Buffered size 1; sends are non-blocking and coalesced.+ // Asks the supervision loop to SIGTERM the current pg_receivewal and respawn+ // (sent by the back-pressure monitor and the slot-LSN watcher). Buffered size+ // 1; sends are non-blocking and coalesced.restartSignal chan struct{}- // isPaused holds the supervision loop between pg_receivewal runs so a slot- // rebuild can drop+recreate the slot without the receiver re-attaching.+ // Holds the supervision loop between pg_receivewal runs so a slot rebuild can+ // drop+recreate the slot without the receiver re-attaching.isPaused atomic.Bool// rebuildMu serializes slot rebuilds in this process; rebuildTimestamps powers@@ -124,9 +118,6 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {OnGapDetected: spec.OnGapDetected,})- // HIGH scales up for clusters with a non-default wal_segment_size so one- // segment can never single-handedly stop the receiver; LOW is HIGH/5 for the- // 5x hysteresis that prevents flapping on the boundary.highWatermarkBytes := max(walLocalMinHighWatermarkBytes, 4*walSegmentSizeBytes(spec.SourceDB))return &WalStreamSupervisor{@@ -140,10 +131,6 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {}}-// Run starts the uploader, the back-pressure monitor, the slot-LSN watcher, the-// lag monitor, and the pg_receivewal supervision loop, blocking until ctx is-// cancelled. The persistent slot is created if missing; torn *.partial files are-// cleared before the first spawn.func (s *WalStreamSupervisor) Run(ctx context.Context) error {logger := s.spec.Logger.With("database_id", s.spec.DatabaseID, "slot_name", s.slotName)@@ -199,10 +186,9 @@ func (s *WalStreamSupervisor) Run(ctx context.Context) error {return nil}-// runReceivewalSupervision is the pg_receivewal lifecycle loop: drain back-// pressure, clear partials, spawn, and react to the run's disposition. It returns-// a non-nil error only when the receiver is unrecoverable here (fatal exit or a-// crash loop), so Run can mark the streamer FAILED for reclaim on a later tick.+// Returns a non-nil error only when the receiver is unrecoverable here (fatal+// exit or a crash loop), so Run can mark the streamer FAILED for reclaim on a+// later tick.func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logger *slog.Logger) error {pgBin := tools.GetPostgresqlExecutable(s.spec.SourceDB.Version, tools.PostgresqlExecutablePgReceivewal)respawnBackoff := receivewalRespawnBackoff@@ -268,8 +254,6 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg}}-// waitWhilePaused blocks the supervision loop while a slot rebuild holds the-// receiver down. Returns false if ctx is cancelled while paused.func (s *WalStreamSupervisor) waitWhilePaused(ctx context.Context) bool {if !s.isPaused.Load() {return true@@ -305,8 +289,6 @@ func (s *WalStreamSupervisor) drainRestartSignal() {}}-// walSegmentSizeBytes returns the source cluster's captured wal_segment_size, or-// the 16 MB default when it has not been captured yet.func walSegmentSizeBytes(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) int64 {if sourceDB.WalSegmentSizeBytes != nil && *sourceDB.WalSegmentSizeBytes > 0 {return *sourceDB.WalSegmentSizeBytesdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.goindex 8687b8b9..7549d9ac 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go@@ -18,10 +18,8 @@ import ("databasus-backend/internal/util/walmath")-// startStreamerForTest runs a WalStreamSupervisor against the fixture's source PG-// in a goroutine and returns a stop func that cancels and waits for full drain —-// ensuring pg_receivewal has released the slot before the DB (and its slot) are-// torn down by later cleanups.+// The stop func waits for full drain, so pg_receivewal has released the slot+// before the DB (and its slot) are torn down by later cleanups.func startStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store storages.StorageFileSaver) func() {t.Helper()@@ -109,8 +107,6 @@ func Test_WalStream_FullIncrementalAndWalStream_StreamerArchivesSegments(t *testrequire.GreaterOrEqual(t, committed, 1, "at least one rotated segment must be archived")}-// assertDbSegmentsArchivedOnlyIn asserts every committed segment of databaseID-// is present in ownStore and absent from otherStore — per-DB streamer isolation.func assertDbSegmentsArchivedOnlyIn(t *testing.T,databaseID uuid.UUID,@@ -138,8 +134,6 @@ func assertDbSegmentsArchivedOnlyIn(require.GreaterOrEqual(t, committed, 1, "database %s must archive at least one segment", databaseID)}-// committedSegmentsInOrder returns the database's committed (file_name NOT NULL)-// WAL segments ordered by start_lsn (FindByChainSpan already sorts ascending).func committedSegmentsInOrder(t *testing.T, databaseID uuid.UUID) []*physical_models.PhysicalWalSegment {t.Helper()@@ -309,12 +303,12 @@ func Test_WalStream_CustomWalSegmentSize_LsnMathCorrect(t *testing.T) {name := "000000010000000200000003"require.NoError(t, uploader.ProcessSegment(context.Background(), writeWalFile(t, dir, name), name))- wantStart := walmath.LSN((uint64(2) << 32) + 3*uint64(customSegSize))+ expectedStartLSN := walmath.LSN((uint64(2) << 32) + 3*uint64(customSegSize))- row := findWalSegment(t, fixture.DB.ID, 1, wantStart)+ row := findWalSegment(t, fixture.DB.ID, 1, expectedStartLSN)require.NotNil(t, row, "segment LSN must be derived from the DB's segsize, not the walmath global")- require.Equal(t, wantStart, row.StartLSN)- require.Equal(t, wantStart+walmath.LSN(customSegSize), row.EndLSN)+ require.Equal(t, expectedStartLSN, row.StartLSN)+ require.Equal(t, expectedStartLSN+walmath.LSN(customSegSize), row.EndLSN)}func Test_Cleaner_AbandonedNullClaim_OlderThanGrace_DeletedYoungerSurvives(t *testing.T) {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.goindex 08fd6a89..3ce9a62d 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go@@ -30,10 +30,7 @@ const testWalSegmentSize = int64(16 * 1024 * 1024)// chain-span query should return every segment regardless of position.const lsnSpanUpperBoundForTests = walmath.LSN(1) << 62-// mockWalStorage is a controllable StorageFileSaver for WAL uploader tests: it-// records saved/deleted object names, can fail the first N SaveFile calls, and-// can block one SaveFile until released (to interleave the DeleteFull cascade-// race).+// blockOn exists to interleave the DeleteFull cascade race.type mockWalStorage struct {mu sync.Mutexsaved map[string][]bytediff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.goindex a746879a..c27f42ca 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go@@ -13,13 +13,10 @@ import ("databasus-backend/internal/util/walmath")-// uploaderPollInterval — the uploader scans watch_dir this often for newly-// rotated segments. Segments rotate at the source write rate; a tight loop-// keeps local dwell time low without measurable CPU.+// Segments rotate at the source write rate; a tight loop keeps local dwell time+// low without measurable CPU.const uploaderPollInterval = 1 * time.Second-// runUploaderLoop scans the watch dir on a tight interval and hands each-// finalized segment / .history file to the appropriate handler.func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.Logger) {ticker := time.NewTicker(uploaderPollInterval)defer ticker.Stop()@@ -35,10 +32,8 @@ func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.}}-// recoverLocalSegmentsOnStartup sweeps watch_dir once at startup, taking over any-// finalized segment / .history left by a crash. Uses the uploader's takeover path-// so a segment whose pre-crash claim row is still file_name NULL gets finished-// rather than left for the cleaner.+// Uses the uploader's takeover path so a segment whose pre-crash claim row is+// still file_name NULL gets finished rather than left for the cleaner.func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context, logger *slog.Logger) {entries, err := os.ReadDir(s.watchDir)if err != nil {@@ -59,7 +54,7 @@ func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context,}case strings.HasSuffix(name, ".history"):- s.handleHistoryFile(ctx, logger, name)+ s.archiveTimelineHistoryFile(ctx, logger, name)}}}@@ -86,15 +81,14 @@ func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Lo}case strings.HasSuffix(name, ".history"):- s.handleHistoryFile(ctx, logger, name)+ s.archiveTimelineHistoryFile(ctx, logger, name)}}}-// handleHistoryFile uploads a .history file the receiver dropped into watch_dir-// (reusing UploadHistoryFile, which reads the body from the source cluster and is-// idempotent on (database_id, timeline_id)), then removes the local copy.-func (s *WalStreamSupervisor) handleHistoryFile(ctx context.Context, logger *slog.Logger, name string) {+// UploadHistoryFile reads the body from the source cluster and is idempotent on+// (database_id, timeline_id), so re-processing the same file is free.+func (s *WalStreamSupervisor) archiveTimelineHistoryFile(ctx context.Context, logger *slog.Logger, name string) {timelineID, err := parseHistoryTimeline(name)if err != nil {logger.Warn("skip unparseable history file", "name", name, "error", err)@@ -143,7 +137,6 @@ func (s *WalStreamSupervisor) removePartials(logger *slog.Logger) {}}-// parseHistoryTimeline extracts the timeline id from a "%08X.history" filename.func parseHistoryTimeline(name string) (int, error) {trimmed := strings.TrimSuffix(name, ".history")if trimmed == name {