Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

REFACTOR (physical): Clean up wal streaming names and comments

Rostislav Dugin2e4e17f241de
diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
index 43cb034e..6b06eb67 100644
--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
@@ -28,11 +28,9 @@ import (
"databasus-backend/internal/util/walmath"
)
-// PhysicalWalStreamSupervisor is the background service that owns long-running
-// pg_receivewal streamers. Ownership and restart recovery run through the
-// physical_wal_streamers heartbeat table: every tick it discovers WAL_STREAM
-// databases, CAS-claims any that are unclaimed / FAILED / stale, runs one
-// WalStreamSupervisor goroutine per owned DB, and heartbeats the rows it owns.
+// Ownership and restart recovery run through the physical_wal_streamers
+// heartbeat table: every tick CAS-claims the databases that are unclaimed /
+// FAILED / stale, so exactly one process streams a given database at a time.
type PhysicalWalStreamSupervisor struct {
databaseService *databases.DatabaseService
backupConfigService *backups_config_physical.BackupConfigService
@@ -55,7 +53,6 @@ type PhysicalWalStreamSupervisor struct {
isReady atomic.Bool
}
-// runningStreamer is the handle to one locally-owned streamer goroutine.
type runningStreamer struct {
cancel context.CancelFunc
done chan struct{}
@@ -133,9 +130,6 @@ func (s *PhysicalWalStreamSupervisor) recoverStreamersOnStartup() error {
return nil
}
-// reconcile starts streamers for newly-claimable WAL_STREAM databases, stops
-// local streamers whose database is no longer a candidate (disabled or demoted),
-// and heartbeats the rows we own.
func (s *PhysicalWalStreamSupervisor) reconcile(ctx context.Context) {
configs, err := s.backupConfigService.GetBackupConfigsWithEnabledBackups()
if err != nil {
@@ -168,8 +162,6 @@ func (s *PhysicalWalStreamSupervisor) reconcile(ctx context.Context) {
s.heartbeatOwnedStreamers()
}
-// isWalStreamCandidate reports whether a config should have a running streamer:
-// WAL_STREAM backup type with storage configured.
func isWalStreamCandidate(backupConfig *backups_config_physical.PhysicalBackupConfig) bool {
if backupConfig.PostgresqlPhysical == nil ||
backupConfig.PostgresqlPhysical.BackupType != postgresql_physical.BackupTypeFullIncrementalAndWalStream {
@@ -179,8 +171,6 @@ func isWalStreamCandidate(backupConfig *backups_config_physical.PhysicalBackupCo
return backupConfig.StorageID != nil
}
-// ensureStreamerRunning starts a streamer for backupConfig when we don't already
-// run one locally and we can win the heartbeat-table claim.
func (s *PhysicalWalStreamSupervisor) ensureStreamerRunning(
ctx context.Context,
logger *slog.Logger,
@@ -317,8 +307,6 @@ func (s *PhysicalWalStreamSupervisor) resolveMasterKey(
return key, true
}
-// gapNotifier builds the post-upload gap-probe callback: it sends a BackupFailed
-// notification (when the config opts in) to each of the database's notifiers.
func (s *PhysicalWalStreamSupervisor) gapNotifier(
db *databases.Database,
backupConfig *backups_config_physical.PhysicalBackupConfig,
@@ -445,8 +433,7 @@ func (s *PhysicalWalStreamSupervisor) removeWatchDirIfRequested(logger *slog.Log
}
}
-// releaseClaim marks a just-won streamer row FAILED when we could not actually
-// start the streamer, so the slot is immediately reclaimable rather than looking
+// Marking FAILED makes the slot immediately reclaimable rather than looking
// alive until the heartbeat goes stale.
func (s *PhysicalWalStreamSupervisor) releaseClaim(logger *slog.Logger, databaseID uuid.UUID) {
if err := s.walStreamerRepo.MarkFailed(databaseID); err != nil {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
index ab87f592..db4280b0 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/lag_monitor.go
@@ -7,27 +7,26 @@ import (
)
const (
- // lagMonitorPollInterval — cadence for reading pg_replication_slots. Balances
- // detection latency against query load on the source (one cheap indexed row).
+ // Balances detection latency against query load on the source (one cheap
+ // indexed row).
lagMonitorPollInterval = 30 * time.Second
- // extendedSlotStatusHoldPeriod — an 'extended' slot status must persist this
- // long before we treat it as a real lag (vs a transient write burst).
+ // An 'extended' slot status must persist this long before we treat it as a
+ // real lag (vs a transient write burst).
extendedSlotStatusHoldPeriod = 5 * time.Minute
- // slotRebuildMaxAttemptsPerHour — beyond this many rebuild ATTEMPTS in a
- // sliding hour (counted regardless of outcome), mechanical retry won't help
- // (creds rotated, pg_hba changed, source dead); stop and surface the
- // condition instead of dropping+recreating in a loop.
+ // Beyond this many rebuild ATTEMPTS in a sliding hour (counted regardless of
+ // outcome), mechanical retry won't help (creds rotated, pg_hba changed,
+ // source dead); stop and surface the condition instead of dropping+recreating
+ // in a loop.
slotRebuildMaxAttemptsPerHour = 3
- // rebuildReceiverStopTimeout — how long to wait for our own pg_receivewal to
- // release the slot during a rebuild before concluding another consumer holds it.
+ // How long to wait for our own pg_receivewal to release the slot during a
+ // rebuild before concluding another consumer holds it.
rebuildReceiverStopTimeout = 30 * time.Second
rebuildReceiverStopPoll = 1 * time.Second
)
-// walBreakReason is a structured-log value for an observed stream break. It is
// NOT a catalog enum: WAL chain breaks are derived from LSN gaps between segment
// rows, never stored — the log carries the human-readable "why".
type walBreakReason string
@@ -38,9 +37,8 @@ const (
breakReasonSlotStolen walBreakReason = "SLOT_STOLEN"
)
-// runLagMonitor polls the source slot and triggers a rebuild on slot loss /
-// unreserved / sustained-extended / lag over threshold. Source-side slot state
-// only; consumer-side liveness is the slot-LSN watcher's job (wal_stream.go).
+// Source-side slot state only; consumer-side liveness is the slot-LSN watcher's
+// job (slot_lsn_watcher.go).
func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Logger) {
ticker := time.NewTicker(lagMonitorPollInterval)
defer ticker.Stop()
@@ -67,7 +65,6 @@ func (s *WalStreamSupervisor) runLagMonitor(ctx context.Context, logger *slog.Lo
}
}
-// evaluateSlotForBreak inspects the slot and decides whether a rebuild is due.
// extendedSince tracks how long wal_status has been 'extended' across ticks.
func (s *WalStreamSupervisor) evaluateSlotForBreak(
ctx context.Context,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
index 0e2f5bf0..d7e3fff0 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process.go
@@ -17,8 +17,7 @@ import (
const receivewalApplicationNamePrefix = "databasus_wal_receiver_"
-// receiverExit is the disposition of one pg_receivewal run, decided by
-// spawnAndSupervise and acted on by runReceivewalSupervision.
+// receiverExit is decided by spawnAndSupervise and acted on by runReceivewalSupervision.
type receiverExit int
const (
@@ -28,9 +27,14 @@ const (
receiverFatal // non-retryable exit — escalate so the supervisor reclaims it on a later tick
)
-// spawnAndSupervise starts one pg_receivewal process and blocks until it exits,
-// the restart signal fires, or ctx is cancelled. It reports the run's
-// disposition, how long the process streamed, and (for receiverFatal) the error.
+type receivewalCommandSpec struct {
+ PgBin string
+ SourceDB *postgresql_physical.PostgresqlPhysicalDatabase
+ Creds *postgresql_shared.CredentialTempFiles
+ WatchDir string
+ SlotName string
+}
+
func (s *WalStreamSupervisor) spawnAndSupervise(
ctx context.Context,
logger *slog.Logger,
@@ -56,7 +60,13 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
procCtx, procCancel := context.WithCancel(ctx)
defer procCancel()
- cmd, err := newReceivewalCommand(procCtx, pgBin, s.spec.SourceDB, creds, s.watchDir, s.slotName)
+ cmd, err := newReceivewalCommand(procCtx, receivewalCommandSpec{
+ PgBin: pgBin,
+ SourceDB: s.spec.SourceDB,
+ Creds: creds,
+ WatchDir: s.watchDir,
+ SlotName: s.slotName,
+ })
if err != nil {
logger.Error("build pg_receivewal command", "error", err)
@@ -127,11 +137,9 @@ func (s *WalStreamSupervisor) spawnAndSupervise(
}
}
-// isFatalReceivewalError reports whether a pg_receivewal stderr indicates a
-// condition a local respawn can never clear — a full / unwritable local disk,
-// rejected credentials, or the replication slot being held by another consumer.
-// These escalate to streamer-FAILED so the supervisor can reclaim it on a later
-// tick instead of this process crash-looping on an unfixable cause.
+// A full / unwritable local disk, rejected credentials or a slot held by another
+// consumer escalate to streamer-FAILED so the supervisor can reclaim it on a
+// later tick, instead of this process crash-looping on an unfixable cause.
func isFatalReceivewalError(stderr []byte) bool {
// Lower-case both sides: OS errno strings vary in case ("Permission denied")
// while PG messages are lower-case, and we want to match either.
@@ -153,57 +161,49 @@ func isFatalReceivewalError(stderr []byte) bool {
return false
}
-// newReceivewalCommand builds the pg_receivewal invocation. WAL is left
-// uncompressed locally (no --compress) because the uploader re-compresses with
-// zstd on upload; --no-loop makes the process exit on connection loss so the
-// supervision loop owns retry; --synchronous flushes each segment promptly. SSL
-// is supplied through the same PGSSL* env path pg_basebackup uses, so mTLS needs
-// no extra handling here.
-func newReceivewalCommand(
- ctx context.Context,
- pgBin string,
- sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,
- creds *postgresql_shared.CredentialTempFiles,
- watchDir string,
- slotName string,
-) (*exec.Cmd, error) {
- if _, err := exec.LookPath(pgBin); err != nil {
- return nil, fmt.Errorf("pg_receivewal binary not found at %s: %w", pgBin, err)
+// WAL is left uncompressed locally (no --compress) because the uploader
+// re-compresses with zstd on upload; --no-loop makes the process exit on
+// connection loss so the supervision loop owns retry; --synchronous flushes each
+// segment promptly. SSL is supplied through the same PGSSL* env path
+// pg_basebackup uses, so mTLS needs no extra handling here.
+func newReceivewalCommand(ctx context.Context, spec receivewalCommandSpec) (*exec.Cmd, error) {
+ if _, err := exec.LookPath(spec.PgBin); err != nil {
+ return nil, fmt.Errorf("pg_receivewal binary not found at %s: %w", spec.PgBin, err)
}
args := []string{
- "--directory=" + watchDir,
- "--slot=" + slotName,
+ "--directory=" + spec.WatchDir,
+ "--slot=" + spec.SlotName,
"--no-loop",
"--synchronous",
"--verbose",
"--no-password",
- "-h", sourceDB.Host,
- "-p", strconv.Itoa(sourceDB.Port),
- "-U", sourceDB.Username,
+ "-h", spec.SourceDB.Host,
+ "-p", strconv.Itoa(spec.SourceDB.Port),
+ "-U", spec.SourceDB.Username,
}
- cmd := exec.CommandContext(ctx, pgBin, args...)
+ cmd := exec.CommandContext(ctx, spec.PgBin, args...)
cmd.Env = append(os.Environ(),
- "PGPASSFILE="+creds.PgpassPath,
- "PGAPPNAME="+receivewalApplicationName(sourceDB),
+ "PGPASSFILE="+spec.Creds.PgpassPath,
+ "PGAPPNAME="+receivewalApplicationName(spec.SourceDB),
"PGCLIENTENCODING=UTF8",
"PGCONNECT_TIMEOUT=30",
"LC_ALL=C.UTF-8",
"LANG=C.UTF-8",
)
- sslMode := sourceDB.SslMode
+ sslMode := spec.SourceDB.SslMode
if sslMode == "" {
sslMode = postgresql_shared.PostgresSslModeDisable
}
cmd.Env = append(cmd.Env,
"PGSSLMODE="+string(sslMode),
- "PGSSLCERT="+creds.ClientCertPath,
- "PGSSLKEY="+creds.ClientKeyPath,
- "PGSSLROOTCERT="+creds.RootCertPath,
+ "PGSSLCERT="+spec.Creds.ClientCertPath,
+ "PGSSLKEY="+spec.Creds.ClientKeyPath,
+ "PGSSLROOTCERT="+spec.Creds.RootCertPath,
"PGSSLCRL=",
)
@@ -225,10 +225,10 @@ func receivewalApplicationName(sourceDB *postgresql_physical.PostgresqlPhysicalD
return receivewalApplicationNamePrefix + sourceDB.DatabaseID.String()
}
-// setReceivewalProcessAttributes makes the kernel send pg_receivewal a SIGTERM
-// if the Databasus process dies, so a crashed supervisor never leaks an orphaned
-// receiver that keeps holding the replication slot. Pdeathsig is Linux-only;
-// Databasus ships Linux containers exclusively.
+// Pdeathsig makes the kernel SIGTERM pg_receivewal if the Databasus process
+// dies, so a crashed supervisor never leaks an orphaned receiver that keeps
+// holding the replication slot. Linux-only; Databasus ships Linux containers
+// exclusively.
func setReceivewalProcessAttributes(cmd *exec.Cmd) {
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGTERM}
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
index 8ca52637..73dcf401 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/receivewal_process_test.go
@@ -22,14 +22,13 @@ func Test_NewReceivewalCommand_SetsParentDeathSignalAndApplicationName(t *testin
ReplicationSlotName: "slot",
}
- cmd, err := newReceivewalCommand(
- t.Context(),
- "sh",
- sourceDB,
- &postgresql_shared.CredentialTempFiles{PgpassPath: "/tmp/pgpass"},
- t.TempDir(),
- "slot",
- )
+ cmd, err := newReceivewalCommand(t.Context(), receivewalCommandSpec{
+ PgBin: "sh",
+ SourceDB: sourceDB,
+ Creds: &postgresql_shared.CredentialTempFiles{PgpassPath: "/tmp/pgpass"},
+ WatchDir: t.TempDir(),
+ SlotName: "slot",
+ })
require.NoError(t, err)
require.NotNil(t, cmd.SysProcAttr)
require.Equal(t, syscall.SIGTERM, cmd.SysProcAttr.Pdeathsig)
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
index e970fa75..40c13c8b 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_lsn_watcher.go
@@ -9,28 +9,24 @@ import (
)
const (
- // slotLsnWatcherPollInterval — consumer-side liveness poll. Tighter than the
- // lag monitor because it tests whether OUR pg_receivewal is actually flushing.
+ // Tighter than the lag monitor because it tests whether OUR pg_receivewal is
+ // actually flushing.
slotLsnWatcherPollInterval = 10 * time.Second
- // slotLsnStallTimeout — restart_lsn unchanged this long on a healthy PG means
- // pg_receivewal is stuck (alive but flushing nothing); restart it locally.
+ // restart_lsn unchanged this long on a healthy PG means pg_receivewal is
+ // stuck (alive but flushing nothing); restart it locally.
slotLsnStallTimeout = 60 * time.Second
)
-// stallTracker tracks slot restart_lsn advance across watcher ticks. A changed
-// LSN (or the very first sample) re-arms the advance clock; an LSN that stays
-// frozen past the stall timeout is the signal that pg_receivewal is alive but
-// flushing nothing on a reachable source.
type stallTracker struct {
lastRestartLSN walmath.LSN
lastAdvanceAt time.Time
hasSample bool
}
-// observe records a restart_lsn sample taken at now and reports whether the slot
-// has stalled longer than stallTimeout. On a positive result it re-arms the clock
-// so the caller restarts at most once per stallTimeout window.
+// A changed LSN (or the very first sample) re-arms the advance clock. On a
+// positive result the clock re-arms too, so the caller restarts at most once per
+// stallTimeout window.
func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeout time.Duration) bool {
if !t.hasSample || restartLSN != t.lastRestartLSN {
t.lastRestartLSN = restartLSN
@@ -49,10 +45,9 @@ func (t *stallTracker) observe(restartLSN walmath.LSN, now time.Time, stallTimeo
return false
}
-// runSlotLsnWatcher restarts pg_receivewal when the slot's restart_lsn has not
-// advanced for slotLsnStallTimeout while the source PG is still reachable — a
-// stuck consumer on a healthy server. A stall with an unreachable server is left
-// to the lag monitor (slot loss / network down).
+// A frozen restart_lsn on a reachable source is a stuck consumer on a healthy
+// server. A stall with an unreachable server is left to the lag monitor (slot
+// loss / network down).
func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slog.Logger) {
ticker := time.NewTicker(slotLsnWatcherPollInterval)
defer ticker.Stop()
@@ -67,7 +62,6 @@ func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slo
case <-ticker.C:
restartLSN, pgReachable := s.sampleSlotRestartLSN(ctx, logger)
if !pgReachable {
- // Connectivity / slot loss is the lag monitor's job, not ours.
continue
}
@@ -81,8 +75,7 @@ func (s *WalStreamSupervisor) runSlotLsnWatcher(ctx context.Context, logger *slo
}
}
-// sampleSlotRestartLSN reads the slot's restart_lsn and confirms the source is
-// reachable with a keepalive. pgReachable=false means defer to the lag monitor.
+// pgReachable=false means defer to the lag monitor.
func (s *WalStreamSupervisor) sampleSlotRestartLSN(ctx context.Context, logger *slog.Logger) (walmath.LSN, bool) {
conn, err := s.spec.SourceDB.OpenInspectionConn(ctx, s.spec.FieldEncryptor)
if err != nil {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go
index fdbe3c57..fc22e93c 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild.go
@@ -72,8 +72,6 @@ func (s *WalStreamSupervisor) rebuildSlot(ctx context.Context, logger *slog.Logg
return nil
}
-// isOwnedReceiverBackend reports whether an active slot is held by one of our own
-// pg_receivewal processes — its PGAPPNAME carries receivewalApplicationNamePrefix.
// A slot held by anything else (a third-party consumer, or a backend we cannot
// attribute) must never be force-terminated or dropped during a rebuild.
func isOwnedReceiverBackend(state *SlotState) bool {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go
index ab009377..4e331d15 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/slot_rebuild_test.go
@@ -6,9 +6,6 @@ import (
"github.com/stretchr/testify/require"
)
-// Test_IsOwnedReceiverBackend_WhenOurReceiverHoldsSlot_ReturnsTrue confirms the
-// rebuild attribution accepts a slot held by our own pg_receivewal (active PID +
-// our application-name prefix) — the only case where force-termination is safe.
func Test_IsOwnedReceiverBackend_WhenOurReceiverHoldsSlot_ReturnsTrue(t *testing.T) {
state := &SlotState{
Active: true,
@@ -19,9 +16,6 @@ func Test_IsOwnedReceiverBackend_WhenOurReceiverHoldsSlot_ReturnsTrue(t *testing
require.True(t, isOwnedReceiverBackend(state))
}
-// Test_IsOwnedReceiverBackend_WhenForeignConsumerHoldsSlot_ReturnsFalse is the
-// slot-stolen guard: a slot held by a consumer whose application name is not ours
-// must not be attributed to us, so rebuildSlot refuses to terminate or drop it.
func Test_IsOwnedReceiverBackend_WhenForeignConsumerHoldsSlot_ReturnsFalse(t *testing.T) {
state := &SlotState{
Active: true,
@@ -32,8 +26,6 @@ func Test_IsOwnedReceiverBackend_WhenForeignConsumerHoldsSlot_ReturnsFalse(t *te
require.False(t, isOwnedReceiverBackend(state))
}
-// Test_IsOwnedReceiverBackend_WhenNoActiveBackend_ReturnsFalse: a slot with no
-// active PID has no walsender to terminate — there is nothing to attribute.
func Test_IsOwnedReceiverBackend_WhenNoActiveBackend_ReturnsFalse(t *testing.T) {
state := &SlotState{
Active: false,
@@ -44,9 +36,6 @@ func Test_IsOwnedReceiverBackend_WhenNoActiveBackend_ReturnsFalse(t *testing.T)
require.False(t, isOwnedReceiverBackend(state))
}
-// Test_IsOwnedReceiverBackend_WhenActiveButEmptyApplicationName_ReturnsFalse:
-// an active backend we cannot name (no pg_stat_replication join, empty app name)
-// is treated as foreign — we never drop a slot we cannot prove is ours.
func Test_IsOwnedReceiverBackend_WhenActiveButEmptyApplicationName_ReturnsFalse(t *testing.T) {
state := &SlotState{
Active: true,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
index 7470bf7f..9b4e43d3 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
@@ -763,10 +763,8 @@ func StartWalStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store sto
}
}
-// MarkFullCompleted promotes the fixture's IN_PROGRESS FULL to COMPLETED with the
-// given LSN bounds, so chain_view treats it as a real chain anchor whose span
-// (GetChainSpan / FindWalGapsInChain) covers the streamed WAL segments. Streamer
-// tests that assert on chain shape need a COMPLETED FULL to anchor against.
+// chain_view only treats a COMPLETED FULL as a chain anchor, so streamer tests
+// that assert on chain shape need one to anchor against.
func MarkFullCompleted(t *testing.T, fullID uuid.UUID, timelineID int, startLSN, stopLSN walmath.LSN) {
t.Helper()
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
index f46713a6..0c0ae05c 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_backpressure.go
@@ -10,19 +10,16 @@ import (
)
const (
- // walLocalMinHighWatermarkBytes — pg_receivewal has no
- // kernel-pipe back pressure (it writes files), so the watch dir IS the buffer.
- // Above HIGH we SIGTERM it; we resume only once uploads drain below LOW. The
- // 5x hysteresis prevents flapping on the boundary. HIGH scales up for clusters
- // with non-default wal_segment_size so one segment does not stop the receiver.
+ // pg_receivewal has no kernel-pipe back pressure (it writes files), so the
+ // watch dir IS the buffer. Above HIGH we SIGTERM it; we resume only once
+ // uploads drain below LOW. The 5x hysteresis prevents flapping on the
+ // boundary. HIGH scales up for clusters with non-default wal_segment_size so
+ // one segment does not stop the receiver.
walLocalMinHighWatermarkBytes int64 = 100 * 1024 * 1024
backpressurePollInterval = 1 * time.Second
)
-// runBackpressureMonitor SIGTERMs pg_receivewal (via the restart signal) when the
-// local watch-dir backlog crosses the high watermark; the supervision loop then
-// waits for the uploader to drain below the low watermark before respawning.
func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slog.Logger) {
ticker := time.NewTicker(backpressurePollInterval)
defer ticker.Stop()
@@ -40,9 +37,6 @@ func (s *WalStreamSupervisor) runBackpressureMonitor(ctx context.Context, _ *slo
}
}
-// waitForBacklogBelowLow blocks while the backlog is at/over the high watermark,
-// returning once it drains below the low watermark. Returns false if ctx is
-// cancelled while waiting.
func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger *slog.Logger) bool {
if s.backlogBytes() < s.highWatermarkBytes {
return true
@@ -68,9 +62,8 @@ func (s *WalStreamSupervisor) waitForBacklogBelowLow(ctx context.Context, logger
}
}
-// backlogBytes sums the sizes of finalized (not-yet-uploaded) WAL segments in the
-// watch dir. Uploaded segments are removed by the uploader, so this is the local
-// queue depth; .partial and .history files are excluded.
+// Uploaded segments are removed by the uploader, so summing what is left is the
+// local queue depth; .partial and .history files are excluded.
func (s *WalStreamSupervisor) backlogBytes() int64 {
entries, err := os.ReadDir(s.watchDir)
if err != nil {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
index f9a5541a..f7949d85 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
@@ -22,29 +22,27 @@ import (
)
const (
- // receivewalRespawnBackoff — initial pause between a pg_receivewal exit and respawn so
- // a hard-failing source (auth, pg_hba) is not hammered. --no-loop makes
- // pg_receivewal exit on connection loss; this loop is its supervision.
+ // Initial pause between a pg_receivewal exit and respawn so a hard-failing
+ // source (auth, pg_hba) is not hammered. --no-loop makes pg_receivewal exit
+ // on connection loss; this loop is its supervision.
receivewalRespawnBackoff = 2 * time.Second
receivewalRespawnMaxBackoff = 30 * time.Minute
- // receivewalMinHealthyUptime — a receiver that streamed at least this long
- // before exiting counts as a transient blip (network drop, slot resend) and
- // resets the crash-loop counter; a shorter run counts toward escalation.
+ // A receiver that streamed at least this long before exiting counts as a
+ // transient blip (network drop, slot resend) and resets the crash-loop
+ // counter; a shorter run counts toward escalation.
receivewalMinHealthyUptime = 15 * time.Second
- // receivewalMaxRapidFailures — this many back-to-back sub-uptime exits escalate
- // to a fatal supervisor error so the streamer row is marked FAILED and the
- // supervisor can reclaim it on a later tick, instead of crash-looping locally
- // forever on a condition a local respawn can never fix (ENOSPC, bad creds, a
- // slot held by a thief).
+ // This many back-to-back sub-uptime exits escalate to a fatal supervisor
+ // error so the streamer row is marked FAILED and the supervisor can reclaim
+ // it on a later tick, instead of crash-looping locally forever on a condition
+ // a local respawn can never fix (ENOSPC, bad creds, a slot held by a thief).
receivewalMaxRapidFailures = 5
pausePollInterval = 1 * time.Second
)
-// WalStreamSpec is the immutable configuration of one database's WAL streamer.
type WalStreamSpec struct {
DatabaseID uuid.UUID
SourceDB *postgresql_physical.PostgresqlPhysicalDatabase
@@ -56,49 +54,45 @@ type WalStreamSpec struct {
WalSegmentRepo *physical_repositories.PhysicalWalSegmentRepository
HistoryRepo *physical_repositories.PhysicalWalHistoryRepository
- // WatchDirRoot is config.DataFolder; the per-DB queue lives under
+ // config.DataFolder; the per-DB queue lives under
// <root>/wal-queue/<database_id>/. It must survive a process restart so crash
// recovery can re-process finalized-but-not-uploaded segments.
WatchDirRoot string
- // WalLagThresholdBytes drives the lag monitor (lag_monitor.go): a slot lag over
- // this many bytes triggers a slot rebuild.
+ // A slot lag over this many bytes triggers a rebuild (lag_monitor.go).
WalLagThresholdBytes int64
- // OnGapDetected fires once per newly-observed WAL gap (see WalUploader); nil
- // disables notification.
+ // Fires once per newly-observed WAL gap (see WalUploader); nil disables
+ // notification.
OnGapDetected func(gapStart, gapEnd walmath.LSN)
- // OnSlotRebuilt fires after the persistent slot has been recreated. Callers use
- // it to request a fresh base backup that anchors the new WAL chain.
+ // Fires after the persistent slot has been recreated. Callers use it to
+ // request a fresh base backup that anchors the new WAL chain.
OnSlotRebuilt func(ctx context.Context, reason string) error
Logger *slog.Logger
}
-// WalStreamSupervisor runs and supervises one pg_receivewal process per database:
-// it spawns the receiver, archives every fully-rotated segment via the
-// insert-first WalUploader, applies disk back pressure, restarts a stalled
-// receiver, forwards .history files, and (lag_monitor.go) rebuilds the slot on
-// lag/loss. Run blocks until ctx is cancelled.
+// One supervisor owns one database's pg_receivewal process. Run blocks until ctx
+// is cancelled.
type WalStreamSupervisor struct {
spec WalStreamSpec
uploader *WalUploader
watchDir string
slotName string
- // Back-pressure watermarks derived once from the source's (immutable)
- // wal_segment_size; recomputing them on every poll tick would be wasted work.
+ // Derived once from the source's (immutable) wal_segment_size; recomputing
+ // them on every poll tick would be wasted work.
highWatermarkBytes int64
lowWatermarkBytes int64
- // restartSignal asks the supervision loop to SIGTERM the current
- // pg_receivewal and respawn (sent by the back-pressure monitor and the
- // slot-LSN watcher). Buffered size 1; sends are non-blocking and coalesced.
+ // Asks the supervision loop to SIGTERM the current pg_receivewal and respawn
+ // (sent by the back-pressure monitor and the slot-LSN watcher). Buffered size
+ // 1; sends are non-blocking and coalesced.
restartSignal chan struct{}
- // isPaused holds the supervision loop between pg_receivewal runs so a slot
- // rebuild can drop+recreate the slot without the receiver re-attaching.
+ // Holds the supervision loop between pg_receivewal runs so a slot rebuild can
+ // drop+recreate the slot without the receiver re-attaching.
isPaused atomic.Bool
// rebuildMu serializes slot rebuilds in this process; rebuildTimestamps powers
@@ -124,9 +118,6 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {
OnGapDetected: spec.OnGapDetected,
})
- // HIGH scales up for clusters with a non-default wal_segment_size so one
- // segment can never single-handedly stop the receiver; LOW is HIGH/5 for the
- // 5x hysteresis that prevents flapping on the boundary.
highWatermarkBytes := max(walLocalMinHighWatermarkBytes, 4*walSegmentSizeBytes(spec.SourceDB))
return &WalStreamSupervisor{
@@ -140,10 +131,6 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {
}
}
-// Run starts the uploader, the back-pressure monitor, the slot-LSN watcher, the
-// lag monitor, and the pg_receivewal supervision loop, blocking until ctx is
-// cancelled. The persistent slot is created if missing; torn *.partial files are
-// cleared before the first spawn.
func (s *WalStreamSupervisor) Run(ctx context.Context) error {
logger := s.spec.Logger.With("database_id", s.spec.DatabaseID, "slot_name", s.slotName)
@@ -199,10 +186,9 @@ func (s *WalStreamSupervisor) Run(ctx context.Context) error {
return nil
}
-// runReceivewalSupervision is the pg_receivewal lifecycle loop: drain back
-// pressure, clear partials, spawn, and react to the run's disposition. It returns
-// a non-nil error only when the receiver is unrecoverable here (fatal exit or a
-// crash loop), so Run can mark the streamer FAILED for reclaim on a later tick.
+// Returns a non-nil error only when the receiver is unrecoverable here (fatal
+// exit or a crash loop), so Run can mark the streamer FAILED for reclaim on a
+// later tick.
func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logger *slog.Logger) error {
pgBin := tools.GetPostgresqlExecutable(s.spec.SourceDB.Version, tools.PostgresqlExecutablePgReceivewal)
respawnBackoff := receivewalRespawnBackoff
@@ -268,8 +254,6 @@ func (s *WalStreamSupervisor) runReceivewalSupervision(ctx context.Context, logg
}
}
-// waitWhilePaused blocks the supervision loop while a slot rebuild holds the
-// receiver down. Returns false if ctx is cancelled while paused.
func (s *WalStreamSupervisor) waitWhilePaused(ctx context.Context) bool {
if !s.isPaused.Load() {
return true
@@ -305,8 +289,6 @@ func (s *WalStreamSupervisor) drainRestartSignal() {
}
}
-// walSegmentSizeBytes returns the source cluster's captured wal_segment_size, or
-// the 16 MB default when it has not been captured yet.
func walSegmentSizeBytes(sourceDB *postgresql_physical.PostgresqlPhysicalDatabase) int64 {
if sourceDB.WalSegmentSizeBytes != nil && *sourceDB.WalSegmentSizeBytes > 0 {
return *sourceDB.WalSegmentSizeBytes
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
index 8687b8b9..7549d9ac 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
@@ -18,10 +18,8 @@ import (
"databasus-backend/internal/util/walmath"
)
-// startStreamerForTest runs a WalStreamSupervisor against the fixture's source PG
-// in a goroutine and returns a stop func that cancels and waits for full drain —
-// ensuring pg_receivewal has released the slot before the DB (and its slot) are
-// torn down by later cleanups.
+// The stop func waits for full drain, so pg_receivewal has released the slot
+// before the DB (and its slot) are torn down by later cleanups.
func startStreamerForTest(t *testing.T, fixture *PhysicalDBFixture, store storages.StorageFileSaver) func() {
t.Helper()
@@ -109,8 +107,6 @@ func Test_WalStream_FullIncrementalAndWalStream_StreamerArchivesSegments(t *test
require.GreaterOrEqual(t, committed, 1, "at least one rotated segment must be archived")
}
-// assertDbSegmentsArchivedOnlyIn asserts every committed segment of databaseID
-// is present in ownStore and absent from otherStore — per-DB streamer isolation.
func assertDbSegmentsArchivedOnlyIn(
t *testing.T,
databaseID uuid.UUID,
@@ -138,8 +134,6 @@ func assertDbSegmentsArchivedOnlyIn(
require.GreaterOrEqual(t, committed, 1, "database %s must archive at least one segment", databaseID)
}
-// committedSegmentsInOrder returns the database's committed (file_name NOT NULL)
-// WAL segments ordered by start_lsn (FindByChainSpan already sorts ascending).
func committedSegmentsInOrder(t *testing.T, databaseID uuid.UUID) []*physical_models.PhysicalWalSegment {
t.Helper()
@@ -309,12 +303,12 @@ func Test_WalStream_CustomWalSegmentSize_LsnMathCorrect(t *testing.T) {
name := "000000010000000200000003"
require.NoError(t, uploader.ProcessSegment(context.Background(), writeWalFile(t, dir, name), name))
- wantStart := walmath.LSN((uint64(2) << 32) + 3*uint64(customSegSize))
+ expectedStartLSN := walmath.LSN((uint64(2) << 32) + 3*uint64(customSegSize))
- row := findWalSegment(t, fixture.DB.ID, 1, wantStart)
+ row := findWalSegment(t, fixture.DB.ID, 1, expectedStartLSN)
require.NotNil(t, row, "segment LSN must be derived from the DB's segsize, not the walmath global")
- require.Equal(t, wantStart, row.StartLSN)
- require.Equal(t, wantStart+walmath.LSN(customSegSize), row.EndLSN)
+ require.Equal(t, expectedStartLSN, row.StartLSN)
+ require.Equal(t, expectedStartLSN+walmath.LSN(customSegSize), row.EndLSN)
}
func Test_Cleaner_AbandonedNullClaim_OlderThanGrace_DeletedYoungerSurvives(t *testing.T) {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
index 08fd6a89..3ce9a62d 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
@@ -30,10 +30,7 @@ const testWalSegmentSize = int64(16 * 1024 * 1024)
// chain-span query should return every segment regardless of position.
const lsnSpanUpperBoundForTests = walmath.LSN(1) << 62
-// mockWalStorage is a controllable StorageFileSaver for WAL uploader tests: it
-// records saved/deleted object names, can fail the first N SaveFile calls, and
-// can block one SaveFile until released (to interleave the DeleteFull cascade
-// race).
+// blockOn exists to interleave the DeleteFull cascade race.
type mockWalStorage struct {
mu sync.Mutex
saved map[string][]byte
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go
index a746879a..c27f42ca 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir.go
@@ -13,13 +13,10 @@ import (
"databasus-backend/internal/util/walmath"
)
-// uploaderPollInterval — the uploader scans watch_dir this often for newly
-// rotated segments. Segments rotate at the source write rate; a tight loop
-// keeps local dwell time low without measurable CPU.
+// Segments rotate at the source write rate; a tight loop keeps local dwell time
+// low without measurable CPU.
const uploaderPollInterval = 1 * time.Second
-// runUploaderLoop scans the watch dir on a tight interval and hands each
-// finalized segment / .history file to the appropriate handler.
func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.Logger) {
ticker := time.NewTicker(uploaderPollInterval)
defer ticker.Stop()
@@ -35,10 +32,8 @@ func (s *WalStreamSupervisor) runUploaderLoop(ctx context.Context, logger *slog.
}
}
-// recoverLocalSegmentsOnStartup sweeps watch_dir once at startup, taking over any
-// finalized segment / .history left by a crash. Uses the uploader's takeover path
-// so a segment whose pre-crash claim row is still file_name NULL gets finished
-// rather than left for the cleaner.
+// Uses the uploader's takeover path so a segment whose pre-crash claim row is
+// still file_name NULL gets finished rather than left for the cleaner.
func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context, logger *slog.Logger) {
entries, err := os.ReadDir(s.watchDir)
if err != nil {
@@ -59,7 +54,7 @@ func (s *WalStreamSupervisor) recoverLocalSegmentsOnStartup(ctx context.Context,
}
case strings.HasSuffix(name, ".history"):
- s.handleHistoryFile(ctx, logger, name)
+ s.archiveTimelineHistoryFile(ctx, logger, name)
}
}
}
@@ -86,15 +81,14 @@ func (s *WalStreamSupervisor) scanAndUpload(ctx context.Context, logger *slog.Lo
}
case strings.HasSuffix(name, ".history"):
- s.handleHistoryFile(ctx, logger, name)
+ s.archiveTimelineHistoryFile(ctx, logger, name)
}
}
}
-// handleHistoryFile uploads a .history file the receiver dropped into watch_dir
-// (reusing UploadHistoryFile, which reads the body from the source cluster and is
-// idempotent on (database_id, timeline_id)), then removes the local copy.
-func (s *WalStreamSupervisor) handleHistoryFile(ctx context.Context, logger *slog.Logger, name string) {
+// UploadHistoryFile reads the body from the source cluster and is idempotent on
+// (database_id, timeline_id), so re-processing the same file is free.
+func (s *WalStreamSupervisor) archiveTimelineHistoryFile(ctx context.Context, logger *slog.Logger, name string) {
timelineID, err := parseHistoryTimeline(name)
if err != nil {
logger.Warn("skip unparseable history file", "name", name, "error", err)
@@ -143,7 +137,6 @@ func (s *WalStreamSupervisor) removePartials(logger *slog.Logger) {
}
}
-// parseHistoryTimeline extracts the timeline id from a "%08X.history" filename.
func parseHistoryTimeline(name string) (int, error) {
trimmed := strings.TrimSuffix(name, ".history")
if trimmed == name {