Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (physical): Don't break incremental chain when WAL summarizer is merely behind

Rostislav Duginf8ffaf7346ba
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc_test.go
index 0ad65be4..e26691b7 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc_test.go
@@ -88,3 +88,21 @@ func Test_CreateIncremental_WhenSummarizerHealthy_GoesIncremental(t *testing.T)
require.NotNil(t, incrRow.FileName, "a healthy incremental must upload an artifact")
require.NotNil(t, incrRow.StopLSN, "a completed incremental must record stop_lsn")
}
+
+func Test_CreateIncremental_WhenSummarizerBehindOnIdleDB_StillCompletes(t *testing.T) {
+ fixture := postgresql_executor.SetupPhysicalDBForBackup(t)
+
+ backuping_physical.CreateTestPhysicalBackuper(nil).MakeBackup(fixture.BackupID, false)
+ postgresql_executor.WaitForBackupStatus(t, fixture.BackupID, physical_enums.PhysicalBackupTypeFull,
+ physical_enums.PhysicalBackupStatusCompleted, nil, 3*time.Minute)
+
+ incrID := postgresql_executor.BuildAndClaimIncremental(t, fixture, nil)
+
+ backuping_physical.CreateTestPhysicalBackuper(nil).MakeBackup(incrID, false)
+ postgresql_executor.WaitForBackupStatus(t, incrID, physical_enums.PhysicalBackupTypeIncremental,
+ physical_enums.PhysicalBackupStatusCompleted, nil, 3*time.Minute)
+
+ incrRow, err := physical_repositories.GetIncrementalBackupRepository().FindByID(incrID)
+ require.NoError(t, err)
+ require.NotNil(t, incrRow.FileName, "an incremental on an idle DB must still upload an artifact")
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer.go
index 60a7e070..0cbe8d4d 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer.go
@@ -82,12 +82,14 @@ func CheckSummarizerReadiness(
}, nil
}
- covered, err := summarizerCoversLSN(ctx, conn, prevStopLSN)
+ window, err := readSummaryWindow(ctx, conn)
if err != nil {
return SummarizerResult{}, err
}
- if !covered {
+ // A parent stop_lsn below the oldest retained summary has aged out for good — no
+ // future summary will ever cover it, so the chain must re-anchor on a fresh FULL.
+ if window.hasAny && prevStopLSN < window.oldestStart {
reason := physical_enums.PhysicalBackupErrorSummariesExpired
return SummarizerResult{
@@ -96,6 +98,16 @@ func CheckSummarizerReadiness(
}, nil
}
+ // Summarizer on but nothing produced yet: not expiry, just not ready. Wait for the
+ // first summary rather than mismeasuring lag against a NULL MAX(end_lsn).
+ if !window.hasAny {
+ return SummarizerResult{
+ Decision: DecisionWait,
+ WaitFor: waitWindowForCadence(incrementalCadence),
+ PollEvery: summarizerWaitPollInterval,
+ }, nil
+ }
+
lag, err := measureSummarizerLag(ctx, conn)
if err != nil {
return SummarizerResult{}, err
@@ -110,10 +122,10 @@ func CheckSummarizerReadiness(
return SummarizerResult{}, err
}
- // A trailing lag within the active-segment band is the healthy steady
- // state — go incremental. pg_basebackup --incremental needs summaries only
- // from prevStopLSN onward (verified above) and waits for the last segment
- // to be summarized itself.
+ // A trailing lag within the active-segment band is the healthy steady state —
+ // go incremental. prevStopLSN is at or above the oldest retained summary (the
+ // aged-out case bailed above); if it sits in the last unsummarized sliver,
+ // pg_basebackup --incremental waits that segment out itself.
if lag < goThreshold {
return SummarizerResult{Decision: DecisionGoIncremental}, nil
}
@@ -225,26 +237,40 @@ func isSummarizerEnabled(ctx context.Context, conn *pgx.Conn) (bool, error) {
return setting == "on", nil
}
-// summarizerCoversLSN returns true when pg_available_wal_summaries reports a
-// summary that contains lsn (start_lsn <= lsn <= end_lsn). Empty result set
-// means the LSN has aged out of the kept window — that's
-// SUMMARIES_EXPIRED.
-func summarizerCoversLSN(ctx context.Context, conn *pgx.Conn, lsn walmath.LSN) (bool, error) {
- var exists bool
+type summaryWindow struct {
+ // hasAny is false when the summarizer is on but has produced no summary file yet —
+ // just enabled, or idle before the first checkpoint.
+ hasAny bool
+ oldestStart walmath.LSN
+ newestEnd walmath.LSN
+}
+
+func readSummaryWindow(ctx context.Context, conn *pgx.Conn) (summaryWindow, error) {
+ var oldestStart, newestEnd *string
err := conn.QueryRow(ctx, `
- SELECT EXISTS (
- SELECT 1
- FROM pg_available_wal_summaries()
- WHERE start_lsn <= $1::pg_lsn
- AND end_lsn >= $1::pg_lsn
- )
- `, lsn.String()).Scan(&exists)
+ SELECT MIN(start_lsn)::text, MAX(end_lsn)::text
+ FROM pg_available_wal_summaries()
+ `).Scan(&oldestStart, &newestEnd)
+ if err != nil {
+ return summaryWindow{}, fmt.Errorf("read WAL summary window: %w", err)
+ }
+
+ if oldestStart == nil || newestEnd == nil {
+ return summaryWindow{}, nil
+ }
+
+ start, err := walmath.ParseLSN(*oldestStart)
+ if err != nil {
+ return summaryWindow{}, fmt.Errorf("parse oldest summary start_lsn: %w", err)
+ }
+
+ end, err := walmath.ParseLSN(*newestEnd)
if err != nil {
- return false, fmt.Errorf("query pg_available_wal_summaries: %w", err)
+ return summaryWindow{}, fmt.Errorf("parse newest summary end_lsn: %w", err)
}
- return exists, nil
+ return summaryWindow{hasAny: true, oldestStart: start, newestEnd: end}, nil
}
// measureSummarizerLag returns how far the summarizer's coverage trails
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer_test.go
index 4db0c97e..1f94dd37 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer_test.go
@@ -46,14 +46,13 @@ func Test_CheckSummarizerReadiness_WhenSummarizerOff_FallsBackToFullNewChain(t *
require.Equal(t, physical_enums.PhysicalBackupErrorSummarizerOff, *result.Reason)
}
-// Test_CheckSummarizerReadiness_WhenSummariesDoNotCoverTarget_FallsBackToFullNewChain
-// covers the second WAL-gap fallback: the summarizer is on but no summary covers
-// the parent's stop LSN (the real-world cause is summary-file expiry). An LSN that
-// predates every available summary is, by definition, uncovered — so the check
+// Test_CheckSummarizerReadiness_WhenStopLsnPredatesOldestSummary_FallsBackToFullNewChain
+// covers genuine expiry: the summarizer is on but the parent's stop LSN sits below
+// the oldest retained summary, so no summary can ever cover it again — the check
// must steer to a fresh FULL and tag it SUMMARIES_EXPIRED. ExpireWalSummaries is
// applied first so the path mirrors production expiry rather than a synthetic LSN
// alone.
-func Test_CheckSummarizerReadiness_WhenSummariesDoNotCoverTarget_FallsBackToFullNewChain(t *testing.T) {
+func Test_CheckSummarizerReadiness_WhenStopLsnPredatesOldestSummary_FallsBackToFullNewChain(t *testing.T) {
fixture := SetupPhysicalDBForBackup(t)
conn := OpenAdminConn(t, fixture)
@@ -63,11 +62,11 @@ func Test_CheckSummarizerReadiness_WhenSummariesDoNotCoverTarget_FallsBackToFull
require.NoError(t, err)
require.True(t, enabled, "the standard fixture runs with summarize_wal on")
- // LSN 0/1 sits before any summary the running cluster can hold, so coverage
- // is guaranteed false regardless of where the summarizer currently starts.
- uncoveredTargetLSN := walmath.LSN(1)
+ // LSN 0/1 sits before any summary the running cluster can hold, so it predates
+ // the oldest retained summary regardless of where the summarizer currently starts.
+ expiredTargetLSN := walmath.LSN(1)
- result, err := CheckSummarizerReadiness(context.Background(), conn, uncoveredTargetLSN, time.Hour)
+ result, err := CheckSummarizerReadiness(context.Background(), conn, expiredTargetLSN, time.Hour)
require.NoError(t, err)
require.Equal(t, DecisionFullNewChain, result.Decision)
@@ -75,6 +74,41 @@ func Test_CheckSummarizerReadiness_WhenSummariesDoNotCoverTarget_FallsBackToFull
require.Equal(t, physical_enums.PhysicalBackupErrorSummariesExpired, *result.Reason)
}
+// Test_CheckSummarizerReadiness_WhenStopLsnAheadOfSummaries_DoesNotBreakChain is the
+// regression guard for the field report: right after a FULL on a near-idle DB the
+// WAL summarizer trails the FULL's stop_lsn (it never summarizes the active segment).
+// A stop_lsn the summarizer simply hasn't reached yet is NOT expiry — the check must
+// fall through to the lag path (GoIncremental / Wait), never CHAIN_BROKEN the chain
+// with SUMMARIES_EXPIRED. Runs on both PG majors since PG 18 is the reported engine.
+func Test_CheckSummarizerReadiness_WhenStopLsnAheadOfSummaries_DoesNotBreakChain(t *testing.T) {
+ for _, version := range []string{"17", "18"} {
+ t.Run("PostgreSQL "+version, func(t *testing.T) {
+ fixture := SetupPhysicalDBForBackupVersion(t, version)
+ conn := OpenAdminConn(t, fixture)
+
+ ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
+ defer cancel()
+
+ // Advance WAL a little so the current tip sits past every flushed summary,
+ // reproducing the "summarizer behind a fresh FULL's stop_lsn" state.
+ _, err := GenerateWalActivity(ctx, conn, 1)
+ require.NoError(t, err)
+
+ var walTipLSN walmath.LSN
+ require.NoError(t, conn.QueryRow(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&walTipLSN))
+
+ result, err := CheckSummarizerReadiness(ctx, conn, walTipLSN, time.Hour)
+ require.NoError(t, err)
+
+ require.NotEqual(t, DecisionFullNewChain, result.Decision,
+ "a stop_lsn the summarizer has not reached yet must not break the chain")
+ require.Nil(t, result.Reason)
+ require.Contains(t,
+ []SummarizerDecision{DecisionGoIncremental, DecisionWait}, result.Decision)
+ })
+ }
+}
+
// stubSummarizerCheck installs a scripted readiness probe in place of the live
// CheckSummarizerReadiness and restores the original when the test ends. Each
// call returns the next scripted step; the final step repeats once the script