PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (physical): Don't break incremental chain when WAL summarizer is merely behind
Rostislav Duginf8ffaf7346ba
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc_test.goindex 0ad65be4..e26691b7 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc_test.go@@ -88,3 +88,21 @@ func Test_CreateIncremental_WhenSummarizerHealthy_GoesIncremental(t *testing.T)require.NotNil(t, incrRow.FileName, "a healthy incremental must upload an artifact")require.NotNil(t, incrRow.StopLSN, "a completed incremental must record stop_lsn")}++func Test_CreateIncremental_WhenSummarizerBehindOnIdleDB_StillCompletes(t *testing.T) {+ fixture := postgresql_executor.SetupPhysicalDBForBackup(t)++ backuping_physical.CreateTestPhysicalBackuper(nil).MakeBackup(fixture.BackupID, false)+ postgresql_executor.WaitForBackupStatus(t, fixture.BackupID, physical_enums.PhysicalBackupTypeFull,+ physical_enums.PhysicalBackupStatusCompleted, nil, 3*time.Minute)++ incrID := postgresql_executor.BuildAndClaimIncremental(t, fixture, nil)++ backuping_physical.CreateTestPhysicalBackuper(nil).MakeBackup(incrID, false)+ postgresql_executor.WaitForBackupStatus(t, incrID, physical_enums.PhysicalBackupTypeIncremental,+ physical_enums.PhysicalBackupStatusCompleted, nil, 3*time.Minute)++ incrRow, err := physical_repositories.GetIncrementalBackupRepository().FindByID(incrID)+ require.NoError(t, err)+ require.NotNil(t, incrRow.FileName, "an incremental on an idle DB must still upload an artifact")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer.goindex 60a7e070..0cbe8d4d 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer.go@@ -82,12 +82,14 @@ func CheckSummarizerReadiness(}, nil}- covered, err := summarizerCoversLSN(ctx, conn, prevStopLSN)+ window, err := readSummaryWindow(ctx, conn)if err != nil {return SummarizerResult{}, err}- if !covered {+ // A parent stop_lsn below the oldest retained summary has aged out for good — no+ // future summary will ever cover it, so the chain must re-anchor on a fresh FULL.+ if window.hasAny && prevStopLSN < window.oldestStart {reason := physical_enums.PhysicalBackupErrorSummariesExpiredreturn SummarizerResult{@@ -96,6 +98,16 @@ func CheckSummarizerReadiness(}, nil}+ // Summarizer on but nothing produced yet: not expiry, just not ready. Wait for the+ // first summary rather than mismeasuring lag against a NULL MAX(end_lsn).+ if !window.hasAny {+ return SummarizerResult{+ Decision: DecisionWait,+ WaitFor: waitWindowForCadence(incrementalCadence),+ PollEvery: summarizerWaitPollInterval,+ }, nil+ }+lag, err := measureSummarizerLag(ctx, conn)if err != nil {return SummarizerResult{}, err@@ -110,10 +122,10 @@ func CheckSummarizerReadiness(return SummarizerResult{}, err}- // A trailing lag within the active-segment band is the healthy steady- // state — go incremental. pg_basebackup --incremental needs summaries only- // from prevStopLSN onward (verified above) and waits for the last segment- // to be summarized itself.+ // A trailing lag within the active-segment band is the healthy steady state —+ // go incremental. prevStopLSN is at or above the oldest retained summary (the+ // aged-out case bailed above); if it sits in the last unsummarized sliver,+ // pg_basebackup --incremental waits that segment out itself.if lag < goThreshold {return SummarizerResult{Decision: DecisionGoIncremental}, nil}@@ -225,26 +237,40 @@ func isSummarizerEnabled(ctx context.Context, conn *pgx.Conn) (bool, error) {return setting == "on", nil}-// summarizerCoversLSN returns true when pg_available_wal_summaries reports a-// summary that contains lsn (start_lsn <= lsn <= end_lsn). Empty result set-// means the LSN has aged out of the kept window — that's-// SUMMARIES_EXPIRED.-func summarizerCoversLSN(ctx context.Context, conn *pgx.Conn, lsn walmath.LSN) (bool, error) {- var exists bool+type summaryWindow struct {+ // hasAny is false when the summarizer is on but has produced no summary file yet —+ // just enabled, or idle before the first checkpoint.+ hasAny bool+ oldestStart walmath.LSN+ newestEnd walmath.LSN+}++func readSummaryWindow(ctx context.Context, conn *pgx.Conn) (summaryWindow, error) {+ var oldestStart, newestEnd *stringerr := conn.QueryRow(ctx, `- SELECT EXISTS (- SELECT 1- FROM pg_available_wal_summaries()- WHERE start_lsn <= $1::pg_lsn- AND end_lsn >= $1::pg_lsn- )- `, lsn.String()).Scan(&exists)+ SELECT MIN(start_lsn)::text, MAX(end_lsn)::text+ FROM pg_available_wal_summaries()+ `).Scan(&oldestStart, &newestEnd)+ if err != nil {+ return summaryWindow{}, fmt.Errorf("read WAL summary window: %w", err)+ }++ if oldestStart == nil || newestEnd == nil {+ return summaryWindow{}, nil+ }++ start, err := walmath.ParseLSN(*oldestStart)+ if err != nil {+ return summaryWindow{}, fmt.Errorf("parse oldest summary start_lsn: %w", err)+ }++ end, err := walmath.ParseLSN(*newestEnd)if err != nil {- return false, fmt.Errorf("query pg_available_wal_summaries: %w", err)+ return summaryWindow{}, fmt.Errorf("parse newest summary end_lsn: %w", err)}- return exists, nil+ return summaryWindow{hasAny: true, oldestStart: start, newestEnd: end}, nil}// measureSummarizerLag returns how far the summarizer's coverage trailsdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer_test.goindex 4db0c97e..1f94dd37 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/summarizer_test.go@@ -46,14 +46,13 @@ func Test_CheckSummarizerReadiness_WhenSummarizerOff_FallsBackToFullNewChain(t *require.Equal(t, physical_enums.PhysicalBackupErrorSummarizerOff, *result.Reason)}-// Test_CheckSummarizerReadiness_WhenSummariesDoNotCoverTarget_FallsBackToFullNewChain-// covers the second WAL-gap fallback: the summarizer is on but no summary covers-// the parent's stop LSN (the real-world cause is summary-file expiry). An LSN that-// predates every available summary is, by definition, uncovered — so the check+// Test_CheckSummarizerReadiness_WhenStopLsnPredatesOldestSummary_FallsBackToFullNewChain+// covers genuine expiry: the summarizer is on but the parent's stop LSN sits below+// the oldest retained summary, so no summary can ever cover it again — the check// must steer to a fresh FULL and tag it SUMMARIES_EXPIRED. ExpireWalSummaries is// applied first so the path mirrors production expiry rather than a synthetic LSN// alone.-func Test_CheckSummarizerReadiness_WhenSummariesDoNotCoverTarget_FallsBackToFullNewChain(t *testing.T) {+func Test_CheckSummarizerReadiness_WhenStopLsnPredatesOldestSummary_FallsBackToFullNewChain(t *testing.T) {fixture := SetupPhysicalDBForBackup(t)conn := OpenAdminConn(t, fixture)@@ -63,11 +62,11 @@ func Test_CheckSummarizerReadiness_WhenSummariesDoNotCoverTarget_FallsBackToFullrequire.NoError(t, err)require.True(t, enabled, "the standard fixture runs with summarize_wal on")- // LSN 0/1 sits before any summary the running cluster can hold, so coverage- // is guaranteed false regardless of where the summarizer currently starts.- uncoveredTargetLSN := walmath.LSN(1)+ // LSN 0/1 sits before any summary the running cluster can hold, so it predates+ // the oldest retained summary regardless of where the summarizer currently starts.+ expiredTargetLSN := walmath.LSN(1)- result, err := CheckSummarizerReadiness(context.Background(), conn, uncoveredTargetLSN, time.Hour)+ result, err := CheckSummarizerReadiness(context.Background(), conn, expiredTargetLSN, time.Hour)require.NoError(t, err)require.Equal(t, DecisionFullNewChain, result.Decision)@@ -75,6 +74,41 @@ func Test_CheckSummarizerReadiness_WhenSummariesDoNotCoverTarget_FallsBackToFullrequire.Equal(t, physical_enums.PhysicalBackupErrorSummariesExpired, *result.Reason)}+// Test_CheckSummarizerReadiness_WhenStopLsnAheadOfSummaries_DoesNotBreakChain is the+// regression guard for the field report: right after a FULL on a near-idle DB the+// WAL summarizer trails the FULL's stop_lsn (it never summarizes the active segment).+// A stop_lsn the summarizer simply hasn't reached yet is NOT expiry — the check must+// fall through to the lag path (GoIncremental / Wait), never CHAIN_BROKEN the chain+// with SUMMARIES_EXPIRED. Runs on both PG majors since PG 18 is the reported engine.+func Test_CheckSummarizerReadiness_WhenStopLsnAheadOfSummaries_DoesNotBreakChain(t *testing.T) {+ for _, version := range []string{"17", "18"} {+ t.Run("PostgreSQL "+version, func(t *testing.T) {+ fixture := SetupPhysicalDBForBackupVersion(t, version)+ conn := OpenAdminConn(t, fixture)++ ctx, cancel := context.WithTimeout(t.Context(), time.Minute)+ defer cancel()++ // Advance WAL a little so the current tip sits past every flushed summary,+ // reproducing the "summarizer behind a fresh FULL's stop_lsn" state.+ _, err := GenerateWalActivity(ctx, conn, 1)+ require.NoError(t, err)++ var walTipLSN walmath.LSN+ require.NoError(t, conn.QueryRow(ctx, "SELECT pg_current_wal_lsn()::text").Scan(&walTipLSN))++ result, err := CheckSummarizerReadiness(ctx, conn, walTipLSN, time.Hour)+ require.NoError(t, err)++ require.NotEqual(t, DecisionFullNewChain, result.Decision,+ "a stop_lsn the summarizer has not reached yet must not break the chain")+ require.Nil(t, result.Reason)+ require.Contains(t,+ []SummarizerDecision{DecisionGoIncremental, DecisionWait}, result.Decision)+ })+ }+}+// stubSummarizerCheck installs a scripted readiness probe in place of the live// CheckSummarizerReadiness and restores the original when the test ends. Each// call returns the next scripted step; the final step repeats once the script