PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Stop the physical scheduler and cleaner from rebuilding every chain with all its WAL on each tick, so an idle instance no longer burns CPU in proportion to retained WAL
Rostislav Dugin7e7699b0c363
diff --git a/backend/internal/features/backups/backups/backuping/physical/cleaner.go b/backend/internal/features/backups/backups/backuping/physical/cleaner.goindex 039cb249..9e9b076e 100644--- a/backend/internal/features/backups/backups/backuping/physical/cleaner.go+++ b/backend/internal/features/backups/backups/backuping/physical/cleaner.go@@ -127,7 +127,7 @@ func (c *PhysicalBackupCleaner) nonExtendableChainsNewestEndFirst(candidates := make([]chainCandidate, 0, len(views))for _, view := range views {- endTs, err := c.chainViewService.GetChainEndTimestamp(view.RootFull.ID)+ endTs, err := c.chainViewService.GetChainEndTimestamp(view)if err != nil {c.logger.Error("failed to get chain end timestamp","root_full_backup_id", view.RootFull.ID, "error", err)@@ -220,21 +220,12 @@ func (c *PhysicalBackupCleaner) walDeleteBudgetMB(ctx context.Context, databaseI// isChainWithinGrace reports whether a chain is too young to evict: its end// timestamp is within max(full, incr cadence) × 2 (floored at the 60-minute-// per-backup grace). A failure to read the timestamp is treated as within grace-// so a transient error never causes a premature delete.-func (c *PhysicalBackupCleaner) isChainWithinGrace(- ctx context.Context,+// per-backup grace). A chain whose end timestamp failed to load never becomes a+// candidate, so a transient error never causes a premature delete.+func isChainWithinGrace(backupConfig *backups_config_physical.PhysicalBackupConfig,- rootFullBackupID uuid.UUID,+ chainEndTs time.Time,) bool {- endTs, err := c.chainViewService.GetChainEndTimestamp(rootFullBackupID)- if err != nil {- c.logger.WarnContext(ctx, "failed to get chain end timestamp; treating as within grace",- "root_full_backup_id", rootFullBackupID, "error", err)-- return true- }-grace := max(approxIntervalDuration(backupConfig.FullBackupInterval),approxIntervalDuration(backupConfig.IncrementalBackupInterval),@@ -242,7 +233,7 @@ func (c *PhysicalBackupCleaner) isChainWithinGrace(grace = max(grace, recentBackupGracePeriod)- return time.Since(endTs) < grace+ return time.Since(chainEndTs) < grace}// cleanByChains keeps the N newest non-extendable chains (by chain-end@@ -277,7 +268,7 @@ func (c *PhysicalBackupCleaner) cleanByChains(for _, candidate := range candidates[keepCount:] {rootFullBackupID := candidate.view.RootFull.ID- if c.isChainWithinGrace(ctx, backupConfig, rootFullBackupID) {+ if isChainWithinGrace(backupConfig, candidate.endTs) {withinGraceCount++logger.DebugContext(ctx, "keeping chain, it is still within the retention grace period",@@ -334,7 +325,7 @@ func (c *PhysicalBackupCleaner) cleanByFulls(for _, candidate := range candidates {rootFullBackupID := candidate.view.RootFull.ID- if c.isChainWithinGrace(ctx, backupConfig, rootFullBackupID) {+ if isChainWithinGrace(backupConfig, candidate.endTs) {withinGraceCount++logger.DebugContext(ctx, "keeping chain, it is still within the retention grace period",@@ -460,7 +451,7 @@ func (c *PhysicalBackupCleaner) cleanByCombined(continue}- if c.isChainWithinGrace(ctx, backupConfig, rootFullBackupID) {+ if isChainWithinGrace(backupConfig, candidate.endTs) {withinGraceCount++logger.DebugContext(ctx, "keeping chain, it is still within the retention grace period",diff --git a/backend/internal/features/backups/backups/backuping/physical/constants.go b/backend/internal/features/backups/backups/backuping/physical/constants.goindex 0800a970..0405b6a9 100644--- a/backend/internal/features/backups/backups/backuping/physical/constants.go+++ b/backend/internal/features/backups/backups/backuping/physical/constants.go@@ -23,10 +23,11 @@ const (// rename would silently break log queries.schedulerJobName = "physical_backup_scheduler"- // Cleaner ticks every 3 s so retention decisions become visible almost- // immediately, while the per-tick WAL byte budget keeps a single tick bounded- // even at this cadence. Mirrors logical.- cleanerTickInterval = 3 * time.Second+ // Each cleaner tick reads every closed chain of every enabled database and+ // runs the orphan WAL anti-join. Retention grace is at least an hour and+ // abandoned WAL claims are reaped after one, so a minute of extra delay+ // changes no outcome while keeping that work off an idle instance.+ cleanerTickInterval = 1 * time.Minute// Never delete a chain whose end timestamp is younger than// max(full, incr cadence) × 2 — protects a chain that just completed or isdiff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler.go b/backend/internal/features/backups/backups/backuping/physical/scheduler.goindex b3eec17f..1b6115d4 100644--- a/backend/internal/features/backups/backups/backuping/physical/scheduler.go+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler.go@@ -232,13 +232,6 @@ func (s *PhysicalBackupsScheduler) decideBackupKind(return backupDecision{}, false}- extendableChain, err := s.chainViewService.FindLastExtendableChainByDatabase(backupConfig.DatabaseID)- if err != nil {- logger.Error("failed to find extendable chain", "error", err)-- return backupDecision{}, false- }-lastBackupTime := newestCreatedAt(lastFull, lastIncr)if !backupConfig.IncrementalBackupInterval.ShouldTriggerBackup(now, lastBackupTime) {@@ -247,6 +240,13 @@ func (s *PhysicalBackupsScheduler) decideBackupKind(return backupDecision{}, false}+ extendableChain, err := s.chainViewService.FindLastExtendableChainByDatabase(backupConfig.DatabaseID)+ if err != nil {+ logger.Error("failed to find extendable chain", "error", err)++ return backupDecision{}, false+ }+if extendableChain == nil {return s.decideReAnchoringFull(logger, backupConfig, lastIncr)}diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/enums.go b/backend/internal/features/backups/backups/core/physical/chain_view/enums.goindex 1b8190b8..c2e8d547 100644--- a/backend/internal/features/backups/backups/core/physical/chain_view/enums.go+++ b/backend/internal/features/backups/backups/core/physical/chain_view/enums.go@@ -1,17 +1,5 @@package chain_view-// ChainState describes whether a chain is still extendable, dead or-// closed by a successor FULL. WAL gaps inside a chain DO NOT change this-// state — lossy chains stay EXTENDABLE; see FindWalGapsInChain to surface the-// unreachable PITR windows.-type ChainState string--const (- ChainStateExtendable ChainState = "EXTENDABLE"- ChainStateBrokenByIncr ChainState = "BROKEN_BY_INCR"- ChainStateClosedByNewerFull ChainState = "CLOSED_BY_NEWER_FULL"-)-type ValidationStatus stringconst (diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/service.go b/backend/internal/features/backups/backups/core/physical/chain_view/service.goindex 7295e313..0e7e9f25 100644--- a/backend/internal/features/backups/backups/core/physical/chain_view/service.go+++ b/backend/internal/features/backups/backups/core/physical/chain_view/service.go@@ -13,7 +13,6 @@ import (physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories""databasus-backend/internal/storage"- "databasus-backend/internal/util/walmath")type ChainViewService struct {@@ -29,18 +28,20 @@ func (s *ChainViewService) FindLastExtendableChainByDatabase(databaseID uuid.UUIreturn nil, err}- for _, full := range fulls {- state, err := s.getChainState(full.ID)- if err != nil {- return nil, err- }+ if len(fulls) == 0 {+ return nil, nil+ }- if state == ChainStateExtendable {- return s.buildChainView(full)- }+ newestView, err := s.buildChainView(fulls[0])+ if err != nil {+ return nil, err+ }++ if !isExtendable(newestView) {+ return nil, nil}- return nil, nil+ return newestView, nil}func (s *ChainViewService) FindNonExtendableChainsByDatabase(databaseID uuid.UUID) ([]*ChainView, error) {@@ -50,24 +51,17 @@ func (s *ChainViewService) FindNonExtendableChainsByDatabase(databaseID uuid.UUI}views := make([]*ChainView, 0, len(fulls))- sawActive := false- for _, full := range fulls {- state, err := s.getChainState(full.ID)+ for i, full := range fulls {+ view, err := s.buildChainView(full)if err != nil {return nil, err}- if !sawActive && state == ChainStateExtendable {- sawActive = true+ if i == 0 && isExtendable(view) {continue}- view, err := s.buildChainView(full)- if err != nil {- return nil, err- }-views = append(views, view)}@@ -82,47 +76,8 @@ func (s *ChainViewService) GetChainSpan(rootFullBackupID uuid.UUID) (LSNRange, eif full == nil {return LSNRange{}, fmt.Errorf("chain_view: full backup not found: %s", rootFullBackupID)}- if full.StartLSN == nil {- return LSNRange{}, errors.New("chain_view: cannot compute span — root FULL has no start_lsn")- }- start := *full.StartLSN-- var successor physical_models.PhysicalFullBackup-- err = storage.- GetDb().- Where(- "database_id = ? AND timeline_id = ? AND status = ? AND start_lsn > ?::pg_lsn",- full.DatabaseID,- full.TimelineID,- physical_enums.PhysicalBackupStatusCompleted,- start.String(),- ).- Order("start_lsn ASC").- First(&successor).Error- if err != nil {- if errors.Is(err, gorm.ErrRecordNotFound) {- return LSNRange{Start: start, End: LSNMax}, nil- }-- return LSNRange{}, err- }-- end := LSNMax- if successor.StartLSN != nil {- end = *successor.StartLSN- }-- return LSNRange{Start: start, End: end}, nil-}--func (s *ChainViewService) FindWalSegmentsInSpan(- databaseID uuid.UUID,- timelineID int,- startLSN, endLSN walmath.LSN,-) ([]*physical_models.PhysicalWalSegment, error) {- return s.walSegmentRepository.FindByChainSpan(databaseID, timelineID, startLSN, endLSN)+ return s.getChainSpanForFull(full)}func (s *ChainViewService) FindWalGapsInChain(rootFullBackupID uuid.UUID) ([]LSNRange, error) {@@ -192,47 +147,27 @@ func (s *ChainViewService) CheckHistoryFilePresence(}, nil}-func (s *ChainViewService) GetChainEndTimestamp(rootFullBackupID uuid.UUID) (time.Time, error) {- full, err := s.fullBackupRepository.FindByID(rootFullBackupID)- if err != nil {- return time.Time{}, err- }- if full == nil {- return time.Time{}, fmt.Errorf("chain_view: full backup not found: %s", rootFullBackupID)- }-+func (s *ChainViewService) GetChainEndTimestamp(view *ChainView) (time.Time, error) {latest := time.Time{}- if full.CompletedAt != nil {- latest = *full.CompletedAt+ if view.RootFull.CompletedAt != nil {+ latest = *view.RootFull.CompletedAt}- incrementalBackups, err := s.incrementalBackupRepository.FindAllByRootFull(rootFullBackupID)- if err != nil {- return time.Time{}, err- }-- for _, incrementalBackup := range incrementalBackups {+ for _, incrementalBackup := range view.Incrementals {if incrementalBackup.CompletedAt != nil && incrementalBackup.CompletedAt.After(latest) {latest = *incrementalBackup.CompletedAt}}- span, err := s.GetChainSpan(rootFullBackupID)- if err != nil {- return time.Time{}, err- }-- segments, err := s.walSegmentRepository.FindByChainSpan(- full.DatabaseID, full.TimelineID, span.Start, span.End,+ latestWalReceivedAt, err := s.walSegmentRepository.GetLatestReceivedAtInChainSpan(+ view.RootFull.DatabaseID, view.RootFull.TimelineID, view.Span.Start, view.Span.End,)if err != nil {return time.Time{}, err}- for _, segment := range segments {- if segment.ReceivedAt.After(latest) {- latest = segment.ReceivedAt- }+ if latestWalReceivedAt != nil && latestWalReceivedAt.After(latest) {+ latest = *latestWalReceivedAt}return latest, nil@@ -252,71 +187,59 @@ func (s *ChainViewService) FindWalOrphansByDatabase(databaseID uuid.UUID) ([]Walreturn refs, nil}-// Precedence:-// 1. A newer COMPLETED FULL on the same database → CLOSED_BY_NEWER_FULL.-// 2. Any downstream INCR with status=CHAIN_BROKEN → BROKEN_BY_INCR.-// 3. Otherwise → EXTENDABLE.-func (s *ChainViewService) getChainState(rootFullBackupID uuid.UUID) (ChainState, error) {- full, err := s.fullBackupRepository.FindByID(rootFullBackupID)- if err != nil {- return "", err- }- if full == nil {- return "", fmt.Errorf("chain_view: full backup not found: %s", rootFullBackupID)+func (s *ChainViewService) getChainSpanForFull(full *physical_models.PhysicalFullBackup) (LSNRange, error) {+ if full.StartLSN == nil {+ return LSNRange{}, errors.New("chain_view: cannot compute span — root FULL has no start_lsn")}- newerFulls, err := s.findNewerCompletedFulls(full)+ start := *full.StartLSN++ var successor physical_models.PhysicalFullBackup++ err := storage.+ GetDb().+ Where(+ "database_id = ? AND timeline_id = ? AND status = ? AND start_lsn > ?::pg_lsn",+ full.DatabaseID,+ full.TimelineID,+ physical_enums.PhysicalBackupStatusCompleted,+ start.String(),+ ).+ Order("start_lsn ASC").+ First(&successor).Errorif err != nil {- return "", err- }- if len(newerFulls) > 0 {- return ChainStateClosedByNewerFull, nil+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return LSNRange{Start: start, End: LSNMax}, nil+ }++ return LSNRange{}, err}- incrementalBackups, err := s.incrementalBackupRepository.FindAllByRootFull(rootFullBackupID)- if err != nil {- return "", err+ end := LSNMax+ if successor.StartLSN != nil {+ end = *successor.StartLSN}- chainBroken := slices.ContainsFunc(- incrementalBackups,+ return LSNRange{Start: start, End: end}, nil+}++// Any newer COMPLETED FULL closes a chain, so only the newest one can still be+// extended, and only until one of its INCRs breaks the chain. WAL gaps never+// change this: a lossy chain stays extendable, and FindWalGapsInChain surfaces+// the unreachable PITR windows.+func isExtendable(newestView *ChainView) bool {+ return !slices.ContainsFunc(+ newestView.Incrementals,func(incrementalBackup *physical_models.PhysicalIncrementalBackup) bool {return incrementalBackup.Status == physical_enums.PhysicalBackupStatusChainBroken},)- if chainBroken {- return ChainStateBrokenByIncr, nil- }-- return ChainStateExtendable, nil-}--func (s *ChainViewService) findNewerCompletedFulls(- full *physical_models.PhysicalFullBackup,-) ([]*physical_models.PhysicalFullBackup, error) {- allCompletedFulls, err := s.fullBackupRepository.FindCompletedNewestFirstByDatabase(full.DatabaseID)- if err != nil {- return nil, err- }-- newerCompletedFulls := make([]*physical_models.PhysicalFullBackup, 0)- for _, candidate := range allCompletedFulls {- if candidate.ID == full.ID {- continue- }-- if candidate.CreatedAt.After(full.CreatedAt) {- newerCompletedFulls = append(newerCompletedFulls, candidate)- }- }-- return newerCompletedFulls, nil}func (s *ChainViewService) buildChainView(full *physical_models.PhysicalFullBackup,) (*ChainView, error) {- span, err := s.GetChainSpan(full.ID)+ span, err := s.getChainSpanForFull(full)if err != nil {return nil, err}@@ -326,44 +249,9 @@ func (s *ChainViewService) buildChainView(return nil, err}- segments, err := s.walSegmentRepository.FindByChainSpan(- full.DatabaseID, full.TimelineID, span.Start, span.End,- )- if err != nil {- return nil, err- }-- gaps, err := s.FindWalGapsInChain(full.ID)- if err != nil {- return nil, err- }-- maxReplayableLSN := walmath.LSN(0)- if full.StopLSN != nil {- maxReplayableLSN = *full.StopLSN- }-- for _, segment := range segments {- segmentEndLSN := segment.EndLSN-- insideGap := slices.ContainsFunc(gaps, func(gap LSNRange) bool {- return segmentEndLSN > gap.Start- })- if insideGap {- break- }-- if segmentEndLSN > maxReplayableLSN {- maxReplayableLSN = segmentEndLSN- }- }-return &ChainView{- RootFull: full,- Incrementals: incrementalBackups,- WalSegments: segments,- Gaps: gaps,- Span: span,- MaxReplayableLSN: maxReplayableLSN,+ RootFull: full,+ Incrementals: incrementalBackups,+ Span: span,}, nil}diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go b/backend/internal/features/backups/backups/core/physical/chain_view/service_test.goindex 2d44e305..bffebd06 100644--- a/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go+++ b/backend/internal/features/backups/backups/core/physical/chain_view/service_test.go@@ -1,6 +1,7 @@package chain_view_testimport (+ "fmt""testing""time"@@ -11,6 +12,8 @@ import ("databasus-backend/internal/features/backups/backups/core/physical/chain_view"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"+ physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"physical_testing "databasus-backend/internal/features/backups/backups/core/physical/testing""databasus-backend/internal/features/databases""databasus-backend/internal/features/notifiers"@@ -97,8 +100,6 @@ func Test_FindLastExtendableChainByDatabase_WhenChainHasOnlyHealthyFull_ReturnsCrequire.NoError(t, err)require.NotNil(t, view)assert.Equal(t, full.ID, view.RootFull.ID)- assert.Len(t, view.WalSegments, 1)- assert.Empty(t, view.Gaps)}func Test_FindLastExtendableChainByDatabase_WhenChainHasBrokenIncr_ReturnsNil(t *testing.T) {@@ -239,7 +240,9 @@ func Test_GetChainSpan_WhenSuccessorFullExists_EndIsSuccessorStartLSN(t *testingassert.Equal(t, olderStart, span.Start)assert.Equal(t, newerStart, span.End, "older FULL's span ends at next FULL's start_lsn")- walInSpan, err := service.FindWalSegmentsInSpan(databaseID, 1, span.Start, span.End)+ walInSpan, err := physical_repositories.GetWalSegmentRepository().FindByChainSpan(+ databaseID, 1, span.Start, span.End,+ )require.NoError(t, err)assert.Empty(t, walInSpan)}@@ -271,7 +274,9 @@ func Test_GetChainSpan_WhenNoSuccessorFull_EndIsLSNMax(t *testing.T) {assert.Equal(t, fullStart, span.Start)assert.Equal(t, chain_view.LSNMax, span.End, "latest FULL has unbounded span")- walInSpan, err := service.FindWalSegmentsInSpan(databaseID, 1, span.Start, span.End)+ walInSpan, err := physical_repositories.GetWalSegmentRepository().FindByChainSpan(+ databaseID, 1, span.Start, span.End,+ )require.NoError(t, err)assert.Len(t, walInSpan, 1, "WAL after the FULL's stop_lsn belongs to its open-ended span")}@@ -339,7 +344,11 @@ func Test_GetChainEndTimestamp_WhenWalReceivedAfterIncr_ReturnsWalReceivedAt(t *service := chain_view.GetChainViewService()- end, err := service.GetChainEndTimestamp(full.ID)+ view, err := service.FindLastExtendableChainByDatabase(databaseID)+ require.NoError(t, err)+ require.NotNil(t, view)++ end, err := service.GetChainEndTimestamp(view)require.NoError(t, err)assert.WithinDuration(t, walReceived, end, time.Second,"WAL.ReceivedAt is the latest of the three; should win the max")@@ -453,38 +462,202 @@ func Test_FindWalGapsInChain_WhenMultipleGapsExist_ReturnsAllGaps(t *testing.T)assert.Equal(t, seg3Start, gaps[1].End)}-func Test_FindLastExtendableChainByDatabase_WhenChainHasGap_MaxReplayableLSNStopsAtFirstGap(t *testing.T) {+func Test_GetChainEndTimestamp_WhenChainHasNoWal_ReturnsLatestBackupCompletion(t *testing.T) {t.Parallel()prereqs := createChainViewTestPrereqs(t)databaseID := prereqs.database.ID- full := physical_testing.CreateTestFullBackup(t,+ full := physical_testing.NewTestCompletedFullBackup(+ databaseID, prereqs.storage.ID, 1,+ walmath.LSN(0), walmath.LSN(segmentBytes),+ )+ full.CompletedAt = new(time.Now().UTC().Add(-2 * time.Hour))+ physical_testing.CreateTestFullBackup(t, full)++ incrCompletedAt := time.Now().UTC().Add(-30 * time.Minute)+ incr := physical_testing.NewTestCompletedIncrementalBackup(+ databaseID, prereqs.storage.ID, full.ID, nil, 1,+ walmath.LSN(segmentBytes), walmath.LSN(2*segmentBytes),+ )+ incr.CompletedAt = new(incrCompletedAt)+ physical_testing.CreateTestIncrementalBackup(t, incr)++ service := chain_view.GetChainViewService()++ view, err := service.FindLastExtendableChainByDatabase(databaseID)+ require.NoError(t, err)+ require.NotNil(t, view)++ end, err := service.GetChainEndTimestamp(view)+ require.NoError(t, err)+ assert.WithinDuration(t, incrCompletedAt, end, time.Second)+}++func Test_FindNonExtendableChainsByDatabase_WhenNewestChainBroken_ListsItWithOlderChains(t *testing.T) {+ t.Parallel()++ prereqs := createChainViewTestPrereqs(t)+ databaseID := prereqs.database.ID++ olderFull := physical_testing.NewTestCompletedFullBackup(+ databaseID, prereqs.storage.ID, 1,+ walmath.LSN(0), walmath.LSN(segmentBytes),+ )+ olderFull.CreatedAt = time.Now().UTC().Add(-time.Hour)+ olderFull.CompletedAt = new(olderFull.CreatedAt.Add(time.Minute))+ physical_testing.CreateTestFullBackup(t, olderFull)++ newerFull := physical_testing.CreateTestFullBackup(t,physical_testing.NewTestCompletedFullBackup(databaseID, prereqs.storage.ID, 1,- walmath.LSN(0), walmath.LSN(segmentBytes),+ walmath.LSN(4*segmentBytes), walmath.LSN(5*segmentBytes),))- beforeGapEnd := walmath.LSN(2 * segmentBytes)- afterGapStart := walmath.LSN(4 * segmentBytes)+ brokenIncr := physical_testing.NewTestCompletedIncrementalBackup(+ databaseID, prereqs.storage.ID, newerFull.ID, nil, 1,+ walmath.LSN(5*segmentBytes), walmath.LSN(6*segmentBytes),+ )+ brokenIncr.Status = physical_enums.PhysicalBackupStatusChainBroken+ brokenIncr.ErrorReason = new(physical_enums.PhysicalBackupErrorSummariesExpired)+ physical_testing.CreateTestIncrementalBackup(t, brokenIncr)++ service := chain_view.GetChainViewService()++ extendableView, err := service.FindLastExtendableChainByDatabase(databaseID)+ require.NoError(t, err)+ assert.Nil(t, extendableView)++ closedViews, err := service.FindNonExtendableChainsByDatabase(databaseID)+ require.NoError(t, err)+ require.Len(t, closedViews, 2)+ assert.Equal(t, newerFull.ID, closedViews[0].RootFull.ID)+ assert.Equal(t, olderFull.ID, closedViews[1].RootFull.ID)+}++func Test_FindByChainSpan_ForMiddleChain_ReturnsOnlyItsSegmentsIncludingBoundary(t *testing.T) {+ t.Parallel()++ prereqs := createChainViewTestPrereqs(t)+ databaseID := prereqs.database.ID++ fullIDsBySegmentIndex := make(map[int]uuid.UUID)++ for _, segmentIndex := range []int{0, 4, 8} {+ fullStart := walmath.LSN(segmentIndex*segmentBytes + firstRecordOffset)+ full := physical_testing.CreateTestFullBackup(t,+ physical_testing.NewTestCompletedFullBackup(+ databaseID, prereqs.storage.ID, 1, fullStart, fullStart+fullLSNSpan,+ ))++ fullIDsBySegmentIndex[segmentIndex] = full.ID+ }++ segmentIDsByIndex := createContiguousWalSegments(t, prereqs, databaseID, 1, 0, 10)++ middleSpan, err := chain_view.GetChainViewService().GetChainSpan(fullIDsBySegmentIndex[4])+ require.NoError(t, err)++ middleSegments, err := physical_repositories.GetWalSegmentRepository().FindByChainSpan(+ databaseID, 1, middleSpan.Start, middleSpan.End,+ )+ require.NoError(t, err)++ assert.Equal(t,+ []uuid.UUID{+ segmentIDsByIndex[4], segmentIDsByIndex[5], segmentIDsByIndex[6],+ segmentIDsByIndex[7], segmentIDsByIndex[8],+ },+ getWalSegmentIDs(middleSegments),+ "the segment holding the middle FULL's start begins below the span and is included; "++ "segments of the first chain are not",+ )+}++func Test_FindByChainSpan_WhenSpanStartsOnSegmentBoundary_ExcludesSegmentEndingThere(t *testing.T) {+ t.Parallel()++ prereqs := createChainViewTestPrereqs(t)+ databaseID := prereqs.database.ID++ segmentIDsByIndex := createContiguousWalSegments(t, prereqs, databaseID, 1, 3, 6)++ spanSegments, err := physical_repositories.GetWalSegmentRepository().FindByChainSpan(+ databaseID, 1, walmath.LSN(4*segmentBytes), chain_view.LSNMax,+ )+ require.NoError(t, err)++ assert.Equal(t,+ []uuid.UUID{segmentIDsByIndex[4], segmentIDsByIndex[5]},+ getWalSegmentIDs(spanSegments),+ )+}++func Test_FindByChainSpan_WhenOtherTimelineAndDatabaseHaveSegmentsInRange_IgnoresThem(t *testing.T) {+ t.Parallel()++ prereqs := createChainViewTestPrereqs(t)+ otherPrereqs := createChainViewTestPrereqs(t)+ databaseID := prereqs.database.ID++ segmentIDsByIndex := createContiguousWalSegments(t, prereqs, databaseID, 1, 3, 6)++ spanStart := walmath.LSN(4*segmentBytes + firstRecordOffset)+ closerToSpanStart := walmath.LSN(4*segmentBytes + firstRecordOffset/2)physical_testing.CreateTestWalSegment(t,physical_testing.NewTestWalSegment(- databaseID, prereqs.storage.ID, 1, "000000010000000000000001",- walmath.LSN(segmentBytes), beforeGapEnd,+ databaseID, prereqs.storage.ID, 2, "000000020000000000000004",+ closerToSpanStart, closerToSpanStart+segmentBytes,))physical_testing.CreateTestWalSegment(t,physical_testing.NewTestWalSegment(- databaseID, prereqs.storage.ID, 1, "000000010000000000000003",- afterGapStart, walmath.LSN(5*segmentBytes),+ otherPrereqs.database.ID, otherPrereqs.storage.ID, 1, "000000010000000000000004",+ closerToSpanStart, closerToSpanStart+segmentBytes,))- service := chain_view.GetChainViewService()-- view, err := service.FindLastExtendableChainByDatabase(databaseID)+ spanSegments, err := physical_repositories.GetWalSegmentRepository().FindByChainSpan(+ databaseID, 1, spanStart, chain_view.LSNMax,+ )require.NoError(t, err)- require.NotNil(t, view)- assert.Equal(t, full.ID, view.RootFull.ID)- assert.Equal(t, beforeGapEnd, view.MaxReplayableLSN,- "PITR horizon stops at the end of contiguous WAL before the first gap")++ assert.Equal(t,+ []uuid.UUID{segmentIDsByIndex[4], segmentIDsByIndex[5]},+ getWalSegmentIDs(spanSegments),+ "segments of another timeline or database must not move the boundary segment",+ )+}++func createContiguousWalSegments(+ t *testing.T,+ prereqs *chainViewTestPrereqs,+ databaseID uuid.UUID,+ timelineID int,+ firstSegmentIndex, endSegmentIndex int,+) map[int]uuid.UUID {+ t.Helper()++ segmentIDsByIndex := make(map[int]uuid.UUID)++ for segmentIndex := firstSegmentIndex; segmentIndex < endSegmentIndex; segmentIndex++ {+ segment := physical_testing.CreateTestWalSegment(t,+ physical_testing.NewTestWalSegment(+ databaseID, prereqs.storage.ID, timelineID,+ fmt.Sprintf("%08X%016X", timelineID, segmentIndex),+ walmath.LSN(segmentIndex*segmentBytes), walmath.LSN((segmentIndex+1)*segmentBytes),+ ))++ segmentIDsByIndex[segmentIndex] = segment.ID+ }++ return segmentIDsByIndex+}++func getWalSegmentIDs(segments []*physical_models.PhysicalWalSegment) []uuid.UUID {+ segmentIDs := make([]uuid.UUID, 0, len(segments))+ for _, segment := range segments {+ segmentIDs = append(segmentIDs, segment.ID)+ }++ return segmentIDs}diff --git a/backend/internal/features/backups/backups/core/physical/chain_view/view.go b/backend/internal/features/backups/backups/core/physical/chain_view/view.goindex 3e7d7b90..a7162079 100644--- a/backend/internal/features/backups/backups/core/physical/chain_view/view.go+++ b/backend/internal/features/backups/backups/core/physical/chain_view/view.go@@ -5,33 +5,16 @@ import ("databasus-backend/internal/util/walmath")-// ChainView is one backup chain — its FULL, its INCRs and the WAL between.-// It represents a single chain from one FULL to the next FULL.-//-// LSN layout of the fields:-//-// Span: [════════════════════════════════════════]-// WalSegments: [seg1][seg2] [seg4][seg5]-// Gaps: [══════]-// MaxReplayableLSN: ↑ (end of seg2 — gap blocks further replay)+// ChainView is one backup chain — its FULL and its INCRs, from one FULL to the+// next. It carries no WAL: callers that need the chain's WAL look it up by Span.type ChainView struct {RootFull *physical_models.PhysicalFullBackupIncrementals []*physical_models.PhysicalIncrementalBackup- WalSegments []*physical_models.PhysicalWalSegment// Span is the LSN range owned by this chain: from RootFull.StartLSN up to// the next FULL's StartLSN (or LSNMax if no next FULL exists yet). Only// FULLs bound a chain — INCRs sit inside Span, not at its edge.Span LSNRange-- // Gaps are holes between WalSegments inside Span — a missing segment or a- // mid-chain timeline switch. The chain stays usable, but PITR into a gap- // is impossible.- Gaps []LSNRange-- // MaxReplayableLSN is the furthest LSN PITR can reach: RootFull.StopLSN- // plus every contiguous WAL segment up to (but not past) the first Gap.- MaxReplayableLSN walmath.LSN}// LSNMax is the unbounded upper sentinel for the latest chain on a timelinediff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.goindex 8dd0b4e6..820b9aed 100644--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go@@ -1,6 +1,7 @@package physical_repositoriesimport (+ "database/sql""errors""time"@@ -64,12 +65,7 @@ func (r *PhysicalWalSegmentRepository) FindByChainSpan() ([]*physical_models.PhysicalWalSegment, error) {var segments []*physical_models.PhysicalWalSegment- if err := storage.- GetDb().- Where(- "database_id = ? AND timeline_id = ? AND end_lsn > ?::pg_lsn AND start_lsn < ?::pg_lsn",- databaseID, timelineID, startLSN.String(), endLSN.String(),- ).+ if err := whereInChainSpan(storage.GetDb(), databaseID, timelineID, startLSN, endLSN).Order("start_lsn ASC").Find(&segments).Error; err != nil {return nil, err@@ -78,6 +74,59 @@ func (r *PhysicalWalSegmentRepository) FindByChainSpan(return segments, nil}+// GetLatestReceivedAtInChainSpan returns nil when no segment overlaps the span.+func (r *PhysicalWalSegmentRepository) GetLatestReceivedAtInChainSpan(+ databaseID uuid.UUID,+ timelineID int,+ startLSN, endLSN walmath.LSN,+) (*time.Time, error) {+ var latestReceivedAt sql.NullTime++ if err := whereInChainSpan(storage.GetDb(), databaseID, timelineID, startLSN, endLSN).+ Model(&physical_models.PhysicalWalSegment{}).+ Select("MAX(received_at)").+ Row().+ Scan(&latestReceivedAt); err != nil {+ return nil, err+ }++ if !latestReceivedAt.Valid {+ return nil, nil+ }++ return &latestReceivedAt.Time, nil+}++// whereInChainSpan selects the segments overlapping [startLSN, endLSN). The+// end_lsn filter alone leaves the start_lsn index bounded only from above, so+// every lookup would read the timeline's WAL from its first segment. Segments of+// one timeline never overlap, so any segment ending after startLSN starts at or+// after the segment holding startLSN, and that segment's start_lsn is a lower+// index bound that keeps the boundary segment in.+func whereInChainSpan(+ db *gorm.DB,+ databaseID uuid.UUID,+ timelineID int,+ startLSN, endLSN walmath.LSN,+) *gorm.DB {+ return db.Where(`+ database_id = ? AND timeline_id = ?+ AND start_lsn >= COALESCE(+ (SELECT boundary.start_lsn+ FROM physical_wal_segments boundary+ WHERE boundary.database_id = ? AND boundary.timeline_id = ? AND boundary.start_lsn <= ?::pg_lsn+ ORDER BY boundary.start_lsn DESC+ LIMIT 1),+ '0/0'::pg_lsn)+ AND start_lsn < ?::pg_lsn+ AND end_lsn > ?::pg_lsn`,+ databaseID, timelineID,+ databaseID, timelineID, startLSN.String(),+ endLSN.String(),+ startLSN.String(),+ )+}+// Anti-join not covered by idx_physical_wal_segments_database_id_received_at;// expect a seq scan on physical_full_backups when invoked in bulk.//diff --git a/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/.openspec.yaml b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/.openspec.yamlnew file mode 100644index 00000000..53f62d70--- /dev/null+++ b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/.openspec.yaml@@ -0,0 +1,2 @@+schema: spec-driven+created: 2026-10-10diff --git a/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/design.md b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/design.mdnew file mode 100644index 00000000..9af50143--- /dev/null+++ b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/design.md@@ -0,0 +1,131 @@+## Context++See proposal.md for the report and the measured load. Paths below are relative to `backend/internal/features/backups/backups/`.++Two background loops drive the load:++- The physical scheduler ticks every second (`backuping/physical/constants.go:16`). The comment above it (`constants.go:11`) assumes each tick is "a handful of cheap indexed queries per enabled DB". `decideBackupKind` calls `FindLastExtendableChainByDatabase` at `backuping/physical/scheduler.go:291`, before it checks the incremental cadence at `scheduler.go:300`, so the chain is built on every tick and discarded on almost all of them.+- The retention cleaner ticks every three seconds (`constants.go:29`). Each retention policy calls `nonExtendableChainsNewestEndFirst` (`backuping/physical/cleaner.go:119`), which builds every closed chain through `FindNonExtendableChainsByDatabase` and then calls `GetChainEndTimestamp` per chain (`cleaner.go:130`). `isChainWithinGrace` (`cleaner.go:225`) calls `GetChainEndTimestamp` a second time for every candidate it weighs.++A chain is built by the chain service in `core/physical/chain_view/service.go`:++- `getChainState` (`service.go:259`) loads the full backup by ID, then `findNewerCompletedFulls` (`service.go:294`) reloads every completed full backup of the database to look for a newer one, then loads the chain's incremental backups. It runs once per full backup, so building all chains reads the full backup list N times.+- `buildChainView` (`service.go:316`) computes the span, loads the incremental backups, loads the chain's WAL with `FindByChainSpan`, and calls `FindWalGapsInChain` (`service.go:128`), which recomputes the span and loads the same WAL again.+- `GetChainEndTimestamp` (`service.go:195`) reloads the full backup, the incremental backups and the span, and loads the chain's WAL a third time to take the newest `received_at`.++No production code reads the WAL that `buildChainView` loads. `ChainView.WalSegments`, `Gaps` and `MaxReplayableLSN` (`core/physical/chain_view/view.go:20-34`) are read only by tests. The scheduler and the backup service use the chain's root full backup; the cleaner uses the root full backup and the end time; restore uses the root full backup, the span start and the incremental backups, and loads its own WAL (`core/physical/chain_view/restore_set.go:61`).++The WAL lookup `FindByChainSpan` (`core/physical/repositories/wal_segment_repository.go:60`) filters on `end_lsn > start AND start_lsn < end`. The only index that matches is the unique key on `(database_id, timeline_id, start_lsn)` (`backend/migrations/20260523120000_add_physical_pg_17_backups.sql`), which bounds the scan from above only. Every lookup reads the database's WAL from its first segment up to the chain's end, and for the newest chain the end is `LSNMax`. Forced WAL rotation every five minutes produces at least 288 segments per database per day, so each lookup reads thousands of rows after a few weeks of retention.++Constraints from `backend/AGENTS.md`: tests live next to their source and follow `Test_WhatWeDo_WhichConditions_WhatWeExpect`; background services keep the `hasRun` guard; migrations, if any, use separate statements per index. Constraints from `AGENTS.md`: no backward-compatibility shims for internal callers; names state intent, with getters taking `Get`; comments carry only the reason the code cannot.++## Goals / Non-Goals++**Goals:**++- Scheduler and cleaner passes whose catalog reads do not depend on how many WAL segments a database retains.+- Each chain built with a constant number of indexed reads, and the full backup list read once per database per pass.+- No change to any scheduling, retention or restore outcome.++**Non-Goals:**++- Changing what the restore path, the backup list or the WAL range rows read, beyond the shared WAL lookup bound.+- Adding instrumentation or a query budget to the test suite.++## Decisions++### Decision 1: Ask the narrow question instead of materializing the answer++The background loops need two facts: which chain is extendable, and how old each closed chain is. The change makes those two questions cheap to answer from the catalog on every pass. It does not store chain state or chain end time anywhere.++Alternatives rejected:++- **Columns or triggers holding chain state and end time.** Every event that changes them would need to update them: a full backup completing, an incremental backup breaking the chain, a WAL segment arriving, retention deleting rows, a timeline switch. A missed update shows up as retention deleting a chain it should keep or keeping one it should delete. After this change the direct answer costs a few indexed reads, so the stored copy would buy nothing worth that risk.+- **An in-memory cache invalidated on writes.** It has the same invalidation problem, and it diverges as soon as a second process writes to the catalog.+- **Only lowering the tick frequency.** The per-pass cost would still grow with retention. Doubling the retained days would double the idle load again.++### Decision 2: The chain model drops WAL; chains are built from one list of full backups++`ChainView` keeps `RootFull`, `Incrementals` and `Span`, and loses `WalSegments`, `Gaps` and `MaxReplayableLSN`. `buildChainView` stops reading WAL. The view's doc diagram shrinks to the span. `FindWalGapsInChain` stays as a standalone method, because a test helper polls it (`usecases/physical/postgresql/testing.go:968`), and it keeps loading the chain's WAL once on its own.++`FindLastExtendableChainByDatabase` and `FindNonExtendableChainsByDatabase` read the completed full backups once, newest first. `getChainState` closes a chain whenever any newer completed full backup exists, so only the first entry of that list can be extendable. It is extendable when none of its incremental backups has status `CHAIN_BROKEN`, which the built view already holds. Every later entry is closed. `getChainState`, `findNewerCompletedFulls` and the `ChainState` enum are deleted. The rule that a WAL gap never changes chain state, today written on the enum (`core/physical/chain_view/enums.go:3-6`), moves to the check that decides extendability.++`GetChainSpan(rootFullBackupID)` today reloads the root full backup with `FindByID` before its successor query (`service.go:77-80`). Its body moves into a private `getChainSpanForFull(full)`, which `GetChainSpan` wraps after the lookup and `buildChainView` calls with the full backup it already holds. Building a chain then costs the successor query plus one read of its incremental backups.++`FindWalSegmentsInSpan` (`service.go:120`) has no production caller and is deleted. The two span tests that call it read through the repository instead.++Alternatives rejected:++- **Keep the WAL fields and load them lazily.** Lazy loading in a plain struct means a method with hidden I/O, and no caller asks for the data.+- **Compute every span in memory from the loaded full backup list.** It saves one indexed read per chain but duplicates the successor rule, including its timeline filter and its treatment of a missing `start_lsn`. Keeping one implementation is worth more than that read.+- **Batch every chain's incremental backups into one query.** It saves N small indexed reads per database per minute and adds a grouping step. The chain count per database is small, and the cleaner runs once a minute after this change.++A tie on `created_at` between two completed full backups is the one input where the new rule can differ: today both count as not closed by a newer backup. Two full backups of one database cannot complete in the same instant, because the scheduler serializes backups per database through the in-flight claim, so this case is not handled.++### Decision 3: A chain's end time is computed from the built chain, with one WAL aggregate++`GetChainEndTimestamp` takes the `*ChainView` the caller already holds. It takes the latest of the root full backup's `CompletedAt` and the incremental backups' `CompletedAt` from the view, and asks the WAL repository for one value: the newest `received_at` among the segments overlapping the chain's span. A new repository method, `GetLatestReceivedAtInChainSpan`, returns that value with `MAX(received_at)` under the same span condition as `FindByChainSpan`, and returns nil when the span holds no segment.++The cleaner computes each candidate's end time once in `nonExtendableChainsNewestEndFirst` and passes it to `isChainWithinGrace`, which takes the end time instead of a full backup ID. The second computation per candidate disappears. Today `isChainWithinGrace` treats a failure to read the end time as "within grace" (`cleaner.go:225`), and `nonExtendableChainsNewestEndFirst` already skips a chain whose end time fails to load (`cleaner.go:130`). A skipped chain is never deleted, so dropping the second read keeps the safety property.++Alternatives rejected:++- **Use the `received_at` of the last segment by position.** It is one index probe, but it equals the maximum only while segments arrive in position order. A segment re-received after its abandoned claim was reaped breaks that order, and the result would make a chain look older than it is to the grace check.+- **One query per database that joins every chain's span to the WAL.** A range join between spans and segments has no hash or merge strategy, so PostgreSQL would run it as a nested loop. It ends up doing the same work as one aggregate per chain, with a harder query to read.++### Decision 4: Bound the WAL span lookup from below through the segment that holds the span start++The span condition gains a lower bound on the indexed column. Segments of one database and timeline never overlap, which the orphan sweep already relies on (`cleaner.go:526`). Any segment that ends after the span start therefore starts at or after the start of the segment holding the span start, meaning the greatest `start_lsn` at or below it. The condition becomes:++```sql+database_id = ? AND timeline_id = ?+AND start_lsn >= COALESCE(+ (SELECT start_lsn FROM physical_wal_segments+ WHERE database_id = ? AND timeline_id = ? AND start_lsn <= ?::pg_lsn+ ORDER BY start_lsn DESC LIMIT 1),+ '0/0'::pg_lsn)+AND start_lsn < ?::pg_lsn+AND end_lsn > ?::pg_lsn+```++PostgreSQL evaluates the uncorrelated subquery once as an init plan and uses its result as the lower index bound. The scan covers the chain's segments and nothing older. The `end_lsn` filter stays, so the result set is identical to today's. `FindByChainSpan` and `GetLatestReceivedAtInChainSpan` share one private helper that applies this condition, so the two cannot drift.++Alternatives rejected:++- **An index on `(database_id, timeline_id, end_lsn)`.** It bounds the scan from below only, so a lookup for an old chain would read every newer segment instead of every older one. It also adds a write to the most frequently inserted catalog table and a migration.+- **A lower bound of `start - wal_segment_size`.** The segment size is a per-cluster setting from 1 MB to 1 GB, and the catalog does not store it per database.+- **A range type with a GiST index.** It answers overlap queries directly but needs a schema change and a new column kept in sync for a lookup that the init plan already bounds.++### Decision 5: The scheduler checks the cadence before it looks up the chain++In `decideBackupKind`, the `IncrementalBackupInterval.ShouldTriggerBackup` check moves above the `FindLastExtendableChainByDatabase` call. Nothing between them reads the chain: the cadence check uses only the newest full and incremental backups, which are already loaded. The forced-incremental branch above it keeps its own lookup, because a request must be honored regardless of cadence.++The scheduler tick stays at one second. With the cadence check first, an idle pass reads the newest full backup, the system identifier mismatch marker and the newest incremental backup, all through existing indexes. The comment at `constants.go:11` becomes true again and stays.++Alternative rejected: **a longer scheduler tick.** A user pressing "back up now" would wait up to the new interval, and the idle cost is already flat after this change.++### Decision 6: The retention cleaner runs once a minute++`cleanerTickInterval` becomes one minute and its comment states the reason: retention grace periods are at least an hour (`recentBackupGracePeriod`), abandoned WAL claims are reaped after an hour (`walClaimGracePeriod`), so a minute of extra delay changes no outcome, while each pass still reads every closed chain of every database and runs the orphan anti-join. The per-pass WAL deletion budget stays as it is.++Alternatives rejected:++- **Keep three seconds.** After Decisions 2 to 4 a pass is cheap, but it still scales with the number of chains, and nothing gains from running it twenty times a minute.+- **Five seconds.** It cuts load less and protects nothing that one minute does not.+- **Event-driven retention.** It needs a signal from every write path that can change retention eligibility, including the clock crossing a grace boundary. Polling at a minute is simpler and robust to restarts.++## Risks / Trade-offs++- [Two segments of one database and timeline overlap, for example after a database record is pointed at a different cluster with another segment size] → The lower bound could skip the older, larger segment. The orphan sweep already assumes no overlap, and automatic scheduling stops after a system identifier mismatch, so such a catalog is already outside supported behavior. A repository test pins the boundary segment case that does occur: a full backup starting partway into a segment.+- [Large WAL backlog deletes more slowly] → A chain whose WAL exceeds one pass's budget now takes several minutes to delete instead of several seconds. The budget is one full backup's size per pass, which outpaces WAL production by a wide margin, so storage still shrinks.+- [Tests relied on the removed fields] → `Test_FindLastExtendableChainByDatabase_WhenChainHasGap_MaxReplayableLSNStopsAtFirstGap` tests a field that no longer exists and is deleted. Its gap behavior stays covered by the `FindWalGapsInChain` tests and by the restore set's own `MaxReplayableLSN` tests. The assertions on `WalSegments` and `Gaps` in the healthy-chain test are removed.+- [No automated guard against the load returning] → No test measures load or query counts; the tests pin outcomes only. The guard is structural: the chain model has no WAL field, so no caller can load WAL through it by accident, and the end time path exposes only an aggregate. A query-count test was rejected because it needs new GORM callback plumbing in shared test setup and breaks on unrelated query additions. A plan-shape test on the WAL lookup was considered and left out by decision: this change pins correctness, not performance.+- [Three spec scenarios have no test] → They are confirmed by code structure and review instead:+ - A scheduling pass with nothing due reads no chain history. This holds because the cadence check sits above the chain lookup in `decideBackupKind` (Decision 5); observing the absent read would need the query counting rejected above.+ - Retention runs once a minute. This is the value of `cleanerTickInterval`; a test of a constant checks nothing.+ - Each closed chain's end time is computed once per pass. This holds because `isChainWithinGrace` takes the end time as an argument and has no way to compute it (Decision 3).+- [The bounded WAL lookup decides which WAL a restore ships] → Repository tests pin the boundary cases: a full backup starting partway into a segment, a span starting exactly on a segment boundary, a span starting before the first archived segment, and segments of other timelines and databases at nearby LSNs. The backup and restore cycle packages for PostgreSQL 17 and 18 run as the final check.++## Migration Plan++No schema or data migration. The change ships with the next release; rollback is a revert of the code.diff --git a/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/proposal.md b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/proposal.mdnew file mode 100644index 00000000..3ae773f4--- /dev/null+++ b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/proposal.md@@ -0,0 +1,42 @@+## Why++A Databasus instance with physical PostgreSQL backups keeps one CPU core about 60% busy while no backup is running. A user with four databases measured about 440 committed transactions per second on the internal database, nearly all of them reads of full backups, incremental backups and WAL segments. The load does not depend on WAL streaming being on, comes back within minutes of a restart, and grows with every day of retained WAL.++The cause is the shape of the reads, not their frequency alone. Every second, the physical backup scheduler asks whether each database has a chain it can extend. Every three seconds, the retention cleaner asks how old each closed chain is. Both questions are answered by building the complete chain model: every chain's full backup list is reloaded once per chain, and every WAL segment row of every chain is loaded twice, only to be thrown away. Neither loop uses those WAL rows. The lookup that loads them is bounded on one side only, so each call reads the WAL from the start of the database's history up to the chain's end.++## What Changes++- The chain model carries a chain's full backup, its incremental backups and its LSN span. It no longer loads WAL segments, WAL gaps or the furthest replayable LSN. Production code never read those three fields; the restore path loads its own WAL.+- Chain state comes from one list of completed full backups. Only the newest full backup can head an extendable chain, so state is decided without reloading the list for every chain.+- A chain's end time is computed from the chain the cleaner already holds. The WAL part is a single aggregate over the chain's span, not a load of every row in it, and the cleaner computes it once per chain per pass instead of twice.+- The WAL span lookup is bounded on both sides, so it reads the segments of the requested chain and nothing older. No new index or migration is needed.+- The scheduler checks the incremental cadence before it looks up the extendable chain, so a pass with nothing due reads no chain data.+- The retention cleaner runs once a minute instead of every three seconds.+- **BREAKING (internal only):** the chain model loses its WAL segment, gap and replayable-LSN fields; the unused span-segment lookup on the chain service, the chain state enum and the per-chain state helper are deleted; the chain end time method takes a built chain instead of a full backup ID. No HTTP API, database schema or configuration changes.++## Capabilities++### New Capabilities++- `physical-backups/background-load`: how much catalog work the physical backup scheduler and retention cleaner do while idle, how often retention runs, and that a chain's WAL lookup reads only that chain's WAL.++### Modified Capabilities++None. Retention decisions, the grace period, chain end time, restore selection and the next-backup times reported to the interface keep their current behavior.++## Out of scope++- The scheduler tick interval. It stays at one second so out-of-cadence backup requests and crash recovery stay near-immediate; after this change a pass with nothing due is a few indexed reads per database.+- The orphan WAL sweep query, which already expects a sequential scan of full backups. It runs on the slower cleaner cadence and is not otherwise changed.+- Materializing chain state or chain end time in columns, triggers or an in-memory cache.+- Autovacuum settings of the bundled PostgreSQL and the shutdown grace period of the container, both mentioned in the report.+- The logical backup cleaner, which has its own three-second cadence and different costs.+- Other unused chain service methods that this change does not touch, such as the history file presence check.+- The backup list, WAL range rows and restore download, apart from the WAL lookup bound they share.++## Impact++- Backend only: `backend/internal/features/backups/backups/core/physical/chain_view` (chain model, chain service and its tests), `backend/internal/features/backups/backups/core/physical/repositories` (WAL segment span lookup and a new span aggregate), and `backend/internal/features/backups/backups/backuping/physical` (scheduler, cleaner and their cadence constants).+- Answers to `AGENTS.md` and `backend/AGENTS.md`.+- No security impact: no authorization path, credential or stored data changes.+- Trade-off: a chain that becomes eligible for deletion starts being deleted up to a minute later than today, and a chain whose WAL exceeds one pass's deletion budget finishes over several minutes instead of several seconds. Retention grace periods are at least an hour, so no restore point is affected.diff --git a/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/specs/physical-backups/background-load/spec.md b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/specs/physical-backups/background-load/spec.mdnew file mode 100644index 00000000..52b5b892--- /dev/null+++ b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/specs/physical-backups/background-load/spec.md@@ -0,0 +1,90 @@+## Purpose++Bounds the catalog work that physical backup scheduling and retention do in the background, so an idle instance stays idle no matter how many chains and how much archived WAL it retains, while every scheduling, retention and restore decision stays the same.++## ADDED Requirements++### Requirement: A scheduling pass with nothing due does not read chain history++When the scheduler evaluates a physical database and neither a full backup, an incremental backup nor an out-of-cadence request is due, it SHALL read only the newest full backup record, the newest incremental backup record and the system identifier mismatch check of that database, and SHALL NOT read archived WAL segments or build any chain. The work of such a pass SHALL NOT grow with the number of retained chains or archived WAL segments.++#### Scenario: Idle database with weeks of retained WAL++- **WHEN** a physical database retains many chains and thousands of archived WAL segments, and no backup is due+- **THEN** the scheduler's pass for that database reads no archived WAL segment and builds no chain+- **AND** its reads are limited to the newest full backup, the newest incremental backup and the system identifier mismatch check++#### Scenario: Incremental cadence becomes due++- **WHEN** the incremental cadence of a database with an extendable chain becomes due+- **THEN** the scheduler starts an incremental backup on the newest chain, as it does today+- **AND** it decides this without reading the chain's archived WAL segments++### Requirement: Only the newest completed full backup can head an extendable chain++A chain SHALL be extendable only when its full backup is the newest successfully completed full backup of the database and none of the chain's incremental backups ended with a broken chain. Every older chain SHALL be closed by its newer full backup. WAL gaps inside a chain SHALL NOT change its state.++#### Scenario: Newest chain is healthy++- **WHEN** the newest completed full backup has no incremental backup with a broken chain+- **THEN** that chain is the extendable chain+- **AND** every older chain is reported as closed++#### Scenario: Newest chain is broken by an incremental backup++- **WHEN** an incremental backup of the newest chain ended with a broken chain+- **THEN** the database has no extendable chain+- **AND** the newest chain is listed with the closed chains that retention and restore consider++#### Scenario: Newest chain has a WAL gap++- **WHEN** the newest chain is missing a WAL segment in the middle of its span+- **THEN** the chain stays extendable++### Requirement: Retention runs once a minute and reads each closed chain's WAL once++The retention cleaner SHALL run once a minute. To decide which closed chains are eligible for deletion and in what order, including the grace check, it SHALL read each closed chain's archived WAL at most once per pass, as a summary rather than as individual segment records, and SHALL NOT read WAL outside that chain's span. Deleting an eligible chain and sweeping WAL that belongs to no chain are outside this rule and keep reading individual segments.++#### Scenario: Idle instance with retained chains++- **WHEN** a retention pass runs and no chain is eligible for deletion+- **THEN** no individual archived WAL segment record is loaded to decide that++#### Scenario: A chain becomes eligible for deletion++- **WHEN** a closed chain falls outside the configured retention and outside the grace period+- **THEN** its deletion begins within about one minute++### Requirement: A chain's end time is unchanged++A chain's end time, which drives retention ordering and the grace period, SHALL be the latest of three moments: the completion of its full backup, the completion of any of its incremental backups, and the receipt of any archived WAL segment inside its span.++#### Scenario: WAL arrives after the last incremental backup++- **WHEN** a chain's newest archived WAL segment was received after its last incremental backup completed+- **THEN** the chain's end time is the receipt time of that segment++#### Scenario: Chain without archived WAL++- **WHEN** a chain has no archived WAL segment in its span+- **THEN** the chain's end time is the latest completion time among its full and incremental backups++### Requirement: A chain's WAL lookup returns exactly the chain's segments++Looking up the archived WAL of a chain SHALL return every segment that overlaps the chain's span, ordered by position, and SHALL NOT read segments that end at or before the span begins. A segment that starts before the span but carries data from the span's start SHALL be included.++#### Scenario: Older chain among many++- **WHEN** the WAL of a chain in the middle of a database's history is looked up+- **THEN** the result holds that chain's segments only+- **AND** no segment of an earlier chain is read++#### Scenario: Full backup starts in the middle of a segment++- **WHEN** a chain's full backup starts partway into an archived segment+- **THEN** that segment is included in the chain's WAL++#### Scenario: Newest chain++- **WHEN** the WAL of the newest chain is looked up+- **THEN** the result runs from the segment holding the chain's start to the newest archived segmentdiff --git a/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/tasks.md b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/tasks.mdnew file mode 100644index 00000000..1ffced0f--- /dev/null+++ b/openspec/changes/archive/2026-10-10-reduce-physical-idle-load/tasks.md@@ -0,0 +1,33 @@+Paths are relative to `backend/internal/features/backups/backups/`. Test commands run from `backend/`.++## 1. Bounded WAL span lookup++- [x] 1.1 In `core/physical/repositories/wal_segment_repository.go`, move the span condition of `FindByChainSpan` into a private helper that adds the lower bound from design Decision 4: `start_lsn` at or above the greatest `start_lsn` at or below the span start, falling back to `'0/0'`. Keep the `end_lsn` filter and the `start_lsn ASC` order. Verify with `go test ./internal/features/backups/backups/core/physical/... -count=1`; the existing boundary-segment tests in `chain_view/restore_set_test.go` and `backuping/physical/cleaner_test.go` must keep passing.+- [x] 1.2 Add `GetLatestReceivedAtInChainSpan(databaseID, timelineID, startLSN, endLSN) (*time.Time, error)` beside `FindByChainSpan`, returning `MAX(received_at)` under the same helper and nil when the span holds no segment. Verify with `go build ./...`.+- [x] 1.3 In `core/physical/chain_view/service_test.go`, add `Test_FindByChainSpan_ForMiddleChain_ReturnsOnlyItsSegmentsIncludingBoundary`: three chains on one timeline, the middle full backup starting partway into a segment; the lookup for the middle span returns that boundary segment and the middle chain's later segments, and no segment of the first or third chain. Verify with `go test ./internal/features/backups/backups/core/physical/chain_view/... -run Test_FindByChainSpan -count=1`.+- [x] 1.4 Add `Test_FindByChainSpan_WhenSpanStartsOnSegmentBoundary_ExcludesSegmentEndingThere` to the same file: the span starts exactly where one segment ends and the next begins; the lookup returns the segment starting at the span start and not the one ending there. Verify with the same command as 1.3.+- [x] 1.5 Add `Test_FindByChainSpan_WhenOtherTimelineAndDatabaseHaveSegmentsInRange_IgnoresThem` to the same file: segments of a second timeline and of a second database sit at LSNs inside and just below the span start; the lookup returns only the requested database's segments on the requested timeline, with the same boundary segment as without them. Verify with the same command as 1.3.++## 2. Chain model without WAL++- [x] 2.1 Remove `WalSegments`, `Gaps` and `MaxReplayableLSN` from `ChainView` in `core/physical/chain_view/view.go`, shrink its doc diagram to the span, and stop `buildChainView` from loading WAL or calling `FindWalGapsInChain`. Delete `FindWalSegmentsInSpan`. Update `service_test.go`: drop the `WalSegments` and `Gaps` assertions from the healthy-chain test, delete `Test_FindLastExtendableChainByDatabase_WhenChainHasGap_MaxReplayableLSNStopsAtFirstGap`, and make the two span tests read through the repository. Verify with `go build ./...` and `go test ./internal/features/backups/backups/core/physical/chain_view/... -count=1`.+- [x] 2.2 Rewrite `FindLastExtendableChainByDatabase` and `FindNonExtendableChainsByDatabase` on one read of the completed full backups, per design Decision 2: only the newest can be extendable, and only when none of its incremental backups is `CHAIN_BROKEN`. Move the body of `GetChainSpan` into a private `getChainSpanForFull(full)` that `GetChainSpan` wraps and `buildChainView` calls with the full backup it holds. Delete `getChainState`, `findNewerCompletedFulls` and the `ChainState` enum, and move the rule that WAL gaps never change chain state into a comment on the extendability check. Verify with `go test ./internal/features/backups/backups/core/physical/chain_view/... -count=1`; the existing extendable, broken and newer-full tests must pass unchanged.+- [x] 2.3 Add `Test_FindNonExtendableChainsByDatabase_WhenNewestChainBroken_ListsItWithOlderChains` to `service_test.go`: with two completed full backups and a `CHAIN_BROKEN` incremental on the newer one, the extendable lookup returns nil and the closed list holds both chains, newest first. Verify with `go test ./internal/features/backups/backups/core/physical/chain_view/... -run Test_FindNonExtendableChainsByDatabase -count=1`.++## 3. Chain end time from the built chain++- [x] 3.1 Change `GetChainEndTimestamp` to take a `*ChainView` and compute the latest of the root full backup's completion, the incremental backups' completion and `GetLatestReceivedAtInChainSpan` over the view's span, per design Decision 3. Update `Test_GetChainEndTimestamp_WhenWalReceivedAfterIncr_ReturnsWalReceivedAt` to build the view first, and add `Test_GetChainEndTimestamp_WhenChainHasNoWal_ReturnsLatestBackupCompletion`. Verify with `go test ./internal/features/backups/backups/core/physical/chain_view/... -run Test_GetChainEndTimestamp -count=1`.+- [x] 3.2 In `backuping/physical/cleaner.go`, pass each view to `GetChainEndTimestamp` in `nonExtendableChainsNewestEndFirst`, and change `isChainWithinGrace` to take the candidate's end time instead of a full backup ID, so each chain's end time is computed once per pass. Verify with `go test ./internal/features/backups/backups/backuping/physical/... -run "Test_Clean" -count=1`; the grace and keep-count tests must pass unchanged.++## 4. Background cadence++- [x] 4.1 In `decideBackupKind` in `backuping/physical/scheduler.go`, move the `IncrementalBackupInterval.ShouldTriggerBackup` check above the `FindLastExtendableChainByDatabase` call, per design Decision 5. Verify with `go test ./internal/features/backups/backups/backuping/physical/... -run Test_DecideBackupKind -count=1`.+- [x] 4.2 In `backuping/physical/constants.go`, set `cleanerTickInterval` to one minute and rewrite its comment with the reason from design Decision 6. Verify with `go test ./internal/features/backups/backups/backuping/physical/... -count=1`.++## 5. Verification++- [x] 5.1 Confirm that no reference to the removed names is left: `grep -rnE "FindWalSegmentsInSpan|ChainState|getChainState|findNewerCompletedFulls|\bview\.(WalSegments|Gaps|MaxReplayableLSN)" backend/` returns nothing.+- [x] 5.2 Run `just lint` in `backend/` and fix every finding in the touched files.+- [x] 5.3 Run the touched packages and confirm that they pass: `go test ./internal/features/backups/backups/core/physical/... ./internal/features/backups/backups/backuping/physical/... ./internal/features/backups/backups/services/... -count=1`.+- [x] 5.4 Run the physical backup and restore cycle packages, which prove end to end that restore still ships the right WAL through the bounded lookup: `go test ./internal/features/tests/physical/postgresql/pg17/... ./internal/features/tests/physical/postgresql/pg18/... -count=1 -timeout 60m`.+- [x] 5.5 Run `openspec validate reduce-physical-idle-load --strict` and confirm that it passes.diff --git a/openspec/specs/physical-backups/background-load/spec.md b/openspec/specs/physical-backups/background-load/spec.mdnew file mode 100644index 00000000..e12b529e--- /dev/null+++ b/openspec/specs/physical-backups/background-load/spec.md@@ -0,0 +1,91 @@+# physical-backups/background-load Specification++## Purpose+Bounds the catalog work that physical backup scheduling and retention do in the background, so an idle instance stays idle no matter how many chains and how much archived WAL it retains, while every scheduling, retention and restore decision stays the same.++## Requirements++### Requirement: A scheduling pass with nothing due does not read chain history++When the scheduler evaluates a physical database and neither a full backup, an incremental backup nor an out-of-cadence request is due, it SHALL read only the newest full backup record, the newest incremental backup record and the system identifier mismatch check of that database, and SHALL NOT read archived WAL segments or build any chain. The work of such a pass SHALL NOT grow with the number of retained chains or archived WAL segments.++#### Scenario: Idle database with weeks of retained WAL++- **WHEN** a physical database retains many chains and thousands of archived WAL segments, and no backup is due+- **THEN** the scheduler's pass for that database reads no archived WAL segment and builds no chain+- **AND** its reads are limited to the newest full backup, the newest incremental backup and the system identifier mismatch check++#### Scenario: Incremental cadence becomes due++- **WHEN** the incremental cadence of a database with an extendable chain becomes due+- **THEN** the scheduler starts an incremental backup on the newest chain, as it does today+- **AND** it decides this without reading the chain's archived WAL segments++### Requirement: Only the newest completed full backup can head an extendable chain++A chain SHALL be extendable only when its full backup is the newest successfully completed full backup of the database and none of the chain's incremental backups ended with a broken chain. Every older chain SHALL be closed by its newer full backup. WAL gaps inside a chain SHALL NOT change its state.++#### Scenario: Newest chain is healthy++- **WHEN** the newest completed full backup has no incremental backup with a broken chain+- **THEN** that chain is the extendable chain+- **AND** every older chain is reported as closed++#### Scenario: Newest chain is broken by an incremental backup++- **WHEN** an incremental backup of the newest chain ended with a broken chain+- **THEN** the database has no extendable chain+- **AND** the newest chain is listed with the closed chains that retention and restore consider++#### Scenario: Newest chain has a WAL gap++- **WHEN** the newest chain is missing a WAL segment in the middle of its span+- **THEN** the chain stays extendable++### Requirement: Retention runs once a minute and reads each closed chain's WAL once++The retention cleaner SHALL run once a minute. To decide which closed chains are eligible for deletion and in what order, including the grace check, it SHALL read each closed chain's archived WAL at most once per pass, as a summary rather than as individual segment records, and SHALL NOT read WAL outside that chain's span. Deleting an eligible chain and sweeping WAL that belongs to no chain are outside this rule and keep reading individual segments.++#### Scenario: Idle instance with retained chains++- **WHEN** a retention pass runs and no chain is eligible for deletion+- **THEN** no individual archived WAL segment record is loaded to decide that++#### Scenario: A chain becomes eligible for deletion++- **WHEN** a closed chain falls outside the configured retention and outside the grace period+- **THEN** its deletion begins within about one minute++### Requirement: A chain's end time is unchanged++A chain's end time, which drives retention ordering and the grace period, SHALL be the latest of three moments: the completion of its full backup, the completion of any of its incremental backups, and the receipt of any archived WAL segment inside its span.++#### Scenario: WAL arrives after the last incremental backup++- **WHEN** a chain's newest archived WAL segment was received after its last incremental backup completed+- **THEN** the chain's end time is the receipt time of that segment++#### Scenario: Chain without archived WAL++- **WHEN** a chain has no archived WAL segment in its span+- **THEN** the chain's end time is the latest completion time among its full and incremental backups++### Requirement: A chain's WAL lookup returns exactly the chain's segments++Looking up the archived WAL of a chain SHALL return every segment that overlaps the chain's span, ordered by position, and SHALL NOT read segments that end at or before the span begins. A segment that starts before the span but carries data from the span's start SHALL be included.++#### Scenario: Older chain among many++- **WHEN** the WAL of a chain in the middle of a database's history is looked up+- **THEN** the result holds that chain's segments only+- **AND** no segment of an earlier chain is read++#### Scenario: Full backup starts in the middle of a segment++- **WHEN** a chain's full backup starts partway into an archived segment+- **THEN** that segment is included in the chain's WAL++#### Scenario: Newest chain++- **WHEN** the WAL of the newest chain is looked up+- **THEN** the result runs from the segment holding the chain's start to the newest archived segment