PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Stop endless basebackup loop from issue #533
Rostislav Dugin643cb0479c85
diff --git a/agent/internal/features/full_backup/backuper.go b/agent/internal/features/full_backup/backuper.goindex 1e122214..7526364b 100644--- a/agent/internal/features/full_backup/backuper.go+++ b/agent/internal/features/full_backup/backuper.go@@ -88,6 +88,20 @@ func (backuper *FullBackuper) checkAndRunIfNeeded(ctx context.Context) {return}+ // Fetch next-time first: a nil response is the backend's signal that backups are disabled,+ // and we must short-circuit before the chain-validity check (which would otherwise trigger+ // a basebackup on `error=no_full_backup`).+ nextTimeResp, err := backuper.apiClient.GetNextFullBackupTime(ctx)+ if err != nil {+ backuper.log.Error("Failed to check next full backup time", "error", err)+ return+ }++ if nextTimeResp.NextFullBackupTime == nil {+ backuper.log.Debug("Backups disabled, skipping basebackup check")+ return+ }+chainResp, err := backuper.apiClient.CheckWalChainValidity(ctx)if err != nil {backuper.log.Error("Failed to check WAL chain validity", "error", err)@@ -105,13 +119,7 @@ func (backuper *FullBackuper) checkAndRunIfNeeded(ctx context.Context) {return}- nextTimeResp, err := backuper.apiClient.GetNextFullBackupTime(ctx)- if err != nil {- backuper.log.Error("Failed to check next full backup time", "error", err)- return- }-- if nextTimeResp.NextFullBackupTime == nil || !nextTimeResp.NextFullBackupTime.After(time.Now().UTC()) {+ if !nextTimeResp.NextFullBackupTime.After(time.Now().UTC()) {backuper.log.Info("Scheduled full backup is due, triggering basebackup")backuper.runBasebackupWithRetry(ctx)diff --git a/agent/internal/features/full_backup/backuper_test.go b/agent/internal/features/full_backup/backuper_test.goindex 610b3fb2..79f17e31 100644--- a/agent/internal/features/full_backup/backuper_test.go+++ b/agent/internal/features/full_backup/backuper_test.go@@ -40,7 +40,7 @@ func Test_RunFullBackup_WhenChainBroken_BasebackupTriggered(t *testing.T) {var finalizeReceived boolvar finalizeBody map[string]any- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {case testChainValidPath:writeJSON(w, api.WalChainValidityResponse{@@ -66,7 +66,7 @@ func Test_RunFullBackup_WhenChainBroken_BasebackupTriggered(t *testing.T) {default:w.WriteHeader(http.StatusNotFound)}- })+ }))fb := newTestFullBackuper(server.URL)fb.cmdBuilder = mockCmdBuilder(t, "test-backup-data", validStderr())@@ -145,7 +145,7 @@ func Test_RunFullBackup_WhenNoFullBackupExists_ImmediateBasebackupTriggered(t *tvar mu sync.Mutexvar finalizeReceived bool- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {case testChainValidPath:writeJSON(w, api.WalChainValidityResponse{@@ -164,7 +164,7 @@ func Test_RunFullBackup_WhenNoFullBackupExists_ImmediateBasebackupTriggered(t *tdefault:w.WriteHeader(http.StatusNotFound)}- })+ }))fb := newTestFullBackuper(server.URL)fb.cmdBuilder = mockCmdBuilder(t, "first-backup-data", validStderr())@@ -191,7 +191,7 @@ func Test_RunFullBackup_WhenUploadFails_RetriesAfterDelay(t *testing.T) {var uploadAttempts intvar errorReported bool- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {case testChainValidPath:writeJSON(w, api.WalChainValidityResponse{@@ -224,7 +224,7 @@ func Test_RunFullBackup_WhenUploadFails_RetriesAfterDelay(t *testing.T) {default:w.WriteHeader(http.StatusNotFound)}- })+ }))fb := newTestFullBackuper(server.URL)fb.cmdBuilder = mockCmdBuilder(t, "retry-backup-data", validStderr())@@ -292,7 +292,7 @@ func Test_RunFullBackup_WhenAlreadyRunning_SkipsExecution(t *testing.T) {}func Test_RunFullBackup_WhenContextCancelled_StopsCleanly(t *testing.T) {- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {case testChainValidPath:writeJSON(w, api.WalChainValidityResponse{@@ -309,7 +309,7 @@ func Test_RunFullBackup_WhenContextCancelled_StopsCleanly(t *testing.T) {default:w.WriteHeader(http.StatusNotFound)}- })+ }))fb := newTestFullBackuper(server.URL)fb.cmdBuilder = mockCmdBuilder(t, "data", validStderr())@@ -376,7 +376,7 @@ func Test_RunFullBackup_WhenStderrParsingFails_FinalizesWithErrorAndRetries(t *tvar finalizeWithErrorReceived boolvar finalizeBody map[string]any- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {case testChainValidPath:writeJSON(w, api.WalChainValidityResponse{@@ -402,7 +402,7 @@ func Test_RunFullBackup_WhenStderrParsingFails_FinalizesWithErrorAndRetries(t *tdefault:w.WriteHeader(http.StatusNotFound)}- })+ }))fb := newTestFullBackuper(server.URL)fb.cmdBuilder = mockCmdBuilder(t, "data", "pg_basebackup: unexpected output with no LSN info")@@ -431,24 +431,29 @@ func Test_RunFullBackup_WhenStderrParsingFails_FinalizesWithErrorAndRetries(t *tassert.NotNil(t, finalizeBody["error"], "finalize should include error message")}-func Test_RunFullBackup_WhenNextBackupTimeNull_BasebackupTriggered(t *testing.T) {- var mu sync.Mutex- var finalizeReceived bool+func Test_RunFullBackup_WhenBackupsDisabled_NoBasebackupTriggered(t *testing.T) {+ var chainCalled atomic.Bool+ var uploadCalled atomic.Boolserver := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {- case testChainValidPath:- writeJSON(w, api.WalChainValidityResponse{IsValid: true})case testNextBackupTimePath:writeJSON(w, api.NextFullBackupTimeResponse{NextFullBackupTime: nil})+ case testChainValidPath:+ // Reproduces the issue #533 condition: even if the chain reports+ // no_full_backup, the disabled gate must fire first and prevent any basebackup.+ chainCalled.Store(true)++ writeJSON(w, api.WalChainValidityResponse{+ IsValid: false,+ Error: "no_full_backup",+ })case testFullStartPath:+ uploadCalled.Store(true)+_, _ = io.ReadAll(r.Body)writeJSON(w, map[string]string{"backupId": testBackupID})case testFullCompletePath:- mu.Lock()- finalizeReceived = true- mu.Unlock()-w.WriteHeader(http.StatusOK)default:w.WriteHeader(http.StatusNotFound)@@ -456,29 +461,25 @@ func Test_RunFullBackup_WhenNextBackupTimeNull_BasebackupTriggered(t *testing.T)})fb := newTestFullBackuper(server.URL)- fb.cmdBuilder = mockCmdBuilder(t, "first-run-data", validStderr())+ fb.cmdBuilder = mockCmdBuilder(t, "should-not-run", validStderr())- ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)defer cancel()go fb.Run(ctx)- waitForCondition(t, func() bool {- mu.Lock()- defer mu.Unlock()- return finalizeReceived- }, 5*time.Second)+ time.Sleep(500 * time.Millisecond)cancel()- mu.Lock()- defer mu.Unlock()-- assert.True(t, finalizeReceived)+ assert.False(t, chainCalled.Load(),+ "chain validity must not be called when backups are disabled")+ assert.False(t, uploadCalled.Load(),+ "no basebackup upload must happen when backups are disabled")}func Test_RunFullBackup_WhenChainValidityReturns401_NoBasebackupTriggered(t *testing.T) {var uploadReceived atomic.Bool- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {case testChainValidPath:w.WriteHeader(http.StatusUnauthorized)@@ -493,7 +494,7 @@ func Test_RunFullBackup_WhenChainValidityReturns401_NoBasebackupTriggered(t *tesdefault:w.WriteHeader(http.StatusNotFound)}- })+ }))fb := newTestFullBackuper(server.URL)fb.cmdBuilder = mockCmdBuilder(t, "data", validStderr())@@ -512,7 +513,7 @@ func Test_RunFullBackup_WhenUploadSucceeds_BodyIsZstdCompressed(t *testing.T) {var mu sync.Mutexvar receivedBody []byte- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {case testChainValidPath:writeJSON(w, api.WalChainValidityResponse{@@ -532,7 +533,7 @@ func Test_RunFullBackup_WhenUploadSucceeds_BodyIsZstdCompressed(t *testing.T) {default:w.WriteHeader(http.StatusNotFound)}- })+ }))originalContent := "test-backup-content-for-compression-check"fb := newTestFullBackuper(server.URL)@@ -702,6 +703,26 @@ pg_basebackup: syncing data to disk ...pg_basebackup: base backup completed`}+func writeEnabledFutureTime(w http.ResponseWriter) {+ futureTime := time.Now().UTC().Add(24 * time.Hour)+ writeJSON(w, api.NextFullBackupTimeResponse{NextFullBackupTime: &futureTime})+}++// withEnabledNextTime makes the test server respond to the next-full-backup-time endpoint+// with "enabled, ~24h away" so tests that exercise other branches don't have to repeat+// the boilerplate. Tests that care about a specific next-time (past, nil) handle it inline+// and skip this wrapper.+func withEnabledNextTime(inner http.HandlerFunc) http.HandlerFunc {+ return func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == testNextBackupTimePath {+ writeEnabledFutureTime(w)+ return+ }++ inner(w, r)+ }+}+func writeJSON(w http.ResponseWriter, v any) {w.Header().Set("Content-Type", "application/json")diff --git a/backend/internal/features/backups/backups/controllers/postgres_wal_controller_test.go b/backend/internal/features/backups/backups/controllers/postgres_wal_controller_test.goindex 5255a906..d505020a 100644--- a/backend/internal/features/backups/backups/controllers/postgres_wal_controller_test.go+++ b/backend/internal/features/backups/backups/controllers/postgres_wal_controller_test.go@@ -538,10 +538,12 @@ func Test_ReportError_WithMissingErrorField_ReturnsBadRequest(t *testing.T) {assert.Equal(t, http.StatusBadRequest, w.Code)}-func Test_GetNextFullBackupTime_WithValidToken_NoFullBackup_ReturnsNull(t *testing.T) {+func Test_GetNextFullBackupTime_NoFullBackup_ReturnsNowSoAgentRunsImmediately(t *testing.T) {router, db, storage, agentToken, _ := createWalTestSetup(t)defer removeWalTestSetup(db, storage)+ now := time.Now().UTC()+var response backups_dto.GetNextFullBackupTimeResponsetest_utils.MakeGetRequestAndUnmarshal(t,@@ -552,7 +554,53 @@ func Test_GetNextFullBackupTime_WithValidToken_NoFullBackup_ReturnsNull(t *testi&response,)- assert.Nil(t, response.NextFullBackupTime, "should be nil when no full backup exists")+ require.NotNil(t, response.NextFullBackupTime,+ "nil is reserved for disabled backups; first run must return a concrete time")+ assert.WithinDuration(t, now, response.NextFullBackupTime.UTC(), 5*time.Second,+ "first run should return ~now so the agent triggers immediately")+}++func Test_GetNextFullBackupTime_WhenBackupsDisabled_ReturnsNull(t *testing.T) {+ router, db, storage, agentToken, ownerToken := createWalTestSetup(t)+ defer removeWalTestSetup(db, storage)++ setBackupsEnabled(t, router, db.ID, ownerToken, false)++ var response backups_dto.GetNextFullBackupTimeResponse+ test_utils.MakeGetRequestAndUnmarshal(+ t,+ router,+ "/api/v1/backups/postgres/wal/next-full-backup-time",+ agentToken,+ http.StatusOK,+ &response,+ )++ assert.Nil(t, response.NextFullBackupTime,+ "disabled backups must return nil so the agent skips this cycle")+}++func Test_GetNextFullBackupTime_WhenBackupsDisabledWithExistingFullBackup_ReturnsNull(t *testing.T) {+ router, db, storage, agentToken, ownerToken := createWalTestSetup(t)+ defer removeWalTestSetup(db, storage)++ uploadBasebackup(t, router, agentToken,+ "000000010000000100000001", "000000010000000100000010")++ setBackupsEnabled(t, router, db.ID, ownerToken, false)++ var response backups_dto.GetNextFullBackupTimeResponse+ test_utils.MakeGetRequestAndUnmarshal(+ t,+ router,+ "/api/v1/backups/postgres/wal/next-full-backup-time",+ agentToken,+ http.StatusOK,+ &response,+ )++ assert.Nil(t, response.NextFullBackupTime,+ "disabled backups must return nil even when a prior full backup exists")}func Test_GetNextFullBackupTime_WithValidToken_HasFullBackup_ReturnsTime(t *testing.T) {@@ -1566,12 +1614,12 @@ func updateLastFullBackupTime(t *testing.T, databaseID uuid.UUID, createdAt time}}-func setEncryption(+func mutateBackupConfig(t *testing.T,router *gin.Engine,databaseID uuid.UUID,ownerToken string,- encryption backups_config.BackupEncryption,+ mutate func(*backups_config.BackupConfig),) {t.Helper()@@ -1583,7 +1631,7 @@ func setEncryption(http.StatusOK, &cfg,)- cfg.Encryption = encryption+ mutate(&cfg)test_utils.MakePostRequestAndUnmarshal(t, router,@@ -1594,24 +1642,32 @@ func setEncryption()}-func setHourlyInterval(t *testing.T, router *gin.Engine, databaseID uuid.UUID, ownerToken string) {- t.Helper()-- var cfg backups_config.BackupConfig- test_utils.MakeGetRequestAndUnmarshal(- t, router,- "/api/v1/backup-configs/database/"+databaseID.String(),- "Bearer "+ownerToken,- http.StatusOK, &cfg,- )+func setEncryption(+ t *testing.T,+ router *gin.Engine,+ databaseID uuid.UUID,+ ownerToken string,+ encryption backups_config.BackupEncryption,+) {+ mutateBackupConfig(t, router, databaseID, ownerToken, func(cfg *backups_config.BackupConfig) {+ cfg.Encryption = encryption+ })+}- cfg.BackupInterval = &intervals.Interval{Interval: intervals.IntervalHourly}+func setBackupsEnabled(+ t *testing.T,+ router *gin.Engine,+ databaseID uuid.UUID,+ ownerToken string,+ enabled bool,+) {+ mutateBackupConfig(t, router, databaseID, ownerToken, func(cfg *backups_config.BackupConfig) {+ cfg.IsBackupsEnabled = enabled+ })+}- test_utils.MakePostRequestAndUnmarshal(- t, router,- "/api/v1/backup-configs/save",- "Bearer "+ownerToken,- cfg,- http.StatusOK, &cfg,- )+func setHourlyInterval(t *testing.T, router *gin.Engine, databaseID uuid.UUID, ownerToken string) {+ mutateBackupConfig(t, router, databaseID, ownerToken, func(cfg *backups_config.BackupConfig) {+ cfg.BackupInterval = &intervals.Interval{Interval: intervals.IntervalHourly}+ })}diff --git a/backend/internal/features/backups/backups/services/postgres_wal_service.go b/backend/internal/features/backups/backups/services/postgres_wal_service.goindex c8730c22..2f3e58ba 100644--- a/backend/internal/features/backups/backups/services/postgres_wal_service.go+++ b/backend/internal/features/backups/backups/services/postgres_wal_service.go@@ -311,6 +311,10 @@ func (s *PostgreWalBackupService) DownloadBackupFile(return s.backupService.GetBackupReader(backupID)}+// GetNextFullBackupTime returns when the agent should next run pg_basebackup.+//+// nil response is the "do not run" signal — the agent treats it as "backups disabled."+// A concrete past/now time tells the agent to run immediately (first run, or schedule overdue).func (s *PostgreWalBackupService) GetNextFullBackupTime(database *databases.Database,) (*backups_dto.GetNextFullBackupTimeResponse, error) {@@ -323,6 +327,10 @@ func (s *PostgreWalBackupService) GetNextFullBackupTime(return nil, fmt.Errorf("failed to get backup config: %w", err)}+ if !backupConfig.IsBackupsEnabled {+ return &backups_dto.GetNextFullBackupTimeResponse{NextFullBackupTime: nil}, nil+ }+if backupConfig.BackupInterval == nil {return nil, fmt.Errorf("no backup interval configured for database %s", database.ID)}@@ -342,6 +350,10 @@ func (s *PostgreWalBackupService) GetNextFullBackupTime(now := time.Now().UTC()nextTime := backupConfig.BackupInterval.NextTriggerTime(now, lastBackupTime)+ if nextTime == nil {+ nextTime = &now+ }+return &backups_dto.GetNextFullBackupTimeResponse{NextFullBackupTime: nextTime,}, nildiff --git a/docker-compose.yml b/docker-compose.ymlindex 44eb13dc..297456f9 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -641,10 +641,6 @@ services:# handshake regressions without multiplying the version matrix.# ================================================================- # Certs are baked in via build context (COPY) instead of bind-mounted.- # CI runs Docker-in-Docker where host paths don't resolve to the job- # container's workspace, but build contexts are uploaded to the daemon- # as a tarball so they always reach the container.test-postgres-ssl:build:context: ./backend/internal/features/tests/testdata/ssl