Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Stop endless basebackup loop from issue #533

Rostislav Dugin643cb0479c85
diff --git a/agent/internal/features/full_backup/backuper.go b/agent/internal/features/full_backup/backuper.go
index 1e122214..7526364b 100644
--- a/agent/internal/features/full_backup/backuper.go
+++ b/agent/internal/features/full_backup/backuper.go
@@ -88,6 +88,20 @@ func (backuper *FullBackuper) checkAndRunIfNeeded(ctx context.Context) {
return
}
+ // Fetch next-time first: a nil response is the backend's signal that backups are disabled,
+ // and we must short-circuit before the chain-validity check (which would otherwise trigger
+ // a basebackup on `error=no_full_backup`).
+ nextTimeResp, err := backuper.apiClient.GetNextFullBackupTime(ctx)
+ if err != nil {
+ backuper.log.Error("Failed to check next full backup time", "error", err)
+ return
+ }
+
+ if nextTimeResp.NextFullBackupTime == nil {
+ backuper.log.Debug("Backups disabled, skipping basebackup check")
+ return
+ }
+
chainResp, err := backuper.apiClient.CheckWalChainValidity(ctx)
if err != nil {
backuper.log.Error("Failed to check WAL chain validity", "error", err)
@@ -105,13 +119,7 @@ func (backuper *FullBackuper) checkAndRunIfNeeded(ctx context.Context) {
return
}
- nextTimeResp, err := backuper.apiClient.GetNextFullBackupTime(ctx)
- if err != nil {
- backuper.log.Error("Failed to check next full backup time", "error", err)
- return
- }
-
- if nextTimeResp.NextFullBackupTime == nil || !nextTimeResp.NextFullBackupTime.After(time.Now().UTC()) {
+ if !nextTimeResp.NextFullBackupTime.After(time.Now().UTC()) {
backuper.log.Info("Scheduled full backup is due, triggering basebackup")
backuper.runBasebackupWithRetry(ctx)
diff --git a/agent/internal/features/full_backup/backuper_test.go b/agent/internal/features/full_backup/backuper_test.go
index 610b3fb2..79f17e31 100644
--- a/agent/internal/features/full_backup/backuper_test.go
+++ b/agent/internal/features/full_backup/backuper_test.go
@@ -40,7 +40,7 @@ func Test_RunFullBackup_WhenChainBroken_BasebackupTriggered(t *testing.T) {
var finalizeReceived bool
var finalizeBody map[string]any
- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case testChainValidPath:
writeJSON(w, api.WalChainValidityResponse{
@@ -66,7 +66,7 @@ func Test_RunFullBackup_WhenChainBroken_BasebackupTriggered(t *testing.T) {
default:
w.WriteHeader(http.StatusNotFound)
}
- })
+ }))
fb := newTestFullBackuper(server.URL)
fb.cmdBuilder = mockCmdBuilder(t, "test-backup-data", validStderr())
@@ -145,7 +145,7 @@ func Test_RunFullBackup_WhenNoFullBackupExists_ImmediateBasebackupTriggered(t *t
var mu sync.Mutex
var finalizeReceived bool
- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case testChainValidPath:
writeJSON(w, api.WalChainValidityResponse{
@@ -164,7 +164,7 @@ func Test_RunFullBackup_WhenNoFullBackupExists_ImmediateBasebackupTriggered(t *t
default:
w.WriteHeader(http.StatusNotFound)
}
- })
+ }))
fb := newTestFullBackuper(server.URL)
fb.cmdBuilder = mockCmdBuilder(t, "first-backup-data", validStderr())
@@ -191,7 +191,7 @@ func Test_RunFullBackup_WhenUploadFails_RetriesAfterDelay(t *testing.T) {
var uploadAttempts int
var errorReported bool
- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case testChainValidPath:
writeJSON(w, api.WalChainValidityResponse{
@@ -224,7 +224,7 @@ func Test_RunFullBackup_WhenUploadFails_RetriesAfterDelay(t *testing.T) {
default:
w.WriteHeader(http.StatusNotFound)
}
- })
+ }))
fb := newTestFullBackuper(server.URL)
fb.cmdBuilder = mockCmdBuilder(t, "retry-backup-data", validStderr())
@@ -292,7 +292,7 @@ func Test_RunFullBackup_WhenAlreadyRunning_SkipsExecution(t *testing.T) {
}
func Test_RunFullBackup_WhenContextCancelled_StopsCleanly(t *testing.T) {
- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case testChainValidPath:
writeJSON(w, api.WalChainValidityResponse{
@@ -309,7 +309,7 @@ func Test_RunFullBackup_WhenContextCancelled_StopsCleanly(t *testing.T) {
default:
w.WriteHeader(http.StatusNotFound)
}
- })
+ }))
fb := newTestFullBackuper(server.URL)
fb.cmdBuilder = mockCmdBuilder(t, "data", validStderr())
@@ -376,7 +376,7 @@ func Test_RunFullBackup_WhenStderrParsingFails_FinalizesWithErrorAndRetries(t *t
var finalizeWithErrorReceived bool
var finalizeBody map[string]any
- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case testChainValidPath:
writeJSON(w, api.WalChainValidityResponse{
@@ -402,7 +402,7 @@ func Test_RunFullBackup_WhenStderrParsingFails_FinalizesWithErrorAndRetries(t *t
default:
w.WriteHeader(http.StatusNotFound)
}
- })
+ }))
fb := newTestFullBackuper(server.URL)
fb.cmdBuilder = mockCmdBuilder(t, "data", "pg_basebackup: unexpected output with no LSN info")
@@ -431,24 +431,29 @@ func Test_RunFullBackup_WhenStderrParsingFails_FinalizesWithErrorAndRetries(t *t
assert.NotNil(t, finalizeBody["error"], "finalize should include error message")
}
-func Test_RunFullBackup_WhenNextBackupTimeNull_BasebackupTriggered(t *testing.T) {
- var mu sync.Mutex
- var finalizeReceived bool
+func Test_RunFullBackup_WhenBackupsDisabled_NoBasebackupTriggered(t *testing.T) {
+ var chainCalled atomic.Bool
+ var uploadCalled atomic.Bool
server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
- case testChainValidPath:
- writeJSON(w, api.WalChainValidityResponse{IsValid: true})
case testNextBackupTimePath:
writeJSON(w, api.NextFullBackupTimeResponse{NextFullBackupTime: nil})
+ case testChainValidPath:
+ // Reproduces the issue #533 condition: even if the chain reports
+ // no_full_backup, the disabled gate must fire first and prevent any basebackup.
+ chainCalled.Store(true)
+
+ writeJSON(w, api.WalChainValidityResponse{
+ IsValid: false,
+ Error: "no_full_backup",
+ })
case testFullStartPath:
+ uploadCalled.Store(true)
+
_, _ = io.ReadAll(r.Body)
writeJSON(w, map[string]string{"backupId": testBackupID})
case testFullCompletePath:
- mu.Lock()
- finalizeReceived = true
- mu.Unlock()
-
w.WriteHeader(http.StatusOK)
default:
w.WriteHeader(http.StatusNotFound)
@@ -456,29 +461,25 @@ func Test_RunFullBackup_WhenNextBackupTimeNull_BasebackupTriggered(t *testing.T)
})
fb := newTestFullBackuper(server.URL)
- fb.cmdBuilder = mockCmdBuilder(t, "first-run-data", validStderr())
+ fb.cmdBuilder = mockCmdBuilder(t, "should-not-run", validStderr())
- ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
defer cancel()
go fb.Run(ctx)
- waitForCondition(t, func() bool {
- mu.Lock()
- defer mu.Unlock()
- return finalizeReceived
- }, 5*time.Second)
+ time.Sleep(500 * time.Millisecond)
cancel()
- mu.Lock()
- defer mu.Unlock()
-
- assert.True(t, finalizeReceived)
+ assert.False(t, chainCalled.Load(),
+ "chain validity must not be called when backups are disabled")
+ assert.False(t, uploadCalled.Load(),
+ "no basebackup upload must happen when backups are disabled")
}
func Test_RunFullBackup_WhenChainValidityReturns401_NoBasebackupTriggered(t *testing.T) {
var uploadReceived atomic.Bool
- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case testChainValidPath:
w.WriteHeader(http.StatusUnauthorized)
@@ -493,7 +494,7 @@ func Test_RunFullBackup_WhenChainValidityReturns401_NoBasebackupTriggered(t *tes
default:
w.WriteHeader(http.StatusNotFound)
}
- })
+ }))
fb := newTestFullBackuper(server.URL)
fb.cmdBuilder = mockCmdBuilder(t, "data", validStderr())
@@ -512,7 +513,7 @@ func Test_RunFullBackup_WhenUploadSucceeds_BodyIsZstdCompressed(t *testing.T) {
var mu sync.Mutex
var receivedBody []byte
- server := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ server := newTestServer(t, withEnabledNextTime(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case testChainValidPath:
writeJSON(w, api.WalChainValidityResponse{
@@ -532,7 +533,7 @@ func Test_RunFullBackup_WhenUploadSucceeds_BodyIsZstdCompressed(t *testing.T) {
default:
w.WriteHeader(http.StatusNotFound)
}
- })
+ }))
originalContent := "test-backup-content-for-compression-check"
fb := newTestFullBackuper(server.URL)
@@ -702,6 +703,26 @@ pg_basebackup: syncing data to disk ...
pg_basebackup: base backup completed`
}
+func writeEnabledFutureTime(w http.ResponseWriter) {
+ futureTime := time.Now().UTC().Add(24 * time.Hour)
+ writeJSON(w, api.NextFullBackupTimeResponse{NextFullBackupTime: &futureTime})
+}
+
+// withEnabledNextTime makes the test server respond to the next-full-backup-time endpoint
+// with "enabled, ~24h away" so tests that exercise other branches don't have to repeat
+// the boilerplate. Tests that care about a specific next-time (past, nil) handle it inline
+// and skip this wrapper.
+func withEnabledNextTime(inner http.HandlerFunc) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == testNextBackupTimePath {
+ writeEnabledFutureTime(w)
+ return
+ }
+
+ inner(w, r)
+ }
+}
+
func writeJSON(w http.ResponseWriter, v any) {
w.Header().Set("Content-Type", "application/json")
diff --git a/backend/internal/features/backups/backups/controllers/postgres_wal_controller_test.go b/backend/internal/features/backups/backups/controllers/postgres_wal_controller_test.go
index 5255a906..d505020a 100644
--- a/backend/internal/features/backups/backups/controllers/postgres_wal_controller_test.go
+++ b/backend/internal/features/backups/backups/controllers/postgres_wal_controller_test.go
@@ -538,10 +538,12 @@ func Test_ReportError_WithMissingErrorField_ReturnsBadRequest(t *testing.T) {
assert.Equal(t, http.StatusBadRequest, w.Code)
}
-func Test_GetNextFullBackupTime_WithValidToken_NoFullBackup_ReturnsNull(t *testing.T) {
+func Test_GetNextFullBackupTime_NoFullBackup_ReturnsNowSoAgentRunsImmediately(t *testing.T) {
router, db, storage, agentToken, _ := createWalTestSetup(t)
defer removeWalTestSetup(db, storage)
+ now := time.Now().UTC()
+
var response backups_dto.GetNextFullBackupTimeResponse
test_utils.MakeGetRequestAndUnmarshal(
t,
@@ -552,7 +554,53 @@ func Test_GetNextFullBackupTime_WithValidToken_NoFullBackup_ReturnsNull(t *testi
&response,
)
- assert.Nil(t, response.NextFullBackupTime, "should be nil when no full backup exists")
+ require.NotNil(t, response.NextFullBackupTime,
+ "nil is reserved for disabled backups; first run must return a concrete time")
+ assert.WithinDuration(t, now, response.NextFullBackupTime.UTC(), 5*time.Second,
+ "first run should return ~now so the agent triggers immediately")
+}
+
+func Test_GetNextFullBackupTime_WhenBackupsDisabled_ReturnsNull(t *testing.T) {
+ router, db, storage, agentToken, ownerToken := createWalTestSetup(t)
+ defer removeWalTestSetup(db, storage)
+
+ setBackupsEnabled(t, router, db.ID, ownerToken, false)
+
+ var response backups_dto.GetNextFullBackupTimeResponse
+ test_utils.MakeGetRequestAndUnmarshal(
+ t,
+ router,
+ "/api/v1/backups/postgres/wal/next-full-backup-time",
+ agentToken,
+ http.StatusOK,
+ &response,
+ )
+
+ assert.Nil(t, response.NextFullBackupTime,
+ "disabled backups must return nil so the agent skips this cycle")
+}
+
+func Test_GetNextFullBackupTime_WhenBackupsDisabledWithExistingFullBackup_ReturnsNull(t *testing.T) {
+ router, db, storage, agentToken, ownerToken := createWalTestSetup(t)
+ defer removeWalTestSetup(db, storage)
+
+ uploadBasebackup(t, router, agentToken,
+ "000000010000000100000001", "000000010000000100000010")
+
+ setBackupsEnabled(t, router, db.ID, ownerToken, false)
+
+ var response backups_dto.GetNextFullBackupTimeResponse
+ test_utils.MakeGetRequestAndUnmarshal(
+ t,
+ router,
+ "/api/v1/backups/postgres/wal/next-full-backup-time",
+ agentToken,
+ http.StatusOK,
+ &response,
+ )
+
+ assert.Nil(t, response.NextFullBackupTime,
+ "disabled backups must return nil even when a prior full backup exists")
}
func Test_GetNextFullBackupTime_WithValidToken_HasFullBackup_ReturnsTime(t *testing.T) {
@@ -1566,12 +1614,12 @@ func updateLastFullBackupTime(t *testing.T, databaseID uuid.UUID, createdAt time
}
}
-func setEncryption(
+func mutateBackupConfig(
t *testing.T,
router *gin.Engine,
databaseID uuid.UUID,
ownerToken string,
- encryption backups_config.BackupEncryption,
+ mutate func(*backups_config.BackupConfig),
) {
t.Helper()
@@ -1583,7 +1631,7 @@ func setEncryption(
http.StatusOK, &cfg,
)
- cfg.Encryption = encryption
+ mutate(&cfg)
test_utils.MakePostRequestAndUnmarshal(
t, router,
@@ -1594,24 +1642,32 @@ func setEncryption(
)
}
-func setHourlyInterval(t *testing.T, router *gin.Engine, databaseID uuid.UUID, ownerToken string) {
- t.Helper()
-
- var cfg backups_config.BackupConfig
- test_utils.MakeGetRequestAndUnmarshal(
- t, router,
- "/api/v1/backup-configs/database/"+databaseID.String(),
- "Bearer "+ownerToken,
- http.StatusOK, &cfg,
- )
+func setEncryption(
+ t *testing.T,
+ router *gin.Engine,
+ databaseID uuid.UUID,
+ ownerToken string,
+ encryption backups_config.BackupEncryption,
+) {
+ mutateBackupConfig(t, router, databaseID, ownerToken, func(cfg *backups_config.BackupConfig) {
+ cfg.Encryption = encryption
+ })
+}
- cfg.BackupInterval = &intervals.Interval{Interval: intervals.IntervalHourly}
+func setBackupsEnabled(
+ t *testing.T,
+ router *gin.Engine,
+ databaseID uuid.UUID,
+ ownerToken string,
+ enabled bool,
+) {
+ mutateBackupConfig(t, router, databaseID, ownerToken, func(cfg *backups_config.BackupConfig) {
+ cfg.IsBackupsEnabled = enabled
+ })
+}
- test_utils.MakePostRequestAndUnmarshal(
- t, router,
- "/api/v1/backup-configs/save",
- "Bearer "+ownerToken,
- cfg,
- http.StatusOK, &cfg,
- )
+func setHourlyInterval(t *testing.T, router *gin.Engine, databaseID uuid.UUID, ownerToken string) {
+ mutateBackupConfig(t, router, databaseID, ownerToken, func(cfg *backups_config.BackupConfig) {
+ cfg.BackupInterval = &intervals.Interval{Interval: intervals.IntervalHourly}
+ })
}
diff --git a/backend/internal/features/backups/backups/services/postgres_wal_service.go b/backend/internal/features/backups/backups/services/postgres_wal_service.go
index c8730c22..2f3e58ba 100644
--- a/backend/internal/features/backups/backups/services/postgres_wal_service.go
+++ b/backend/internal/features/backups/backups/services/postgres_wal_service.go
@@ -311,6 +311,10 @@ func (s *PostgreWalBackupService) DownloadBackupFile(
return s.backupService.GetBackupReader(backupID)
}
+// GetNextFullBackupTime returns when the agent should next run pg_basebackup.
+//
+// nil response is the "do not run" signal — the agent treats it as "backups disabled."
+// A concrete past/now time tells the agent to run immediately (first run, or schedule overdue).
func (s *PostgreWalBackupService) GetNextFullBackupTime(
database *databases.Database,
) (*backups_dto.GetNextFullBackupTimeResponse, error) {
@@ -323,6 +327,10 @@ func (s *PostgreWalBackupService) GetNextFullBackupTime(
return nil, fmt.Errorf("failed to get backup config: %w", err)
}
+ if !backupConfig.IsBackupsEnabled {
+ return &backups_dto.GetNextFullBackupTimeResponse{NextFullBackupTime: nil}, nil
+ }
+
if backupConfig.BackupInterval == nil {
return nil, fmt.Errorf("no backup interval configured for database %s", database.ID)
}
@@ -342,6 +350,10 @@ func (s *PostgreWalBackupService) GetNextFullBackupTime(
now := time.Now().UTC()
nextTime := backupConfig.BackupInterval.NextTriggerTime(now, lastBackupTime)
+ if nextTime == nil {
+ nextTime = &now
+ }
+
return &backups_dto.GetNextFullBackupTimeResponse{
NextFullBackupTime: nextTime,
}, nil
diff --git a/docker-compose.yml b/docker-compose.yml
index 44eb13dc..297456f9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -641,10 +641,6 @@ services:
# handshake regressions without multiplying the version matrix.
# ================================================================
- # Certs are baked in via build context (COPY) instead of bind-mounted.
- # CI runs Docker-in-Docker where host paths don't resolve to the job
- # container's workspace, but build contexts are uploaded to the daemon
- # as a tarball so they always reach the container.
test-postgres-ssl:
build:
context: ./backend/internal/features/tests/testdata/ssl