PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (agent): Cleanup history files after base backup
Rostislav Dugine7d2ff8e35b9
diff --git a/agent/Makefile b/agent/Makefileindex fe95ee81..5c3e0285 100644--- a/agent/Makefile+++ b/agent/Makefile@@ -17,6 +17,8 @@ run:--pg-wal-dir ../wal-queue \--skip-update+# We are using it to make PostgreSQL produce WAL segments+# so out development agent can capture it and we can developwal-generator:go run cmd/wal-generator/main.go@@ -30,6 +32,8 @@ lint:golangci-lint fmt ./cmd/... ./internal/... ./e2e/... && golangci-lint run ./cmd/... ./internal/... ./e2e/...e2e:+ cd e2e && docker compose down -v --remove-orphans 2>/dev/null || true+ rm -rf e2e/pgdata e2e/artifactscd e2e && docker compose build --no-cache e2e-mock-servercd e2e && docker compose buildcd e2e && docker compose run --rm e2e-agent-builderdiff --git a/agent/e2e/scripts/backup-restore-helpers.sh b/agent/e2e/scripts/backup-restore-helpers.shindex c6d44e54..3022d806 100644--- a/agent/e2e/scripts/backup-restore-helpers.sh+++ b/agent/e2e/scripts/backup-restore-helpers.sh@@ -334,6 +334,28 @@ verify_restored_data() {return 0}+verify_wal_queue_no_backup_files() {+ local wal_queue="$1"+ local timeout="${2:-60}"++ echo "Verifying WAL archive contains no .backup history files..."++ for i in $(seq 1 "$timeout"); do+ if ! find "$wal_queue" -maxdepth 1 -name "*.backup" -type f 2>/dev/null | grep -q .; then+ echo "PASS: WAL archive contains no .backup history files"+ return 0+ fi++ sleep 1+ done++ echo "FAIL: WAL archive still contains .backup history files after ${timeout}s:"+ find "$wal_queue" -maxdepth 1 -name "*.backup" -type f -printf " %f (%s bytes)\n" 2>/dev/null || true+ echo "Agent output:"+ cat /tmp/agent-output.log 2>/dev/null || true+ return 1+}+find_pg_bin_dir() {# Find the PG bin dir from the installed versionlocal pg_config_pathdiff --git a/agent/e2e/scripts/run-all.sh b/agent/e2e/scripts/run-all.shindex 33cc86b6..4d78a62e 100644--- a/agent/e2e/scripts/run-all.sh+++ b/agent/e2e/scripts/run-all.sh@@ -32,9 +32,10 @@ if [ "$MODE" = "host" ]; thenrun_test "Test 3: Background upgrade (v1 -> v2 while running)" "$SCRIPT_DIR/test-upgrade-background.sh"run_test "Test 4: Backup-restore via host PATH" "$SCRIPT_DIR/test-pg-host-path.sh"run_test "Test 5: Backup-restore via host bindir" "$SCRIPT_DIR/test-pg-host-bindir.sh"+ run_test "Test 6: WAL archive cleanup of .backup history files" "$SCRIPT_DIR/test-wal-archive-cleanup.sh"elif [ "$MODE" = "docker" ]; then- run_test "Test 6: Backup-restore via docker exec" "$SCRIPT_DIR/test-pg-docker-exec.sh"+ run_test "Test 7: Backup-restore via docker exec" "$SCRIPT_DIR/test-pg-docker-exec.sh"elseecho "Unknown mode: $MODE (expected 'host' or 'docker')"diff --git a/agent/e2e/scripts/test-wal-archive-cleanup.sh b/agent/e2e/scripts/test-wal-archive-cleanup.shnew file mode 100644index 00000000..db9ec51e--- /dev/null+++ b/agent/e2e/scripts/test-wal-archive-cleanup.sh@@ -0,0 +1,46 @@+#!/bin/bash+set -euo pipefail++SCRIPT_DIR="$(dirname "$0")"+source "$SCRIPT_DIR/backup-restore-helpers.sh"++MOCK_SERVER="${MOCK_SERVER_OVERRIDE:-http://e2e-mock-server:4050}"+PGDATA="/tmp/pgdata-walcleanup"+WAL_QUEUE="/tmp/wal-queue-walcleanup"+PG_PORT=5435++PG_BIN_DIR=$(find_pg_bin_dir)+echo "Using PG bin dir: $PG_BIN_DIR"++echo "=== Phase 1: Setup agent ==="+setup_agent++echo "=== Phase 2: Initialize PostgreSQL ==="+init_pg_local "$PGDATA" "$PG_PORT" "$WAL_QUEUE" "$PG_BIN_DIR"++echo "=== Phase 3: Insert test data ==="+insert_test_data "$PG_PORT" "$PG_BIN_DIR"++echo "=== Phase 4: Force checkpoint and start agent backup ==="+force_checkpoint "$PG_PORT" "$PG_BIN_DIR"+run_agent_backup "$MOCK_SERVER" "127.0.0.1" "$PG_PORT" "$WAL_QUEUE" "host"++echo "=== Phase 5: Generate WAL in background ==="+generate_wal_background "$PG_PORT" "$PG_BIN_DIR" &+WAL_GEN_PID=$!++echo "=== Phase 6: Wait for base backup to complete ==="+wait_for_backup_complete "$MOCK_SERVER" 120++echo "=== Phase 7: Stop WAL generator and PostgreSQL (agent keeps running) ==="+kill $WAL_GEN_PID 2>/dev/null || true+wait $WAL_GEN_PID 2>/dev/null || true+stop_pg "$PGDATA" "$PG_BIN_DIR"++echo "=== Phase 8: Verify agent cleaned .backup history files from WAL archive ==="+verify_wal_queue_no_backup_files "$WAL_QUEUE" 60++echo "=== Phase 9: Cleanup ==="+stop_agent++echo "wal-archive-cleanup: .backup history files cleaned from WAL queue"diff --git a/agent/internal/features/wal/streamer.go b/agent/internal/features/wal/streamer.goindex d5b5702b..533c9fa1 100644--- a/agent/internal/features/wal/streamer.go+++ b/agent/internal/features/wal/streamer.go@@ -2,6 +2,7 @@ package walimport ("context"+ "errors""fmt""io""log/slog"@@ -25,7 +26,11 @@ const (uploadTimeout = 5 * time.Minute)-var segmentNameRegex = regexp.MustCompile(`^[0-9A-Fa-f]{24}$`)+var (+ segmentNameRegex = regexp.MustCompile(`^[0-9A-Fa-f]{24}$`)+ backupHistoryRegex = regexp.MustCompile(`^[0-9A-Fa-f]{24}(\.[0-9A-Fa-f]+)?\.backup$`)+ timelineHistoryRegex = regexp.MustCompile(`^[0-9A-Fa-f]{24}\.history$`)+)type Streamer struct {cfg *config.Config@@ -108,11 +113,12 @@ func (s *Streamer) listSegments() ([]string, error) {continue}- if !segmentNameRegex.MatchString(name) {- continue+ switch {+ case segmentNameRegex.MatchString(name):+ segments = append(segments, name)+ case backupHistoryRegex.MatchString(name), timelineHistoryRegex.MatchString(name):+ s.removeSidecar(name)}-- segments = append(segments, name)}slices.Sort(segments)@@ -172,6 +178,26 @@ func (s *Streamer) uploadSegment(ctx context.Context, segmentName string) errorreturn nil}+// removeSidecar deletes Postgres backup-history (.backup) and timeline-history+// (.history) files left in the WAL archive by archive_command. Safe to delete:+// the agent uploads each pg_basebackup tarball directly to the backend with+// start/stop LSNs, and timeline-history files are already inside that tarball,+// so the on-disk copies are inert in this architecture.+func (s *Streamer) removeSidecar(name string) {+ path := filepath.Join(s.cfg.PgWalDir, name)++ if err := os.Remove(path); err != nil {+ if errors.Is(err, os.ErrNotExist) {+ return+ }++ s.log.Warn("failed to delete WAL sidecar file", "file", name, "error", err)+ return+ }++ s.log.Info("deleted WAL sidecar file", "file", name)+}+func (s *Streamer) compressAndStream(pw *io.PipeWriter, filePath string) {f, err := os.Open(filePath)if err != nil {diff --git a/agent/internal/features/wal/streamer_test.go b/agent/internal/features/wal/streamer_test.goindex 5647cbf8..0aef741a 100644--- a/agent/internal/features/wal/streamer_test.go+++ b/agent/internal/features/wal/streamer_test.go@@ -332,6 +332,183 @@ func Test_UploadSegment_WhenUploadStalls_FailsWithIdleTimeout(t *testing.T) {assert.NoError(t, statErr, "segment file should remain in queue after idle timeout")}+func Test_UploadSegments_DirectoryHasBackupHistoryFile_BackupFileDeletedAndNotUploaded(t *testing.T) {+ walDir := createTestWalDir(t)+ segmentName := "000000010000000100000001"+ backupName := "000000010000000000000003.00000028.backup"+ writeTestSegment(t, walDir, segmentName, []byte("real segment"))+ writeTestSegment(t, walDir, backupName, []byte("backup history"))++ var mu sync.Mutex+ var uploadedSegments []string++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ mu.Lock()+ uploadedSegments = append(uploadedSegments, r.Header.Get("X-Wal-Segment-Name"))+ mu.Unlock()++ _, _ = io.ReadAll(r.Body)+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ streamer := newTestStreamer(walDir, server.URL)++ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)+ defer cancel()++ go streamer.Run(ctx)+ time.Sleep(500 * time.Millisecond)+ cancel()++ mu.Lock()+ defer mu.Unlock()++ require.Len(t, uploadedSegments, 1)+ assert.Equal(t, segmentName, uploadedSegments[0])++ _, err := os.Stat(filepath.Join(walDir, segmentName))+ assert.True(t, os.IsNotExist(err), "uploaded segment should be deleted")++ _, err = os.Stat(filepath.Join(walDir, backupName))+ assert.True(t, os.IsNotExist(err), ".backup history file should be deleted from archive")+}++func Test_UploadSegments_DirectoryHasTimelineHistoryFile_HistoryFileDeletedAndNotUploaded(t *testing.T) {+ walDir := createTestWalDir(t)+ segmentName := "000000010000000100000001"+ historyName := "000000000000000000000002.history"+ writeTestSegment(t, walDir, segmentName, []byte("real segment"))+ writeTestSegment(t, walDir, historyName, []byte("timeline history"))++ var mu sync.Mutex+ var uploadedSegments []string++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ mu.Lock()+ uploadedSegments = append(uploadedSegments, r.Header.Get("X-Wal-Segment-Name"))+ mu.Unlock()++ _, _ = io.ReadAll(r.Body)+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ streamer := newTestStreamer(walDir, server.URL)++ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)+ defer cancel()++ go streamer.Run(ctx)+ time.Sleep(500 * time.Millisecond)+ cancel()++ mu.Lock()+ defer mu.Unlock()++ require.Len(t, uploadedSegments, 1)+ assert.Equal(t, segmentName, uploadedSegments[0])++ _, err := os.Stat(filepath.Join(walDir, historyName))+ assert.True(t, os.IsNotExist(err), ".history file should be deleted from archive")+}++func Test_UploadSegments_BackupHistoryFileWithoutOffset_FileDeleted(t *testing.T) {+ walDir := createTestWalDir(t)+ backupName := "000000010000000000000003.backup"+ writeTestSegment(t, walDir, backupName, []byte("backup history"))++ uploadCount := 0++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ uploadCount+++ _, _ = io.ReadAll(r.Body)+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ streamer := newTestStreamer(walDir, server.URL)++ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)+ defer cancel()++ go streamer.Run(ctx)+ time.Sleep(500 * time.Millisecond)+ cancel()++ assert.Equal(t, 0, uploadCount, "sidecar files should not trigger uploads")++ _, err := os.Stat(filepath.Join(walDir, backupName))+ assert.True(t, os.IsNotExist(err), ".backup file without offset suffix should be deleted")+}++func Test_UploadSegments_DeleteDisabled_SidecarsStillDeleted(t *testing.T) {+ walDir := createTestWalDir(t)+ segmentName := "000000010000000100000001"+ backupName := "000000010000000000000003.00000028.backup"+ historyName := "000000000000000000000002.history"+ writeTestSegment(t, walDir, segmentName, []byte("real segment"))+ writeTestSegment(t, walDir, backupName, []byte("backup history"))+ writeTestSegment(t, walDir, historyName, []byte("timeline history"))++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.ReadAll(r.Body)+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ isDeleteDisabled := false+ cfg := createTestConfig(walDir, server.URL)+ cfg.IsDeleteWalAfterUpload = &isDeleteDisabled+ apiClient := api.NewClient(server.URL, cfg.Token, logger.GetLogger())+ streamer := NewStreamer(cfg, apiClient, logger.GetLogger())++ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)+ defer cancel()++ go streamer.Run(ctx)+ time.Sleep(500 * time.Millisecond)+ cancel()++ _, err := os.Stat(filepath.Join(walDir, segmentName))+ assert.NoError(t, err, "segment file should be kept when delete is disabled")++ _, err = os.Stat(filepath.Join(walDir, backupName))+ assert.True(t, os.IsNotExist(err), ".backup sidecar should be deleted regardless of IsDeleteWalAfterUpload")++ _, err = os.Stat(filepath.Join(walDir, historyName))+ assert.True(t, os.IsNotExist(err), ".history sidecar should be deleted regardless of IsDeleteWalAfterUpload")+}++func Test_UploadSegments_NonMatchingDotBackupFile_FileKept(t *testing.T) {+ walDir := createTestWalDir(t)+ unrelatedBackup := "notawal.backup"+ unrelatedHistory := "random.history"+ writeTestSegment(t, walDir, unrelatedBackup, []byte("user file"))+ writeTestSegment(t, walDir, unrelatedHistory, []byte("user file"))++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.ReadAll(r.Body)+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ streamer := newTestStreamer(walDir, server.URL)++ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)+ defer cancel()++ go streamer.Run(ctx)+ time.Sleep(500 * time.Millisecond)+ cancel()++ _, err := os.Stat(filepath.Join(walDir, unrelatedBackup))+ assert.NoError(t, err, "files not matching the Postgres sidecar pattern must be left alone")++ _, err = os.Stat(filepath.Join(walDir, unrelatedHistory))+ assert.NoError(t, err, "files not matching the Postgres sidecar pattern must be left alone")+}+func newTestStreamer(walDir, serverURL string) *Streamer {cfg := createTestConfig(walDir, serverURL)apiClient := api.NewClient(serverURL, cfg.Token, logger.GetLogger())