Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (agent): Cleanup history files after base backup

Rostislav Dugine7d2ff8e35b9
diff --git a/agent/Makefile b/agent/Makefile
index fe95ee81..5c3e0285 100644
--- a/agent/Makefile
+++ b/agent/Makefile
@@ -17,6 +17,8 @@ run:
--pg-wal-dir ../wal-queue \
--skip-update
+# We are using it to make PostgreSQL produce WAL segments
+# so out development agent can capture it and we can develop
wal-generator:
go run cmd/wal-generator/main.go
@@ -30,6 +32,8 @@ lint:
golangci-lint fmt ./cmd/... ./internal/... ./e2e/... && golangci-lint run ./cmd/... ./internal/... ./e2e/...
e2e:
+ cd e2e && docker compose down -v --remove-orphans 2>/dev/null || true
+ rm -rf e2e/pgdata e2e/artifacts
cd e2e && docker compose build --no-cache e2e-mock-server
cd e2e && docker compose build
cd e2e && docker compose run --rm e2e-agent-builder
diff --git a/agent/e2e/scripts/backup-restore-helpers.sh b/agent/e2e/scripts/backup-restore-helpers.sh
index c6d44e54..3022d806 100644
--- a/agent/e2e/scripts/backup-restore-helpers.sh
+++ b/agent/e2e/scripts/backup-restore-helpers.sh
@@ -334,6 +334,28 @@ verify_restored_data() {
return 0
}
+verify_wal_queue_no_backup_files() {
+ local wal_queue="$1"
+ local timeout="${2:-60}"
+
+ echo "Verifying WAL archive contains no .backup history files..."
+
+ for i in $(seq 1 "$timeout"); do
+ if ! find "$wal_queue" -maxdepth 1 -name "*.backup" -type f 2>/dev/null | grep -q .; then
+ echo "PASS: WAL archive contains no .backup history files"
+ return 0
+ fi
+
+ sleep 1
+ done
+
+ echo "FAIL: WAL archive still contains .backup history files after ${timeout}s:"
+ find "$wal_queue" -maxdepth 1 -name "*.backup" -type f -printf " %f (%s bytes)\n" 2>/dev/null || true
+ echo "Agent output:"
+ cat /tmp/agent-output.log 2>/dev/null || true
+ return 1
+}
+
find_pg_bin_dir() {
# Find the PG bin dir from the installed version
local pg_config_path
diff --git a/agent/e2e/scripts/run-all.sh b/agent/e2e/scripts/run-all.sh
index 33cc86b6..4d78a62e 100644
--- a/agent/e2e/scripts/run-all.sh
+++ b/agent/e2e/scripts/run-all.sh
@@ -32,9 +32,10 @@ if [ "$MODE" = "host" ]; then
run_test "Test 3: Background upgrade (v1 -> v2 while running)" "$SCRIPT_DIR/test-upgrade-background.sh"
run_test "Test 4: Backup-restore via host PATH" "$SCRIPT_DIR/test-pg-host-path.sh"
run_test "Test 5: Backup-restore via host bindir" "$SCRIPT_DIR/test-pg-host-bindir.sh"
+ run_test "Test 6: WAL archive cleanup of .backup history files" "$SCRIPT_DIR/test-wal-archive-cleanup.sh"
elif [ "$MODE" = "docker" ]; then
- run_test "Test 6: Backup-restore via docker exec" "$SCRIPT_DIR/test-pg-docker-exec.sh"
+ run_test "Test 7: Backup-restore via docker exec" "$SCRIPT_DIR/test-pg-docker-exec.sh"
else
echo "Unknown mode: $MODE (expected 'host' or 'docker')"
diff --git a/agent/e2e/scripts/test-wal-archive-cleanup.sh b/agent/e2e/scripts/test-wal-archive-cleanup.sh
new file mode 100644
index 00000000..db9ec51e
--- /dev/null
+++ b/agent/e2e/scripts/test-wal-archive-cleanup.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(dirname "$0")"
+source "$SCRIPT_DIR/backup-restore-helpers.sh"
+
+MOCK_SERVER="${MOCK_SERVER_OVERRIDE:-http://e2e-mock-server:4050}"
+PGDATA="/tmp/pgdata-walcleanup"
+WAL_QUEUE="/tmp/wal-queue-walcleanup"
+PG_PORT=5435
+
+PG_BIN_DIR=$(find_pg_bin_dir)
+echo "Using PG bin dir: $PG_BIN_DIR"
+
+echo "=== Phase 1: Setup agent ==="
+setup_agent
+
+echo "=== Phase 2: Initialize PostgreSQL ==="
+init_pg_local "$PGDATA" "$PG_PORT" "$WAL_QUEUE" "$PG_BIN_DIR"
+
+echo "=== Phase 3: Insert test data ==="
+insert_test_data "$PG_PORT" "$PG_BIN_DIR"
+
+echo "=== Phase 4: Force checkpoint and start agent backup ==="
+force_checkpoint "$PG_PORT" "$PG_BIN_DIR"
+run_agent_backup "$MOCK_SERVER" "127.0.0.1" "$PG_PORT" "$WAL_QUEUE" "host"
+
+echo "=== Phase 5: Generate WAL in background ==="
+generate_wal_background "$PG_PORT" "$PG_BIN_DIR" &
+WAL_GEN_PID=$!
+
+echo "=== Phase 6: Wait for base backup to complete ==="
+wait_for_backup_complete "$MOCK_SERVER" 120
+
+echo "=== Phase 7: Stop WAL generator and PostgreSQL (agent keeps running) ==="
+kill $WAL_GEN_PID 2>/dev/null || true
+wait $WAL_GEN_PID 2>/dev/null || true
+stop_pg "$PGDATA" "$PG_BIN_DIR"
+
+echo "=== Phase 8: Verify agent cleaned .backup history files from WAL archive ==="
+verify_wal_queue_no_backup_files "$WAL_QUEUE" 60
+
+echo "=== Phase 9: Cleanup ==="
+stop_agent
+
+echo "wal-archive-cleanup: .backup history files cleaned from WAL queue"
diff --git a/agent/internal/features/wal/streamer.go b/agent/internal/features/wal/streamer.go
index d5b5702b..533c9fa1 100644
--- a/agent/internal/features/wal/streamer.go
+++ b/agent/internal/features/wal/streamer.go
@@ -2,6 +2,7 @@ package wal
import (
"context"
+ "errors"
"fmt"
"io"
"log/slog"
@@ -25,7 +26,11 @@ const (
uploadTimeout = 5 * time.Minute
)
-var segmentNameRegex = regexp.MustCompile(`^[0-9A-Fa-f]{24}$`)
+var (
+ segmentNameRegex = regexp.MustCompile(`^[0-9A-Fa-f]{24}$`)
+ backupHistoryRegex = regexp.MustCompile(`^[0-9A-Fa-f]{24}(\.[0-9A-Fa-f]+)?\.backup$`)
+ timelineHistoryRegex = regexp.MustCompile(`^[0-9A-Fa-f]{24}\.history$`)
+)
type Streamer struct {
cfg *config.Config
@@ -108,11 +113,12 @@ func (s *Streamer) listSegments() ([]string, error) {
continue
}
- if !segmentNameRegex.MatchString(name) {
- continue
+ switch {
+ case segmentNameRegex.MatchString(name):
+ segments = append(segments, name)
+ case backupHistoryRegex.MatchString(name), timelineHistoryRegex.MatchString(name):
+ s.removeSidecar(name)
}
-
- segments = append(segments, name)
}
slices.Sort(segments)
@@ -172,6 +178,26 @@ func (s *Streamer) uploadSegment(ctx context.Context, segmentName string) error
return nil
}
+// removeSidecar deletes Postgres backup-history (.backup) and timeline-history
+// (.history) files left in the WAL archive by archive_command. Safe to delete:
+// the agent uploads each pg_basebackup tarball directly to the backend with
+// start/stop LSNs, and timeline-history files are already inside that tarball,
+// so the on-disk copies are inert in this architecture.
+func (s *Streamer) removeSidecar(name string) {
+ path := filepath.Join(s.cfg.PgWalDir, name)
+
+ if err := os.Remove(path); err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return
+ }
+
+ s.log.Warn("failed to delete WAL sidecar file", "file", name, "error", err)
+ return
+ }
+
+ s.log.Info("deleted WAL sidecar file", "file", name)
+}
+
func (s *Streamer) compressAndStream(pw *io.PipeWriter, filePath string) {
f, err := os.Open(filePath)
if err != nil {
diff --git a/agent/internal/features/wal/streamer_test.go b/agent/internal/features/wal/streamer_test.go
index 5647cbf8..0aef741a 100644
--- a/agent/internal/features/wal/streamer_test.go
+++ b/agent/internal/features/wal/streamer_test.go
@@ -332,6 +332,183 @@ func Test_UploadSegment_WhenUploadStalls_FailsWithIdleTimeout(t *testing.T) {
assert.NoError(t, statErr, "segment file should remain in queue after idle timeout")
}
+func Test_UploadSegments_DirectoryHasBackupHistoryFile_BackupFileDeletedAndNotUploaded(t *testing.T) {
+ walDir := createTestWalDir(t)
+ segmentName := "000000010000000100000001"
+ backupName := "000000010000000000000003.00000028.backup"
+ writeTestSegment(t, walDir, segmentName, []byte("real segment"))
+ writeTestSegment(t, walDir, backupName, []byte("backup history"))
+
+ var mu sync.Mutex
+ var uploadedSegments []string
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ uploadedSegments = append(uploadedSegments, r.Header.Get("X-Wal-Segment-Name"))
+ mu.Unlock()
+
+ _, _ = io.ReadAll(r.Body)
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ streamer := newTestStreamer(walDir, server.URL)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+
+ go streamer.Run(ctx)
+ time.Sleep(500 * time.Millisecond)
+ cancel()
+
+ mu.Lock()
+ defer mu.Unlock()
+
+ require.Len(t, uploadedSegments, 1)
+ assert.Equal(t, segmentName, uploadedSegments[0])
+
+ _, err := os.Stat(filepath.Join(walDir, segmentName))
+ assert.True(t, os.IsNotExist(err), "uploaded segment should be deleted")
+
+ _, err = os.Stat(filepath.Join(walDir, backupName))
+ assert.True(t, os.IsNotExist(err), ".backup history file should be deleted from archive")
+}
+
+func Test_UploadSegments_DirectoryHasTimelineHistoryFile_HistoryFileDeletedAndNotUploaded(t *testing.T) {
+ walDir := createTestWalDir(t)
+ segmentName := "000000010000000100000001"
+ historyName := "000000000000000000000002.history"
+ writeTestSegment(t, walDir, segmentName, []byte("real segment"))
+ writeTestSegment(t, walDir, historyName, []byte("timeline history"))
+
+ var mu sync.Mutex
+ var uploadedSegments []string
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ uploadedSegments = append(uploadedSegments, r.Header.Get("X-Wal-Segment-Name"))
+ mu.Unlock()
+
+ _, _ = io.ReadAll(r.Body)
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ streamer := newTestStreamer(walDir, server.URL)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+
+ go streamer.Run(ctx)
+ time.Sleep(500 * time.Millisecond)
+ cancel()
+
+ mu.Lock()
+ defer mu.Unlock()
+
+ require.Len(t, uploadedSegments, 1)
+ assert.Equal(t, segmentName, uploadedSegments[0])
+
+ _, err := os.Stat(filepath.Join(walDir, historyName))
+ assert.True(t, os.IsNotExist(err), ".history file should be deleted from archive")
+}
+
+func Test_UploadSegments_BackupHistoryFileWithoutOffset_FileDeleted(t *testing.T) {
+ walDir := createTestWalDir(t)
+ backupName := "000000010000000000000003.backup"
+ writeTestSegment(t, walDir, backupName, []byte("backup history"))
+
+ uploadCount := 0
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ uploadCount++
+ _, _ = io.ReadAll(r.Body)
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ streamer := newTestStreamer(walDir, server.URL)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+
+ go streamer.Run(ctx)
+ time.Sleep(500 * time.Millisecond)
+ cancel()
+
+ assert.Equal(t, 0, uploadCount, "sidecar files should not trigger uploads")
+
+ _, err := os.Stat(filepath.Join(walDir, backupName))
+ assert.True(t, os.IsNotExist(err), ".backup file without offset suffix should be deleted")
+}
+
+func Test_UploadSegments_DeleteDisabled_SidecarsStillDeleted(t *testing.T) {
+ walDir := createTestWalDir(t)
+ segmentName := "000000010000000100000001"
+ backupName := "000000010000000000000003.00000028.backup"
+ historyName := "000000000000000000000002.history"
+ writeTestSegment(t, walDir, segmentName, []byte("real segment"))
+ writeTestSegment(t, walDir, backupName, []byte("backup history"))
+ writeTestSegment(t, walDir, historyName, []byte("timeline history"))
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.ReadAll(r.Body)
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ isDeleteDisabled := false
+ cfg := createTestConfig(walDir, server.URL)
+ cfg.IsDeleteWalAfterUpload = &isDeleteDisabled
+ apiClient := api.NewClient(server.URL, cfg.Token, logger.GetLogger())
+ streamer := NewStreamer(cfg, apiClient, logger.GetLogger())
+
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+
+ go streamer.Run(ctx)
+ time.Sleep(500 * time.Millisecond)
+ cancel()
+
+ _, err := os.Stat(filepath.Join(walDir, segmentName))
+ assert.NoError(t, err, "segment file should be kept when delete is disabled")
+
+ _, err = os.Stat(filepath.Join(walDir, backupName))
+ assert.True(t, os.IsNotExist(err), ".backup sidecar should be deleted regardless of IsDeleteWalAfterUpload")
+
+ _, err = os.Stat(filepath.Join(walDir, historyName))
+ assert.True(t, os.IsNotExist(err), ".history sidecar should be deleted regardless of IsDeleteWalAfterUpload")
+}
+
+func Test_UploadSegments_NonMatchingDotBackupFile_FileKept(t *testing.T) {
+ walDir := createTestWalDir(t)
+ unrelatedBackup := "notawal.backup"
+ unrelatedHistory := "random.history"
+ writeTestSegment(t, walDir, unrelatedBackup, []byte("user file"))
+ writeTestSegment(t, walDir, unrelatedHistory, []byte("user file"))
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.ReadAll(r.Body)
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ streamer := newTestStreamer(walDir, server.URL)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+
+ go streamer.Run(ctx)
+ time.Sleep(500 * time.Millisecond)
+ cancel()
+
+ _, err := os.Stat(filepath.Join(walDir, unrelatedBackup))
+ assert.NoError(t, err, "files not matching the Postgres sidecar pattern must be left alone")
+
+ _, err = os.Stat(filepath.Join(walDir, unrelatedHistory))
+ assert.NoError(t, err, "files not matching the Postgres sidecar pattern must be left alone")
+}
+
func newTestStreamer(walDir, serverURL string) *Streamer {
cfg := createTestConfig(walDir, serverURL)
apiClient := api.NewClient(serverURL, cfg.Token, logger.GetLogger())