Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

TEST (physical): Cover WAL history cataloging across a timeline switch (issue #643)

Rostislav Dugin82d83fad7f2b
diff --git a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
index 3e8089dc..a42974a7 100644
--- a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
+++ b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go
@@ -27,6 +27,10 @@ func Test_PhysicalRestore_WhenWalGapBeforeTarget_TokenRequestReturns422(t *testi
physicaltesting.RunWhenWalGapBeforeTargetTokenRequestReturns422(t, pgVersion, pgImage)
}
+func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t *testing.T) {
+ physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)
+}
+
func Test_PhysicalRestore_BootViaEntrypointVolumeMount_RecoversBaseRows(t *testing.T) {
physicaltesting.RunBootViaEntrypointVolumeMountRecoversBaseRows(t, pgVersion, pgImage)
}
diff --git a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
index a2a5f26e..87c2910f 100644
--- a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
+++ b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go
@@ -27,6 +27,10 @@ func Test_PhysicalRestore_WhenWalGapBeforeTarget_TokenRequestReturns422(t *testi
physicaltesting.RunWhenWalGapBeforeTargetTokenRequestReturns422(t, pgVersion, pgImage)
}
+func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t *testing.T) {
+ physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)
+}
+
func Test_PhysicalRestore_BootViaEntrypointVolumeMount_RecoversBaseRows(t *testing.T) {
physicaltesting.RunBootViaEntrypointVolumeMountRecoversBaseRows(t, pgVersion, pgImage)
}
diff --git a/backend/internal/features/tests/physical/postgresql/shared/bodies.go b/backend/internal/features/tests/physical/postgresql/shared/bodies.go
index 3ccc8a9e..9e75e821 100644
--- a/backend/internal/features/tests/physical/postgresql/shared/bodies.go
+++ b/backend/internal/features/tests/physical/postgresql/shared/bodies.go
@@ -116,6 +116,68 @@ func RunFullTwoIncrementalsPlusWalRecoversToTarget(t *testing.T, version, image
"PITR must replay the chain plus rows committed at/before the target and drop the row after it")
}
+// RunWalStreamCatalogsHistoryOnTimelineSwitch exercises the failover path no other test covers:
+// Databasus streams WAL from a standby, the standby is promoted to a new timeline mid-stream, and
+// the live WAL stream supervisor must observe the switch and catalog the new timeline's .history
+// row. Promotion makes pg_receivewal fetch 00000002.history, driving the supervisor's
+// handleHistoryFile -> UploadHistoryFile path — the exact insert that regressed in issue #643:
+// keyed on the postgresql_physical_databases PK it violated fk_physical_wal_history_files_database_id
+// and aborted the whole WAL stream. The test asserts the history row is cataloged on the parent
+// databases.id (and not the physical PK), proving the failover path runs end to end.
+//
+// It deliberately stops at the catalog assertion rather than running a cross-timeline PITR restore:
+// the restore-set resolver loads a chain's WAL on the FULL's timeline only
+// (buildChainView -> FindByChainSpan keys on full.TimelineID), so it cannot ship post-fork WAL and a
+// cross-timeline restore is unsupported by the current resolver regardless of issue #643. The
+// single-timeline PITR restore is already covered by RunFullTwoIncrementalsPlusWalRecoversToTarget.
+func RunWalStreamCatalogsHistoryOnTimelineSwitch(t *testing.T, version, image string) {
+ if testing.Short() {
+ t.Skip("clones a standby and promotes it across a timeline switch; skipped in -short")
+ }
+
+ primary, standby := containers.StartPhysicalPrimaryWithStandby(t, image)
+
+ // Databasus backs up the STANDBY: pg_basebackup and pg_receivewal run against it, so promoting it
+ // makes the live stream follow the timeline switch — the production failover path. Backups run as
+ // the seeded superuser (a standby is read-only, so the replication-only-user provisioning the
+ // single-node fixtures do would fail its CREATE ROLE here).
+ router := newPhysicalTestRouter()
+ fixture := postgresql_executor.SetupPhysicalDBForScheduledBackupVersion(
+ t, standby.Host, standby.Port, version, postgresql_physical.BackupTypeFullIncrementalAndWalStream)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
+ defer cancel()
+
+ primaryConn := openConnAt(t, primary.Host, primary.Port)
+ standbyConn := openSourceTestDBConn(t, fixture)
+
+ // Timeline 1: seed and drive WAL on the primary, let it replicate to the standby, and back the
+ // standby up. Writes must hit the primary — the standby is read-only until promotion.
+ createMarkerTable(t, ctx, primaryConn)
+ insertMarker(t, ctx, primaryConn, "tl1-row", "row-on-tl1")
+ waitForMarkerOnStandby(t, ctx, standbyConn, "tl1-row", 60*time.Second)
+
+ enablePhysicalBackupsViaAPI(t, router, fixture, true)
+ chain := waitForChainBackups(t, router, fixture, 0, 3*time.Minute)
+
+ // Stream a couple of committed TL1 segments so the slot is actively consuming when we promote.
+ streamPostFullSegments(t, ctx, router, primaryConn, fixture, chainTipStopLSN(t, chain), 2, 90*time.Second)
+
+ // Promote: the standby becomes a primary on timeline 2. pg_receivewal follows the switch and
+ // uploads 00000002.history, driving the issue #643 path in the live supervisor.
+ PromoteStandby(t, ctx, standbyConn)
+
+ // A write on the new timeline guarantees TL2 WAL, so pg_receivewal advances onto timeline 2 and
+ // ships its .history.
+ insertMarker(t, ctx, standbyConn, "tl2-row", "row-on-tl2")
+ _, err := postgresql_executor.GenerateWalActivity(ctx, standbyConn, 16*1024*1024)
+ require.NoError(t, err)
+
+ // The regression guard: the supervisor must catalog the timeline-2 history row on the parent
+ // databases.id. Pre-fix the FK violation drops it and the wait fails.
+ waitForTimelineHistoryOnParent(t, fixture, 2, 60*time.Second)
+}
+
// RunBootViaEntrypointVolumeMountRecoversBaseRows reproduces the user's docker-compose flow: restore on
// the host in --combine-image mode, then serve the result through the postgres image's own entrypoint
// with the output bind-mounted at the image VOLUME (the volume root on PG 18). The booted cluster must
diff --git a/backend/internal/features/tests/physical/postgresql/shared/helpers.go b/backend/internal/features/tests/physical/postgresql/shared/helpers.go
index 63d9f85d..2573a17c 100644
--- a/backend/internal/features/tests/physical/postgresql/shared/helpers.go
+++ b/backend/internal/features/tests/physical/postgresql/shared/helpers.go
@@ -24,6 +24,7 @@ import (
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
"databasus-backend/internal/features/backups/backups/core/physical/chain_view"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
backups_dto_physical "databasus-backend/internal/features/backups/backups/dto/physical"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
backups_config_physical "databasus-backend/internal/features/backups/config/physical"
@@ -971,13 +972,17 @@ func queryMarkerRowsAt(t *testing.T, host string, port int) []string {
func openSourceTestDBConn(t *testing.T, fixture *postgresql_executor.PhysicalDBFixture) *pgx.Conn {
t.Helper()
+ return openConnAt(t, fixture.DB.PostgresqlPhysical.Host, fixture.DB.PostgresqlPhysical.Port)
+}
+
+// openConnAt opens a superuser connection (the seeded testuser) to a PostgreSQL endpoint, closed on
+// cleanup. The cross-timeline test reaches the primary through this, since the fixture's own
+// connection targets the standby Databasus backs up.
+func openConnAt(t *testing.T, host string, port int) *pgx.Conn {
+ t.Helper()
+
dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
- fixture.DB.PostgresqlPhysical.Host,
- fixture.DB.PostgresqlPhysical.Port,
- restoredPgUser,
- restoredPgPassword,
- restoredPgDatabase,
- )
+ host, port, restoredPgUser, restoredPgPassword, restoredPgDatabase)
conn, err := pgx.Connect(t.Context(), dsn)
require.NoError(t, err)
@@ -986,6 +991,99 @@ func openSourceTestDBConn(t *testing.T, fixture *postgresql_executor.PhysicalDBF
return conn
}
+// waitForMarkerOnStandby blocks until phase is visible on the standby, i.e. streaming replication
+// has carried the primary's write across. The FULL is taken from the standby, so a row must land
+// there before it can appear in the base backup. The marker table itself replicates via WAL too, so
+// a query before it exists errors with undefined_table — treated here as "not yet replicated".
+func waitForMarkerOnStandby(
+ t *testing.T,
+ ctx context.Context,
+ standbyConn *pgx.Conn,
+ phase string,
+ timeout time.Duration,
+) {
+ t.Helper()
+
+ deadline := time.Now().UTC().Add(timeout)
+ for time.Now().UTC().Before(deadline) {
+ var exists bool
+ if err := standbyConn.QueryRow(ctx,
+ `SELECT EXISTS(SELECT 1 FROM restore_marker WHERE phase = $1)`, phase,
+ ).Scan(&exists); err == nil && exists {
+ return
+ }
+
+ time.Sleep(250 * time.Millisecond)
+ }
+
+ t.Fatalf("marker %q never replicated to the standby within %s", phase, timeout)
+}
+
+// PromoteStandby promotes a streaming standby to a primary on a fresh timeline and waits until it
+// has left recovery. pg_promote keeps the postmaster up, so a pg_receivewal already streaming from
+// this node follows the switch and fetches the new timeline's .history file — the path issue #643
+// regressed. Callers can write on the new timeline as soon as it returns.
+func PromoteStandby(t *testing.T, ctx context.Context, standbyConn *pgx.Conn) {
+ t.Helper()
+
+ var promoted bool
+ require.NoError(t, standbyConn.QueryRow(ctx,
+ `SELECT pg_promote(wait => true, wait_seconds => 60)`).Scan(&promoted))
+ require.True(t, promoted, "pg_promote must finish promotion within its wait window")
+
+ deadline := time.Now().UTC().Add(30 * time.Second)
+ for time.Now().UTC().Before(deadline) {
+ var isInRecovery bool
+ require.NoError(t, standbyConn.QueryRow(ctx, `SELECT pg_is_in_recovery()`).Scan(&isInRecovery))
+ if !isInRecovery {
+ return
+ }
+
+ time.Sleep(250 * time.Millisecond)
+ }
+
+ t.Fatal("standby still in recovery 30s after pg_promote reported success")
+}
+
+// waitForTimelineHistoryOnParent blocks until the WAL stream supervisor has cataloged the timeline's
+// .history row, then asserts it is keyed on the parent databases.id and not the
+// postgresql_physical_databases PK. Pre-fix (issue #643) the insert violated
+// fk_physical_wal_history_files_database_id and the row never appeared, so this both proves the
+// failover path runs end to end and guards the regression.
+func waitForTimelineHistoryOnParent(
+ t *testing.T,
+ fixture *postgresql_executor.PhysicalDBFixture,
+ timelineID int,
+ timeout time.Duration,
+) {
+ t.Helper()
+
+ historyRepo := physical_repositories.GetWalHistoryRepository()
+ parentDatabaseID := fixture.DB.ID
+ physicalPK := fixture.DB.PostgresqlPhysical.ID
+
+ deadline := time.Now().UTC().Add(timeout)
+ for time.Now().UTC().Before(deadline) {
+ byParent, err := historyRepo.FindByDatabaseTimeline(parentDatabaseID, timelineID)
+ require.NoError(t, err)
+
+ if byParent != nil {
+ byPhysicalPK, err := historyRepo.FindByDatabaseTimeline(physicalPK, timelineID)
+ require.NoError(t, err)
+ require.Nil(t, byPhysicalPK,
+ "timeline-%d history must be keyed on the parent databases.id, not the physical PK", timelineID)
+
+ return
+ }
+
+ time.Sleep(500 * time.Millisecond)
+ }
+
+ t.Fatalf("WAL stream supervisor never cataloged the timeline-%d history row on the parent databases.id "+
+ "within %s (issue #643: keying on the physical PK fails fk_physical_wal_history_files_database_id)",
+ timelineID, timeout)
+}
+
func connectWithRetry(t *testing.T, dsn string, timeout time.Duration) *pgx.Conn {
t.Helper()
diff --git a/backend/internal/util/testing/containers/postgres_physical.go b/backend/internal/util/testing/containers/postgres_physical.go
index 96df4d54..836130de 100644
--- a/backend/internal/util/testing/containers/postgres_physical.go
+++ b/backend/internal/util/testing/containers/postgres_physical.go
@@ -12,9 +12,19 @@ import (
"github.com/moby/moby/api/types/container"
"github.com/testcontainers/testcontainers-go"
tcexec "github.com/testcontainers/testcontainers-go/exec"
+ "github.com/testcontainers/testcontainers-go/network"
"github.com/testcontainers/testcontainers-go/wait"
)
+// physicalPrimaryAlias is the network alias the standby resolves its primary by. Both containers
+// must share a user-defined network for DNS by alias — the default bridge resolves no names.
+const physicalPrimaryAlias = "databasus-primary"
+
+// physicalStandbySlotName is the physical replication slot the standby creates on the primary to
+// pin WAL until it has streamed it. Distinct from the databasus_* slots Databasus manages on the
+// node it backs up.
+const physicalStandbySlotName = "databasus_test_standby"
+
// physicalDataDirTmpfsOptions sizes the tmpfs for physical containers well above the generic
// logical size: a replication source retains WAL (wal_keep_size plus a slot that pins every
// streamed segment until the uploader commits it), and a WAL-stream test forces a 16 MB segment
@@ -25,6 +35,7 @@ const physicalDataDirTmpfsOptions = "rw,size=2g"
// physicalPostgresOptions configures a replication-capable source container.
type physicalPostgresOptions struct {
summarizeWal bool
+ fullPageWrites bool
withTablespace bool
omitReplicationHbaEntry bool
maxConnections int
@@ -93,8 +104,16 @@ func physicalPostgresCmd(opts physicalPostgresOptions) []string {
summarize = "summarize_wal=off"
}
+ // pg_basebackup from a standby refuses a cluster whose WAL was generated with full_page_writes=off
+ // ("backup ... is corrupt"), so the replication-pair callers turn it on; single-node sources keep
+ // it off for the throwaway-durability speedup.
+ fullPageWrites := "full_page_writes=off"
+ if opts.fullPageWrites {
+ fullPageWrites = "full_page_writes=on"
+ }
+
cmd := []string{
- "-c", "fsync=off", "-c", "full_page_writes=off", "-c", "synchronous_commit=off",
+ "-c", "fsync=off", "-c", fullPageWrites, "-c", "synchronous_commit=off",
"-c", "wal_level=logical", "-c", summarize,
"-c", "max_wal_senders=10", "-c", "max_replication_slots=10", "-c", "wal_keep_size=512MB",
}
@@ -137,6 +156,137 @@ func StartPhysicalPostgres(t *testing.T, image string, opts ...PhysicalPostgresO
return start(t, req, postgresPort)
}
+// StartPhysicalPrimaryWithStandby boots a replication-capable primary and a streaming standby that
+// clones it with pg_basebackup over a shared network, returning both endpoints (primary first).
+// Promoting the standby (PromoteStandby) bumps its timeline while the postmaster stays up, so a
+// pg_receivewal already streaming from it observes the switch and fetches the new timeline's
+// .history file — the failover path issue #643 regressed. The network and both containers are torn
+// down with the test.
+func StartPhysicalPrimaryWithStandby(
+ t *testing.T,
+ image string,
+ opts ...PhysicalPostgresOption,
+) (Endpoint, Endpoint) {
+ t.Helper()
+
+ options := physicalPostgresOptions{summarizeWal: true}
+ for _, apply := range opts {
+ apply(&options)
+ }
+
+ // A base backup taken on the standby requires full_page_writes — force it on for both nodes.
+ options.fullPageWrites = true
+
+ replicationNetwork, err := network.New(context.Background())
+ if err != nil {
+ t.Fatalf("failed to create replication test network: %v", err)
+ }
+ t.Cleanup(func() { _ = replicationNetwork.Remove(context.Background()) })
+
+ primary := startPhysicalPrimaryOnNetwork(t, image, options, replicationNetwork.Name)
+ standby := startPhysicalStandbyOnNetwork(t, image, options, replicationNetwork.Name)
+
+ return primary, standby
+}
+
+// startPhysicalPrimaryOnNetwork boots the replication source joined to networkName under
+// physicalPrimaryAlias, so the standby can reach it by name. It mirrors StartPhysicalPostgres'
+// request otherwise (replication pg_hba rule, durability and WAL flags).
+func startPhysicalPrimaryOnNetwork(
+ t *testing.T,
+ image string,
+ options physicalPostgresOptions,
+ networkName string,
+) Endpoint {
+ t.Helper()
+
+ var files []testcontainers.ContainerFile
+ if !options.omitReplicationHbaEntry {
+ files = append(files, allowReplicationInitScript())
+ }
+ if options.withTablespace {
+ files = append(files, createTablespaceInitScript())
+ }
+
+ req := testcontainers.ContainerRequest{
+ Image: image,
+ ExposedPorts: []string{postgresPort},
+ Env: postgresEnv(),
+ Cmd: physicalPostgresCmd(options),
+ Files: files,
+ Tmpfs: map[string]string{postgresDataDir(image): physicalDataDirTmpfsOptions},
+ Networks: []string{networkName},
+ NetworkAliases: map[string][]string{networkName: {physicalPrimaryAlias}},
+ WaitingFor: postgresReady(),
+ }
+
+ return start(t, req, postgresPort)
+}
+
+// startPhysicalStandbyOnNetwork boots a standby that clones the primary (reached by alias on
+// networkName) before postgres starts. A custom entrypoint runs pg_basebackup -R into PGDATA, then
+// hands the same WAL flags to the image entrypoint, which serves the cloned cluster as a streaming
+// standby. The readiness log differs from a primary ("read-only connections"), so it gets its own
+// wait.
+func startPhysicalStandbyOnNetwork(
+ t *testing.T,
+ image string,
+ options physicalPostgresOptions,
+ networkName string,
+) Endpoint {
+ t.Helper()
+
+ req := testcontainers.ContainerRequest{
+ Image: image,
+ ExposedPorts: []string{postgresPort},
+ Env: postgresEnv(),
+ Entrypoint: []string{"/usr/local/bin/databasus-standby-entrypoint.sh"},
+ Cmd: physicalPostgresCmd(options),
+ Files: []testcontainers.ContainerFile{{
+ Reader: strings.NewReader(standbyEntrypointScript()),
+ ContainerFilePath: "/usr/local/bin/databasus-standby-entrypoint.sh",
+ FileMode: 0o755,
+ }},
+ Tmpfs: map[string]string{postgresDataDir(image): physicalDataDirTmpfsOptions},
+ Networks: []string{networkName},
+ WaitingFor: standbyReady(),
+ }
+
+ return start(t, req, postgresPort)
+}
+
+// standbyEntrypointScript clones the primary into an empty PGDATA and writes the standby recovery
+// config (-R records the alias conninfo, password included so streaming reconnects). It runs as
+// root (the image default) and drops to postgres for the clone so the cluster files are owned by
+// the server uid, then exec's the image entrypoint to serve it. The final "$@" forwards the WAL
+// flags passed as the container command.
+func standbyEntrypointScript() string {
+ return `#!/bin/bash
+set -euo pipefail
+mkdir -p "$PGDATA"
+chown -R postgres:postgres "$PGDATA"
+echo "databasus-standby: waiting for primary to accept connections"
+until pg_isready -h ` + physicalPrimaryAlias + ` -p 5432 -U ` + PostgresUsername + ` >/dev/null 2>&1; do sleep 1; done
+echo "databasus-standby: cloning primary via pg_basebackup"
+gosu postgres pg_basebackup -D "$PGDATA" -R -X stream -C -S ` + physicalStandbySlotName + ` -w \
+ -d "host=` + physicalPrimaryAlias + ` port=5432 user=` + PostgresUsername +
+ ` password=` + PostgresPassword + ` dbname=postgres"
+echo "databasus-standby: starting as a streaming standby"
+exec docker-entrypoint.sh postgres "$@"
+`
+}
+
+// standbyReady waits on the standby's own readiness line. A node in archive recovery logs "ready to
+// accept read-only connections" (once, after reaching consistency), not the primary's twice-logged
+// "ready to accept connections", so postgresReady would hang here.
+func standbyReady() wait.Strategy {
+ return wait.ForAll(
+ wait.ForLog("database system is ready to accept read-only connections").
+ WithStartupTimeout(postgresStartupTimeout),
+ wait.ForListeningPort(postgresPort),
+ )
+}
+
// StartPostgresWithBoundDataDir boots a normal postgres container through its image entrypoint (not
// pg_ctl) with hostVolumeDir bind-mounted at the image's data VOLUME, reproducing a docker-compose
// '- ./pgdata:<volume>' mount. This is how a restored cluster is actually served, so it exercises the