PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
TEST (physical): Cover WAL history cataloging across a timeline switch (issue #643)
Rostislav Dugin82d83fad7f2b
diff --git a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.goindex 3e8089dc..a42974a7 100644--- a/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go+++ b/backend/internal/features/tests/physical/postgresql/pg17/backup_restore_test.go@@ -27,6 +27,10 @@ func Test_PhysicalRestore_WhenWalGapBeforeTarget_TokenRequestReturns422(t *testiphysicaltesting.RunWhenWalGapBeforeTargetTokenRequestReturns422(t, pgVersion, pgImage)}+func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t *testing.T) {+ physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)+}+func Test_PhysicalRestore_BootViaEntrypointVolumeMount_RecoversBaseRows(t *testing.T) {physicaltesting.RunBootViaEntrypointVolumeMountRecoversBaseRows(t, pgVersion, pgImage)}diff --git a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.goindex a2a5f26e..87c2910f 100644--- a/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go+++ b/backend/internal/features/tests/physical/postgresql/pg18/backup_restore_test.go@@ -27,6 +27,10 @@ func Test_PhysicalRestore_WhenWalGapBeforeTarget_TokenRequestReturns422(t *testiphysicaltesting.RunWhenWalGapBeforeTargetTokenRequestReturns422(t, pgVersion, pgImage)}+func Test_PhysicalWalStream_OnTimelineSwitch_CatalogsHistoryOnParentDatabaseID(t *testing.T) {+ physicaltesting.RunWalStreamCatalogsHistoryOnTimelineSwitch(t, pgVersion, pgImage)+}+func Test_PhysicalRestore_BootViaEntrypointVolumeMount_RecoversBaseRows(t *testing.T) {physicaltesting.RunBootViaEntrypointVolumeMountRecoversBaseRows(t, pgVersion, pgImage)}diff --git a/backend/internal/features/tests/physical/postgresql/shared/bodies.go b/backend/internal/features/tests/physical/postgresql/shared/bodies.goindex 3ccc8a9e..9e75e821 100644--- a/backend/internal/features/tests/physical/postgresql/shared/bodies.go+++ b/backend/internal/features/tests/physical/postgresql/shared/bodies.go@@ -116,6 +116,68 @@ func RunFullTwoIncrementalsPlusWalRecoversToTarget(t *testing.T, version, image"PITR must replay the chain plus rows committed at/before the target and drop the row after it")}+// RunWalStreamCatalogsHistoryOnTimelineSwitch exercises the failover path no other test covers:+// Databasus streams WAL from a standby, the standby is promoted to a new timeline mid-stream, and+// the live WAL stream supervisor must observe the switch and catalog the new timeline's .history+// row. Promotion makes pg_receivewal fetch 00000002.history, driving the supervisor's+// handleHistoryFile -> UploadHistoryFile path — the exact insert that regressed in issue #643:+// keyed on the postgresql_physical_databases PK it violated fk_physical_wal_history_files_database_id+// and aborted the whole WAL stream. The test asserts the history row is cataloged on the parent+// databases.id (and not the physical PK), proving the failover path runs end to end.+//+// It deliberately stops at the catalog assertion rather than running a cross-timeline PITR restore:+// the restore-set resolver loads a chain's WAL on the FULL's timeline only+// (buildChainView -> FindByChainSpan keys on full.TimelineID), so it cannot ship post-fork WAL and a+// cross-timeline restore is unsupported by the current resolver regardless of issue #643. The+// single-timeline PITR restore is already covered by RunFullTwoIncrementalsPlusWalRecoversToTarget.+func RunWalStreamCatalogsHistoryOnTimelineSwitch(t *testing.T, version, image string) {+ if testing.Short() {+ t.Skip("clones a standby and promotes it across a timeline switch; skipped in -short")+ }++ primary, standby := containers.StartPhysicalPrimaryWithStandby(t, image)++ // Databasus backs up the STANDBY: pg_basebackup and pg_receivewal run against it, so promoting it+ // makes the live stream follow the timeline switch — the production failover path. Backups run as+ // the seeded superuser (a standby is read-only, so the replication-only-user provisioning the+ // single-node fixtures do would fail its CREATE ROLE here).+ router := newPhysicalTestRouter()+ fixture := postgresql_executor.SetupPhysicalDBForScheduledBackupVersion(+ t, standby.Host, standby.Port, version, postgresql_physical.BackupTypeFullIncrementalAndWalStream)++ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)+ defer cancel()++ primaryConn := openConnAt(t, primary.Host, primary.Port)+ standbyConn := openSourceTestDBConn(t, fixture)++ // Timeline 1: seed and drive WAL on the primary, let it replicate to the standby, and back the+ // standby up. Writes must hit the primary — the standby is read-only until promotion.+ createMarkerTable(t, ctx, primaryConn)+ insertMarker(t, ctx, primaryConn, "tl1-row", "row-on-tl1")+ waitForMarkerOnStandby(t, ctx, standbyConn, "tl1-row", 60*time.Second)++ enablePhysicalBackupsViaAPI(t, router, fixture, true)+ chain := waitForChainBackups(t, router, fixture, 0, 3*time.Minute)++ // Stream a couple of committed TL1 segments so the slot is actively consuming when we promote.+ streamPostFullSegments(t, ctx, router, primaryConn, fixture, chainTipStopLSN(t, chain), 2, 90*time.Second)++ // Promote: the standby becomes a primary on timeline 2. pg_receivewal follows the switch and+ // uploads 00000002.history, driving the issue #643 path in the live supervisor.+ PromoteStandby(t, ctx, standbyConn)++ // A write on the new timeline guarantees TL2 WAL, so pg_receivewal advances onto timeline 2 and+ // ships its .history.+ insertMarker(t, ctx, standbyConn, "tl2-row", "row-on-tl2")+ _, err := postgresql_executor.GenerateWalActivity(ctx, standbyConn, 16*1024*1024)+ require.NoError(t, err)++ // The regression guard: the supervisor must catalog the timeline-2 history row on the parent+ // databases.id. Pre-fix the FK violation drops it and the wait fails.+ waitForTimelineHistoryOnParent(t, fixture, 2, 60*time.Second)+}+// RunBootViaEntrypointVolumeMountRecoversBaseRows reproduces the user's docker-compose flow: restore on// the host in --combine-image mode, then serve the result through the postgres image's own entrypoint// with the output bind-mounted at the image VOLUME (the volume root on PG 18). The booted cluster mustdiff --git a/backend/internal/features/tests/physical/postgresql/shared/helpers.go b/backend/internal/features/tests/physical/postgresql/shared/helpers.goindex 63d9f85d..2573a17c 100644--- a/backend/internal/features/tests/physical/postgresql/shared/helpers.go+++ b/backend/internal/features/tests/physical/postgresql/shared/helpers.go@@ -24,6 +24,7 @@ import (backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums""databasus-backend/internal/features/backups/backups/core/physical/chain_view"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"+ physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"backups_dto_physical "databasus-backend/internal/features/backups/backups/dto/physical"postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"backups_config_physical "databasus-backend/internal/features/backups/config/physical"@@ -971,13 +972,17 @@ func queryMarkerRowsAt(t *testing.T, host string, port int) []string {func openSourceTestDBConn(t *testing.T, fixture *postgresql_executor.PhysicalDBFixture) *pgx.Conn {t.Helper()+ return openConnAt(t, fixture.DB.PostgresqlPhysical.Host, fixture.DB.PostgresqlPhysical.Port)+}++// openConnAt opens a superuser connection (the seeded testuser) to a PostgreSQL endpoint, closed on+// cleanup. The cross-timeline test reaches the primary through this, since the fixture's own+// connection targets the standby Databasus backs up.+func openConnAt(t *testing.T, host string, port int) *pgx.Conn {+ t.Helper()+dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",- fixture.DB.PostgresqlPhysical.Host,- fixture.DB.PostgresqlPhysical.Port,- restoredPgUser,- restoredPgPassword,- restoredPgDatabase,- )+ host, port, restoredPgUser, restoredPgPassword, restoredPgDatabase)conn, err := pgx.Connect(t.Context(), dsn)require.NoError(t, err)@@ -986,6 +991,99 @@ func openSourceTestDBConn(t *testing.T, fixture *postgresql_executor.PhysicalDBFreturn conn}+// waitForMarkerOnStandby blocks until phase is visible on the standby, i.e. streaming replication+// has carried the primary's write across. The FULL is taken from the standby, so a row must land+// there before it can appear in the base backup. The marker table itself replicates via WAL too, so+// a query before it exists errors with undefined_table — treated here as "not yet replicated".+func waitForMarkerOnStandby(+ t *testing.T,+ ctx context.Context,+ standbyConn *pgx.Conn,+ phase string,+ timeout time.Duration,+) {+ t.Helper()++ deadline := time.Now().UTC().Add(timeout)+ for time.Now().UTC().Before(deadline) {+ var exists bool+ if err := standbyConn.QueryRow(ctx,+ `SELECT EXISTS(SELECT 1 FROM restore_marker WHERE phase = $1)`, phase,+ ).Scan(&exists); err == nil && exists {+ return+ }++ time.Sleep(250 * time.Millisecond)+ }++ t.Fatalf("marker %q never replicated to the standby within %s", phase, timeout)+}++// PromoteStandby promotes a streaming standby to a primary on a fresh timeline and waits until it+// has left recovery. pg_promote keeps the postmaster up, so a pg_receivewal already streaming from+// this node follows the switch and fetches the new timeline's .history file — the path issue #643+// regressed. Callers can write on the new timeline as soon as it returns.+func PromoteStandby(t *testing.T, ctx context.Context, standbyConn *pgx.Conn) {+ t.Helper()++ var promoted bool+ require.NoError(t, standbyConn.QueryRow(ctx,+ `SELECT pg_promote(wait => true, wait_seconds => 60)`).Scan(&promoted))+ require.True(t, promoted, "pg_promote must finish promotion within its wait window")++ deadline := time.Now().UTC().Add(30 * time.Second)+ for time.Now().UTC().Before(deadline) {+ var isInRecovery bool+ require.NoError(t, standbyConn.QueryRow(ctx, `SELECT pg_is_in_recovery()`).Scan(&isInRecovery))+ if !isInRecovery {+ return+ }++ time.Sleep(250 * time.Millisecond)+ }++ t.Fatal("standby still in recovery 30s after pg_promote reported success")+}++// waitForTimelineHistoryOnParent blocks until the WAL stream supervisor has cataloged the timeline's+// .history row, then asserts it is keyed on the parent databases.id and not the+// postgresql_physical_databases PK. Pre-fix (issue #643) the insert violated+// fk_physical_wal_history_files_database_id and the row never appeared, so this both proves the+// failover path runs end to end and guards the regression.+func waitForTimelineHistoryOnParent(+ t *testing.T,+ fixture *postgresql_executor.PhysicalDBFixture,+ timelineID int,+ timeout time.Duration,+) {+ t.Helper()++ historyRepo := physical_repositories.GetWalHistoryRepository()+ parentDatabaseID := fixture.DB.ID+ physicalPK := fixture.DB.PostgresqlPhysical.ID++ deadline := time.Now().UTC().Add(timeout)+ for time.Now().UTC().Before(deadline) {+ byParent, err := historyRepo.FindByDatabaseTimeline(parentDatabaseID, timelineID)+ require.NoError(t, err)++ if byParent != nil {+ byPhysicalPK, err := historyRepo.FindByDatabaseTimeline(physicalPK, timelineID)+ require.NoError(t, err)+ require.Nil(t, byPhysicalPK,+ "timeline-%d history must be keyed on the parent databases.id, not the physical PK", timelineID)++ return+ }++ time.Sleep(500 * time.Millisecond)+ }++ t.Fatalf("WAL stream supervisor never cataloged the timeline-%d history row on the parent databases.id "++ "within %s (issue #643: keying on the physical PK fails fk_physical_wal_history_files_database_id)",+ timelineID, timeout)+}+func connectWithRetry(t *testing.T, dsn string, timeout time.Duration) *pgx.Conn {t.Helper()diff --git a/backend/internal/util/testing/containers/postgres_physical.go b/backend/internal/util/testing/containers/postgres_physical.goindex 96df4d54..836130de 100644--- a/backend/internal/util/testing/containers/postgres_physical.go+++ b/backend/internal/util/testing/containers/postgres_physical.go@@ -12,9 +12,19 @@ import ("github.com/moby/moby/api/types/container""github.com/testcontainers/testcontainers-go"tcexec "github.com/testcontainers/testcontainers-go/exec"+ "github.com/testcontainers/testcontainers-go/network""github.com/testcontainers/testcontainers-go/wait")+// physicalPrimaryAlias is the network alias the standby resolves its primary by. Both containers+// must share a user-defined network for DNS by alias — the default bridge resolves no names.+const physicalPrimaryAlias = "databasus-primary"++// physicalStandbySlotName is the physical replication slot the standby creates on the primary to+// pin WAL until it has streamed it. Distinct from the databasus_* slots Databasus manages on the+// node it backs up.+const physicalStandbySlotName = "databasus_test_standby"+// physicalDataDirTmpfsOptions sizes the tmpfs for physical containers well above the generic// logical size: a replication source retains WAL (wal_keep_size plus a slot that pins every// streamed segment until the uploader commits it), and a WAL-stream test forces a 16 MB segment@@ -25,6 +35,7 @@ const physicalDataDirTmpfsOptions = "rw,size=2g"// physicalPostgresOptions configures a replication-capable source container.type physicalPostgresOptions struct {summarizeWal bool+ fullPageWrites boolwithTablespace boolomitReplicationHbaEntry boolmaxConnections int@@ -93,8 +104,16 @@ func physicalPostgresCmd(opts physicalPostgresOptions) []string {summarize = "summarize_wal=off"}+ // pg_basebackup from a standby refuses a cluster whose WAL was generated with full_page_writes=off+ // ("backup ... is corrupt"), so the replication-pair callers turn it on; single-node sources keep+ // it off for the throwaway-durability speedup.+ fullPageWrites := "full_page_writes=off"+ if opts.fullPageWrites {+ fullPageWrites = "full_page_writes=on"+ }+cmd := []string{- "-c", "fsync=off", "-c", "full_page_writes=off", "-c", "synchronous_commit=off",+ "-c", "fsync=off", "-c", fullPageWrites, "-c", "synchronous_commit=off","-c", "wal_level=logical", "-c", summarize,"-c", "max_wal_senders=10", "-c", "max_replication_slots=10", "-c", "wal_keep_size=512MB",}@@ -137,6 +156,137 @@ func StartPhysicalPostgres(t *testing.T, image string, opts ...PhysicalPostgresOreturn start(t, req, postgresPort)}+// StartPhysicalPrimaryWithStandby boots a replication-capable primary and a streaming standby that+// clones it with pg_basebackup over a shared network, returning both endpoints (primary first).+// Promoting the standby (PromoteStandby) bumps its timeline while the postmaster stays up, so a+// pg_receivewal already streaming from it observes the switch and fetches the new timeline's+// .history file — the failover path issue #643 regressed. The network and both containers are torn+// down with the test.+func StartPhysicalPrimaryWithStandby(+ t *testing.T,+ image string,+ opts ...PhysicalPostgresOption,+) (Endpoint, Endpoint) {+ t.Helper()++ options := physicalPostgresOptions{summarizeWal: true}+ for _, apply := range opts {+ apply(&options)+ }++ // A base backup taken on the standby requires full_page_writes — force it on for both nodes.+ options.fullPageWrites = true++ replicationNetwork, err := network.New(context.Background())+ if err != nil {+ t.Fatalf("failed to create replication test network: %v", err)+ }+ t.Cleanup(func() { _ = replicationNetwork.Remove(context.Background()) })++ primary := startPhysicalPrimaryOnNetwork(t, image, options, replicationNetwork.Name)+ standby := startPhysicalStandbyOnNetwork(t, image, options, replicationNetwork.Name)++ return primary, standby+}++// startPhysicalPrimaryOnNetwork boots the replication source joined to networkName under+// physicalPrimaryAlias, so the standby can reach it by name. It mirrors StartPhysicalPostgres'+// request otherwise (replication pg_hba rule, durability and WAL flags).+func startPhysicalPrimaryOnNetwork(+ t *testing.T,+ image string,+ options physicalPostgresOptions,+ networkName string,+) Endpoint {+ t.Helper()++ var files []testcontainers.ContainerFile+ if !options.omitReplicationHbaEntry {+ files = append(files, allowReplicationInitScript())+ }+ if options.withTablespace {+ files = append(files, createTablespaceInitScript())+ }++ req := testcontainers.ContainerRequest{+ Image: image,+ ExposedPorts: []string{postgresPort},+ Env: postgresEnv(),+ Cmd: physicalPostgresCmd(options),+ Files: files,+ Tmpfs: map[string]string{postgresDataDir(image): physicalDataDirTmpfsOptions},+ Networks: []string{networkName},+ NetworkAliases: map[string][]string{networkName: {physicalPrimaryAlias}},+ WaitingFor: postgresReady(),+ }++ return start(t, req, postgresPort)+}++// startPhysicalStandbyOnNetwork boots a standby that clones the primary (reached by alias on+// networkName) before postgres starts. A custom entrypoint runs pg_basebackup -R into PGDATA, then+// hands the same WAL flags to the image entrypoint, which serves the cloned cluster as a streaming+// standby. The readiness log differs from a primary ("read-only connections"), so it gets its own+// wait.+func startPhysicalStandbyOnNetwork(+ t *testing.T,+ image string,+ options physicalPostgresOptions,+ networkName string,+) Endpoint {+ t.Helper()++ req := testcontainers.ContainerRequest{+ Image: image,+ ExposedPorts: []string{postgresPort},+ Env: postgresEnv(),+ Entrypoint: []string{"/usr/local/bin/databasus-standby-entrypoint.sh"},+ Cmd: physicalPostgresCmd(options),+ Files: []testcontainers.ContainerFile{{+ Reader: strings.NewReader(standbyEntrypointScript()),+ ContainerFilePath: "/usr/local/bin/databasus-standby-entrypoint.sh",+ FileMode: 0o755,+ }},+ Tmpfs: map[string]string{postgresDataDir(image): physicalDataDirTmpfsOptions},+ Networks: []string{networkName},+ WaitingFor: standbyReady(),+ }++ return start(t, req, postgresPort)+}++// standbyEntrypointScript clones the primary into an empty PGDATA and writes the standby recovery+// config (-R records the alias conninfo, password included so streaming reconnects). It runs as+// root (the image default) and drops to postgres for the clone so the cluster files are owned by+// the server uid, then exec's the image entrypoint to serve it. The final "$@" forwards the WAL+// flags passed as the container command.+func standbyEntrypointScript() string {+ return `#!/bin/bash+set -euo pipefail+mkdir -p "$PGDATA"+chown -R postgres:postgres "$PGDATA"+echo "databasus-standby: waiting for primary to accept connections"+until pg_isready -h ` + physicalPrimaryAlias + ` -p 5432 -U ` + PostgresUsername + ` >/dev/null 2>&1; do sleep 1; done+echo "databasus-standby: cloning primary via pg_basebackup"+gosu postgres pg_basebackup -D "$PGDATA" -R -X stream -C -S ` + physicalStandbySlotName + ` -w \+ -d "host=` + physicalPrimaryAlias + ` port=5432 user=` + PostgresUsername ++ ` password=` + PostgresPassword + ` dbname=postgres"+echo "databasus-standby: starting as a streaming standby"+exec docker-entrypoint.sh postgres "$@"+`+}++// standbyReady waits on the standby's own readiness line. A node in archive recovery logs "ready to+// accept read-only connections" (once, after reaching consistency), not the primary's twice-logged+// "ready to accept connections", so postgresReady would hang here.+func standbyReady() wait.Strategy {+ return wait.ForAll(+ wait.ForLog("database system is ready to accept read-only connections").+ WithStartupTimeout(postgresStartupTimeout),+ wait.ForListeningPort(postgresPort),+ )+}+// StartPostgresWithBoundDataDir boots a normal postgres container through its image entrypoint (not// pg_ctl) with hostVolumeDir bind-mounted at the image's data VOLUME, reproducing a docker-compose// '- ./pgdata:<volume>' mount. This is how a restored cluster is actually served, so it exercises the