Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Read the timeline from the WAL file name, not the control file

Rostislav Dugin7d4fc630e254
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
index 99f6f5a6..ad7814c7 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
@@ -328,20 +328,67 @@ func UploadHistoryFile(
return row, nil
}
-func readClusterIdentity(ctx context.Context, conn *pgx.Conn) (int, string, error) {
- var tli int
- var sysID string
+type clusterIdentitySources struct {
+ // NULL during recovery, where pg_current_wal_lsn() cannot be called.
+ CurrentWalFilename *string
+ ControlFileTimelineID int
+ SystemIdentifier string
+}
+
+func readClusterIdentitySources(ctx context.Context, conn *pgx.Conn) (clusterIdentitySources, error) {
+ var sources clusterIdentitySources
err := conn.QueryRow(ctx, `
SELECT
+ CASE WHEN pg_is_in_recovery() THEN NULL ELSE pg_walfile_name(pg_current_wal_lsn()) END,
(SELECT timeline_id FROM pg_control_checkpoint()),
(SELECT system_identifier::text FROM pg_control_system())
- `).Scan(&tli, &sysID)
+ `).Scan(
+ &sources.CurrentWalFilename,
+ &sources.ControlFileTimelineID,
+ &sources.SystemIdentifier,
+ )
+ if err != nil {
+ return clusterIdentitySources{}, fmt.Errorf("read cluster identity: %w", err)
+ }
+
+ return sources, nil
+}
+
+func readClusterIdentity(ctx context.Context, conn *pgx.Conn) (int, string, error) {
+ sources, err := readClusterIdentitySources(ctx, conn)
if err != nil {
- return 0, "", fmt.Errorf("read cluster identity: %w", err)
+ return 0, "", err
+ }
+
+ timelineID, err := resolveLiveTimelineID(sources)
+ if err != nil {
+ return 0, "", err
+ }
+
+ return timelineID, sources.SystemIdentifier, nil
+}
+
+// Every cluster answers the control file, but it lags a promotion: PostgreSQL stamps it
+// at checkpoints, and the checkpoint a promotion requests is spread across
+// checkpoint_completion_target, so a promoted primary keeps naming the pre-failover
+// timeline for minutes. A stale reading lets an incremental extend a chain anchored to a
+// timeline the cluster has already left.
+//
+// A source still in recovery has no fresher reading to offer: pg_walfile_name() refuses
+// to run there, and pg_stat_wal_receiver hides received_tli from a role without
+// pg_read_all_stats, which the replication user this product provisions does not have.
+func resolveLiveTimelineID(sources clusterIdentitySources) (int, error) {
+ if sources.CurrentWalFilename != nil {
+ timelineID, err := walmath.ParseWALFilenameTimeline(*sources.CurrentWalFilename)
+ if err != nil {
+ return 0, fmt.Errorf("read timeline from current WAL file name: %w", err)
+ }
+
+ return int(timelineID), nil
}
- return tli, sysID, nil
+ return sources.ControlFileTimelineID, nil
}
func newestKnownTimeline(
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go
index e9b9aa49..75dba2cf 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go
@@ -6,10 +6,16 @@ import (
"io"
"log/slog"
"testing"
+ "time"
+ "github.com/jackc/pgx/v5"
"github.com/stretchr/testify/require"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ "databasus-backend/internal/features/databases"
+ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/testing/containers"
)
func Test_DecideTimelineCompatibility_WithMatchingAndConflictingIdentity_ReturnsExpectedDecision(t *testing.T) {
@@ -219,3 +225,121 @@ func Test_RecheckStreamFailure_WhenIdentityProbeFails_PreservesOriginalResult(t
require.Equal(t, originalResult, fullResult)
require.Equal(t, originalResult, incrementalResult)
}
+
+func Test_ResolveLiveTimelineID_WhenSourcesDisagree_PrefersTheFreshestReading(t *testing.T) {
+ promotedPrimaryWalFilename := "000000020000000000000005"
+
+ tests := []struct {
+ name string
+ sources clusterIdentitySources
+ resolvedTimelineID int
+ }{
+ {
+ name: "promoted primary whose control file has not caught up",
+ sources: clusterIdentitySources{
+ CurrentWalFilename: &promotedPrimaryWalFilename,
+ ControlFileTimelineID: 1,
+ },
+ resolvedTimelineID: 2,
+ },
+ {
+ name: "cluster in recovery, which can only answer the control file",
+ sources: clusterIdentitySources{ControlFileTimelineID: 3},
+ resolvedTimelineID: 3,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ timelineID, err := resolveLiveTimelineID(test.sources)
+
+ require.NoError(t, err)
+ require.Equal(t, test.resolvedTimelineID, timelineID)
+ })
+ }
+}
+
+func Test_ResolveLiveTimelineID_WhenWalFilenameIsUnreadable_ReturnsError(t *testing.T) {
+ unreadableWalFilename := "not-a-wal-file"
+
+ _, err := resolveLiveTimelineID(clusterIdentitySources{
+ CurrentWalFilename: &unreadableWalFilename,
+ ControlFileTimelineID: 1,
+ })
+
+ require.Error(
+ t,
+ err,
+ "a source naming its WAL file in a shape we cannot read must not fall back to a stale timeline",
+ )
+}
+
+const timelineTestPostgresVersion = "17"
+
+func Test_ReadClusterIdentity_WhenTheControlFileLagsAPromotion_ReportsTheLiveTimeline(t *testing.T) {
+ if testing.Short() {
+ t.Skip("clones a standby and promotes it across a timeline switch; skipped in -short")
+ }
+
+ primary, standby := containers.StartPhysicalPrimaryWithStandby(
+ t, "postgres:"+timelineTestPostgresVersion, containers.WithCheckpointsSpreadOverAnHour())
+
+ ctx := t.Context()
+ primaryConn := openPhysicalTestConn(t, ctx, primary)
+ standbyConn := openPhysicalTestConn(t, ctx, standby)
+
+ // Replaying these rows dirties buffers on the standby, which is what leaves the
+ // post-promotion checkpoint work to spread instead of finishing at once.
+ _, err := primaryConn.Exec(ctx, "CREATE TABLE promotion_ballast AS SELECT generate_series(1, 300000) AS n")
+ require.NoError(t, err)
+ waitForReplayedRelation(t, ctx, standbyConn, "promotion_ballast")
+
+ var isPromoted bool
+ require.NoError(t, standbyConn.QueryRow(ctx,
+ "SELECT pg_promote(wait => true, wait_seconds => 60)").Scan(&isPromoted))
+ require.True(t, isPromoted, "pg_promote must finish promotion within its wait window")
+
+ var controlFileTimelineID int
+ require.NoError(t, standbyConn.QueryRow(ctx,
+ "SELECT timeline_id FROM pg_control_checkpoint()").Scan(&controlFileTimelineID))
+ require.Equal(t, 1, controlFileTimelineID,
+ "the spread-checkpoint source must keep the control file behind, or this test proves nothing")
+
+ liveTimelineID, _, err := readClusterIdentity(ctx, standbyConn)
+ require.NoError(t, err)
+ require.Equal(t, 2, liveTimelineID,
+ "a promoted source must report the timeline it writes WAL on, not the one its control file was stamped with")
+}
+
+func openPhysicalTestConn(t *testing.T, ctx context.Context, endpoint containers.Endpoint) *pgx.Conn {
+ t.Helper()
+
+ sourceDB := databases.GetTestPhysicalPostgresConfigWithType(
+ endpoint.Host, endpoint.Port, timelineTestPostgresVersion, postgresql_physical.BackupTypeFullOnly)
+
+ conn, err := sourceDB.OpenInspectionConn(ctx, encryption.GetFieldEncryptor())
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = conn.Close(context.Background()) })
+
+ return conn
+}
+
+func waitForReplayedRelation(t *testing.T, ctx context.Context, standbyConn *pgx.Conn, relation string) {
+ t.Helper()
+
+ deadline := time.Now().UTC().Add(60 * time.Second)
+
+ for time.Now().UTC().Before(deadline) {
+ var replayedRelation *string
+
+ require.NoError(t, standbyConn.QueryRow(ctx, "SELECT to_regclass($1)::text", relation).Scan(&replayedRelation))
+
+ if replayedRelation != nil {
+ return
+ }
+
+ time.Sleep(200 * time.Millisecond)
+ }
+
+ t.Fatalf("relation %q never replayed onto the standby", relation)
+}
diff --git a/backend/internal/features/tests/physical/postgresql/shared/bodies.go b/backend/internal/features/tests/physical/postgresql/shared/bodies.go
index 49e60e9d..787c3b7f 100644
--- a/backend/internal/features/tests/physical/postgresql/shared/bodies.go
+++ b/backend/internal/features/tests/physical/postgresql/shared/bodies.go
@@ -6,17 +6,13 @@ import (
"net/http"
"os"
"path/filepath"
- "slices"
"testing"
"time"
- "github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
backuping_physical "databasus-backend/internal/features/backups/backups/backuping/physical"
- physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
- physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
@@ -215,36 +211,7 @@ func RunPromotionReanchorsIncrementalChain(t *testing.T, version, image string)
waitForTimelineHistoryOnParent(t, fixture, 2, 60*time.Second)
triggerIncrementalViaAPI(t, router, fixture)
- deadline := time.Now().UTC().Add(4 * time.Minute)
- var replacementFullID uuid.UUID
- for time.Now().UTC().Before(deadline) {
- incrementals, err := physical_repositories.GetIncrementalBackupRepository().FindAllByRootFull(initialFullID)
- require.NoError(t, err)
- fulls, err := physical_repositories.GetFullBackupRepository().FindCompletedNewestFirstByDatabase(fixture.DB.ID)
- require.NoError(t, err)
-
- hasTimelineRefusal := slices.ContainsFunc(
- incrementals,
- func(incremental *physical_models.PhysicalIncrementalBackup) bool {
- return incremental.Status == physical_enums.PhysicalBackupStatusChainBroken &&
- incremental.ErrorReason != nil &&
- *incremental.ErrorReason == physical_enums.PhysicalBackupErrorTimelineSwitchDetected
- },
- )
- if hasTimelineRefusal && len(fulls) >= 2 && fulls[0].TimelineID == 2 {
- replacementFullID = fulls[0].ID
-
- break
- }
-
- time.Sleep(500 * time.Millisecond)
- }
- require.NotEqual(
- t,
- uuid.Nil,
- replacementFullID,
- "promotion must refuse the stale incremental and complete a timeline-2 FULL",
- )
+ waitForReanchoredFull(t, fixture, initialFullID, 2, 4*time.Minute)
backuping_physical.RunOrphanWalCleanupForTest(t, fixture.DB.ID)
bundle := downloadRestoreBundleViaAPI(t, router, fixture, nil)
diff --git a/backend/internal/features/tests/physical/postgresql/shared/helpers.go b/backend/internal/features/tests/physical/postgresql/shared/helpers.go
index d9833999..572bec41 100644
--- a/backend/internal/features/tests/physical/postgresql/shared/helpers.go
+++ b/backend/internal/features/tests/physical/postgresql/shared/helpers.go
@@ -24,6 +24,7 @@ import (
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
"databasus-backend/internal/features/backups/backups/core/physical/chain_view"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
backups_dto_physical "databasus-backend/internal/features/backups/backups/dto/physical"
postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"
@@ -1137,6 +1138,138 @@ func waitForTimelineHistoryOnParent(
timelineID, timeout)
}
+// Re-anchoring crosses the incremental executor, the scheduler and the FULL executor,
+// and each of them can stop it for its own reason, so the timeout prints the catalog
+// rows it last read rather than only the verdict.
+func waitForReanchoredFull(
+ t *testing.T,
+ fixture *postgresql_executor.PhysicalDBFixture,
+ staleFullID uuid.UUID,
+ newTimelineID int,
+ timeout time.Duration,
+) {
+ t.Helper()
+
+ deadline := time.Now().UTC().Add(timeout)
+
+ var state reanchorCatalogState
+
+ for time.Now().UTC().Before(deadline) {
+ state = readReanchorCatalogState(t, fixture, staleFullID)
+
+ hasReplacementFull := len(state.CompletedFulls) >= 2 &&
+ state.CompletedFulls[0].TimelineID == newTimelineID
+
+ if hasTimelineRefusal(state.StaleChainIncrementals) && hasReplacementFull {
+ return
+ }
+
+ time.Sleep(500 * time.Millisecond)
+ }
+
+ t.Fatalf("promotion did not re-anchor within %s: the stale chain must refuse an incremental with %s, "+
+ "and a COMPLETED FULL on timeline %d must follow it\n"+
+ " stale-chain incrementals: %s\n"+
+ " completed fulls: %s\n"+
+ " newest full (any status): %s",
+ timeout, physical_enums.PhysicalBackupErrorTimelineSwitchDetected, newTimelineID,
+ describeIncrementalOutcomes(state.StaleChainIncrementals),
+ describeFullOutcomes(state.CompletedFulls),
+ describeFullOutcome(state.NewestFullAnyStatus))
+}
+
+type reanchorCatalogState struct {
+ StaleChainIncrementals []*physical_models.PhysicalIncrementalBackup
+ CompletedFulls []*physical_models.PhysicalFullBackup
+ // A replacement FULL that failed is absent from the completed list.
+ NewestFullAnyStatus *physical_models.PhysicalFullBackup
+}
+
+// waitForReanchoredFull polls for a typed error reason, which only the repository rows
+// carry; the backup list DTO has just the free-text fail message.
+func readReanchorCatalogState(
+ t *testing.T,
+ fixture *postgresql_executor.PhysicalDBFixture,
+ staleFullID uuid.UUID,
+) reanchorCatalogState {
+ t.Helper()
+
+ incrementalRepo := physical_repositories.GetIncrementalBackupRepository()
+ fullRepo := physical_repositories.GetFullBackupRepository()
+
+ staleChainIncrementals, err := incrementalRepo.FindAllByRootFull(staleFullID)
+ require.NoError(t, err)
+
+ completedFulls, err := fullRepo.FindCompletedNewestFirstByDatabase(fixture.DB.ID)
+ require.NoError(t, err)
+
+ newestFullAnyStatus, err := fullRepo.FindLastFullAnyStatusByDatabase(fixture.DB.ID)
+ require.NoError(t, err)
+
+ return reanchorCatalogState{
+ StaleChainIncrementals: staleChainIncrementals,
+ CompletedFulls: completedFulls,
+ NewestFullAnyStatus: newestFullAnyStatus,
+ }
+}
+
+func hasTimelineRefusal(incrementals []*physical_models.PhysicalIncrementalBackup) bool {
+ return slices.ContainsFunc(
+ incrementals,
+ func(incremental *physical_models.PhysicalIncrementalBackup) bool {
+ return incremental.Status == physical_enums.PhysicalBackupStatusChainBroken &&
+ incremental.ErrorReason != nil &&
+ *incremental.ErrorReason == physical_enums.PhysicalBackupErrorTimelineSwitchDetected
+ },
+ )
+}
+
+func describeIncrementalOutcomes(incrementals []*physical_models.PhysicalIncrementalBackup) string {
+ descriptions := make([]string, 0, len(incrementals))
+
+ for _, incremental := range incrementals {
+ descriptions = append(descriptions, fmt.Sprintf("%s %s (%s)",
+ incremental.ID, incremental.Status, describeErrorReason(incremental.ErrorReason)))
+ }
+
+ return joinOutcomesOrNone(descriptions)
+}
+
+func describeFullOutcomes(fulls []*physical_models.PhysicalFullBackup) string {
+ descriptions := make([]string, 0, len(fulls))
+
+ for _, full := range fulls {
+ descriptions = append(descriptions, describeFullOutcome(full))
+ }
+
+ return joinOutcomesOrNone(descriptions)
+}
+
+func describeFullOutcome(full *physical_models.PhysicalFullBackup) string {
+ if full == nil {
+ return "none"
+ }
+
+ return fmt.Sprintf("%s %s on TL %d (%s)",
+ full.ID, full.Status, full.TimelineID, describeErrorReason(full.ErrorReason))
+}
+
+func joinOutcomesOrNone(descriptions []string) string {
+ if len(descriptions) == 0 {
+ return "none"
+ }
+
+ return strings.Join(descriptions, "; ")
+}
+
+func describeErrorReason(reason *physical_enums.PhysicalBackupErrorReason) string {
+ if reason == nil {
+ return "no error reason"
+ }
+
+ return string(*reason)
+}
+
func connectWithRetry(t *testing.T, dsn string, timeout time.Duration) *pgx.Conn {
t.Helper()
diff --git a/backend/internal/util/testing/containers/postgres_physical.go b/backend/internal/util/testing/containers/postgres_physical.go
index 57890754..b10f3a2a 100644
--- a/backend/internal/util/testing/containers/postgres_physical.go
+++ b/backend/internal/util/testing/containers/postgres_physical.go
@@ -32,13 +32,13 @@ const physicalStandbySlotName = "databasus_test_standby"
// logical default fills mid-test and the source crashes with "No space left on device".
const physicalDataDirTmpfsOptions = "rw,size=2g"
-// physicalPostgresOptions configures a replication-capable source container.
type physicalPostgresOptions struct {
- summarizeWal bool
- fullPageWrites bool
- withTablespace bool
- omitReplicationHbaEntry bool
- maxConnections int
+ shouldSummarizeWal bool
+ shouldEnableFullPageWrites bool
+ shouldCreateTablespace bool
+ shouldOmitReplicationHbaEntry bool
+ shouldSpreadCheckpoints bool
+ maxConnections int
}
// PhysicalPostgresOption tunes a physical source container away from its replication-ready default.
@@ -47,20 +47,20 @@ type PhysicalPostgresOption func(*physicalPostgresOptions)
// WithoutSummarizer starts the cluster with summarize_wal=off, so an incremental pre-flight reaches
// the SUMMARIZER_OFF fallback deterministically.
func WithoutSummarizer() PhysicalPostgresOption {
- return func(o *physicalPostgresOptions) { o.summarizeWal = false }
+ return func(o *physicalPostgresOptions) { o.shouldSummarizeWal = false }
}
// WithTablespace pre-creates a custom tablespace at first boot so the pre-flight rejects the cluster
// per ADR-0010 (physical backups do not support custom tablespaces).
func WithTablespace() PhysicalPostgresOption {
- return func(o *physicalPostgresOptions) { o.withTablespace = true }
+ return func(o *physicalPostgresOptions) { o.shouldCreateTablespace = true }
}
// WithoutReplicationHbaEntry omits the "host replication" pg_hba line, leaving only the image's
// default "host all all all" rule. Such a cluster accepts ordinary and logical-replication
// connections but refuses physical replication — the mode pg_basebackup / pg_receivewal actually use.
func WithoutReplicationHbaEntry() PhysicalPostgresOption {
- return func(o *physicalPostgresOptions) { o.omitReplicationHbaEntry = true }
+ return func(o *physicalPostgresOptions) { o.shouldOmitReplicationHbaEntry = true }
}
// WithMaxConnections runs the source with a raised max_connections so a restored cluster booted at
@@ -70,6 +70,14 @@ func WithMaxConnections(connections int) PhysicalPostgresOption {
return func(o *physicalPostgresOptions) { o.maxConnections = connections }
}
+// A promotion only requests its checkpoint, so on a node carrying dirty buffers that request stays
+// outstanding for the length of a test and pg_control_checkpoint() keeps answering the pre-failover
+// timeline — the stale reading a timeline probe must not trust. The completion target is pinned so
+// the pacing does not depend on the image's default.
+func WithCheckpointsSpreadOverAnHour() PhysicalPostgresOption {
+ return func(o *physicalPostgresOptions) { o.shouldSpreadCheckpoints = true }
+}
+
// allowReplicationInitScript appends the pg_hba replication rule at first boot. "host all all" does
// not cover replication connections, so pg_basebackup / pg_receivewal need this explicit line.
func allowReplicationInitScript() testcontainers.ContainerFile {
@@ -100,7 +108,7 @@ func createTablespaceInitScript() testcontainers.ContainerFile {
func physicalPostgresCmd(opts physicalPostgresOptions) []string {
summarize := "summarize_wal=on"
- if !opts.summarizeWal {
+ if !opts.shouldSummarizeWal {
summarize = "summarize_wal=off"
}
@@ -108,7 +116,7 @@ func physicalPostgresCmd(opts physicalPostgresOptions) []string {
// ("backup ... is corrupt"), so the replication-pair callers turn it on; single-node sources keep
// it off for the throwaway-durability speedup.
fullPageWrites := "full_page_writes=off"
- if opts.fullPageWrites {
+ if opts.shouldEnableFullPageWrites {
fullPageWrites = "full_page_writes=on"
}
@@ -121,6 +129,10 @@ func physicalPostgresCmd(opts physicalPostgresOptions) []string {
cmd = append(cmd, "-c", fmt.Sprintf("max_connections=%d", opts.maxConnections))
}
+ if opts.shouldSpreadCheckpoints {
+ cmd = append(cmd, "-c", "checkpoint_timeout=1h", "-c", "checkpoint_completion_target=0.9")
+ }
+
return cmd
}
@@ -161,7 +173,7 @@ func StartPhysicalPostgresBehindSshBastion(
}
func resolvePhysicalPostgresOptions(opts []PhysicalPostgresOption) physicalPostgresOptions {
- options := physicalPostgresOptions{summarizeWal: true}
+ options := physicalPostgresOptions{shouldSummarizeWal: true}
for _, apply := range opts {
apply(&options)
}
@@ -171,10 +183,10 @@ func resolvePhysicalPostgresOptions(opts []PhysicalPostgresOption) physicalPostg
func physicalPostgresRequest(image string, options physicalPostgresOptions) testcontainers.ContainerRequest {
var files []testcontainers.ContainerFile
- if !options.omitReplicationHbaEntry {
+ if !options.shouldOmitReplicationHbaEntry {
files = append(files, allowReplicationInitScript())
}
- if options.withTablespace {
+ if options.shouldCreateTablespace {
files = append(files, createTablespaceInitScript())
}
@@ -202,13 +214,10 @@ func StartPhysicalPrimaryWithStandby(
) (Endpoint, Endpoint) {
t.Helper()
- options := physicalPostgresOptions{summarizeWal: true}
- for _, apply := range opts {
- apply(&options)
- }
+ options := resolvePhysicalPostgresOptions(opts)
// A base backup taken on the standby requires full_page_writes — force it on for both nodes.
- options.fullPageWrites = true
+ options.shouldEnableFullPageWrites = true
replicationNetwork, err := network.New(context.Background())
if err != nil {
@@ -285,7 +294,7 @@ chown -R postgres:postgres "$PGDATA"
echo "databasus-standby: waiting for primary to accept connections"
until pg_isready -h ` + physicalPrimaryAlias + ` -p 5432 -U ` + PostgresUsername + ` >/dev/null 2>&1; do sleep 1; done
echo "databasus-standby: cloning primary via pg_basebackup"
-gosu postgres pg_basebackup -D "$PGDATA" -R -X stream -C -S ` + physicalStandbySlotName + ` -w \
+gosu postgres pg_basebackup -D "$PGDATA" -R -X stream -c fast -C -S ` + physicalStandbySlotName + ` -w \
-d "host=` + physicalPrimaryAlias + ` port=5432 user=` + PostgresUsername +
` password=` + PostgresPassword + ` dbname=postgres"
echo "databasus-standby: starting as a streaming standby"
diff --git a/backend/internal/util/walmath/constants.go b/backend/internal/util/walmath/constants.go
index 8cb227e5..540031e6 100644
--- a/backend/internal/util/walmath/constants.go
+++ b/backend/internal/util/walmath/constants.go
@@ -16,6 +16,9 @@ const (
maxCountOfLSN = 2
hexUint32Bits = 32
+ hexUint64Bits = 64
+
+ walSegmentFilenameLength = 24
)
func SetWalSize(sizeMb uint64) {
diff --git a/backend/internal/util/walmath/wal_filename.go b/backend/internal/util/walmath/wal_filename.go
index eb97fb9d..d75fc10f 100644
--- a/backend/internal/util/walmath/wal_filename.go
+++ b/backend/internal/util/walmath/wal_filename.go
@@ -5,6 +5,7 @@ package walmath
import (
"fmt"
+ "math"
"regexp"
"strconv"
)
@@ -15,39 +16,53 @@ func ParseWALFilename(name string) (timelineID uint32, logSegNo uint64, err erro
return ParseWALFilenameWithSize(name, WalSegmentSize)
}
+// Unlike ParseWALFilename, this holds for a cluster whose WAL segment size differs
+// from the configured one: the timeline field does not depend on it.
+func ParseWALFilenameTimeline(name string) (uint32, error) {
+ timelineID, _, err := parseWalSegmentFields(name)
+
+ return timelineID, err
+}
+
func ParseWALFilenameWithSize(name string, segmentSize uint64) (timelineID uint32, logSegNo uint64, err error) {
- if len(name) != 24 {
- return 0, 0, NotWalFilenameError{Filename: name}
+ timelineID, logSegmentFields, err := parseWalSegmentFields(name)
+ if err != nil {
+ return 0, 0, err
}
if segmentSize == 0 || 0x100000000%segmentSize != 0 {
return 0, 0, IncorrectLogSegNoError{Filename: name}
}
- timelineID64, err := strconv.ParseUint(name[0:8], 0x10, hexUint32Bits)
- if err != nil {
- return 0, 0, NotWalFilenameError{Filename: name}
+ logSegNoHi := logSegmentFields >> hexUint32Bits
+ logSegNoLo := logSegmentFields & math.MaxUint32
+
+ segmentsPerLogID := 0x100000000 / segmentSize
+ if logSegNoLo >= segmentsPerLogID {
+ return 0, 0, IncorrectLogSegNoError{Filename: name}
}
- logSegNoHi, err := strconv.ParseUint(name[8:16], 0x10, hexUint32Bits)
- if err != nil {
+ logSegNo = logSegNoHi*segmentsPerLogID + logSegNoLo
+
+ return timelineID, logSegNo, nil
+}
+
+func parseWalSegmentFields(name string) (timelineID uint32, logSegmentFields uint64, err error) {
+ if len(name) != walSegmentFilenameLength {
return 0, 0, NotWalFilenameError{Filename: name}
}
- logSegNoLo, err := strconv.ParseUint(name[16:24], 0x10, hexUint32Bits)
+ parsedTimelineID, err := strconv.ParseUint(name[0:8], 0x10, hexUint32Bits)
if err != nil {
return 0, 0, NotWalFilenameError{Filename: name}
}
- segmentsPerLogID := 0x100000000 / segmentSize
- if logSegNoLo >= segmentsPerLogID {
- return 0, 0, IncorrectLogSegNoError{Filename: name}
+ logSegmentFields, err = strconv.ParseUint(name[8:walSegmentFilenameLength], 0x10, hexUint64Bits)
+ if err != nil {
+ return 0, 0, NotWalFilenameError{Filename: name}
}
- timelineID = uint32(timelineID64)
- logSegNo = logSegNoHi*segmentsPerLogID + logSegNoLo
-
- return timelineID, logSegNo, nil
+ return uint32(parsedTimelineID), logSegmentFields, nil
}
func formatWALFileName(timeline uint32, logSegNo uint64) string {
diff --git a/backend/internal/util/walmath/wal_filename_test.go b/backend/internal/util/walmath/wal_filename_test.go
index 0abb3643..6a76ff06 100644
--- a/backend/internal/util/walmath/wal_filename_test.go
+++ b/backend/internal/util/walmath/wal_filename_test.go
@@ -146,3 +146,32 @@ func Test_FormatHistoryFilename_KnownTimeline_FormatsAsPg(t *testing.T) {
assert.Equal(t, "00000003.history", walmath.FormatHistoryFilename(3))
assert.Equal(t, "0000000A.history", walmath.FormatHistoryFilename(10))
}
+
+func Test_ParseWALFilenameTimeline_ValidName_ReadsTimelineWithoutSegmentSize(t *testing.T) {
+ // A cluster with segments smaller than 16 MB numbers them past what 16 MB
+ // arithmetic accepts. The timeline field reads either way.
+ segmentOfASmallerSegmentedCluster := "000000020000000000000100"
+
+ timeline, err := walmath.ParseWALFilenameTimeline(segmentOfASmallerSegmentedCluster)
+ assert.NoError(t, err)
+ assert.Equal(t, uint32(2), timeline)
+
+ _, _, err = walmath.ParseWALFilenameWithSize(segmentOfASmallerSegmentedCluster, 16*1024*1024)
+ assert.Error(t, err, "the segment-size-aware parser is the one that rejects such a name")
+}
+
+func Test_ParseWALFilenameTimeline_MalformedNames_ReturnsError(t *testing.T) {
+ cases := []string{
+ "",
+ "00000002",
+ "000xYz020000000000000030",
+ "00000002000xYz0000000030",
+ "0000000200000000000xYz30",
+ "0000000200000000000000300",
+ }
+
+ for _, name := range cases {
+ _, err := walmath.ParseWALFilenameTimeline(name)
+ assert.Error(t, err, name)
+ }
+}