PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Read the timeline from the WAL file name, not the control file
Rostislav Dugin7d4fc630e254
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.goindex 99f6f5a6..ad7814c7 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go@@ -328,20 +328,67 @@ func UploadHistoryFile(return row, nil}-func readClusterIdentity(ctx context.Context, conn *pgx.Conn) (int, string, error) {- var tli int- var sysID string+type clusterIdentitySources struct {+ // NULL during recovery, where pg_current_wal_lsn() cannot be called.+ CurrentWalFilename *string+ ControlFileTimelineID int+ SystemIdentifier string+}++func readClusterIdentitySources(ctx context.Context, conn *pgx.Conn) (clusterIdentitySources, error) {+ var sources clusterIdentitySourceserr := conn.QueryRow(ctx, `SELECT+ CASE WHEN pg_is_in_recovery() THEN NULL ELSE pg_walfile_name(pg_current_wal_lsn()) END,(SELECT timeline_id FROM pg_control_checkpoint()),(SELECT system_identifier::text FROM pg_control_system())- `).Scan(&tli, &sysID)+ `).Scan(+ &sources.CurrentWalFilename,+ &sources.ControlFileTimelineID,+ &sources.SystemIdentifier,+ )+ if err != nil {+ return clusterIdentitySources{}, fmt.Errorf("read cluster identity: %w", err)+ }++ return sources, nil+}++func readClusterIdentity(ctx context.Context, conn *pgx.Conn) (int, string, error) {+ sources, err := readClusterIdentitySources(ctx, conn)if err != nil {- return 0, "", fmt.Errorf("read cluster identity: %w", err)+ return 0, "", err+ }++ timelineID, err := resolveLiveTimelineID(sources)+ if err != nil {+ return 0, "", err+ }++ return timelineID, sources.SystemIdentifier, nil+}++// Every cluster answers the control file, but it lags a promotion: PostgreSQL stamps it+// at checkpoints, and the checkpoint a promotion requests is spread across+// checkpoint_completion_target, so a promoted primary keeps naming the pre-failover+// timeline for minutes. A stale reading lets an incremental extend a chain anchored to a+// timeline the cluster has already left.+//+// A source still in recovery has no fresher reading to offer: pg_walfile_name() refuses+// to run there, and pg_stat_wal_receiver hides received_tli from a role without+// pg_read_all_stats, which the replication user this product provisions does not have.+func resolveLiveTimelineID(sources clusterIdentitySources) (int, error) {+ if sources.CurrentWalFilename != nil {+ timelineID, err := walmath.ParseWALFilenameTimeline(*sources.CurrentWalFilename)+ if err != nil {+ return 0, fmt.Errorf("read timeline from current WAL file name: %w", err)+ }++ return int(timelineID), nil}- return tli, sysID, nil+ return sources.ControlFileTimelineID, nil}func newestKnownTimeline(diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.goindex e9b9aa49..75dba2cf 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline_test.go@@ -6,10 +6,16 @@ import ("io""log/slog""testing"+ "time"+ "github.com/jackc/pgx/v5""github.com/stretchr/testify/require"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"+ "databasus-backend/internal/features/databases"+ postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"+ "databasus-backend/internal/util/encryption"+ "databasus-backend/internal/util/testing/containers")func Test_DecideTimelineCompatibility_WithMatchingAndConflictingIdentity_ReturnsExpectedDecision(t *testing.T) {@@ -219,3 +225,121 @@ func Test_RecheckStreamFailure_WhenIdentityProbeFails_PreservesOriginalResult(trequire.Equal(t, originalResult, fullResult)require.Equal(t, originalResult, incrementalResult)}++func Test_ResolveLiveTimelineID_WhenSourcesDisagree_PrefersTheFreshestReading(t *testing.T) {+ promotedPrimaryWalFilename := "000000020000000000000005"++ tests := []struct {+ name string+ sources clusterIdentitySources+ resolvedTimelineID int+ }{+ {+ name: "promoted primary whose control file has not caught up",+ sources: clusterIdentitySources{+ CurrentWalFilename: &promotedPrimaryWalFilename,+ ControlFileTimelineID: 1,+ },+ resolvedTimelineID: 2,+ },+ {+ name: "cluster in recovery, which can only answer the control file",+ sources: clusterIdentitySources{ControlFileTimelineID: 3},+ resolvedTimelineID: 3,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ timelineID, err := resolveLiveTimelineID(test.sources)++ require.NoError(t, err)+ require.Equal(t, test.resolvedTimelineID, timelineID)+ })+ }+}++func Test_ResolveLiveTimelineID_WhenWalFilenameIsUnreadable_ReturnsError(t *testing.T) {+ unreadableWalFilename := "not-a-wal-file"++ _, err := resolveLiveTimelineID(clusterIdentitySources{+ CurrentWalFilename: &unreadableWalFilename,+ ControlFileTimelineID: 1,+ })++ require.Error(+ t,+ err,+ "a source naming its WAL file in a shape we cannot read must not fall back to a stale timeline",+ )+}++const timelineTestPostgresVersion = "17"++func Test_ReadClusterIdentity_WhenTheControlFileLagsAPromotion_ReportsTheLiveTimeline(t *testing.T) {+ if testing.Short() {+ t.Skip("clones a standby and promotes it across a timeline switch; skipped in -short")+ }++ primary, standby := containers.StartPhysicalPrimaryWithStandby(+ t, "postgres:"+timelineTestPostgresVersion, containers.WithCheckpointsSpreadOverAnHour())++ ctx := t.Context()+ primaryConn := openPhysicalTestConn(t, ctx, primary)+ standbyConn := openPhysicalTestConn(t, ctx, standby)++ // Replaying these rows dirties buffers on the standby, which is what leaves the+ // post-promotion checkpoint work to spread instead of finishing at once.+ _, err := primaryConn.Exec(ctx, "CREATE TABLE promotion_ballast AS SELECT generate_series(1, 300000) AS n")+ require.NoError(t, err)+ waitForReplayedRelation(t, ctx, standbyConn, "promotion_ballast")++ var isPromoted bool+ require.NoError(t, standbyConn.QueryRow(ctx,+ "SELECT pg_promote(wait => true, wait_seconds => 60)").Scan(&isPromoted))+ require.True(t, isPromoted, "pg_promote must finish promotion within its wait window")++ var controlFileTimelineID int+ require.NoError(t, standbyConn.QueryRow(ctx,+ "SELECT timeline_id FROM pg_control_checkpoint()").Scan(&controlFileTimelineID))+ require.Equal(t, 1, controlFileTimelineID,+ "the spread-checkpoint source must keep the control file behind, or this test proves nothing")++ liveTimelineID, _, err := readClusterIdentity(ctx, standbyConn)+ require.NoError(t, err)+ require.Equal(t, 2, liveTimelineID,+ "a promoted source must report the timeline it writes WAL on, not the one its control file was stamped with")+}++func openPhysicalTestConn(t *testing.T, ctx context.Context, endpoint containers.Endpoint) *pgx.Conn {+ t.Helper()++ sourceDB := databases.GetTestPhysicalPostgresConfigWithType(+ endpoint.Host, endpoint.Port, timelineTestPostgresVersion, postgresql_physical.BackupTypeFullOnly)++ conn, err := sourceDB.OpenInspectionConn(ctx, encryption.GetFieldEncryptor())+ require.NoError(t, err)+ t.Cleanup(func() { _ = conn.Close(context.Background()) })++ return conn+}++func waitForReplayedRelation(t *testing.T, ctx context.Context, standbyConn *pgx.Conn, relation string) {+ t.Helper()++ deadline := time.Now().UTC().Add(60 * time.Second)++ for time.Now().UTC().Before(deadline) {+ var replayedRelation *string++ require.NoError(t, standbyConn.QueryRow(ctx, "SELECT to_regclass($1)::text", relation).Scan(&replayedRelation))++ if replayedRelation != nil {+ return+ }++ time.Sleep(200 * time.Millisecond)+ }++ t.Fatalf("relation %q never replayed onto the standby", relation)+}diff --git a/backend/internal/features/tests/physical/postgresql/shared/bodies.go b/backend/internal/features/tests/physical/postgresql/shared/bodies.goindex 49e60e9d..787c3b7f 100644--- a/backend/internal/features/tests/physical/postgresql/shared/bodies.go+++ b/backend/internal/features/tests/physical/postgresql/shared/bodies.go@@ -6,17 +6,13 @@ import ("net/http""os""path/filepath"- "slices""testing""time"- "github.com/google/uuid""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"backuping_physical "databasus-backend/internal/features/backups/backups/backuping/physical"- physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"- physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"@@ -215,36 +211,7 @@ func RunPromotionReanchorsIncrementalChain(t *testing.T, version, image string)waitForTimelineHistoryOnParent(t, fixture, 2, 60*time.Second)triggerIncrementalViaAPI(t, router, fixture)- deadline := time.Now().UTC().Add(4 * time.Minute)- var replacementFullID uuid.UUID- for time.Now().UTC().Before(deadline) {- incrementals, err := physical_repositories.GetIncrementalBackupRepository().FindAllByRootFull(initialFullID)- require.NoError(t, err)- fulls, err := physical_repositories.GetFullBackupRepository().FindCompletedNewestFirstByDatabase(fixture.DB.ID)- require.NoError(t, err)-- hasTimelineRefusal := slices.ContainsFunc(- incrementals,- func(incremental *physical_models.PhysicalIncrementalBackup) bool {- return incremental.Status == physical_enums.PhysicalBackupStatusChainBroken &&- incremental.ErrorReason != nil &&- *incremental.ErrorReason == physical_enums.PhysicalBackupErrorTimelineSwitchDetected- },- )- if hasTimelineRefusal && len(fulls) >= 2 && fulls[0].TimelineID == 2 {- replacementFullID = fulls[0].ID-- break- }-- time.Sleep(500 * time.Millisecond)- }- require.NotEqual(- t,- uuid.Nil,- replacementFullID,- "promotion must refuse the stale incremental and complete a timeline-2 FULL",- )+ waitForReanchoredFull(t, fixture, initialFullID, 2, 4*time.Minute)backuping_physical.RunOrphanWalCleanupForTest(t, fixture.DB.ID)bundle := downloadRestoreBundleViaAPI(t, router, fixture, nil)diff --git a/backend/internal/features/tests/physical/postgresql/shared/helpers.go b/backend/internal/features/tests/physical/postgresql/shared/helpers.goindex d9833999..572bec41 100644--- a/backend/internal/features/tests/physical/postgresql/shared/helpers.go+++ b/backend/internal/features/tests/physical/postgresql/shared/helpers.go@@ -24,6 +24,7 @@ import (backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums""databasus-backend/internal/features/backups/backups/core/physical/chain_view"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"backups_dto_physical "databasus-backend/internal/features/backups/backups/dto/physical"postgresql_executor "databasus-backend/internal/features/backups/backups/usecases/physical/postgresql"@@ -1137,6 +1138,138 @@ func waitForTimelineHistoryOnParent(timelineID, timeout)}+// Re-anchoring crosses the incremental executor, the scheduler and the FULL executor,+// and each of them can stop it for its own reason, so the timeout prints the catalog+// rows it last read rather than only the verdict.+func waitForReanchoredFull(+ t *testing.T,+ fixture *postgresql_executor.PhysicalDBFixture,+ staleFullID uuid.UUID,+ newTimelineID int,+ timeout time.Duration,+) {+ t.Helper()++ deadline := time.Now().UTC().Add(timeout)++ var state reanchorCatalogState++ for time.Now().UTC().Before(deadline) {+ state = readReanchorCatalogState(t, fixture, staleFullID)++ hasReplacementFull := len(state.CompletedFulls) >= 2 &&+ state.CompletedFulls[0].TimelineID == newTimelineID++ if hasTimelineRefusal(state.StaleChainIncrementals) && hasReplacementFull {+ return+ }++ time.Sleep(500 * time.Millisecond)+ }++ t.Fatalf("promotion did not re-anchor within %s: the stale chain must refuse an incremental with %s, "++ "and a COMPLETED FULL on timeline %d must follow it\n"++ " stale-chain incrementals: %s\n"++ " completed fulls: %s\n"++ " newest full (any status): %s",+ timeout, physical_enums.PhysicalBackupErrorTimelineSwitchDetected, newTimelineID,+ describeIncrementalOutcomes(state.StaleChainIncrementals),+ describeFullOutcomes(state.CompletedFulls),+ describeFullOutcome(state.NewestFullAnyStatus))+}++type reanchorCatalogState struct {+ StaleChainIncrementals []*physical_models.PhysicalIncrementalBackup+ CompletedFulls []*physical_models.PhysicalFullBackup+ // A replacement FULL that failed is absent from the completed list.+ NewestFullAnyStatus *physical_models.PhysicalFullBackup+}++// waitForReanchoredFull polls for a typed error reason, which only the repository rows+// carry; the backup list DTO has just the free-text fail message.+func readReanchorCatalogState(+ t *testing.T,+ fixture *postgresql_executor.PhysicalDBFixture,+ staleFullID uuid.UUID,+) reanchorCatalogState {+ t.Helper()++ incrementalRepo := physical_repositories.GetIncrementalBackupRepository()+ fullRepo := physical_repositories.GetFullBackupRepository()++ staleChainIncrementals, err := incrementalRepo.FindAllByRootFull(staleFullID)+ require.NoError(t, err)++ completedFulls, err := fullRepo.FindCompletedNewestFirstByDatabase(fixture.DB.ID)+ require.NoError(t, err)++ newestFullAnyStatus, err := fullRepo.FindLastFullAnyStatusByDatabase(fixture.DB.ID)+ require.NoError(t, err)++ return reanchorCatalogState{+ StaleChainIncrementals: staleChainIncrementals,+ CompletedFulls: completedFulls,+ NewestFullAnyStatus: newestFullAnyStatus,+ }+}++func hasTimelineRefusal(incrementals []*physical_models.PhysicalIncrementalBackup) bool {+ return slices.ContainsFunc(+ incrementals,+ func(incremental *physical_models.PhysicalIncrementalBackup) bool {+ return incremental.Status == physical_enums.PhysicalBackupStatusChainBroken &&+ incremental.ErrorReason != nil &&+ *incremental.ErrorReason == physical_enums.PhysicalBackupErrorTimelineSwitchDetected+ },+ )+}++func describeIncrementalOutcomes(incrementals []*physical_models.PhysicalIncrementalBackup) string {+ descriptions := make([]string, 0, len(incrementals))++ for _, incremental := range incrementals {+ descriptions = append(descriptions, fmt.Sprintf("%s %s (%s)",+ incremental.ID, incremental.Status, describeErrorReason(incremental.ErrorReason)))+ }++ return joinOutcomesOrNone(descriptions)+}++func describeFullOutcomes(fulls []*physical_models.PhysicalFullBackup) string {+ descriptions := make([]string, 0, len(fulls))++ for _, full := range fulls {+ descriptions = append(descriptions, describeFullOutcome(full))+ }++ return joinOutcomesOrNone(descriptions)+}++func describeFullOutcome(full *physical_models.PhysicalFullBackup) string {+ if full == nil {+ return "none"+ }++ return fmt.Sprintf("%s %s on TL %d (%s)",+ full.ID, full.Status, full.TimelineID, describeErrorReason(full.ErrorReason))+}++func joinOutcomesOrNone(descriptions []string) string {+ if len(descriptions) == 0 {+ return "none"+ }++ return strings.Join(descriptions, "; ")+}++func describeErrorReason(reason *physical_enums.PhysicalBackupErrorReason) string {+ if reason == nil {+ return "no error reason"+ }++ return string(*reason)+}+func connectWithRetry(t *testing.T, dsn string, timeout time.Duration) *pgx.Conn {t.Helper()diff --git a/backend/internal/util/testing/containers/postgres_physical.go b/backend/internal/util/testing/containers/postgres_physical.goindex 57890754..b10f3a2a 100644--- a/backend/internal/util/testing/containers/postgres_physical.go+++ b/backend/internal/util/testing/containers/postgres_physical.go@@ -32,13 +32,13 @@ const physicalStandbySlotName = "databasus_test_standby"// logical default fills mid-test and the source crashes with "No space left on device".const physicalDataDirTmpfsOptions = "rw,size=2g"-// physicalPostgresOptions configures a replication-capable source container.type physicalPostgresOptions struct {- summarizeWal bool- fullPageWrites bool- withTablespace bool- omitReplicationHbaEntry bool- maxConnections int+ shouldSummarizeWal bool+ shouldEnableFullPageWrites bool+ shouldCreateTablespace bool+ shouldOmitReplicationHbaEntry bool+ shouldSpreadCheckpoints bool+ maxConnections int}// PhysicalPostgresOption tunes a physical source container away from its replication-ready default.@@ -47,20 +47,20 @@ type PhysicalPostgresOption func(*physicalPostgresOptions)// WithoutSummarizer starts the cluster with summarize_wal=off, so an incremental pre-flight reaches// the SUMMARIZER_OFF fallback deterministically.func WithoutSummarizer() PhysicalPostgresOption {- return func(o *physicalPostgresOptions) { o.summarizeWal = false }+ return func(o *physicalPostgresOptions) { o.shouldSummarizeWal = false }}// WithTablespace pre-creates a custom tablespace at first boot so the pre-flight rejects the cluster// per ADR-0010 (physical backups do not support custom tablespaces).func WithTablespace() PhysicalPostgresOption {- return func(o *physicalPostgresOptions) { o.withTablespace = true }+ return func(o *physicalPostgresOptions) { o.shouldCreateTablespace = true }}// WithoutReplicationHbaEntry omits the "host replication" pg_hba line, leaving only the image's// default "host all all all" rule. Such a cluster accepts ordinary and logical-replication// connections but refuses physical replication — the mode pg_basebackup / pg_receivewal actually use.func WithoutReplicationHbaEntry() PhysicalPostgresOption {- return func(o *physicalPostgresOptions) { o.omitReplicationHbaEntry = true }+ return func(o *physicalPostgresOptions) { o.shouldOmitReplicationHbaEntry = true }}// WithMaxConnections runs the source with a raised max_connections so a restored cluster booted at@@ -70,6 +70,14 @@ func WithMaxConnections(connections int) PhysicalPostgresOption {return func(o *physicalPostgresOptions) { o.maxConnections = connections }}+// A promotion only requests its checkpoint, so on a node carrying dirty buffers that request stays+// outstanding for the length of a test and pg_control_checkpoint() keeps answering the pre-failover+// timeline — the stale reading a timeline probe must not trust. The completion target is pinned so+// the pacing does not depend on the image's default.+func WithCheckpointsSpreadOverAnHour() PhysicalPostgresOption {+ return func(o *physicalPostgresOptions) { o.shouldSpreadCheckpoints = true }+}+// allowReplicationInitScript appends the pg_hba replication rule at first boot. "host all all" does// not cover replication connections, so pg_basebackup / pg_receivewal need this explicit line.func allowReplicationInitScript() testcontainers.ContainerFile {@@ -100,7 +108,7 @@ func createTablespaceInitScript() testcontainers.ContainerFile {func physicalPostgresCmd(opts physicalPostgresOptions) []string {summarize := "summarize_wal=on"- if !opts.summarizeWal {+ if !opts.shouldSummarizeWal {summarize = "summarize_wal=off"}@@ -108,7 +116,7 @@ func physicalPostgresCmd(opts physicalPostgresOptions) []string {// ("backup ... is corrupt"), so the replication-pair callers turn it on; single-node sources keep// it off for the throwaway-durability speedup.fullPageWrites := "full_page_writes=off"- if opts.fullPageWrites {+ if opts.shouldEnableFullPageWrites {fullPageWrites = "full_page_writes=on"}@@ -121,6 +129,10 @@ func physicalPostgresCmd(opts physicalPostgresOptions) []string {cmd = append(cmd, "-c", fmt.Sprintf("max_connections=%d", opts.maxConnections))}+ if opts.shouldSpreadCheckpoints {+ cmd = append(cmd, "-c", "checkpoint_timeout=1h", "-c", "checkpoint_completion_target=0.9")+ }+return cmd}@@ -161,7 +173,7 @@ func StartPhysicalPostgresBehindSshBastion(}func resolvePhysicalPostgresOptions(opts []PhysicalPostgresOption) physicalPostgresOptions {- options := physicalPostgresOptions{summarizeWal: true}+ options := physicalPostgresOptions{shouldSummarizeWal: true}for _, apply := range opts {apply(&options)}@@ -171,10 +183,10 @@ func resolvePhysicalPostgresOptions(opts []PhysicalPostgresOption) physicalPostgfunc physicalPostgresRequest(image string, options physicalPostgresOptions) testcontainers.ContainerRequest {var files []testcontainers.ContainerFile- if !options.omitReplicationHbaEntry {+ if !options.shouldOmitReplicationHbaEntry {files = append(files, allowReplicationInitScript())}- if options.withTablespace {+ if options.shouldCreateTablespace {files = append(files, createTablespaceInitScript())}@@ -202,13 +214,10 @@ func StartPhysicalPrimaryWithStandby() (Endpoint, Endpoint) {t.Helper()- options := physicalPostgresOptions{summarizeWal: true}- for _, apply := range opts {- apply(&options)- }+ options := resolvePhysicalPostgresOptions(opts)// A base backup taken on the standby requires full_page_writes — force it on for both nodes.- options.fullPageWrites = true+ options.shouldEnableFullPageWrites = truereplicationNetwork, err := network.New(context.Background())if err != nil {@@ -285,7 +294,7 @@ chown -R postgres:postgres "$PGDATA"echo "databasus-standby: waiting for primary to accept connections"until pg_isready -h ` + physicalPrimaryAlias + ` -p 5432 -U ` + PostgresUsername + ` >/dev/null 2>&1; do sleep 1; doneecho "databasus-standby: cloning primary via pg_basebackup"-gosu postgres pg_basebackup -D "$PGDATA" -R -X stream -C -S ` + physicalStandbySlotName + ` -w \+gosu postgres pg_basebackup -D "$PGDATA" -R -X stream -c fast -C -S ` + physicalStandbySlotName + ` -w \-d "host=` + physicalPrimaryAlias + ` port=5432 user=` + PostgresUsername +` password=` + PostgresPassword + ` dbname=postgres"echo "databasus-standby: starting as a streaming standby"diff --git a/backend/internal/util/walmath/constants.go b/backend/internal/util/walmath/constants.goindex 8cb227e5..540031e6 100644--- a/backend/internal/util/walmath/constants.go+++ b/backend/internal/util/walmath/constants.go@@ -16,6 +16,9 @@ const (maxCountOfLSN = 2hexUint32Bits = 32+ hexUint64Bits = 64++ walSegmentFilenameLength = 24)func SetWalSize(sizeMb uint64) {diff --git a/backend/internal/util/walmath/wal_filename.go b/backend/internal/util/walmath/wal_filename.goindex eb97fb9d..d75fc10f 100644--- a/backend/internal/util/walmath/wal_filename.go+++ b/backend/internal/util/walmath/wal_filename.go@@ -5,6 +5,7 @@ package walmathimport ("fmt"+ "math""regexp""strconv")@@ -15,39 +16,53 @@ func ParseWALFilename(name string) (timelineID uint32, logSegNo uint64, err erroreturn ParseWALFilenameWithSize(name, WalSegmentSize)}+// Unlike ParseWALFilename, this holds for a cluster whose WAL segment size differs+// from the configured one: the timeline field does not depend on it.+func ParseWALFilenameTimeline(name string) (uint32, error) {+ timelineID, _, err := parseWalSegmentFields(name)++ return timelineID, err+}+func ParseWALFilenameWithSize(name string, segmentSize uint64) (timelineID uint32, logSegNo uint64, err error) {- if len(name) != 24 {- return 0, 0, NotWalFilenameError{Filename: name}+ timelineID, logSegmentFields, err := parseWalSegmentFields(name)+ if err != nil {+ return 0, 0, err}if segmentSize == 0 || 0x100000000%segmentSize != 0 {return 0, 0, IncorrectLogSegNoError{Filename: name}}- timelineID64, err := strconv.ParseUint(name[0:8], 0x10, hexUint32Bits)- if err != nil {- return 0, 0, NotWalFilenameError{Filename: name}+ logSegNoHi := logSegmentFields >> hexUint32Bits+ logSegNoLo := logSegmentFields & math.MaxUint32++ segmentsPerLogID := 0x100000000 / segmentSize+ if logSegNoLo >= segmentsPerLogID {+ return 0, 0, IncorrectLogSegNoError{Filename: name}}- logSegNoHi, err := strconv.ParseUint(name[8:16], 0x10, hexUint32Bits)- if err != nil {+ logSegNo = logSegNoHi*segmentsPerLogID + logSegNoLo++ return timelineID, logSegNo, nil+}++func parseWalSegmentFields(name string) (timelineID uint32, logSegmentFields uint64, err error) {+ if len(name) != walSegmentFilenameLength {return 0, 0, NotWalFilenameError{Filename: name}}- logSegNoLo, err := strconv.ParseUint(name[16:24], 0x10, hexUint32Bits)+ parsedTimelineID, err := strconv.ParseUint(name[0:8], 0x10, hexUint32Bits)if err != nil {return 0, 0, NotWalFilenameError{Filename: name}}- segmentsPerLogID := 0x100000000 / segmentSize- if logSegNoLo >= segmentsPerLogID {- return 0, 0, IncorrectLogSegNoError{Filename: name}+ logSegmentFields, err = strconv.ParseUint(name[8:walSegmentFilenameLength], 0x10, hexUint64Bits)+ if err != nil {+ return 0, 0, NotWalFilenameError{Filename: name}}- timelineID = uint32(timelineID64)- logSegNo = logSegNoHi*segmentsPerLogID + logSegNoLo-- return timelineID, logSegNo, nil+ return uint32(parsedTimelineID), logSegmentFields, nil}func formatWALFileName(timeline uint32, logSegNo uint64) string {diff --git a/backend/internal/util/walmath/wal_filename_test.go b/backend/internal/util/walmath/wal_filename_test.goindex 0abb3643..6a76ff06 100644--- a/backend/internal/util/walmath/wal_filename_test.go+++ b/backend/internal/util/walmath/wal_filename_test.go@@ -146,3 +146,32 @@ func Test_FormatHistoryFilename_KnownTimeline_FormatsAsPg(t *testing.T) {assert.Equal(t, "00000003.history", walmath.FormatHistoryFilename(3))assert.Equal(t, "0000000A.history", walmath.FormatHistoryFilename(10))}++func Test_ParseWALFilenameTimeline_ValidName_ReadsTimelineWithoutSegmentSize(t *testing.T) {+ // A cluster with segments smaller than 16 MB numbers them past what 16 MB+ // arithmetic accepts. The timeline field reads either way.+ segmentOfASmallerSegmentedCluster := "000000020000000000000100"++ timeline, err := walmath.ParseWALFilenameTimeline(segmentOfASmallerSegmentedCluster)+ assert.NoError(t, err)+ assert.Equal(t, uint32(2), timeline)++ _, _, err = walmath.ParseWALFilenameWithSize(segmentOfASmallerSegmentedCluster, 16*1024*1024)+ assert.Error(t, err, "the segment-size-aware parser is the one that rejects such a name")+}++func Test_ParseWALFilenameTimeline_MalformedNames_ReturnsError(t *testing.T) {+ cases := []string{+ "",+ "00000002",+ "000xYz020000000000000030",+ "00000002000xYz0000000030",+ "0000000200000000000xYz30",+ "0000000200000000000000300",+ }++ for _, name := range cases {+ _, err := walmath.ParseWALFilenameTimeline(name)+ assert.Error(t, err, name)+ }+}