Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (physical): Derive manifest sidecar upload timeout from its size (issue #679)

Rostislav Dugin4e3347b87da7
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go
index 5f2b7075..ef4557c8 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go
@@ -20,7 +20,11 @@ import (
"databasus-backend/internal/util/tools"
)
-const metadataUploadTimeout = 30 * time.Second
+// The `.metadata` sidecar is a few KB of JSON, so its deadline covers endpoint
+// latency rather than transfer time: a slow S3-compatible provider is the only
+// realistic way this upload runs long, and losing it fails the whole backup.
+// Aligned with walSegmentUploadTimeout and the manifest sidecar's floor.
+const metadataUploadTimeout = 2 * time.Minute
// uploadFullMetadata writes the `<artifact>.metadata` sidecar describing a
// completed FULL, next to the artifact / manifest / history the stream already
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go
index 3988d42b..10028f43 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go
@@ -20,10 +20,18 @@ import (
util_encryption "databasus-backend/internal/util/encryption"
)
-// manifestSaveTimeout bounds the post-success PUT of the reconstructed
-// `.manifest` sidecar. The manifest is fully materialized in memory (KBs–MBs),
-// so this is a plain buffered upload, not a multi-hour stream.
-const manifestSaveTimeout = 30 * time.Second
+// Bounds for the post-success PUT of the reconstructed `.manifest` sidecar.
+// serializeManifest emits ~210 bytes per source file, so a 500k-file cluster
+// produces a ~100 MB sidecar that a slow S3-compatible endpoint cannot absorb
+// under a flat deadline — and losing the sidecar marks an already-stored backup
+// CHAIN_BROKEN. The floor matches walSegmentUploadTimeout and covers endpoint
+// latency on the common few-MB manifest; the ceiling keeps a wedged endpoint
+// from pinning a backup worker indefinitely.
+const (
+ manifestSaveMinTimeout = 2 * time.Minute
+ manifestSaveMaxTimeout = 15 * time.Minute
+ manifestSaveThroughputBytesPerSec = 256 * 1024
+)
// runStreamParams carries everything runStream needs that does not come from the
// *exec.Cmd itself. Grouped into a struct so FULL and INCR pass the same shape.
@@ -333,6 +341,12 @@ func buildSuccessOutcome(
}, nil
}
+func manifestSaveTimeout(payloadSizeBytes int) time.Duration {
+ budget := time.Duration(payloadSizeBytes/manifestSaveThroughputBytesPerSec) * time.Second
+
+ return min(max(budget, manifestSaveMinTimeout), manifestSaveMaxTimeout)
+}
+
// saveManifestSidecar uploads the reconstructed manifest. When encryption is on
// it uses a FRESH salt+nonce (SetupEncryptionWriter) — never the tar's — because
// the chunk nonce is baseNonce||chunkIndex, so reusing the tar's (salt, nonce)
@@ -344,43 +358,48 @@ func saveManifestSidecar(
manifestFileName string,
manifestBytes []byte,
) (saltBase64, nonceBase64 string, err error) {
- saveCtx, cancel := context.WithTimeout(ctx, manifestSaveTimeout)
- defer cancel()
+ payload := bytes.NewReader(manifestBytes)
+ payloadSizeBytes := len(manifestBytes)
+
+ if p.Encryption == backups_core_enums.BackupEncryptionEncrypted {
+ var encrypted bytes.Buffer
- if p.Encryption != backups_core_enums.BackupEncryptionEncrypted {
- if err := p.Storage.SaveFile(
- saveCtx,
- p.FieldEncryptor,
- p.Logger,
- manifestFileName,
- bytes.NewReader(manifestBytes),
- ); err != nil {
- return "", "", err
+ encSetup, setupErr := backup_encryption.SetupEncryptionWriter(&encrypted, p.MasterKey, p.BackupID)
+ if setupErr != nil {
+ return "", "", fmt.Errorf("setup manifest encryption: %w", setupErr)
}
- return "", "", nil
- }
+ if _, writeErr := encSetup.Writer.Write(manifestBytes); writeErr != nil {
+ return "", "", fmt.Errorf("encrypt manifest: %w", writeErr)
+ }
- var encrypted bytes.Buffer
+ if closeErr := encSetup.Writer.Close(); closeErr != nil {
+ return "", "", fmt.Errorf("flush manifest encryption: %w", closeErr)
+ }
- encSetup, err := backup_encryption.SetupEncryptionWriter(&encrypted, p.MasterKey, p.BackupID)
- if err != nil {
- return "", "", fmt.Errorf("setup manifest encryption: %w", err)
+ payload = bytes.NewReader(encrypted.Bytes())
+ payloadSizeBytes = encrypted.Len()
+ saltBase64, nonceBase64 = encSetup.SaltBase64, encSetup.NonceBase64
}
- if _, err := encSetup.Writer.Write(manifestBytes); err != nil {
- return "", "", fmt.Errorf("encrypt manifest: %w", err)
- }
+ // The deadline starts here, after encryption, so CPU-bound work never eats the
+ // upload's budget, and it is sized from the bytes that actually go over the wire.
+ saveTimeout := manifestSaveTimeout(payloadSizeBytes)
- if err := encSetup.Writer.Close(); err != nil {
- return "", "", fmt.Errorf("flush manifest encryption: %w", err)
- }
+ p.Logger.Debug(
+ fmt.Sprintf("uploading manifest sidecar: %.2f MB, timeout %s",
+ float64(payloadSizeBytes)/(1024*1024), saveTimeout),
+ "backup_id", p.BackupID,
+ )
+
+ saveCtx, cancel := context.WithTimeout(ctx, saveTimeout)
+ defer cancel()
- if err := p.Storage.SaveFile(saveCtx, p.FieldEncryptor, p.Logger, manifestFileName, &encrypted); err != nil {
+ if err := p.Storage.SaveFile(saveCtx, p.FieldEncryptor, p.Logger, manifestFileName, payload); err != nil {
return "", "", err
}
- return encSetup.SaltBase64, encSetup.NonceBase64, nil
+ return saltBase64, nonceBase64, nil
}
// isCompressionUnsupportedError reports whether stderr carries pg_basebackup's
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
index 8ddd572b..60b2195b 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
@@ -34,14 +34,21 @@ type fakeStorageFileSaver struct {
mu sync.Mutex
saved map[string][]byte
saveErr error
+
+ // The deadline is recorded rather than the context itself so no context is ever
+ // parked in a struct field.
+ savedDeadlines map[string]time.Time
}
func newFakeStorage() *fakeStorageFileSaver {
- return &fakeStorageFileSaver{saved: map[string][]byte{}}
+ return &fakeStorageFileSaver{
+ saved: map[string][]byte{},
+ savedDeadlines: map[string]time.Time{},
+ }
}
func (f *fakeStorageFileSaver) SaveFile(
- _ context.Context,
+ ctx context.Context,
_ encryption.FieldEncryptor,
_ *slog.Logger,
fileName string,
@@ -58,11 +65,25 @@ func (f *fakeStorageFileSaver) SaveFile(
f.mu.Lock()
f.saved[fileName] = data
+
+ if deadline, hasDeadline := ctx.Deadline(); hasDeadline {
+ f.savedDeadlines[fileName] = deadline
+ }
+
f.mu.Unlock()
return nil
}
+func (f *fakeStorageFileSaver) deadlineFor(fileName string) (time.Time, bool) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+
+ deadline, hasDeadline := f.savedDeadlines[fileName]
+
+ return deadline, hasDeadline
+}
+
func (f *fakeStorageFileSaver) GetFile(_ encryption.FieldEncryptor, fileName string) (io.ReadCloser, error) {
f.mu.Lock()
defer f.mu.Unlock()
@@ -318,6 +339,58 @@ func Test_SaveManifestSidecar_WhenEncrypted_RoundTripsWithOwnSaltIV(t *testing.T
assert.Equal(t, manifest, plaintext, "manifest must decrypt with its own salt/IV")
}
+func Test_ManifestSaveTimeout_ScalesWithSizeBetweenFloorAndCeiling(t *testing.T) {
+ cases := []struct {
+ name string
+ payloadSizeBytes int
+ timeout time.Duration
+ }{
+ {"empty manifest", 0, manifestSaveMinTimeout},
+ {"1 MiB manifest", 1024 * 1024, manifestSaveMinTimeout},
+ {"100 MB manifest", 100 * 1024 * 1024, 400 * time.Second},
+ {"1 GiB manifest", 1024 * 1024 * 1024, manifestSaveMaxTimeout},
+ }
+
+ for _, testCase := range cases {
+ t.Run(testCase.name, func(t *testing.T) {
+ assert.Equal(t, testCase.timeout, manifestSaveTimeout(testCase.payloadSizeBytes))
+ })
+ }
+}
+
+func Test_SaveManifestSidecar_WhenManifestIsSmall_UsesFloorDeadline(t *testing.T) {
+ storage := newFakeStorage()
+ params := testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone)
+
+ _, _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", []byte(`{ "Files": [] }`))
+ require.NoError(t, err)
+
+ deadline, hasDeadline := storage.deadlineFor("obj.manifest")
+ require.True(t, hasDeadline, "the sidecar upload must stay bounded")
+
+ assert.InDelta(t, manifestSaveMinTimeout.Seconds(), time.Until(deadline).Seconds(), 5,
+ "a few-KB manifest gets the floor, not a size-derived budget")
+}
+
+// A 40 MiB sidecar sits past the knee where the size-derived budget overtakes the
+// floor (manifestSaveMinTimeout worth of throughput is 30 MiB), so it proves the
+// deadline actually tracks the payload instead of being a renamed constant.
+func Test_SaveManifestSidecar_WhenManifestIsLarge_ExtendsDeadlineBeyondFloor(t *testing.T) {
+ storage := newFakeStorage()
+ params := testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone)
+ largeManifest := bytes.Repeat([]byte("m"), 40*1024*1024)
+
+ _, _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", largeManifest)
+ require.NoError(t, err)
+
+ deadline, hasDeadline := storage.deadlineFor("obj.manifest")
+ require.True(t, hasDeadline, "the sidecar upload must stay bounded")
+
+ assert.Greater(t, time.Until(deadline), manifestSaveMinTimeout,
+ "a manifest past the floor's throughput budget must get more time")
+ assert.Len(t, storage.saved["obj.manifest"], len(largeManifest))
+}
+
func Test_IsCompressionUnsupportedError_MatchesBuildRejection(t *testing.T) {
cases := []struct {
stderr string