PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (physical): Derive manifest sidecar upload timeout from its size (issue #679)
Rostislav Dugin4e3347b87da7
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.goindex 5f2b7075..ef4557c8 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go@@ -20,7 +20,11 @@ import ("databasus-backend/internal/util/tools")-const metadataUploadTimeout = 30 * time.Second+// The `.metadata` sidecar is a few KB of JSON, so its deadline covers endpoint+// latency rather than transfer time: a slow S3-compatible provider is the only+// realistic way this upload runs long, and losing it fails the whole backup.+// Aligned with walSegmentUploadTimeout and the manifest sidecar's floor.+const metadataUploadTimeout = 2 * time.Minute// uploadFullMetadata writes the `<artifact>.metadata` sidecar describing a// completed FULL, next to the artifact / manifest / history the stream alreadydiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.goindex 3988d42b..10028f43 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go@@ -20,10 +20,18 @@ import (util_encryption "databasus-backend/internal/util/encryption")-// manifestSaveTimeout bounds the post-success PUT of the reconstructed-// `.manifest` sidecar. The manifest is fully materialized in memory (KBs–MBs),-// so this is a plain buffered upload, not a multi-hour stream.-const manifestSaveTimeout = 30 * time.Second+// Bounds for the post-success PUT of the reconstructed `.manifest` sidecar.+// serializeManifest emits ~210 bytes per source file, so a 500k-file cluster+// produces a ~100 MB sidecar that a slow S3-compatible endpoint cannot absorb+// under a flat deadline — and losing the sidecar marks an already-stored backup+// CHAIN_BROKEN. The floor matches walSegmentUploadTimeout and covers endpoint+// latency on the common few-MB manifest; the ceiling keeps a wedged endpoint+// from pinning a backup worker indefinitely.+const (+ manifestSaveMinTimeout = 2 * time.Minute+ manifestSaveMaxTimeout = 15 * time.Minute+ manifestSaveThroughputBytesPerSec = 256 * 1024+)// runStreamParams carries everything runStream needs that does not come from the// *exec.Cmd itself. Grouped into a struct so FULL and INCR pass the same shape.@@ -333,6 +341,12 @@ func buildSuccessOutcome(}, nil}+func manifestSaveTimeout(payloadSizeBytes int) time.Duration {+ budget := time.Duration(payloadSizeBytes/manifestSaveThroughputBytesPerSec) * time.Second++ return min(max(budget, manifestSaveMinTimeout), manifestSaveMaxTimeout)+}+// saveManifestSidecar uploads the reconstructed manifest. When encryption is on// it uses a FRESH salt+nonce (SetupEncryptionWriter) — never the tar's — because// the chunk nonce is baseNonce||chunkIndex, so reusing the tar's (salt, nonce)@@ -344,43 +358,48 @@ func saveManifestSidecar(manifestFileName string,manifestBytes []byte,) (saltBase64, nonceBase64 string, err error) {- saveCtx, cancel := context.WithTimeout(ctx, manifestSaveTimeout)- defer cancel()+ payload := bytes.NewReader(manifestBytes)+ payloadSizeBytes := len(manifestBytes)++ if p.Encryption == backups_core_enums.BackupEncryptionEncrypted {+ var encrypted bytes.Buffer- if p.Encryption != backups_core_enums.BackupEncryptionEncrypted {- if err := p.Storage.SaveFile(- saveCtx,- p.FieldEncryptor,- p.Logger,- manifestFileName,- bytes.NewReader(manifestBytes),- ); err != nil {- return "", "", err+ encSetup, setupErr := backup_encryption.SetupEncryptionWriter(&encrypted, p.MasterKey, p.BackupID)+ if setupErr != nil {+ return "", "", fmt.Errorf("setup manifest encryption: %w", setupErr)}- return "", "", nil- }+ if _, writeErr := encSetup.Writer.Write(manifestBytes); writeErr != nil {+ return "", "", fmt.Errorf("encrypt manifest: %w", writeErr)+ }- var encrypted bytes.Buffer+ if closeErr := encSetup.Writer.Close(); closeErr != nil {+ return "", "", fmt.Errorf("flush manifest encryption: %w", closeErr)+ }- encSetup, err := backup_encryption.SetupEncryptionWriter(&encrypted, p.MasterKey, p.BackupID)- if err != nil {- return "", "", fmt.Errorf("setup manifest encryption: %w", err)+ payload = bytes.NewReader(encrypted.Bytes())+ payloadSizeBytes = encrypted.Len()+ saltBase64, nonceBase64 = encSetup.SaltBase64, encSetup.NonceBase64}- if _, err := encSetup.Writer.Write(manifestBytes); err != nil {- return "", "", fmt.Errorf("encrypt manifest: %w", err)- }+ // The deadline starts here, after encryption, so CPU-bound work never eats the+ // upload's budget, and it is sized from the bytes that actually go over the wire.+ saveTimeout := manifestSaveTimeout(payloadSizeBytes)- if err := encSetup.Writer.Close(); err != nil {- return "", "", fmt.Errorf("flush manifest encryption: %w", err)- }+ p.Logger.Debug(+ fmt.Sprintf("uploading manifest sidecar: %.2f MB, timeout %s",+ float64(payloadSizeBytes)/(1024*1024), saveTimeout),+ "backup_id", p.BackupID,+ )++ saveCtx, cancel := context.WithTimeout(ctx, saveTimeout)+ defer cancel()- if err := p.Storage.SaveFile(saveCtx, p.FieldEncryptor, p.Logger, manifestFileName, &encrypted); err != nil {+ if err := p.Storage.SaveFile(saveCtx, p.FieldEncryptor, p.Logger, manifestFileName, payload); err != nil {return "", "", err}- return encSetup.SaltBase64, encSetup.NonceBase64, nil+ return saltBase64, nonceBase64, nil}// isCompressionUnsupportedError reports whether stderr carries pg_basebackup'sdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.goindex 8ddd572b..60b2195b 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go@@ -34,14 +34,21 @@ type fakeStorageFileSaver struct {mu sync.Mutexsaved map[string][]bytesaveErr error++ // The deadline is recorded rather than the context itself so no context is ever+ // parked in a struct field.+ savedDeadlines map[string]time.Time}func newFakeStorage() *fakeStorageFileSaver {- return &fakeStorageFileSaver{saved: map[string][]byte{}}+ return &fakeStorageFileSaver{+ saved: map[string][]byte{},+ savedDeadlines: map[string]time.Time{},+ }}func (f *fakeStorageFileSaver) SaveFile(- _ context.Context,+ ctx context.Context,_ encryption.FieldEncryptor,_ *slog.Logger,fileName string,@@ -58,11 +65,25 @@ func (f *fakeStorageFileSaver) SaveFile(f.mu.Lock()f.saved[fileName] = data++ if deadline, hasDeadline := ctx.Deadline(); hasDeadline {+ f.savedDeadlines[fileName] = deadline+ }+f.mu.Unlock()return nil}+func (f *fakeStorageFileSaver) deadlineFor(fileName string) (time.Time, bool) {+ f.mu.Lock()+ defer f.mu.Unlock()++ deadline, hasDeadline := f.savedDeadlines[fileName]++ return deadline, hasDeadline+}+func (f *fakeStorageFileSaver) GetFile(_ encryption.FieldEncryptor, fileName string) (io.ReadCloser, error) {f.mu.Lock()defer f.mu.Unlock()@@ -318,6 +339,58 @@ func Test_SaveManifestSidecar_WhenEncrypted_RoundTripsWithOwnSaltIV(t *testing.Tassert.Equal(t, manifest, plaintext, "manifest must decrypt with its own salt/IV")}+func Test_ManifestSaveTimeout_ScalesWithSizeBetweenFloorAndCeiling(t *testing.T) {+ cases := []struct {+ name string+ payloadSizeBytes int+ timeout time.Duration+ }{+ {"empty manifest", 0, manifestSaveMinTimeout},+ {"1 MiB manifest", 1024 * 1024, manifestSaveMinTimeout},+ {"100 MB manifest", 100 * 1024 * 1024, 400 * time.Second},+ {"1 GiB manifest", 1024 * 1024 * 1024, manifestSaveMaxTimeout},+ }++ for _, testCase := range cases {+ t.Run(testCase.name, func(t *testing.T) {+ assert.Equal(t, testCase.timeout, manifestSaveTimeout(testCase.payloadSizeBytes))+ })+ }+}++func Test_SaveManifestSidecar_WhenManifestIsSmall_UsesFloorDeadline(t *testing.T) {+ storage := newFakeStorage()+ params := testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone)++ _, _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", []byte(`{ "Files": [] }`))+ require.NoError(t, err)++ deadline, hasDeadline := storage.deadlineFor("obj.manifest")+ require.True(t, hasDeadline, "the sidecar upload must stay bounded")++ assert.InDelta(t, manifestSaveMinTimeout.Seconds(), time.Until(deadline).Seconds(), 5,+ "a few-KB manifest gets the floor, not a size-derived budget")+}++// A 40 MiB sidecar sits past the knee where the size-derived budget overtakes the+// floor (manifestSaveMinTimeout worth of throughput is 30 MiB), so it proves the+// deadline actually tracks the payload instead of being a renamed constant.+func Test_SaveManifestSidecar_WhenManifestIsLarge_ExtendsDeadlineBeyondFloor(t *testing.T) {+ storage := newFakeStorage()+ params := testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone)+ largeManifest := bytes.Repeat([]byte("m"), 40*1024*1024)++ _, _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", largeManifest)+ require.NoError(t, err)++ deadline, hasDeadline := storage.deadlineFor("obj.manifest")+ require.True(t, hasDeadline, "the sidecar upload must stay bounded")++ assert.Greater(t, time.Until(deadline), manifestSaveMinTimeout,+ "a manifest past the floor's throughput budget must get more time")+ assert.Len(t, storage.saved["obj.manifest"], len(largeManifest))+}+func Test_IsCompressionUnsupportedError_MatchesBuildRejection(t *testing.T) {cases := []struct {stderr string