Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Publish physical backups and their files in one transaction

Rostislav Dugin897b347fc3d1
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper.go b/backend/internal/features/backups/backups/backuping/physical/backuper.go
index 1a47dd46..74799609 100644
--- a/backend/internal/features/backups/backups/backuping/physical/backuper.go
+++ b/backend/internal/features/backups/backups/backuping/physical/backuper.go
@@ -32,6 +32,9 @@ import (
// PhysicalBackuper drives a FULL or INCR through the postgresql executor.
// The scheduler invokes MakeBackup directly in a goroutine.
+// Every stored artifact carries a sidecar under the same name plus this suffix.
+const metadataSuffix = ".metadata"
+
type PhysicalBackuper struct {
databaseService *databases.DatabaseService
fieldEncryptor util_encryption.FieldEncryptor
@@ -153,7 +156,7 @@ func (b *PhysicalBackuper) runFullBackup(
"message", backupResult.ErrorMessage)
}
- if err := b.persistFullResult(fullBackup, backupResult, rawSizeMb); err != nil {
+ if err := b.persistFullResult(ctx, fullBackup, backupResult, rawSizeMb); err != nil {
logger.ErrorContext(ctx, "failed to persist full result", "error", err)
return
@@ -249,7 +252,7 @@ func (b *PhysicalBackuper) runIncrementalBackup(
"message", backupResult.ErrorMessage)
}
- if err := b.persistIncrResult(incrBackup, backupResult); err != nil {
+ if err := b.persistIncrResult(ctx, incrBackup, backupResult); err != nil {
logger.ErrorContext(ctx, "failed to persist incremental result", "error", err)
return
@@ -414,6 +417,7 @@ func (b *PhysicalBackuper) getSourceClusterSizeMb(
}
func (b *PhysicalBackuper) persistFullResult(
+ ctx context.Context,
fullBackup *physical_models.PhysicalFullBackup,
backupResult postgresql_executor.PhysicalBackupResult,
rawSizeMb *float64,
@@ -446,10 +450,12 @@ func (b *PhysicalBackuper) persistFullResult(
}
fullBackup.CompletedAt = &completedAt
- return b.saveTerminalResultIfInProgress(
- fullBackup.DatabaseID,
- fullBackup.ID,
- func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error) {
+ return b.saveTerminalResultIfInProgress(ctx, terminalPersistSpec{
+ DatabaseID: fullBackup.DatabaseID,
+ BackupID: fullBackup.ID,
+ StorageID: fullBackup.StorageID,
+ Result: backupResult,
+ LoadStatus: func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error) {
var current physical_models.PhysicalFullBackup
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
@@ -461,11 +467,12 @@ func (b *PhysicalBackuper) persistFullResult(
return current.Status, nil
},
- func(tx *gorm.DB) error { return tx.Save(fullBackup).Error },
- )
+ Save: func(tx *gorm.DB) error { return tx.Save(fullBackup).Error },
+ })
}
func (b *PhysicalBackuper) persistIncrResult(
+ ctx context.Context,
incrBackup *physical_models.PhysicalIncrementalBackup,
backupResult postgresql_executor.PhysicalBackupResult,
) error {
@@ -496,10 +503,12 @@ func (b *PhysicalBackuper) persistIncrResult(
}
incrBackup.CompletedAt = &completedAt
- return b.saveTerminalResultIfInProgress(
- incrBackup.DatabaseID,
- incrBackup.ID,
- func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error) {
+ return b.saveTerminalResultIfInProgress(ctx, terminalPersistSpec{
+ DatabaseID: incrBackup.DatabaseID,
+ BackupID: incrBackup.ID,
+ StorageID: incrBackup.StorageID,
+ Result: backupResult,
+ LoadStatus: func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error) {
var current physical_models.PhysicalIncrementalBackup
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
@@ -511,8 +520,8 @@ func (b *PhysicalBackuper) persistIncrResult(
return current.Status, nil
},
- func(tx *gorm.DB) error { return tx.Save(incrBackup).Error },
- )
+ Save: func(tx *gorm.DB) error { return tx.Save(incrBackup).Error },
+ })
}
// The current attempt's row is already IN_PROGRESS in the table (claimAndInsert
@@ -562,15 +571,23 @@ func (b *PhysicalBackuper) isIncrRetryBudgetExhausted(
// guarded read (locked FOR UPDATE to serialize against the sweep's conditional
// update) skips the write instead. The claim delete is
// scoped to backupID so it can never remove the newer backup's claim.
-func (b *PhysicalBackuper) saveTerminalResultIfInProgress(
- databaseID, backupID uuid.UUID,
- loadStatus func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error),
- save func(tx *gorm.DB) error,
-) error {
+type terminalPersistSpec struct {
+ DatabaseID uuid.UUID
+ BackupID uuid.UUID
+ StorageID uuid.UUID
+ Result postgresql_executor.PhysicalBackupResult
+ LoadStatus func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error)
+ Save func(tx *gorm.DB) error
+}
+
+// The files an attempt wrote are kept only by the transaction that publishes it.
+// A failed attempt, and a superseded one whose row a later attempt already took,
+// both hand their files back to cleanup in that same transaction.
+func (b *PhysicalBackuper) saveTerminalResultIfInProgress(ctx context.Context, spec terminalPersistSpec) error {
superseded := false
err := storage.GetDb().Transaction(func(tx *gorm.DB) error {
- status, err := loadStatus(tx)
+ status, err := spec.LoadStatus(tx)
if err != nil {
return err
}
@@ -578,18 +595,26 @@ func (b *PhysicalBackuper) saveTerminalResultIfInProgress(
if status != physical_enums.PhysicalBackupStatusInProgress {
superseded = true
- return nil
+ return b.fileStore.RequestFileDeletions(ctx, tx, b.attemptFileReferences(spec))
}
- if err := save(tx); err != nil {
+ if spec.Result.Status == physical_enums.PhysicalBackupStatusCompleted {
+ if err := b.fileStore.ConfirmFileWrites(ctx, tx, spec.Result.Receipts); err != nil {
+ return err
+ }
+ } else if err := b.fileStore.RequestFileDeletions(ctx, tx, b.attemptFileReferences(spec)); err != nil {
+ return err
+ }
+
+ if err := spec.Save(tx); err != nil {
return err
}
return tx.Delete(
&physical_models.PhysicalInFlightBackup{},
"database_id = ? AND backup_id = ?",
- databaseID,
- backupID,
+ spec.DatabaseID,
+ spec.BackupID,
).Error
})
if err != nil {
@@ -597,12 +622,35 @@ func (b *PhysicalBackuper) saveTerminalResultIfInProgress(
}
if superseded {
- b.logger.Warn("backup row no longer in progress; skipping terminal persist", "backup_id", backupID)
+ b.logger.Warn("backup row no longer in progress; skipping terminal persist", "backup_id", spec.BackupID)
}
return nil
}
+// The attempt owns its artifact plus the two sidecars derived from it. Requesting
+// a name nothing wrote costs one idempotent provider call and keeps the caller
+// from having to know which stage the attempt reached.
+func (b *PhysicalBackuper) attemptFileReferences(
+ spec terminalPersistSpec,
+) []storage_files.StoredFileReference {
+ if spec.Result.FileName == "" {
+ return nil
+ }
+
+ references := []storage_files.StoredFileReference{
+ {StorageID: spec.StorageID, FileName: spec.Result.FileName},
+ {StorageID: spec.StorageID, FileName: spec.Result.FileName + metadataSuffix},
+ }
+
+ if spec.Result.ManifestFileName != "" {
+ references = append(references,
+ storage_files.StoredFileReference{StorageID: spec.StorageID, FileName: spec.Result.ManifestFileName})
+ }
+
+ return references
+}
+
func (b *PhysicalBackuper) finalizeFullAsError(
fullBackup *physical_models.PhysicalFullBackup,
reason physical_enums.PhysicalBackupErrorReason,
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go b/backend/internal/features/backups/backups/backuping/physical/backuper_test.go
index 77de91fc..3fe64022 100644
--- a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/backuper_test.go
@@ -2,6 +2,7 @@ package backuping_physical
import (
"errors"
+ "strings"
"testing"
"time"
@@ -19,8 +20,11 @@ import (
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
notifier_models "databasus-backend/internal/features/notifiers/models"
+ "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"databasus-backend/internal/storage"
+ "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/walmath"
)
@@ -140,7 +144,7 @@ func Test_PersistFullResult_WhenCompleted_CopiesCompressionManifestFieldsAndRele
CompletedAt: time.Now().UTC(),
}
- require.NoError(t, backuper.persistFullResult(fullBackup, result, nil))
+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))
persisted, err := physical_repositories.GetFullBackupRepository().FindByID(fullBackup.ID)
require.NoError(t, err)
@@ -174,7 +178,7 @@ func Test_PersistFullResult_WhenCompletedWithMeasuredClusterSize_StoresRawSizeMb
CompletedAt: time.Now().UTC(),
}
- require.NoError(t, backuper.persistFullResult(fullBackup, result, new(4321.5)))
+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, new(4321.5)))
persisted, err := physical_repositories.GetFullBackupRepository().FindByID(fullBackup.ID)
require.NoError(t, err)
@@ -197,7 +201,7 @@ func Test_PersistFullResult_WhenClusterSizeUnmeasured_LeavesRawSizeMbNil(t *test
CompletedAt: time.Now().UTC(),
}
- require.NoError(t, backuper.persistFullResult(fullBackup, result, nil))
+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))
persisted, err := physical_repositories.GetFullBackupRepository().FindByID(fullBackup.ID)
require.NoError(t, err)
@@ -221,7 +225,7 @@ func Test_PersistFullResult_WhenErrorStatus_RecordsTerminalTimeAndReleasesInFlig
ManifestFileName: "should-not-be-copied",
}
- require.NoError(t, backuper.persistFullResult(fullBackup, result, nil))
+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))
persisted, err := physical_repositories.GetFullBackupRepository().FindByID(fullBackup.ID)
require.NoError(t, err)
@@ -247,7 +251,7 @@ func Test_PersistIncrementalResult_WhenErrorStatus_RecordsTerminalTime(t *testin
Status: physical_enums.PhysicalBackupStatusChainBroken,
ErrorReason: new(physical_enums.PhysicalBackupErrorTimelineSwitchDetected),
}
- require.NoError(t, backuper.persistIncrResult(incremental, result))
+ require.NoError(t, backuper.persistIncrResult(t.Context(), incremental, result))
persistedIncremental, err := physical_repositories.GetIncrementalBackupRepository().FindByID(incremental.ID)
require.NoError(t, err)
@@ -275,7 +279,7 @@ func Test_PersistIncrementalResult_WhenCompleted_CopiesFieldsAndReleasesInFlight
CompletedAt: time.Now().UTC(),
}
- require.NoError(t, backuper.persistIncrResult(incrBackup, result))
+ require.NoError(t, backuper.persistIncrResult(t.Context(), incrBackup, result))
persisted, err := physical_repositories.GetIncrementalBackupRepository().FindByID(incrBackup.ID)
require.NoError(t, err)
@@ -811,7 +815,7 @@ func Test_PersistFullResult_WhenRowNoLongerInProgress_DoesNotResurrect(t *testin
newBackupID := uuid.New()
claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeFull, newBackupID)
- err := backuper.persistFullResult(full, postgresql_executor.PhysicalBackupResult{
+ err := backuper.persistFullResult(t.Context(), full, postgresql_executor.PhysicalBackupResult{
Status: physical_enums.PhysicalBackupStatusCompleted,
}, nil)
require.NoError(t, err)
@@ -911,3 +915,76 @@ func assertInFlightReleased(t *testing.T, databaseID uuid.UUID) {
require.NoError(t, err)
assert.Nil(t, inFlight, "in-flight claim must be released")
}
+
+func Test_PersistFullResult_WhenCompleted_KeepsTheFilesTheAttemptWrote(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ backuper := CreateTestPhysicalBackuper(nil)
+
+ fullBackup := seedInProgressFull(t, prereqs)
+ claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeFull, fullBackup.ID)
+
+ fileName := "physical-completed-" + fullBackup.ID.String()
+
+ receipt, err := storages.GetStorageFileStore().WriteFile(
+ t.Context(),
+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},
+ strings.NewReader("physical artifact"),
+ )
+ require.NoError(t, err)
+
+ result := postgresql_executor.PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusCompleted,
+ FileName: fileName,
+ Receipts: []storage_files.WriteReceipt{receipt},
+ CompletedAt: time.Now().UTC(),
+ }
+
+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))
+
+ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),
+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},
+ ))
+
+ reader, err := prereqs.Storage.GetFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName,
+ )
+ require.NoError(t, err, "a published backup keeps its artifact")
+ require.NoError(t, reader.Close())
+}
+
+func Test_PersistFullResult_WhenFailed_HandsTheFilesBackToCleanup(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ backuper := CreateTestPhysicalBackuper(nil)
+
+ fullBackup := seedInProgressFull(t, prereqs)
+ claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeFull, fullBackup.ID)
+
+ fileName := "physical-failed-" + fullBackup.ID.String()
+
+ _, err := storages.GetStorageFileStore().WriteFile(
+ t.Context(),
+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},
+ strings.NewReader("partial physical artifact"),
+ )
+ require.NoError(t, err)
+
+ reason := physical_enums.PhysicalBackupErrorPgBasebackupFailed
+ result := postgresql_executor.PhysicalBackupResult{
+ Status: physical_enums.PhysicalBackupStatusError,
+ ErrorReason: &reason,
+ ErrorMessage: "pg_basebackup failed after producing output",
+ FileName: fileName,
+ CompletedAt: time.Now().UTC(),
+ }
+
+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))
+
+ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),
+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},
+ ))
+
+ _, err = prereqs.Storage.GetFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName,
+ )
+ assert.Error(t, err, "a failed physical backup must not leave its artifact behind")
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
index 61d9fd92..fe49b71f 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
@@ -27,7 +27,7 @@ type CommonBackupSpec struct {
// Storage stays for the reads this package still does, the parent manifest
// fetch among them. Everything it writes goes through FileStore.
Storage storages.StorageFileSaver
- FileStore storage_files.FileStore
+ FileStore *storage_files.Store
Encryption backups_core_enums.BackupEncryption
MasterKey string
FieldEncryptor util_encryption.FieldEncryptor
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
index 53565200..07990dc1 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
@@ -10,6 +10,8 @@ import (
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"
+ storage_files "databasus-backend/internal/features/storages/files"
+ db "databasus-backend/internal/storage"
files_utils "databasus-backend/internal/util/files"
"databasus-backend/internal/util/tools"
)
@@ -110,6 +112,12 @@ func streamWithCodecFallback(
fmt.Sprintf("compression downgraded: %s -> %s", codec, codecFallbackOrder[i+1]),
"backup_id", spec.BackupID)
+ // The next attempt writes a different name, so this one's bytes are
+ // owned by nobody. Handing them back now rather than letting the
+ // commit window expire keeps the rejected attempt from sitting in
+ // the user's storage for as long as a healthy upload would.
+ discardAttemptFiles(ctx, spec.Common, fileName)
+
continue
}
@@ -126,6 +134,16 @@ func streamWithCodecFallback(
return resultFromOutcome(settled, settledFileName), nil
}
+func discardAttemptFiles(ctx context.Context, common CommonBackupSpec, fileName string) {
+ err := common.FileStore.RequestFileDeletions(ctx, db.GetDb(), []storage_files.StoredFileReference{
+ {StorageID: common.StorageID, FileName: fileName},
+ })
+ if err != nil {
+ common.Logger.ErrorContext(ctx, "failed to discard a rejected codec attempt",
+ "file_name", fileName, "error", err)
+ }
+}
+
func resultFromOutcome(outcome streamOutcome, fileName string) PhysicalBackupResult {
return PhysicalBackupResult{
Status: outcome.Status,
diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.md
index 845da54f..04f22f78 100644
--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md
+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md
@@ -37,7 +37,7 @@
## 6. Integrate physical full and incremental backups
- [x] 6.1 Route full and incremental stream uploads and their sidecars through the file store and return receipts only after writer goroutines are joined. Give `buildObjectName` (`.../executor.go:152`) an attempt UUID, since its four current inputs are constant across the codec fallback loop, and split it instead into `buildBackupLabel`, which keeps the four inputs it already had, and `buildObjectName(label, attemptID)`. That satisfies the rule the struct was there to protect without a struct: neither function grows a fifth positional value, and neither takes two adjacent `uuid.UUID` values. Have `streamWithCodecFallback` (`.../executor.go:50`) mint each name through a `mintAndSaveAttemptName` callback its two callers supply, named for both of its effects, because the loop holds neither the naming inputs nor the repository that persists `file_name`, and move that function's stream inputs into a `streamAttemptSpec` struct rather than growing an already eight-value positional list. The callback becomes the only producer of object names: the pre-loop `buildObjectName` call and its `Save` (`create_full_uc.go:40` with its assignment and `Save` at `:42` and `:43`, `create_incremental_uc.go:76` with its own at `:78` and `:80`) stop minting the object key and keep only the stable pg_basebackup label, which `newPgBasebackupCommand` takes at `pg_basebackup.go:50` and which names the backup rather than the object. Return the settled attempt's name and take every consumer from that result rather than the caller's variable: `runStreamParams` (`.../executor.go:79`) so the upload-time manifest at `stream.go:317` follows it, the chain-broken removal and the reported `FileName` (`create_full_uc.go:84`, `:92`), `streamResult.FileName` (`create_full_uc.go:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`). Correct the two comments that state the old shape, `executor.go:45` ("every attempt streams to the same object key") and the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and move `executor_test.go` onto the new shape, renaming the table field `expected` (declared at `:19`, set at `:25` and `:31`, read at `:39`) to `objectName` and updating its values for the attempt UUID; verify stream success, a zstd rejection that downgrades to gzip and then completes without the second attempt being refused for a name already in use and with the manifest landing under the settled name, pg_basebackup failure after partial output, cancellation and provider failure with focused tests, renaming `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact` to `..._LeavesNoReceiptToPublish` and asserting the outcome carries no receipts. `runStream` sits below the layer that owns cleanup and its tests drive a fake store, so removal cannot be observed there; task 6.3 covers it against a real store.
-- [ ] 6.2 Confirm physical receipts in the same transaction that publishes the backup and leave pending deletions in place on terminal failure paths, including manifest, metadata and incremental-chain failures; verify rollback and restart behavior with service and scheduler tests.
+- [x] 6.2 Confirm physical receipts in the same transaction that publishes the backup and leave pending deletions in place on terminal failure paths, including manifest, metadata and incremental-chain failures; verify rollback and restart behavior with service and scheduler tests. A superseded attempt, whose row a later attempt already moved out of `IN_PROGRESS`, hands its files back in that same transaction, and a codec attempt the fallback rejects hands its own back immediately rather than waiting out the commit window.
- [ ] 6.3 Convert the physical reproduction tests into passing cleanup tests on local storage and MinIO; run the worker until idle and assert that partial base files, manifests, exact chunks, multipart uploads and metadata are absent after ordinary errors and cancellation.
- [ ] 6.4 Rewrite restart recovery so it no longer assumes a missing backup file name means no object exists, replacing the stale premise recorded at `scheduler.go:710`, and request deletion for a claimless `IN_PROGRESS` row that carries a file name in the same transaction that records its terminal state, gated on that conditional `IN_PROGRESS` transition affecting one row so a late completion that already wrote a terminal status keeps its file; verify with `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact` renamed to `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_ArtifactRemoved` and rewritten to assert eventual absence, and that recovery neither deletes a confirmed successful backup nor duplicates an existing pending deletion.