PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Publish physical backups and their files in one transaction
Rostislav Dugin897b347fc3d1
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper.go b/backend/internal/features/backups/backups/backuping/physical/backuper.goindex 1a47dd46..74799609 100644--- a/backend/internal/features/backups/backups/backuping/physical/backuper.go+++ b/backend/internal/features/backups/backups/backuping/physical/backuper.go@@ -32,6 +32,9 @@ import (// PhysicalBackuper drives a FULL or INCR through the postgresql executor.// The scheduler invokes MakeBackup directly in a goroutine.+// Every stored artifact carries a sidecar under the same name plus this suffix.+const metadataSuffix = ".metadata"+type PhysicalBackuper struct {databaseService *databases.DatabaseServicefieldEncryptor util_encryption.FieldEncryptor@@ -153,7 +156,7 @@ func (b *PhysicalBackuper) runFullBackup("message", backupResult.ErrorMessage)}- if err := b.persistFullResult(fullBackup, backupResult, rawSizeMb); err != nil {+ if err := b.persistFullResult(ctx, fullBackup, backupResult, rawSizeMb); err != nil {logger.ErrorContext(ctx, "failed to persist full result", "error", err)return@@ -249,7 +252,7 @@ func (b *PhysicalBackuper) runIncrementalBackup("message", backupResult.ErrorMessage)}- if err := b.persistIncrResult(incrBackup, backupResult); err != nil {+ if err := b.persistIncrResult(ctx, incrBackup, backupResult); err != nil {logger.ErrorContext(ctx, "failed to persist incremental result", "error", err)return@@ -414,6 +417,7 @@ func (b *PhysicalBackuper) getSourceClusterSizeMb(}func (b *PhysicalBackuper) persistFullResult(+ ctx context.Context,fullBackup *physical_models.PhysicalFullBackup,backupResult postgresql_executor.PhysicalBackupResult,rawSizeMb *float64,@@ -446,10 +450,12 @@ func (b *PhysicalBackuper) persistFullResult(}fullBackup.CompletedAt = &completedAt- return b.saveTerminalResultIfInProgress(- fullBackup.DatabaseID,- fullBackup.ID,- func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error) {+ return b.saveTerminalResultIfInProgress(ctx, terminalPersistSpec{+ DatabaseID: fullBackup.DatabaseID,+ BackupID: fullBackup.ID,+ StorageID: fullBackup.StorageID,+ Result: backupResult,+ LoadStatus: func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error) {var current physical_models.PhysicalFullBackupif err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).@@ -461,11 +467,12 @@ func (b *PhysicalBackuper) persistFullResult(return current.Status, nil},- func(tx *gorm.DB) error { return tx.Save(fullBackup).Error },- )+ Save: func(tx *gorm.DB) error { return tx.Save(fullBackup).Error },+ })}func (b *PhysicalBackuper) persistIncrResult(+ ctx context.Context,incrBackup *physical_models.PhysicalIncrementalBackup,backupResult postgresql_executor.PhysicalBackupResult,) error {@@ -496,10 +503,12 @@ func (b *PhysicalBackuper) persistIncrResult(}incrBackup.CompletedAt = &completedAt- return b.saveTerminalResultIfInProgress(- incrBackup.DatabaseID,- incrBackup.ID,- func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error) {+ return b.saveTerminalResultIfInProgress(ctx, terminalPersistSpec{+ DatabaseID: incrBackup.DatabaseID,+ BackupID: incrBackup.ID,+ StorageID: incrBackup.StorageID,+ Result: backupResult,+ LoadStatus: func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error) {var current physical_models.PhysicalIncrementalBackupif err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).@@ -511,8 +520,8 @@ func (b *PhysicalBackuper) persistIncrResult(return current.Status, nil},- func(tx *gorm.DB) error { return tx.Save(incrBackup).Error },- )+ Save: func(tx *gorm.DB) error { return tx.Save(incrBackup).Error },+ })}// The current attempt's row is already IN_PROGRESS in the table (claimAndInsert@@ -562,15 +571,23 @@ func (b *PhysicalBackuper) isIncrRetryBudgetExhausted(// guarded read (locked FOR UPDATE to serialize against the sweep's conditional// update) skips the write instead. The claim delete is// scoped to backupID so it can never remove the newer backup's claim.-func (b *PhysicalBackuper) saveTerminalResultIfInProgress(- databaseID, backupID uuid.UUID,- loadStatus func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error),- save func(tx *gorm.DB) error,-) error {+type terminalPersistSpec struct {+ DatabaseID uuid.UUID+ BackupID uuid.UUID+ StorageID uuid.UUID+ Result postgresql_executor.PhysicalBackupResult+ LoadStatus func(tx *gorm.DB) (physical_enums.PhysicalBackupStatus, error)+ Save func(tx *gorm.DB) error+}++// The files an attempt wrote are kept only by the transaction that publishes it.+// A failed attempt, and a superseded one whose row a later attempt already took,+// both hand their files back to cleanup in that same transaction.+func (b *PhysicalBackuper) saveTerminalResultIfInProgress(ctx context.Context, spec terminalPersistSpec) error {superseded := falseerr := storage.GetDb().Transaction(func(tx *gorm.DB) error {- status, err := loadStatus(tx)+ status, err := spec.LoadStatus(tx)if err != nil {return err}@@ -578,18 +595,26 @@ func (b *PhysicalBackuper) saveTerminalResultIfInProgress(if status != physical_enums.PhysicalBackupStatusInProgress {superseded = true- return nil+ return b.fileStore.RequestFileDeletions(ctx, tx, b.attemptFileReferences(spec))}- if err := save(tx); err != nil {+ if spec.Result.Status == physical_enums.PhysicalBackupStatusCompleted {+ if err := b.fileStore.ConfirmFileWrites(ctx, tx, spec.Result.Receipts); err != nil {+ return err+ }+ } else if err := b.fileStore.RequestFileDeletions(ctx, tx, b.attemptFileReferences(spec)); err != nil {+ return err+ }++ if err := spec.Save(tx); err != nil {return err}return tx.Delete(&physical_models.PhysicalInFlightBackup{},"database_id = ? AND backup_id = ?",- databaseID,- backupID,+ spec.DatabaseID,+ spec.BackupID,).Error})if err != nil {@@ -597,12 +622,35 @@ func (b *PhysicalBackuper) saveTerminalResultIfInProgress(}if superseded {- b.logger.Warn("backup row no longer in progress; skipping terminal persist", "backup_id", backupID)+ b.logger.Warn("backup row no longer in progress; skipping terminal persist", "backup_id", spec.BackupID)}return nil}+// The attempt owns its artifact plus the two sidecars derived from it. Requesting+// a name nothing wrote costs one idempotent provider call and keeps the caller+// from having to know which stage the attempt reached.+func (b *PhysicalBackuper) attemptFileReferences(+ spec terminalPersistSpec,+) []storage_files.StoredFileReference {+ if spec.Result.FileName == "" {+ return nil+ }++ references := []storage_files.StoredFileReference{+ {StorageID: spec.StorageID, FileName: spec.Result.FileName},+ {StorageID: spec.StorageID, FileName: spec.Result.FileName + metadataSuffix},+ }++ if spec.Result.ManifestFileName != "" {+ references = append(references,+ storage_files.StoredFileReference{StorageID: spec.StorageID, FileName: spec.Result.ManifestFileName})+ }++ return references+}+func (b *PhysicalBackuper) finalizeFullAsError(fullBackup *physical_models.PhysicalFullBackup,reason physical_enums.PhysicalBackupErrorReason,diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go b/backend/internal/features/backups/backups/backuping/physical/backuper_test.goindex 77de91fc..3fe64022 100644--- a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/backuper_test.go@@ -2,6 +2,7 @@ package backuping_physicalimport ("errors"+ "strings""testing""time"@@ -19,8 +20,11 @@ import ("databasus-backend/internal/features/databases""databasus-backend/internal/features/notifiers"notifier_models "databasus-backend/internal/features/notifiers/models"+ "databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"workspaces_services "databasus-backend/internal/features/workspaces/services""databasus-backend/internal/storage"+ "databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger""databasus-backend/internal/util/walmath")@@ -140,7 +144,7 @@ func Test_PersistFullResult_WhenCompleted_CopiesCompressionManifestFieldsAndReleCompletedAt: time.Now().UTC(),}- require.NoError(t, backuper.persistFullResult(fullBackup, result, nil))+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))persisted, err := physical_repositories.GetFullBackupRepository().FindByID(fullBackup.ID)require.NoError(t, err)@@ -174,7 +178,7 @@ func Test_PersistFullResult_WhenCompletedWithMeasuredClusterSize_StoresRawSizeMbCompletedAt: time.Now().UTC(),}- require.NoError(t, backuper.persistFullResult(fullBackup, result, new(4321.5)))+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, new(4321.5)))persisted, err := physical_repositories.GetFullBackupRepository().FindByID(fullBackup.ID)require.NoError(t, err)@@ -197,7 +201,7 @@ func Test_PersistFullResult_WhenClusterSizeUnmeasured_LeavesRawSizeMbNil(t *testCompletedAt: time.Now().UTC(),}- require.NoError(t, backuper.persistFullResult(fullBackup, result, nil))+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))persisted, err := physical_repositories.GetFullBackupRepository().FindByID(fullBackup.ID)require.NoError(t, err)@@ -221,7 +225,7 @@ func Test_PersistFullResult_WhenErrorStatus_RecordsTerminalTimeAndReleasesInFligManifestFileName: "should-not-be-copied",}- require.NoError(t, backuper.persistFullResult(fullBackup, result, nil))+ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))persisted, err := physical_repositories.GetFullBackupRepository().FindByID(fullBackup.ID)require.NoError(t, err)@@ -247,7 +251,7 @@ func Test_PersistIncrementalResult_WhenErrorStatus_RecordsTerminalTime(t *testinStatus: physical_enums.PhysicalBackupStatusChainBroken,ErrorReason: new(physical_enums.PhysicalBackupErrorTimelineSwitchDetected),}- require.NoError(t, backuper.persistIncrResult(incremental, result))+ require.NoError(t, backuper.persistIncrResult(t.Context(), incremental, result))persistedIncremental, err := physical_repositories.GetIncrementalBackupRepository().FindByID(incremental.ID)require.NoError(t, err)@@ -275,7 +279,7 @@ func Test_PersistIncrementalResult_WhenCompleted_CopiesFieldsAndReleasesInFlightCompletedAt: time.Now().UTC(),}- require.NoError(t, backuper.persistIncrResult(incrBackup, result))+ require.NoError(t, backuper.persistIncrResult(t.Context(), incrBackup, result))persisted, err := physical_repositories.GetIncrementalBackupRepository().FindByID(incrBackup.ID)require.NoError(t, err)@@ -811,7 +815,7 @@ func Test_PersistFullResult_WhenRowNoLongerInProgress_DoesNotResurrect(t *testinnewBackupID := uuid.New()claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeFull, newBackupID)- err := backuper.persistFullResult(full, postgresql_executor.PhysicalBackupResult{+ err := backuper.persistFullResult(t.Context(), full, postgresql_executor.PhysicalBackupResult{Status: physical_enums.PhysicalBackupStatusCompleted,}, nil)require.NoError(t, err)@@ -911,3 +915,76 @@ func assertInFlightReleased(t *testing.T, databaseID uuid.UUID) {require.NoError(t, err)assert.Nil(t, inFlight, "in-flight claim must be released")}++func Test_PersistFullResult_WhenCompleted_KeepsTheFilesTheAttemptWrote(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ backuper := CreateTestPhysicalBackuper(nil)++ fullBackup := seedInProgressFull(t, prereqs)+ claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeFull, fullBackup.ID)++ fileName := "physical-completed-" + fullBackup.ID.String()++ receipt, err := storages.GetStorageFileStore().WriteFile(+ t.Context(),+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},+ strings.NewReader("physical artifact"),+ )+ require.NoError(t, err)++ result := postgresql_executor.PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusCompleted,+ FileName: fileName,+ Receipts: []storage_files.WriteReceipt{receipt},+ CompletedAt: time.Now().UTC(),+ }++ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))++ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},+ ))++ reader, err := prereqs.Storage.GetFile(+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName,+ )+ require.NoError(t, err, "a published backup keeps its artifact")+ require.NoError(t, reader.Close())+}++func Test_PersistFullResult_WhenFailed_HandsTheFilesBackToCleanup(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ backuper := CreateTestPhysicalBackuper(nil)++ fullBackup := seedInProgressFull(t, prereqs)+ claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeFull, fullBackup.ID)++ fileName := "physical-failed-" + fullBackup.ID.String()++ _, err := storages.GetStorageFileStore().WriteFile(+ t.Context(),+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},+ strings.NewReader("partial physical artifact"),+ )+ require.NoError(t, err)++ reason := physical_enums.PhysicalBackupErrorPgBasebackupFailed+ result := postgresql_executor.PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusError,+ ErrorReason: &reason,+ ErrorMessage: "pg_basebackup failed after producing output",+ FileName: fileName,+ CompletedAt: time.Now().UTC(),+ }++ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))++ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},+ ))++ _, err = prereqs.Storage.GetFile(+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName,+ )+ assert.Error(t, err, "a failed physical backup must not leave its artifact behind")+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.goindex 61d9fd92..fe49b71f 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go@@ -27,7 +27,7 @@ type CommonBackupSpec struct {// Storage stays for the reads this package still does, the parent manifest// fetch among them. Everything it writes goes through FileStore.Storage storages.StorageFileSaver- FileStore storage_files.FileStore+ FileStore *storage_files.StoreEncryption backups_core_enums.BackupEncryptionMasterKey stringFieldEncryptor util_encryption.FieldEncryptordiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.goindex 53565200..07990dc1 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go@@ -10,6 +10,8 @@ import (physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"+ storage_files "databasus-backend/internal/features/storages/files"+ db "databasus-backend/internal/storage"files_utils "databasus-backend/internal/util/files""databasus-backend/internal/util/tools")@@ -110,6 +112,12 @@ func streamWithCodecFallback(fmt.Sprintf("compression downgraded: %s -> %s", codec, codecFallbackOrder[i+1]),"backup_id", spec.BackupID)+ // The next attempt writes a different name, so this one's bytes are+ // owned by nobody. Handing them back now rather than letting the+ // commit window expire keeps the rejected attempt from sitting in+ // the user's storage for as long as a healthy upload would.+ discardAttemptFiles(ctx, spec.Common, fileName)+continue}@@ -126,6 +134,16 @@ func streamWithCodecFallback(return resultFromOutcome(settled, settledFileName), nil}+func discardAttemptFiles(ctx context.Context, common CommonBackupSpec, fileName string) {+ err := common.FileStore.RequestFileDeletions(ctx, db.GetDb(), []storage_files.StoredFileReference{+ {StorageID: common.StorageID, FileName: fileName},+ })+ if err != nil {+ common.Logger.ErrorContext(ctx, "failed to discard a rejected codec attempt",+ "file_name", fileName, "error", err)+ }+}+func resultFromOutcome(outcome streamOutcome, fileName string) PhysicalBackupResult {return PhysicalBackupResult{Status: outcome.Status,diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.mdindex 845da54f..04f22f78 100644--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md@@ -37,7 +37,7 @@## 6. Integrate physical full and incremental backups- [x] 6.1 Route full and incremental stream uploads and their sidecars through the file store and return receipts only after writer goroutines are joined. Give `buildObjectName` (`.../executor.go:152`) an attempt UUID, since its four current inputs are constant across the codec fallback loop, and split it instead into `buildBackupLabel`, which keeps the four inputs it already had, and `buildObjectName(label, attemptID)`. That satisfies the rule the struct was there to protect without a struct: neither function grows a fifth positional value, and neither takes two adjacent `uuid.UUID` values. Have `streamWithCodecFallback` (`.../executor.go:50`) mint each name through a `mintAndSaveAttemptName` callback its two callers supply, named for both of its effects, because the loop holds neither the naming inputs nor the repository that persists `file_name`, and move that function's stream inputs into a `streamAttemptSpec` struct rather than growing an already eight-value positional list. The callback becomes the only producer of object names: the pre-loop `buildObjectName` call and its `Save` (`create_full_uc.go:40` with its assignment and `Save` at `:42` and `:43`, `create_incremental_uc.go:76` with its own at `:78` and `:80`) stop minting the object key and keep only the stable pg_basebackup label, which `newPgBasebackupCommand` takes at `pg_basebackup.go:50` and which names the backup rather than the object. Return the settled attempt's name and take every consumer from that result rather than the caller's variable: `runStreamParams` (`.../executor.go:79`) so the upload-time manifest at `stream.go:317` follows it, the chain-broken removal and the reported `FileName` (`create_full_uc.go:84`, `:92`), `streamResult.FileName` (`create_full_uc.go:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`). Correct the two comments that state the old shape, `executor.go:45` ("every attempt streams to the same object key") and the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and move `executor_test.go` onto the new shape, renaming the table field `expected` (declared at `:19`, set at `:25` and `:31`, read at `:39`) to `objectName` and updating its values for the attempt UUID; verify stream success, a zstd rejection that downgrades to gzip and then completes without the second attempt being refused for a name already in use and with the manifest landing under the settled name, pg_basebackup failure after partial output, cancellation and provider failure with focused tests, renaming `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact` to `..._LeavesNoReceiptToPublish` and asserting the outcome carries no receipts. `runStream` sits below the layer that owns cleanup and its tests drive a fake store, so removal cannot be observed there; task 6.3 covers it against a real store.-- [ ] 6.2 Confirm physical receipts in the same transaction that publishes the backup and leave pending deletions in place on terminal failure paths, including manifest, metadata and incremental-chain failures; verify rollback and restart behavior with service and scheduler tests.+- [x] 6.2 Confirm physical receipts in the same transaction that publishes the backup and leave pending deletions in place on terminal failure paths, including manifest, metadata and incremental-chain failures; verify rollback and restart behavior with service and scheduler tests. A superseded attempt, whose row a later attempt already moved out of `IN_PROGRESS`, hands its files back in that same transaction, and a codec attempt the fallback rejects hands its own back immediately rather than waiting out the commit window.- [ ] 6.3 Convert the physical reproduction tests into passing cleanup tests on local storage and MinIO; run the worker until idle and assert that partial base files, manifests, exact chunks, multipart uploads and metadata are absent after ordinary errors and cancellation.- [ ] 6.4 Rewrite restart recovery so it no longer assumes a missing backup file name means no object exists, replacing the stale premise recorded at `scheduler.go:710`, and request deletion for a claimless `IN_PROGRESS` row that carries a file name in the same transaction that records its terminal state, gated on that conditional `IN_PROGRESS` transition affecting one row so a late completion that already wrote a terminal status keeps its file; verify with `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact` renamed to `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_ArtifactRemoved` and rewritten to assert eventual absence, and that recovery neither deletes a confirmed successful backup nor duplicates an existing pending deletion.