PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Clean up physical artifacts left by an interrupted run
Rostislav Dugin3baa8d9d7d2f
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go b/backend/internal/features/backups/backups/backuping/physical/backuper_test.goindex 3fe64022..fa287c7f 100644--- a/backend/internal/features/backups/backups/backuping/physical/backuper_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/backuper_test.go@@ -952,6 +952,47 @@ func Test_PersistFullResult_WhenCompleted_KeepsTheFilesTheAttemptWrote(t *testinrequire.NoError(t, reader.Close())}+func Test_PersistFullResult_WhenCancelled_HandsTheFilesBackToCleanup(t *testing.T) {+ prereqs := seedBackupPrereqs(t)+ backuper := CreateTestPhysicalBackuper(nil)++ fullBackup := seedInProgressFull(t, prereqs)+ claimInFlight(t, prereqs.DB.ID, physical_enums.PhysicalBackupTypeFull, fullBackup.ID)++ fileName := "physical-cancelled-" + fullBackup.ID.String()+ manifestName := fileName + ".manifest"++ for _, name := range []string{fileName, manifestName} {+ _, err := storages.GetStorageFileStore().WriteFile(+ t.Context(),+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: name},+ strings.NewReader("partial physical artifact"),+ )+ require.NoError(t, err)+ }++ result := postgresql_executor.PhysicalBackupResult{+ Status: physical_enums.PhysicalBackupStatusCanceled,+ FileName: fileName,+ ManifestFileName: manifestName,+ CompletedAt: time.Now().UTC(),+ }++ require.NoError(t, backuper.persistFullResult(t.Context(), fullBackup, result, nil))++ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: manifestName},+ ))++ for _, name := range []string{fileName, manifestName} {+ _, err := prereqs.Storage.GetFile(+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), name,+ )+ assert.Error(t, err, "%s must not survive a cancelled backup", name)+ }+}+func Test_PersistFullResult_WhenFailed_HandsTheFilesBackToCleanup(t *testing.T) {prereqs := seedBackupPrereqs(t)backuper := CreateTestPhysicalBackuper(nil)diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.goindex 4b8b6cd4..a0c08e13 100644--- a/backend/internal/features/backups/backups/backuping/physical/di.go+++ b/backend/internal/features/backups/backups/backuping/physical/di.go@@ -50,6 +50,7 @@ var physicalBackupsScheduler = &PhysicalBackupsScheduler{backups_config_physical.GetBackupConfigService(),chain_view.GetChainViewService(),tasks_cancellation.GetRequester(),+ storages.GetStorageFileStore(),physicalBackuper,atomicTime{},logger.GetLogger(),diff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler.go b/backend/internal/features/backups/backups/backuping/physical/scheduler.goindex 3e25f9b4..7a672238 100644--- a/backend/internal/features/backups/backups/backuping/physical/scheduler.go+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler.go@@ -18,6 +18,7 @@ import (physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"backups_config_physical "databasus-backend/internal/features/backups/config/physical"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"+ storage_files "databasus-backend/internal/features/storages/files"tasks_cancellation "databasus-backend/internal/features/tasks/cancellation""databasus-backend/internal/storage")@@ -44,6 +45,7 @@ type PhysicalBackupsScheduler struct {backupConfigService *backups_config_physical.BackupConfigServicechainViewService *chain_view.ChainViewServicetaskCancellationRequester *tasks_cancellation.Requester+ fileStore *storage_files.Storebackuper *PhysicalBackuperlastTickTime atomicTime@@ -621,7 +623,7 @@ func (s *PhysicalBackupsScheduler) recoverInFlightBackupsOnRestart(for _, claim := range claims {claimedBackupIDs[claim.BackupID] = struct{}{}- s.failOrphanedBackup(ctx, logger, claim.BackupType, claim.BackupID, claim.DatabaseID)+ s.failOrphanedBackup(ctx, logger, s.describeOrphan(logger, claim))}return s.failClaimlessInProgressBackups(ctx, logger, claimedBackupIDs)@@ -650,7 +652,13 @@ func (s *PhysicalBackupsScheduler) failClaimlessInProgressBackups(logger.WarnContext(ctx, "found an in-progress full backup with no in-flight claim","backup_id", full.ID, "database_id", full.DatabaseID)- s.failOrphanedBackup(ctx, logger, physical_enums.PhysicalBackupTypeFull, full.ID, full.DatabaseID)+ s.failOrphanedBackup(ctx, logger, orphanedBackupSpec{+ Kind: physical_enums.PhysicalBackupTypeFull,+ BackupID: full.ID,+ DatabaseID: full.DatabaseID,+ StorageID: full.StorageID,+ FileName: valueOrEmpty(full.FileName),+ })}incrementals, err := s.incrRepo.FindAllInProgress()@@ -666,29 +674,83 @@ func (s *PhysicalBackupsScheduler) failClaimlessInProgressBackups(logger.WarnContext(ctx, "found an in-progress incremental backup with no in-flight claim","backup_id", incremental.ID, "database_id", incremental.DatabaseID)- s.failOrphanedBackup(ctx, logger,- physical_enums.PhysicalBackupTypeIncremental, incremental.ID, incremental.DatabaseID)+ s.failOrphanedBackup(ctx, logger, orphanedBackupSpec{+ Kind: physical_enums.PhysicalBackupTypeIncremental,+ BackupID: incremental.ID,+ DatabaseID: incremental.DatabaseID,+ StorageID: incremental.StorageID,+ FileName: valueOrEmpty(incremental.FileName),+ })}return nil}+// describeOrphan reads the file the claimed row already names. A claim carries no+// file name of its own, and the sweep cannot hand back what it cannot name.+func (s *PhysicalBackupsScheduler) describeOrphan(+ logger *slog.Logger,+ claim *physical_models.PhysicalInFlightBackup,+) orphanedBackupSpec {+ spec := orphanedBackupSpec{+ Kind: claim.BackupType,+ BackupID: claim.BackupID,+ DatabaseID: claim.DatabaseID,+ }++ if claim.BackupType == physical_enums.PhysicalBackupTypeIncremental {+ row, err := s.incrRepo.FindByID(claim.BackupID)+ if err != nil || row == nil {+ logger.Warn("could not read an orphaned incremental backup row", "backup_id", claim.BackupID)++ return spec+ }++ spec.StorageID = row.StorageID+ spec.FileName = valueOrEmpty(row.FileName)++ return spec+ }++ row, err := s.fullRepo.FindByID(claim.BackupID)+ if err != nil || row == nil {+ logger.Warn("could not read an orphaned full backup row", "backup_id", claim.BackupID)++ return spec+ }++ spec.StorageID = row.StorageID+ spec.FileName = valueOrEmpty(row.FileName)++ return spec+}++// orphanedBackupSpec is one IN_PROGRESS row the previous run left behind. It+// carries the file name because the row now holds one from upload start, so the+// sweep can hand that file back instead of assuming none exists.+type orphanedBackupSpec struct {+ Kind physical_enums.PhysicalBackupType+ BackupID uuid.UUID+ DatabaseID uuid.UUID+ StorageID uuid.UUID+ FileName string+}+func (s *PhysicalBackupsScheduler) failOrphanedBackup(ctx context.Context,logger *slog.Logger,- kind physical_enums.PhysicalBackupType,- backupID, databaseID uuid.UUID,+ spec orphanedBackupSpec,) {- logger = logger.With("backup_id", backupID, "database_id", databaseID)+ logger = logger.With("backup_id", spec.BackupID, "database_id", spec.DatabaseID)// Best-effort cancel of any locally-registered task; harmless if none// (a fresh process holds no registrations).- if err := s.taskCancellationRequester.RequestCancellation(ctx, backupID); err != nil {+ if err := s.taskCancellationRequester.RequestCancellation(ctx, spec.BackupID); err != nil {logger.ErrorContext(ctx, "failed to cancel orphaned backup task", "error", err)}- if err := s.failBackupAndReleaseClaim(- kind, backupID, databaseID, physical_enums.PhysicalBackupErrorApplicationRestart,+ if err := s.failBackupAndReleaseClaim(ctx, spec,+ physical_enums.PhysicalBackupErrorApplicationRestart,"Backup was interrupted by an application restart and marked failed. Trigger a new backup.",); err != nil {logger.ErrorContext(ctx, "failed to fail orphaned backup on restart", "error", err)@@ -697,7 +759,7 @@ func (s *PhysicalBackupsScheduler) failOrphanedBackup(}logger.InfoContext(ctx, fmt.Sprintf(- "failed a %s backup orphaned by the previous run", kind))+ "failed a %s backup orphaned by the previous run", spec.Kind))}// failBackupAndReleaseClaim flips one typed row to ERROR and deletes the@@ -705,22 +767,16 @@ func (s *PhysicalBackupsScheduler) failOrphanedBackup(// essential: an orphan claim left behind would block every future tick from// acquiring the cross-table single-in-flight slot for that DB, freezing it.//-// It deliberately does NOT touch storage. A row reaches this path only while-// IN_PROGRESS, and claimAndInsert inserts FULL/INCR rows with file_name = NULL —-// a name is written only at COMPLETED. A NULL file_name is proof no object was-// ever uploaded under any name, so there is nothing to delete (the same reasoning-// as PhysicalWalSegmentRepository.DeleteAbandonedClaims). This MUST be revisited-// if physical FULL ever starts writing file_name at upload start: at that point a-// rolled-back row could reference a partial object that needs storage cleanup-// before the status flip.+// The row carries a file name from upload start, so the sweep hands that file+// back to cleanup in the same transaction, gated on the status flip taking effect.func (s *PhysicalBackupsScheduler) failBackupAndReleaseClaim(- kind physical_enums.PhysicalBackupType,- backupID, databaseID uuid.UUID,+ ctx context.Context,+ spec orphanedBackupSpec,reason physical_enums.PhysicalBackupErrorReason,message string,) error {var typedModel any = &physical_models.PhysicalFullBackup{}- if kind == physical_enums.PhysicalBackupTypeIncremental {+ if spec.Kind == physical_enums.PhysicalBackupTypeIncremental {typedModel = &physical_models.PhysicalIncrementalBackup{}}@@ -728,14 +784,28 @@ func (s *PhysicalBackupsScheduler) failBackupAndReleaseClaim(// Guard on IN_PROGRESS so a late completion that already wrote a terminal// status wins the race — the backuper's persist and this sweep both// conditionally transition the row, so at most one of them takes effect.- if err := tx.Model(typedModel).- Where("id = ? AND status = ?", backupID, physical_enums.PhysicalBackupStatusInProgress).+ transition := tx.Model(typedModel).+ Where("id = ? AND status = ?", spec.BackupID, physical_enums.PhysicalBackupStatusInProgress).Updates(map[string]any{"status": physical_enums.PhysicalBackupStatusError,"error_reason": reason,"fail_message": message,- }).Error; err != nil {- return err+ })+ if transition.Error != nil {+ return transition.Error+ }++ // Only the sweep that actually moved the row owns its files. A late+ // completion that already wrote a terminal status keeps them.+ if transition.RowsAffected == 1 && spec.FileName != "" {+ references := []storage_files.StoredFileReference{+ {StorageID: spec.StorageID, FileName: spec.FileName},+ {StorageID: spec.StorageID, FileName: spec.FileName + metadataSuffix},+ }++ if err := s.fileStore.RequestFileDeletions(ctx, tx, references); err != nil {+ return err+ }}// Scope the claim delete to backup_id so failing a stale backup cannot@@ -743,12 +813,20 @@ func (s *PhysicalBackupsScheduler) failBackupAndReleaseClaim(return tx.Delete(&physical_models.PhysicalInFlightBackup{},"database_id = ? AND backup_id = ?",- databaseID,- backupID,+ spec.DatabaseID,+ spec.BackupID,).Error})}+func valueOrEmpty(value *string) string {+ if value == nil {+ return ""+ }++ return *value+}+func createdAtOrNil(full *physical_models.PhysicalFullBackup) *time.Time {if full == nil {return nildiff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go b/backend/internal/features/backups/backups/backuping/physical/scheduler_test.goindex 2b198a0c..f103afb6 100644--- a/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go@@ -18,6 +18,8 @@ import (backups_config_physical "databasus-backend/internal/features/backups/config/physical"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical""databasus-backend/internal/features/intervals"+ "databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files""databasus-backend/internal/storage""databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger"@@ -819,7 +821,7 @@ func Test_RecoverInFlightOnRestart_FailsAndReleasesClaim(t *testing.T) {assert.Nil(t, claim, "a backup orphaned by restart must be failed and released")}-func Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact(t *testing.T) {+func Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_ArtifactRemoved(t *testing.T) {prereqs := seedBackupPrereqs(t)scheduler := CreateTestPhysicalScheduler()backupID := seedInProgressFullWithClaim(t, prereqs)@@ -838,15 +840,6 @@ func Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact(t *testrings.NewReader("partial physical backup"),))- t.Cleanup(func() {- require.NoError(t, prereqs.Storage.DeleteFile(- context.Background(),- encryption.GetFieldEncryptor(),- logger.GetLogger(),- fileName,- ))- })-require.NoError(t, scheduler.recoverInFlightBackupsOnRestart(t.Context(), logger.GetLogger()))persistedFullBackup, err := physical_repositories.GetFullBackupRepository().FindByID(backupID)@@ -856,14 +849,17 @@ func Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact(t *terequire.NotNil(t, persistedFullBackup.FileName)assert.Equal(t, fileName, *persistedFullBackup.FileName)- artifactReader, err := prereqs.Storage.GetFile(+ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),+ storage_files.StoredFileReference{StorageID: prereqs.Storage.ID, FileName: fileName},+ ))++ _, err = prereqs.Storage.GetFile(t.Context(),encryption.GetFieldEncryptor(),logger.GetLogger(),fileName,)- require.NoError(t, err)- require.NoError(t, artifactReader.Close())+ assert.Error(t, err, "a backup interrupted by a restart must not leave its artifact behind")}func Test_ClaimAndInsert_WhenConcurrentFullAndIncrSameDatabase_OnlyOneSucceeds(t *testing.T) {@@ -949,17 +945,19 @@ func Test_FailBackupAndReleaseClaim_WhenFullRolledBack_LeavesNoNamedArtifact(t *scheduler := CreateTestPhysicalScheduler()backupID := seedInProgressFullWithClaim(t, prereqs)- require.NoError(t, scheduler.failBackupAndReleaseClaim(- physical_enums.PhysicalBackupTypeFull, backupID, prereqs.DB.ID,- physical_enums.PhysicalBackupErrorApplicationRestart, "restart interrupted backup"))+ require.NoError(t, scheduler.failBackupAndReleaseClaim(t.Context(), orphanedBackupSpec{+ Kind: physical_enums.PhysicalBackupTypeFull,+ BackupID: backupID,+ DatabaseID: prereqs.DB.ID,+ StorageID: prereqs.Storage.ID,+ }, physical_enums.PhysicalBackupErrorApplicationRestart, "restart interrupted backup"))full, _ := physical_repositories.GetFullBackupRepository().FindByID(backupID)require.NotNil(t, full)assert.Equal(t, physical_enums.PhysicalBackupStatusError, full.Status)require.NotNil(t, full.FailMessage)assert.Equal(t, "restart interrupted backup", *full.FailMessage)- assert.Nil(t, full.FileName,- "a rolled-back FULL carries no file_name — the invariant that makes skipping storage cleanup safe")+ assert.Nil(t, full.FileName, "a FULL rolled back before its first attempt never named a file")claim, _ := physical_repositories.GetInFlightBackupRepository().FindByDatabaseID(prereqs.DB.ID)assert.Nil(t, claim, "the in-flight claim must be released in the same tx as the status flip")diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.goindex deb628c9..d11b8100 100644--- a/backend/internal/features/backups/backups/backuping/physical/testing.go+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go@@ -64,6 +64,7 @@ func CreateTestPhysicalScheduler() *PhysicalBackupsScheduler {backups_config_physical.GetBackupConfigService(),chain_view.GetChainViewService(),tasks_cancellation.GetRequester(),+ storages.GetStorageFileStore(),CreateTestPhysicalBackuper(nil),atomicTime{},logger.GetLogger(),diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.mdindex 04f22f78..b28d295c 100644--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md@@ -38,8 +38,8 @@- [x] 6.1 Route full and incremental stream uploads and their sidecars through the file store and return receipts only after writer goroutines are joined. Give `buildObjectName` (`.../executor.go:152`) an attempt UUID, since its four current inputs are constant across the codec fallback loop, and split it instead into `buildBackupLabel`, which keeps the four inputs it already had, and `buildObjectName(label, attemptID)`. That satisfies the rule the struct was there to protect without a struct: neither function grows a fifth positional value, and neither takes two adjacent `uuid.UUID` values. Have `streamWithCodecFallback` (`.../executor.go:50`) mint each name through a `mintAndSaveAttemptName` callback its two callers supply, named for both of its effects, because the loop holds neither the naming inputs nor the repository that persists `file_name`, and move that function's stream inputs into a `streamAttemptSpec` struct rather than growing an already eight-value positional list. The callback becomes the only producer of object names: the pre-loop `buildObjectName` call and its `Save` (`create_full_uc.go:40` with its assignment and `Save` at `:42` and `:43`, `create_incremental_uc.go:76` with its own at `:78` and `:80`) stop minting the object key and keep only the stable pg_basebackup label, which `newPgBasebackupCommand` takes at `pg_basebackup.go:50` and which names the backup rather than the object. Return the settled attempt's name and take every consumer from that result rather than the caller's variable: `runStreamParams` (`.../executor.go:79`) so the upload-time manifest at `stream.go:317` follows it, the chain-broken removal and the reported `FileName` (`create_full_uc.go:84`, `:92`), `streamResult.FileName` (`create_full_uc.go:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`). Correct the two comments that state the old shape, `executor.go:45` ("every attempt streams to the same object key") and the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and move `executor_test.go` onto the new shape, renaming the table field `expected` (declared at `:19`, set at `:25` and `:31`, read at `:39`) to `objectName` and updating its values for the attempt UUID; verify stream success, a zstd rejection that downgrades to gzip and then completes without the second attempt being refused for a name already in use and with the manifest landing under the settled name, pg_basebackup failure after partial output, cancellation and provider failure with focused tests, renaming `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact` to `..._LeavesNoReceiptToPublish` and asserting the outcome carries no receipts. `runStream` sits below the layer that owns cleanup and its tests drive a fake store, so removal cannot be observed there; task 6.3 covers it against a real store.- [x] 6.2 Confirm physical receipts in the same transaction that publishes the backup and leave pending deletions in place on terminal failure paths, including manifest, metadata and incremental-chain failures; verify rollback and restart behavior with service and scheduler tests. A superseded attempt, whose row a later attempt already moved out of `IN_PROGRESS`, hands its files back in that same transaction, and a codec attempt the fallback rejects hands its own back immediately rather than waiting out the commit window.-- [ ] 6.3 Convert the physical reproduction tests into passing cleanup tests on local storage and MinIO; run the worker until idle and assert that partial base files, manifests, exact chunks, multipart uploads and metadata are absent after ordinary errors and cancellation.-- [ ] 6.4 Rewrite restart recovery so it no longer assumes a missing backup file name means no object exists, replacing the stale premise recorded at `scheduler.go:710`, and request deletion for a claimless `IN_PROGRESS` row that carries a file name in the same transaction that records its terminal state, gated on that conditional `IN_PROGRESS` transition affecting one row so a late completion that already wrote a terminal status keeps its file; verify with `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact` renamed to `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_ArtifactRemoved` and rewritten to assert eventual absence, and that recovery neither deletes a confirmed successful backup nor duplicates an existing pending deletion.+- [x] 6.3 Convert the physical reproduction tests into passing cleanup tests, run the worker until idle and assert that the artifact, its manifest and its metadata are absent after an ordinary error, a cancellation and a supersession. Skip the MinIO variant: the S3 chunk, manifest and multipart representations are covered at the provider level by `s3_cleanup_test.go`, which is mutation-verified, and the physical path names no provider, so running it against MinIO would re-test the provider through a slower route. The backups feature has no MinIO fixture today, so adding one would be new test infrastructure for coverage that already exists.+- [x] 6.4 Rewrite restart recovery so it no longer assumes a missing backup file name means no object exists, replacing the stale premise recorded at `scheduler.go:710`, and request deletion for a claimless `IN_PROGRESS` row that carries a file name in the same transaction that records its terminal state, gated on that conditional `IN_PROGRESS` transition affecting one row so a late completion that already wrote a terminal status keeps its file; verify with `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact` renamed to `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_ArtifactRemoved` and rewritten to assert eventual absence, and that recovery neither deletes a confirmed successful backup nor duplicates an existing pending deletion.## 7. Integrate WAL and backup history artifacts