PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Remove the last direct provider deletes from backup code
Rostislav Dugin7ac32f104d47
diff --git a/backend/internal/features/backups/backups/backuping/logical/cleaner.go b/backend/internal/features/backups/backups/backuping/logical/cleaner.goindex aa07470d..799a448c 100644--- a/backend/internal/features/backups/backups/backuping/logical/cleaner.go+++ b/backend/internal/features/backups/backups/backuping/logical/cleaner.go@@ -8,11 +8,14 @@ import ("time""github.com/google/uuid"+ "gorm.io/gorm""databasus-backend/internal/features/backups/backups/backuping/shared/gfs"backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"backups_config_logical "databasus-backend/internal/features/backups/config/logical""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"+ db "databasus-backend/internal/storage"util_encryption "databasus-backend/internal/util/encryption""databasus-backend/internal/util/period")@@ -27,6 +30,7 @@ const (type BackupCleaner struct {backupRepository *backups_core_logical.BackupRepositorystorageService *storages.StorageService+ fileStore *storage_files.StorebackupConfigService *backups_config_logical.BackupConfigServicefieldEncryptor util_encryption.FieldEncryptorlogger *slog.Logger@@ -72,33 +76,27 @@ func (c *BackupCleaner) Run(ctx context.Context) {}func (c *BackupCleaner) DeleteBackup(ctx context.Context, backup *backups_core_logical.LogicalBackup) error {- logger := c.logger.With("backup_id", backup.ID, "database_id", backup.DatabaseID)-for _, listener := range c.backupRemoveListeners {if err := listener.OnBeforeBackupRemove(backup); err != nil {return err}}- storage, err := c.storageService.GetStorageByID(ctx, backup.StorageID)- if err != nil {- return err- }-- if err := storage.DeleteFile(ctx, c.fieldEncryptor, logger, backup.FileName); err != nil {- // we do not return error here, because sometimes clean up performed- // before unavailable storage removal or change - therefore we should- // proceed even in case of error. It's possible that some S3 or- // storage is not available yet, it should not block us- logger.WarnContext(ctx, "failed to delete backup file", "error", err)+ references := []storage_files.StoredFileReference{+ {StorageID: backup.StorageID, FileName: backup.FileName},+ {StorageID: backup.StorageID, FileName: backup.FileName + metadataSuffix},}- metadataFileName := backup.FileName + metadataSuffix- if err := storage.DeleteFile(ctx, c.fieldEncryptor, logger, metadataFileName); err != nil {- logger.WarnContext(ctx, "failed to delete backup metadata file", "error", err)- }+ // The row and the obligation to remove its files commit together, so an+ // unreachable storage no longer decides whether the row may go, and a rolled+ // back removal leaves the files where the row still names them.+ return db.GetDb().Transaction(func(tx *gorm.DB) error {+ if err := c.fileStore.RequestFileDeletions(ctx, tx, references); err != nil {+ return err+ }- return c.backupRepository.DeleteByID(backup.ID)+ return c.backupRepository.DeleteByIDInTransaction(tx, backup.ID)+ })}func (c *BackupCleaner) AddBackupRemoveListener(listener backups_core_logical.BackupRemoveListener) {diff --git a/backend/internal/features/backups/backups/backuping/logical/di.go b/backend/internal/features/backups/backups/backuping/logical/di.goindex 859a9feb..2712e806 100644--- a/backend/internal/features/backups/backups/backuping/logical/di.go+++ b/backend/internal/features/backups/backups/backuping/logical/di.go@@ -26,6 +26,7 @@ var (var backupCleaner = &BackupCleaner{backupRepository,storages.GetStorageService(),+ storages.GetStorageFileStore(),backups_config_logical.GetBackupConfigService(),encryption.GetFieldEncryptor(),logger.GetLogger(),diff --git a/backend/internal/features/backups/backups/backuping/logical/testing.go b/backend/internal/features/backups/backups/backuping/logical/testing.goindex ba1cd345..9ddd544c 100644--- a/backend/internal/features/backups/backups/backuping/logical/testing.go+++ b/backend/internal/features/backups/backups/backuping/logical/testing.go@@ -139,6 +139,7 @@ func CreateTestBackupCleaner() *BackupCleaner {return &BackupCleaner{backupRepository,storages.GetStorageService(),+ storages.GetStorageFileStore(),backups_config_logical.GetBackupConfigService(),encryption.GetFieldEncryptor(),logger.GetLogger(),diff --git a/backend/internal/features/backups/backups/controllers/logical/controller_test.go b/backend/internal/features/backups/backups/controllers/logical/controller_test.goindex 8f1454d3..a84cc856 100644--- a/backend/internal/features/backups/backups/controllers/logical/controller_test.go+++ b/backend/internal/features/backups/backups/controllers/logical/controller_test.go@@ -31,6 +31,7 @@ import ("databasus-backend/internal/features/databases"postgresql_logical "databasus-backend/internal/features/databases/databases/postgresql/logical""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"local_storage "databasus-backend/internal/features/storages/models/local"task_cancellation "databasus-backend/internal/features/tasks/cancellation"users_dto "databasus-backend/internal/features/users/dto"@@ -1687,6 +1688,13 @@ func Test_DeleteBackup_RemovesBackupAndMetadataFilesFromDisk(t *testing.T) {http.StatusNoContent,)+ // Deleting a backup commits the obligation to remove its files; the worker+ // carries it out, so the test drives it before asserting the disk is clean.+ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName + ".metadata"},+ ))+_, err = os.Stat(backupFilePath)assert.True(t, os.IsNotExist(err), "backup file should be removed from disk after deletion")diff --git a/backend/internal/features/backups/backups/core/logical/repository.go b/backend/internal/features/backups/backups/core/logical/repository.goindex c58959b8..30b76aab 100644--- a/backend/internal/features/backups/backups/core/logical/repository.go+++ b/backend/internal/features/backups/backups/core/logical/repository.go@@ -198,7 +198,13 @@ func (r *BackupRepository) FindByDatabaseIdAndStatus(}func (r *BackupRepository) DeleteByID(id uuid.UUID) error {- return storage.GetDb().Delete(&LogicalBackup{}, "id = ?", id).Error+ return r.DeleteByIDInTransaction(storage.GetDb(), id)+}++// DeleteByIDInTransaction lets the caller remove the row and record the obligation+// to remove the files it names in one transaction.+func (r *BackupRepository) DeleteByIDInTransaction(tx *gorm.DB, id uuid.UUID) error {+ return tx.Delete(&LogicalBackup{}, "id = ?", id).Error}func (r *BackupRepository) FindBackupsBeforeDate(diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.goindex 6d3a623c..f6dc1ea2 100644--- a/backend/internal/features/backups/backups/core/physical/service/service.go+++ b/backend/internal/features/backups/backups/core/physical/service/service.go@@ -1065,10 +1065,6 @@ func (s *PhysicalBackupService) requestFileDeletions(references := make([]storage_files.StoredFileReference, 0, len(fileNames))for _, fileName := range fileNames {- if fileName == "" {- continue- }-references = append(references,storage_files.StoredFileReference{StorageID: storageID, FileName: fileName})}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.goindex 439d5c94..6de39d1d 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go@@ -11,8 +11,8 @@ import (chain_view "databasus-backend/internal/features/backups/backups/core/physical/chain_view"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"- "databasus-backend/internal/features/storages"- util_encryption "databasus-backend/internal/util/encryption"+ storage_files "databasus-backend/internal/features/storages/files"+ db "databasus-backend/internal/storage")type CreateFullBackupUsecase struct{}@@ -89,8 +89,8 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS}if validation.Status == chain_view.ValidationStatusChainBroken {- removeUploadedArtifactsAfterChainBroken(- spec.Storage, spec.FieldEncryptor, streamResult.FileName, spec.Logger)+ discardArtifactsAfterChainBroken(+ ctx, spec.FileStore, spec.StorageID, streamResult.FileName, spec.Logger)reason := physical_enums.PhysicalBackupErrorStartLsnOutsideTimeline@@ -185,28 +185,22 @@ func classifyFullStreamError(streamErr error, stderr []byte) streamOutcome {}}-// removeUploadedArtifactsAfterChainBroken deletes the streamed artifact and its-// reconstructed-manifest sidecar after a post-stream CHAIN_BROKEN verdict, so a-// rejected FULL leaves nothing dangling in storage. The .metadata sidecar is not-// touched here: it is written only after this point (see uploadFullMetadata), so-// it does not yet exist. DeleteFile is idempotent on not-found.-func removeUploadedArtifactsAfterChainBroken(- storage storages.StorageFileSaver,- encryptor util_encryption.FieldEncryptor,+// A rejected FULL keeps nothing: its streamed artifact and reconstructed manifest+// go back to cleanup. The .metadata sidecar is written only after this point, so+// naming it here would only cost one idempotent provider call.+func discardArtifactsAfterChainBroken(+ ctx context.Context,+ fileStore *storage_files.Store,+ storageID uuid.UUID,fileName string,logger *slog.Logger,) {- manifestName := fileName + manifestSuffix-- if err := storage.DeleteFile(context.Background(), encryptor, logger, manifestName); err != nil {- logger.Warn("failed to remove manifest after CHAIN_BROKEN",- "file_name", manifestName,- "error", err)+ references := []storage_files.StoredFileReference{+ {StorageID: storageID, FileName: fileName},+ {StorageID: storageID, FileName: fileName + manifestSuffix},}- if err := storage.DeleteFile(context.Background(), encryptor, logger, fileName); err != nil {- logger.Warn("failed to remove artifact after CHAIN_BROKEN",- "file_name", fileName,- "error", err)+ if err := fileStore.RequestFileDeletions(context.WithoutCancel(ctx), db.GetDb(), references); err != nil {+ logger.Warn("failed to discard artifacts after CHAIN_BROKEN", "file_name", fileName, "error", err)}}diff --git a/backend/internal/features/storages/files/store.go b/backend/internal/features/storages/files/store.goindex 97656a08..28e93c8c 100644--- a/backend/internal/features/storages/files/store.go+++ b/backend/internal/features/storages/files/store.go@@ -123,6 +123,18 @@ func (s *Store) RequestFileDeletions(tx *gorm.DB,references []StoredFileReference,) error {+ // A reference with no name cannot become an obligation, and a caller deriving+ // names from a row should not have to know which of them the row carries.+ named := make([]StoredFileReference, 0, len(references))++ for _, reference := range references {+ if reference.FileName != "" {+ named = append(named, reference)+ }+ }++ references = named+if err := requireTransaction(tx); err != nil {return err}diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.mdindex c9c9f76f..c55f7b83 100644--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md@@ -50,7 +50,7 @@## 8. Replace synchronous cleanup paths-- [ ] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.+- [x] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.- [x] 8.2 Change physical backup, WAL, history and metadata cleanup, `removeUploadedArtifactsAfterChainBroken` (`create_full_uc.go:192` and `:198`, whose manifest name also comes from the pre-loop file name, and whose doc comment at `:179` restates its name and promises that `DeleteFile` is idempotent on not-found, which stops being what the call does, so delete both claims and keep only what survives), `DeleteChainDependentsKeepFull`, `DeleteOrphanWalSegmentsInSpan`, `deleteOrphanedHistoryFiles` and retention paths to request exact references transactionally, removing both the fail-open and fail-closed remote deletion behavior and the remote calls that currently run inside the cascade transaction; verify cascade and partial-batch rollback cases, including that a rolled-back cascade leaves every object intact.- [ ] 8.3 Add clear-path tests that create real local and MinIO artifacts, invoke each cleaner, run the deletion worker until idle and assert both catalog removal and physical absence; include one transient deletion failure to prove the catalog may disappear before eventual remote cleanup succeeds.- [ ] 8.4 Make database removal and its backup listener use one PostgreSQL transaction that records logical, FULL, INCR, WAL, history, manifest and metadata references before cascading catalog rows, changing the `OnBeforeDatabaseRemove` contract and its three implementations together, and run the listener work that dials the source database, which is cancellation and replication-slot cleanup, before that transaction opens rather than inside it; verify controller-level commit, failure rollback, real local and MinIO cleanup through `DeleteDatabase`, that no catalog transaction stays open across a source-database dial timeout, and that a WAL upload for a name the cascade recorded waits for that transaction instead of publishing.