Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Remove the last direct provider deletes from backup code

Rostislav Dugin7ac32f104d47
diff --git a/backend/internal/features/backups/backups/backuping/logical/cleaner.go b/backend/internal/features/backups/backups/backuping/logical/cleaner.go
index aa07470d..799a448c 100644
--- a/backend/internal/features/backups/backups/backuping/logical/cleaner.go
+++ b/backend/internal/features/backups/backups/backuping/logical/cleaner.go
@@ -8,11 +8,14 @@ import (
"time"
"github.com/google/uuid"
+ "gorm.io/gorm"
"databasus-backend/internal/features/backups/backups/backuping/shared/gfs"
backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
backups_config_logical "databasus-backend/internal/features/backups/config/logical"
"databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
+ db "databasus-backend/internal/storage"
util_encryption "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/period"
)
@@ -27,6 +30,7 @@ const (
type BackupCleaner struct {
backupRepository *backups_core_logical.BackupRepository
storageService *storages.StorageService
+ fileStore *storage_files.Store
backupConfigService *backups_config_logical.BackupConfigService
fieldEncryptor util_encryption.FieldEncryptor
logger *slog.Logger
@@ -72,33 +76,27 @@ func (c *BackupCleaner) Run(ctx context.Context) {
}
func (c *BackupCleaner) DeleteBackup(ctx context.Context, backup *backups_core_logical.LogicalBackup) error {
- logger := c.logger.With("backup_id", backup.ID, "database_id", backup.DatabaseID)
-
for _, listener := range c.backupRemoveListeners {
if err := listener.OnBeforeBackupRemove(backup); err != nil {
return err
}
}
- storage, err := c.storageService.GetStorageByID(ctx, backup.StorageID)
- if err != nil {
- return err
- }
-
- if err := storage.DeleteFile(ctx, c.fieldEncryptor, logger, backup.FileName); err != nil {
- // we do not return error here, because sometimes clean up performed
- // before unavailable storage removal or change - therefore we should
- // proceed even in case of error. It's possible that some S3 or
- // storage is not available yet, it should not block us
- logger.WarnContext(ctx, "failed to delete backup file", "error", err)
+ references := []storage_files.StoredFileReference{
+ {StorageID: backup.StorageID, FileName: backup.FileName},
+ {StorageID: backup.StorageID, FileName: backup.FileName + metadataSuffix},
}
- metadataFileName := backup.FileName + metadataSuffix
- if err := storage.DeleteFile(ctx, c.fieldEncryptor, logger, metadataFileName); err != nil {
- logger.WarnContext(ctx, "failed to delete backup metadata file", "error", err)
- }
+ // The row and the obligation to remove its files commit together, so an
+ // unreachable storage no longer decides whether the row may go, and a rolled
+ // back removal leaves the files where the row still names them.
+ return db.GetDb().Transaction(func(tx *gorm.DB) error {
+ if err := c.fileStore.RequestFileDeletions(ctx, tx, references); err != nil {
+ return err
+ }
- return c.backupRepository.DeleteByID(backup.ID)
+ return c.backupRepository.DeleteByIDInTransaction(tx, backup.ID)
+ })
}
func (c *BackupCleaner) AddBackupRemoveListener(listener backups_core_logical.BackupRemoveListener) {
diff --git a/backend/internal/features/backups/backups/backuping/logical/di.go b/backend/internal/features/backups/backups/backuping/logical/di.go
index 859a9feb..2712e806 100644
--- a/backend/internal/features/backups/backups/backuping/logical/di.go
+++ b/backend/internal/features/backups/backups/backuping/logical/di.go
@@ -26,6 +26,7 @@ var (
var backupCleaner = &BackupCleaner{
backupRepository,
storages.GetStorageService(),
+ storages.GetStorageFileStore(),
backups_config_logical.GetBackupConfigService(),
encryption.GetFieldEncryptor(),
logger.GetLogger(),
diff --git a/backend/internal/features/backups/backups/backuping/logical/testing.go b/backend/internal/features/backups/backups/backuping/logical/testing.go
index ba1cd345..9ddd544c 100644
--- a/backend/internal/features/backups/backups/backuping/logical/testing.go
+++ b/backend/internal/features/backups/backups/backuping/logical/testing.go
@@ -139,6 +139,7 @@ func CreateTestBackupCleaner() *BackupCleaner {
return &BackupCleaner{
backupRepository,
storages.GetStorageService(),
+ storages.GetStorageFileStore(),
backups_config_logical.GetBackupConfigService(),
encryption.GetFieldEncryptor(),
logger.GetLogger(),
diff --git a/backend/internal/features/backups/backups/controllers/logical/controller_test.go b/backend/internal/features/backups/backups/controllers/logical/controller_test.go
index 8f1454d3..a84cc856 100644
--- a/backend/internal/features/backups/backups/controllers/logical/controller_test.go
+++ b/backend/internal/features/backups/backups/controllers/logical/controller_test.go
@@ -31,6 +31,7 @@ import (
"databasus-backend/internal/features/databases"
postgresql_logical "databasus-backend/internal/features/databases/databases/postgresql/logical"
"databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
local_storage "databasus-backend/internal/features/storages/models/local"
task_cancellation "databasus-backend/internal/features/tasks/cancellation"
users_dto "databasus-backend/internal/features/users/dto"
@@ -1687,6 +1688,13 @@ func Test_DeleteBackup_RemovesBackupAndMetadataFilesFromDisk(t *testing.T) {
http.StatusNoContent,
)
+ // Deleting a backup commits the obligation to remove its files; the worker
+ // carries it out, so the test drives it before asserting the disk is clean.
+ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName + ".metadata"},
+ ))
+
_, err = os.Stat(backupFilePath)
assert.True(t, os.IsNotExist(err), "backup file should be removed from disk after deletion")
diff --git a/backend/internal/features/backups/backups/core/logical/repository.go b/backend/internal/features/backups/backups/core/logical/repository.go
index c58959b8..30b76aab 100644
--- a/backend/internal/features/backups/backups/core/logical/repository.go
+++ b/backend/internal/features/backups/backups/core/logical/repository.go
@@ -198,7 +198,13 @@ func (r *BackupRepository) FindByDatabaseIdAndStatus(
}
func (r *BackupRepository) DeleteByID(id uuid.UUID) error {
- return storage.GetDb().Delete(&LogicalBackup{}, "id = ?", id).Error
+ return r.DeleteByIDInTransaction(storage.GetDb(), id)
+}
+
+// DeleteByIDInTransaction lets the caller remove the row and record the obligation
+// to remove the files it names in one transaction.
+func (r *BackupRepository) DeleteByIDInTransaction(tx *gorm.DB, id uuid.UUID) error {
+ return tx.Delete(&LogicalBackup{}, "id = ?", id).Error
}
func (r *BackupRepository) FindBackupsBeforeDate(
diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.go
index 6d3a623c..f6dc1ea2 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service.go
@@ -1065,10 +1065,6 @@ func (s *PhysicalBackupService) requestFileDeletions(
references := make([]storage_files.StoredFileReference, 0, len(fileNames))
for _, fileName := range fileNames {
- if fileName == "" {
- continue
- }
-
references = append(references,
storage_files.StoredFileReference{StorageID: storageID, FileName: fileName})
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
index 439d5c94..6de39d1d 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
@@ -11,8 +11,8 @@ import (
chain_view "databasus-backend/internal/features/backups/backups/core/physical/chain_view"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"
- "databasus-backend/internal/features/storages"
- util_encryption "databasus-backend/internal/util/encryption"
+ storage_files "databasus-backend/internal/features/storages/files"
+ db "databasus-backend/internal/storage"
)
type CreateFullBackupUsecase struct{}
@@ -89,8 +89,8 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS
}
if validation.Status == chain_view.ValidationStatusChainBroken {
- removeUploadedArtifactsAfterChainBroken(
- spec.Storage, spec.FieldEncryptor, streamResult.FileName, spec.Logger)
+ discardArtifactsAfterChainBroken(
+ ctx, spec.FileStore, spec.StorageID, streamResult.FileName, spec.Logger)
reason := physical_enums.PhysicalBackupErrorStartLsnOutsideTimeline
@@ -185,28 +185,22 @@ func classifyFullStreamError(streamErr error, stderr []byte) streamOutcome {
}
}
-// removeUploadedArtifactsAfterChainBroken deletes the streamed artifact and its
-// reconstructed-manifest sidecar after a post-stream CHAIN_BROKEN verdict, so a
-// rejected FULL leaves nothing dangling in storage. The .metadata sidecar is not
-// touched here: it is written only after this point (see uploadFullMetadata), so
-// it does not yet exist. DeleteFile is idempotent on not-found.
-func removeUploadedArtifactsAfterChainBroken(
- storage storages.StorageFileSaver,
- encryptor util_encryption.FieldEncryptor,
+// A rejected FULL keeps nothing: its streamed artifact and reconstructed manifest
+// go back to cleanup. The .metadata sidecar is written only after this point, so
+// naming it here would only cost one idempotent provider call.
+func discardArtifactsAfterChainBroken(
+ ctx context.Context,
+ fileStore *storage_files.Store,
+ storageID uuid.UUID,
fileName string,
logger *slog.Logger,
) {
- manifestName := fileName + manifestSuffix
-
- if err := storage.DeleteFile(context.Background(), encryptor, logger, manifestName); err != nil {
- logger.Warn("failed to remove manifest after CHAIN_BROKEN",
- "file_name", manifestName,
- "error", err)
+ references := []storage_files.StoredFileReference{
+ {StorageID: storageID, FileName: fileName},
+ {StorageID: storageID, FileName: fileName + manifestSuffix},
}
- if err := storage.DeleteFile(context.Background(), encryptor, logger, fileName); err != nil {
- logger.Warn("failed to remove artifact after CHAIN_BROKEN",
- "file_name", fileName,
- "error", err)
+ if err := fileStore.RequestFileDeletions(context.WithoutCancel(ctx), db.GetDb(), references); err != nil {
+ logger.Warn("failed to discard artifacts after CHAIN_BROKEN", "file_name", fileName, "error", err)
}
}
diff --git a/backend/internal/features/storages/files/store.go b/backend/internal/features/storages/files/store.go
index 97656a08..28e93c8c 100644
--- a/backend/internal/features/storages/files/store.go
+++ b/backend/internal/features/storages/files/store.go
@@ -123,6 +123,18 @@ func (s *Store) RequestFileDeletions(
tx *gorm.DB,
references []StoredFileReference,
) error {
+ // A reference with no name cannot become an obligation, and a caller deriving
+ // names from a row should not have to know which of them the row carries.
+ named := make([]StoredFileReference, 0, len(references))
+
+ for _, reference := range references {
+ if reference.FileName != "" {
+ named = append(named, reference)
+ }
+ }
+
+ references = named
+
if err := requireTransaction(tx); err != nil {
return err
}
diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.md
index c9c9f76f..c55f7b83 100644
--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md
+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md
@@ -50,7 +50,7 @@
## 8. Replace synchronous cleanup paths
-- [ ] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.
+- [x] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.
- [x] 8.2 Change physical backup, WAL, history and metadata cleanup, `removeUploadedArtifactsAfterChainBroken` (`create_full_uc.go:192` and `:198`, whose manifest name also comes from the pre-loop file name, and whose doc comment at `:179` restates its name and promises that `DeleteFile` is idempotent on not-found, which stops being what the call does, so delete both claims and keep only what survives), `DeleteChainDependentsKeepFull`, `DeleteOrphanWalSegmentsInSpan`, `deleteOrphanedHistoryFiles` and retention paths to request exact references transactionally, removing both the fail-open and fail-closed remote deletion behavior and the remote calls that currently run inside the cascade transaction; verify cascade and partial-batch rollback cases, including that a rolled-back cascade leaves every object intact.
- [ ] 8.3 Add clear-path tests that create real local and MinIO artifacts, invoke each cleaner, run the deletion worker until idle and assert both catalog removal and physical absence; include one transient deletion failure to prove the catalog may disappear before eventual remote cleanup succeeds.
- [ ] 8.4 Make database removal and its backup listener use one PostgreSQL transaction that records logical, FULL, INCR, WAL, history, manifest and metadata references before cascading catalog rows, changing the `OnBeforeDatabaseRemove` contract and its three implementations together, and run the listener work that dials the source database, which is cancellation and replication-slot cleanup, before that transaction opens rather than inside it; verify controller-level commit, failure rollback, real local and MinIO cleanup through `DeleteDatabase`, that no catalog transaction stays open across a source-database dial timeout, and that a WAL upload for a name the cascade recorded waits for that transaction instead of publishing.