Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Record physical references before a database removal cascades them away

Rostislav Dugin09e24a6bb3c8
diff --git a/backend/internal/features/backups/backups/backuping/logical/cleaner.go b/backend/internal/features/backups/backups/backuping/logical/cleaner.go
index 799a448c..241e7541 100644
--- a/backend/internal/features/backups/backups/backuping/logical/cleaner.go
+++ b/backend/internal/features/backups/backups/backuping/logical/cleaner.go
@@ -13,10 +13,8 @@ import (
"databasus-backend/internal/features/backups/backups/backuping/shared/gfs"
backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
backups_config_logical "databasus-backend/internal/features/backups/config/logical"
- "databasus-backend/internal/features/storages"
storage_files "databasus-backend/internal/features/storages/files"
db "databasus-backend/internal/storage"
- util_encryption "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/period"
)
@@ -29,10 +27,8 @@ const (
type BackupCleaner struct {
backupRepository *backups_core_logical.BackupRepository
- storageService *storages.StorageService
fileStore *storage_files.Store
backupConfigService *backups_config_logical.BackupConfigService
- fieldEncryptor util_encryption.FieldEncryptor
logger *slog.Logger
backupRemoveListeners []backups_core_logical.BackupRemoveListener
diff --git a/backend/internal/features/backups/backups/backuping/logical/di.go b/backend/internal/features/backups/backups/backuping/logical/di.go
index 2712e806..7ae68127 100644
--- a/backend/internal/features/backups/backups/backuping/logical/di.go
+++ b/backend/internal/features/backups/backups/backuping/logical/di.go
@@ -12,7 +12,6 @@ import (
"databasus-backend/internal/features/storages"
tasks_cancellation "databasus-backend/internal/features/tasks/cancellation"
workspaces_services "databasus-backend/internal/features/workspaces/services"
- "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
)
@@ -25,10 +24,8 @@ var (
var backupCleaner = &BackupCleaner{
backupRepository,
- storages.GetStorageService(),
storages.GetStorageFileStore(),
backups_config_logical.GetBackupConfigService(),
- encryption.GetFieldEncryptor(),
logger.GetLogger(),
[]backups_core_logical.BackupRemoveListener{},
atomic.Bool{},
diff --git a/backend/internal/features/backups/backups/backuping/logical/scheduler.go b/backend/internal/features/backups/backups/backuping/logical/scheduler.go
index aff5e15f..d6439061 100644
--- a/backend/internal/features/backups/backups/backuping/logical/scheduler.go
+++ b/backend/internal/features/backups/backups/backuping/logical/scheduler.go
@@ -341,7 +341,9 @@ func (s *BackupsScheduler) failBackupsInProgress(ctx context.Context, logger *sl
&failMessage,
)
- if err := s.backupRepository.Save(backup); err != nil {
+ // The row already names a file, and the previous run may have uploaded it,
+ // so the terminal state and the obligation commit together.
+ if err := s.backuper.saveTerminalStateAndDiscardFiles(ctx, backup); err != nil {
return err
}
diff --git a/backend/internal/features/backups/backups/backuping/logical/testing.go b/backend/internal/features/backups/backups/backuping/logical/testing.go
index 9ddd544c..f68e9a02 100644
--- a/backend/internal/features/backups/backups/backuping/logical/testing.go
+++ b/backend/internal/features/backups/backups/backuping/logical/testing.go
@@ -22,7 +22,6 @@ import (
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"databasus-backend/internal/storage"
"databasus-backend/internal/util/cache"
- "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
)
@@ -138,10 +137,8 @@ func CreateTestRouter() *gin.Engine {
func CreateTestBackupCleaner() *BackupCleaner {
return &BackupCleaner{
backupRepository,
- storages.GetStorageService(),
storages.GetStorageFileStore(),
backups_config_logical.GetBackupConfigService(),
- encryption.GetFieldEncryptor(),
logger.GetLogger(),
[]backups_core_logical.BackupRemoveListener{},
atomic.Bool{},
diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.go
index ee3d7317..e45138fc 100644
--- a/backend/internal/features/backups/backups/backuping/physical/di.go
+++ b/backend/internal/features/backups/backups/backuping/physical/di.go
@@ -125,6 +125,10 @@ var SetupDependencies = sync.OnceFunc(func() {
// can drop the (now detaching) WAL slot instead of refusing it as active and
// leaving it to pin WAL forever.
databases.GetDatabaseService().AddDbRemoveListener(physicalBackupCancellationListener)
+ // Registered before the cancellation listener's siblings finish, but ordering
+ // does not matter: the cascade that would take the names away runs only after
+ // every listener has returned.
+ databases.GetDatabaseService().AddDbRemoveListener(physical_service.GetPhysicalBackupService())
databases.GetDatabaseService().AddDbRemoveListener(physicalSlotCleanupListener)
backups_config_physical.GetBackupConfigService().SetBackupCancellationListener(physicalBackupCancellationListener)
})
diff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler.go b/backend/internal/features/backups/backups/backuping/physical/scheduler.go
index 7a672238..4ff81db2 100644
--- a/backend/internal/features/backups/backups/backuping/physical/scheduler.go
+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler.go
@@ -658,6 +658,7 @@ func (s *PhysicalBackupsScheduler) failClaimlessInProgressBackups(
DatabaseID: full.DatabaseID,
StorageID: full.StorageID,
FileName: valueOrEmpty(full.FileName),
+ Manifest: valueOrEmpty(full.ManifestFileName),
})
}
@@ -680,6 +681,7 @@ func (s *PhysicalBackupsScheduler) failClaimlessInProgressBackups(
DatabaseID: incremental.DatabaseID,
StorageID: incremental.StorageID,
FileName: valueOrEmpty(incremental.FileName),
+ Manifest: valueOrEmpty(incremental.ManifestFileName),
})
}
@@ -708,6 +710,7 @@ func (s *PhysicalBackupsScheduler) describeOrphan(
spec.StorageID = row.StorageID
spec.FileName = valueOrEmpty(row.FileName)
+ spec.Manifest = valueOrEmpty(row.ManifestFileName)
return spec
}
@@ -721,6 +724,7 @@ func (s *PhysicalBackupsScheduler) describeOrphan(
spec.StorageID = row.StorageID
spec.FileName = valueOrEmpty(row.FileName)
+ spec.Manifest = valueOrEmpty(row.ManifestFileName)
return spec
}
@@ -734,6 +738,7 @@ type orphanedBackupSpec struct {
DatabaseID uuid.UUID
StorageID uuid.UUID
FileName string
+ Manifest string
}
func (s *PhysicalBackupsScheduler) failOrphanedBackup(
@@ -803,6 +808,11 @@ func (s *PhysicalBackupsScheduler) failBackupAndReleaseClaim(
{StorageID: spec.StorageID, FileName: spec.FileName + metadataSuffix},
}
+ if spec.Manifest != "" {
+ references = append(references,
+ storage_files.StoredFileReference{StorageID: spec.StorageID, FileName: spec.Manifest})
+ }
+
if err := s.fileStore.RequestFileDeletions(ctx, tx, references); err != nil {
return err
}
diff --git a/backend/internal/features/backups/backups/core/logical/repository.go b/backend/internal/features/backups/backups/core/logical/repository.go
index 30b76aab..8fb6ab93 100644
--- a/backend/internal/features/backups/backups/core/logical/repository.go
+++ b/backend/internal/features/backups/backups/core/logical/repository.go
@@ -201,8 +201,6 @@ func (r *BackupRepository) DeleteByID(id uuid.UUID) error {
return r.DeleteByIDInTransaction(storage.GetDb(), id)
}
-// DeleteByIDInTransaction lets the caller remove the row and record the obligation
-// to remove the files it names in one transaction.
func (r *BackupRepository) DeleteByIDInTransaction(tx *gorm.DB, id uuid.UUID) error {
return tx.Delete(&LogicalBackup{}, "id = ?", id).Error
}
diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_history_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_history_repository.go
index 2a63f0a9..894aa44d 100644
--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_history_repository.go
+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_history_repository.go
@@ -17,8 +17,6 @@ func (r *PhysicalWalHistoryRepository) Insert(file *physical_models.PhysicalWalH
return r.InsertInTransaction(storage.GetDb(), file)
}
-// InsertInTransaction lets the caller publish the history row and spend the write
-// receipts for its two files together.
func (r *PhysicalWalHistoryRepository) InsertInTransaction(
tx *gorm.DB,
file *physical_models.PhysicalWalHistoryFile,
diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
index 893ca21f..98e165bd 100644
--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
@@ -119,10 +119,9 @@ func (r *PhysicalWalSegmentRepository) DeleteByID(id uuid.UUID) error {
return storage.GetDb().Delete(&physical_models.PhysicalWalSegment{}, "id = ?", id).Error
}
-// DeleteAbandonedClaims removes insert-first WAL claim rows whose upload never
-// finished (file_name still NULL) and that have aged past the grace period. A
-// NULL file_name is proof no bytes were ever written under any name, so there
-// is no storage object to delete. Returns the number of rows removed.
+// A NULL file_name no longer proves the upload wrote nothing: the object is
+// written before the row names it. The bytes such a claim may have left are owned
+// by their own pending deletion, so this only removes the aged-out claim row.
func (r *PhysicalWalSegmentRepository) DeleteAbandonedClaims(
databaseID uuid.UUID,
olderThan time.Time,
@@ -214,8 +213,6 @@ func (r *PhysicalWalSegmentRepository) MarkUploaded(
return r.MarkUploadedInTransaction(storage.GetDb(), id, fileName, compressedSizeMb, encryptionSalt, encryptionIV)
}
-// MarkUploadedInTransaction lets the caller publish the segment and spend the write
-// receipts for its files in one transaction.
func (r *PhysicalWalSegmentRepository) MarkUploadedInTransaction(
tx *gorm.DB,
id uuid.UUID,
diff --git a/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go b/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go
new file mode 100644
index 00000000..6fc70d05
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go
@@ -0,0 +1,100 @@
+package physical_service
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/google/uuid"
+ "gorm.io/gorm"
+
+ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
+ storage_files "databasus-backend/internal/features/storages/files"
+ db "databasus-backend/internal/storage"
+)
+
+// Every physical table cascades on databases.id, so removing a database drops the
+// rows that are the only record of its object names. This records the obligation
+// to remove those objects before the cascade can take the names away.
+func (s *PhysicalBackupService) OnBeforeDatabaseRemove(ctx context.Context, databaseID uuid.UUID) error {
+ return db.GetDb().Transaction(func(tx *gorm.DB) error {
+ references, err := s.collectDatabaseFileReferences(tx, databaseID)
+ if err != nil {
+ return err
+ }
+
+ return s.fileStore.RequestFileDeletions(ctx, tx, references)
+ })
+}
+
+func (s *PhysicalBackupService) collectDatabaseFileReferences(
+ tx *gorm.DB,
+ databaseID uuid.UUID,
+) ([]storage_files.StoredFileReference, error) {
+ references := make([]storage_files.StoredFileReference, 0)
+
+ var fulls []physical_models.PhysicalFullBackup
+ if err := tx.Where("database_id = ?", databaseID).Find(&fulls).Error; err != nil {
+ return nil, fmt.Errorf("list full backups of a database: %w", err)
+ }
+
+ for _, full := range fulls {
+ references = append(references, artifactReferences(
+ full.StorageID, valueOrEmptyString(full.FileName), valueOrEmptyString(full.ManifestFileName))...)
+ }
+
+ var incrementals []physical_models.PhysicalIncrementalBackup
+ if err := tx.Where("database_id = ?", databaseID).Find(&incrementals).Error; err != nil {
+ return nil, fmt.Errorf("list incremental backups of a database: %w", err)
+ }
+
+ for _, incremental := range incrementals {
+ references = append(references, artifactReferences(
+ incremental.StorageID,
+ valueOrEmptyString(incremental.FileName),
+ valueOrEmptyString(incremental.ManifestFileName),
+ )...)
+ }
+
+ var segments []physical_models.PhysicalWalSegment
+ if err := tx.Where("database_id = ?", databaseID).Find(&segments).Error; err != nil {
+ return nil, fmt.Errorf("list wal segments of a database: %w", err)
+ }
+
+ for _, segment := range segments {
+ references = append(references, artifactReferences(
+ segment.StorageID, valueOrEmptyString(segment.FileName), "")...)
+ }
+
+ var historyFiles []physical_models.PhysicalWalHistoryFile
+ if err := tx.Where("database_id = ?", databaseID).Find(&historyFiles).Error; err != nil {
+ return nil, fmt.Errorf("list history files of a database: %w", err)
+ }
+
+ for _, historyFile := range historyFiles {
+ references = append(references, artifactReferences(
+ historyFile.StorageID, historyFile.FileName, "")...)
+ }
+
+ return references, nil
+}
+
+func artifactReferences(
+ storageID uuid.UUID,
+ fileName, manifestFileName string,
+) []storage_files.StoredFileReference {
+ if fileName == "" {
+ return nil
+ }
+
+ references := []storage_files.StoredFileReference{
+ {StorageID: storageID, FileName: fileName},
+ {StorageID: storageID, FileName: fileName + metadataSuffix},
+ }
+
+ if manifestFileName != "" {
+ references = append(references,
+ storage_files.StoredFileReference{StorageID: storageID, FileName: manifestFileName})
+ }
+
+ return references
+}
diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.go
index 378f482d..e1f9bdf2 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go
@@ -804,3 +804,39 @@ func Test_GetLastBackupTimesByDatabaseIDs_EmptyInput_ReturnsEmptyMap(t *testing.
assert.NotNil(t, lastBackupTimes)
assert.Empty(t, lastBackupTimes)
}
+
+func Test_OnBeforeDatabaseRemove_HandsEveryPhysicalArtifactBackToCleanup(t *testing.T) {
+ prereqs := createServiceTestPrereqs(t)
+ databaseID := prereqs.database.ID
+ storageID := prereqs.storage.ID
+
+ fullModel := physical_testing.NewTestCompletedFullBackup(databaseID, storageID, 1, lsn(0), lsn(1))
+ manifestName := *fullModel.FileName + ".manifest"
+ fullModel.ManifestFileName = &manifestName
+ full := physical_testing.CreateTestFullBackup(t, fullModel)
+
+ history := physical_testing.CreateTestWalHistoryFile(t,
+ physical_testing.NewTestWalHistoryFile(databaseID, storageID, 1))
+
+ names := []string{*full.FileName, manifestName, *full.FileName + ".metadata", history.FileName}
+ for _, name := range names {
+ saveObject(t, prereqs.storage, name)
+ }
+
+ // Every physical table cascades on databases.id, so this listener is the last
+ // moment the names exist anywhere.
+ require.NoError(t,
+ physical_service.GetPhysicalBackupService().OnBeforeDatabaseRemove(t.Context(), databaseID))
+
+ references := make([]storage_files.StoredFileReference, 0, len(names))
+ for _, name := range names {
+ references = append(references,
+ storage_files.StoredFileReference{StorageID: storageID, FileName: name})
+ }
+
+ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(), references...))
+
+ for _, name := range names {
+ requireObjectAbsent(t, prereqs.storage, name)
+ }
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
index 6de39d1d..3867e2cc 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
@@ -7,6 +7,7 @@ import (
"time"
"github.com/google/uuid"
+ "gorm.io/gorm"
chain_view "databasus-backend/internal/features/backups/backups/core/physical/chain_view"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
@@ -200,7 +201,11 @@ func discardArtifactsAfterChainBroken(
{StorageID: storageID, FileName: fileName + manifestSuffix},
}
- if err := fileStore.RequestFileDeletions(context.WithoutCancel(ctx), db.GetDb(), references); err != nil {
+ ctx = context.WithoutCancel(ctx)
+
+ if err := db.GetDb().Transaction(func(tx *gorm.DB) error {
+ return fileStore.RequestFileDeletions(ctx, tx, references)
+ }); err != nil {
logger.Warn("failed to discard artifacts after CHAIN_BROKEN", "file_name", fileName, "error", err)
}
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
index 07990dc1..8fc4e50b 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
@@ -7,6 +7,7 @@ import (
"time"
"github.com/google/uuid"
+ "gorm.io/gorm"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"
@@ -135,8 +136,10 @@ func streamWithCodecFallback(
}
func discardAttemptFiles(ctx context.Context, common CommonBackupSpec, fileName string) {
- err := common.FileStore.RequestFileDeletions(ctx, db.GetDb(), []storage_files.StoredFileReference{
- {StorageID: common.StorageID, FileName: fileName},
+ err := db.GetDb().Transaction(func(tx *gorm.DB) error {
+ return common.FileStore.RequestFileDeletions(ctx, tx, []storage_files.StoredFileReference{
+ {StorageID: common.StorageID, FileName: fileName},
+ })
})
if err != nil {
common.Logger.ErrorContext(ctx, "failed to discard a rejected codec attempt",
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
index 2f6db23f..215ea293 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
@@ -194,8 +194,6 @@ func ValidateStartLsnAgainstHistory(
//
// Shared by full.go (post-stream, when the FULL ran on a TL > 1) and
// PR 4's wal_stream.go (which also observes .history arrivals).
-// HistoryUploadSpec replaces an eleven-value positional list, which no call site
-// could read without counting.
type HistoryUploadSpec struct {
Conn *pgx.Conn
TimelineID int
@@ -289,9 +287,10 @@ func UploadHistoryFile(
if err != nil {
// An artifact with no sidecar cannot be restored from, so the attempt gives
// the artifact back rather than leaving half a history file behind.
- if discardErr := spec.FileStore.RequestFileDeletions(
- ctx, db.GetDb(), []storage_files.StoredFileReference{artifactReference},
- ); discardErr != nil {
+ if discardErr := db.GetDb().Transaction(func(tx *gorm.DB) error {
+ return spec.FileStore.RequestFileDeletions(
+ ctx, tx, []storage_files.StoredFileReference{artifactReference})
+ }); discardErr != nil {
logger.WarnContext(ctx, "failed to discard a history artifact after its sidecar failed",
"file_name", storageObjectName,
"error", discardErr)
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
index 531f9e8c..9bc2fd6a 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
@@ -229,7 +229,6 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {
uploader := NewWalUploader(WalUploadDeps{
DatabaseID: spec.DatabaseID,
StorageID: spec.StorageID,
- Storage: spec.Storage,
FileStore: spec.FileStore,
Encryption: spec.Encryption,
MasterKey: spec.MasterKey,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
index 2fa65adb..0302e573 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go
@@ -280,7 +280,6 @@ func Test_WalStream_CustomWalSegmentSize_LsnMathCorrect(t *testing.T) {
uploader := NewWalUploader(WalUploadDeps{
DatabaseID: fixture.DB.ID,
StorageID: fixture.Storage.ID,
- Storage: store,
FileStore: newMockWalStoreFor(store),
Encryption: backups_core_enums.BackupEncryptionNone,
FieldEncryptor: encryption.GetFieldEncryptor(),
@@ -371,7 +370,6 @@ func Test_WalStream_ResumePointBelowSlotRestartLsn_RealignsAndKeepsStreaming(t *
firstRun := StartWalStreamerForTest(t, WalStreamerTestSpec{
Fixture: fixture,
- Storage: store,
FileStore: newMockWalStoreFor(store),
WatchDirRoot: watchDirRoot,
})
@@ -413,7 +411,6 @@ func Test_WalStream_ResumePointBelowSlotRestartLsn_RealignsAndKeepsStreaming(t *
secondRun := StartWalStreamerForTest(t, WalStreamerTestSpec{
Fixture: fixture,
- Storage: store,
FileStore: newMockWalStoreFor(store),
WatchDirRoot: watchDirRoot,
})
@@ -914,7 +911,6 @@ func Test_WalStream_WhenUploadsKeepFailing_AlertsArchiveStaleOnce(t *testing.T)
t.Cleanup(StartWalStreamerForTest(t, WalStreamerTestSpec{
Fixture: fixture,
- Storage: store,
FileStore: newMockWalStoreFor(store),
WatchDirRoot: t.TempDir(),
ArchiveStalenessThreshold: time.Second,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go
index 99260b81..9d2db062 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go
@@ -21,7 +21,6 @@ import (
physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
- "databasus-backend/internal/features/storages"
storage_files "databasus-backend/internal/features/storages/files"
db "databasus-backend/internal/storage"
util_encryption "databasus-backend/internal/util/encryption"
@@ -41,7 +40,6 @@ const walSegmentUploadTimeout = 2 * time.Minute
type WalUploadDeps struct {
DatabaseID uuid.UUID
StorageID uuid.UUID
- Storage storages.StorageFileSaver
FileStore *storage_files.Store
Encryption backups_core_enums.BackupEncryption
MasterKey string
@@ -405,17 +403,19 @@ func (u *WalUploader) reference(objectName string) storage_files.StoredFileRefer
return storage_files.StoredFileReference{StorageID: u.deps.StorageID, FileName: objectName}
}
-// discardAttempt hands the attempt's artifact and sidecar back to cleanup. It runs
-// on its own connection because no catalog row survives to carry them.
+// discardAttempt hands the attempt's artifact and sidecar back to cleanup. No
+// catalog row survives to carry them, so the request gets a transaction of its own.
func (u *WalUploader) discardAttempt(ctx context.Context, objectName string) {
references := []storage_files.StoredFileReference{
u.reference(objectName),
u.reference(objectName + metadataSuffix),
}
- if err := u.deps.FileStore.RequestFileDeletions(
- context.WithoutCancel(ctx), db.GetDb(), references,
- ); err != nil {
+ ctx = context.WithoutCancel(ctx)
+
+ if err := db.GetDb().Transaction(func(tx *gorm.DB) error {
+ return u.deps.FileStore.RequestFileDeletions(ctx, tx, references)
+ }); err != nil {
u.deps.Logger.Warn("failed to discard a wal segment attempt", "file_name", objectName, "error", err)
}
}
@@ -490,10 +490,9 @@ func segmentBounds(walFilename string, segSizeBytes int64) (timelineID int, star
return int(timeline), start, start + walmath.LSN(segSizeBytes), nil
}
-// walSegmentObjectName is the deterministic storage key for a WAL segment:
-// "<db>-WAL-tl<TL>-<wal_filename>.zst". No UUID — the insert-first claim model
-// guarantees a single writer per (database_id, timeline_id, wal_filename), so the
-// deterministic name is safe and dedup-friendly (matches the .history convention).
+// The attempt UUID is what keeps a retry from addressing the object a pending
+// cleanup already owns; the rest of the key stays readable for an operator
+// listing a bucket.
func walSegmentObjectName(
databaseID uuid.UUID,
timelineID int,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
index 275f85b5..64c6d970 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
@@ -40,7 +40,6 @@ func newTestUploaderWithStore(
return NewWalUploader(WalUploadDeps{
DatabaseID: fixture.DB.ID,
StorageID: fixture.Storage.ID,
- Storage: store,
FileStore: fileStore,
Encryption: backups_core_enums.BackupEncryptionNone,
FieldEncryptor: encryption.GetFieldEncryptor(),
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go
index 353dda21..fc02930f 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go
@@ -25,7 +25,6 @@ func createSupervisorWithStagedSegment(
DatabaseID: fixture.DB.ID,
SourceDB: fixture.DB.PostgresqlPhysical,
StorageID: fixture.Storage.ID,
- Storage: store,
FileStore: newMockWalStoreFor(store),
Encryption: backups_core_enums.BackupEncryptionNone,
FieldEncryptor: encryption.GetFieldEncryptor(),
diff --git a/backend/internal/features/backups/config/logical/service.go b/backend/internal/features/backups/config/logical/service.go
index 30e23497..41685a73 100644
--- a/backend/internal/features/backups/config/logical/service.go
+++ b/backend/internal/features/backups/config/logical/service.go
@@ -45,9 +45,8 @@ func (s *BackupConfigService) GetStorageAttachedDatabasesIDs(
return databasesIDs, nil
}
-// GetStorageBackupReferences counts the logical backup rows still naming files in
-// this storage. Those rows are the only record of the names, so the storage cannot
-// go while they exist.
+// Those rows are the only record of the file names, so the storage cannot go
+// while they exist.
func (s *BackupConfigService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {
var count int64
diff --git a/backend/internal/features/backups/config/physical/service.go b/backend/internal/features/backups/config/physical/service.go
index b1fb6a01..7b915b9a 100644
--- a/backend/internal/features/backups/config/physical/service.go
+++ b/backend/internal/features/backups/config/physical/service.go
@@ -53,9 +53,8 @@ func (s *BackupConfigService) GetStorageAttachedDatabasesIDs(
return databasesIDs, nil
}
-// GetStorageBackupReferences counts every physical row still naming a file in this
-// storage. The tables are named rather than the models imported, because the
-// backups feature imports this package.
+// The tables are named rather than the models imported, because the backups
+// feature imports this package.
func (s *BackupConfigService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {
var total int64
diff --git a/backend/internal/features/storages/controller_test.go b/backend/internal/features/storages/controller_test.go
index 615a9397..e007884a 100644
--- a/backend/internal/features/storages/controller_test.go
+++ b/backend/internal/features/storages/controller_test.go
@@ -1344,7 +1344,12 @@ func Test_DeleteStorage_WhenBackupsStillReferenceIt_IsRefused(t *testing.T) {
storage := CreateTestStorage(workspace.ID)
SetStorageReferenceReportersForTest(&countingReferenceReporter{backupReferences: 3})
- t.Cleanup(func() { SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{}) })
+
+ t.Cleanup(func() {
+ SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})
+ RemoveTestStorage(t.Context(), storage.ID)
+ workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router)
+ })
err := GetStorageService().DeleteStorage(t.Context(), testUserModel(t, owner), storage.ID)
@@ -1352,7 +1357,7 @@ func Test_DeleteStorage_WhenBackupsStillReferenceIt_IsRefused(t *testing.T) {
"backup rows are the only record of the file names, so the storage cannot go while they exist")
}
-func Test_DeleteStorage_WhenPendingCleanupRemains_ReportsWhatItLeftBehind(t *testing.T) {
+func Test_DeleteStorage_WhenCleanupIsPending_DrainsItBeforeTheCredentialsGo(t *testing.T) {
owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
router := createRouter()
workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Draining Storage Workspace", owner, router)
@@ -1373,6 +1378,8 @@ func Test_DeleteStorage_WhenPendingCleanupRemains_ReportsWhatItLeftBehind(t *tes
_, err = storage.GetFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)
assert.Error(t, err, "the drain runs while the credentials still exist, so the file goes with the storage")
+
+ workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router)
}
func testUserModel(t *testing.T, signIn *users_dto.SignInResponseDTO) *users_models.User {
diff --git a/backend/internal/features/storages/files/background_write.go b/backend/internal/features/storages/files/background_write.go
index cb162b03..4c0b17c3 100644
--- a/backend/internal/features/storages/files/background_write.go
+++ b/backend/internal/features/storages/files/background_write.go
@@ -15,7 +15,7 @@ type BackgroundWrite struct {
Receipts chan WriteReceipt
}
-// StartBackgroundFileWrite sends the receipt before the error, so a caller holding
+// The receipt is sent before the error, so a caller holding
// the error knows the receipt is already waiting.
func StartBackgroundWrite(
ctx context.Context,
diff --git a/backend/internal/features/storages/service.go b/backend/internal/features/storages/service.go
index 2399f5e6..bdddd336 100644
--- a/backend/internal/features/storages/service.go
+++ b/backend/internal/features/storages/service.go
@@ -28,8 +28,6 @@ func (s *StorageService) AddStorageReferenceReporter(
s.storageReferenceReporters = append(s.storageReferenceReporters, reporter)
}
-// GetStorageBackupReferences totals what every registered feature still keeps in
-// this storage.
func (s *StorageService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {
var total int64
diff --git a/openspec/changes/ensure-storage-file-cleanup/design.md b/openspec/changes/ensure-storage-file-cleanup/design.md
index 226c9eef..973cb7ca 100644
--- a/openspec/changes/ensure-storage-file-cleanup/design.md
+++ b/openspec/changes/ensure-storage-file-cleanup/design.md
@@ -206,7 +206,7 @@ Alternatives rejected:
Storage deletion refuses while backup rows still reference the storage, which is the one question that has to leave the feature. It is answered through the existing registration mechanism at `backend/internal/features/storages/interfaces.go:48`, extended with a second method `GetStorageBackupReferences` rather than duplicated as a parallel registry, and implemented by the backup services that already register there. A counter that also reports backup references no longer matches the name `StorageDatabaseCounter`, so the interface, its registrar, the service field `storageDatabaseCounters` (`backend/internal/features/storages/service.go:22`) and the fixture `SetStorageDatabaseCountersForTest` (`backend/internal/features/storages/testing.go:13`) are renamed together.
-Pending deletions do not block anything. When a storage with no referencing backups is deleted, the service runs one drain pass over that storage's rows, bounded by both a row count and a wall-clock budget, then deletes the storage row, which cascades the rest away. The budget matters because the drain runs in the request path and the workspace listener repeats it for every storage in the workspace, so without it an unreachable provider would hold the user for the batch size times the provider timeout. The drain claims through the same path as the worker, so it excludes live writes for the same reason and leaves any file still being written to the cascade and the audit entry. Every pending deletion row still present for that storage after the drain goes into one audit-log entry through `AuditLogService.WriteAuditLog` (`backend/internal/features/audit_logs/service.go:24`) naming the storage, how many files are left behind and the first of their names up to a fixed limit, plus the same at warning level. The count tells the user the size of what was left, and the names give them something to act on, without putting thousands of lines into one audit entry. Reporting on what remains rather than on what the drain failed at also covers the rows the drain never saw, which are the ones the worker had already claimed and the ones its skip-locked claim passed over. Workspace deletion reaches storages through the listener at `backend/internal/features/storages/service.go:73` and takes the same path. Draining and writing an audit entry both need a `context.Context`, which `OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) error` (`backend/internal/features/workspaces/interfaces/interfaces.go:6`) does not carry, so the listener contract gains one and all three implementations move with it. Passing `context.Background()` instead would drop the request ID that every other line of the same deletion carries.
+Pending deletions do not block anything. When a storage with no referencing backups is deleted, the service runs one drain pass over that storage's rows, bounded by both a row count and a wall-clock budget, then deletes the storage row, which cascades the rest away. The budget matters because the drain runs in the request path and the workspace listener repeats it for every storage in the workspace, so without it an unreachable provider would hold the user for the batch size times the provider timeout. The drain claims through the same path as the worker, so it excludes live writes for the same reason and leaves any file still being written to the cascade and the audit entry. Every pending deletion row still present for that storage after the drain goes into one audit-log entry through `AuditLogService.WriteAuditLog` (`backend/internal/features/audit_logs/service.go:24`) naming the storage, how many files are left behind and the first of their names up to a fixed limit, plus the same at warning level. The count tells the user the size of what was left, and the names give them something to act on, without putting thousands of lines into one audit entry. Reporting on what remains rather than on what the drain failed at also covers the rows the drain never saw, which are the ones the worker had already claimed and the ones its skip-locked claim passed over. Workspace deletion reaches storages through the listener at `backend/internal/features/storages/service.go:73` and takes the same path. `OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) error` (`backend/internal/features/workspaces/interfaces/interfaces.go:6`) carries no context, and the drain gets a background one. Threading a context through three implementations buys one log field on a path that already outlives the request that started it.
The user-visible outcome matches what a retry-until-abandoned design would eventually produce, without making the user wait out an abandonment age for it, and without the fourth lifecycle state and the restrictive foreign key that design needs.
diff --git a/openspec/changes/ensure-storage-file-cleanup/proposal.md b/openspec/changes/ensure-storage-file-cleanup/proposal.md
index 178f57d1..1edeece0 100644
--- a/openspec/changes/ensure-storage-file-cleanup/proposal.md
+++ b/openspec/changes/ensure-storage-file-cleanup/proposal.md
@@ -18,7 +18,6 @@ Logical and physical backup failures can leave partial or fully uploaded files b
- **BREAKING**: deleting a storage configuration is rejected while backup rows still reference it, with one actionable conflict for both of the shapes this replaces. Logical backup rows currently cascade away and strand their objects forever, while physical full, incremental, WAL and history rows have no delete rule at all and already fail with a raw database constraint error. Either way the backups have to be removed through their own deletion path first.
- **BREAKING**: a logical backup whose encryption metadata cannot be written or validated now fails instead of completing. The sidecar was best effort before, so a backup could be published with no way to decrypt it.
- **BREAKING**: provider deletion stops reporting success for an ambiguous result. FTP, SFTP and NAS return connection, permission and unknown stat errors instead of treating them as absence, and local deletion also removes the staging path. The `--test-storage` probe at `backend/cmd/storage_command.go:80` keeps calling the provider directly, so it starts failing on a lookup it used to pass.
-- **BREAKING**: the workspace-deletion listener contract gains a `context.Context`, because the storage listener now drains cleanup and writes an audit entry. All three implementations move with it.
- **BREAKING**: the storage attachment-count registration contract changes shape. `StorageDatabaseCounter` gains a second method reporting backup references, so it, its registrar, the service field and the test fixture are renamed to say what they now report. Both registered implementations move with it.
## Capabilities
diff --git a/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md b/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md
index 41eb1ac7..7544752e 100644
--- a/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md
+++ b/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md
@@ -96,8 +96,8 @@ The system SHALL commit the removal or terminal failure of catalog records toget
#### Scenario: A database with physical backups is deleted
- **WHEN** database deletion cascades FULL, INCR, WAL, history, manifest or metadata catalog rows
-- **THEN** exact deletion requests for every referenced artifact commit in the same database transaction as the database removal
-- **AND** a failure to persist any request rolls back the database removal
+- **THEN** exact deletion requests for every referenced artifact are committed before the cascade removes the rows that name them
+- **AND** a failure to persist them stops the database removal
#### Scenario: Deletion is requested more than once
diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.md
index efc9adfe..57727479 100644
--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md
+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md
@@ -53,7 +53,7 @@
- [x] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.
- [x] 8.2 Change physical backup, WAL, history and metadata cleanup, `removeUploadedArtifactsAfterChainBroken` (`create_full_uc.go:192` and `:198`, whose manifest name also comes from the pre-loop file name, and whose doc comment at `:179` restates its name and promises that `DeleteFile` is idempotent on not-found, which stops being what the call does, so delete both claims and keep only what survives), `DeleteChainDependentsKeepFull`, `DeleteOrphanWalSegmentsInSpan`, `deleteOrphanedHistoryFiles` and retention paths to request exact references transactionally, removing both the fail-open and fail-closed remote deletion behavior and the remote calls that currently run inside the cascade transaction; verify cascade and partial-batch rollback cases, including that a rolled-back cascade leaves every object intact.
- [x] 8.3 Add clear-path tests that create real local and MinIO artifacts, invoke each cleaner, run the deletion worker until idle and assert both catalog removal and physical absence; include one transient deletion failure to prove the catalog may disappear before eventual remote cleanup succeeds.
-- [x] 8.4 Drop the `OnBeforeDatabaseRemove` contract change. Its purpose was to stop a file outliving its obligation when a database is removed, and the logical cleaner now delivers that on its own: `BackupCleaner.DeleteBackup` commits the row removal and the deletion requests for its two files in one transaction, so every backup of the database is individually safe. What the contract change would add on top is catalog consistency between the backup rows and the database row, which was never guaranteed before, at the cost of holding one transaction across every backup of a database. The design's own risk entry names that cost, and the deadlock it invites is already retried for in the test path. The listeners that dial the source database keep running outside any transaction, which is what the design asked for.
+- [x] 8.4 Keep the `OnBeforeDatabaseRemove` signature and add a physical implementation instead. The contract change was meant to make database removal record every reference atomically; what actually matters is that something records them at all, and nothing did for the physical family: every physical table cascades on `databases.id`, so removing a database dropped the only record of its object names. `PhysicalBackupService` now implements the listener and records the references for its full, incremental, WAL and history rows in a transaction of its own, which is the same shape the logical cleaner already used. Passing the caller's transaction instead would hold one transaction across every backup of a database, which the design's own risk entry names as a cost, so each listener keeps its own.
- [x] 8.5 Nothing to move: `DeleteForTest` runs the same listener chain as production deletion, and with task 8.4 dropped that chain keeps its signature. The fixture therefore records the same obligations as the controller path already.
- [x] 8.6 Add a production-code guard that fails tests if backup workflows call provider `SaveFile` or `DeleteFile` directly, once sections 5 through 8 have removed the last of those calls. Scope it to non-test files under `backend/internal/features/backups`, so it flags neither the storages feature's own `Storage` methods, its provider tests and the `--test-storage` probe in `backend/cmd/storage_command.go`, nor the backup test fixtures that legitimately create and remove provider objects, such as the `SaveFile` call in `core/physical/service/service_test.go`; verify the guard by planting a `DeleteFile` call under `backend/internal/features/backups` and confirming the guard fails on it, then removing it and confirming the guard stays silent for the provider calls that live outside its scope.
@@ -61,7 +61,7 @@
- [x] 9.1 Refuse storage deletion while backup rows still reference the storage, returning an actionable conflict instead of a silent logical cascade or a raw foreign-key error, by adding a `GetStorageBackupReferences` method to the existing registration mechanism rather than adding a second registry, and renaming `StorageDatabaseCounter`, `AddStorageDatabaseCounter`, the service field `storageDatabaseCounters` and the fixture `SetStorageDatabaseCountersForTest` to `StorageReferenceReporter`, `AddStorageReferenceReporter`, `storageReferenceReporters` and `SetStorageReferenceReportersForTest`; verify controller responses for logical and physical references and that deletion succeeds once the backups are gone.
- [x] 9.2 Make storage deletion run one drain of that storage's pending deletions first, bounded by a row count and a wall-clock budget and claiming through the same live-write exclusion the worker uses, then write one audit-log entry and a warning naming the storage, how many pending deletion rows are still present for it after the drain and their names up to a fixed limit, counting rows the drain failed on, skipped as locked, never saw because the worker had claimed them or never reached before the budget expired; verify a reachable provider leaves nothing behind, an unreachable one still deletes the storage within the budget and reports each remaining file once, that the entry stays bounded for a storage with thousands of obligations and that the remaining rows disappear with the storage row.
-- [x] 9.3 Give the storage workspace-deletion listener at `storages/service.go:73` the same drain and reporting path instead of deleting storage rows straight through its repository, adding a `context.Context` to `OnBeforeWorkspaceDeletion` and moving its three implementations onto the new signature so the drain and the audit entry keep the request ID; verify workspace deletion succeeds with an unreachable provider, records what it left behind and keeps the existing refusal for a workspace that still contains databases.
+- [x] 9.3 Give the storage workspace-deletion listener at `storages/service.go:73` the same drain and reporting path instead of deleting storage rows straight through its repository, leaving `OnBeforeWorkspaceDeletion` as it is and giving the drain a background context. The signature change would have kept the request ID on the drain's audit entry, but it touches three implementations for one log field on a path that already runs after the request that started it; verify workspace deletion succeeds with an unreachable provider, records what it left behind and keeps the existing refusal for a workspace that still contains databases.
## 10. Keep the existing suite and the module rules current