Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

REFACTOR (backups): Let each backup feature count its own storage references

Rostislav Duginec9bce5de85d
diff --git a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go b/backend/internal/features/backups/backups/backuping/logical/backuper_test.go
index 34311c27..393b0655 100644
--- a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go
+++ b/backend/internal/features/backups/backups/backuping/logical/backuper_test.go
@@ -414,6 +414,8 @@ func Test_MakeBackup_WhenPublicationFails_EveryFileOfTheAttemptIsRemoved(t *test
assert.Equal(t, backups_core_logical.BackupStatusFailed, persistedBackup.Status,
"a backup whose files cannot be claimed must not be published")
+ // The caller names the files its scenario actually produced: asserting the
+ // absence of a file nothing ever wrote would pass with no cleanup at all.
assertBackupFilesRemoved(t, fixture, backup.FileName, backup.FileName+metadataSuffix)
}
diff --git a/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go
index 01e66644..46399dd2 100644
--- a/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go
+++ b/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go
@@ -612,8 +612,8 @@ func createTestInterval() intervals.Interval {
}
}
-// The point of the whole change: the catalog row goes even when the provider is
-// refusing, and the file still disappears once it stops refusing.
+// The catalog row goes while the provider is refusing, and the file follows once
+// it stops.
func Test_DeleteBackup_WhenProviderRefusesThenRecovers_RowGoesFirstAndFileFollows(t *testing.T) {
fixture := CreateBackupTestFixture(t, "Flaky Cleanup Workspace")
backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)
diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.go
index e45138fc..b07c2f90 100644
--- a/backend/internal/features/backups/backups/backuping/physical/di.go
+++ b/backend/internal/features/backups/backups/backuping/physical/di.go
@@ -125,10 +125,10 @@ var SetupDependencies = sync.OnceFunc(func() {
// can drop the (now detaching) WAL slot instead of refusing it as active and
// leaving it to pin WAL forever.
databases.GetDatabaseService().AddDbRemoveListener(physicalBackupCancellationListener)
- // Registered before the cancellation listener's siblings finish, but ordering
- // does not matter: the cascade that would take the names away runs only after
- // every listener has returned.
+ // The cascade that would take the object names away runs only after every
+ // listener has returned.
databases.GetDatabaseService().AddDbRemoveListener(physical_service.GetPhysicalBackupService())
+ storages.GetStorageService().AddStorageBackupCounter(physical_service.GetPhysicalBackupService())
databases.GetDatabaseService().AddDbRemoveListener(physicalSlotCleanupListener)
backups_config_physical.GetBackupConfigService().SetBackupCancellationListener(physicalBackupCancellationListener)
})
diff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler.go b/backend/internal/features/backups/backups/backuping/physical/scheduler.go
index 4ff81db2..b3eec17f 100644
--- a/backend/internal/features/backups/backups/backuping/physical/scheduler.go
+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler.go
@@ -653,12 +653,12 @@ func (s *PhysicalBackupsScheduler) failClaimlessInProgressBackups(
"backup_id", full.ID, "database_id", full.DatabaseID)
s.failOrphanedBackup(ctx, logger, orphanedBackupSpec{
- Kind: physical_enums.PhysicalBackupTypeFull,
- BackupID: full.ID,
- DatabaseID: full.DatabaseID,
- StorageID: full.StorageID,
- FileName: valueOrEmpty(full.FileName),
- Manifest: valueOrEmpty(full.ManifestFileName),
+ Kind: physical_enums.PhysicalBackupTypeFull,
+ BackupID: full.ID,
+ DatabaseID: full.DatabaseID,
+ StorageID: full.StorageID,
+ FileName: valueOrEmpty(full.FileName),
+ ManifestFileName: valueOrEmpty(full.ManifestFileName),
})
}
@@ -676,20 +676,20 @@ func (s *PhysicalBackupsScheduler) failClaimlessInProgressBackups(
"backup_id", incremental.ID, "database_id", incremental.DatabaseID)
s.failOrphanedBackup(ctx, logger, orphanedBackupSpec{
- Kind: physical_enums.PhysicalBackupTypeIncremental,
- BackupID: incremental.ID,
- DatabaseID: incremental.DatabaseID,
- StorageID: incremental.StorageID,
- FileName: valueOrEmpty(incremental.FileName),
- Manifest: valueOrEmpty(incremental.ManifestFileName),
+ Kind: physical_enums.PhysicalBackupTypeIncremental,
+ BackupID: incremental.ID,
+ DatabaseID: incremental.DatabaseID,
+ StorageID: incremental.StorageID,
+ FileName: valueOrEmpty(incremental.FileName),
+ ManifestFileName: valueOrEmpty(incremental.ManifestFileName),
})
}
return nil
}
-// describeOrphan reads the file the claimed row already names. A claim carries no
-// file name of its own, and the sweep cannot hand back what it cannot name.
+// A claim carries no file name of its own, and the sweep cannot hand back what it
+// cannot name.
func (s *PhysicalBackupsScheduler) describeOrphan(
logger *slog.Logger,
claim *physical_models.PhysicalInFlightBackup,
@@ -710,7 +710,7 @@ func (s *PhysicalBackupsScheduler) describeOrphan(
spec.StorageID = row.StorageID
spec.FileName = valueOrEmpty(row.FileName)
- spec.Manifest = valueOrEmpty(row.ManifestFileName)
+ spec.ManifestFileName = valueOrEmpty(row.ManifestFileName)
return spec
}
@@ -724,21 +724,20 @@ func (s *PhysicalBackupsScheduler) describeOrphan(
spec.StorageID = row.StorageID
spec.FileName = valueOrEmpty(row.FileName)
- spec.Manifest = valueOrEmpty(row.ManifestFileName)
+ spec.ManifestFileName = valueOrEmpty(row.ManifestFileName)
return spec
}
-// orphanedBackupSpec is one IN_PROGRESS row the previous run left behind. It
-// carries the file name because the row now holds one from upload start, so the
-// sweep can hand that file back instead of assuming none exists.
+// orphanedBackupSpec is one IN_PROGRESS row the previous run left behind, with
+// the names its files carry so the sweep can hand them back.
type orphanedBackupSpec struct {
- Kind physical_enums.PhysicalBackupType
- BackupID uuid.UUID
- DatabaseID uuid.UUID
- StorageID uuid.UUID
- FileName string
- Manifest string
+ Kind physical_enums.PhysicalBackupType
+ BackupID uuid.UUID
+ DatabaseID uuid.UUID
+ StorageID uuid.UUID
+ FileName string
+ ManifestFileName string
}
func (s *PhysicalBackupsScheduler) failOrphanedBackup(
@@ -772,8 +771,8 @@ func (s *PhysicalBackupsScheduler) failOrphanedBackup(
// essential: an orphan claim left behind would block every future tick from
// acquiring the cross-table single-in-flight slot for that DB, freezing it.
//
-// The row carries a file name from upload start, so the sweep hands that file
-// back to cleanup in the same transaction, gated on the status flip taking effect.
+// The sweep hands the row's files back to cleanup in the same transaction, gated
+// on the status flip taking effect.
func (s *PhysicalBackupsScheduler) failBackupAndReleaseClaim(
ctx context.Context,
spec orphanedBackupSpec,
@@ -808,9 +807,9 @@ func (s *PhysicalBackupsScheduler) failBackupAndReleaseClaim(
{StorageID: spec.StorageID, FileName: spec.FileName + metadataSuffix},
}
- if spec.Manifest != "" {
+ if spec.ManifestFileName != "" {
references = append(references,
- storage_files.StoredFileReference{StorageID: spec.StorageID, FileName: spec.Manifest})
+ storage_files.StoredFileReference{StorageID: spec.StorageID, FileName: spec.ManifestFileName})
}
if err := s.fileStore.RequestFileDeletions(ctx, tx, references); err != nil {
diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
index b35025e3..7d7b6835 100644
--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go
@@ -276,7 +276,6 @@ func (s *PhysicalWalStreamSupervisor) startStreamer(
SourceDB: tunneledDatabase.GetDatabaseThroughTunnel(),
IsBastionReachable: tunneledDatabase.IsBastionReachable,
StorageID: storage.ID,
- Storage: storage,
FileStore: s.fileStore,
Encryption: backupConfig.Encryption,
MasterKey: masterKey,
diff --git a/backend/internal/features/backups/backups/core/logical/repository.go b/backend/internal/features/backups/backups/core/logical/repository.go
index 8fb6ab93..6880c502 100644
--- a/backend/internal/features/backups/backups/core/logical/repository.go
+++ b/backend/internal/features/backups/backups/core/logical/repository.go
@@ -2,6 +2,7 @@ package backups_core_logical
import (
"errors"
+ "fmt"
"time"
"github.com/google/uuid"
@@ -197,6 +198,19 @@ func (r *BackupRepository) FindByDatabaseIdAndStatus(
return backups, nil
}
+func (r *BackupRepository) CountByStorageID(storageID uuid.UUID) (int64, error) {
+ var count int64
+
+ if err := storage.GetDb().
+ Model(&LogicalBackup{}).
+ Where("storage_id = ?", storageID).
+ Count(&count).Error; err != nil {
+ return 0, fmt.Errorf("count backups of a storage: %w", err)
+ }
+
+ return count, nil
+}
+
func (r *BackupRepository) DeleteByID(id uuid.UUID) error {
return r.DeleteByIDInTransaction(storage.GetDb(), id)
}
diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
index 98e165bd..5644748c 100644
--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go
@@ -119,9 +119,9 @@ func (r *PhysicalWalSegmentRepository) DeleteByID(id uuid.UUID) error {
return storage.GetDb().Delete(&physical_models.PhysicalWalSegment{}, "id = ?", id).Error
}
-// A NULL file_name no longer proves the upload wrote nothing: the object is
-// written before the row names it. The bytes such a claim may have left are owned
-// by their own pending deletion, so this only removes the aged-out claim row.
+// A NULL file_name does not prove the upload wrote nothing: the object is written
+// before the row names it. The bytes such a claim may have left are owned by their
+// own pending deletion, so this only removes the aged-out claim row.
func (r *PhysicalWalSegmentRepository) DeleteAbandonedClaims(
databaseID uuid.UUID,
olderThan time.Time,
@@ -204,30 +204,30 @@ func (r *PhysicalWalSegmentRepository) FindByChainKey(
// makes this a no-op. updated=true means the upload is durably committed;
// updated=false means the NULL claim no longer exists (cascade caught it) and the
// caller must DeleteFile the now-orphaned storage object.
-func (r *PhysicalWalSegmentRepository) MarkUploaded(
- id uuid.UUID,
- fileName string,
- compressedSizeMb float64,
- encryptionSalt, encryptionIV *string,
-) (updated bool, err error) {
- return r.MarkUploadedInTransaction(storage.GetDb(), id, fileName, compressedSizeMb, encryptionSalt, encryptionIV)
+type WalSegmentUpload struct {
+ SegmentID uuid.UUID
+ FileName string
+ CompressedSizeMb float64
+ EncryptionSalt *string
+ EncryptionIV *string
+}
+
+func (r *PhysicalWalSegmentRepository) MarkUploaded(upload WalSegmentUpload) (updated bool, err error) {
+ return r.MarkUploadedInTransaction(storage.GetDb(), upload)
}
func (r *PhysicalWalSegmentRepository) MarkUploadedInTransaction(
tx *gorm.DB,
- id uuid.UUID,
- fileName string,
- compressedSizeMb float64,
- encryptionSalt, encryptionIV *string,
+ upload WalSegmentUpload,
) (updated bool, err error) {
result := tx.
Model(&physical_models.PhysicalWalSegment{}).
- Where("id = ? AND file_name IS NULL", id).
+ Where("id = ? AND file_name IS NULL", upload.SegmentID).
Updates(map[string]any{
- "file_name": fileName,
- "compressed_size_mb": compressedSizeMb,
- "encryption_salt": encryptionSalt,
- "encryption_iv": encryptionIV,
+ "file_name": upload.FileName,
+ "compressed_size_mb": upload.CompressedSizeMb,
+ "encryption_salt": upload.EncryptionSalt,
+ "encryption_iv": upload.EncryptionIV,
})
if result.Error != nil {
return false, result.Error
diff --git a/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go b/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go
index 6fc70d05..ee4d5f47 100644
--- a/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go
+++ b/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go
@@ -26,6 +26,29 @@ func (s *PhysicalBackupService) OnBeforeDatabaseRemove(ctx context.Context, data
})
}
+// The physical rows are the only record of the file names in a storage, so the
+// storage cannot go while they exist.
+func (s *PhysicalBackupService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {
+ var total int64
+
+ for _, model := range []any{
+ &physical_models.PhysicalFullBackup{},
+ &physical_models.PhysicalIncrementalBackup{},
+ &physical_models.PhysicalWalSegment{},
+ &physical_models.PhysicalWalHistoryFile{},
+ } {
+ var count int64
+
+ if err := db.GetDb().Model(model).Where("storage_id = ?", storageID).Count(&count).Error; err != nil {
+ return 0, fmt.Errorf("count physical rows of a storage: %w", err)
+ }
+
+ total += count
+ }
+
+ return total, nil
+}
+
func (s *PhysicalBackupService) collectDatabaseFileReferences(
tx *gorm.DB,
databaseID uuid.UUID,
@@ -82,18 +105,15 @@ func artifactReferences(
storageID uuid.UUID,
fileName, manifestFileName string,
) []storage_files.StoredFileReference {
- if fileName == "" {
- return nil
- }
+ references := make([]storage_files.StoredFileReference, 0, 3)
- references := []storage_files.StoredFileReference{
- {StorageID: storageID, FileName: fileName},
- {StorageID: storageID, FileName: fileName + metadataSuffix},
- }
+ for _, name := range []string{fileName, fileName + metadataSuffix, manifestFileName} {
+ if name == "" || name == metadataSuffix {
+ continue
+ }
- if manifestFileName != "" {
references = append(references,
- storage_files.StoredFileReference{StorageID: storageID, FileName: manifestFileName})
+ storage_files.StoredFileReference{StorageID: storageID, FileName: name})
}
return references
diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.go
index f6dc1ea2..6f7fa922 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service.go
@@ -1054,8 +1054,8 @@ func (s *PhysicalBackupService) deleteFullArtifactAndRow(
// requestFileDeletions records the obligation to remove these files in the same
// transaction that removes the rows naming them. Nothing reaches the provider
-// here: a rolled back cascade leaves every object intact, and a storage outage
-// no longer decides whether a catalog row may go.
+// here, so a rolled back cascade leaves every object intact and a storage outage
+// does not decide whether a catalog row may go.
func (s *PhysicalBackupService) requestFileDeletions(
ctx context.Context,
tx *gorm.DB,
diff --git a/backend/internal/features/backups/backups/services/di.go b/backend/internal/features/backups/backups/services/di.go
index c534ac21..09dd3911 100644
--- a/backend/internal/features/backups/backups/services/di.go
+++ b/backend/internal/features/backups/backups/services/di.go
@@ -78,5 +78,6 @@ var SetupDependencies = sync.OnceFunc(func() {
SetDatabaseStorageChangeListener(backupService)
databases.GetDatabaseService().AddDbRemoveListener(backupService)
+ storages.GetStorageService().AddStorageBackupCounter(backupService)
databases.GetDatabaseService().AddDbCopyListener(backups_config_logical.GetBackupConfigService())
})
diff --git a/backend/internal/features/backups/backups/services/logical_service.go b/backend/internal/features/backups/backups/services/logical_service.go
index 256a91cc..a82467e8 100644
--- a/backend/internal/features/backups/backups/services/logical_service.go
+++ b/backend/internal/features/backups/backups/services/logical_service.go
@@ -81,6 +81,12 @@ func (s *LogicalBackupService) OnBeforeBackupsStorageChange(ctx context.Context,
return nil
}
+// The logical_backups rows are the only record of the file names in a storage, so
+// the storage cannot go while they exist.
+func (s *LogicalBackupService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {
+ return s.backupRepository.CountByStorageID(storageID)
+}
+
func (s *LogicalBackupService) OnBeforeDatabaseRemove(ctx context.Context, databaseID uuid.UUID) error {
err := s.deleteDbBackups(ctx, databaseID)
if err != nil {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
index 8fc4e50b..d7b1b9d9 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
@@ -41,21 +41,14 @@ func (s CommonBackupSpec) runStreamParams(
}
}
-// The ZSTD -> GZIP -> NONE fallback runs INSIDE the per-backup replication slot,
-// so the slot is held across every attempt; only the --compress flag, the recorded
-// codec and the object key differ between them. A rejected attempt leaves an object
-// of its own for cleanup rather than being overwritten.
-//
-// IncrementalManifestPath is "" for a FULL; a non-empty path is the downloaded
-// parent-manifest temp file that makes this an INCR (--incremental=<path>).
-// streamAttemptSpec carries what one backup needs to stream, so the codec loop
-// does not grow a positional list every time the caller learns something new.
type streamAttemptSpec struct {
- Common CommonBackupSpec
- BackupID uuid.UUID
- Creds *postgresql_shared.CredentialTempFiles
- Label string
- SystemID uint64
+ Common CommonBackupSpec
+ BackupID uuid.UUID
+ Creds *postgresql_shared.CredentialTempFiles
+ Label string
+ SystemID uint64
+ // "" for a FULL; a non-empty path is the downloaded parent-manifest temp file
+ // that makes this an INCR (--incremental=<path>).
IncrementalManifestPath string
Classify streamErrorClassifier
@@ -66,6 +59,10 @@ type streamAttemptSpec struct {
MintAndSaveAttemptName func() (string, error)
}
+// The ZSTD -> GZIP -> NONE fallback runs INSIDE the per-backup replication slot,
+// so the slot is held across every attempt; only the --compress flag, the recorded
+// codec and the object key differ between them. A rejected attempt leaves an object
+// of its own for cleanup rather than being overwritten.
func streamWithCodecFallback(
ctx context.Context,
spec streamAttemptSpec,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go
index 0ce01e2b..deadbc3e 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go
@@ -47,8 +47,8 @@ type mockWalStorage struct {
}
// hasObjectFor reports whether any stored object belongs to the given segment.
-// Per-attempt names mean a test can no longer rebuild the exact key, so it matches
-// on the segment identity the key is derived from.
+// A test cannot rebuild the exact key, which carries a per-attempt UUID, so it
+// matches on the segment identity the key derives from.
func (m *mockWalStorage) hasObjectFor(databaseID uuid.UUID, timelineID int, walFilename string) bool {
m.mu.Lock()
defer m.mu.Unlock()
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
index 4a4bfd0d..58d92bd3 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
@@ -116,8 +116,8 @@ func (f *fakeStorageFileSaver) DeleteFile(context.Context, encryption.FieldEncry
func (f *fakeStorageFileSaver) Validate(encryption.FieldEncryptor) error { return nil }
func (f *fakeStorageFileSaver) TestConnection(encryption.FieldEncryptor) error { return nil }
-// The physical stream writes through the file store now, and the fake keeps its
-// own recording so the deadline and payload assertions stay where they were.
+// The physical stream writes through the file store, and the fake records what it
+// receives so the deadline and payload assertions have something to read.
func (f *fakeStorageFileSaver) WriteFile(
ctx context.Context,
reference storage_files.StoredFileReference,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
index 13679f69..477f5bfe 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go
@@ -843,7 +843,6 @@ func StartWalStreamerForTest(t *testing.T, spec WalStreamerTestSpec) *WalStreame
DatabaseID: fixture.DB.ID,
SourceDB: fixture.DB.PostgresqlPhysical,
StorageID: fixture.Storage.ID,
- Storage: spec.Storage,
FileStore: spec.FileStore,
Encryption: backups_core_enums.BackupEncryptionNone,
FieldEncryptor: encryption.GetFieldEncryptor(),
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
index 215ea293..99f6f5a6 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go
@@ -186,14 +186,6 @@ func ValidateStartLsnAgainstHistory(
return chain_view.ValidationResult{Status: chain_view.ValidationStatusOK}, nil
}
-// UploadHistoryFile reads the .history file for timelineID from the source
-// cluster's pg_wal/, compresses with zstd, optionally encrypts, uploads
-// artifact + sidecar to storage, and inserts the physical_wal_history_files
-// row. Idempotent on (database_id, timeline_id) via the UNIQUE constraint:
-// a duplicate insert returns nil after observing the existing row.
-//
-// Shared by full.go (post-stream, when the FULL ran on a TL > 1) and
-// PR 4's wal_stream.go (which also observes .history arrivals).
type HistoryUploadSpec struct {
Conn *pgx.Conn
TimelineID int
@@ -207,6 +199,8 @@ type HistoryUploadSpec struct {
Logger *slog.Logger
}
+// Idempotent on (database_id, timeline_id) through the UNIQUE constraint: a
+// duplicate insert returns the existing row instead of an error.
func UploadHistoryFile(
ctx context.Context,
spec HistoryUploadSpec,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
index 9bc2fd6a..3b59ed1d 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go
@@ -15,7 +15,6 @@ import (
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
- "databasus-backend/internal/features/storages"
storage_files "databasus-backend/internal/features/storages/files"
util_encryption "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
@@ -137,7 +136,6 @@ type WalStreamSpec struct {
DatabaseID uuid.UUID
SourceDB *postgresql_physical.PostgresqlPhysicalDatabase
StorageID uuid.UUID
- Storage storages.StorageFileSaver
FileStore *storage_files.Store
Encryption backups_core_enums.BackupEncryption
MasterKey string
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go
index 9d2db062..6e9ca975 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go
@@ -403,8 +403,8 @@ func (u *WalUploader) reference(objectName string) storage_files.StoredFileRefer
return storage_files.StoredFileReference{StorageID: u.deps.StorageID, FileName: objectName}
}
-// discardAttempt hands the attempt's artifact and sidecar back to cleanup. No
-// catalog row survives to carry them, so the request gets a transaction of its own.
+// No catalog row survives to carry these files, so the request gets a transaction
+// of its own.
func (u *WalUploader) discardAttempt(ctx context.Context, objectName string) {
references := []storage_files.StoredFileReference{
u.reference(objectName),
@@ -435,9 +435,13 @@ func (u *WalUploader) commitSegment(ctx context.Context, spec commitSegmentSpec)
committed := false
err := db.GetDb().Transaction(func(tx *gorm.DB) error {
- updated, err := u.deps.WalSegmentRepo.MarkUploadedInTransaction(
- tx, spec.Claim.ID, spec.ObjectName, spec.CompressedSizeMb, nilIfEmpty(spec.Salt), nilIfEmpty(spec.IV),
- )
+ updated, err := u.deps.WalSegmentRepo.MarkUploadedInTransaction(tx, physical_repositories.WalSegmentUpload{
+ SegmentID: spec.Claim.ID,
+ FileName: spec.ObjectName,
+ CompressedSizeMb: spec.CompressedSizeMb,
+ EncryptionSalt: nilIfEmpty(spec.Salt),
+ EncryptionIV: nilIfEmpty(spec.IV),
+ })
if err != nil {
return err
}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
index 64c6d970..acacf918 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go
@@ -81,8 +81,8 @@ func Test_WalUpload_ClaimSucceeds_SaveFileLands_FileNameUpdatedFromNullToNonNull
require.NotNil(t, row)
require.NotNil(t, row.FileName, "file_name must be flipped from NULL to the object key")
- // The name carries a per-attempt UUID now, so the row is the only place that
- // knows it, which is where restore reads it from too.
+ // The name carries a per-attempt UUID, so the row is the only place that knows
+ // it, which is where restore reads it from too.
objectName := *row.FileName
require.True(t, store.hasObject(objectName), "artifact must be in storage under the name the row carries")
require.True(t, store.hasObject(objectName+metadataSuffix), "sidecar must be uploaded")
diff --git a/backend/internal/features/backups/config/logical/di.go b/backend/internal/features/backups/config/logical/di.go
index 16e8d083..b08f7aae 100644
--- a/backend/internal/features/backups/config/logical/di.go
+++ b/backend/internal/features/backups/config/logical/di.go
@@ -34,5 +34,5 @@ func GetBackupConfigService() *BackupConfigService {
}
var SetupDependencies = sync.OnceFunc(func() {
- storages.GetStorageService().AddStorageReferenceReporter(backupConfigService)
+ storages.GetStorageService().AddStorageDatabaseCounter(backupConfigService)
})
diff --git a/backend/internal/features/backups/config/logical/service.go b/backend/internal/features/backups/config/logical/service.go
index 41685a73..f4f1bbef 100644
--- a/backend/internal/features/backups/config/logical/service.go
+++ b/backend/internal/features/backups/config/logical/service.go
@@ -3,7 +3,6 @@ package backups_config_logical
import (
"context"
"errors"
- "fmt"
"github.com/google/uuid"
@@ -14,7 +13,6 @@ import (
"databasus-backend/internal/features/storages"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
- "databasus-backend/internal/storage"
"databasus-backend/internal/util/period"
)
@@ -45,24 +43,6 @@ func (s *BackupConfigService) GetStorageAttachedDatabasesIDs(
return databasesIDs, nil
}
-// Those rows are the only record of the file names, so the storage cannot go
-// while they exist.
-func (s *BackupConfigService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {
- var count int64
-
- // The table is named rather than the model imported: the backups feature imports
- // this package, so the dependency cannot run the other way.
- err := storage.GetDb().
- Table("logical_backups").
- Where("storage_id = ?", storageID).
- Count(&count).Error
- if err != nil {
- return 0, fmt.Errorf("count logical backups of a storage: %w", err)
- }
-
- return count, nil
-}
-
func (s *BackupConfigService) SaveBackupConfigWithAuth(
ctx context.Context,
user *users_models.User,
diff --git a/backend/internal/features/backups/config/physical/di.go b/backend/internal/features/backups/config/physical/di.go
index 6266bf4e..287ef6d2 100644
--- a/backend/internal/features/backups/config/physical/di.go
+++ b/backend/internal/features/backups/config/physical/di.go
@@ -37,6 +37,6 @@ func GetBackupConfigService() *BackupConfigService {
}
var SetupDependencies = sync.OnceFunc(func() {
- storages.GetStorageService().AddStorageReferenceReporter(backupConfigService)
+ storages.GetStorageService().AddStorageDatabaseCounter(backupConfigService)
databases.GetDatabaseService().AddDbBackupTypeChangeListener(backupConfigService)
})
diff --git a/backend/internal/features/backups/config/physical/service.go b/backend/internal/features/backups/config/physical/service.go
index 7b915b9a..0d5d1fe3 100644
--- a/backend/internal/features/backups/config/physical/service.go
+++ b/backend/internal/features/backups/config/physical/service.go
@@ -15,7 +15,6 @@ import (
"databasus-backend/internal/features/storages"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
- "databasus-backend/internal/storage"
)
type BackupConfigService struct {
@@ -53,32 +52,6 @@ func (s *BackupConfigService) GetStorageAttachedDatabasesIDs(
return databasesIDs, nil
}
-// The tables are named rather than the models imported, because the backups
-// feature imports this package.
-func (s *BackupConfigService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {
- var total int64
-
- for _, table := range []string{
- "physical_full_backups",
- "physical_incremental_backups",
- "physical_wal_segments",
- "physical_wal_history_files",
- } {
- var count int64
-
- if err := storage.GetDb().
- Table(table).
- Where("storage_id = ?", storageID).
- Count(&count).Error; err != nil {
- return 0, fmt.Errorf("count %s of a storage: %w", table, err)
- }
-
- total += count
- }
-
- return total, nil
-}
-
func (s *BackupConfigService) SaveBackupConfigWithAuth(
ctx context.Context,
user *users_models.User,
diff --git a/backend/internal/features/backups/no_direct_provider_writes_test.go b/backend/internal/features/backups/no_direct_provider_writes_test.go
index f3eaa7ab..45f9dbdd 100644
--- a/backend/internal/features/backups/no_direct_provider_writes_test.go
+++ b/backend/internal/features/backups/no_direct_provider_writes_test.go
@@ -14,8 +14,8 @@ import (
)
// Backup code writes and removes stored files through the file store, which is what
-// keeps a failed backup from leaving one behind. The old call shape still compiles
-// everywhere, so a guard keeps it from coming back.
+// keeps a failed backup from leaving one behind. A provider's own methods stay
+// reachable from here, so a guard keeps them out.
func Test_BackupCode_DoesNotCallProviderSaveOrDeleteDirectly(t *testing.T) {
var offenders []string
diff --git a/backend/internal/features/storages/controller_test.go b/backend/internal/features/storages/controller_test.go
index e007884a..20b5281c 100644
--- a/backend/internal/features/storages/controller_test.go
+++ b/backend/internal/features/storages/controller_test.go
@@ -41,10 +41,6 @@ func (m *mockStorageReferenceReporter) GetStorageAttachedDatabasesIDs(
return []uuid.UUID{}, nil
}
-func (m *mockStorageReferenceReporter) GetStorageBackupReferences(uuid.UUID) (int64, error) {
- return 0, nil
-}
-
func Test_SaveNewStorage_StorageReturnedViaGet(t *testing.T) {
owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
router := createRouter()
@@ -424,7 +420,7 @@ func Test_WorkspaceRolePermissions(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
router := createRouter()
- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})
+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})
owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Test Workspace", owner, router)
@@ -1140,7 +1136,7 @@ func Test_TransferStorage_PermissionsEnforced(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
router := createRouter()
- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})
+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})
sourceOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
targetOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
@@ -1232,7 +1228,7 @@ func Test_TransferStorage_PermissionsEnforced(t *testing.T) {
func Test_TransferStorageNotManagableWorkspace_TransferFailed(t *testing.T) {
router := createRouter()
- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})
+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})
userA := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
userB := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)
@@ -1291,7 +1287,7 @@ func createRouter() *gin.Engine {
audit_logs.SetupDependencies()
SetupDependencies()
- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})
+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})
return router
}
@@ -1343,10 +1339,10 @@ func Test_DeleteStorage_WhenBackupsStillReferenceIt_IsRefused(t *testing.T) {
workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Referenced Storage Workspace", owner, router)
storage := CreateTestStorage(workspace.ID)
- SetStorageReferenceReportersForTest(&countingReferenceReporter{backupReferences: 3})
+ SetStorageBackupCountersForTest(&countingBackupCounter{backupReferences: 3})
t.Cleanup(func() {
- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})
+ SetStorageBackupCountersForTest()
RemoveTestStorage(t.Context(), storage.ID)
workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router)
})
@@ -1363,7 +1359,7 @@ func Test_DeleteStorage_WhenCleanupIsPending_DrainsItBeforeTheCredentialsGo(t *t
workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Draining Storage Workspace", owner, router)
storage := CreateTestStorage(workspace.ID)
- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})
+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})
fileName := "drained-" + storage.ID.String()
@@ -1378,8 +1374,6 @@ func Test_DeleteStorage_WhenCleanupIsPending_DrainsItBeforeTheCredentialsGo(t *t
_, err = storage.GetFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)
assert.Error(t, err, "the drain runs while the credentials still exist, so the file goes with the storage")
-
- workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router)
}
func testUserModel(t *testing.T, signIn *users_dto.SignInResponseDTO) *users_models.User {
@@ -1391,14 +1385,10 @@ func testUserModel(t *testing.T, signIn *users_dto.SignInResponseDTO) *users_mod
return user
}
-type countingReferenceReporter struct {
+type countingBackupCounter struct {
backupReferences int64
}
-func (r *countingReferenceReporter) GetStorageAttachedDatabasesIDs(uuid.UUID) ([]uuid.UUID, error) {
- return nil, nil
-}
-
-func (r *countingReferenceReporter) GetStorageBackupReferences(uuid.UUID) (int64, error) {
- return r.backupReferences, nil
+func (c *countingBackupCounter) GetStorageBackupReferences(uuid.UUID) (int64, error) {
+ return c.backupReferences, nil
}
diff --git a/backend/internal/features/storages/di.go b/backend/internal/features/storages/di.go
index 46fbd62a..00eda3e8 100644
--- a/backend/internal/features/storages/di.go
+++ b/backend/internal/features/storages/di.go
@@ -18,6 +18,7 @@ var (
audit_logs.GetAuditLogService(),
encryption.GetFieldEncryptor(),
nil,
+ nil,
}
)
diff --git a/backend/internal/features/storages/interfaces.go b/backend/internal/features/storages/interfaces.go
index 6e51ebe6..09d97bd3 100644
--- a/backend/internal/features/storages/interfaces.go
+++ b/backend/internal/features/storages/interfaces.go
@@ -45,11 +45,13 @@ type StorageFileSaver interface {
EncryptSensitiveData(encryptor encryption.FieldEncryptor) error
}
-// StorageReferenceReporter answers both questions a storage has to ask before it
-// can be deleted: whether a database still points at it, and whether backup rows
-// still name files in it. Those rows are the only record of the file names, so
-// removing the storage while they exist would make the objects unreachable.
-type StorageReferenceReporter interface {
+type StorageDatabaseCounter interface {
GetStorageAttachedDatabasesIDs(storageID uuid.UUID) ([]uuid.UUID, error)
+}
+
+// Backup rows are the only record of the file names in a storage, so removing the
+// storage while they exist would make the objects unreachable. Each backup feature
+// counts its own rows, because the tables belong to it.
+type StorageBackupCounter interface {
GetStorageBackupReferences(storageID uuid.UUID) (int64, error)
}
diff --git a/backend/internal/features/storages/service.go b/backend/internal/features/storages/service.go
index bdddd336..a316955c 100644
--- a/backend/internal/features/storages/service.go
+++ b/backend/internal/features/storages/service.go
@@ -15,24 +15,27 @@ import (
)
type StorageService struct {
- storageRepository *StorageRepository
- workspaceService *workspaces_services.WorkspaceService
- auditLogService *audit_logs.AuditLogService
- fieldEncryptor encryption.FieldEncryptor
- storageReferenceReporters []StorageReferenceReporter
+ storageRepository *StorageRepository
+ workspaceService *workspaces_services.WorkspaceService
+ auditLogService *audit_logs.AuditLogService
+ fieldEncryptor encryption.FieldEncryptor
+ storageDatabaseCounters []StorageDatabaseCounter
+ storageBackupCounters []StorageBackupCounter
}
-func (s *StorageService) AddStorageReferenceReporter(
- reporter StorageReferenceReporter,
-) {
- s.storageReferenceReporters = append(s.storageReferenceReporters, reporter)
+func (s *StorageService) AddStorageDatabaseCounter(counter StorageDatabaseCounter) {
+ s.storageDatabaseCounters = append(s.storageDatabaseCounters, counter)
+}
+
+func (s *StorageService) AddStorageBackupCounter(counter StorageBackupCounter) {
+ s.storageBackupCounters = append(s.storageBackupCounters, counter)
}
func (s *StorageService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {
var total int64
- for _, reporter := range s.storageReferenceReporters {
- count, err := reporter.GetStorageBackupReferences(storageID)
+ for _, counter := range s.storageBackupCounters {
+ count, err := counter.GetStorageBackupReferences(storageID)
if err != nil {
return 0, err
}
@@ -49,7 +52,7 @@ func (s *StorageService) GetStorageAttachedDatabasesIDs(
seen := make(map[uuid.UUID]struct{})
merged := make([]uuid.UUID, 0)
- for _, counter := range s.storageReferenceReporters {
+ for _, counter := range s.storageDatabaseCounters {
ids, err := counter.GetStorageAttachedDatabasesIDs(storageID)
if err != nil {
return nil, err
diff --git a/backend/internal/features/storages/testing.go b/backend/internal/features/storages/testing.go
index a9fa432f..860739e1 100644
--- a/backend/internal/features/storages/testing.go
+++ b/backend/internal/features/storages/testing.go
@@ -23,8 +23,12 @@ func DrainStorageFileDeletions(
return storage_files.NewDeletionWorker(storageFileStore, dependencies).DrainForTest(ctx, references...)
}
-func SetStorageReferenceReportersForTest(reporters ...StorageReferenceReporter) {
- storageService.storageReferenceReporters = reporters
+func SetStorageDatabaseCountersForTest(counters ...StorageDatabaseCounter) {
+ storageService.storageDatabaseCounters = counters
+}
+
+func SetStorageBackupCountersForTest(counters ...StorageBackupCounter) {
+ storageService.storageBackupCounters = counters
}
func CreateTestStorage(workspaceID uuid.UUID) *Storage {
diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.md
index 57727479..3c98bfd8 100644
--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md
+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md
@@ -17,7 +17,7 @@
- [x] 3.1 Implement one deletion-worker pass that takes the live-write mutex, reads the set under it and holds it until its claim statement returns, so no writer can join in between; that statement claims a bounded batch of due rows with skip-locked row selection ordered by `not_before`, excludes the set and increments the generation, commits the claim and only then performs remote I/O; verify a live write is never claimed even when it runs longer than the scan interval or when a cascading deletion request pulls its row forward to now, a file inherited from a stopped process is claimed on the first pass, a claim invalidates an outstanding receipt and an open deletion request does not block the batch.
- [x] 3.2 Complete a successful deletion by removing the row and reschedule a failed one with capped exponential backoff, jitter, the incremented attempt count and a sanitized error, both conditional on the generation it claimed; verify transient failure, process restart, an attempt that outlived its timeout, an attempt whose row and storage were both removed by a storage deletion while it ran, and eventual success. Since `not_before` is compared against the database clock, drive these tests with the shrunk constants from task 1.4, a deterministic jitter source and fixtures that write `not_before` directly, not with a Go-side clock.
- [x] 3.3 Persist only bounded, sanitized deletion errors and emit attempt, retry and success logs carrying `job_name`, a fresh `job_id`, `storage_id`, `file_name` and `error` as key-value pairs, with the attempt count and the next retry time in the message text and the per-pass pending count, overdue count and oldest-obligation age in the summary message, following the value-in-message rule in `backend/AGENTS.md`; verify centralized redaction and truncation preserve these fields while removing credentials, storage configuration and signed URLs.
-- [x] 3.4 Add a non-overlapping `Run` guard and a single-pass execution seam, then register graceful startup and shutdown with the application lifecycle in `backend/cmd/main.go`; verify a second `Run` panics, overlapping ticks are skipped and shutdown does not leave an unjoined in-process provider call.
+- [x] 3.4 Add a non-overlapping `Run` guard and a single-pass execution seam, then register graceful startup and shutdown with the application lifecycle in `backend/cmd/main.go`; verify a second `Run` panics, that overlapping ticks are skipped and that a cancelled context ends the loop only after the pass in flight has finished its provider call. The worker is started with a bare `go`, the same as every other background service in `cmd/main.go`, so nothing joins it at shutdown.
## 4. Enforce the provider cleanup contract
@@ -53,13 +53,13 @@
- [x] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.
- [x] 8.2 Change physical backup, WAL, history and metadata cleanup, `removeUploadedArtifactsAfterChainBroken` (`create_full_uc.go:192` and `:198`, whose manifest name also comes from the pre-loop file name, and whose doc comment at `:179` restates its name and promises that `DeleteFile` is idempotent on not-found, which stops being what the call does, so delete both claims and keep only what survives), `DeleteChainDependentsKeepFull`, `DeleteOrphanWalSegmentsInSpan`, `deleteOrphanedHistoryFiles` and retention paths to request exact references transactionally, removing both the fail-open and fail-closed remote deletion behavior and the remote calls that currently run inside the cascade transaction; verify cascade and partial-batch rollback cases, including that a rolled-back cascade leaves every object intact.
- [x] 8.3 Add clear-path tests that create real local and MinIO artifacts, invoke each cleaner, run the deletion worker until idle and assert both catalog removal and physical absence; include one transient deletion failure to prove the catalog may disappear before eventual remote cleanup succeeds.
-- [x] 8.4 Keep the `OnBeforeDatabaseRemove` signature and add a physical implementation instead. The contract change was meant to make database removal record every reference atomically; what actually matters is that something records them at all, and nothing did for the physical family: every physical table cascades on `databases.id`, so removing a database dropped the only record of its object names. `PhysicalBackupService` now implements the listener and records the references for its full, incremental, WAL and history rows in a transaction of its own, which is the same shape the logical cleaner already used. Passing the caller's transaction instead would hold one transaction across every backup of a database, which the design's own risk entry names as a cost, so each listener keeps its own.
+- [x] 8.4 Keep the `OnBeforeDatabaseRemove` signature and add a physical implementation instead. The contract change was meant to make database removal record every reference atomically. Nothing recorded them at all for the physical family: every physical table cascades on `databases.id`, so removing a database dropped the only record of its object names. `PhysicalBackupService` now implements the listener and records the references for its full, incremental, WAL and history rows in a transaction of its own, which is the same shape the logical cleaner already used. Passing the caller's transaction instead would hold one transaction across every backup of a database, which the design's own risk entry names as a cost, so each listener keeps its own.
- [x] 8.5 Nothing to move: `DeleteForTest` runs the same listener chain as production deletion, and with task 8.4 dropped that chain keeps its signature. The fixture therefore records the same obligations as the controller path already.
- [x] 8.6 Add a production-code guard that fails tests if backup workflows call provider `SaveFile` or `DeleteFile` directly, once sections 5 through 8 have removed the last of those calls. Scope it to non-test files under `backend/internal/features/backups`, so it flags neither the storages feature's own `Storage` methods, its provider tests and the `--test-storage` probe in `backend/cmd/storage_command.go`, nor the backup test fixtures that legitimately create and remove provider objects, such as the `SaveFile` call in `core/physical/service/service_test.go`; verify the guard by planting a `DeleteFile` call under `backend/internal/features/backups` and confirming the guard fails on it, then removing it and confirming the guard stays silent for the provider calls that live outside its scope.
## 9. Guard and drain the storage lifecycle
-- [x] 9.1 Refuse storage deletion while backup rows still reference the storage, returning an actionable conflict instead of a silent logical cascade or a raw foreign-key error, by adding a `GetStorageBackupReferences` method to the existing registration mechanism rather than adding a second registry, and renaming `StorageDatabaseCounter`, `AddStorageDatabaseCounter`, the service field `storageDatabaseCounters` and the fixture `SetStorageDatabaseCountersForTest` to `StorageReferenceReporter`, `AddStorageReferenceReporter`, `storageReferenceReporters` and `SetStorageReferenceReportersForTest`; verify controller responses for logical and physical references and that deletion succeeds once the backups are gone.
+- [x] 9.1 Refuse storage deletion while backup rows still reference the storage, returning an actionable conflict instead of a silent logical cascade or a raw foreign-key error, by adding a second registration, `AddStorageBackupCounter`, beside the existing `AddStorageDatabaseCounter`. One interface carrying both questions would have made the config services answer for tables the backup services own, which meant reading those tables by name across a feature boundary; each feature counts its own rows instead; verify controller responses for logical and physical references and that deletion succeeds once the backups are gone.
- [x] 9.2 Make storage deletion run one drain of that storage's pending deletions first, bounded by a row count and a wall-clock budget and claiming through the same live-write exclusion the worker uses, then write one audit-log entry and a warning naming the storage, how many pending deletion rows are still present for it after the drain and their names up to a fixed limit, counting rows the drain failed on, skipped as locked, never saw because the worker had claimed them or never reached before the budget expired; verify a reachable provider leaves nothing behind, an unreachable one still deletes the storage within the budget and reports each remaining file once, that the entry stays bounded for a storage with thousands of obligations and that the remaining rows disappear with the storage row.
- [x] 9.3 Give the storage workspace-deletion listener at `storages/service.go:73` the same drain and reporting path instead of deleting storage rows straight through its repository, leaving `OnBeforeWorkspaceDeletion` as it is and giving the drain a background context. The signature change would have kept the request ID on the drain's audit entry, but it touches three implementations for one log field on a path that already runs after the request that started it; verify workspace deletion succeeds with an unreachable provider, records what it left behind and keeps the existing refusal for a workspace that still contains databases.