PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
REFACTOR (backups): Let each backup feature count its own storage references
Rostislav Duginec9bce5de85d
diff --git a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go b/backend/internal/features/backups/backups/backuping/logical/backuper_test.goindex 34311c27..393b0655 100644--- a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go+++ b/backend/internal/features/backups/backups/backuping/logical/backuper_test.go@@ -414,6 +414,8 @@ func Test_MakeBackup_WhenPublicationFails_EveryFileOfTheAttemptIsRemoved(t *testassert.Equal(t, backups_core_logical.BackupStatusFailed, persistedBackup.Status,"a backup whose files cannot be claimed must not be published")+ // The caller names the files its scenario actually produced: asserting the+ // absence of a file nothing ever wrote would pass with no cleanup at all.assertBackupFilesRemoved(t, fixture, backup.FileName, backup.FileName+metadataSuffix)}diff --git a/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/logical/cleaner_test.goindex 01e66644..46399dd2 100644--- a/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go+++ b/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go@@ -612,8 +612,8 @@ func createTestInterval() intervals.Interval {}}-// The point of the whole change: the catalog row goes even when the provider is-// refusing, and the file still disappears once it stops refusing.+// The catalog row goes while the provider is refusing, and the file follows once+// it stops.func Test_DeleteBackup_WhenProviderRefusesThenRecovers_RowGoesFirstAndFileFollows(t *testing.T) {fixture := CreateBackupTestFixture(t, "Flaky Cleanup Workspace")backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.goindex e45138fc..b07c2f90 100644--- a/backend/internal/features/backups/backups/backuping/physical/di.go+++ b/backend/internal/features/backups/backups/backuping/physical/di.go@@ -125,10 +125,10 @@ var SetupDependencies = sync.OnceFunc(func() {// can drop the (now detaching) WAL slot instead of refusing it as active and// leaving it to pin WAL forever.databases.GetDatabaseService().AddDbRemoveListener(physicalBackupCancellationListener)- // Registered before the cancellation listener's siblings finish, but ordering- // does not matter: the cascade that would take the names away runs only after- // every listener has returned.+ // The cascade that would take the object names away runs only after every+ // listener has returned.databases.GetDatabaseService().AddDbRemoveListener(physical_service.GetPhysicalBackupService())+ storages.GetStorageService().AddStorageBackupCounter(physical_service.GetPhysicalBackupService())databases.GetDatabaseService().AddDbRemoveListener(physicalSlotCleanupListener)backups_config_physical.GetBackupConfigService().SetBackupCancellationListener(physicalBackupCancellationListener)})diff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler.go b/backend/internal/features/backups/backups/backuping/physical/scheduler.goindex 4ff81db2..b3eec17f 100644--- a/backend/internal/features/backups/backups/backuping/physical/scheduler.go+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler.go@@ -653,12 +653,12 @@ func (s *PhysicalBackupsScheduler) failClaimlessInProgressBackups("backup_id", full.ID, "database_id", full.DatabaseID)s.failOrphanedBackup(ctx, logger, orphanedBackupSpec{- Kind: physical_enums.PhysicalBackupTypeFull,- BackupID: full.ID,- DatabaseID: full.DatabaseID,- StorageID: full.StorageID,- FileName: valueOrEmpty(full.FileName),- Manifest: valueOrEmpty(full.ManifestFileName),+ Kind: physical_enums.PhysicalBackupTypeFull,+ BackupID: full.ID,+ DatabaseID: full.DatabaseID,+ StorageID: full.StorageID,+ FileName: valueOrEmpty(full.FileName),+ ManifestFileName: valueOrEmpty(full.ManifestFileName),})}@@ -676,20 +676,20 @@ func (s *PhysicalBackupsScheduler) failClaimlessInProgressBackups("backup_id", incremental.ID, "database_id", incremental.DatabaseID)s.failOrphanedBackup(ctx, logger, orphanedBackupSpec{- Kind: physical_enums.PhysicalBackupTypeIncremental,- BackupID: incremental.ID,- DatabaseID: incremental.DatabaseID,- StorageID: incremental.StorageID,- FileName: valueOrEmpty(incremental.FileName),- Manifest: valueOrEmpty(incremental.ManifestFileName),+ Kind: physical_enums.PhysicalBackupTypeIncremental,+ BackupID: incremental.ID,+ DatabaseID: incremental.DatabaseID,+ StorageID: incremental.StorageID,+ FileName: valueOrEmpty(incremental.FileName),+ ManifestFileName: valueOrEmpty(incremental.ManifestFileName),})}return nil}-// describeOrphan reads the file the claimed row already names. A claim carries no-// file name of its own, and the sweep cannot hand back what it cannot name.+// A claim carries no file name of its own, and the sweep cannot hand back what it+// cannot name.func (s *PhysicalBackupsScheduler) describeOrphan(logger *slog.Logger,claim *physical_models.PhysicalInFlightBackup,@@ -710,7 +710,7 @@ func (s *PhysicalBackupsScheduler) describeOrphan(spec.StorageID = row.StorageIDspec.FileName = valueOrEmpty(row.FileName)- spec.Manifest = valueOrEmpty(row.ManifestFileName)+ spec.ManifestFileName = valueOrEmpty(row.ManifestFileName)return spec}@@ -724,21 +724,20 @@ func (s *PhysicalBackupsScheduler) describeOrphan(spec.StorageID = row.StorageIDspec.FileName = valueOrEmpty(row.FileName)- spec.Manifest = valueOrEmpty(row.ManifestFileName)+ spec.ManifestFileName = valueOrEmpty(row.ManifestFileName)return spec}-// orphanedBackupSpec is one IN_PROGRESS row the previous run left behind. It-// carries the file name because the row now holds one from upload start, so the-// sweep can hand that file back instead of assuming none exists.+// orphanedBackupSpec is one IN_PROGRESS row the previous run left behind, with+// the names its files carry so the sweep can hand them back.type orphanedBackupSpec struct {- Kind physical_enums.PhysicalBackupType- BackupID uuid.UUID- DatabaseID uuid.UUID- StorageID uuid.UUID- FileName string- Manifest string+ Kind physical_enums.PhysicalBackupType+ BackupID uuid.UUID+ DatabaseID uuid.UUID+ StorageID uuid.UUID+ FileName string+ ManifestFileName string}func (s *PhysicalBackupsScheduler) failOrphanedBackup(@@ -772,8 +771,8 @@ func (s *PhysicalBackupsScheduler) failOrphanedBackup(// essential: an orphan claim left behind would block every future tick from// acquiring the cross-table single-in-flight slot for that DB, freezing it.//-// The row carries a file name from upload start, so the sweep hands that file-// back to cleanup in the same transaction, gated on the status flip taking effect.+// The sweep hands the row's files back to cleanup in the same transaction, gated+// on the status flip taking effect.func (s *PhysicalBackupsScheduler) failBackupAndReleaseClaim(ctx context.Context,spec orphanedBackupSpec,@@ -808,9 +807,9 @@ func (s *PhysicalBackupsScheduler) failBackupAndReleaseClaim({StorageID: spec.StorageID, FileName: spec.FileName + metadataSuffix},}- if spec.Manifest != "" {+ if spec.ManifestFileName != "" {references = append(references,- storage_files.StoredFileReference{StorageID: spec.StorageID, FileName: spec.Manifest})+ storage_files.StoredFileReference{StorageID: spec.StorageID, FileName: spec.ManifestFileName})}if err := s.fileStore.RequestFileDeletions(ctx, tx, references); err != nil {diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.goindex b35025e3..7d7b6835 100644--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go@@ -276,7 +276,6 @@ func (s *PhysicalWalStreamSupervisor) startStreamer(SourceDB: tunneledDatabase.GetDatabaseThroughTunnel(),IsBastionReachable: tunneledDatabase.IsBastionReachable,StorageID: storage.ID,- Storage: storage,FileStore: s.fileStore,Encryption: backupConfig.Encryption,MasterKey: masterKey,diff --git a/backend/internal/features/backups/backups/core/logical/repository.go b/backend/internal/features/backups/backups/core/logical/repository.goindex 8fb6ab93..6880c502 100644--- a/backend/internal/features/backups/backups/core/logical/repository.go+++ b/backend/internal/features/backups/backups/core/logical/repository.go@@ -2,6 +2,7 @@ package backups_core_logicalimport ("errors"+ "fmt""time""github.com/google/uuid"@@ -197,6 +198,19 @@ func (r *BackupRepository) FindByDatabaseIdAndStatus(return backups, nil}+func (r *BackupRepository) CountByStorageID(storageID uuid.UUID) (int64, error) {+ var count int64++ if err := storage.GetDb().+ Model(&LogicalBackup{}).+ Where("storage_id = ?", storageID).+ Count(&count).Error; err != nil {+ return 0, fmt.Errorf("count backups of a storage: %w", err)+ }++ return count, nil+}+func (r *BackupRepository) DeleteByID(id uuid.UUID) error {return r.DeleteByIDInTransaction(storage.GetDb(), id)}diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.goindex 98e165bd..5644748c 100644--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go@@ -119,9 +119,9 @@ func (r *PhysicalWalSegmentRepository) DeleteByID(id uuid.UUID) error {return storage.GetDb().Delete(&physical_models.PhysicalWalSegment{}, "id = ?", id).Error}-// A NULL file_name no longer proves the upload wrote nothing: the object is-// written before the row names it. The bytes such a claim may have left are owned-// by their own pending deletion, so this only removes the aged-out claim row.+// A NULL file_name does not prove the upload wrote nothing: the object is written+// before the row names it. The bytes such a claim may have left are owned by their+// own pending deletion, so this only removes the aged-out claim row.func (r *PhysicalWalSegmentRepository) DeleteAbandonedClaims(databaseID uuid.UUID,olderThan time.Time,@@ -204,30 +204,30 @@ func (r *PhysicalWalSegmentRepository) FindByChainKey(// makes this a no-op. updated=true means the upload is durably committed;// updated=false means the NULL claim no longer exists (cascade caught it) and the// caller must DeleteFile the now-orphaned storage object.-func (r *PhysicalWalSegmentRepository) MarkUploaded(- id uuid.UUID,- fileName string,- compressedSizeMb float64,- encryptionSalt, encryptionIV *string,-) (updated bool, err error) {- return r.MarkUploadedInTransaction(storage.GetDb(), id, fileName, compressedSizeMb, encryptionSalt, encryptionIV)+type WalSegmentUpload struct {+ SegmentID uuid.UUID+ FileName string+ CompressedSizeMb float64+ EncryptionSalt *string+ EncryptionIV *string+}++func (r *PhysicalWalSegmentRepository) MarkUploaded(upload WalSegmentUpload) (updated bool, err error) {+ return r.MarkUploadedInTransaction(storage.GetDb(), upload)}func (r *PhysicalWalSegmentRepository) MarkUploadedInTransaction(tx *gorm.DB,- id uuid.UUID,- fileName string,- compressedSizeMb float64,- encryptionSalt, encryptionIV *string,+ upload WalSegmentUpload,) (updated bool, err error) {result := tx.Model(&physical_models.PhysicalWalSegment{}).- Where("id = ? AND file_name IS NULL", id).+ Where("id = ? AND file_name IS NULL", upload.SegmentID).Updates(map[string]any{- "file_name": fileName,- "compressed_size_mb": compressedSizeMb,- "encryption_salt": encryptionSalt,- "encryption_iv": encryptionIV,+ "file_name": upload.FileName,+ "compressed_size_mb": upload.CompressedSizeMb,+ "encryption_salt": upload.EncryptionSalt,+ "encryption_iv": upload.EncryptionIV,})if result.Error != nil {return false, result.Errordiff --git a/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go b/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.goindex 6fc70d05..ee4d5f47 100644--- a/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go+++ b/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go@@ -26,6 +26,29 @@ func (s *PhysicalBackupService) OnBeforeDatabaseRemove(ctx context.Context, data})}+// The physical rows are the only record of the file names in a storage, so the+// storage cannot go while they exist.+func (s *PhysicalBackupService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {+ var total int64++ for _, model := range []any{+ &physical_models.PhysicalFullBackup{},+ &physical_models.PhysicalIncrementalBackup{},+ &physical_models.PhysicalWalSegment{},+ &physical_models.PhysicalWalHistoryFile{},+ } {+ var count int64++ if err := db.GetDb().Model(model).Where("storage_id = ?", storageID).Count(&count).Error; err != nil {+ return 0, fmt.Errorf("count physical rows of a storage: %w", err)+ }++ total += count+ }++ return total, nil+}+func (s *PhysicalBackupService) collectDatabaseFileReferences(tx *gorm.DB,databaseID uuid.UUID,@@ -82,18 +105,15 @@ func artifactReferences(storageID uuid.UUID,fileName, manifestFileName string,) []storage_files.StoredFileReference {- if fileName == "" {- return nil- }+ references := make([]storage_files.StoredFileReference, 0, 3)- references := []storage_files.StoredFileReference{- {StorageID: storageID, FileName: fileName},- {StorageID: storageID, FileName: fileName + metadataSuffix},- }+ for _, name := range []string{fileName, fileName + metadataSuffix, manifestFileName} {+ if name == "" || name == metadataSuffix {+ continue+ }- if manifestFileName != "" {references = append(references,- storage_files.StoredFileReference{StorageID: storageID, FileName: manifestFileName})+ storage_files.StoredFileReference{StorageID: storageID, FileName: name})}return referencesdiff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.goindex f6dc1ea2..6f7fa922 100644--- a/backend/internal/features/backups/backups/core/physical/service/service.go+++ b/backend/internal/features/backups/backups/core/physical/service/service.go@@ -1054,8 +1054,8 @@ func (s *PhysicalBackupService) deleteFullArtifactAndRow(// requestFileDeletions records the obligation to remove these files in the same// transaction that removes the rows naming them. Nothing reaches the provider-// here: a rolled back cascade leaves every object intact, and a storage outage-// no longer decides whether a catalog row may go.+// here, so a rolled back cascade leaves every object intact and a storage outage+// does not decide whether a catalog row may go.func (s *PhysicalBackupService) requestFileDeletions(ctx context.Context,tx *gorm.DB,diff --git a/backend/internal/features/backups/backups/services/di.go b/backend/internal/features/backups/backups/services/di.goindex c534ac21..09dd3911 100644--- a/backend/internal/features/backups/backups/services/di.go+++ b/backend/internal/features/backups/backups/services/di.go@@ -78,5 +78,6 @@ var SetupDependencies = sync.OnceFunc(func() {SetDatabaseStorageChangeListener(backupService)databases.GetDatabaseService().AddDbRemoveListener(backupService)+ storages.GetStorageService().AddStorageBackupCounter(backupService)databases.GetDatabaseService().AddDbCopyListener(backups_config_logical.GetBackupConfigService())})diff --git a/backend/internal/features/backups/backups/services/logical_service.go b/backend/internal/features/backups/backups/services/logical_service.goindex 256a91cc..a82467e8 100644--- a/backend/internal/features/backups/backups/services/logical_service.go+++ b/backend/internal/features/backups/backups/services/logical_service.go@@ -81,6 +81,12 @@ func (s *LogicalBackupService) OnBeforeBackupsStorageChange(ctx context.Context,return nil}+// The logical_backups rows are the only record of the file names in a storage, so+// the storage cannot go while they exist.+func (s *LogicalBackupService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {+ return s.backupRepository.CountByStorageID(storageID)+}+func (s *LogicalBackupService) OnBeforeDatabaseRemove(ctx context.Context, databaseID uuid.UUID) error {err := s.deleteDbBackups(ctx, databaseID)if err != nil {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.goindex 8fc4e50b..d7b1b9d9 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go@@ -41,21 +41,14 @@ func (s CommonBackupSpec) runStreamParams(}}-// The ZSTD -> GZIP -> NONE fallback runs INSIDE the per-backup replication slot,-// so the slot is held across every attempt; only the --compress flag, the recorded-// codec and the object key differ between them. A rejected attempt leaves an object-// of its own for cleanup rather than being overwritten.-//-// IncrementalManifestPath is "" for a FULL; a non-empty path is the downloaded-// parent-manifest temp file that makes this an INCR (--incremental=<path>).-// streamAttemptSpec carries what one backup needs to stream, so the codec loop-// does not grow a positional list every time the caller learns something new.type streamAttemptSpec struct {- Common CommonBackupSpec- BackupID uuid.UUID- Creds *postgresql_shared.CredentialTempFiles- Label string- SystemID uint64+ Common CommonBackupSpec+ BackupID uuid.UUID+ Creds *postgresql_shared.CredentialTempFiles+ Label string+ SystemID uint64+ // "" for a FULL; a non-empty path is the downloaded parent-manifest temp file+ // that makes this an INCR (--incremental=<path>).IncrementalManifestPath stringClassify streamErrorClassifier@@ -66,6 +59,10 @@ type streamAttemptSpec struct {MintAndSaveAttemptName func() (string, error)}+// The ZSTD -> GZIP -> NONE fallback runs INSIDE the per-backup replication slot,+// so the slot is held across every attempt; only the --compress flag, the recorded+// codec and the object key differ between them. A rejected attempt leaves an object+// of its own for cleanup rather than being overwritten.func streamWithCodecFallback(ctx context.Context,spec streamAttemptSpec,diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.goindex 0ce01e2b..deadbc3e 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go@@ -47,8 +47,8 @@ type mockWalStorage struct {}// hasObjectFor reports whether any stored object belongs to the given segment.-// Per-attempt names mean a test can no longer rebuild the exact key, so it matches-// on the segment identity the key is derived from.+// A test cannot rebuild the exact key, which carries a per-attempt UUID, so it+// matches on the segment identity the key derives from.func (m *mockWalStorage) hasObjectFor(databaseID uuid.UUID, timelineID int, walFilename string) bool {m.mu.Lock()defer m.mu.Unlock()diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.goindex 4a4bfd0d..58d92bd3 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go@@ -116,8 +116,8 @@ func (f *fakeStorageFileSaver) DeleteFile(context.Context, encryption.FieldEncryfunc (f *fakeStorageFileSaver) Validate(encryption.FieldEncryptor) error { return nil }func (f *fakeStorageFileSaver) TestConnection(encryption.FieldEncryptor) error { return nil }-// The physical stream writes through the file store now, and the fake keeps its-// own recording so the deadline and payload assertions stay where they were.+// The physical stream writes through the file store, and the fake records what it+// receives so the deadline and payload assertions have something to read.func (f *fakeStorageFileSaver) WriteFile(ctx context.Context,reference storage_files.StoredFileReference,diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.goindex 13679f69..477f5bfe 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go@@ -843,7 +843,6 @@ func StartWalStreamerForTest(t *testing.T, spec WalStreamerTestSpec) *WalStreameDatabaseID: fixture.DB.ID,SourceDB: fixture.DB.PostgresqlPhysical,StorageID: fixture.Storage.ID,- Storage: spec.Storage,FileStore: spec.FileStore,Encryption: backups_core_enums.BackupEncryptionNone,FieldEncryptor: encryption.GetFieldEncryptor(),diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.goindex 215ea293..99f6f5a6 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/timeline.go@@ -186,14 +186,6 @@ func ValidateStartLsnAgainstHistory(return chain_view.ValidationResult{Status: chain_view.ValidationStatusOK}, nil}-// UploadHistoryFile reads the .history file for timelineID from the source-// cluster's pg_wal/, compresses with zstd, optionally encrypts, uploads-// artifact + sidecar to storage, and inserts the physical_wal_history_files-// row. Idempotent on (database_id, timeline_id) via the UNIQUE constraint:-// a duplicate insert returns nil after observing the existing row.-//-// Shared by full.go (post-stream, when the FULL ran on a TL > 1) and-// PR 4's wal_stream.go (which also observes .history arrivals).type HistoryUploadSpec struct {Conn *pgx.ConnTimelineID int@@ -207,6 +199,8 @@ type HistoryUploadSpec struct {Logger *slog.Logger}+// Idempotent on (database_id, timeline_id) through the UNIQUE constraint: a+// duplicate insert returns the existing row instead of an error.func UploadHistoryFile(ctx context.Context,spec HistoryUploadSpec,diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.goindex 9bc2fd6a..3b59ed1d 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go@@ -15,7 +15,6 @@ import (backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"- "databasus-backend/internal/features/storages"storage_files "databasus-backend/internal/features/storages/files"util_encryption "databasus-backend/internal/util/encryption""databasus-backend/internal/util/tools"@@ -137,7 +136,6 @@ type WalStreamSpec struct {DatabaseID uuid.UUIDSourceDB *postgresql_physical.PostgresqlPhysicalDatabaseStorageID uuid.UUID- Storage storages.StorageFileSaverFileStore *storage_files.StoreEncryption backups_core_enums.BackupEncryptionMasterKey stringdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.goindex 9d2db062..6e9ca975 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go@@ -403,8 +403,8 @@ func (u *WalUploader) reference(objectName string) storage_files.StoredFileReferreturn storage_files.StoredFileReference{StorageID: u.deps.StorageID, FileName: objectName}}-// discardAttempt hands the attempt's artifact and sidecar back to cleanup. No-// catalog row survives to carry them, so the request gets a transaction of its own.+// No catalog row survives to carry these files, so the request gets a transaction+// of its own.func (u *WalUploader) discardAttempt(ctx context.Context, objectName string) {references := []storage_files.StoredFileReference{u.reference(objectName),@@ -435,9 +435,13 @@ func (u *WalUploader) commitSegment(ctx context.Context, spec commitSegmentSpec)committed := falseerr := db.GetDb().Transaction(func(tx *gorm.DB) error {- updated, err := u.deps.WalSegmentRepo.MarkUploadedInTransaction(- tx, spec.Claim.ID, spec.ObjectName, spec.CompressedSizeMb, nilIfEmpty(spec.Salt), nilIfEmpty(spec.IV),- )+ updated, err := u.deps.WalSegmentRepo.MarkUploadedInTransaction(tx, physical_repositories.WalSegmentUpload{+ SegmentID: spec.Claim.ID,+ FileName: spec.ObjectName,+ CompressedSizeMb: spec.CompressedSizeMb,+ EncryptionSalt: nilIfEmpty(spec.Salt),+ EncryptionIV: nilIfEmpty(spec.IV),+ })if err != nil {return err}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.goindex 64c6d970..acacf918 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go@@ -81,8 +81,8 @@ func Test_WalUpload_ClaimSucceeds_SaveFileLands_FileNameUpdatedFromNullToNonNullrequire.NotNil(t, row)require.NotNil(t, row.FileName, "file_name must be flipped from NULL to the object key")- // The name carries a per-attempt UUID now, so the row is the only place that- // knows it, which is where restore reads it from too.+ // The name carries a per-attempt UUID, so the row is the only place that knows+ // it, which is where restore reads it from too.objectName := *row.FileNamerequire.True(t, store.hasObject(objectName), "artifact must be in storage under the name the row carries")require.True(t, store.hasObject(objectName+metadataSuffix), "sidecar must be uploaded")diff --git a/backend/internal/features/backups/config/logical/di.go b/backend/internal/features/backups/config/logical/di.goindex 16e8d083..b08f7aae 100644--- a/backend/internal/features/backups/config/logical/di.go+++ b/backend/internal/features/backups/config/logical/di.go@@ -34,5 +34,5 @@ func GetBackupConfigService() *BackupConfigService {}var SetupDependencies = sync.OnceFunc(func() {- storages.GetStorageService().AddStorageReferenceReporter(backupConfigService)+ storages.GetStorageService().AddStorageDatabaseCounter(backupConfigService)})diff --git a/backend/internal/features/backups/config/logical/service.go b/backend/internal/features/backups/config/logical/service.goindex 41685a73..f4f1bbef 100644--- a/backend/internal/features/backups/config/logical/service.go+++ b/backend/internal/features/backups/config/logical/service.go@@ -3,7 +3,6 @@ package backups_config_logicalimport ("context""errors"- "fmt""github.com/google/uuid"@@ -14,7 +13,6 @@ import ("databasus-backend/internal/features/storages"users_models "databasus-backend/internal/features/users/models"workspaces_services "databasus-backend/internal/features/workspaces/services"- "databasus-backend/internal/storage""databasus-backend/internal/util/period")@@ -45,24 +43,6 @@ func (s *BackupConfigService) GetStorageAttachedDatabasesIDs(return databasesIDs, nil}-// Those rows are the only record of the file names, so the storage cannot go-// while they exist.-func (s *BackupConfigService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {- var count int64-- // The table is named rather than the model imported: the backups feature imports- // this package, so the dependency cannot run the other way.- err := storage.GetDb().- Table("logical_backups").- Where("storage_id = ?", storageID).- Count(&count).Error- if err != nil {- return 0, fmt.Errorf("count logical backups of a storage: %w", err)- }-- return count, nil-}-func (s *BackupConfigService) SaveBackupConfigWithAuth(ctx context.Context,user *users_models.User,diff --git a/backend/internal/features/backups/config/physical/di.go b/backend/internal/features/backups/config/physical/di.goindex 6266bf4e..287ef6d2 100644--- a/backend/internal/features/backups/config/physical/di.go+++ b/backend/internal/features/backups/config/physical/di.go@@ -37,6 +37,6 @@ func GetBackupConfigService() *BackupConfigService {}var SetupDependencies = sync.OnceFunc(func() {- storages.GetStorageService().AddStorageReferenceReporter(backupConfigService)+ storages.GetStorageService().AddStorageDatabaseCounter(backupConfigService)databases.GetDatabaseService().AddDbBackupTypeChangeListener(backupConfigService)})diff --git a/backend/internal/features/backups/config/physical/service.go b/backend/internal/features/backups/config/physical/service.goindex 7b915b9a..0d5d1fe3 100644--- a/backend/internal/features/backups/config/physical/service.go+++ b/backend/internal/features/backups/config/physical/service.go@@ -15,7 +15,6 @@ import ("databasus-backend/internal/features/storages"users_models "databasus-backend/internal/features/users/models"workspaces_services "databasus-backend/internal/features/workspaces/services"- "databasus-backend/internal/storage")type BackupConfigService struct {@@ -53,32 +52,6 @@ func (s *BackupConfigService) GetStorageAttachedDatabasesIDs(return databasesIDs, nil}-// The tables are named rather than the models imported, because the backups-// feature imports this package.-func (s *BackupConfigService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {- var total int64-- for _, table := range []string{- "physical_full_backups",- "physical_incremental_backups",- "physical_wal_segments",- "physical_wal_history_files",- } {- var count int64-- if err := storage.GetDb().- Table(table).- Where("storage_id = ?", storageID).- Count(&count).Error; err != nil {- return 0, fmt.Errorf("count %s of a storage: %w", table, err)- }-- total += count- }-- return total, nil-}-func (s *BackupConfigService) SaveBackupConfigWithAuth(ctx context.Context,user *users_models.User,diff --git a/backend/internal/features/backups/no_direct_provider_writes_test.go b/backend/internal/features/backups/no_direct_provider_writes_test.goindex f3eaa7ab..45f9dbdd 100644--- a/backend/internal/features/backups/no_direct_provider_writes_test.go+++ b/backend/internal/features/backups/no_direct_provider_writes_test.go@@ -14,8 +14,8 @@ import ()// Backup code writes and removes stored files through the file store, which is what-// keeps a failed backup from leaving one behind. The old call shape still compiles-// everywhere, so a guard keeps it from coming back.+// keeps a failed backup from leaving one behind. A provider's own methods stay+// reachable from here, so a guard keeps them out.func Test_BackupCode_DoesNotCallProviderSaveOrDeleteDirectly(t *testing.T) {var offenders []stringdiff --git a/backend/internal/features/storages/controller_test.go b/backend/internal/features/storages/controller_test.goindex e007884a..20b5281c 100644--- a/backend/internal/features/storages/controller_test.go+++ b/backend/internal/features/storages/controller_test.go@@ -41,10 +41,6 @@ func (m *mockStorageReferenceReporter) GetStorageAttachedDatabasesIDs(return []uuid.UUID{}, nil}-func (m *mockStorageReferenceReporter) GetStorageBackupReferences(uuid.UUID) (int64, error) {- return 0, nil-}-func Test_SaveNewStorage_StorageReturnedViaGet(t *testing.T) {owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)router := createRouter()@@ -424,7 +420,7 @@ func Test_WorkspaceRolePermissions(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {router := createRouter()- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Test Workspace", owner, router)@@ -1140,7 +1136,7 @@ func Test_TransferStorage_PermissionsEnforced(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {router := createRouter()- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})sourceOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)targetOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)@@ -1232,7 +1228,7 @@ func Test_TransferStorage_PermissionsEnforced(t *testing.T) {func Test_TransferStorageNotManagableWorkspace_TransferFailed(t *testing.T) {router := createRouter()- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})userA := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)userB := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)@@ -1291,7 +1287,7 @@ func createRouter() *gin.Engine {audit_logs.SetupDependencies()SetupDependencies()- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})return router}@@ -1343,10 +1339,10 @@ func Test_DeleteStorage_WhenBackupsStillReferenceIt_IsRefused(t *testing.T) {workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Referenced Storage Workspace", owner, router)storage := CreateTestStorage(workspace.ID)- SetStorageReferenceReportersForTest(&countingReferenceReporter{backupReferences: 3})+ SetStorageBackupCountersForTest(&countingBackupCounter{backupReferences: 3})t.Cleanup(func() {- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})+ SetStorageBackupCountersForTest()RemoveTestStorage(t.Context(), storage.ID)workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router)})@@ -1363,7 +1359,7 @@ func Test_DeleteStorage_WhenCleanupIsPending_DrainsItBeforeTheCredentialsGo(t *tworkspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Draining Storage Workspace", owner, router)storage := CreateTestStorage(workspace.ID)- SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})fileName := "drained-" + storage.ID.String()@@ -1378,8 +1374,6 @@ func Test_DeleteStorage_WhenCleanupIsPending_DrainsItBeforeTheCredentialsGo(t *t_, err = storage.GetFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)assert.Error(t, err, "the drain runs while the credentials still exist, so the file goes with the storage")-- workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router)}func testUserModel(t *testing.T, signIn *users_dto.SignInResponseDTO) *users_models.User {@@ -1391,14 +1385,10 @@ func testUserModel(t *testing.T, signIn *users_dto.SignInResponseDTO) *users_modreturn user}-type countingReferenceReporter struct {+type countingBackupCounter struct {backupReferences int64}-func (r *countingReferenceReporter) GetStorageAttachedDatabasesIDs(uuid.UUID) ([]uuid.UUID, error) {- return nil, nil-}--func (r *countingReferenceReporter) GetStorageBackupReferences(uuid.UUID) (int64, error) {- return r.backupReferences, nil+func (c *countingBackupCounter) GetStorageBackupReferences(uuid.UUID) (int64, error) {+ return c.backupReferences, nil}diff --git a/backend/internal/features/storages/di.go b/backend/internal/features/storages/di.goindex 46fbd62a..00eda3e8 100644--- a/backend/internal/features/storages/di.go+++ b/backend/internal/features/storages/di.go@@ -18,6 +18,7 @@ var (audit_logs.GetAuditLogService(),encryption.GetFieldEncryptor(),nil,+ nil,})diff --git a/backend/internal/features/storages/interfaces.go b/backend/internal/features/storages/interfaces.goindex 6e51ebe6..09d97bd3 100644--- a/backend/internal/features/storages/interfaces.go+++ b/backend/internal/features/storages/interfaces.go@@ -45,11 +45,13 @@ type StorageFileSaver interface {EncryptSensitiveData(encryptor encryption.FieldEncryptor) error}-// StorageReferenceReporter answers both questions a storage has to ask before it-// can be deleted: whether a database still points at it, and whether backup rows-// still name files in it. Those rows are the only record of the file names, so-// removing the storage while they exist would make the objects unreachable.-type StorageReferenceReporter interface {+type StorageDatabaseCounter interface {GetStorageAttachedDatabasesIDs(storageID uuid.UUID) ([]uuid.UUID, error)+}++// Backup rows are the only record of the file names in a storage, so removing the+// storage while they exist would make the objects unreachable. Each backup feature+// counts its own rows, because the tables belong to it.+type StorageBackupCounter interface {GetStorageBackupReferences(storageID uuid.UUID) (int64, error)}diff --git a/backend/internal/features/storages/service.go b/backend/internal/features/storages/service.goindex bdddd336..a316955c 100644--- a/backend/internal/features/storages/service.go+++ b/backend/internal/features/storages/service.go@@ -15,24 +15,27 @@ import ()type StorageService struct {- storageRepository *StorageRepository- workspaceService *workspaces_services.WorkspaceService- auditLogService *audit_logs.AuditLogService- fieldEncryptor encryption.FieldEncryptor- storageReferenceReporters []StorageReferenceReporter+ storageRepository *StorageRepository+ workspaceService *workspaces_services.WorkspaceService+ auditLogService *audit_logs.AuditLogService+ fieldEncryptor encryption.FieldEncryptor+ storageDatabaseCounters []StorageDatabaseCounter+ storageBackupCounters []StorageBackupCounter}-func (s *StorageService) AddStorageReferenceReporter(- reporter StorageReferenceReporter,-) {- s.storageReferenceReporters = append(s.storageReferenceReporters, reporter)+func (s *StorageService) AddStorageDatabaseCounter(counter StorageDatabaseCounter) {+ s.storageDatabaseCounters = append(s.storageDatabaseCounters, counter)+}++func (s *StorageService) AddStorageBackupCounter(counter StorageBackupCounter) {+ s.storageBackupCounters = append(s.storageBackupCounters, counter)}func (s *StorageService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {var total int64- for _, reporter := range s.storageReferenceReporters {- count, err := reporter.GetStorageBackupReferences(storageID)+ for _, counter := range s.storageBackupCounters {+ count, err := counter.GetStorageBackupReferences(storageID)if err != nil {return 0, err}@@ -49,7 +52,7 @@ func (s *StorageService) GetStorageAttachedDatabasesIDs(seen := make(map[uuid.UUID]struct{})merged := make([]uuid.UUID, 0)- for _, counter := range s.storageReferenceReporters {+ for _, counter := range s.storageDatabaseCounters {ids, err := counter.GetStorageAttachedDatabasesIDs(storageID)if err != nil {return nil, errdiff --git a/backend/internal/features/storages/testing.go b/backend/internal/features/storages/testing.goindex a9fa432f..860739e1 100644--- a/backend/internal/features/storages/testing.go+++ b/backend/internal/features/storages/testing.go@@ -23,8 +23,12 @@ func DrainStorageFileDeletions(return storage_files.NewDeletionWorker(storageFileStore, dependencies).DrainForTest(ctx, references...)}-func SetStorageReferenceReportersForTest(reporters ...StorageReferenceReporter) {- storageService.storageReferenceReporters = reporters+func SetStorageDatabaseCountersForTest(counters ...StorageDatabaseCounter) {+ storageService.storageDatabaseCounters = counters+}++func SetStorageBackupCountersForTest(counters ...StorageBackupCounter) {+ storageService.storageBackupCounters = counters}func CreateTestStorage(workspaceID uuid.UUID) *Storage {diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.mdindex 57727479..3c98bfd8 100644--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md@@ -17,7 +17,7 @@- [x] 3.1 Implement one deletion-worker pass that takes the live-write mutex, reads the set under it and holds it until its claim statement returns, so no writer can join in between; that statement claims a bounded batch of due rows with skip-locked row selection ordered by `not_before`, excludes the set and increments the generation, commits the claim and only then performs remote I/O; verify a live write is never claimed even when it runs longer than the scan interval or when a cascading deletion request pulls its row forward to now, a file inherited from a stopped process is claimed on the first pass, a claim invalidates an outstanding receipt and an open deletion request does not block the batch.- [x] 3.2 Complete a successful deletion by removing the row and reschedule a failed one with capped exponential backoff, jitter, the incremented attempt count and a sanitized error, both conditional on the generation it claimed; verify transient failure, process restart, an attempt that outlived its timeout, an attempt whose row and storage were both removed by a storage deletion while it ran, and eventual success. Since `not_before` is compared against the database clock, drive these tests with the shrunk constants from task 1.4, a deterministic jitter source and fixtures that write `not_before` directly, not with a Go-side clock.- [x] 3.3 Persist only bounded, sanitized deletion errors and emit attempt, retry and success logs carrying `job_name`, a fresh `job_id`, `storage_id`, `file_name` and `error` as key-value pairs, with the attempt count and the next retry time in the message text and the per-pass pending count, overdue count and oldest-obligation age in the summary message, following the value-in-message rule in `backend/AGENTS.md`; verify centralized redaction and truncation preserve these fields while removing credentials, storage configuration and signed URLs.-- [x] 3.4 Add a non-overlapping `Run` guard and a single-pass execution seam, then register graceful startup and shutdown with the application lifecycle in `backend/cmd/main.go`; verify a second `Run` panics, overlapping ticks are skipped and shutdown does not leave an unjoined in-process provider call.+- [x] 3.4 Add a non-overlapping `Run` guard and a single-pass execution seam, then register graceful startup and shutdown with the application lifecycle in `backend/cmd/main.go`; verify a second `Run` panics, that overlapping ticks are skipped and that a cancelled context ends the loop only after the pass in flight has finished its provider call. The worker is started with a bare `go`, the same as every other background service in `cmd/main.go`, so nothing joins it at shutdown.## 4. Enforce the provider cleanup contract@@ -53,13 +53,13 @@- [x] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.- [x] 8.2 Change physical backup, WAL, history and metadata cleanup, `removeUploadedArtifactsAfterChainBroken` (`create_full_uc.go:192` and `:198`, whose manifest name also comes from the pre-loop file name, and whose doc comment at `:179` restates its name and promises that `DeleteFile` is idempotent on not-found, which stops being what the call does, so delete both claims and keep only what survives), `DeleteChainDependentsKeepFull`, `DeleteOrphanWalSegmentsInSpan`, `deleteOrphanedHistoryFiles` and retention paths to request exact references transactionally, removing both the fail-open and fail-closed remote deletion behavior and the remote calls that currently run inside the cascade transaction; verify cascade and partial-batch rollback cases, including that a rolled-back cascade leaves every object intact.- [x] 8.3 Add clear-path tests that create real local and MinIO artifacts, invoke each cleaner, run the deletion worker until idle and assert both catalog removal and physical absence; include one transient deletion failure to prove the catalog may disappear before eventual remote cleanup succeeds.-- [x] 8.4 Keep the `OnBeforeDatabaseRemove` signature and add a physical implementation instead. The contract change was meant to make database removal record every reference atomically; what actually matters is that something records them at all, and nothing did for the physical family: every physical table cascades on `databases.id`, so removing a database dropped the only record of its object names. `PhysicalBackupService` now implements the listener and records the references for its full, incremental, WAL and history rows in a transaction of its own, which is the same shape the logical cleaner already used. Passing the caller's transaction instead would hold one transaction across every backup of a database, which the design's own risk entry names as a cost, so each listener keeps its own.+- [x] 8.4 Keep the `OnBeforeDatabaseRemove` signature and add a physical implementation instead. The contract change was meant to make database removal record every reference atomically. Nothing recorded them at all for the physical family: every physical table cascades on `databases.id`, so removing a database dropped the only record of its object names. `PhysicalBackupService` now implements the listener and records the references for its full, incremental, WAL and history rows in a transaction of its own, which is the same shape the logical cleaner already used. Passing the caller's transaction instead would hold one transaction across every backup of a database, which the design's own risk entry names as a cost, so each listener keeps its own.- [x] 8.5 Nothing to move: `DeleteForTest` runs the same listener chain as production deletion, and with task 8.4 dropped that chain keeps its signature. The fixture therefore records the same obligations as the controller path already.- [x] 8.6 Add a production-code guard that fails tests if backup workflows call provider `SaveFile` or `DeleteFile` directly, once sections 5 through 8 have removed the last of those calls. Scope it to non-test files under `backend/internal/features/backups`, so it flags neither the storages feature's own `Storage` methods, its provider tests and the `--test-storage` probe in `backend/cmd/storage_command.go`, nor the backup test fixtures that legitimately create and remove provider objects, such as the `SaveFile` call in `core/physical/service/service_test.go`; verify the guard by planting a `DeleteFile` call under `backend/internal/features/backups` and confirming the guard fails on it, then removing it and confirming the guard stays silent for the provider calls that live outside its scope.## 9. Guard and drain the storage lifecycle-- [x] 9.1 Refuse storage deletion while backup rows still reference the storage, returning an actionable conflict instead of a silent logical cascade or a raw foreign-key error, by adding a `GetStorageBackupReferences` method to the existing registration mechanism rather than adding a second registry, and renaming `StorageDatabaseCounter`, `AddStorageDatabaseCounter`, the service field `storageDatabaseCounters` and the fixture `SetStorageDatabaseCountersForTest` to `StorageReferenceReporter`, `AddStorageReferenceReporter`, `storageReferenceReporters` and `SetStorageReferenceReportersForTest`; verify controller responses for logical and physical references and that deletion succeeds once the backups are gone.+- [x] 9.1 Refuse storage deletion while backup rows still reference the storage, returning an actionable conflict instead of a silent logical cascade or a raw foreign-key error, by adding a second registration, `AddStorageBackupCounter`, beside the existing `AddStorageDatabaseCounter`. One interface carrying both questions would have made the config services answer for tables the backup services own, which meant reading those tables by name across a feature boundary; each feature counts its own rows instead; verify controller responses for logical and physical references and that deletion succeeds once the backups are gone.- [x] 9.2 Make storage deletion run one drain of that storage's pending deletions first, bounded by a row count and a wall-clock budget and claiming through the same live-write exclusion the worker uses, then write one audit-log entry and a warning naming the storage, how many pending deletion rows are still present for it after the drain and their names up to a fixed limit, counting rows the drain failed on, skipped as locked, never saw because the worker had claimed them or never reached before the budget expired; verify a reachable provider leaves nothing behind, an unreachable one still deletes the storage within the budget and reports each remaining file once, that the entry stays bounded for a storage with thousands of obligations and that the remaining rows disappear with the storage row.- [x] 9.3 Give the storage workspace-deletion listener at `storages/service.go:73` the same drain and reporting path instead of deleting storage rows straight through its repository, leaving `OnBeforeWorkspaceDeletion` as it is and giving the drain a background context. The signature change would have kept the request ID on the drain's audit entry, but it touches three implementations for one log field on a path that already runs after the request that started it; verify workspace deletion succeeds with an unreachable provider, records what it left behind and keeps the existing refusal for a workspace that still contains databases.