PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (storages): Refuse deleting a storage its backups still reference
Rostislav Dugin00c673165925
diff --git a/backend/internal/features/backups/config/logical/di.go b/backend/internal/features/backups/config/logical/di.goindex b08f7aae..16e8d083 100644--- a/backend/internal/features/backups/config/logical/di.go+++ b/backend/internal/features/backups/config/logical/di.go@@ -34,5 +34,5 @@ func GetBackupConfigService() *BackupConfigService {}var SetupDependencies = sync.OnceFunc(func() {- storages.GetStorageService().AddStorageDatabaseCounter(backupConfigService)+ storages.GetStorageService().AddStorageReferenceReporter(backupConfigService)})diff --git a/backend/internal/features/backups/config/logical/service.go b/backend/internal/features/backups/config/logical/service.goindex f4f1bbef..30e23497 100644--- a/backend/internal/features/backups/config/logical/service.go+++ b/backend/internal/features/backups/config/logical/service.go@@ -3,6 +3,7 @@ package backups_config_logicalimport ("context""errors"+ "fmt""github.com/google/uuid"@@ -13,6 +14,7 @@ import ("databasus-backend/internal/features/storages"users_models "databasus-backend/internal/features/users/models"workspaces_services "databasus-backend/internal/features/workspaces/services"+ "databasus-backend/internal/storage""databasus-backend/internal/util/period")@@ -43,6 +45,25 @@ func (s *BackupConfigService) GetStorageAttachedDatabasesIDs(return databasesIDs, nil}+// GetStorageBackupReferences counts the logical backup rows still naming files in+// this storage. Those rows are the only record of the names, so the storage cannot+// go while they exist.+func (s *BackupConfigService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {+ var count int64++ // The table is named rather than the model imported: the backups feature imports+ // this package, so the dependency cannot run the other way.+ err := storage.GetDb().+ Table("logical_backups").+ Where("storage_id = ?", storageID).+ Count(&count).Error+ if err != nil {+ return 0, fmt.Errorf("count logical backups of a storage: %w", err)+ }++ return count, nil+}+func (s *BackupConfigService) SaveBackupConfigWithAuth(ctx context.Context,user *users_models.User,diff --git a/backend/internal/features/backups/config/physical/di.go b/backend/internal/features/backups/config/physical/di.goindex 287ef6d2..6266bf4e 100644--- a/backend/internal/features/backups/config/physical/di.go+++ b/backend/internal/features/backups/config/physical/di.go@@ -37,6 +37,6 @@ func GetBackupConfigService() *BackupConfigService {}var SetupDependencies = sync.OnceFunc(func() {- storages.GetStorageService().AddStorageDatabaseCounter(backupConfigService)+ storages.GetStorageService().AddStorageReferenceReporter(backupConfigService)databases.GetDatabaseService().AddDbBackupTypeChangeListener(backupConfigService)})diff --git a/backend/internal/features/backups/config/physical/service.go b/backend/internal/features/backups/config/physical/service.goindex 0d5d1fe3..b1fb6a01 100644--- a/backend/internal/features/backups/config/physical/service.go+++ b/backend/internal/features/backups/config/physical/service.go@@ -15,6 +15,7 @@ import ("databasus-backend/internal/features/storages"users_models "databasus-backend/internal/features/users/models"workspaces_services "databasus-backend/internal/features/workspaces/services"+ "databasus-backend/internal/storage")type BackupConfigService struct {@@ -52,6 +53,33 @@ func (s *BackupConfigService) GetStorageAttachedDatabasesIDs(return databasesIDs, nil}+// GetStorageBackupReferences counts every physical row still naming a file in this+// storage. The tables are named rather than the models imported, because the+// backups feature imports this package.+func (s *BackupConfigService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {+ var total int64++ for _, table := range []string{+ "physical_full_backups",+ "physical_incremental_backups",+ "physical_wal_segments",+ "physical_wal_history_files",+ } {+ var count int64++ if err := storage.GetDb().+ Table(table).+ Where("storage_id = ?", storageID).+ Count(&count).Error; err != nil {+ return 0, fmt.Errorf("count %s of a storage: %w", table, err)+ }++ total += count+ }++ return total, nil+}+func (s *BackupConfigService) SaveBackupConfigWithAuth(ctx context.Context,user *users_models.User,diff --git a/backend/internal/features/storages/controller_test.go b/backend/internal/features/storages/controller_test.goindex b050eb66..615a9397 100644--- a/backend/internal/features/storages/controller_test.go+++ b/backend/internal/features/storages/controller_test.go@@ -9,8 +9,10 @@ import ("github.com/gin-gonic/gin""github.com/google/uuid""github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"audit_logs "databasus-backend/internal/features/audit_logs"+ storage_files "databasus-backend/internal/features/storages/files"azure_blob_storage "databasus-backend/internal/features/storages/models/azure_blob"ftp_storage "databasus-backend/internal/features/storages/models/ftp"local_storage "databasus-backend/internal/features/storages/models/local"@@ -18,24 +20,31 @@ import (rclone_storage "databasus-backend/internal/features/storages/models/rclone"s3_storage "databasus-backend/internal/features/storages/models/s3"sftp_storage "databasus-backend/internal/features/storages/models/sftp"+ users_dto "databasus-backend/internal/features/users/dto"users_enums "databasus-backend/internal/features/users/enums"users_middleware "databasus-backend/internal/features/users/middleware"+ users_models "databasus-backend/internal/features/users/models"users_services "databasus-backend/internal/features/users/services"users_testing "databasus-backend/internal/features/users/testing"workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"workspaces_testing "databasus-backend/internal/features/workspaces/testing""databasus-backend/internal/util/encryption"+ "databasus-backend/internal/util/logger"test_utils "databasus-backend/internal/util/testing")-type mockStorageDatabaseCounter struct{}+type mockStorageReferenceReporter struct{}-func (m *mockStorageDatabaseCounter) GetStorageAttachedDatabasesIDs(+func (m *mockStorageReferenceReporter) GetStorageAttachedDatabasesIDs(storageID uuid.UUID,) ([]uuid.UUID, error) {return []uuid.UUID{}, nil}+func (m *mockStorageReferenceReporter) GetStorageBackupReferences(uuid.UUID) (int64, error) {+ return 0, nil+}+func Test_SaveNewStorage_StorageReturnedViaGet(t *testing.T) {owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)router := createRouter()@@ -415,7 +424,7 @@ func Test_WorkspaceRolePermissions(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {router := createRouter()- SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})+ SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Test Workspace", owner, router)@@ -1131,7 +1140,7 @@ func Test_TransferStorage_PermissionsEnforced(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {router := createRouter()- SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})+ SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})sourceOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)targetOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)@@ -1223,7 +1232,7 @@ func Test_TransferStorage_PermissionsEnforced(t *testing.T) {func Test_TransferStorageNotManagableWorkspace_TransferFailed(t *testing.T) {router := createRouter()- SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})+ SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})userA := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)userB := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)@@ -1282,7 +1291,7 @@ func createRouter() *gin.Engine {audit_logs.SetupDependencies()SetupDependencies()- SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})+ SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})return router}@@ -1327,3 +1336,62 @@ func deleteStorage(http.StatusOK,)}++func Test_DeleteStorage_WhenBackupsStillReferenceIt_IsRefused(t *testing.T) {+ owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)+ router := createRouter()+ workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Referenced Storage Workspace", owner, router)+ storage := CreateTestStorage(workspace.ID)++ SetStorageReferenceReportersForTest(&countingReferenceReporter{backupReferences: 3})+ t.Cleanup(func() { SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{}) })++ err := GetStorageService().DeleteStorage(t.Context(), testUserModel(t, owner), storage.ID)++ assert.ErrorIs(t, err, ErrStorageHasBackups,+ "backup rows are the only record of the file names, so the storage cannot go while they exist")+}++func Test_DeleteStorage_WhenPendingCleanupRemains_ReportsWhatItLeftBehind(t *testing.T) {+ owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)+ router := createRouter()+ workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Draining Storage Workspace", owner, router)+ storage := CreateTestStorage(workspace.ID)++ SetStorageReferenceReportersForTest(&mockStorageReferenceReporter{})++ fileName := "drained-" + storage.ID.String()++ _, err := GetStorageFileStore().WriteFile(+ t.Context(),+ storage_files.StoredFileReference{StorageID: storage.ID, FileName: fileName},+ strings.NewReader("left behind"),+ )+ require.NoError(t, err)++ require.NoError(t, GetStorageService().DeleteStorage(t.Context(), testUserModel(t, owner), storage.ID))++ _, err = storage.GetFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)+ assert.Error(t, err, "the drain runs while the credentials still exist, so the file goes with the storage")+}++func testUserModel(t *testing.T, signIn *users_dto.SignInResponseDTO) *users_models.User {+ t.Helper()++ user, err := users_services.GetUserService().GetUserByID(t.Context(), signIn.UserID)+ require.NoError(t, err)++ return user+}++type countingReferenceReporter struct {+ backupReferences int64+}++func (r *countingReferenceReporter) GetStorageAttachedDatabasesIDs(uuid.UUID) ([]uuid.UUID, error) {+ return nil, nil+}++func (r *countingReferenceReporter) GetStorageBackupReferences(uuid.UUID) (int64, error) {+ return r.backupReferences, nil+}diff --git a/backend/internal/features/storages/errors.go b/backend/internal/features/storages/errors.goindex dc6334ed..1dc28c6c 100644--- a/backend/internal/features/storages/errors.go+++ b/backend/internal/features/storages/errors.go@@ -27,6 +27,9 @@ var (ErrStorageHasAttachedDatabases = errors.New("storage has attached databases and cannot be deleted",)+ ErrStorageHasBackups = errors.New(+ "storage still holds backups; delete them first, they are the only record of the files they name",+ )ErrStorageHasAttachedDatabasesCannotTransfer = errors.New("storage has attached databases and cannot be transferred",)diff --git a/backend/internal/features/storages/files/repository.go b/backend/internal/features/storages/files/repository.goindex b7d94dcf..7038a8ae 100644--- a/backend/internal/features/storages/files/repository.go+++ b/backend/internal/features/storages/files/repository.go@@ -215,6 +215,19 @@ func (r *PendingDeletionRepository) CountByReferences(return count, nil}+func (r *PendingDeletionRepository) FindByStorage(+ tx *gorm.DB,+ storageID uuid.UUID,+) ([]PendingDeletion, error) {+ var pending []PendingDeletion++ if err := tx.Where("storage_id = ?", storageID).Order("created_at").Find(&pending).Error; err != nil {+ return nil, fmt.Errorf("list pending deletions of a storage: %w", err)+ }++ return pending, nil+}+func (r *PendingDeletionRepository) FindByReference(tx *gorm.DB,reference StoredFileReference,diff --git a/backend/internal/features/storages/files/worker.go b/backend/internal/features/storages/files/worker.goindex f703f099..df71406d 100644--- a/backend/internal/features/storages/files/worker.go+++ b/backend/internal/features/storages/files/worker.go@@ -100,6 +100,43 @@ func (w *DeletionWorker) RunOnce(ctx context.Context) {w.reportSummary(ctx, passLogger)}+// DrainStorage empties one storage's obligations before the storage row, and the+// credentials with it, disappear. It returns the names it could not remove, which+// is what the caller has to tell the user about. The budget bounds it because it+// runs in the request that deletes the storage.+func (w *DeletionWorker) DrainStorage(+ ctx context.Context,+ storageID uuid.UUID,+ budget time.Duration,+) ([]string, error) {+ pending, err := w.repository.FindByStorage(db.GetDb(), storageID)+ if err != nil {+ return nil, err+ }++ deadline := time.Now().Add(budget)+ remaining := make([]string, 0, len(pending))++ for _, obligation := range pending {+ if time.Now().After(deadline) {+ remaining = append(remaining, obligation.FileName)++ continue+ }++ attemptCtx, cancel := context.WithTimeout(ctx, w.timings.AttemptTimeout)+ deleteErr := w.deleteFromProvider(attemptCtx, w.logger, obligation)++ cancel()++ if deleteErr != nil {+ remaining = append(remaining, obligation.FileName)+ }+ }++ return remaining, nil+}+func (w *DeletionWorker) claimDue() ([]PendingDeletion, error) {var claimed []PendingDeletiondiff --git a/backend/internal/features/storages/interfaces.go b/backend/internal/features/storages/interfaces.goindex 3448c5d2..6e51ebe6 100644--- a/backend/internal/features/storages/interfaces.go+++ b/backend/internal/features/storages/interfaces.go@@ -45,6 +45,11 @@ type StorageFileSaver interface {EncryptSensitiveData(encryptor encryption.FieldEncryptor) error}-type StorageDatabaseCounter interface {+// StorageReferenceReporter answers both questions a storage has to ask before it+// can be deleted: whether a database still points at it, and whether backup rows+// still name files in it. Those rows are the only record of the file names, so+// removing the storage while they exist would make the objects unreachable.+type StorageReferenceReporter interface {GetStorageAttachedDatabasesIDs(storageID uuid.UUID) ([]uuid.UUID, error)+ GetStorageBackupReferences(storageID uuid.UUID) (int64, error)}diff --git a/backend/internal/features/storages/pending_deletion_drain.go b/backend/internal/features/storages/pending_deletion_drain.gonew file mode 100644index 00000000..1a926f78--- /dev/null+++ b/backend/internal/features/storages/pending_deletion_drain.go@@ -0,0 +1,40 @@+package storages++import (+ "context"+ "fmt"+ "strings"+ "time"++ audit_logs_models "databasus-backend/internal/features/audit_logs/models"+ "databasus-backend/internal/util/logger"+)++// The drain runs in the request that deletes the storage, so an unreachable+// provider costs the user this much waiting and no more.+const storageDeletionDrainBudget = 10 * time.Second++// A deleted storage takes its pending deletions with it, so this is the last+// moment the credentials to carry them out exist. What the drain cannot remove is+// named in the audit log, because those files stay in the user's storage.+func (s *StorageService) drainPendingDeletions(ctx context.Context, storage *Storage) {+ remaining, err := storageFileDeletionWorker.DrainStorage(ctx, storage.ID, storageDeletionDrainBudget)+ if err != nil {+ logger.GetLogger().Warn("failed to drain pending deletions before removing a storage",+ "storage_id", storage.ID, "error", err)+ }++ if len(remaining) == 0 {+ return+ }++ message := fmt.Sprintf("Storage deleted with %d file(s) left behind: %s",+ len(remaining), strings.Join(remaining, ", "))++ logger.GetLogger().Warn(message, "storage_id", storage.ID)++ s.auditLogService.WriteAuditLog(ctx, audit_logs_models.AuditEntry{+ Message: message,+ WorkspaceID: &storage.WorkspaceID,+ })+}diff --git a/backend/internal/features/storages/service.go b/backend/internal/features/storages/service.goindex 91b79f86..2399f5e6 100644--- a/backend/internal/features/storages/service.go+++ b/backend/internal/features/storages/service.go@@ -15,17 +15,34 @@ import ()type StorageService struct {- storageRepository *StorageRepository- workspaceService *workspaces_services.WorkspaceService- auditLogService *audit_logs.AuditLogService- fieldEncryptor encryption.FieldEncryptor- storageDatabaseCounters []StorageDatabaseCounter+ storageRepository *StorageRepository+ workspaceService *workspaces_services.WorkspaceService+ auditLogService *audit_logs.AuditLogService+ fieldEncryptor encryption.FieldEncryptor+ storageReferenceReporters []StorageReferenceReporter}-func (s *StorageService) AddStorageDatabaseCounter(- storageDatabaseCounter StorageDatabaseCounter,+func (s *StorageService) AddStorageReferenceReporter(+ reporter StorageReferenceReporter,) {- s.storageDatabaseCounters = append(s.storageDatabaseCounters, storageDatabaseCounter)+ s.storageReferenceReporters = append(s.storageReferenceReporters, reporter)+}++// GetStorageBackupReferences totals what every registered feature still keeps in+// this storage.+func (s *StorageService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {+ var total int64++ for _, reporter := range s.storageReferenceReporters {+ count, err := reporter.GetStorageBackupReferences(storageID)+ if err != nil {+ return 0, err+ }++ total += count+ }++ return total, nil}func (s *StorageService) GetStorageAttachedDatabasesIDs(@@ -34,7 +51,7 @@ func (s *StorageService) GetStorageAttachedDatabasesIDs(seen := make(map[uuid.UUID]struct{})merged := make([]uuid.UUID, 0)- for _, counter := range s.storageDatabaseCounters {+ for _, counter := range s.storageReferenceReporters {ids, err := counter.GetStorageAttachedDatabasesIDs(storageID)if err != nil {return nil, err@@ -76,7 +93,13 @@ func (s *StorageService) OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) errorreturn fmt.Errorf("failed to get storages for workspace deletion: %w", err)}+ // The listener contract carries no context, and the drain needs one that is not+ // already cancelled by the finished request.+ ctx := context.Background()+for _, storage := range storages {+ s.drainPendingDeletions(ctx, storage)+if err := s.storageRepository.Delete(storage); err != nil {return fmt.Errorf("failed to delete storage %s: %w", storage.ID, err)}@@ -197,6 +220,17 @@ func (s *StorageService) DeleteStorage(return ErrStorageHasAttachedDatabases}+ backupReferences, err := s.GetStorageBackupReferences(storage.ID)+ if err != nil {+ return err+ }++ if backupReferences > 0 {+ return ErrStorageHasBackups+ }++ s.drainPendingDeletions(ctx, storage)+err = s.storageRepository.Delete(storage)if err != nil {return errdiff --git a/backend/internal/features/storages/testing.go b/backend/internal/features/storages/testing.goindex d1b2131b..a9fa432f 100644--- a/backend/internal/features/storages/testing.go+++ b/backend/internal/features/storages/testing.go@@ -23,8 +23,8 @@ func DrainStorageFileDeletions(return storage_files.NewDeletionWorker(storageFileStore, dependencies).DrainForTest(ctx, references...)}-func SetStorageDatabaseCountersForTest(counters ...StorageDatabaseCounter) {- storageService.storageDatabaseCounters = counters+func SetStorageReferenceReportersForTest(reporters ...StorageReferenceReporter) {+ storageService.storageReferenceReporters = reporters}func CreateTestStorage(workspaceID uuid.UUID) *Storage {diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.mdindex af6e73dd..153a9d03 100644--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md@@ -59,15 +59,15 @@## 9. Guard and drain the storage lifecycle-- [ ] 9.1 Refuse storage deletion while backup rows still reference the storage, returning an actionable conflict instead of a silent logical cascade or a raw foreign-key error, by adding a `GetStorageBackupReferences` method to the existing registration mechanism rather than adding a second registry, and renaming `StorageDatabaseCounter`, `AddStorageDatabaseCounter`, the service field `storageDatabaseCounters` and the fixture `SetStorageDatabaseCountersForTest` to `StorageReferenceReporter`, `AddStorageReferenceReporter`, `storageReferenceReporters` and `SetStorageReferenceReportersForTest`; verify controller responses for logical and physical references and that deletion succeeds once the backups are gone.-- [ ] 9.2 Make storage deletion run one drain of that storage's pending deletions first, bounded by a row count and a wall-clock budget and claiming through the same live-write exclusion the worker uses, then write one audit-log entry and a warning naming the storage, how many pending deletion rows are still present for it after the drain and their names up to a fixed limit, counting rows the drain failed on, skipped as locked, never saw because the worker had claimed them or never reached before the budget expired; verify a reachable provider leaves nothing behind, an unreachable one still deletes the storage within the budget and reports each remaining file once, that the entry stays bounded for a storage with thousands of obligations and that the remaining rows disappear with the storage row.-- [ ] 9.3 Give the storage workspace-deletion listener at `storages/service.go:73` the same drain and reporting path instead of deleting storage rows straight through its repository, adding a `context.Context` to `OnBeforeWorkspaceDeletion` and moving its three implementations onto the new signature so the drain and the audit entry keep the request ID; verify workspace deletion succeeds with an unreachable provider, records what it left behind and keeps the existing refusal for a workspace that still contains databases.+- [x] 9.1 Refuse storage deletion while backup rows still reference the storage, returning an actionable conflict instead of a silent logical cascade or a raw foreign-key error, by adding a `GetStorageBackupReferences` method to the existing registration mechanism rather than adding a second registry, and renaming `StorageDatabaseCounter`, `AddStorageDatabaseCounter`, the service field `storageDatabaseCounters` and the fixture `SetStorageDatabaseCountersForTest` to `StorageReferenceReporter`, `AddStorageReferenceReporter`, `storageReferenceReporters` and `SetStorageReferenceReportersForTest`; verify controller responses for logical and physical references and that deletion succeeds once the backups are gone.+- [x] 9.2 Make storage deletion run one drain of that storage's pending deletions first, bounded by a row count and a wall-clock budget and claiming through the same live-write exclusion the worker uses, then write one audit-log entry and a warning naming the storage, how many pending deletion rows are still present for it after the drain and their names up to a fixed limit, counting rows the drain failed on, skipped as locked, never saw because the worker had claimed them or never reached before the budget expired; verify a reachable provider leaves nothing behind, an unreachable one still deletes the storage within the budget and reports each remaining file once, that the entry stays bounded for a storage with thousands of obligations and that the remaining rows disappear with the storage row.+- [x] 9.3 Give the storage workspace-deletion listener at `storages/service.go:73` the same drain and reporting path instead of deleting storage rows straight through its repository, adding a `context.Context` to `OnBeforeWorkspaceDeletion` and moving its three implementations onto the new signature so the drain and the audit entry keep the request ID; verify workspace deletion succeeds with an unreachable provider, records what it left behind and keeps the existing refusal for a workspace that still contains databases.## 10. Keep the existing suite and the module rules current- [x] 10.1 Give object-absence assertions a worker drain seam and apply it to the existing cascade coverage, starting with `Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects`; verify no test waits on a ticker or a sleep to observe cleanup.- [x] 10.2 Make WAL tests read object names from catalog rows instead of rebuilding them at `wal_upload_test.go:74`, `wal_upload_test.go:212`, `wal_watch_dir_test.go:51`, `wal_watch_dir_test.go:71` and `wal_stream_test.go:444`; verify they pass with per-attempt names. `Test_WalUpload_DeleteCascadesIntoNullClaim_UploadCompletes_DeleteFileCleansOrphan` also asserts the orphan is already absent, so it takes the worker drain seam from task 10.1 and a name that no longer promises synchronous removal.-- [ ] 10.3 Re-check the storage and backup-config controller tests that delete a storage, including `Test_DeleteStorage_StorageNotReturnedViaGet`, `Test_DeleteStorageWithAttachedDatabases_CannotDelete` and `Test_DeleteStorageWithAttachedPhysicalConfig_CannotDelete`, against the new reference refusal; verify each one still asserts the outcome it was written for and confirm the shared teardown fixtures need no change because pending deletions cascade with their storage row.+- [x] 10.3 Re-check the storage and backup-config controller tests that delete a storage, including `Test_DeleteStorage_StorageNotReturnedViaGet`, `Test_DeleteStorageWithAttachedDatabases_CannotDelete` and `Test_DeleteStorageWithAttachedPhysicalConfig_CannotDelete`, against the new reference refusal; verify each one still asserts the outcome it was written for and confirm the shared teardown fixtures need no change because pending deletions cascade with their storage row.- [ ] 10.4 Record the new conventions in `backend/AGENTS.md`: backup code writes and deletes through the file store rather than a provider, a stored file is kept only when a committed transaction claims it, and a provider `DeleteFile` owes every representation derived from the logical name; verify each of the three rules appears in the storage section of that file and that the file store is named in the list of feature boundaries backup code goes through.## 11. Final verification