Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Remove stored files left by failed logical backups

Rostislav Dugin554dfc781176
diff --git a/backend/cmd/main.go b/backend/cmd/main.go
index 167c7593..27367901 100644
--- a/backend/cmd/main.go
+++ b/backend/cmd/main.go
@@ -376,6 +376,10 @@ func runBackgroundTasks(log *slog.Logger) {
backuping_physical.GetPhysicalWalStreamSupervisor().Run(ctx)
})
+ go runWithPanicLogging(log, "storage file deletion background service", func() {
+ storages.GetStorageFileDeletionWorker().Run(ctx)
+ })
+
go runWithPanicLogging(log, "telemetry background service", func() {
telemetry.GetTelemetryBackgroundService().Run(ctx)
})
diff --git a/backend/internal/features/backups/backups/backuping/logical/backuper.go b/backend/internal/features/backups/backups/backuping/logical/backuper.go
index 333cd5af..93b81167 100644
--- a/backend/internal/features/backups/backups/backuping/logical/backuper.go
+++ b/backend/internal/features/backups/backups/backuping/logical/backuper.go
@@ -12,26 +12,30 @@ import (
"time"
"github.com/google/uuid"
+ "gorm.io/gorm"
backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
backups_config_logical "databasus-backend/internal/features/backups/config/logical"
"databasus-backend/internal/features/databases"
notifier_models "databasus-backend/internal/features/notifiers/models"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
tasks_cancellation "databasus-backend/internal/features/tasks/cancellation"
workspaces_services "databasus-backend/internal/features/workspaces/services"
- util_encryption "databasus-backend/internal/util/encryption"
+ db "databasus-backend/internal/storage"
)
-const partialBackupCleanupTimeout = 30 * time.Second
+const (
+ metadataWriteTimeout = 30 * time.Second
+
+ metadataSuffix = ".metadata"
+)
type Backuper struct {
databaseService *databases.DatabaseService
- fieldEncryptor util_encryption.FieldEncryptor
workspaceService *workspaces_services.WorkspaceService
backupRepository *backups_core_logical.BackupRepository
backupConfigService *backups_config_logical.BackupConfigService
- storageService *storages.StorageService
+ fileStore *storage_files.Store
notificationSender backups_core_logical.NotificationSender
taskCancellationRegistry *tasks_cancellation.Registry
logger *slog.Logger
@@ -70,12 +74,6 @@ func (b *Backuper) MakeBackup(ctx context.Context, backupID uuid.UUID, isCallNot
b.taskCancellationRegistry.RegisterTask(backup.ID, cancel)
defer b.taskCancellationRegistry.UnregisterTask(backup.ID)
- storage, err := b.storageService.GetStorageByID(executionCtx, *backupConfig.StorageID)
- if err != nil {
- logger.ErrorContext(ctx, "failed to get storage by ID", "error", err)
- return
- }
-
start := time.Now().UTC()
backupProgressListener := func(
@@ -89,12 +87,12 @@ func (b *Backuper) MakeBackup(ctx context.Context, backupID uuid.UUID, isCallNot
}
}
- backupMetadata, err := b.createBackupUseCase.Execute(
+ artifacts, err := b.createBackupUseCase.Execute(
executionCtx,
backup,
backupConfig,
database,
- storage,
+ b.fileStore,
backupProgressListener,
)
if err != nil {
@@ -128,8 +126,7 @@ func (b *Backuper) MakeBackup(ctx context.Context, backupID uuid.UUID, isCallNot
"backup_id", backup.ID,
"database_id", databaseID,
"database_type", database.Type,
- "storage_id", storage.ID,
- "storage_type", storage.Type,
+ "storage_id", backup.StorageID,
"error", err,
)
@@ -150,33 +147,10 @@ func (b *Backuper) MakeBackup(ctx context.Context, backupID uuid.UUID, isCallNot
backup.BackupDurationMs = time.Since(start).Milliseconds()
backup.BackupSizeMb = 0
- if err := b.backupRepository.Save(backup); err != nil {
+ if err := b.saveTerminalStateAndDiscardFiles(ctx, backup); err != nil {
logger.ErrorContext(ctx, "failed to save cancelled backup", "error", err)
}
- // This branch is reached because executionCtx was cancelled, so the cleanup needs a live
- // context of its own or the partial file is never deleted.
- cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), partialBackupCleanupTimeout)
- defer cancelCleanup()
-
- partialBackupStorage, storageErr := b.storageService.GetStorageByID(cleanupCtx, backup.StorageID)
- if storageErr == nil {
- if deleteErr := partialBackupStorage.DeleteFile(
- cleanupCtx,
- b.fieldEncryptor,
- logger,
- backup.FileName,
- ); deleteErr != nil {
- logger.ErrorContext(ctx,
- "failed to delete partial backup file",
- "backup_id",
- backup.ID,
- "error",
- deleteErr,
- )
- }
- }
-
return
}
@@ -193,7 +167,7 @@ func (b *Backuper) MakeBackup(ctx context.Context, backupID uuid.UUID, isCallNot
)
}
- if err := b.backupRepository.Save(backup); err != nil {
+ if err := b.saveTerminalStateAndDiscardFiles(ctx, backup); err != nil {
logger.ErrorContext(ctx, "failed to save backup", "error", err)
}
@@ -212,55 +186,36 @@ func (b *Backuper) MakeBackup(ctx context.Context, backupID uuid.UUID, isCallNot
backup.BackupDurationMs = time.Since(start).Milliseconds()
- // Update backup with encryption metadata if provided
- if backupMetadata != nil {
- backupMetadata.BackupID = backup.ID
+ receipts := artifacts.Receipts
+
+ if artifacts.Metadata != nil {
+ artifacts.Metadata.BackupID = backup.ID
+
+ if err := artifacts.Metadata.Validate(); err != nil {
+ b.failBackup(ctx, logger, backup, backupConfig, fmt.Errorf("validate backup metadata: %w", err))
- if err := backupMetadata.Validate(); err != nil {
- logger.ErrorContext(ctx, "failed to validate backup metadata", "error", err)
return
}
- backup.EncryptionSalt = backupMetadata.EncryptionSalt
- backup.EncryptionIV = backupMetadata.EncryptionIV
- backup.Encryption = backupMetadata.Encryption
- }
+ backup.EncryptionSalt = artifacts.Metadata.EncryptionSalt
+ backup.EncryptionIV = artifacts.Metadata.EncryptionIV
+ backup.Encryption = artifacts.Metadata.Encryption
- if backupMetadata != nil {
- metadataJSON, err := json.Marshal(backupMetadata)
+ metadataReceipt, err := b.writeMetadataFile(backup, artifacts.Metadata)
if err != nil {
- logger.ErrorContext(ctx, "failed to marshal backup metadata to JSON",
- "backup_id", backup.ID,
- "error", err,
- )
- } else {
- metadataReader := bytes.NewReader(metadataJSON)
- metadataFileName := backup.FileName + ".metadata"
-
- // Not executionCtx: the dump itself already succeeded, and a cancel landing here would
- // leave a completed backup whose encryption metadata is missing, making it unrestorable.
- if err := storage.SaveFile(
- context.Background(),
- b.fieldEncryptor,
- logger,
- metadataFileName,
- metadataReader,
- ); err != nil {
- logger.ErrorContext(ctx, "failed to save backup metadata file to storage",
- "backup_id", backup.ID,
- "file_name", metadataFileName,
- "error", err,
- )
- } else {
- logger.DebugContext(ctx, "backup metadata file saved", "file_name", metadataFileName)
- }
+ b.failBackup(ctx, logger, backup, backupConfig, err)
+
+ return
}
+
+ receipts = append(receipts, metadataReceipt)
}
backup.Status = backups_core_logical.BackupStatusCompleted
- if err := b.backupRepository.Save(backup); err != nil {
- logger.ErrorContext(ctx, "failed to save backup", "error", err)
+ if err := b.publishBackup(ctx, backup, receipts); err != nil {
+ b.failBackup(ctx, logger, backup, backupConfig, err)
+
return
}
@@ -378,3 +333,98 @@ func (b *Backuper) SendBackupNotification(
)
}
}
+
+// The sidecar carries its own context because the dump already succeeded: a caller
+// cancelling here would otherwise fail a backup whose expensive part is done.
+func (b *Backuper) writeMetadataFile(
+ backup *backups_core_logical.LogicalBackup,
+ metadata *backups_core_logical.BackupMetadata,
+) (storage_files.WriteReceipt, error) {
+ metadataJSON, err := json.Marshal(metadata)
+ if err != nil {
+ return storage_files.WriteReceipt{}, fmt.Errorf("marshal backup metadata: %w", err)
+ }
+
+ writeCtx, cancel := context.WithTimeout(context.Background(), metadataWriteTimeout)
+ defer cancel()
+
+ receipt, err := b.fileStore.WriteFile(
+ writeCtx,
+ storage_files.StoredFileReference{
+ StorageID: backup.StorageID,
+ FileName: backup.FileName + metadataSuffix,
+ },
+ bytes.NewReader(metadataJSON),
+ )
+ if err != nil {
+ return storage_files.WriteReceipt{}, fmt.Errorf("save backup metadata file: %w", err)
+ }
+
+ return receipt, nil
+}
+
+// Every file the receipts name is kept only if the row marking the backup completed
+// commits with them.
+func (b *Backuper) publishBackup(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ receipts []storage_files.WriteReceipt,
+) error {
+ return db.GetDb().Transaction(func(tx *gorm.DB) error {
+ if err := b.fileStore.ConfirmFileWrites(ctx, tx, receipts); err != nil {
+ return err
+ }
+
+ return b.backupRepository.SaveInTransaction(tx, backup)
+ })
+}
+
+// Per-attempt naming makes the two names exact, so nothing else can be writing
+// them while this runs.
+func (b *Backuper) saveTerminalStateAndDiscardFiles(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+) error {
+ references := []storage_files.StoredFileReference{
+ {StorageID: backup.StorageID, FileName: backup.FileName},
+ {StorageID: backup.StorageID, FileName: backup.FileName + metadataSuffix},
+ }
+
+ // MakeBackup runs on a context detached from the caller, but a terminal state has
+ // to be recorded even when that context is the one that was cancelled.
+ ctx = context.WithoutCancel(ctx)
+
+ return db.GetDb().Transaction(func(tx *gorm.DB) error {
+ if err := b.fileStore.RequestFileDeletions(ctx, tx, references); err != nil {
+ return err
+ }
+
+ return b.backupRepository.SaveInTransaction(tx, backup)
+ })
+}
+
+func (b *Backuper) failBackup(
+ ctx context.Context,
+ logger *slog.Logger,
+ backup *backups_core_logical.LogicalBackup,
+ backupConfig *backups_config_logical.LogicalBackupConfig,
+ cause error,
+) {
+ message := cause.Error()
+
+ backup.Status = backups_core_logical.BackupStatusFailed
+ backup.FailMessage = &message
+ backup.BackupSizeMb = 0
+
+ logger.ErrorContext(ctx, "logical backup failed after the dump", "error", cause)
+
+ if err := b.databaseService.SetBackupError(backup.DatabaseID, message); err != nil {
+ logger.ErrorContext(ctx, "failed to record the backup error on the database", "error", err)
+ }
+
+ if err := b.saveTerminalStateAndDiscardFiles(ctx, backup); err != nil {
+ logger.ErrorContext(ctx, "failed to save failed backup", "error", err)
+ }
+
+ b.SendBackupNotification(ctx, backupConfig, backup, backups_config_logical.NotificationBackupFailed, &message)
+}
diff --git a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go b/backend/internal/features/backups/backups/backuping/logical/backuper_test.go
index 479d2eb4..34311c27 100644
--- a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go
+++ b/backend/internal/features/backups/backups/backuping/logical/backuper_test.go
@@ -2,28 +2,59 @@ package backuping_logical
import (
"context"
+ "errors"
+ "fmt"
"log/slog"
"strings"
"sync"
"testing"
"time"
+ "github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+ backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
backups_core_logical "databasus-backend/internal/features/backups/backups/core/logical"
backups_config_logical "databasus-backend/internal/features/backups/config/logical"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
notifier_models "databasus-backend/internal/features/notifiers/models"
"databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
users_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"databasus-backend/internal/util/cache"
+ "databasus-backend/internal/util/encryption"
util_logger "databasus-backend/internal/util/logger"
)
+const partialLogicalArtifact = "partial logical backup"
+
+type createStoredArtifactThenFailBackupUsecase struct{}
+
+func (*createStoredArtifactThenFailBackupUsecase) Execute(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ _ *backups_config_logical.LogicalBackupConfig,
+ _ *databases.Database,
+ fileStore backups_core_logical.BackupFileStore,
+ _ func(completedMBs float64),
+) (*backups_core_logical.BackupArtifacts, error) {
+ _, err := fileStore.WriteFile(
+ ctx,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ strings.NewReader(partialLogicalArtifact),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return nil, errors.New("database dump failed after producing output")
+}
+
func Test_BackupExecuted_NotificationSent(t *testing.T) {
cache.GetStore().Clear(t.Context())
user := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleAdmin)
@@ -58,6 +89,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
backup := &backups_core_logical.LogicalBackup{
DatabaseID: database.ID,
StorageID: storage.ID,
+ FileName: "notification-" + uuid.New().String(),
Status: backups_core_logical.BackupStatusInProgress,
CreatedAt: time.Now().UTC(),
}
@@ -90,6 +122,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
backup := &backups_core_logical.LogicalBackup{
DatabaseID: database.ID,
StorageID: storage.ID,
+ FileName: "notification-" + uuid.New().String(),
Status: backups_core_logical.BackupStatusInProgress,
CreatedAt: time.Now().UTC(),
}
@@ -122,6 +155,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
backup := &backups_core_logical.LogicalBackup{
DatabaseID: database.ID,
StorageID: storage.ID,
+ FileName: "notification-" + uuid.New().String(),
Status: backups_core_logical.BackupStatusInProgress,
CreatedAt: time.Now().UTC(),
}
@@ -180,6 +214,36 @@ func Test_MakeBackup_WhenCallerContextCarriesRequestID_FinishLineIsAttributed(t
assert.Equal(t, requestID, capturedRequestID)
}
+func Test_MakeBackup_WhenUsecaseStoresArtifactThenFails_ArtifactRemoved(t *testing.T) {
+ fixture := CreateBackupTestFixture(t, "Failed Backup Artifact Workspace")
+ backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)
+ backup.FileName = "failed-logical-" + backup.ID.String()
+ require.NoError(t, backupRepository.Save(backup))
+
+ notificationSender := &MockNotificationSender{}
+ notificationSender.On("SendNotification", mock.Anything, mock.Anything).Maybe()
+
+ backuper := CreateTestBackuperWithUseCase(&createStoredArtifactThenFailBackupUsecase{})
+ backuper.notificationSender = notificationSender
+ backuper.MakeBackup(t.Context(), backup.ID, true)
+
+ persistedBackup, err := backupRepository.FindByID(backup.ID)
+ require.NoError(t, err)
+ assert.Equal(t, backups_core_logical.BackupStatusFailed, persistedBackup.Status)
+
+ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(),
+ storage_files.StoredFileReference{StorageID: fixture.Storage.ID, FileName: backup.FileName},
+ ))
+
+ _, err = fixture.Storage.GetFile(
+ t.Context(),
+ encryption.GetFieldEncryptor(),
+ util_logger.GetLogger(),
+ backup.FileName,
+ )
+ assert.Error(t, err, "a dump that failed after producing output must not leave its artifact behind")
+}
+
// A record logged on the detached execution context is indistinguishable from one that never
// carried a request at all, which is what this pins.
type requestIDCapturingHandler struct {
@@ -214,3 +278,162 @@ func (h *requestIDCapturingHandler) GetRequestIDForMessagePrefix(prefix string)
return "", false
}
+
+// Cancellation and metadata failure are backuper concerns, not engine concerns:
+// every engine reaches storage through the same file store, so the discard logic
+// these cover is the same code for MySQL, MariaDB, MongoDB and PostgreSQL.
+type createStoredArtifactThenCancelBackupUsecase struct{}
+
+func (*createStoredArtifactThenCancelBackupUsecase) Execute(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ _ *backups_config_logical.LogicalBackupConfig,
+ _ *databases.Database,
+ fileStore backups_core_logical.BackupFileStore,
+ _ func(completedMBs float64),
+) (*backups_core_logical.BackupArtifacts, error) {
+ if _, err := fileStore.WriteFile(
+ ctx,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ strings.NewReader(partialLogicalArtifact),
+ ); err != nil {
+ return nil, err
+ }
+
+ return nil, fmt.Errorf("backup cancelled: %w", context.Canceled)
+}
+
+type createUnusableMetadataBackupUsecase struct{}
+
+func (*createUnusableMetadataBackupUsecase) Execute(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ _ *backups_config_logical.LogicalBackupConfig,
+ _ *databases.Database,
+ fileStore backups_core_logical.BackupFileStore,
+ _ func(completedMBs float64),
+) (*backups_core_logical.BackupArtifacts, error) {
+ receipt, err := fileStore.WriteFile(
+ ctx,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ strings.NewReader(partialLogicalArtifact),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ // Encrypted metadata with no salt or IV cannot be restored, so Validate refuses it.
+ return &backups_core_logical.BackupArtifacts{
+ Metadata: &backups_core_logical.BackupMetadata{Encryption: backups_core_enums.BackupEncryptionEncrypted},
+ Receipts: []storage_files.WriteReceipt{receipt},
+ }, nil
+}
+
+func Test_MakeBackup_WhenCancelledAfterStoringArtifact_ArtifactRemoved(t *testing.T) {
+ fixture := CreateBackupTestFixture(t, "Cancelled Backup Artifact Workspace")
+ backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)
+
+ notificationSender := &MockNotificationSender{}
+ notificationSender.On("SendNotification", mock.Anything, mock.Anything).Maybe()
+
+ backuper := CreateTestBackuperWithUseCase(&createStoredArtifactThenCancelBackupUsecase{})
+ backuper.notificationSender = notificationSender
+ backuper.MakeBackup(t.Context(), backup.ID, true)
+
+ persistedBackup, err := backupRepository.FindByID(backup.ID)
+ require.NoError(t, err)
+ assert.Equal(t, backups_core_logical.BackupStatusCanceled, persistedBackup.Status)
+
+ assertBackupFilesRemoved(t, fixture, backup.FileName)
+}
+
+func Test_MakeBackup_WhenMetadataIsUnusable_BackupFailsAndArtifactRemoved(t *testing.T) {
+ fixture := CreateBackupTestFixture(t, "Unusable Metadata Workspace")
+ backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)
+
+ notificationSender := &MockNotificationSender{}
+ notificationSender.On("SendNotification", mock.Anything, mock.Anything).Maybe()
+
+ backuper := CreateTestBackuperWithUseCase(&createUnusableMetadataBackupUsecase{})
+ backuper.notificationSender = notificationSender
+ backuper.MakeBackup(t.Context(), backup.ID, true)
+
+ persistedBackup, err := backupRepository.FindByID(backup.ID)
+ require.NoError(t, err)
+ assert.Equal(t, backups_core_logical.BackupStatusFailed, persistedBackup.Status,
+ "a backup whose metadata cannot be restored from must not be published")
+
+ assertBackupFilesRemoved(t, fixture, backup.FileName)
+}
+
+// The caller names the files its scenario actually produced: asserting the absence
+// of a file nothing ever wrote would pass with no cleanup at all.
+// A receipt goes stale when something claims the file between the upload and the
+// publishing transaction. Bumping the generation by hand reproduces that without a
+// second goroutine racing the test.
+type createStaleReceiptBackupUsecase struct{}
+
+func (*createStaleReceiptBackupUsecase) Execute(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ _ *backups_config_logical.LogicalBackupConfig,
+ _ *databases.Database,
+ fileStore backups_core_logical.BackupFileStore,
+ _ func(completedMBs float64),
+) (*backups_core_logical.BackupArtifacts, error) {
+ receipt, err := fileStore.WriteFile(
+ ctx,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ strings.NewReader(partialLogicalArtifact),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ receipt.Generation++
+
+ return &backups_core_logical.BackupArtifacts{
+ Metadata: &backups_core_logical.BackupMetadata{Encryption: backups_core_enums.BackupEncryptionNone},
+ Receipts: []storage_files.WriteReceipt{receipt},
+ }, nil
+}
+
+func Test_MakeBackup_WhenPublicationFails_EveryFileOfTheAttemptIsRemoved(t *testing.T) {
+ fixture := CreateBackupTestFixture(t, "Failed Publication Workspace")
+ backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)
+
+ notificationSender := &MockNotificationSender{}
+ notificationSender.On("SendNotification", mock.Anything, mock.Anything).Maybe()
+
+ backuper := CreateTestBackuperWithUseCase(&createStaleReceiptBackupUsecase{})
+ backuper.notificationSender = notificationSender
+ backuper.MakeBackup(t.Context(), backup.ID, true)
+
+ persistedBackup, err := backupRepository.FindByID(backup.ID)
+ require.NoError(t, err)
+ assert.Equal(t, backups_core_logical.BackupStatusFailed, persistedBackup.Status,
+ "a backup whose files cannot be claimed must not be published")
+
+ assertBackupFilesRemoved(t, fixture, backup.FileName, backup.FileName+metadataSuffix)
+}
+
+func assertBackupFilesRemoved(t *testing.T, fixture *BackupTestFixture, fileNames ...string) {
+ t.Helper()
+
+ references := make([]storage_files.StoredFileReference, 0, len(fileNames))
+ for _, name := range fileNames {
+ references = append(references, storage_files.StoredFileReference{
+ StorageID: fixture.Storage.ID,
+ FileName: name,
+ })
+ }
+
+ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(), references...))
+
+ for _, name := range fileNames {
+ _, err := fixture.Storage.GetFile(
+ t.Context(), encryption.GetFieldEncryptor(), util_logger.GetLogger(), name,
+ )
+ assert.Error(t, err, "%s must not survive a backup that never published", name)
+ }
+}
diff --git a/backend/internal/features/backups/backups/backuping/logical/cleaner.go b/backend/internal/features/backups/backups/backuping/logical/cleaner.go
index 30ff36e1..aa07470d 100644
--- a/backend/internal/features/backups/backups/backuping/logical/cleaner.go
+++ b/backend/internal/features/backups/backups/backuping/logical/cleaner.go
@@ -93,7 +93,7 @@ func (c *BackupCleaner) DeleteBackup(ctx context.Context, backup *backups_core_l
logger.WarnContext(ctx, "failed to delete backup file", "error", err)
}
- metadataFileName := backup.FileName + ".metadata"
+ metadataFileName := backup.FileName + metadataSuffix
if err := storage.DeleteFile(ctx, c.fieldEncryptor, logger, metadataFileName); err != nil {
logger.WarnContext(ctx, "failed to delete backup metadata file", "error", err)
}
diff --git a/backend/internal/features/backups/backups/backuping/logical/di.go b/backend/internal/features/backups/backups/backuping/logical/di.go
index 1f4a7302..859a9feb 100644
--- a/backend/internal/features/backups/backups/backuping/logical/di.go
+++ b/backend/internal/features/backups/backups/backuping/logical/di.go
@@ -35,11 +35,10 @@ var backupCleaner = &BackupCleaner{
var backuper = &Backuper{
databases.GetDatabaseService(),
- encryption.GetFieldEncryptor(),
workspaces_services.GetWorkspaceService(),
backupRepository,
backups_config_logical.GetBackupConfigService(),
- storages.GetStorageService(),
+ storages.GetStorageFileStore(),
notifiers.GetNotifierService(),
taskCancellationRegistry,
logger.GetLogger(),
diff --git a/backend/internal/features/backups/backups/backuping/logical/mocks.go b/backend/internal/features/backups/backups/backuping/logical/mocks.go
index efe42149..9287c6eb 100644
--- a/backend/internal/features/backups/backups/backuping/logical/mocks.go
+++ b/backend/internal/features/backups/backups/backuping/logical/mocks.go
@@ -3,6 +3,7 @@ package backuping_logical
import (
"context"
"errors"
+ "strings"
"sync/atomic"
"time"
@@ -15,7 +16,7 @@ import (
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
notifier_models "databasus-backend/internal/features/notifiers/models"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
)
type MockNotificationSender struct {
@@ -30,6 +31,8 @@ func (m *MockNotificationSender) SendNotification(
m.Called(notifier, notification)
}
+const FakeBackupArtifact = "fake logical backup"
+
type CreateFailedBackupUsecase struct{}
func (uc *CreateFailedBackupUsecase) Execute(
@@ -37,9 +40,9 @@ func (uc *CreateFailedBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
database *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
backupProgressListener(10)
return nil, errors.New("backup failed")
}
@@ -51,15 +54,12 @@ func (uc *CreateSuccessBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
database *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
backupProgressListener(10)
- return &backups_core_logical.BackupMetadata{
- EncryptionSalt: nil,
- EncryptionIV: nil,
- Encryption: backups_core_enums.BackupEncryptionNone,
- }, nil
+
+ return writeFakeBackupArtifact(ctx, backup, fileStore)
}
// CreateLargeBackupUsecase simulates a large backup (10000 MB)
@@ -70,15 +70,12 @@ func (uc *CreateLargeBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
database *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
backupProgressListener(10000)
- return &backups_core_logical.BackupMetadata{
- EncryptionSalt: nil,
- EncryptionIV: nil,
- Encryption: backups_core_enums.BackupEncryptionNone,
- }, nil
+
+ return writeFakeBackupArtifact(ctx, backup, fileStore)
}
// CreateProgressiveBackupUsecase simulates progressive size updates that exceed limit
@@ -89,9 +86,9 @@ func (uc *CreateProgressiveBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
database *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
// Simulate progressive backup that grows beyond limit
backupProgressListener(1)
if ctx.Err() != nil {
@@ -114,11 +111,7 @@ func (uc *CreateProgressiveBackupUsecase) Execute(
}
// Should not reach here due to cancellation
- return &backups_core_logical.BackupMetadata{
- EncryptionSalt: nil,
- EncryptionIV: nil,
- Encryption: backups_core_enums.BackupEncryptionNone,
- }, nil
+ return writeFakeBackupArtifact(ctx, backup, fileStore)
}
// CreateMediumBackupUsecase simulates a 50 MB backup
@@ -129,15 +122,11 @@ func (uc *CreateMediumBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
database *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
backupProgressListener(50)
- return &backups_core_logical.BackupMetadata{
- EncryptionSalt: nil,
- EncryptionIV: nil,
- Encryption: backups_core_enums.BackupEncryptionNone,
- }, nil
+ return writeFakeBackupArtifact(ctx, backup, fileStore)
}
// MockTrackingBackupUsecase tracks backup use case calls for testing parallel execution
@@ -157,9 +146,9 @@ func (m *MockTrackingBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
database *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
m.callCount.Add(1)
// Send backup ID to channel (non-blocking)
@@ -172,11 +161,7 @@ func (m *MockTrackingBackupUsecase) Execute(
time.Sleep(100 * time.Millisecond)
backupProgressListener(10)
- return &backups_core_logical.BackupMetadata{
- EncryptionSalt: nil,
- EncryptionIV: nil,
- Encryption: backups_core_enums.BackupEncryptionNone,
- }, nil
+ return writeFakeBackupArtifact(ctx, backup, fileStore)
}
func (m *MockTrackingBackupUsecase) GetCallCount() int32 {
@@ -194,3 +179,26 @@ func (m *MockTrackingBackupUsecase) GetCalledBackupIDs() []uuid.UUID {
}
}
}
+
+// The fakes stand in for a database dump, so they write a real file through the
+// store: a backuper test that never produced an artifact could not tell a kept
+// file from a cleaned one.
+func writeFakeBackupArtifact(
+ ctx context.Context,
+ backup *backups_core_logical.LogicalBackup,
+ fileStore backups_core_logical.BackupFileStore,
+) (*backups_core_logical.BackupArtifacts, error) {
+ receipt, err := fileStore.WriteFile(
+ ctx,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ strings.NewReader(FakeBackupArtifact),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return &backups_core_logical.BackupArtifacts{
+ Metadata: &backups_core_logical.BackupMetadata{Encryption: backups_core_enums.BackupEncryptionNone},
+ Receipts: []storage_files.WriteReceipt{receipt},
+ }, nil
+}
diff --git a/backend/internal/features/backups/backups/backuping/logical/testing.go b/backend/internal/features/backups/backups/backuping/logical/testing.go
index 6c717875..ba1cd345 100644
--- a/backend/internal/features/backups/backups/backuping/logical/testing.go
+++ b/backend/internal/features/backups/backups/backuping/logical/testing.go
@@ -68,10 +68,15 @@ func SeedInProgressTestBackup(
) *backups_core_logical.LogicalBackup {
t.Helper()
+ backupID := uuid.New()
+
+ // The scheduler names a backup before anything writes to storage, so a seeded
+ // row without a name is a state production never reaches.
return seedBackup(t, "in-progress", &backups_core_logical.LogicalBackup{
- ID: uuid.New(),
+ ID: backupID,
DatabaseID: databaseID,
StorageID: storageID,
+ FileName: "seeded-" + backupID.String(),
Status: backups_core_logical.BackupStatusInProgress,
CreatedAt: time.Now().UTC(),
})
@@ -145,11 +150,10 @@ func CreateTestBackupCleaner() *BackupCleaner {
func CreateTestBackuper() *Backuper {
return &Backuper{
databases.GetDatabaseService(),
- encryption.GetFieldEncryptor(),
workspaces_services.GetWorkspaceService(),
backupRepository,
backups_config_logical.GetBackupConfigService(),
- storages.GetStorageService(),
+ storages.GetStorageFileStore(),
notifiers.GetNotifierService(),
taskCancellationRegistry,
logger.GetLogger(),
@@ -160,11 +164,10 @@ func CreateTestBackuper() *Backuper {
func CreateTestBackuperWithUseCase(useCase backups_core_logical.CreateBackupUsecase) *Backuper {
return &Backuper{
databases.GetDatabaseService(),
- encryption.GetFieldEncryptor(),
workspaces_services.GetWorkspaceService(),
backupRepository,
backups_config_logical.GetBackupConfigService(),
- storages.GetStorageService(),
+ storages.GetStorageFileStore(),
notifiers.GetNotifierService(),
taskCancellationRegistry,
logger.GetLogger(),
diff --git a/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go b/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go
index d1f385dc..2b198a0c 100644
--- a/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go
+++ b/backend/internal/features/backups/backups/backuping/physical/scheduler_test.go
@@ -2,6 +2,7 @@ package backuping_physical
import (
"context"
+ "strings"
"sync"
"testing"
"time"
@@ -18,6 +19,7 @@ import (
postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
"databasus-backend/internal/features/intervals"
"databasus-backend/internal/storage"
+ "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/walmath"
)
@@ -817,6 +819,53 @@ func Test_RecoverInFlightOnRestart_FailsAndReleasesClaim(t *testing.T) {
assert.Nil(t, claim, "a backup orphaned by restart must be failed and released")
}
+func Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact(t *testing.T) {
+ prereqs := seedBackupPrereqs(t)
+ scheduler := CreateTestPhysicalScheduler()
+ backupID := seedInProgressFullWithClaim(t, prereqs)
+ fileName := "restart-interrupted-full-" + backupID.String()
+
+ fullBackup, err := physical_repositories.GetFullBackupRepository().FindByID(backupID)
+ require.NoError(t, err)
+ require.NotNil(t, fullBackup)
+ fullBackup.FileName = &fileName
+ require.NoError(t, physical_repositories.GetFullBackupRepository().Save(fullBackup))
+ require.NoError(t, prereqs.Storage.SaveFile(
+ t.Context(),
+ encryption.GetFieldEncryptor(),
+ logger.GetLogger(),
+ fileName,
+ strings.NewReader("partial physical backup"),
+ ))
+
+ t.Cleanup(func() {
+ require.NoError(t, prereqs.Storage.DeleteFile(
+ context.Background(),
+ encryption.GetFieldEncryptor(),
+ logger.GetLogger(),
+ fileName,
+ ))
+ })
+
+ require.NoError(t, scheduler.recoverInFlightBackupsOnRestart(t.Context(), logger.GetLogger()))
+
+ persistedFullBackup, err := physical_repositories.GetFullBackupRepository().FindByID(backupID)
+ require.NoError(t, err)
+ require.NotNil(t, persistedFullBackup)
+ assert.Equal(t, physical_enums.PhysicalBackupStatusError, persistedFullBackup.Status)
+ require.NotNil(t, persistedFullBackup.FileName)
+ assert.Equal(t, fileName, *persistedFullBackup.FileName)
+
+ artifactReader, err := prereqs.Storage.GetFile(
+ t.Context(),
+ encryption.GetFieldEncryptor(),
+ logger.GetLogger(),
+ fileName,
+ )
+ require.NoError(t, err)
+ require.NoError(t, artifactReader.Close())
+}
+
func Test_ClaimAndInsert_WhenConcurrentFullAndIncrSameDatabase_OnlyOneSucceeds(t *testing.T) {
prereqs := seedBackupPrereqs(t)
// The INCR claimer needs a real chain root to satisfy the FK; the FULL claimer
diff --git a/backend/internal/features/backups/backups/core/logical/dto.go b/backend/internal/features/backups/backups/core/logical/dto.go
index 7d1ab545..6ab9a447 100644
--- a/backend/internal/features/backups/backups/core/logical/dto.go
+++ b/backend/internal/features/backups/backups/core/logical/dto.go
@@ -7,6 +7,7 @@ import (
"github.com/google/uuid"
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
+ storage_files "databasus-backend/internal/features/storages/files"
)
type BackupFilters struct {
@@ -42,3 +43,10 @@ func (m *BackupMetadata) Validate() error {
return nil
}
+
+// The receipts are what the publishing transaction spends to keep the files this
+// attempt uploaded.
+type BackupArtifacts struct {
+ Metadata *BackupMetadata
+ Receipts []storage_files.WriteReceipt
+}
diff --git a/backend/internal/features/backups/backups/core/logical/file_write.go b/backend/internal/features/backups/backups/core/logical/file_write.go
new file mode 100644
index 00000000..1032903c
--- /dev/null
+++ b/backend/internal/features/backups/backups/core/logical/file_write.go
@@ -0,0 +1,46 @@
+package backups_core_logical
+
+import (
+ "context"
+ "io"
+
+ storage_files "databasus-backend/internal/features/storages/files"
+)
+
+// BackgroundFileWrite is an upload running beside the dump that feeds it through a
+// pipe. Every engine drives it the same way: peek at the error while the dump is
+// still producing, then take the receipt once the writer is done.
+type BackgroundFileWrite struct {
+ // Errors stays writable because the engines put a peeked value back and their
+ // cancellation cleanup drains it.
+ Errors chan error
+ Receipts chan storage_files.WriteReceipt
+}
+
+// StartBackgroundFileWrite sends the receipt before the error, so a caller holding
+// the error knows the receipt is already waiting.
+func StartBackgroundFileWrite(
+ ctx context.Context,
+ fileStore BackupFileStore,
+ reference storage_files.StoredFileReference,
+ source *io.PipeReader,
+ onFailure context.CancelCauseFunc,
+) BackgroundFileWrite {
+ write := BackgroundFileWrite{
+ Errors: make(chan error, 1),
+ Receipts: make(chan storage_files.WriteReceipt, 1),
+ }
+
+ go func() {
+ receipt, err := fileStore.WriteFile(ctx, reference, source)
+ if err != nil {
+ _ = source.CloseWithError(err)
+ onFailure(err)
+ }
+
+ write.Receipts <- receipt
+ write.Errors <- err
+ }()
+
+ return write
+}
diff --git a/backend/internal/features/backups/backups/core/logical/interfaces.go b/backend/internal/features/backups/backups/core/logical/interfaces.go
index 69b7354b..92298c61 100644
--- a/backend/internal/features/backups/backups/core/logical/interfaces.go
+++ b/backend/internal/features/backups/backups/core/logical/interfaces.go
@@ -2,6 +2,7 @@ package backups_core_logical
import (
"context"
+ "io"
"github.com/google/uuid"
@@ -9,7 +10,7 @@ import (
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
notifier_models "databasus-backend/internal/features/notifiers/models"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
)
type NotificationSender interface {
@@ -20,15 +21,26 @@ type NotificationSender interface {
)
}
+// BackupFileStore is the only way a backup writes to storage. It hands back a
+// receipt instead of a stored file, because the file belongs to nobody until the
+// transaction that publishes the backup claims it.
+type BackupFileStore interface {
+ WriteFile(
+ ctx context.Context,
+ reference storage_files.StoredFileReference,
+ file io.Reader,
+ ) (storage_files.WriteReceipt, error)
+}
+
type CreateBackupUsecase interface {
Execute(
ctx context.Context,
backup *LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
database *databases.Database,
- storage *storages.Storage,
+ fileStore BackupFileStore,
backupProgressListener func(completedMBs float64),
- ) (*BackupMetadata, error)
+ ) (*BackupArtifacts, error)
}
type BackupRemoveListener interface {
diff --git a/backend/internal/features/backups/backups/core/logical/repository.go b/backend/internal/features/backups/backups/core/logical/repository.go
index 5a383297..c58959b8 100644
--- a/backend/internal/features/backups/backups/core/logical/repository.go
+++ b/backend/internal/features/backups/backups/core/logical/repository.go
@@ -13,21 +13,24 @@ import (
type BackupRepository struct{}
func (r *BackupRepository) Save(backup *LogicalBackup) error {
+ return r.SaveInTransaction(storage.GetDb(), backup)
+}
+
+// A stored file is kept only when the row naming it commits, so publication and the
+// receipts it spends have to share one transaction.
+func (r *BackupRepository) SaveInTransaction(tx *gorm.DB, backup *LogicalBackup) error {
if backup.DatabaseID == uuid.Nil || backup.StorageID == uuid.Nil {
return errors.New("database ID and storage ID are required")
}
- db := storage.GetDb()
-
isNew := backup.ID == uuid.Nil
if isNew {
backup.ID = uuid.New()
- return db.Create(backup).
- Error
+
+ return tx.Create(backup).Error
}
- return db.Save(backup).
- Error
+ return tx.Save(backup).Error
}
func (r *BackupRepository) UpdateRestoreVerificationStatus(
diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.go
index 01a3a756..b7eee373 100644
--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go
+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go
@@ -90,6 +90,20 @@ func objectExists(t *testing.T, st *storages.Storage, fileName string) bool {
return true
}
+func requireObjectAbsent(t *testing.T, backupStorage *storages.Storage, fileName string) {
+ t.Helper()
+
+ reader, err := backupStorage.GetFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)
+ if err == nil {
+ require.NoError(t, reader.Close())
+ require.Failf(t, "storage object still exists", "file_name=%s", fileName)
+
+ return
+ }
+
+ require.ErrorContains(t, err, "file not found:")
+}
+
func lsn(segments int) walmath.LSN {
return walmath.LSN(segments * segmentMB * 1024 * 1024)
}
@@ -106,20 +120,25 @@ func Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects(t *testing.T)
fullModel.BackupSizeMb = new(100.0)
full := physical_testing.CreateTestFullBackup(t, fullModel)
- firstIncr := physical_testing.CreateTestIncrementalBackup(t,
- physical_testing.NewTestCompletedIncrementalBackup(databaseID, storageID, full.ID, nil, 1, lsn(1), lsn(2)))
- physical_testing.CreateTestIncrementalBackup(
- t,
- physical_testing.NewTestCompletedIncrementalBackup(
- databaseID,
- storageID,
- full.ID,
- &firstIncr.ID,
- 1,
- lsn(2),
- lsn(3),
- ),
+ firstIncrModel := physical_testing.NewTestCompletedIncrementalBackup(
+ databaseID, storageID, full.ID, nil, 1, lsn(1), lsn(2))
+ firstIncrManifestName := *firstIncrModel.FileName + ".manifest"
+ firstIncrModel.ManifestFileName = &firstIncrManifestName
+ firstIncr := physical_testing.CreateTestIncrementalBackup(t, firstIncrModel)
+
+ secondIncrModel := physical_testing.NewTestCompletedIncrementalBackup(
+ databaseID,
+ storageID,
+ full.ID,
+ &firstIncr.ID,
+ 1,
+ lsn(2),
+ lsn(3),
)
+ secondIncrManifestName := *secondIncrModel.FileName + ".manifest"
+ secondIncrModel.ManifestFileName = &secondIncrManifestName
+ secondIncr := physical_testing.CreateTestIncrementalBackup(t, secondIncrModel)
+ incrementals := []*physical_models.PhysicalIncrementalBackup{firstIncr, secondIncr}
walSegments := []*physical_models.PhysicalWalSegment{
physical_testing.NewTestWalSegment(databaseID, storageID, 1, "000000010000000000000001", lsn(1), lsn(2)),
@@ -141,10 +160,17 @@ func Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects(t *testing.T)
saveObject(t, prereqs.storage, *full.FileName)
saveObject(t, prereqs.storage, manifestName)
saveObject(t, prereqs.storage, *full.FileName+".metadata")
+ for _, incremental := range incrementals {
+ saveObject(t, prereqs.storage, *incremental.FileName)
+ saveObject(t, prereqs.storage, *incremental.ManifestFileName)
+ saveObject(t, prereqs.storage, *incremental.FileName+".metadata")
+ }
for _, walSegment := range walSegments {
saveObject(t, prereqs.storage, *walSegment.FileName)
+ saveObject(t, prereqs.storage, *walSegment.FileName+".metadata")
}
saveObject(t, prereqs.storage, history.FileName)
+ saveObject(t, prereqs.storage, history.FileName+".metadata")
summary, err := service.DeleteFull(t.Context(), full.ID, 1_000_000)
require.NoError(t, err)
@@ -168,12 +194,20 @@ func Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects(t *testing.T)
gotHistory, _ := physical_repositories.GetWalHistoryRepository().FindByDatabaseTimeline(databaseID, 1)
assert.Nil(t, gotHistory, "history row must be gone")
- assert.False(t, objectExists(t, prereqs.storage, *full.FileName), "full artifact gone")
- assert.False(t, objectExists(t, prereqs.storage, manifestName), "full manifest gone")
- assert.False(t, objectExists(t, prereqs.storage, *full.FileName+".metadata"), "full sidecar gone")
+ requireObjectAbsent(t, prereqs.storage, *full.FileName)
+ requireObjectAbsent(t, prereqs.storage, manifestName)
+ requireObjectAbsent(t, prereqs.storage, *full.FileName+".metadata")
+ for _, incremental := range incrementals {
+ requireObjectAbsent(t, prereqs.storage, *incremental.FileName)
+ requireObjectAbsent(t, prereqs.storage, *incremental.ManifestFileName)
+ requireObjectAbsent(t, prereqs.storage, *incremental.FileName+".metadata")
+ }
for _, walSegment := range walSegments {
- assert.False(t, objectExists(t, prereqs.storage, *walSegment.FileName), "wal artifact gone")
+ requireObjectAbsent(t, prereqs.storage, *walSegment.FileName)
+ requireObjectAbsent(t, prereqs.storage, *walSegment.FileName+".metadata")
}
+ requireObjectAbsent(t, prereqs.storage, history.FileName)
+ requireObjectAbsent(t, prereqs.storage, history.FileName+".metadata")
survivingSuccessor, _ := physical_repositories.GetFullBackupRepository().FindByID(successor.ID)
assert.NotNil(t, survivingSuccessor, "successor chain on TL2 must be untouched")
diff --git a/backend/internal/features/backups/backups/usecases/logical/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/create_backup_uc.go
index 30e2148e..bf0b875e 100644
--- a/backend/internal/features/backups/backups/usecases/logical/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/create_backup_uc.go
@@ -11,7 +11,6 @@ import (
usecases_logical_postgresql "databasus-backend/internal/features/backups/backups/usecases/logical/postgresql"
backups_config_logical "databasus-backend/internal/features/backups/config/logical"
"databasus-backend/internal/features/databases"
- "databasus-backend/internal/features/storages"
)
type CreateBackupUsecase struct {
@@ -26,9 +25,9 @@ func (uc *CreateBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
database *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
switch database.Type {
case databases.DatabaseTypePostgresLogical:
return uc.CreatePostgresqlBackupUsecase.Execute(
@@ -36,7 +35,7 @@ func (uc *CreateBackupUsecase) Execute(
backup,
backupConfig,
database,
- storage,
+ fileStore,
backupProgressListener,
)
@@ -46,7 +45,7 @@ func (uc *CreateBackupUsecase) Execute(
backup,
backupConfig,
database,
- storage,
+ fileStore,
backupProgressListener,
)
@@ -56,7 +55,7 @@ func (uc *CreateBackupUsecase) Execute(
backup,
backupConfig,
database,
- storage,
+ fileStore,
backupProgressListener,
)
@@ -66,7 +65,7 @@ func (uc *CreateBackupUsecase) Execute(
backup,
backupConfig,
database,
- storage,
+ fileStore,
backupProgressListener,
)
diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
index a6e30606..06acfff4 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
@@ -24,7 +24,7 @@ import (
"databasus-backend/internal/features/databases"
mariadbtypes "databasus-backend/internal/features/databases/databases/mariadb"
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
"databasus-backend/internal/util/encryption"
io_utils "databasus-backend/internal/util/io"
"databasus-backend/internal/util/namelist"
@@ -69,10 +69,10 @@ func (uc *CreateMariadbBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
db *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
- logger := uc.logger.With("database_id", db.ID, "storage_id", storage.ID)
+) (*backups_core_logical.BackupArtifacts, error) {
+ logger := uc.logger.With("database_id", db.ID, "storage_id", backup.StorageID)
logger.InfoContext(ctx, "creating mariadb backup via mariadb-dump")
@@ -117,7 +117,7 @@ func (uc *CreateMariadbBackupUsecase) Execute(
tools.GetMariadbExecutable(mariadbDatabase.Version, tools.MariadbExecutableMariadbDump),
args,
decryptedPassword,
- storage,
+ fileStore,
backupProgressListener,
mariadbDatabase,
)
@@ -182,10 +182,10 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
mariadbBin string,
args []string,
password string,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
mdbConfig *mariadbtypes.MariadbDatabase,
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
uc.logger.InfoContext(parentCtx, "streaming MariaDB backup to storage", "mariadb_bin", mariadbBin)
ctx, cancel := uc.createBackupContext(parentCtx)
@@ -253,23 +253,17 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
return nil, fmt.Errorf("failed to create zstd writer: %w", err)
}
- saveErrCh := make(chan error, 1)
- go func() {
- saveErr := storage.SaveFile(
- ctx,
- uc.fieldEncryptor,
- uc.logger,
- backup.FileName,
- storageReader,
- )
- if saveErr != nil {
- _ = storageReader.CloseWithError(saveErr)
- cancel(saveErr)
- }
- saveErrCh <- saveErr
- }()
+ fileWrite := backups_core_logical.StartBackgroundFileWrite(
+ ctx,
+ fileStore,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ storageReader,
+ cancel,
+ )
if err = cmd.Start(); err != nil {
+ uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, fileWrite.Errors)
+
return nil, fmt.Errorf("start %s: %w", filepath.Base(mariadbBin), err)
}
@@ -287,19 +281,19 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
waitErr := cmd.Wait()
select {
- case earlySaveErr := <-saveErrCh:
+ case earlySaveErr := <-fileWrite.Errors:
if earlySaveErr != nil {
_ = zstdWriter.Close()
_ = uc.closeWriters(encryptionWriter, storageWriter)
return nil, fmt.Errorf("save to storage: %w", earlySaveErr)
}
- saveErrCh <- nil
+ fileWrite.Errors <- nil
default:
}
select {
case <-ctx.Done():
- uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)
+ uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, fileWrite.Errors)
return nil, uc.classifyCancellation(ctx)
default:
}
@@ -308,11 +302,11 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
uc.logger.ErrorContext(parentCtx, "failed to close zstd writer", "error", err)
}
if err := uc.closeWriters(encryptionWriter, storageWriter); err != nil {
- <-saveErrCh
+ <-fileWrite.Errors
return nil, err
}
- saveErr := <-saveErrCh
+ saveErr := <-fileWrite.Errors
stderrOutput := <-stderrCh
if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil {
@@ -329,7 +323,10 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
return nil, fmt.Errorf("save to storage: %w", saveErr)
}
- return &backupMetadata, nil
+ return &backups_core_logical.BackupArtifacts{
+ Metadata: &backupMetadata,
+ Receipts: []storage_files.WriteReceipt{<-fileWrite.Receipts},
+ }, nil
}
func (uc *CreateMariadbBackupUsecase) createTempMyCnfFile(
@@ -515,7 +512,7 @@ func (uc *CreateMariadbBackupUsecase) cleanupOnCancellation(
zstdWriter *zstd.Encoder,
encryptionWriter *backup_encryption.EncryptionWriter,
storageWriter io.WriteCloser,
- saveErrCh chan error,
+ writeErrors chan error,
) {
if zstdWriter != nil {
go func() {
@@ -545,7 +542,7 @@ func (uc *CreateMariadbBackupUsecase) cleanupOnCancellation(
uc.logger.Error("failed to close pipe writer during cancellation", "error", err)
}
- <-saveErrCh
+ <-writeErrors
}
func (uc *CreateMariadbBackupUsecase) closeWriters(
diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
index b9ab09ed..7686060c 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
@@ -21,7 +21,7 @@ import (
"databasus-backend/internal/features/databases"
mongodbtypes "databasus-backend/internal/features/databases/databases/mongodb"
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/namelist"
"databasus-backend/internal/util/tools"
@@ -55,10 +55,10 @@ func (uc *CreateMongodbBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
db *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
- logger := uc.logger.With("database_id", db.ID, "storage_id", storage.ID)
+) (*backups_core_logical.BackupArtifacts, error) {
+ logger := uc.logger.With("database_id", db.ID, "storage_id", backup.StorageID)
logger.InfoContext(ctx, "creating mongodb backup via mongodump")
@@ -102,7 +102,7 @@ func (uc *CreateMongodbBackupUsecase) Execute(
backupConfig,
tools.GetMongodbExecutable(tools.MongodbExecutableMongodump),
args,
- storage,
+ fileStore,
backupProgressListener,
)
}
@@ -139,9 +139,9 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
backupConfig *backups_config_logical.LogicalBackupConfig,
mongodumpBin string,
args []string,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
uc.logger.InfoContext(parentCtx, "streaming MongoDB backup to storage", "mongodump_bin", mongodumpBin)
ctx, cancel := uc.createBackupContext(parentCtx)
@@ -192,23 +192,17 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
return nil, err
}
- saveErrCh := make(chan error, 1)
- go func() {
- saveErr := storage.SaveFile(
- ctx,
- uc.fieldEncryptor,
- uc.logger,
- backup.FileName,
- storageReader,
- )
- if saveErr != nil {
- _ = storageReader.CloseWithError(saveErr)
- cancel(saveErr)
- }
- saveErrCh <- saveErr
- }()
+ fileWrite := backups_core_logical.StartBackgroundFileWrite(
+ ctx,
+ fileStore,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ storageReader,
+ cancel,
+ )
if err = cmd.Start(); err != nil {
+ uc.cleanupOnCancellation(encryptionWriter, storageWriter, fileWrite.Errors)
+
return nil, fmt.Errorf("start %s: %w", filepath.Base(mongodumpBin), err)
}
@@ -230,28 +224,28 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
waitErr := cmd.Wait()
select {
- case earlySaveErr := <-saveErrCh:
+ case earlySaveErr := <-fileWrite.Errors:
if earlySaveErr != nil {
_ = uc.closeWriters(encryptionWriter, storageWriter)
return nil, fmt.Errorf("save to storage: %w", earlySaveErr)
}
- saveErrCh <- nil
+ fileWrite.Errors <- nil
default:
}
select {
case <-ctx.Done():
- uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)
+ uc.cleanupOnCancellation(encryptionWriter, storageWriter, fileWrite.Errors)
return nil, uc.classifyCancellation(ctx)
default:
}
if err := uc.closeWriters(encryptionWriter, storageWriter); err != nil {
- <-saveErrCh
+ <-fileWrite.Errors
return nil, err
}
- saveErr := <-saveErrCh
+ saveErr := <-fileWrite.Errors
stderrOutput := <-stderrCh
if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil {
@@ -268,7 +262,10 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
return nil, fmt.Errorf("save to storage: %w", saveErr)
}
- return &backupMetadata, nil
+ return &backups_core_logical.BackupArtifacts{
+ Metadata: &backupMetadata,
+ Receipts: []storage_files.WriteReceipt{<-fileWrite.Receipts},
+ }, nil
}
func (uc *CreateMongodbBackupUsecase) createBackupContext(
@@ -405,13 +402,13 @@ func (uc *CreateMongodbBackupUsecase) copyWithShutdownCheck(
func (uc *CreateMongodbBackupUsecase) cleanupOnCancellation(
encryptionWriter *backup_encryption.EncryptionWriter,
storageWriter *io.PipeWriter,
- saveErrCh chan error,
+ writeErrors chan error,
) {
if encryptionWriter != nil {
_ = encryptionWriter.Close()
}
_ = storageWriter.CloseWithError(errors.New("backup cancelled"))
- <-saveErrCh
+ <-writeErrors
}
func (uc *CreateMongodbBackupUsecase) closeWriters(
diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
index 96b47d50..89f0c82d 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
@@ -24,7 +24,7 @@ import (
"databasus-backend/internal/features/databases"
mysqltypes "databasus-backend/internal/features/databases/databases/mysql"
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
"databasus-backend/internal/util/encryption"
io_utils "databasus-backend/internal/util/io"
"databasus-backend/internal/util/namelist"
@@ -69,10 +69,10 @@ func (uc *CreateMysqlBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
db *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
- logger := uc.logger.With("database_id", db.ID, "storage_id", storage.ID)
+) (*backups_core_logical.BackupArtifacts, error) {
+ logger := uc.logger.With("database_id", db.ID, "storage_id", backup.StorageID)
logger.InfoContext(ctx, "creating mysql backup via mysqldump")
@@ -117,7 +117,7 @@ func (uc *CreateMysqlBackupUsecase) Execute(
tools.GetMysqlExecutable(mysqlDatabase.Version, tools.MysqlExecutableMysqldump),
args,
decryptedPassword,
- storage,
+ fileStore,
backupProgressListener,
mysqlDatabase,
)
@@ -180,10 +180,10 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
mysqlBin string,
args []string,
password string,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(completedMBs float64),
myConfig *mysqltypes.MysqlDatabase,
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
uc.logger.InfoContext(parentCtx, "streaming MySQL backup to storage", "mysql_bin", mysqlBin)
ctx, cancel := uc.createBackupContext(parentCtx)
@@ -250,23 +250,17 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
return nil, fmt.Errorf("failed to create zstd writer: %w", err)
}
- saveErrCh := make(chan error, 1)
- go func() {
- saveErr := storage.SaveFile(
- ctx,
- uc.fieldEncryptor,
- uc.logger,
- backup.FileName,
- storageReader,
- )
- if saveErr != nil {
- _ = storageReader.CloseWithError(saveErr)
- cancel(saveErr)
- }
- saveErrCh <- saveErr
- }()
+ fileWrite := backups_core_logical.StartBackgroundFileWrite(
+ ctx,
+ fileStore,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ storageReader,
+ cancel,
+ )
if err = cmd.Start(); err != nil {
+ uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, fileWrite.Errors)
+
return nil, fmt.Errorf("start %s: %w", filepath.Base(mysqlBin), err)
}
@@ -284,19 +278,19 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
waitErr := cmd.Wait()
select {
- case earlySaveErr := <-saveErrCh:
+ case earlySaveErr := <-fileWrite.Errors:
if earlySaveErr != nil {
_ = zstdWriter.Close()
_ = uc.closeWriters(encryptionWriter, storageWriter)
return nil, fmt.Errorf("save to storage: %w", earlySaveErr)
}
- saveErrCh <- nil
+ fileWrite.Errors <- nil
default:
}
select {
case <-ctx.Done():
- uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh)
+ uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, fileWrite.Errors)
return nil, uc.classifyCancellation(ctx)
default:
}
@@ -305,11 +299,11 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
uc.logger.ErrorContext(parentCtx, "failed to close zstd writer", "error", err)
}
if err := uc.closeWriters(encryptionWriter, storageWriter); err != nil {
- <-saveErrCh
+ <-fileWrite.Errors
return nil, err
}
- saveErr := <-saveErrCh
+ saveErr := <-fileWrite.Errors
stderrOutput := <-stderrCh
if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil {
@@ -326,7 +320,10 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
return nil, fmt.Errorf("save to storage: %w", saveErr)
}
- return &backupMetadata, nil
+ return &backups_core_logical.BackupArtifacts{
+ Metadata: &backupMetadata,
+ Receipts: []storage_files.WriteReceipt{<-fileWrite.Receipts},
+ }, nil
}
func (uc *CreateMysqlBackupUsecase) createTempMyCnfFile(
@@ -512,7 +509,7 @@ func (uc *CreateMysqlBackupUsecase) cleanupOnCancellation(
zstdWriter *zstd.Encoder,
encryptionWriter *backup_encryption.EncryptionWriter,
storageWriter io.WriteCloser,
- saveErrCh chan error,
+ writeErrors chan error,
) {
if zstdWriter != nil {
go func() {
@@ -542,7 +539,7 @@ func (uc *CreateMysqlBackupUsecase) cleanupOnCancellation(
uc.logger.Error("failed to close pipe writer during cancellation", "error", err)
}
- <-saveErrCh
+ <-writeErrors
}
func (uc *CreateMysqlBackupUsecase) closeWriters(
diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
index d3db88f7..cf77398e 100644
--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
@@ -24,7 +24,7 @@ import (
pgtypes "databasus-backend/internal/features/databases/databases/postgresql/logical"
postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/namelist"
"databasus-backend/internal/util/tools"
@@ -63,12 +63,12 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
backup *backups_core_logical.LogicalBackup,
backupConfig *backups_config_logical.LogicalBackupConfig,
db *databases.Database,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
backupProgressListener func(
completedMBs float64,
),
-) (*backups_core_logical.BackupMetadata, error) {
- logger := uc.logger.With("database_id", db.ID, "storage_id", storage.ID, "backup_id", backup.ID)
+) (*backups_core_logical.BackupArtifacts, error) {
+ logger := uc.logger.With("database_id", db.ID, "storage_id", backup.StorageID, "backup_id", backup.ID)
logger.InfoContext(ctx, "creating postgresql backup via pg_dump custom format")
@@ -122,13 +122,12 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
tools.GetPostgresqlExecutable(pg.Version, "pg_dump"),
args,
decryptedPassword,
- storage,
+ fileStore,
databaseThroughTunnel,
backupProgressListener,
)
}
-// streamToStorage streams pg_dump output directly to storage
func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
parentCtx context.Context,
backup *backups_core_logical.LogicalBackup,
@@ -136,10 +135,10 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
pgBin string,
args []string,
password string,
- storage *storages.Storage,
+ fileStore backups_core_logical.BackupFileStore,
db *databases.Database,
backupProgressListener func(completedMBs float64),
-) (*backups_core_logical.BackupMetadata, error) {
+) (*backups_core_logical.BackupArtifacts, error) {
uc.logger.InfoContext(parentCtx, "streaming PostgreSQL backup to storage", "pg_bin", pgBin, "args", args)
ctx, cancel := uc.createBackupContext(parentCtx)
@@ -194,23 +193,17 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
return nil, err
}
- saveErrCh := make(chan error, 1)
- go func() {
- saveErr := storage.SaveFile(
- ctx,
- uc.fieldEncryptor,
- uc.logger,
- backup.FileName,
- storageReader,
- )
- if saveErr != nil {
- _ = storageReader.CloseWithError(saveErr)
- cancel(saveErr)
- }
- saveErrCh <- saveErr
- }()
+ fileWrite := backups_core_logical.StartBackgroundFileWrite(
+ ctx,
+ fileStore,
+ storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
+ storageReader,
+ cancel,
+ )
if err = cmd.Start(); err != nil {
+ uc.cleanupOnCancellation(encryptionWriter, storageWriter, fileWrite.Errors)
+
return nil, fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)
}
@@ -232,28 +225,28 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
waitErr := cmd.Wait()
select {
- case earlySaveErr := <-saveErrCh:
+ case earlySaveErr := <-fileWrite.Errors:
if earlySaveErr != nil {
_ = uc.closeWriters(encryptionWriter, storageWriter)
return nil, fmt.Errorf("save to storage: %w", earlySaveErr)
}
- saveErrCh <- nil
+ fileWrite.Errors <- nil
default:
}
select {
case <-ctx.Done():
- uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)
+ uc.cleanupOnCancellation(encryptionWriter, storageWriter, fileWrite.Errors)
return nil, uc.classifyCancellation(ctx)
default:
}
if err := uc.closeWriters(encryptionWriter, storageWriter); err != nil {
- <-saveErrCh
+ <-fileWrite.Errors
return nil, err
}
- saveErr := <-saveErrCh
+ saveErr := <-fileWrite.Errors
stderrOutput := <-stderrCh
// Send final sizing after backup is completed
@@ -280,7 +273,10 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
return nil, fmt.Errorf("save to storage: %w", saveErr)
}
- return &backupMetadata, nil
+ return &backups_core_logical.BackupArtifacts{
+ Metadata: &backupMetadata,
+ Receipts: []storage_files.WriteReceipt{<-fileWrite.Receipts},
+ }, nil
}
func (uc *CreatePostgresqlBackupUsecase) copyWithShutdownCheck(
@@ -517,7 +513,7 @@ func (uc *CreatePostgresqlBackupUsecase) setupBackupEncryption(
func (uc *CreatePostgresqlBackupUsecase) cleanupOnCancellation(
encryptionWriter *backup_encryption.EncryptionWriter,
storageWriter io.WriteCloser,
- saveErrCh chan error,
+ writeErrors chan error,
) {
if encryptionWriter != nil {
go func() {
@@ -535,7 +531,7 @@ func (uc *CreatePostgresqlBackupUsecase) cleanupOnCancellation(
uc.logger.Error("failed to close pipe writer during cancellation", "error", err)
}
- <-saveErrCh
+ <-writeErrors
}
func (uc *CreatePostgresqlBackupUsecase) closeWriters(
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
index 80d70003..aab07609 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
@@ -266,6 +266,22 @@ func Test_RunStream_WhenStorageSaveFails_ReturnsStorageUploadFailedNotStall(t *t
"a storage failure that cancels the stream must not be reported as a network stall")
}
+func Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact(t *testing.T) {
+ storage := newFakeStorage()
+
+ outcome, err := runStream(
+ t.Context(),
+ testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone),
+ shellBuildCmd(`printf 'partial physical backup'; exit 1`),
+ classifyFullStreamError,
+ )
+ require.NoError(t, err)
+ require.NotNil(t, outcome.ErrorReason)
+ assert.Equal(t, physical_enums.PhysicalBackupStatusError, outcome.Status)
+ assert.Equal(t, physical_enums.PhysicalBackupErrorPgBasebackupFailed, *outcome.ErrorReason)
+ assert.Equal(t, []byte("partial physical backup"), storage.saved["test-obj"])
+}
+
func Test_RunStream_WhenManifestWalkFails_ReturnsManifestCorrupted(t *testing.T) {
// 1 KB of non-tar bytes: pg exits clean, but the walk hits a bad tar header
// (not a truncation), so the manifest goroutine flags genuine corruption.
diff --git a/backend/internal/features/storages/di.go b/backend/internal/features/storages/di.go
index 5259cbd2..46fbd62a 100644
--- a/backend/internal/features/storages/di.go
+++ b/backend/internal/features/storages/di.go
@@ -4,8 +4,10 @@ import (
"sync"
audit_logs "databasus-backend/internal/features/audit_logs"
+ storage_files "databasus-backend/internal/features/storages/files"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
)
var (
@@ -19,6 +21,18 @@ var (
}
)
+var (
+ storageFileDependencies = storage_files.Dependencies{
+ Repository: &storage_files.PendingDeletionRepository{},
+ Locator: storageService,
+ FieldEncryptor: encryption.GetFieldEncryptor(),
+ Logger: logger.GetLogger(),
+ Timings: storage_files.ProductionTimings(),
+ }
+ storageFileStore = storage_files.NewStore(storageFileDependencies)
+ storageFileDeletionWorker = storage_files.NewDeletionWorker(storageFileStore, storageFileDependencies)
+)
+
var storageController = &StorageController{
storageService,
workspaces_services.GetWorkspaceService(),
@@ -32,6 +46,14 @@ func GetStorageController() *StorageController {
return storageController
}
+func GetStorageFileStore() *storage_files.Store {
+ return storageFileStore
+}
+
+func GetStorageFileDeletionWorker() *storage_files.DeletionWorker {
+ return storageFileDeletionWorker
+}
+
var SetupDependencies = sync.OnceFunc(func() {
workspaces_services.GetWorkspaceService().AddWorkspaceDeletionListener(storageService)
})
diff --git a/backend/internal/features/storages/file_locator.go b/backend/internal/features/storages/file_locator.go
new file mode 100644
index 00000000..16398e91
--- /dev/null
+++ b/backend/internal/features/storages/file_locator.go
@@ -0,0 +1,26 @@
+package storages
+
+import (
+ "context"
+
+ "github.com/google/uuid"
+
+ storage_files "databasus-backend/internal/features/storages/files"
+)
+
+// The file store needs a provider, not a configuration, so the lookup stays here
+// where provider selection already lives and the child package keeps no import of
+// this one.
+func (s *StorageService) GetFileWriter(
+ ctx context.Context,
+ storageID uuid.UUID,
+) (storage_files.FileWriter, error) {
+ return s.storageRepository.FindByID(ctx, storageID)
+}
+
+func (s *StorageService) GetFileRemover(
+ ctx context.Context,
+ storageID uuid.UUID,
+) (storage_files.FileRemover, error) {
+ return s.storageRepository.FindByID(ctx, storageID)
+}
diff --git a/backend/internal/features/storages/files/doc.go b/backend/internal/features/storages/files/doc.go
new file mode 100644
index 00000000..bbb21542
--- /dev/null
+++ b/backend/internal/features/storages/files/doc.go
@@ -0,0 +1,5 @@
+// Package storage_files owns the obligation to remove a stored backup file. The
+// obligation is created before the first byte reaches the provider and released
+// only by the transaction that publishes the file as part of a backup, so cleanup
+// never depends on a caller remembering to delete anything.
+package storage_files
diff --git a/backend/internal/features/storages/files/dto.go b/backend/internal/features/storages/files/dto.go
new file mode 100644
index 00000000..bc6efad3
--- /dev/null
+++ b/backend/internal/features/storages/files/dto.go
@@ -0,0 +1,77 @@
+package storage_files
+
+import (
+ "context"
+ "io"
+ "log/slog"
+ "time"
+
+ "github.com/google/uuid"
+
+ "databasus-backend/internal/util/encryption"
+)
+
+// Dependencies is what both the store and the worker need, so the two
+// constructors cannot drift apart or be wired in a different order.
+type Dependencies struct {
+ Repository *PendingDeletionRepository
+ Locator StorageLocator
+ FieldEncryptor encryption.FieldEncryptor
+ Logger *slog.Logger
+ Timings Timings
+}
+
+type ClaimRequest struct {
+ Limit int
+ AttemptTimeout time.Duration
+ // ExcludedIDs are the writes running in this process, which have an
+ // obligation but no owner willing to let it go yet.
+ ExcludedIDs []uuid.UUID
+}
+
+type PendingDeletionSummary struct {
+ Pending int64
+ Overdue int64
+ OldestAgeMs int64
+}
+
+// Backup code owns the file name; this package owns what happens to the file.
+type StoredFileReference struct {
+ StorageID uuid.UUID
+ FileName string
+}
+
+// Permission to publish one uploaded file. Generation is the pending row's
+// attempt count when the upload finished: anything that takes the file away from
+// its writer increments it, so a receipt that no longer matches cannot confirm.
+type WriteReceipt struct {
+ PendingDeletionID uuid.UUID
+ Reference StoredFileReference
+ Generation int
+}
+
+type FileWriter interface {
+ SaveFile(
+ ctx context.Context,
+ encryptor encryption.FieldEncryptor,
+ logger *slog.Logger,
+ fileName string,
+ file io.Reader,
+ ) error
+}
+
+type FileRemover interface {
+ DeleteFile(
+ ctx context.Context,
+ encryptor encryption.FieldEncryptor,
+ logger *slog.Logger,
+ fileName string,
+ ) error
+}
+
+// The parent storages package implements this, which keeps provider selection
+// there and this package free of an import cycle.
+type StorageLocator interface {
+ GetFileWriter(ctx context.Context, storageID uuid.UUID) (FileWriter, error)
+ GetFileRemover(ctx context.Context, storageID uuid.UUID) (FileRemover, error)
+}
diff --git a/backend/internal/features/storages/files/errors.go b/backend/internal/features/storages/files/errors.go
new file mode 100644
index 00000000..51710297
--- /dev/null
+++ b/backend/internal/features/storages/files/errors.go
@@ -0,0 +1,17 @@
+package storage_files
+
+import "errors"
+
+var (
+ // Either a cascading removal reached the file before its write started, or a
+ // second writer picked the same name, which per-attempt naming makes a defect.
+ ErrFileNameAlreadyRegistered = errors.New("file name is already registered for cleanup")
+
+ // The upload finished, but something claimed the file while it ran, so the
+ // caller must not publish it.
+ ErrFileTakenForDeletion = errors.New("file was taken for deletion during the write")
+
+ ErrReceiptNoLongerValid = errors.New("write receipt no longer matches its pending deletion")
+
+ ErrTransactionRequired = errors.New("publication and deletion requests need the caller's transaction")
+)
diff --git a/backend/internal/features/storages/files/model.go b/backend/internal/features/storages/files/model.go
new file mode 100644
index 00000000..5aaba19c
--- /dev/null
+++ b/backend/internal/features/storages/files/model.go
@@ -0,0 +1,31 @@
+package storage_files
+
+import (
+ "time"
+
+ "github.com/google/uuid"
+)
+
+// Row presence is the whole state: there is no status column, because a row exists
+// exactly while nobody has committed to keeping the file it names.
+type PendingDeletion struct {
+ ID uuid.UUID `gorm:"column:id;primaryKey;type:uuid;default:gen_random_uuid()"`
+ StorageID uuid.UUID `gorm:"column:storage_id;not null;type:uuid"`
+ FileName string `gorm:"column:file_name;not null;type:text"`
+ NotBefore time.Time `gorm:"column:not_before;not null"`
+ // Retry counter and receipt generation at once: every event that takes the
+ // file away from its writer increments it, which invalidates the receipt the
+ // writer is holding.
+ AttemptCount int `gorm:"column:attempt_count;not null"`
+ LastError *string `gorm:"column:last_error;type:text"`
+ CreatedAt time.Time `gorm:"column:created_at;not null"`
+ UpdatedAt time.Time `gorm:"column:updated_at;not null"`
+}
+
+func (PendingDeletion) TableName() string {
+ return "storage_pending_deletions"
+}
+
+func (p *PendingDeletion) GetReference() StoredFileReference {
+ return StoredFileReference{StorageID: p.StorageID, FileName: p.FileName}
+}
diff --git a/backend/internal/features/storages/files/publication_test.go b/backend/internal/features/storages/files/publication_test.go
new file mode 100644
index 00000000..c54ce425
--- /dev/null
+++ b/backend/internal/features/storages/files/publication_test.go
@@ -0,0 +1,236 @@
+package storage_files
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+
+ db "databasus-backend/internal/storage"
+)
+
+// Both batch methods refuse a bare handle, so every call here goes through one.
+func inTransaction(t *testing.T, work func(tx *gorm.DB) error) error {
+ t.Helper()
+
+ return db.GetDb().Transaction(work)
+}
+
+func Test_ConfirmFileWrites_WhenGivenNoTransaction_Refuses(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ receipt, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+ require.NoError(t, err)
+
+ assert.ErrorIs(t,
+ store.ConfirmFileWrites(t.Context(), db.GetDb(), []WriteReceipt{receipt}), ErrTransactionRequired)
+ assert.ErrorIs(t, store.ConfirmFileWrites(t.Context(), nil, []WriteReceipt{receipt}), ErrTransactionRequired)
+}
+
+func Test_RequestFileDeletions_WhenGivenNoTransaction_Refuses(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+ references := []StoredFileReference{{StorageID: createStorageRow(t), FileName: "backup-1"}}
+
+ assert.ErrorIs(t,
+ store.RequestFileDeletions(t.Context(), db.GetDb(), references), ErrTransactionRequired)
+ assert.ErrorIs(t, store.RequestFileDeletions(t.Context(), nil, references), ErrTransactionRequired)
+}
+
+func Test_ConfirmFileWrites_WhenTransactionCommits_KeepsTheFile(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+ storageID := createStorageRow(t)
+ base := StoredFileReference{StorageID: storageID, FileName: "backup-1"}
+ sidecar := StoredFileReference{StorageID: storageID, FileName: "backup-1.metadata"}
+
+ baseReceipt, err := store.WriteFile(t.Context(), base, strings.NewReader("payload"))
+ require.NoError(t, err)
+
+ sidecarReceipt, err := store.WriteFile(t.Context(), sidecar, strings.NewReader("meta"))
+ require.NoError(t, err)
+
+ require.NoError(t, db.GetDb().Transaction(func(tx *gorm.DB) error {
+ return store.ConfirmFileWrites(t.Context(), tx, []WriteReceipt{baseReceipt, sidecarReceipt})
+ }))
+
+ for _, reference := range []StoredFileReference{base, sidecar} {
+ pending, findErr := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, findErr)
+ assert.Nil(t, pending, "a published file carries no obligation")
+ }
+}
+
+func Test_ConfirmFileWrites_WhenTransactionRollsBack_KeepsTheObligation(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ receipt, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+ require.NoError(t, err)
+
+ err = db.GetDb().Transaction(func(tx *gorm.DB) error {
+ if confirmErr := store.ConfirmFileWrites(t.Context(), tx, []WriteReceipt{receipt}); confirmErr != nil {
+ return confirmErr
+ }
+
+ return assert.AnError
+ })
+ require.ErrorIs(t, err, assert.AnError)
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ assert.NotNil(t, pending, "an uncommitted publication leaves the file for cleanup")
+}
+
+func Test_ConfirmFileWrites_WhenCalledTwice_FailsTheSecondTime(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ receipt, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+ require.NoError(t, err)
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.ConfirmFileWrites(t.Context(), tx, []WriteReceipt{receipt})
+ }))
+
+ err = inTransaction(t, func(tx *gorm.DB) error {
+ return store.ConfirmFileWrites(t.Context(), tx, []WriteReceipt{receipt})
+ })
+
+ assert.ErrorIs(t, err, ErrReceiptNoLongerValid)
+}
+
+func Test_ConfirmFileWrites_WhenDeletionWasRequestedMeanwhile_Fails(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ receipt, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+ require.NoError(t, err)
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+
+ err = inTransaction(t, func(tx *gorm.DB) error {
+ return store.ConfirmFileWrites(t.Context(), tx, []WriteReceipt{receipt})
+ })
+
+ assert.ErrorIs(t, err, ErrReceiptNoLongerValid)
+
+ pending, findErr := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, findErr)
+ assert.NotNil(t, pending)
+}
+
+func Test_ConfirmFileWrites_WhenOneReceiptIsStale_FailsTheWholeBatch(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+ storageID := createStorageRow(t)
+ good := StoredFileReference{StorageID: storageID, FileName: "backup-1"}
+ stale := StoredFileReference{StorageID: storageID, FileName: "backup-1.metadata"}
+
+ goodReceipt, err := store.WriteFile(t.Context(), good, strings.NewReader("payload"))
+ require.NoError(t, err)
+
+ staleReceipt, err := store.WriteFile(t.Context(), stale, strings.NewReader("meta"))
+ require.NoError(t, err)
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{stale})
+ }))
+
+ err = db.GetDb().Transaction(func(tx *gorm.DB) error {
+ return store.ConfirmFileWrites(t.Context(), tx, []WriteReceipt{goodReceipt, staleReceipt})
+ })
+
+ require.ErrorIs(t, err, ErrReceiptNoLongerValid)
+
+ pending, findErr := store.repository.FindByReference(db.GetDb(), good)
+ require.NoError(t, findErr)
+ assert.NotNil(t, pending, "a failed batch leaves every file in the batch for cleanup")
+}
+
+func Test_RequestFileDeletions_WhenRepeated_KeepsOneObligation(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ for range 2 {
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+ }
+
+ var count int64
+ require.NoError(t, db.GetDb().Model(&PendingDeletion{}).
+ Where("storage_id = ? AND file_name = ?", reference.StorageID, reference.FileName).
+ Count(&count).Error)
+
+ assert.EqualValues(t, 1, count)
+}
+
+func Test_RequestFileDeletions_WhenWriterIsRunning_Succeeds(t *testing.T) {
+ store := newTestStore(nil)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ var requestErr error
+
+ provider := &fakeProvider{onSave: func() {
+ requestErr = inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ })
+ }}
+ store.locator = &fakeLocator{provider: provider}
+
+ _, writeErr := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+
+ assert.NoError(t, requestErr, "a cascade cannot wait for an upload it does not own")
+ assert.ErrorIs(t, writeErr, ErrFileTakenForDeletion)
+
+ pending, findErr := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, findErr)
+ require.NotNil(t, pending)
+ assert.True(t, isDue(t, reference), "the writer must not push a file that was taken from it further out")
+}
+
+func Test_RequestFileDeletions_WhenBatchIsEmpty_Succeeds(t *testing.T) {
+ store := newTestStore(&fakeLocator{provider: &fakeProvider{}})
+
+ assert.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, nil)
+ }))
+}
+
+// A process that stops between the upload and the catalog commit leaves a receipt
+// nobody will ever spend, which is indistinguishable from a caller that simply
+// never published. Both end with the file gone once the commit window closes.
+func Test_WriteFile_WhenNobodyPublishes_TheFileIsRemovedAndConfirmedOnesAreKept(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ storageID := createStorageRow(t)
+ unpublished := StoredFileReference{StorageID: storageID, FileName: "unpublished"}
+ published := StoredFileReference{StorageID: storageID, FileName: "published"}
+
+ if _, err := store.WriteFile(t.Context(), unpublished, strings.NewReader("payload")); err != nil {
+ require.NoError(t, err)
+ }
+
+ publishedReceipt, err := store.WriteFile(t.Context(), published, strings.NewReader("payload"))
+ require.NoError(t, err)
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.ConfirmFileWrites(t.Context(), tx, []WriteReceipt{publishedReceipt})
+ }))
+
+ worker.RunOnce(t.Context())
+ assert.Empty(t, provider.deleted, "a file inside its commit window is not cleanup's yet")
+
+ time.Sleep(store.GetTimings().CommitWindow)
+
+ require.NoError(t, worker.DrainForTest(t.Context(), unpublished))
+
+ assert.Equal(t, []string{"unpublished"}, provider.deleted,
+ "the published file carries no obligation, so nothing can delete it")
+}
diff --git a/backend/internal/features/storages/files/repository.go b/backend/internal/features/storages/files/repository.go
new file mode 100644
index 00000000..b7d94dcf
--- /dev/null
+++ b/backend/internal/features/storages/files/repository.go
@@ -0,0 +1,258 @@
+package storage_files
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+)
+
+type PendingDeletionRepository struct{}
+
+// A conflict means the pair already carries an obligation, so the write must not
+// start: either a cascade reached the file first, or a second writer picked a name
+// per-attempt naming should have made unique.
+func (r *PendingDeletionRepository) InsertIfAbsent(
+ tx *gorm.DB,
+ id uuid.UUID,
+ reference StoredFileReference,
+ dueIn time.Duration,
+) (*PendingDeletion, error) {
+ var inserted []PendingDeletion
+
+ result := tx.Raw(`
+ INSERT INTO storage_pending_deletions (id, storage_id, file_name, not_before)
+ VALUES (?, ?, ?, now() + (? * interval '1 second'))
+ ON CONFLICT (storage_id, file_name) DO NOTHING
+ RETURNING *`,
+ id, reference.StorageID, reference.FileName, dueIn.Seconds(),
+ ).Scan(&inserted)
+ if result.Error != nil {
+ return nil, fmt.Errorf("insert pending deletion: %w", result.Error)
+ }
+
+ if len(inserted) == 0 {
+ return nil, ErrFileNameAlreadyRegistered
+ }
+
+ return &inserted[0], nil
+}
+
+// Against an existing row this also increments the generation, which is what takes
+// the file away from a writer that is still running and stops it publishing.
+func (r *PendingDeletionRepository) InsertOrTake(tx *gorm.DB, references []StoredFileReference) error {
+ unique := deduplicateReferences(references)
+ if len(unique) == 0 {
+ return nil
+ }
+
+ placeholders := make([]string, 0, len(unique))
+ args := make([]any, 0, len(unique)*2)
+
+ for _, reference := range unique {
+ placeholders = append(placeholders, "(?, ?, now())")
+ args = append(args, reference.StorageID, reference.FileName)
+ }
+
+ statement := fmt.Sprintf(`
+ INSERT INTO storage_pending_deletions (storage_id, file_name, not_before)
+ VALUES %s
+ ON CONFLICT (storage_id, file_name) DO UPDATE
+ SET not_before = now(),
+ attempt_count = storage_pending_deletions.attempt_count + 1,
+ updated_at = now()`, strings.Join(placeholders, ", "))
+
+ if err := tx.Exec(statement, args...).Error; err != nil {
+ return fmt.Errorf("record pending deletions: %w", err)
+ }
+
+ return nil
+}
+
+// This is how a publishing transaction keeps a file and how the worker records a
+// finished deletion, so it refuses when the generation moved and someone else owns
+// the outcome.
+func (r *PendingDeletionRepository) CompleteIfGeneration(
+ tx *gorm.DB,
+ id uuid.UUID,
+ generation int,
+) (bool, error) {
+ result := tx.
+ Where("id = ? AND attempt_count = ?", id, generation).
+ Delete(&PendingDeletion{})
+ if result.Error != nil {
+ return false, fmt.Errorf("complete pending deletion: %w", result.Error)
+ }
+
+ return result.RowsAffected == 1, nil
+}
+
+func (r *PendingDeletionRepository) SetNotBeforeIfGeneration(
+ tx *gorm.DB,
+ id uuid.UUID,
+ generation int,
+ dueIn time.Duration,
+) (bool, error) {
+ result := tx.Model(&PendingDeletion{}).
+ Where("id = ? AND attempt_count = ?", id, generation).
+ Updates(map[string]any{
+ "not_before": databaseTimeIn(dueIn),
+ "updated_at": gorm.Expr("now()"),
+ })
+ if result.Error != nil {
+ return false, fmt.Errorf("reschedule pending deletion: %w", result.Error)
+ }
+
+ return result.RowsAffected == 1, nil
+}
+
+func (r *PendingDeletionRepository) RescheduleIfGeneration(
+ tx *gorm.DB,
+ id uuid.UUID,
+ generation int,
+ dueIn time.Duration,
+ lastError string,
+) (bool, error) {
+ result := tx.Model(&PendingDeletion{}).
+ Where("id = ? AND attempt_count = ?", id, generation).
+ Updates(map[string]any{
+ "not_before": databaseTimeIn(dueIn),
+ "last_error": lastError,
+ "updated_at": gorm.Expr("now()"),
+ })
+ if result.Error != nil {
+ return false, fmt.Errorf("reschedule pending deletion: %w", result.Error)
+ }
+
+ return result.RowsAffected == 1, nil
+}
+
+// Incrementing the generation invalidates any receipt still outstanding for these
+// files, and pushing not_before past the attempt keeps a stalled provider call from
+// holding a row forever. SKIP LOCKED keeps an open deletion request from stalling
+// the whole batch.
+func (r *PendingDeletionRepository) ClaimDue(
+ tx *gorm.DB,
+ request ClaimRequest,
+) ([]PendingDeletion, error) {
+ exclusion := ""
+ args := []any{request.AttemptTimeout.Seconds()}
+
+ if len(request.ExcludedIDs) > 0 {
+ exclusion = "AND id NOT IN (?)"
+
+ args = append(args, request.ExcludedIDs)
+ }
+
+ args = append(args, request.Limit)
+
+ statement := fmt.Sprintf(`
+ UPDATE storage_pending_deletions
+ SET attempt_count = attempt_count + 1,
+ not_before = now() + (? * interval '1 second'),
+ updated_at = now()
+ WHERE id IN (
+ SELECT id
+ FROM storage_pending_deletions
+ WHERE not_before <= now() %s
+ ORDER BY not_before
+ LIMIT ?
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING *`, exclusion)
+
+ var claimed []PendingDeletion
+
+ if err := tx.Raw(statement, args...).Scan(&claimed).Error; err != nil {
+ return nil, fmt.Errorf("claim pending deletions: %w", err)
+ }
+
+ return claimed, nil
+}
+
+func (r *PendingDeletionRepository) GetPendingDeletionSummary(tx *gorm.DB) (PendingDeletionSummary, error) {
+ var summary PendingDeletionSummary
+
+ err := tx.Raw(`
+ SELECT count(*) AS pending,
+ count(*) FILTER (WHERE not_before <= now()) AS overdue,
+ coalesce(extract(epoch FROM now() - min(created_at)) * 1000, 0)::bigint AS oldest_age_ms
+ FROM storage_pending_deletions`).Scan(&summary).Error
+ if err != nil {
+ return PendingDeletionSummary{}, fmt.Errorf("summarize pending deletions: %w", err)
+ }
+
+ return summary, nil
+}
+
+func (r *PendingDeletionRepository) CountByReferences(
+ tx *gorm.DB,
+ references []StoredFileReference,
+) (int64, error) {
+ unique := deduplicateReferences(references)
+ if len(unique) == 0 {
+ return 0, nil
+ }
+
+ pairs := make([][]any, 0, len(unique))
+ for _, reference := range unique {
+ pairs = append(pairs, []any{reference.StorageID, reference.FileName})
+ }
+
+ var count int64
+
+ err := tx.Model(&PendingDeletion{}).
+ Where("(storage_id, file_name) IN ?", pairs).
+ Count(&count).Error
+ if err != nil {
+ return 0, fmt.Errorf("count pending deletions: %w", err)
+ }
+
+ return count, nil
+}
+
+func (r *PendingDeletionRepository) FindByReference(
+ tx *gorm.DB,
+ reference StoredFileReference,
+) (*PendingDeletion, error) {
+ var pending PendingDeletion
+
+ err := tx.
+ Where("storage_id = ? AND file_name = ?", reference.StorageID, reference.FileName).
+ First(&pending).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, nil
+ }
+
+ if err != nil {
+ return nil, fmt.Errorf("find pending deletion: %w", err)
+ }
+
+ return &pending, nil
+}
+
+// databaseTimeIn keeps every deadline on the database clock, so app-to-Postgres
+// skew cannot shorten a commit window on one path and lengthen it on another.
+func databaseTimeIn(offset time.Duration) clause.Expr {
+ return gorm.Expr("now() + (? * interval '1 second')", offset.Seconds())
+}
+
+func deduplicateReferences(references []StoredFileReference) []StoredFileReference {
+ seen := make(map[StoredFileReference]struct{}, len(references))
+ unique := make([]StoredFileReference, 0, len(references))
+
+ for _, reference := range references {
+ if _, duplicate := seen[reference]; duplicate {
+ continue
+ }
+
+ seen[reference] = struct{}{}
+ unique = append(unique, reference)
+ }
+
+ return unique
+}
diff --git a/backend/internal/features/storages/files/repository_test.go b/backend/internal/features/storages/files/repository_test.go
new file mode 100644
index 00000000..03bd45c8
--- /dev/null
+++ b/backend/internal/features/storages/files/repository_test.go
@@ -0,0 +1,261 @@
+package storage_files
+
+import (
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+
+ db "databasus-backend/internal/storage"
+)
+
+// The rows go in through raw SQL rather than storages.CreateTestStorage: the
+// storages package imports this one, so an in-package test cannot reach its
+// fixtures without giving up access to the unexported protocol under test.
+func createStorageRow(t *testing.T) uuid.UUID {
+ t.Helper()
+
+ workspaceID := uuid.New()
+ storageID := uuid.New()
+
+ require.NoError(t, db.GetDb().Exec(
+ "INSERT INTO workspaces (id, name, created_at) VALUES (?, ?, now())",
+ workspaceID, "storage-files-"+workspaceID.String(),
+ ).Error)
+
+ require.NoError(t, db.GetDb().Exec(
+ "INSERT INTO storages (id, workspace_id, type, name) VALUES (?, ?, 'LOCAL', ?)",
+ storageID, workspaceID, "storage-files-"+storageID.String(),
+ ).Error)
+
+ t.Cleanup(func() {
+ db.GetDb().Exec("DELETE FROM workspaces WHERE id = ?", workspaceID)
+ })
+
+ return storageID
+}
+
+func Test_InsertIfAbsent_WhenNameIsFree_RegistersObligation(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ pending, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, time.Hour)
+
+ require.NoError(t, err)
+ assert.Equal(t, 0, pending.AttemptCount)
+ assert.Equal(t, reference, pending.GetReference())
+}
+
+func Test_InsertIfAbsent_WhenNameAlreadyRegistered_Refuses(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ _, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, 0)
+ require.NoError(t, err)
+
+ _, err = repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, 0)
+
+ assert.ErrorIs(t, err, ErrFileNameAlreadyRegistered)
+}
+
+func Test_InsertIfAbsent_WhenTransactionRollsBack_LeavesNoRow(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ err := db.GetDb().Transaction(func(tx *gorm.DB) error {
+ _, insertErr := repository.InsertIfAbsent(tx, uuid.New(), reference, 0)
+ require.NoError(t, insertErr)
+
+ return assert.AnError
+ })
+ require.ErrorIs(t, err, assert.AnError)
+
+ found, err := repository.FindByReference(db.GetDb(), reference)
+
+ require.NoError(t, err)
+ assert.Nil(t, found)
+}
+
+func Test_InsertOrTake_WhenRequestedTwice_KeepsOneRowAndRaisesGeneration(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.NoError(t, repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
+ require.NoError(t, repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
+
+ found, err := repository.FindByReference(db.GetDb(), reference)
+
+ require.NoError(t, err)
+ require.NotNil(t, found)
+ assert.Equal(t, 1, found.AttemptCount)
+}
+
+func Test_InsertOrTake_WhenBatchRepeatsAReference_DoesNotFail(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ err := repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference, reference})
+
+ require.NoError(t, err)
+
+ found, findErr := repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, findErr)
+ require.NotNil(t, found)
+ assert.Equal(t, 0, found.AttemptCount)
+}
+
+func Test_InsertOrTake_WhenBatchIsEmpty_DoesNothing(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+
+ assert.NoError(t, repository.InsertOrTake(db.GetDb(), nil))
+}
+
+func Test_InsertOrTake_WhenWriterHoldsTheRow_TakesItByRaisingGeneration(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ pending, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, time.Hour)
+ require.NoError(t, err)
+
+ require.NoError(t, repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
+
+ completed, err := repository.CompleteIfGeneration(db.GetDb(), pending.ID, pending.AttemptCount)
+
+ require.NoError(t, err)
+ assert.False(t, completed, "the writer's receipt must not release an obligation someone else took")
+}
+
+func Test_CompleteIfGeneration_WhenGenerationMatches_ReleasesObligation(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ pending, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, 0)
+ require.NoError(t, err)
+
+ completed, err := repository.CompleteIfGeneration(db.GetDb(), pending.ID, pending.AttemptCount)
+ require.NoError(t, err)
+ assert.True(t, completed)
+
+ found, err := repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ assert.Nil(t, found)
+}
+
+func Test_RescheduleIfGeneration_WhenGenerationIsStale_ChangesNothing(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ pending, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, 0)
+ require.NoError(t, err)
+
+ rescheduled, err := repository.RescheduleIfGeneration(
+ db.GetDb(), pending.ID, pending.AttemptCount+1, time.Hour, "stale",
+ )
+
+ require.NoError(t, err)
+ assert.False(t, rescheduled)
+
+ found, err := repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ require.NotNil(t, found)
+ assert.Nil(t, found.LastError)
+}
+
+func Test_ClaimDue_WhenNothingIsDue_ReturnsEmpty(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ _, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, time.Hour)
+ require.NoError(t, err)
+
+ claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 10, AttemptTimeout: time.Minute})
+
+ require.NoError(t, err)
+ assert.NotContains(t, referencesOf(claimed), reference)
+}
+
+func Test_ClaimDue_WhenRowIsDue_TakesItAndRaisesGeneration(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.NoError(t, repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
+
+ claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 100, AttemptTimeout: time.Minute})
+ require.NoError(t, err)
+ require.Contains(t, referencesOf(claimed), reference)
+
+ found, err := repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ require.NotNil(t, found)
+ assert.Equal(t, 1, found.AttemptCount)
+ assert.False(t, isDue(t, reference), "a claimed row is not due again until its attempt times out")
+}
+
+func Test_ClaimDue_WhenWriteIsLive_SkipsIt(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.NoError(t, repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
+
+ live, err := repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+
+ claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{
+ Limit: 100, AttemptTimeout: time.Minute, ExcludedIDs: []uuid.UUID{live.ID},
+ })
+
+ require.NoError(t, err)
+ assert.NotContains(t, referencesOf(claimed), reference)
+}
+
+func Test_ClaimDue_WhenAnotherTransactionHoldsTheRow_SkipsItInsteadOfWaiting(t *testing.T) {
+ repository := &PendingDeletionRepository{}
+ storageID := createStorageRow(t)
+ held := StoredFileReference{StorageID: storageID, FileName: "held"}
+ free := StoredFileReference{StorageID: storageID, FileName: "free"}
+
+ require.NoError(t, repository.InsertOrTake(db.GetDb(), []StoredFileReference{held, free}))
+
+ blocked := make(chan struct{})
+ released := make(chan struct{})
+
+ go func() {
+ defer close(released)
+
+ _ = db.GetDb().Transaction(func(tx *gorm.DB) error {
+ if err := repository.InsertOrTake(tx, []StoredFileReference{held}); err != nil {
+ return err
+ }
+
+ close(blocked)
+ time.Sleep(2 * time.Second)
+
+ return nil
+ })
+ }()
+
+ <-blocked
+
+ start := time.Now()
+ claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 100, AttemptTimeout: time.Minute})
+ elapsed := time.Since(start)
+
+ require.NoError(t, err)
+ assert.Less(t, elapsed, 2*time.Second, "the claim must skip a locked row rather than wait for it")
+ assert.NotContains(t, referencesOf(claimed), held)
+ assert.Contains(t, referencesOf(claimed), free)
+
+ <-released
+}
+
+func referencesOf(rows []PendingDeletion) []StoredFileReference {
+ references := make([]StoredFileReference, 0, len(rows))
+ for _, row := range rows {
+ references = append(references, row.GetReference())
+ }
+
+ return references
+}
diff --git a/backend/internal/features/storages/files/store.go b/backend/internal/features/storages/files/store.go
new file mode 100644
index 00000000..97656a08
--- /dev/null
+++ b/backend/internal/features/storages/files/store.go
@@ -0,0 +1,191 @@
+package storage_files
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "log/slog"
+ "sync"
+
+ "github.com/google/uuid"
+ "gorm.io/gorm"
+
+ db "databasus-backend/internal/storage"
+ "databasus-backend/internal/util/encryption"
+)
+
+// The only way backup code writes a file to a storage and the only way it hands
+// that file over to a catalog transaction. It owns the obligation to remove the
+// file until such a transaction claims it.
+type Store struct {
+ repository *PendingDeletionRepository
+ locator StorageLocator
+ fieldEncryptor encryption.FieldEncryptor
+ logger *slog.Logger
+ timings Timings
+
+ liveWritesMutex sync.Mutex
+ liveWrites map[uuid.UUID]struct{}
+}
+
+func NewStore(dependencies Dependencies) *Store {
+ return &Store{
+ repository: dependencies.Repository,
+ locator: dependencies.Locator,
+ fieldEncryptor: dependencies.FieldEncryptor,
+ logger: dependencies.Logger,
+ timings: dependencies.Timings,
+ liveWrites: make(map[uuid.UUID]struct{}),
+ }
+}
+
+// The obligation to remove the file exists before the first byte leaves, so a
+// provider error, a crash or a caller that never publishes all end the same way,
+// with the file removed.
+func (s *Store) WriteFile(
+ ctx context.Context,
+ reference StoredFileReference,
+ file io.Reader,
+) (WriteReceipt, error) {
+ pendingID := uuid.New()
+
+ // Joining before the insert is what makes the worker's exclusion sound: a row
+ // it can see belongs to a writer that is already in the set.
+ s.joinLiveWrites(pendingID)
+ defer s.leaveLiveWrites(pendingID)
+
+ pending, err := s.repository.InsertIfAbsent(db.GetDb(), pendingID, reference, s.timings.CommitWindow)
+ if err != nil {
+ return WriteReceipt{}, err
+ }
+
+ writer, err := s.locator.GetFileWriter(ctx, reference.StorageID)
+ if err != nil {
+ s.makeEligibleNow(ctx, pending)
+
+ return WriteReceipt{}, fmt.Errorf("locate storage for write: %w", err)
+ }
+
+ logger := s.logger.With("storage_id", reference.StorageID, "file_name", reference.FileName)
+
+ if saveErr := writer.SaveFile(ctx, s.fieldEncryptor, logger, reference.FileName, file); saveErr != nil {
+ s.makeEligibleNow(ctx, pending)
+
+ return WriteReceipt{}, saveErr
+ }
+
+ kept, err := s.repository.SetNotBeforeIfGeneration(
+ db.GetDb(), pending.ID, pending.AttemptCount, s.timings.CommitWindow,
+ )
+ if err != nil {
+ return WriteReceipt{}, err
+ }
+
+ if !kept {
+ return WriteReceipt{}, ErrFileTakenForDeletion
+ }
+
+ return WriteReceipt{
+ PendingDeletionID: pending.ID,
+ Reference: reference,
+ Generation: pending.AttemptCount,
+ }, nil
+}
+
+// Fails unless every receipt still matches its row, so a file that cleanup already
+// took cannot be published by the transaction that rolls back here.
+func (s *Store) ConfirmFileWrites(ctx context.Context, tx *gorm.DB, receipts []WriteReceipt) error {
+ if err := requireTransaction(tx); err != nil {
+ return err
+ }
+
+ for _, receipt := range receipts {
+ released, err := s.repository.CompleteIfGeneration(
+ tx.WithContext(ctx), receipt.PendingDeletionID, receipt.Generation,
+ )
+ if err != nil {
+ return err
+ }
+
+ if !released {
+ return fmt.Errorf("%w: %s", ErrReceiptNoLongerValid, receipt.Reference.FileName)
+ }
+ }
+
+ return nil
+}
+
+// Never fails because a writer is still running or because the same file was
+// requested before: a caller removing catalog rows has no way to wait for an
+// upload it does not own.
+func (s *Store) RequestFileDeletions(
+ ctx context.Context,
+ tx *gorm.DB,
+ references []StoredFileReference,
+) error {
+ if err := requireTransaction(tx); err != nil {
+ return err
+ }
+
+ return s.repository.InsertOrTake(tx.WithContext(ctx), references)
+}
+
+func (s *Store) GetTimings() Timings {
+ return s.timings
+}
+
+// A false result means something already took the file, which leaves it eligible
+// anyway. The write is over either way, so the failure is logged, not returned.
+func (s *Store) makeEligibleNow(ctx context.Context, pending *PendingDeletion) {
+ if _, err := s.repository.SetNotBeforeIfGeneration(
+ db.GetDb(), pending.ID, pending.AttemptCount, 0,
+ ); err != nil {
+ s.logger.ErrorContext(ctx, "failed to schedule cleanup for a failed write",
+ "storage_id", pending.StorageID,
+ "file_name", pending.FileName,
+ "error", err)
+ }
+}
+
+func (s *Store) joinLiveWrites(id uuid.UUID) {
+ s.liveWritesMutex.Lock()
+ defer s.liveWritesMutex.Unlock()
+
+ s.liveWrites[id] = struct{}{}
+}
+
+func (s *Store) leaveLiveWrites(id uuid.UUID) {
+ s.liveWritesMutex.Lock()
+ defer s.liveWritesMutex.Unlock()
+
+ delete(s.liveWrites, id)
+}
+
+// Holding the mutex across claim is what keeps the exclusion list from going stale
+// between reading it and running the statement.
+func (s *Store) withLiveWrites(claim func(excluded []uuid.UUID) error) error {
+ s.liveWritesMutex.Lock()
+ defer s.liveWritesMutex.Unlock()
+
+ excluded := make([]uuid.UUID, 0, len(s.liveWrites))
+ for id := range s.liveWrites {
+ excluded = append(excluded, id)
+ }
+
+ return claim(excluded)
+}
+
+// Both batch methods are all-or-nothing. On the shared handle a batch that fails
+// partway has already released the receipts it got through, stranding those files
+// forever.
+func requireTransaction(tx *gorm.DB) error {
+ if tx == nil {
+ return ErrTransactionRequired
+ }
+
+ if _, inTransaction := tx.Statement.ConnPool.(gorm.TxCommitter); !inTransaction {
+ return ErrTransactionRequired
+ }
+
+ return nil
+}
diff --git a/backend/internal/features/storages/files/store_test.go b/backend/internal/features/storages/files/store_test.go
new file mode 100644
index 00000000..a4b71fc6
--- /dev/null
+++ b/backend/internal/features/storages/files/store_test.go
@@ -0,0 +1,224 @@
+package storage_files
+
+import (
+ "context"
+ "errors"
+ "io"
+ "log/slog"
+ "strings"
+ "testing"
+
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ db "databasus-backend/internal/storage"
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
+)
+
+type fakeProvider struct {
+ saveErr error
+ deleteErr error
+ saved []string
+ deleted []string
+ onSave func()
+ onDelete func()
+ savePanics bool
+}
+
+func (p *fakeProvider) SaveFile(
+ _ context.Context, _ encryption.FieldEncryptor, _ *slog.Logger, fileName string, file io.Reader,
+) error {
+ if p.onSave != nil {
+ p.onSave()
+ }
+
+ if p.savePanics {
+ panic("provider exploded")
+ }
+
+ if file != nil {
+ _, _ = io.Copy(io.Discard, file)
+ }
+
+ if p.saveErr != nil {
+ return p.saveErr
+ }
+
+ p.saved = append(p.saved, fileName)
+
+ return nil
+}
+
+func (p *fakeProvider) DeleteFile(
+ _ context.Context, _ encryption.FieldEncryptor, _ *slog.Logger, fileName string,
+) error {
+ if p.onDelete != nil {
+ p.onDelete()
+ }
+
+ if p.deleteErr != nil {
+ return p.deleteErr
+ }
+
+ p.deleted = append(p.deleted, fileName)
+
+ return nil
+}
+
+type fakeLocator struct {
+ provider *fakeProvider
+ err error
+}
+
+func (l *fakeLocator) GetFileWriter(_ context.Context, _ uuid.UUID) (FileWriter, error) {
+ if l.err != nil {
+ return nil, l.err
+ }
+
+ return l.provider, nil
+}
+
+func (l *fakeLocator) GetFileRemover(_ context.Context, _ uuid.UUID) (FileRemover, error) {
+ if l.err != nil {
+ return nil, l.err
+ }
+
+ return l.provider, nil
+}
+
+func testDependencies(locator StorageLocator) Dependencies {
+ return Dependencies{
+ Repository: &PendingDeletionRepository{},
+ Locator: locator,
+ FieldEncryptor: encryption.GetFieldEncryptor(),
+ Logger: logger.GetLogger(),
+ Timings: TimingsForTest(),
+ }
+}
+
+func newTestStore(locator StorageLocator) *Store {
+ return NewStore(testDependencies(locator))
+}
+
+// Eligibility is a database-clock question, so the database answers it.
+func isDue(t *testing.T, reference StoredFileReference) bool {
+ t.Helper()
+
+ var count int64
+
+ require.NoError(t, db.GetDb().Model(&PendingDeletion{}).
+ Where("storage_id = ? AND file_name = ? AND not_before <= now()", reference.StorageID, reference.FileName).
+ Count(&count).Error)
+
+ return count == 1
+}
+
+func liveWriteCount(t *testing.T, store *Store) int {
+ t.Helper()
+
+ count := 0
+ require.NoError(t, store.withLiveWrites(func(excluded []uuid.UUID) error {
+ count = len(excluded)
+
+ return nil
+ }))
+
+ return count
+}
+
+func Test_WriteFile_WhenProviderAccepts_ReturnsReceiptAndHoldsObligation(t *testing.T) {
+ provider := &fakeProvider{}
+ store := newTestStore(&fakeLocator{provider: provider})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ receipt, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+
+ require.NoError(t, err)
+ assert.Equal(t, reference, receipt.Reference)
+ assert.Equal(t, 0, receipt.Generation)
+ assert.Equal(t, []string{"backup-1"}, provider.saved)
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ require.NotNil(t, pending, "the obligation stands until a catalog transaction claims it")
+ assert.False(t, isDue(t, reference), "a fresh upload gets its commit window before cleanup may act")
+ assert.Zero(t, liveWriteCount(t, store))
+}
+
+func Test_WriteFile_WhenNameAlreadyRegistered_SendsNoBytes(t *testing.T) {
+ provider := &fakeProvider{}
+ store := newTestStore(&fakeLocator{provider: provider})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.NoError(t, store.repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
+
+ _, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+
+ assert.ErrorIs(t, err, ErrFileNameAlreadyRegistered)
+ assert.Empty(t, provider.saved)
+ assert.Zero(t, liveWriteCount(t, store))
+}
+
+func Test_WriteFile_WhenProviderFails_LeavesFileEligibleOnceWriterIsGone(t *testing.T) {
+ provider := &fakeProvider{saveErr: errors.New("provider is down")}
+ store := newTestStore(&fakeLocator{provider: provider})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ _, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+
+ require.ErrorIs(t, err, provider.saveErr)
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ require.NotNil(t, pending)
+ assert.True(t, isDue(t, reference), "a failed write is eligible immediately")
+ assert.Zero(t, liveWriteCount(t, store))
+}
+
+func Test_WriteFile_WhileRunning_KeepsTheRowOutOfReachOfCleanup(t *testing.T) {
+ store := newTestStore(nil)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ var duringWrite int
+
+ provider := &fakeProvider{onSave: func() {
+ require.NoError(t, store.repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
+
+ duringWrite = liveWriteCount(t, store)
+ }}
+ store.locator = &fakeLocator{provider: provider}
+
+ _, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+
+ assert.ErrorIs(t, err, ErrFileTakenForDeletion)
+ assert.Equal(t, 1, duringWrite, "the write is excluded from cleanup while it runs")
+ assert.Zero(t, liveWriteCount(t, store))
+}
+
+func Test_WriteFile_WhenProviderPanics_LeavesTheLiveSet(t *testing.T) {
+ provider := &fakeProvider{savePanics: true}
+ store := newTestStore(&fakeLocator{provider: provider})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.Panics(t, func() {
+ _, _ = store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+ })
+
+ assert.Zero(t, liveWriteCount(t, store))
+}
+
+func Test_WriteFile_WhenStorageCannotBeLocated_LeavesFileEligible(t *testing.T) {
+ store := newTestStore(&fakeLocator{err: errors.New("no such storage")})
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ _, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+
+ require.Error(t, err)
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ require.NotNil(t, pending)
+ assert.True(t, isDue(t, reference))
+}
diff --git a/backend/internal/features/storages/files/testing.go b/backend/internal/features/storages/files/testing.go
new file mode 100644
index 00000000..2f282b01
--- /dev/null
+++ b/backend/internal/features/storages/files/testing.go
@@ -0,0 +1,50 @@
+package storage_files
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ db "databasus-backend/internal/storage"
+)
+
+// drainDeadlineForTest keeps a provider that never succeeds from hanging a test.
+const drainDeadlineForTest = 30 * time.Second
+
+// TimingsForTest keeps every bound long enough to be observable in a single test
+// and short enough that no test waits on a production interval.
+func TimingsForTest() Timings {
+ return Timings{
+ CommitWindow: 2 * time.Second,
+ AttemptTimeout: time.Second,
+ RetryBaseDelay: 10 * time.Millisecond,
+ RetryMaxDelay: 50 * time.Millisecond,
+ WorkerTick: 10 * time.Millisecond,
+ ClaimBatchSize: 10,
+ }
+}
+
+// It watches only what the caller names, because the slot database is shared with
+// every other test running in parallel.
+func (w *DeletionWorker) DrainForTest(ctx context.Context, references ...StoredFileReference) error {
+ deadline := time.Now().UTC().Add(drainDeadlineForTest)
+
+ for {
+ outstanding, err := w.repository.CountByReferences(db.GetDb(), references)
+ if err != nil {
+ return err
+ }
+
+ if outstanding == 0 {
+ return nil
+ }
+
+ if time.Now().UTC().After(deadline) {
+ return fmt.Errorf("%d file deletions still pending after %s", outstanding, drainDeadlineForTest)
+ }
+
+ w.RunOnce(ctx)
+
+ time.Sleep(w.timings.RetryMaxDelay)
+ }
+}
diff --git a/backend/internal/features/storages/files/timings.go b/backend/internal/features/storages/files/timings.go
new file mode 100644
index 00000000..2829503b
--- /dev/null
+++ b/backend/internal/features/storages/files/timings.go
@@ -0,0 +1,30 @@
+package storage_files
+
+import "time"
+
+// The single policy the store and the worker read at construction: every deadline
+// plus how many rows one claim may take. Tests shrink it instead of waiting, so
+// nothing here is read from a package-level variable at call time.
+type Timings struct {
+ // CommitWindow is how long a file may stay recorded but unclaimed by any
+ // catalog transaction before cleanup assumes its publisher is never coming.
+ CommitWindow time.Duration
+ // AttemptTimeout bounds one provider deletion and is also how far a claim
+ // pushes not_before, so a stalled call cannot hold its row forever.
+ AttemptTimeout time.Duration
+ RetryBaseDelay time.Duration
+ RetryMaxDelay time.Duration
+ WorkerTick time.Duration
+ ClaimBatchSize int
+}
+
+func ProductionTimings() Timings {
+ return Timings{
+ CommitWindow: 15 * time.Minute,
+ AttemptTimeout: 2 * time.Minute,
+ RetryBaseDelay: 30 * time.Second,
+ RetryMaxDelay: time.Hour,
+ WorkerTick: time.Minute,
+ ClaimBatchSize: 50,
+ }
+}
diff --git a/backend/internal/features/storages/files/worker.go b/backend/internal/features/storages/files/worker.go
new file mode 100644
index 00000000..f703f099
--- /dev/null
+++ b/backend/internal/features/storages/files/worker.go
@@ -0,0 +1,231 @@
+package storage_files
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "math"
+ "math/rand/v2"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/uuid"
+
+ db "databasus-backend/internal/storage"
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
+)
+
+const (
+ jobName = "storage_file_deletion"
+
+ maxPersistedErrorLength = 500
+)
+
+// Drains the obligations nobody claimed. It never touches a file whose write is
+// still running in this process, and it commits its claim before it talks to a
+// provider, so a stalled provider call holds no database lock.
+type DeletionWorker struct {
+ store *Store
+ repository *PendingDeletionRepository
+ locator StorageLocator
+ fieldEncryptor encryption.FieldEncryptor
+ logger *slog.Logger
+ timings Timings
+
+ // jitter is injectable so a test can assert the backoff instead of the noise.
+ jitter func(time.Duration) time.Duration
+
+ hasRun atomic.Bool
+}
+
+func NewDeletionWorker(store *Store, dependencies Dependencies) *DeletionWorker {
+ return &DeletionWorker{
+ store: store,
+ repository: dependencies.Repository,
+ locator: dependencies.Locator,
+ fieldEncryptor: dependencies.FieldEncryptor,
+ logger: dependencies.Logger,
+ timings: dependencies.Timings,
+ jitter: defaultJitter,
+ }
+}
+
+func (w *DeletionWorker) Run(ctx context.Context) {
+ if w.hasRun.Swap(true) {
+ panic(fmt.Sprintf("%T.Run() called multiple times", w))
+ }
+
+ lifecycleLogger := w.logger.With("job_name", jobName)
+
+ lifecycleLogger.InfoContext(ctx, "storage file deletion started")
+
+ if ctx.Err() != nil {
+ return
+ }
+
+ w.RunOnce(ctx)
+
+ ticker := time.NewTicker(w.timings.WorkerTick)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ lifecycleLogger.InfoContext(ctx, "storage file deletion stopped")
+
+ return
+ case <-ticker.C:
+ w.RunOnce(ctx)
+ }
+ }
+}
+
+// One pass over the due obligations. Exported so a test can observe cleanup as a
+// sequence instead of waiting on the ticker.
+func (w *DeletionWorker) RunOnce(ctx context.Context) {
+ passLogger := w.logger.With("job_id", uuid.New(), "job_name", jobName)
+
+ claimed, err := w.claimDue()
+ if err != nil {
+ passLogger.ErrorContext(ctx, "failed to claim pending file deletions", "error", err)
+
+ return
+ }
+
+ for _, pending := range claimed {
+ w.attemptDeletion(ctx, passLogger, pending)
+ }
+
+ w.reportSummary(ctx, passLogger)
+}
+
+func (w *DeletionWorker) claimDue() ([]PendingDeletion, error) {
+ var claimed []PendingDeletion
+
+ err := w.store.withLiveWrites(func(excluded []uuid.UUID) error {
+ batch, claimErr := w.repository.ClaimDue(db.GetDb(), ClaimRequest{
+ Limit: w.timings.ClaimBatchSize,
+ AttemptTimeout: w.timings.AttemptTimeout,
+ ExcludedIDs: excluded,
+ })
+ claimed = batch
+
+ return claimErr
+ })
+
+ return claimed, err
+}
+
+func (w *DeletionWorker) attemptDeletion(ctx context.Context, passLogger *slog.Logger, pending PendingDeletion) {
+ attemptLogger := passLogger.With("storage_id", pending.StorageID, "file_name", pending.FileName)
+
+ attemptCtx, cancel := context.WithTimeout(ctx, w.timings.AttemptTimeout)
+ defer cancel()
+
+ err := w.deleteFromProvider(attemptCtx, attemptLogger, pending)
+ if err == nil {
+ released, completeErr := w.repository.CompleteIfGeneration(
+ db.GetDb(), pending.ID, pending.AttemptCount,
+ )
+ if completeErr != nil {
+ attemptLogger.ErrorContext(ctx, "failed to release a completed file deletion", "error", completeErr)
+
+ return
+ }
+
+ if !released {
+ attemptLogger.InfoContext(ctx, fmt.Sprintf(
+ "deleted stored file on attempt %d, a newer request now owns it", pending.AttemptCount))
+
+ return
+ }
+
+ attemptLogger.InfoContext(ctx, fmt.Sprintf("deleted stored file on attempt %d", pending.AttemptCount))
+
+ return
+ }
+
+ retryDelay := w.retryDelay(pending.AttemptCount)
+
+ rescheduled, rescheduleErr := w.repository.RescheduleIfGeneration(
+ db.GetDb(), pending.ID, pending.AttemptCount, retryDelay, sanitizeError(err),
+ )
+ if rescheduleErr != nil {
+ attemptLogger.ErrorContext(ctx, "failed to reschedule a file deletion", "error", rescheduleErr)
+
+ return
+ }
+
+ if !rescheduled {
+ attemptLogger.WarnContext(ctx, fmt.Sprintf(
+ "file deletion attempt %d failed, a newer request now owns it", pending.AttemptCount), "error", err)
+
+ return
+ }
+
+ attemptLogger.WarnContext(ctx, fmt.Sprintf(
+ "file deletion attempt %d failed, retrying in %s", pending.AttemptCount, retryDelay), "error", err)
+}
+
+func (w *DeletionWorker) deleteFromProvider(
+ ctx context.Context,
+ attemptLogger *slog.Logger,
+ pending PendingDeletion,
+) error {
+ remover, err := w.locator.GetFileRemover(ctx, pending.StorageID)
+ if err != nil {
+ return fmt.Errorf("locate storage for deletion: %w", err)
+ }
+
+ return remover.DeleteFile(ctx, w.fieldEncryptor, attemptLogger, pending.FileName)
+}
+
+func (w *DeletionWorker) reportSummary(ctx context.Context, passLogger *slog.Logger) {
+ summary, err := w.repository.GetPendingDeletionSummary(db.GetDb())
+ if err != nil {
+ passLogger.ErrorContext(ctx, "failed to summarize pending file deletions", "error", err)
+
+ return
+ }
+
+ if summary.Pending == 0 {
+ return
+ }
+
+ passLogger.InfoContext(ctx, fmt.Sprintf(
+ "pending file deletions: %d pending, %d overdue, oldest %s",
+ summary.Pending, summary.Overdue, time.Duration(summary.OldestAgeMs)*time.Millisecond))
+}
+
+// Growth stops at the maximum, so an unreachable provider settles into one attempt
+// per maximum delay rather than an ever-widening gap.
+func (w *DeletionWorker) retryDelay(attemptCount int) time.Duration {
+ if attemptCount < 1 {
+ attemptCount = 1
+ }
+
+ growth := math.Pow(2, float64(attemptCount-1))
+
+ delay := time.Duration(float64(w.timings.RetryBaseDelay) * growth)
+ if delay > w.timings.RetryMaxDelay || delay <= 0 {
+ delay = w.timings.RetryMaxDelay
+ }
+
+ return w.jitter(delay)
+}
+
+func defaultJitter(delay time.Duration) time.Duration {
+ return delay/2 + time.Duration(rand.Int64N(int64(delay/2)+1))
+}
+
+// Truncation counts runes, because cutting a multi-byte character in half makes
+// Postgres reject the whole update and lose the backoff with it.
+func sanitizeError(err error) string {
+ message := []rune(logger.Redact(err.Error()))
+ if len(message) > maxPersistedErrorLength {
+ message = message[:maxPersistedErrorLength]
+ }
+
+ return string(message)
+}
diff --git a/backend/internal/features/storages/files/worker_test.go b/backend/internal/features/storages/files/worker_test.go
new file mode 100644
index 00000000..faa89472
--- /dev/null
+++ b/backend/internal/features/storages/files/worker_test.go
@@ -0,0 +1,456 @@
+package storage_files
+
+import (
+ "context"
+ "errors"
+ "log/slog"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+
+ db "databasus-backend/internal/storage"
+)
+
+// A derived handler shares the record list, because the code under test logs
+// through loggers built with With(...).
+type capturingHandler struct {
+ records *[]slog.Record
+ attrs []slog.Attr
+}
+
+func newCapturingHandler() *capturingHandler {
+ return &capturingHandler{records: &[]slog.Record{}}
+}
+
+func (h *capturingHandler) Enabled(context.Context, slog.Level) bool { return true }
+
+func (h *capturingHandler) Handle(_ context.Context, record slog.Record) error {
+ record.AddAttrs(h.attrs...)
+
+ *h.records = append(*h.records, record)
+
+ return nil
+}
+
+func (h *capturingHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+ return &capturingHandler{records: h.records, attrs: append(append([]slog.Attr{}, h.attrs...), attrs...)}
+}
+
+func (h *capturingHandler) WithGroup(string) slog.Handler { return h }
+
+func (h *capturingHandler) attrsOf(index int) map[string]any {
+ found := map[string]any{}
+
+ (*h.records)[index].Attrs(func(attr slog.Attr) bool {
+ found[attr.Key] = attr.Value.Any()
+
+ return true
+ })
+
+ return found
+}
+
+func newTestWorker(t *testing.T, store *Store, locator StorageLocator) *DeletionWorker {
+ t.Helper()
+
+ worker := NewDeletionWorker(store, testDependencies(locator))
+ worker.jitter = func(d time.Duration) time.Duration { return d }
+
+ return worker
+}
+
+func makeDue(t *testing.T, reference StoredFileReference) {
+ t.Helper()
+
+ require.NoError(t, db.GetDb().Model(&PendingDeletion{}).
+ Where("storage_id = ? AND file_name = ?", reference.StorageID, reference.FileName).
+ Update("not_before", gorm.Expr("now() - interval '1 second'")).Error)
+}
+
+func Test_RunOnce_WhenObligationIsDue_DeletesTheFileAndReleasesIt(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+
+ worker.RunOnce(t.Context())
+
+ assert.Equal(t, []string{"backup-1"}, provider.deleted)
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ assert.Nil(t, pending)
+}
+
+func Test_RunOnce_WhenWriteIsLive_LeavesTheFileAloneUntilTheWriterIsGone(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ var deletedDuringWrite []string
+
+ provider.onSave = func() {
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+
+ worker.RunOnce(t.Context())
+
+ deletedDuringWrite = append([]string{}, provider.deleted...)
+ }
+
+ _, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+ require.ErrorIs(t, err, ErrFileTakenForDeletion)
+
+ assert.Empty(t, deletedDuringWrite, "cleanup must not delete a file whose write is still running")
+
+ worker.RunOnce(t.Context())
+
+ assert.Equal(t, []string{"backup-1"}, provider.deleted)
+}
+
+func Test_RunOnce_WhenWriteWasInheritedFromAStoppedProcess_ClaimsItOnTheFirstPass(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ worker := newTestWorker(t, newTestStore(locator), locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ _, err := worker.repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, -time.Minute)
+ require.NoError(t, err)
+
+ worker.RunOnce(t.Context())
+
+ assert.Equal(t, []string{"backup-1"}, provider.deleted)
+}
+
+func Test_RunOnce_WhenClaimingAFile_InvalidatesAnOutstandingReceipt(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ receipt, err := store.WriteFile(t.Context(), reference, strings.NewReader("payload"))
+ require.NoError(t, err)
+
+ makeDue(t, reference)
+ worker.RunOnce(t.Context())
+
+ err = inTransaction(t, func(tx *gorm.DB) error {
+ return store.ConfirmFileWrites(t.Context(), tx, []WriteReceipt{receipt})
+ })
+
+ assert.ErrorIs(t, err, ErrReceiptNoLongerValid)
+ assert.Equal(t, []string{"backup-1"}, provider.deleted)
+}
+
+func Test_RunOnce_WhenProviderFailsThenRecovers_RetriesUntilTheFileIsGone(t *testing.T) {
+ provider := &fakeProvider{deleteErr: errors.New("storage is unavailable")}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+
+ worker.RunOnce(t.Context())
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ require.NotNil(t, pending, "a failed deletion stays an obligation")
+ assert.Equal(t, 1, pending.AttemptCount)
+ require.NotNil(t, pending.LastError)
+ assert.Contains(t, *pending.LastError, "storage is unavailable")
+
+ provider.deleteErr = nil
+ makeDue(t, reference)
+
+ worker.RunOnce(t.Context())
+
+ pending, err = store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ assert.Nil(t, pending)
+}
+
+func Test_RunOnce_WhenAnotherCallerTakesTheFileMidAttempt_DoesNotReleaseTheirObligation(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+
+ provider.onDelete = func() {
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+ }
+
+ worker.RunOnce(t.Context())
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+
+ require.NoError(t, err)
+ assert.NotNil(t, pending, "an attempt that lost its claim must not release the newer one")
+}
+
+func Test_RunOnce_WhenStorageDisappearsMidAttempt_FinishesQuietly(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ storageID := createStorageRow(t)
+ reference := StoredFileReference{StorageID: storageID, FileName: "backup-1"}
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+
+ provider.onDelete = func() {
+ require.NoError(t, db.GetDb().Exec("DELETE FROM storages WHERE id = ?", storageID).Error)
+ }
+
+ require.NotPanics(t, func() { worker.RunOnce(t.Context()) })
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ assert.Nil(t, pending)
+}
+
+func Test_DrainForTest_WhenProviderFailsThenRecovers_ReturnsOnlyAfterTheFilesAreGone(t *testing.T) {
+ provider := &fakeProvider{deleteErr: errors.New("storage is unavailable")}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ storageID := createStorageRow(t)
+ first := StoredFileReference{StorageID: storageID, FileName: "backup-1"}
+ second := StoredFileReference{StorageID: storageID, FileName: "backup-2"}
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{first, second})
+ }))
+
+ // onDelete runs before DeleteFile reads deleteErr, so the first attempt has to
+ // keep the error to fail; clearing it from the second call on lets the drain
+ // recover without writing the field from another goroutine.
+ attempts := 0
+
+ provider.onDelete = func() {
+ attempts++
+
+ if attempts > 1 {
+ provider.deleteErr = nil
+ }
+ }
+
+ assert.NoError(t, worker.DrainForTest(t.Context(), first, second))
+ assert.Greater(t, attempts, 2, "the drain must keep going past a failed attempt")
+
+ for _, reference := range []StoredFileReference{first, second} {
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ assert.Nil(t, pending)
+ }
+}
+
+func Test_DrainForTest_WhenNothingIsPending_ReturnsImmediately(t *testing.T) {
+ locator := &fakeLocator{provider: &fakeProvider{}}
+ worker := newTestWorker(t, newTestStore(locator), locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "never-written"}
+
+ assert.NoError(t, worker.DrainForTest(t.Context(), reference))
+}
+
+func Test_RetryDelay_GrowsExponentiallyAndStopsAtTheMaximum(t *testing.T) {
+ worker := newTestWorker(t, newTestStore(nil), nil)
+ worker.timings.RetryBaseDelay = 10 * time.Millisecond
+ worker.timings.RetryMaxDelay = 40 * time.Millisecond
+
+ assert.Equal(t, 10*time.Millisecond, worker.retryDelay(1))
+ assert.Equal(t, 20*time.Millisecond, worker.retryDelay(2))
+ assert.Equal(t, 40*time.Millisecond, worker.retryDelay(3))
+ assert.Equal(t, 40*time.Millisecond, worker.retryDelay(9))
+ assert.Equal(t, 40*time.Millisecond, worker.retryDelay(500), "an overflowing exponent still lands on the cap")
+}
+
+func Test_SanitizeError_RemovesCredentialsAndBoundsLength(t *testing.T) {
+ sanitized := sanitizeError(errors.New("dial postgres://admin:hunter2@db:5432 failed"))
+
+ assert.NotContains(t, sanitized, "hunter2")
+
+ truncated := sanitizeError(errors.New(strings.Repeat("x", maxPersistedErrorLength*2)))
+
+ assert.Len(t, truncated, maxPersistedErrorLength)
+}
+
+func Test_RunOnce_WhenAttemptFails_LogsJobStorageFileAndRetry(t *testing.T) {
+ handler := newCapturingHandler()
+ provider := &fakeProvider{deleteErr: errors.New("storage is unavailable")}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ worker.logger = slog.New(handler)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+
+ worker.RunOnce(t.Context())
+
+ require.NotEmpty(t, *handler.records)
+
+ attempt := handler.attrsOf(0)
+ assert.Equal(t, jobName, attempt["job_name"])
+ assert.NotNil(t, attempt["job_id"])
+ assert.Equal(t, reference.StorageID, attempt["storage_id"])
+ assert.Equal(t, reference.FileName, attempt["file_name"])
+ assert.Contains(t, (*handler.records)[0].Message, "attempt 1")
+ assert.Contains(t, (*handler.records)[0].Message, "retrying in")
+
+ summary := (*handler.records)[len(*handler.records)-1]
+ assert.Contains(t, summary.Message, "pending")
+ assert.Contains(t, summary.Message, "overdue")
+ assert.Contains(t, summary.Message, "oldest")
+}
+
+func Test_Run_WhenCalledTwice_Panics(t *testing.T) {
+ locator := &fakeLocator{provider: &fakeProvider{}}
+ worker := newTestWorker(t, newTestStore(locator), locator)
+
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+
+ worker.Run(ctx)
+
+ assert.Panics(t, func() { worker.Run(ctx) })
+}
+
+func Test_Run_WhenAPassIsSlow_DoesNotOverlapPasses(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ storageID := createStorageRow(t)
+ // One row per pass and three rows waiting, one of which the startup pass takes
+ // before the ticker exists: a pass that began while another was still running
+ // would find work of its own and the counter would reach two.
+ worker.timings.ClaimBatchSize = 1
+
+ references := []StoredFileReference{
+ {StorageID: storageID, FileName: "backup-1"},
+ {StorageID: storageID, FileName: "backup-2"},
+ {StorageID: storageID, FileName: "backup-3"},
+ }
+
+ var concurrentPasses atomic.Int32
+ var maxConcurrentPasses atomic.Int32
+
+ provider.onDelete = func() {
+ if inFlight := concurrentPasses.Add(1); inFlight > maxConcurrentPasses.Load() {
+ maxConcurrentPasses.Store(inFlight)
+ }
+
+ time.Sleep(5 * worker.timings.WorkerTick)
+
+ concurrentPasses.Add(-1)
+ }
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, references)
+ }))
+
+ ctx, cancel := context.WithCancel(t.Context())
+ stopped := make(chan struct{})
+
+ go func() {
+ defer close(stopped)
+
+ worker.Run(ctx)
+ }()
+
+ time.Sleep(20 * worker.timings.WorkerTick)
+ cancel()
+
+ select {
+ case <-stopped:
+ case <-time.After(5 * time.Second):
+ t.Fatal("worker did not stop")
+ }
+
+ assert.Equal(t, int32(1), maxConcurrentPasses.Load(),
+ "exactly one pass runs at a time: a tick arriving during a pass is skipped, not stacked")
+}
+
+func Test_Run_WhenContextIsCancelledMidPass_FinishesTheProviderCallBeforeReturning(t *testing.T) {
+ provider := &fakeProvider{}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ ctx, cancel := context.WithCancel(t.Context())
+
+ deleteReturned := false
+ provider.onDelete = func() {
+ cancel()
+ time.Sleep(20 * time.Millisecond)
+
+ deleteReturned = true
+ }
+
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+
+ stopped := make(chan struct{})
+
+ go func() {
+ defer close(stopped)
+
+ worker.Run(ctx)
+ }()
+
+ select {
+ case <-stopped:
+ case <-time.After(5 * time.Second):
+ t.Fatal("worker did not stop")
+ }
+
+ assert.True(t, deleteReturned, "shutdown must not leave a provider call unjoined")
+}
+
+func Test_TimingsForTest_ShrinksEveryBoundReachingStoreAndWorker(t *testing.T) {
+ production := ProductionTimings()
+ shrunk := TimingsForTest()
+
+ assert.Less(t, shrunk.CommitWindow, production.CommitWindow)
+ assert.Less(t, shrunk.AttemptTimeout, production.AttemptTimeout)
+ assert.Less(t, shrunk.RetryBaseDelay, production.RetryBaseDelay)
+ assert.Less(t, shrunk.RetryMaxDelay, production.RetryMaxDelay)
+ assert.Less(t, shrunk.WorkerTick, production.WorkerTick)
+
+ locator := &fakeLocator{provider: &fakeProvider{}}
+ store := newTestStore(locator)
+
+ assert.Equal(t, shrunk, store.GetTimings())
+ assert.Equal(t, shrunk, newTestWorker(t, store, locator).timings)
+}
diff --git a/backend/internal/features/storages/models/azure_blob/model.go b/backend/internal/features/storages/models/azure_blob/model.go
index 984a51cf..58e573ff 100644
--- a/backend/internal/features/storages/models/azure_blob/model.go
+++ b/backend/internal/features/storages/models/azure_blob/model.go
@@ -248,6 +248,10 @@ func (s *AzureBlobStorage) DeleteFile(
deleteCtx, cancel := context.WithTimeout(context.Background(), azureDeleteTimeout)
defer cancel()
+ if err := s.discardUncommittedBlocks(deleteCtx, client, blobName); err != nil {
+ return err
+ }
+
_, err = client.DeleteBlob(
deleteCtx,
s.ContainerName,
@@ -255,10 +259,10 @@ func (s *AzureBlobStorage) DeleteFile(
nil,
)
if err != nil {
- var respErr *azcore.ResponseError
- if errors.As(err, &respErr) && respErr.StatusCode == 404 {
+ if isAzureNotFound(err) {
return nil
}
+
return fmt.Errorf("failed to delete blob from Azure: %w", err)
}
@@ -305,12 +309,10 @@ func (s *AzureBlobStorage) TestConnection(encryptor encryption.FieldEncryptor) e
containerClient := client.ServiceClient().NewContainerClient(s.ContainerName)
_, err = containerClient.GetProperties(ctx, nil)
if err != nil {
- var respErr *azcore.ResponseError
- if errors.As(err, &respErr) {
- if respErr.StatusCode == 404 {
- return fmt.Errorf("container '%s' does not exist", s.ContainerName)
- }
+ if isAzureNotFound(err) {
+ return fmt.Errorf("container '%s' does not exist", s.ContainerName)
}
+
if errors.Is(err, context.DeadlineExceeded) {
return errors.New("failed to connect to Azure Blob Storage. Please check params")
}
@@ -387,6 +389,59 @@ func (s *AzureBlobStorage) Update(incoming *AzureBlobStorage) {
}
}
+// An interrupted upload leaves staged blocks that no blob points at, and Azure
+// keeps billing for them for a week. There is no API that drops them, so cleanup
+// commits an empty list, which turns them into a zero-length blob the delete then
+// removes. Existence is the guard rather than the committed block list, because a
+// blob written with a single Put Blob has no committed blocks and committing an
+// empty list over it would truncate real content.
+//
+// This assumes Azure answers Get Block List for a blob that was never committed.
+// Nothing here can detect it if Azure ever answers 404 instead.
+func (s *AzureBlobStorage) discardUncommittedBlocks(
+ ctx context.Context,
+ client *azblob.Client,
+ blobName string,
+) error {
+ containerClient := client.ServiceClient().NewContainerClient(s.ContainerName)
+
+ _, err := containerClient.NewBlobClient(blobName).GetProperties(ctx, nil)
+ if err == nil {
+ return nil
+ }
+
+ if !isAzureNotFound(err) {
+ return fmt.Errorf("failed to inspect azure blob: %w", err)
+ }
+
+ blockBlobClient := containerClient.NewBlockBlobClient(blobName)
+
+ blocks, err := blockBlobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
+ if err != nil {
+ if isAzureNotFound(err) {
+ return nil
+ }
+
+ return fmt.Errorf("failed to list azure blocks: %w", err)
+ }
+
+ if len(blocks.UncommittedBlocks) == 0 {
+ return nil
+ }
+
+ if _, err := blockBlobClient.CommitBlockList(ctx, nil, nil); err != nil {
+ return fmt.Errorf("failed to discard uncommitted azure blocks: %w", err)
+ }
+
+ return nil
+}
+
+func isAzureNotFound(err error) bool {
+ var responseError *azcore.ResponseError
+
+ return errors.As(err, &responseError) && responseError.StatusCode == http.StatusNotFound
+}
+
func (s *AzureBlobStorage) buildBlobName(fileName string) string {
if s.Prefix == "" {
return fileName
diff --git a/backend/internal/features/storages/models/azure_blob/model_test.go b/backend/internal/features/storages/models/azure_blob/model_test.go
new file mode 100644
index 00000000..0651fb15
--- /dev/null
+++ b/backend/internal/features/storages/models/azure_blob/model_test.go
@@ -0,0 +1,125 @@
+package azure_blob_storage
+
+import (
+ "bytes"
+ "encoding/base64"
+ "fmt"
+ "testing"
+
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/util/testing/containers"
+)
+
+func startAzurite(t *testing.T) (*AzureBlobStorage, *azblob.Client, string) {
+ t.Helper()
+
+ endpoint := containers.StartAzurite(t)
+ address := fmt.Sprintf("%s:%d", endpoint.Host, endpoint.Port)
+ containerName := "uncommitted-blocks"
+
+ connectionString := fmt.Sprintf(
+ "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=http://%s/%s",
+ containers.AzuriteAccountName, containers.AzuriteAccountKey, address, containers.AzuriteAccountName,
+ )
+
+ client, err := azblob.NewClientFromConnectionString(connectionString, nil)
+ require.NoError(t, err)
+
+ _, err = client.CreateContainer(t.Context(), containerName, nil)
+ require.NoError(t, err)
+
+ storage := &AzureBlobStorage{
+ StorageID: uuid.New(),
+ AuthMethod: AuthMethodConnectionString,
+ ConnectionString: connectionString,
+ ContainerName: containerName,
+ }
+
+ return storage, client, containerName
+}
+
+func stageBlock(t *testing.T, client *azblob.Client, containerName, blobName string) {
+ t.Helper()
+
+ blockClient := client.ServiceClient().NewContainerClient(containerName).NewBlockBlobClient(blobName)
+
+ _, err := blockClient.StageBlock(
+ t.Context(),
+ base64.StdEncoding.EncodeToString([]byte("000000")),
+ streaming.NopCloser(bytes.NewReader(bytes.Repeat([]byte("x"), 1024))),
+ nil,
+ )
+ require.NoError(t, err)
+}
+
+func getBlockList(t *testing.T, client *azblob.Client, containerName, blobName string) blockblob.GetBlockListResponse {
+ t.Helper()
+
+ blockClient := client.ServiceClient().NewContainerClient(containerName).NewBlockBlobClient(blobName)
+
+ blocks, err := blockClient.GetBlockList(t.Context(), blockblob.BlockListTypeAll, nil)
+ require.NoError(t, err)
+
+ return blocks
+}
+
+// Azurite's DeleteBlob succeeds on a blob holding only uncommitted blocks and drops
+// them, while Azure returns 404 and keeps charging for them for a week. An
+// end-to-end assertion therefore cannot separate a fixed build from an unfixed one,
+// so the discarding step is exercised on its own.
+func Test_DiscardUncommittedBlocks_WhenUploadWasInterrupted_TurnsThemIntoADeletableBlob(t *testing.T) {
+ storage, client, containerName := startAzurite(t)
+ blobName := "interrupted-" + uuid.New().String()
+
+ stageBlock(t, client, containerName, blobName)
+ require.Len(t, getBlockList(t, client, containerName, blobName).UncommittedBlocks, 1)
+
+ require.NoError(t, storage.discardUncommittedBlocks(t.Context(), client, blobName))
+
+ assert.Empty(t, getBlockList(t, client, containerName, blobName).UncommittedBlocks,
+ "the staged blocks must not survive as an invisible charge")
+
+ properties, err := client.ServiceClient().NewContainerClient(containerName).
+ NewBlobClient(blobName).GetProperties(t.Context(), nil)
+
+ require.NoError(t, err, "committing an empty list leaves a zero-length blob that DeleteFile removes")
+ assert.EqualValues(t, 0, *properties.ContentLength)
+}
+
+func Test_DiscardUncommittedBlocks_WhenBlobExists_LeavesItsContentAlone(t *testing.T) {
+ storage, client, containerName := startAzurite(t)
+ blobName := "committed-" + uuid.New().String()
+
+ _, err := client.UploadBuffer(t.Context(), containerName, blobName, []byte("payload"), nil)
+ require.NoError(t, err)
+
+ // A single Put Blob leaves no committed block list, so an existing blob with
+ // stale staged blocks is exactly the case that would be truncated by a guard
+ // that looked at the committed list instead of at existence.
+ stageBlock(t, client, containerName, blobName)
+
+ blocks := getBlockList(t, client, containerName, blobName)
+ require.Empty(t, blocks.CommittedBlocks, "the case only discriminates while Put Blob leaves no committed list")
+ require.Len(t, blocks.UncommittedBlocks, 1)
+
+ require.NoError(t, storage.discardUncommittedBlocks(t.Context(), client, blobName))
+
+ properties, err := client.ServiceClient().NewContainerClient(containerName).
+ NewBlobClient(blobName).GetProperties(t.Context(), nil)
+
+ require.NoError(t, err)
+ assert.EqualValues(t, len("payload"), *properties.ContentLength,
+ "an existing blob must not be emptied by cleanup")
+}
+
+func Test_DiscardUncommittedBlocks_WhenNothingExists_Succeeds(t *testing.T) {
+ storage, client, _ := startAzurite(t)
+
+ assert.NoError(t, storage.discardUncommittedBlocks(t.Context(), client, "absent-"+uuid.New().String()))
+}
diff --git a/backend/internal/features/storages/models/ftp/model.go b/backend/internal/features/storages/models/ftp/model.go
index ef8c157d..90f06ae1 100644
--- a/backend/internal/features/storages/models/ftp/model.go
+++ b/backend/internal/features/storages/models/ftp/model.go
@@ -182,13 +182,7 @@ func (f *FTPStorage) DeleteFile(
filePath := f.getFilePath(fileName)
- _, err = conn.FileSize(filePath)
- if err != nil {
- return nil
- }
-
- err = conn.Delete(filePath)
- if err != nil {
+ if err := conn.Delete(filePath); err != nil && !isFtpFileUnavailable(err) {
return fmt.Errorf("failed to delete file from FTP: %w", err)
}
@@ -445,6 +439,17 @@ func (f *FTPStorage) retrieveFileFromOffset(
}, nil
}
+// 550 is the only reply an FTP server has for a missing file, and also its reply
+// for a refused one, so absence is the closest reading the protocol allows.
+func isFtpFileUnavailable(err error) bool {
+ var protocolError *textproto.Error
+ if !errors.As(err, &protocolError) {
+ return false
+ }
+
+ return protocolError.Code == ftp.StatusFileUnavailable
+}
+
func isUnsupportedFtpCommand(err error) bool {
var protocolError *textproto.Error
if !errors.As(err, &protocolError) {
diff --git a/backend/internal/features/storages/models/ftp/model_test.go b/backend/internal/features/storages/models/ftp/model_test.go
new file mode 100644
index 00000000..01fb4ae1
--- /dev/null
+++ b/backend/internal/features/storages/models/ftp/model_test.go
@@ -0,0 +1,40 @@
+package ftp_storage
+
+import (
+ "errors"
+ "net/textproto"
+ "testing"
+
+ "github.com/jlaffaye/ftp"
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_IsFtpFileUnavailable_TellsAMissingFileFromAFailedConnection(t *testing.T) {
+ cases := map[string]struct {
+ err error
+ isAbsent bool
+ }{
+ "file unavailable": {
+ err: &textproto.Error{Code: ftp.StatusFileUnavailable, Msg: "No such file or directory"},
+ isAbsent: true,
+ },
+ "transient error": {
+ err: &textproto.Error{Code: ftp.StatusFileActionIgnored, Msg: "File busy"},
+ isAbsent: false,
+ },
+ "command not implemented": {
+ err: &textproto.Error{Code: ftp.StatusCommandNotImplemented, Msg: "DELE not supported"},
+ isAbsent: false,
+ },
+ "connection failure": {
+ err: errors.New("connection reset by peer"),
+ isAbsent: false,
+ },
+ }
+
+ for name, testCase := range cases {
+ t.Run(name, func(t *testing.T) {
+ assert.Equal(t, testCase.isAbsent, isFtpFileUnavailable(testCase.err))
+ })
+ }
+}
diff --git a/backend/internal/features/storages/models/google_drive/errors.go b/backend/internal/features/storages/models/google_drive/errors.go
new file mode 100644
index 00000000..9e70e378
--- /dev/null
+++ b/backend/internal/features/storages/models/google_drive/errors.go
@@ -0,0 +1,7 @@
+package google_drive_storage
+
+import "errors"
+
+// A missing backups folder means every file in it is missing too, so deletion
+// treats it as a confirmed absence rather than a failure to retry.
+var errBackupsFolderNotFound = errors.New("databasus_backups folder not found")
diff --git a/backend/internal/features/storages/models/google_drive/model.go b/backend/internal/features/storages/models/google_drive/model.go
index 4d7f72f9..a8ebd512 100644
--- a/backend/internal/features/storages/models/google_drive/model.go
+++ b/backend/internal/features/storages/models/google_drive/model.go
@@ -229,6 +229,10 @@ func (s *GoogleDriveStorage) DeleteFile(
if err := s.withRetryOnAuth(deleteCtx, encryptor, func(driveService *drive.Service) error {
folderID, err := s.findBackupsFolder(driveService)
+ if errors.Is(err, errBackupsFolderNotFound) {
+ return nil
+ }
+
if err != nil {
return fmt.Errorf("failed to find backups folder: %w", err)
}
@@ -716,7 +720,6 @@ func (s *GoogleDriveStorage) ensureBackupsFolderExists(
return folder.Id, nil
}
-// findBackupsFolder finds the databasus_backups folder ID
func (s *GoogleDriveStorage) findBackupsFolder(driveService *drive.Service) (string, error) {
query := "name = 'databasus_backups' and mimeType = 'application/vnd.google-apps.folder' and trashed = false"
@@ -730,7 +733,7 @@ func (s *GoogleDriveStorage) findBackupsFolder(driveService *drive.Service) (str
}
if len(results.Files) == 0 {
- return "", fmt.Errorf("databasus_backups folder not found")
+ return "", errBackupsFolderNotFound
}
return results.Files[0].Id, nil
diff --git a/backend/internal/features/storages/models/local/model.go b/backend/internal/features/storages/models/local/model.go
index 5f7fd7bb..502ea0b9 100644
--- a/backend/internal/features/storages/models/local/model.go
+++ b/backend/internal/features/storages/models/local/model.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
+ "io/fs"
"log/slog"
"os"
"path/filepath"
@@ -166,14 +167,17 @@ func (l *LocalStorage) DeleteFile(
logger *slog.Logger,
fileName string,
) error {
- filePath := filepath.Join(config.GetEnv().DataFolder, fileName)
-
- if _, err := os.Stat(filePath); os.IsNotExist(err) {
- return nil
+ // An interrupted write leaves the staging copy behind, and a cross-filesystem
+ // publication can leave a partial file at the final path.
+ paths := []string{
+ filepath.Join(config.GetEnv().DataFolder, fileName),
+ filepath.Join(config.GetEnv().TempFolder, fileName),
}
- if err := os.Remove(filePath); err != nil {
- return fmt.Errorf("failed to delete file: %w", err)
+ for _, path := range paths {
+ if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return fmt.Errorf("failed to delete file: %w", err)
+ }
}
logger.DebugContext(ctx, "deleted file from local storage", "file_name", fileName)
diff --git a/backend/internal/features/storages/models/local/model_test.go b/backend/internal/features/storages/models/local/model_test.go
new file mode 100644
index 00000000..56e4a0fd
--- /dev/null
+++ b/backend/internal/features/storages/models/local/model_test.go
@@ -0,0 +1,93 @@
+package local_storage
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/config"
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
+)
+
+func Test_DeleteFile_WhenWriteWasInterrupted_RemovesTheStagingFile(t *testing.T) {
+ storage := &LocalStorage{}
+ fileName := "interrupted-" + t.Name()
+ stagingPath := filepath.Join(config.GetEnv().TempFolder, fileName)
+
+ require.NoError(t, os.MkdirAll(config.GetEnv().TempFolder, 0o755))
+ require.NoError(t, os.WriteFile(stagingPath, []byte("partial"), 0o600))
+
+ err := storage.DeleteFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)
+
+ require.NoError(t, err)
+ assert.NoFileExists(t, stagingPath)
+}
+
+func Test_DeleteFile_WhenFileWasPublished_RemovesBothPaths(t *testing.T) {
+ storage := &LocalStorage{}
+ fileName := "published-" + t.Name()
+
+ require.NoError(t, storage.SaveFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName, strings.NewReader("payload"),
+ ))
+
+ stagingPath := filepath.Join(config.GetEnv().TempFolder, fileName)
+ require.NoError(t, os.WriteFile(stagingPath, []byte("partial"), 0o600))
+
+ err := storage.DeleteFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)
+
+ require.NoError(t, err)
+ assert.NoFileExists(t, filepath.Join(config.GetEnv().DataFolder, fileName))
+ assert.NoFileExists(t, stagingPath)
+}
+
+func Test_DeleteFile_WhenNothingExists_Succeeds(t *testing.T) {
+ storage := &LocalStorage{}
+
+ err := storage.DeleteFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), "never-written-"+t.Name(),
+ )
+
+ assert.NoError(t, err)
+}
+
+func Test_DeleteFile_WhenCalledTwice_Succeeds(t *testing.T) {
+ storage := &LocalStorage{}
+ fileName := "twice-" + t.Name()
+
+ require.NoError(t, storage.SaveFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName, strings.NewReader("payload"),
+ ))
+
+ require.NoError(t, storage.DeleteFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName,
+ ))
+
+ assert.NoError(t, storage.DeleteFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName,
+ ))
+}
+
+func Test_DeleteFile_WhenRemovalIsRefused_ReturnsTheFilesystemError(t *testing.T) {
+ storage := &LocalStorage{}
+ fileName := filepath.Join("locked-"+t.Name(), "artifact")
+ directory := filepath.Join(config.GetEnv().DataFolder, filepath.Dir(fileName))
+
+ require.NoError(t, os.MkdirAll(directory, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(config.GetEnv().DataFolder, fileName), []byte("x"), 0o600))
+ require.NoError(t, os.Chmod(directory, 0o500))
+
+ t.Cleanup(func() {
+ _ = os.Chmod(directory, 0o755)
+ _ = os.RemoveAll(directory)
+ })
+
+ err := storage.DeleteFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)
+
+ assert.Error(t, err, "an unconfirmed absence must not be reported as a successful deletion")
+}
diff --git a/backend/internal/features/storages/models/nas/model.go b/backend/internal/features/storages/models/nas/model.go
index 3d0357b7..7e0d5ed2 100644
--- a/backend/internal/features/storages/models/nas/model.go
+++ b/backend/internal/features/storages/models/nas/model.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
+ iofs "io/fs"
"log/slog"
"net"
"path/filepath"
@@ -235,13 +236,7 @@ func (n *NASStorage) DeleteFile(
filePath := n.getFilePath(fileName)
- _, err = fs.Stat(filePath)
- if err != nil {
- return nil
- }
-
- err = fs.Remove(filePath)
- if err != nil {
+ if err := fs.Remove(filePath); err != nil && !errors.Is(err, iofs.ErrNotExist) {
return fmt.Errorf("failed to delete file from NAS: %w", err)
}
diff --git a/backend/internal/features/storages/models/nas/model_test.go b/backend/internal/features/storages/models/nas/model_test.go
new file mode 100644
index 00000000..202561e4
--- /dev/null
+++ b/backend/internal/features/storages/models/nas/model_test.go
@@ -0,0 +1,70 @@
+package nas_storage
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
+ "databasus-backend/internal/util/testing/containers"
+)
+
+func startNasStorage(t *testing.T) *NASStorage {
+ t.Helper()
+
+ endpoint := containers.StartSamba(t)
+
+ return &NASStorage{
+ StorageID: uuid.New(),
+ Host: endpoint.Host,
+ Port: endpoint.Port,
+ Share: containers.SambaShare,
+ Username: containers.SambaUsername,
+ Password: containers.SambaPassword,
+ Path: "cleanup-" + uuid.New().String(),
+ }
+}
+
+// A non-empty directory is the one refusal the provider's own API can set up.
+func Test_DeleteFile_WhenTheServerRefusesTheRemoval_ReturnsTheError(t *testing.T) {
+ storage := startNasStorage(t)
+ encryptor := encryption.GetFieldEncryptor()
+ directory := "refused-" + uuid.New().String()
+
+ session, err := storage.createSessionWithContext(t.Context(), encryptor)
+ require.NoError(t, err)
+
+ share, err := session.Mount(storage.Share)
+ require.NoError(t, err)
+
+ require.NoError(t, share.MkdirAll(storage.getFilePath(directory), 0o755))
+
+ t.Cleanup(func() {
+ _ = share.Umount()
+ _ = session.Logoff()
+ })
+
+ require.NoError(t, storage.SaveFile(
+ t.Context(), encryptor, logger.GetLogger(), directory+"/artifact", strings.NewReader("payload"),
+ ))
+
+ err = storage.DeleteFile(t.Context(), encryptor, logger.GetLogger(), directory)
+
+ assert.Error(t, err, "a removal the server refuses must not count as a completed deletion")
+}
+
+// The configured path is a directory that was never created, so this also covers a
+// removal whose parent is missing.
+func Test_DeleteFile_WhenFileIsAbsent_Succeeds(t *testing.T) {
+ storage := startNasStorage(t)
+
+ err := storage.DeleteFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), "never-written-"+uuid.New().String(),
+ )
+
+ assert.NoError(t, err)
+}
diff --git a/backend/internal/features/storages/models/rclone/model_test.go b/backend/internal/features/storages/models/rclone/model_test.go
index a931e4f0..df2c91c7 100644
--- a/backend/internal/features/storages/models/rclone/model_test.go
+++ b/backend/internal/features/storages/models/rclone/model_test.go
@@ -1,10 +1,16 @@
package rclone_storage
import (
+ "os"
+ "path/filepath"
"testing"
+ "github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
)
func Test_ParseConfigContent_SingleRemote_ParsedCorrectly(t *testing.T) {
@@ -116,3 +122,61 @@ func Test_ParseConfigContent_WhitespaceAroundKeysAndValues_Trimmed(t *testing.T)
assert.Equal(t, "s3", sections["myremote"]["type"])
assert.Equal(t, "us-west-2", sections["myremote"]["region"])
}
+
+// getFs strips a leading slash from RemotePath, so an absolute root would become
+// relative to the working directory. The test moves there instead and leaves the
+// remote path empty, which puts the rclone root on the temporary directory.
+func newLocalRcloneStorage(t *testing.T) (*RcloneStorage, string) {
+ t.Helper()
+
+ // The first Encrypt resolves the secret key through config.GetEnv(), which finds
+ // the module root by walking up from the working directory and loads .env from
+ // above it, so it has to run before the test moves.
+ config, err := encryption.GetFieldEncryptor().Encrypt("[testlocal]\ntype = local\n")
+ require.NoError(t, err)
+
+ root := t.TempDir()
+ t.Chdir(root)
+
+ return &RcloneStorage{
+ StorageID: uuid.New(),
+ ConfigContent: config,
+ }, root
+}
+
+func Test_DeleteFile_WhenObjectExists_RemovesOnlyThatPath(t *testing.T) {
+ storage, root := newLocalRcloneStorage(t)
+
+ for _, name := range []string{"backup-1", "backup-10", "backup-1.metadata"} {
+ require.NoError(t, os.WriteFile(filepath.Join(root, name), []byte("payload"), 0o600))
+ }
+
+ err := storage.DeleteFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), "backup-1")
+
+ require.NoError(t, err)
+ assert.NoFileExists(t, filepath.Join(root, "backup-1"))
+ assert.FileExists(t, filepath.Join(root, "backup-10"), "a similarly named object must survive")
+ assert.FileExists(t, filepath.Join(root, "backup-1.metadata"), "a sidecar is its own logical file")
+}
+
+func Test_DeleteFile_WhenObjectIsAbsent_Succeeds(t *testing.T) {
+ storage, _ := newLocalRcloneStorage(t)
+
+ err := storage.DeleteFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), "never-written")
+
+ assert.NoError(t, err)
+}
+
+func Test_DeleteFile_WhenCalledTwice_Succeeds(t *testing.T) {
+ storage, root := newLocalRcloneStorage(t)
+
+ require.NoError(t, os.WriteFile(filepath.Join(root, "backup-1"), []byte("payload"), 0o600))
+
+ require.NoError(t, storage.DeleteFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), "backup-1",
+ ))
+
+ assert.NoError(t, storage.DeleteFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), "backup-1",
+ ))
+}
diff --git a/backend/internal/features/storages/models/s3/model.go b/backend/internal/features/storages/models/s3/model.go
index b9d054db..7ceb7f69 100644
--- a/backend/internal/features/storages/models/s3/model.go
+++ b/backend/internal/features/storages/models/s3/model.go
@@ -53,6 +53,8 @@ const (
defaultMaxPartsPerObject = 1000
manifestSuffix = ".parts"
+ partSuffix = ".part"
+ partIndexDigits = 6
manifestVersion = 1
s3ErrCodeNoSuchBucket = "NoSuchBucket"
@@ -252,6 +254,9 @@ func (s *S3Storage) GetFile(
}), nil
}
+// Chunks are found by listing rather than only through the manifest, because a
+// write can complete numbered objects before it publishes one. The multipart abort
+// matters too: an interrupted upload keeps consuming storage until it is aborted.
func (s *S3Storage) DeleteFile(
ctx context.Context,
encryptor encryption.FieldEncryptor,
@@ -270,45 +275,30 @@ func (s *S3Storage) DeleteFile(
deleteCtx, cancel := context.WithTimeout(context.Background(), s3DeleteTimeout)
defer cancel()
- manifest, hasManifest, err := s.readManifest(deleteCtx, client, baseKey)
+ keys, err := s.derivedObjectKeys(deleteCtx, client, baseKey)
if err != nil {
+ logger.WarnContext(ctx, "failed to discover s3 objects to delete",
+ "file_name", fileName, "object_key", baseKey, "error", err)
+
return err
}
- if hasManifest {
- keys := make([]string, 0, len(manifest.Parts)+1)
- for _, part := range manifest.Parts {
- keys = append(keys, part.Key)
- }
- keys = append(keys, manifestObjectKey(baseKey))
-
- if err := s.removeObjects(deleteCtx, client, keys); err != nil {
- logger.WarnContext(ctx, "failed to delete chunked file from s3",
- "file_name", fileName, "object_key", baseKey, "error", err)
-
- return fmt.Errorf("failed to delete chunked backup from S3: %w", err)
- }
-
- logger.DebugContext(ctx, fmt.Sprintf("deleted chunked file from s3 (%d parts)", len(manifest.Parts)),
- "file_name", fileName, "object_key", baseKey)
+ if err := s.removeObjects(deleteCtx, client, keys); err != nil {
+ logger.WarnContext(ctx, "failed to delete file from s3",
+ "file_name", fileName, "object_key", baseKey, "error", err)
- return nil
+ return fmt.Errorf("failed to delete file from S3: %w", err)
}
- err = client.RemoveObject(
- deleteCtx,
- s.S3Bucket,
- baseKey,
- minio.RemoveObjectOptions{},
- )
- if err != nil {
- logger.WarnContext(ctx, "failed to delete file from s3",
+ if err := s.abortDerivedMultipartUploads(deleteCtx, encryptor, client, baseKey); err != nil {
+ logger.WarnContext(ctx, "failed to abort incomplete s3 uploads",
"file_name", fileName, "object_key", baseKey, "error", err)
- return fmt.Errorf("failed to delete file from S3: %w", err)
+ return err
}
- logger.DebugContext(ctx, "deleted file from s3", "file_name", fileName, "object_key", baseKey)
+ logger.DebugContext(ctx, fmt.Sprintf("deleted file from s3 (%d objects)", len(keys)),
+ "file_name", fileName, "object_key", baseKey)
return nil
}
@@ -397,6 +387,85 @@ func (s *S3Storage) Update(incoming *S3Storage) {
// otherwise we will have to transfer all the data to the new prefix
}
+// Listing matches the exact derived grammar, so deleting "backup-1" cannot reach
+// "backup-10". The manifest key is removed unconditionally, so no caller may name a
+// sidecar "<base>.parts".
+func (s *S3Storage) derivedObjectKeys(
+ ctx context.Context,
+ client *minio.Client,
+ baseKey string,
+) ([]string, error) {
+ keys := []string{baseKey, manifestObjectKey(baseKey)}
+
+ manifest, hasManifest, err := s.readManifest(ctx, client, baseKey)
+ if err != nil {
+ return nil, err
+ }
+
+ seen := map[string]struct{}{keys[0]: {}, keys[1]: {}}
+
+ if hasManifest {
+ for _, part := range manifest.Parts {
+ if _, duplicate := seen[part.Key]; duplicate {
+ continue
+ }
+
+ seen[part.Key] = struct{}{}
+ keys = append(keys, part.Key)
+ }
+ }
+
+ for object := range client.ListObjects(ctx, s.S3Bucket, minio.ListObjectsOptions{
+ Prefix: baseKey + partSuffix,
+ Recursive: true,
+ }) {
+ if object.Err != nil {
+ return nil, fmt.Errorf("failed to list chunk objects: %w", object.Err)
+ }
+
+ if !isPartObjectKey(baseKey, object.Key) {
+ continue
+ }
+
+ if _, duplicate := seen[object.Key]; duplicate {
+ continue
+ }
+
+ seen[object.Key] = struct{}{}
+ keys = append(keys, object.Key)
+ }
+
+ return keys, nil
+}
+
+func (s *S3Storage) abortDerivedMultipartUploads(
+ ctx context.Context,
+ encryptor encryption.FieldEncryptor,
+ client *minio.Client,
+ baseKey string,
+) error {
+ coreClient, err := s.getCoreClient(encryptor)
+ if err != nil {
+ return err
+ }
+
+ for upload := range client.ListIncompleteUploads(ctx, s.S3Bucket, baseKey, true) {
+ if upload.Err != nil {
+ return fmt.Errorf("failed to list incomplete uploads: %w", upload.Err)
+ }
+
+ if upload.Key != baseKey && !isPartObjectKey(baseKey, upload.Key) {
+ continue
+ }
+
+ if err := coreClient.AbortMultipartUpload(ctx, s.S3Bucket, upload.Key, upload.UploadID); err != nil {
+ return fmt.Errorf("failed to abort incomplete upload for %s: %w", upload.Key, err)
+ }
+ }
+
+ return nil
+}
+
func (s *S3Storage) writeConnectionProbe(
client *minio.Client,
objectKey string,
@@ -830,7 +899,22 @@ type objectSpan struct {
}
func partObjectKey(baseKey string, objectIndex int) string {
- return fmt.Sprintf("%s.part%06d", baseKey, objectIndex)
+ return fmt.Sprintf("%s%s%0*d", baseKey, partSuffix, partIndexDigits, objectIndex)
+}
+
+func isPartObjectKey(baseKey, candidate string) bool {
+ index, found := strings.CutPrefix(candidate, baseKey+partSuffix)
+ if !found || len(index) != partIndexDigits {
+ return false
+ }
+
+ for _, digit := range index {
+ if digit < '0' || digit > '9' {
+ return false
+ }
+ }
+
+ return true
}
func manifestObjectKey(baseKey string) string {
diff --git a/backend/internal/features/storages/models/sftp/model.go b/backend/internal/features/storages/models/sftp/model.go
index cd5fc412..328e3e81 100644
--- a/backend/internal/features/storages/models/sftp/model.go
+++ b/backend/internal/features/storages/models/sftp/model.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
+ "io/fs"
"log/slog"
"net"
"strings"
@@ -194,13 +195,7 @@ func (s *SFTPStorage) DeleteFile(
filePath := s.getFilePath(fileName)
- _, err = client.Stat(filePath)
- if err != nil {
- return nil
- }
-
- err = client.Remove(filePath)
- if err != nil {
+ if err := client.Remove(filePath); err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("failed to delete file from SFTP: %w", err)
}
diff --git a/backend/internal/features/storages/models/sftp/model_test.go b/backend/internal/features/storages/models/sftp/model_test.go
new file mode 100644
index 00000000..02e0fdbc
--- /dev/null
+++ b/backend/internal/features/storages/models/sftp/model_test.go
@@ -0,0 +1,56 @@
+package sftp_storage
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
+ "databasus-backend/internal/util/testing/containers"
+)
+
+func startSftpStorage(t *testing.T) *SFTPStorage {
+ t.Helper()
+
+ endpoint := containers.StartSftp(t)
+
+ return &SFTPStorage{
+ StorageID: uuid.New(),
+ Host: endpoint.Host,
+ Port: endpoint.Port,
+ Username: containers.SftpUsername,
+ Password: containers.SftpPassword,
+ SkipHostKeyVerify: true,
+ Path: containers.SftpUploadDir,
+ }
+}
+
+// A non-empty directory is the one refusal the provider's own API can set up.
+func Test_DeleteFile_WhenTheServerRefusesTheRemoval_ReturnsTheError(t *testing.T) {
+ storage := startSftpStorage(t)
+ encryptor := encryption.GetFieldEncryptor()
+ directory := "refused-" + uuid.New().String()
+
+ client, sshConn, err := storage.connectWithContext(t.Context(), encryptor, 30*time.Second)
+ require.NoError(t, err)
+
+ require.NoError(t, client.Mkdir(storage.getFilePath(directory)))
+
+ t.Cleanup(func() {
+ _ = client.Close()
+ _ = sshConn.Close()
+ })
+
+ require.NoError(t, storage.SaveFile(
+ t.Context(), encryptor, logger.GetLogger(), directory+"/artifact", strings.NewReader("payload"),
+ ))
+
+ err = storage.DeleteFile(t.Context(), encryptor, logger.GetLogger(), directory)
+
+ assert.Error(t, err, "a removal the server refuses must not count as a completed deletion")
+}
diff --git a/backend/internal/features/storages/s3_cleanup_test.go b/backend/internal/features/storages/s3_cleanup_test.go
new file mode 100644
index 00000000..c9b2d628
--- /dev/null
+++ b/backend/internal/features/storages/s3_cleanup_test.go
@@ -0,0 +1,171 @@
+package storages
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/google/uuid"
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ s3_storage "databasus-backend/internal/features/storages/models/s3"
+ "databasus-backend/internal/util/encryption"
+ "databasus-backend/internal/util/logger"
+ "databasus-backend/internal/util/testing/containers"
+)
+
+func startS3CleanupTarget(t *testing.T) (*s3_storage.S3Storage, *minio.Client, string) {
+ t.Helper()
+
+ endpoint := containers.StartMinio(t)
+ address := addressOf(endpoint)
+
+ container, err := setupS3Container(t.Context(), address)
+ require.NoError(t, err, "failed to set up the S3 container")
+
+ storage := &s3_storage.S3Storage{
+ StorageID: uuid.New(),
+ S3Bucket: container.bucketName,
+ S3Region: container.region,
+ S3AccessKey: container.accessKey,
+ S3SecretKey: container.secretKey,
+ S3Endpoint: "http://" + address,
+ }
+
+ client, err := minio.New(address, &minio.Options{
+ Creds: credentials.NewStaticV4(container.accessKey, container.secretKey, ""),
+ Secure: false,
+ Region: container.region,
+ })
+ require.NoError(t, err)
+
+ return storage, client, container.bucketName
+}
+
+func objectExists(t *testing.T, client *minio.Client, bucket, key string) bool {
+ t.Helper()
+
+ _, err := client.StatObject(t.Context(), bucket, key, minio.StatObjectOptions{})
+
+ return err == nil
+}
+
+func putObject(t *testing.T, client *minio.Client, bucket, key, body string) {
+ t.Helper()
+
+ _, err := client.PutObject(
+ t.Context(), bucket, key, strings.NewReader(body), int64(len(body)), minio.PutObjectOptions{},
+ )
+ require.NoError(t, err)
+}
+
+func deleteThroughStorage(t *testing.T, storage *s3_storage.S3Storage, fileName string) error {
+ t.Helper()
+
+ return storage.DeleteFile(t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName)
+}
+
+func Test_S3DeleteFile_WhenChunksHaveNoManifest_RemovesThemAnyway(t *testing.T) {
+ storage, client, bucket := startS3CleanupTarget(t)
+ fileName := "orphan-chunks"
+
+ for index := 1; index <= 3; index++ {
+ putObject(t, client, bucket, fmt.Sprintf("%s.part%06d", fileName, index), "chunk")
+ }
+
+ require.NoError(t, deleteThroughStorage(t, storage, fileName))
+
+ for index := 1; index <= 3; index++ {
+ key := fmt.Sprintf("%s.part%06d", fileName, index)
+ assert.False(t, objectExists(t, client, bucket, key), "chunk %s survived cleanup", key)
+ }
+}
+
+func Test_S3DeleteFile_WhenNeighbourSharesThePrefix_LeavesItAlone(t *testing.T) {
+ storage, client, bucket := startS3CleanupTarget(t)
+
+ putObject(t, client, bucket, "backup-1", "one")
+ putObject(t, client, bucket, "backup-1.part000001", "one chunk")
+ putObject(t, client, bucket, "backup-10", "ten")
+ putObject(t, client, bucket, "backup-10.part000001", "ten chunk")
+ putObject(t, client, bucket, "backup-1.metadata", "sidecar")
+
+ require.NoError(t, deleteThroughStorage(t, storage, "backup-1"))
+
+ assert.False(t, objectExists(t, client, bucket, "backup-1"))
+ assert.False(t, objectExists(t, client, bucket, "backup-1.part000001"))
+ assert.True(t, objectExists(t, client, bucket, "backup-10"), "a similarly named object must survive")
+ assert.True(t, objectExists(t, client, bucket, "backup-10.part000001"))
+ assert.True(t, objectExists(t, client, bucket, "backup-1.metadata"), "a sidecar is its own logical file")
+}
+
+func Test_S3DeleteFile_WhenUploadWasInterrupted_AbortsTheMultipartUpload(t *testing.T) {
+ storage, client, bucket := startS3CleanupTarget(t)
+ fileName := "interrupted-upload"
+
+ coreClient := minio.Core{Client: client}
+
+ uploadID, err := coreClient.NewMultipartUpload(t.Context(), bucket, fileName, minio.PutObjectOptions{})
+ require.NoError(t, err)
+
+ _, err = coreClient.PutObjectPart(
+ t.Context(), bucket, fileName, uploadID, 1,
+ bytes.NewReader(make([]byte, 5*1024*1024)), 5*1024*1024, minio.PutObjectPartOptions{},
+ )
+ require.NoError(t, err)
+
+ require.True(t, hasIncompleteUpload(t, client, bucket, fileName))
+
+ require.NoError(t, deleteThroughStorage(t, storage, fileName))
+
+ assert.False(t, hasIncompleteUpload(t, client, bucket, fileName),
+ "an interrupted upload keeps consuming storage until it is aborted")
+}
+
+func Test_S3DeleteFile_WhenCalledTwice_Succeeds(t *testing.T) {
+ storage, client, bucket := startS3CleanupTarget(t)
+ fileName := "twice"
+
+ putObject(t, client, bucket, fileName, "payload")
+
+ require.NoError(t, deleteThroughStorage(t, storage, fileName))
+ assert.NoError(t, deleteThroughStorage(t, storage, fileName))
+}
+
+func Test_S3DeleteFile_WhenChunkedFileWasPublished_RemovesPartsAndManifest(t *testing.T) {
+ storage, client, bucket := startS3CleanupTarget(t)
+ fileName := "chunked-" + uuid.New().String()
+
+ payload := strings.Repeat("x", 6*1024*1024)
+ require.NoError(t, storage.SaveFile(
+ t.Context(), encryption.GetFieldEncryptor(), logger.GetLogger(), fileName, strings.NewReader(payload),
+ ))
+
+ require.NoError(t, deleteThroughStorage(t, storage, fileName))
+
+ assert.False(t, objectExists(t, client, bucket, fileName))
+ assert.False(t, objectExists(t, client, bucket, fileName+".parts"))
+ assert.False(t, objectExists(t, client, bucket, fileName+".part000001"))
+}
+
+func hasIncompleteUpload(t *testing.T, client *minio.Client, bucket, key string) bool {
+ t.Helper()
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+
+ for upload := range client.ListIncompleteUploads(ctx, bucket, key, true) {
+ require.NoError(t, upload.Err)
+
+ if upload.Key == key {
+ return true
+ }
+ }
+
+ return false
+}
diff --git a/backend/internal/features/storages/testing.go b/backend/internal/features/storages/testing.go
index b837e118..d1b2131b 100644
--- a/backend/internal/features/storages/testing.go
+++ b/backend/internal/features/storages/testing.go
@@ -5,11 +5,24 @@ import (
"github.com/google/uuid"
+ storage_files "databasus-backend/internal/features/storages/files"
local_storage "databasus-backend/internal/features/storages/models/local"
s3_storage "databasus-backend/internal/features/storages/models/s3"
"databasus-backend/internal/util/encryption"
)
+// A test can assert absence right after the operation that caused it instead of
+// waiting on the production ticker.
+func DrainStorageFileDeletions(
+ ctx context.Context,
+ references ...storage_files.StoredFileReference,
+) error {
+ dependencies := storageFileDependencies
+ dependencies.Timings = storage_files.TimingsForTest()
+
+ return storage_files.NewDeletionWorker(storageFileStore, dependencies).DrainForTest(ctx, references...)
+}
+
func SetStorageDatabaseCountersForTest(counters ...StorageDatabaseCounter) {
storageService.storageDatabaseCounters = counters
}
diff --git a/backend/internal/util/logger/redaction.go b/backend/internal/util/logger/redaction.go
index ac8ba349..6f668005 100644
--- a/backend/internal/util/logger/redaction.go
+++ b/backend/internal/util/logger/redaction.go
@@ -94,6 +94,12 @@ func redactAttr(attr slog.Attr) slog.Attr {
return attr
}
+// The entry point for callers that persist a provider error outside a log record,
+// where the sink's redaction never runs but the same rules still have to hold.
+func Redact(message string) string {
+ return redactMessage(message)
+}
+
// Credentials and addresses reach the message through fmt.Sprintf at call sites - audit messages
// in particular embed raw user emails, and those messages are now exported off-box.
// The strings.Contains guards keep three regex scans off the hot path: this runs on every record,
diff --git a/backend/migrations/20260910093602_add_storage_pending_deletions.sql b/backend/migrations/20260910093602_add_storage_pending_deletions.sql
new file mode 100644
index 00000000..c74d9a2c
--- /dev/null
+++ b/backend/migrations/20260910093602_add_storage_pending_deletions.sql
@@ -0,0 +1,58 @@
+-- +goose Up
+-- +goose StatementBegin
+-- A row here is an outstanding obligation to remove one stored file. It is created
+-- before the first byte reaches the provider and removed by the transaction that
+-- publishes the file as part of a backup, so every other outcome (provider error,
+-- crash, rolled back catalog write, cancellation) leaves the obligation standing.
+CREATE TABLE storage_pending_deletions (
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ storage_id UUID NOT NULL,
+ file_name TEXT NOT NULL,
+ -- Earliest moment the cleanup worker may act on this file. A fresh write starts
+ -- one commit window in the future; a failed write and every deletion request
+ -- move it to now.
+ not_before TIMESTAMPTZ NOT NULL DEFAULT now(),
+ -- Two meanings on purpose. It counts deletion attempts for the retry backoff,
+ -- and it is the generation a write receipt carries: anything that takes the file
+ -- away from its writer increments it, so the stale receipt can no longer publish.
+ attempt_count INTEGER NOT NULL DEFAULT 0,
+ last_error TEXT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+ALTER TABLE storage_pending_deletions
+ ADD CONSTRAINT fk_storage_pending_deletions_storage_id
+ FOREIGN KEY (storage_id) REFERENCES storages (id) ON DELETE CASCADE;
+
+ALTER TABLE storage_pending_deletions
+ ADD CONSTRAINT uq_storage_pending_deletions_storage_file UNIQUE (storage_id, file_name);
+
+ALTER TABLE storage_pending_deletions
+ ADD CONSTRAINT ck_storage_pending_deletions_file_name_not_blank CHECK (file_name <> '');
+
+ALTER TABLE storage_pending_deletions
+ ADD CONSTRAINT ck_storage_pending_deletions_attempt_count_not_negative CHECK (attempt_count >= 0);
+
+CREATE INDEX idx_storage_pending_deletions_not_before
+ ON storage_pending_deletions (not_before);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP INDEX IF EXISTS idx_storage_pending_deletions_not_before;
+
+ALTER TABLE storage_pending_deletions
+ DROP CONSTRAINT IF EXISTS ck_storage_pending_deletions_attempt_count_not_negative;
+
+ALTER TABLE storage_pending_deletions
+ DROP CONSTRAINT IF EXISTS ck_storage_pending_deletions_file_name_not_blank;
+
+ALTER TABLE storage_pending_deletions
+ DROP CONSTRAINT IF EXISTS uq_storage_pending_deletions_storage_file;
+
+ALTER TABLE storage_pending_deletions
+ DROP CONSTRAINT IF EXISTS fk_storage_pending_deletions_storage_id;
+
+DROP TABLE IF EXISTS storage_pending_deletions;
+-- +goose StatementEnd
diff --git a/openspec/changes/ensure-storage-file-cleanup/.openspec.yaml b/openspec/changes/ensure-storage-file-cleanup/.openspec.yaml
new file mode 100644
index 00000000..1a62d62b
--- /dev/null
+++ b/openspec/changes/ensure-storage-file-cleanup/.openspec.yaml
@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-09-06
diff --git a/openspec/changes/ensure-storage-file-cleanup/design.md b/openspec/changes/ensure-storage-file-cleanup/design.md
new file mode 100644
index 00000000..226c9eef
--- /dev/null
+++ b/openspec/changes/ensure-storage-file-cleanup/design.md
@@ -0,0 +1,258 @@
+## Context
+
+See `proposal.md` for the user-visible failure. File I/O currently starts from methods on `Storage` (`backend/internal/features/storages/model.go:40` and `:68`), and `StorageService.GetStorageByID` exposes that model to every caller (`backend/internal/features/storages/service.go:331`). This leaves lifecycle policy spread across backup packages.
+
+The logical path starts a storage writer inside each database-specific use case. A producer can write a valid storage object and still return a dump error later (`backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go:199`). `Backuper.MakeBackup` deletes the main file only in its cancellation branch (`backend/internal/features/backups/backups/backuping/logical/backuper.go:157`) and does not delete it for the ordinary failure branch at `:183`. Retention logs storage deletion errors and removes the catalog row anyway (`backend/internal/features/backups/backups/backuping/logical/cleaner.go:88`).
+
+The physical stream has the same split outcome: storage writes in a goroutine (`backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go:148`), while the producer result is classified later. FULL and INCR names are persisted before upload (`backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go:40`), but restart recovery still contains the old assumption that `file_name` is NULL until completion (`backend/internal/features/backups/backups/backuping/physical/scheduler.go:710`). Physical cascade cleanup contains both fail-open and fail-closed direct deletion policies (`backend/internal/features/backups/backups/core/physical/service/service.go:1056` and `:1073`), and both run inside the cascade transaction that opens at `:613`, so a rollback restores rows whose objects are already gone.
+
+Provider cleanup is not yet a uniform logical-file operation. S3 can leave chunk objects without a manifest when its best-effort abandon path fails (`backend/internal/features/storages/models/s3/model.go:535`), while its later deletion discovers chunks only through that manifest (`backend/internal/features/storages/models/s3/model.go:273`). Local storage writes to the temporary folder first (`backend/internal/features/storages/models/local/model.go:53`) but deletion checks only the data folder (`backend/internal/features/storages/models/local/model.go:163`). FTP, SFTP and NAS collapse broad stat failures into successful absence (`ftp/model.go:185`, `sftp/model.go:197`, `nas/model.go:238`).
+
+Two existing guards bound how far storage lifecycle work has to reach. Workspace deletion already refuses while the workspace still contains databases (`backend/internal/features/databases/service.go:662`, reached through the listener loop at `backend/internal/features/workspaces/services/workspace_service.go:182`), so a workspace cannot be removed out from under live backup rows. Storage deletion already refuses while databases are attached to it (`backend/internal/features/storages/service.go:192`). What is missing is the case where a storage keeps historical backup rows with no attached database, and the two artifact families behave differently there. `backups.storage_id` is `ON DELETE CASCADE` (`backend/migrations/20260127132617_add_cascade_delete_for_backups_and_databases.sql:8`), so deleting that storage drops the logical rows and strands every object they name. The physical tables declare no delete rule at all (`backend/migrations/20260523120000_add_physical_pg_17_backups.sql:132`, `:168`, `:212`, `:287` and `:317`), so the same deletion already fails with a raw constraint error instead of an actionable one.
+
+One platform guarantee shapes every coordination decision below. The `single-instance-runtime` capability specifies exactly one application process for each installation, and background `Run` methods already panic on a second in-process call. Writers and the cleanup worker therefore share one address space, which is why an in-memory set is a sound way to tell a live write from a dead one.
+
+The design follows `AGENTS.md` and `backend/AGENTS.md`. In particular, the storage feature keeps its repository instance private and exposes no repository getter (`backend/internal/features/storages/di.go:12`), background `Run` methods reject a second call, files have one responsibility, DI uses positional fields, comments explain hidden constraints only and tests assert observable cleanup rather than log text.
+
+## Goals / Non-Goals
+
+**Goals:**
+
+- Close the crash window before upload and the commit window after upload.
+- Give backup code one transactional way to publish or discard every file it owns.
+- Make a logical file deletion complete and idempotent for every provider.
+- Prevent stale cleanup from deleting a published file or a later write.
+- Keep the existing test suite meaningful once deletion stops being synchronous.
+
+**Non-Goals:**
+
+- Treating storage as the owner of backup file hierarchies or retention decisions.
+- Keeping successful file rows after their catalog transaction commits.
+- Retrying uploads inside the storage file layer.
+- Refactoring the read path in this change.
+- Listing and deleting arbitrary provider objects that no Databasus record can identify.
+- Proving that a provider location is unchanged, whether by contacting the provider or by restricting configuration edits.
+- Adding a coordination protocol for several concurrent cleanup processes, which the deployment model does not produce.
+
+## Decisions
+
+### Invert the default: a file is kept only when a transaction claims it
+
+The whole change rests on one invariant. **A pending deletion row exists for a file from before its first byte is written until a committed catalog transaction takes responsibility for it.** Every failure mode is then the same failure mode: the row stays, and the worker eventually removes the file. There is no separate reasoning for a provider error, a crash, a rolled back catalog transaction or a canceled backup, because none of them removes the row.
+
+This inverts today's default, where a file is kept unless some code path remembers to delete it, and the missing paths are the bug.
+
+Alternatives rejected:
+
+- Keep the file name in the catalog and add a deletion-only queue fed by terminal transitions. Three artifact types already commit their object name before upload (`backuping/logical/scheduler.go:158`, `create_full_uc.go:40`, `create_incremental_uc.go:76`), so this is tempting. It fails on the crash window: a process that stops between the upload and any terminal transition leaves an object that no row and no queue entry names. It also spreads recovery reasoning across one sweep per artifact type, which is where the current bug already lives, at `scheduler.go:710`.
+- A deletion-only queue created after `SaveFile` returns. A process can stop before the insert.
+- A permanent file registry. Backup tables already own successful file relationships, and copying them would create two authorities for retention and restore.
+
+### Put the write and deletion boundary in `storages/files`
+
+Create `backend/internal/features/storages/files` with `Store`, `StoredFileReference`, `WriteReceipt`, `PendingDeletionRepository` and `DeletionWorker`. `Store.WriteFile` accepts a `StoredFileReference` carrying a storage ID and a file name, and backup code still creates the exact name. `ConfirmFileWrites` and `RequestFileDeletions` accept batches because a backup result commonly owns a base file and sidecars.
+
+The directory is `files` and the package is `storage_files`, matching how the repository already aliases provider packages (`local` declares `package local_storage`). The bare name `files` is taken by `backend/internal/util/files`, which declares `package files_utils`. `Store` names what the type owns, which is the state of pending deletions, and `StoredFileReference` names the pair it carries rather than the fact that it points at something. `Store` reads as a whole at its call sites, `storage_files.Store`, and it names a thing that holds state rather than an unnamed activity, which is what `AGENTS.md:126` rules out in `Manager` and `Handler`. A name built around the obligation, such as `CleanupObligations`, would describe one of its three methods and misname `WriteFile`.
+
+The child package declares narrow `FileWriter`, `FileRemover` and `StorageLocator` interfaces and does not import its parent `storages` package. Splitting write from removal keeps each name honest: `Store` uses the writer, `DeletionWorker` uses the remover and neither name hides a second effect. The parent package implements `StorageLocator`, wires `Store` and exposes its getter, which avoids a Go import cycle and leaves provider selection in the feature that owns storage configuration.
+
+Only write and deletion callers move in this change. Reads keep using the existing path because they do not participate in publication or cleanup. `backend/cmd/storage_command.go` is the one production exception: the `--test-storage` startup probe writes and removes its own file before the application exists, so it has no catalog state to publish and no row to own.
+
+Alternatives rejected:
+
+- Add all methods to the current `StorageService`. It already owns configuration CRUD, authorization, audit, attachment counting, testing and transfer, and file lifecycle would mix a control plane with a data path.
+- Move reads at the same time. That adds restore and streaming risk without strengthening cleanup.
+- Name the type `FileManager`. `Manager` names nothing, which `AGENTS.md:126` rules out.
+
+### Use one pending deletion table
+
+Add `storage_pending_deletions` with these columns:
+
+| Column | Purpose |
+| --- | --- |
+| `id` | UUID row and receipt identity |
+| `storage_id`, `file_name` | Logical file reference, unique as a pair |
+| `not_before` | Earliest time the worker may act on this file |
+| `attempt_count` | Retry count, and the generation that invalidates a stale receipt |
+| `last_error` | Sanitized bounded failure text |
+| `created_at`, `updated_at` | Ordering and operational diagnosis |
+
+The table has a unique constraint on `(storage_id, file_name)`, a foreign key to `storages` with `ON DELETE CASCADE` and one index on `not_before`. It holds no state column, no lease token, no backup ID, no owner type and no provider configuration or credentials. Row presence is the state.
+
+`ON DELETE CASCADE` is deliberate. The alternative, a restrictive foreign key, turns every teardown fixture and every workspace cascade into a raw database error and buys nothing: the storage row is what carries the credentials, so once it is gone the obligation is not executable anyway. Storage deletion instead drains what it can and records what it could not, which is covered below.
+
+`attempt_count` carries two meanings, so the migration and the repository both say so. It schedules retries, and it is the generation number that lets a write receipt detect that cleanup, or another caller, took the file away from its writer.
+
+Alternatives rejected:
+
+- Separate lifecycle states for writing, awaiting commit and pending deletion. Each state needs its own expiry, its own conversion pass and its own test matrix, and all three answer the same question that row presence already answers.
+- Separate write-intent and deletion tables. Their transition would need another transaction and duplicate indexes without adding a guarantee.
+- `ON DELETE RESTRICT` with a service-level refusal in front of it. It makes an unreachable provider location permanently block deletion of the storage and its workspace, which then needs an abandonment timer, an audit trail for abandonment and fixture changes in three shared teardown helpers.
+
+### Register before writing, confirm inside the publishing transaction
+
+`Store.WriteFile` generates the row ID, joins an in-process set of live writes, inserts the pending deletion row with `not_before = now() + commitWindow`, then calls the provider. Joining before the insert and starting the row ineligible close the same gap from two sides: a row the worker can see belongs to a writer that already joined, and a freshly inserted row is not due anyway. When the provider returns:
+
+- on success it refreshes `not_before` to `now() + commitWindow` conditionally on `attempt_count` still holding the value it started with, leaves the live set and returns a `WriteReceipt` carrying the row ID and that `attempt_count`;
+- on failure it sets `not_before = now()` under the same condition and only then leaves the live set, so the file becomes eligible exactly when its writer is gone;
+- if either update matches no row, someone requested deletion of this file while the write was running, so it returns an error instead of a receipt and the caller never tries to publish.
+
+The insert takes no action on conflict, and `WriteFile` then fails without sending a byte, reporting that the name is already registered. Two things produce that conflict. A cascading removal can request deletion of a file whose write has not started, which is a legitimate race and the reason the write must not proceed; if the worker drains that row first the later write simply starts a fresh obligation, which is equally safe. A second concurrent write of the same name also collides, and per-attempt naming makes that a defect rather than a case to merge, so the error says what the writer can observe rather than guessing which cause it was.
+
+The caller then does one of two things:
+
+- `ConfirmFileWrites(ctx, tx, receipts)` inside the transaction that makes the backup, WAL, history, manifest or metadata catalog state usable. It deletes each row by `id` conditionally on the receipt's `attempt_count` and fails the transaction unless every receipt matched exactly one row;
+- `RequestFileDeletions(ctx, tx, references)` inside the transaction that records failure, cancellation, retention removal or user deletion.
+
+`commitWindow` is the only deadline in the design. It answers a single question: how long may a file sit recorded but unclaimed by any catalog transaction before cleanup assumes its publisher is never coming. Publication normally follows the upload within the same function, so the window is generous.
+
+There is no heartbeat and no lease renewal. A long physical backup does not need one, because the live-write set already tells the worker that this file has a writer, and the set is empty for every write inherited from a process that stopped. Removing the heartbeat also removes the failure mode where a metadata database blip cancels a multi-hour backup.
+
+Each independent attempt gets an immutable name, and attempt means one call to a provider, not one backup. Logical naming already satisfies that: `GenerateFilename` runs once per backup at scheduling (`backend/internal/features/backups/backups/backuping/logical/scheduler.go:158`) and nothing retries the upload under that name. Three physical paths do not.
+
+FULL and INCR reuse one name across the codec fallback. `streamWithCodecFallback` retries the same backup through zstd, gzip and none with one `fileName` (`backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go:50`), so a server that rejects zstd produces two or three writes of the same name, and its doc comment says so outright at `:45`. Under the rule above the second write would be refused and the downgrade path would stop working. Calling `buildObjectName` (`.../executor.go:152`) once per attempt does not help, because its four inputs are constant across the loop. It gains an attempt UUID instead, which also covers a same-codec retry that a codec suffix would not. Two comments and one test state the old shape and go with it: `executor.go:45`, the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and the `expected` table field in `executor_test.go`.
+
+The loop cannot mint that name by itself. It holds neither the naming inputs nor the repository that persists `file_name`, and its parameter list is already eight positional values, so it takes a per-attempt `mintAndSaveAttemptName` callback from its caller, named for both of its effects because it produces the name and persists it on the row, and the stream inputs move to a named struct rather than growing that list. The two callers own the repositories the callback needs (`create_full_uc.go:40` and `create_incremental_uc.go:76`, each followed by its `Save`). One use of `fileName` deliberately does not follow the attempt: `newPgBasebackupCommand` takes it as the pg_basebackup label (`pg_basebackup.go:50`), which stays stable across attempts because it names the backup, not the object.
+
+Everything downstream then reads the settled name from the stream result rather than from the caller's variable. `streamWithCodecFallback` returns it, `runStreamParams` carries it into the stream (`.../executor.go:79`) so the upload-time manifest derived at `stream.go:317` follows it, and the remaining consumers move with them: the chain-broken removal and the reported `FileName` (`create_full_uc.go:84` and `:92`), `streamResult.FileName` (`:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`).
+
+WAL and history rebuild their keys from identity. `walSegmentObjectName` derives from database, timeline and segment name (`.../wal_upload.go:439`) and is called twice per upload, once for the artifact at `:232` and once for the sidecar at `:380`. The history key derives from database and timeline (`.../timeline.go:231`), and its sidecar appends `.metadata` to that key at `:246`. Both gain a UUID minted once per upload attempt, and both sidecars derive from the minted name rather than calling the generator again, or the artifact and its sidecar would carry different UUIDs and every consumer that reads `file_name` off the row and appends `.metadata` would miss. History already mints `historyFileID` one line above its key (`timeline.go:230`), so it has the value it needs. Restore already reads these names from catalog rows rather than rebuilding them, so only tests that rebuild the key are affected.
+
+Backup packages also stop reaching providers directly, and a test enforces it rather than a convention. It walks the non-test files under `backend/internal/features/backups` for calls to `SaveFile` and `DeleteFile` on a provider value. That scope keeps out the storages feature's own methods, its provider tests, the `--test-storage` probe and the backup test fixtures that legitimately create and remove provider objects to set up a case. The rule is worth a test because the old call shape still compiles everywhere and reintroducing it would silently reopen the leak.
+
+Alternatives rejected:
+
+- Unexport the provider write and delete methods. Provider implementations live in child packages, so the methods have to stay exported for the storages feature itself.
+- Add a custom linter rule. It would live outside the test command every contributor already runs, and the check is one package walk.
+- Treat provider success as publication. This leaves a file unowned when the following catalog write fails.
+- Let `Store` update backup tables. That would teach storage about logical, FULL, INCR, WAL, history, manifest and metadata relationships.
+- Reuse object names across attempts. A pending deletion for the previous attempt would remove the new bytes.
+
+### Let the worker skip live writes instead of leasing them
+
+`DeletionWorker` holds the live-write mutex across its claim statement and passes the set into it: `UPDATE ... SET attempt_count = attempt_count + 1, not_before = now() + attemptTimeout WHERE id IN (SELECT id FROM storage_pending_deletions WHERE not_before <= now() AND id <> ALL(<live writes>) ORDER BY not_before LIMIT <batch> FOR UPDATE SKIP LOCKED) RETURNING *`. It commits the claim, then deletes remotely without holding a database lock. Both completion and rescheduling are conditional on the claimed `attempt_count`: success deletes the row, failure records a sanitized error and a jittered exponential `not_before` capped at a package maximum.
+
+`SKIP LOCKED` matters because `RequestFileDeletions` runs inside a caller's catalog transaction, and task 8.4 makes database removal one transaction that records every FULL, INCR, WAL, history, manifest and metadata reference it owns. Without it, one such transaction would stall the whole claim batch rather than the rows it holds. Holding the mutex across that statement, rather than snapshotting the set before it, is what makes the exclusion sound. A writer joins the set before it inserts, so a writer that joins after the claim has not inserted its row yet and a writer that joined before it is in the exclusion list. Snapshotting first leaves a gap in between, and the gap is reachable: `RequestFileDeletions` can pull a live writer's row forward to `now()`, which a cascade does to a FULL or INCR file whose name the catalog already carries (`create_full_uc.go:40`). The mutex is safe to hold here because `FOR UPDATE SKIP LOCKED` means the statement never waits on a row lock, and no writer holds it across an insert.
+
+Two properties make this enough, and both follow from one process:
+
+- a row absent from the live set has no writer in this address space, and no other address space exists, so nothing is going to publish it;
+- the claiming update is a compare and swap on `attempt_count`, so a receipt issued before the claim can no longer confirm, and a slow attempt that outlived its `attemptTimeout` can neither reschedule nor complete over a newer one.
+
+The live set is a mutex-guarded map of row IDs written by `Store.WriteFile` and read by the worker inside the same process. A writer joins before its insert and leaves only after its own database update has committed, so the worker can never observe a row as both eligible and unowned while its writer is still deciding the outcome.
+
+The worst case this accepts is a provider write that hangs without honoring cancellation. Its row stays in the live set and its file is not cleaned until the call returns or the process stops. Nothing is lost and nothing is deleted early, so it needs no bounded-stop contract and no conformance suite across eight providers.
+
+`Run` uses `atomic.Bool.Swap(true)` and panics on a second call, matching `backend/internal/features/backups/backups/download/download_token/background.go:29`. It scans immediately at startup, then on a ticker and exits on context cancellation. Every pass uses the stable `job_name` `storage_file_deletion` and a fresh `job_id`; attempt logs also retain storage ID, file name, attempt count and next retry time after centralized redaction. Each pass ends with one summary line carrying how many obligations are pending, how many are overdue and the age of the oldest one, because the change adds no interface for reading this table. The age is what separates a queue that is draining from one obligation that will never succeed, which two counts alone cannot show once a capped retry makes every stuck row look briefly overdue. The pass is one exported method that `Run` calls at startup and on every tick, so a test drives cleanup as a sequence rather than sleeping on the ticker. A `DrainForTest` helper in the package's `testing.go` runs passes until the files a test names carry no obligation, following the seam used by `backend/internal/features/backups/backups/backuping/physical/testing.go:221`.
+
+Alternatives rejected:
+
+- Lease tokens with expiry and a termination grace. They reconstruct, across two extra columns and a conversion pass, the fact that the live set already knows.
+- Delete the row first and re-insert it if the provider call fails. It makes the claim trivially safe and reintroduces the exact bug this change exists to fix, because a crash between the delete and the re-insert loses the obligation.
+- Hold `FOR UPDATE` during remote deletion. A storage outage would occupy database locks for the full network timeout.
+- Report queue depth through a metrics endpoint or an API. A summary log line reaches the same operator through the sink they already have.
+
+### Keep catalog transactions in the owning backup service
+
+The backup service owns its PostgreSQL transaction and passes its `*gorm.DB` to the two batch methods. The storage repository remains private behind `Store`; backup packages never receive it. The deletion-request insert and the receipt confirmation use the caller transaction, while remote provider calls always run after commit in the background worker. The registration insert inside `WriteFile` is the one that must not: it commits on its own connection before the provider call, because a caller transaction that later rolled back would take the obligation with it.
+
+Database deletion currently invokes listeners and then performs a separate cascading delete. This change makes database removal transactional and changes its internal listener contract to receive the same `*gorm.DB`. The backup listener enumerates logical, FULL, INCR, WAL, history, manifest and metadata references and requests their deletion before the database row is removed. Cancellation and replication-slot cleanup dial the source database, so that work runs to completion before the transaction opens rather than inside it. Otherwise one catalog transaction would stay open across an SSH or PostgreSQL dial timeout. Inside the transaction there are only catalog writes and deletion records, and catalog removal cannot commit unless every record is durable.
+
+That transaction still holds uncommitted unique-index entries on `(storage_id, file_name)` for every reference it recorded. A `WriteFile` for one of those names blocks until it commits, rather than failing fast, which is the WAL upload racing a database deletion. `SKIP LOCKED` covers the worker, not the writer. The wait is bounded by the transaction, which is why the external listener work is outside it, and blocking is the correct outcome anyway: the write cannot be allowed to publish a file the cascade is removing.
+
+This is a deliberate GORM seam. It is smaller than a general unit-of-work abstraction and matches existing physical services that already coordinate several repositories in one transaction. Method names include `FileWrites` or `FileDeletions`, and their parameters use named structs rather than long positional lists.
+
+Alternatives rejected:
+
+- Record deletions before catalog deletion in a separate transaction. A worker could remove a file while a failed catalog transaction leaves the backup visible.
+- Record them after catalog deletion. A process stop between commits loses the only durable file reference.
+- Give the repository to backup packages. That violates the repository ownership rule and lets callers bypass the contract.
+
+### Accept a deletion request at any point in the write lifecycle
+
+`RequestFileDeletions` never fails because a writer is still running and never fails because the same reference was requested before. It inserts the row with `not_before = now()` when it is missing, because a file nobody is writing has no writer to wait for, and against an existing row it sets `not_before = now()` and increments `attempt_count`. That increment is what makes the request safe against a live writer: the writer's own completion update and its receipt both carry the earlier value, so the writer cannot extend the window and its caller cannot publish the file.
+
+The alternative shape, where the method rejects a request against a live writer, breaks a caller that already exists. `DeleteFull` cascade deletion runs while WAL archiving is active and is documented to catch a claim mid-upload (`.../wal_upload.go:275`). A rejection there would roll back the whole cascade transaction every time a segment happens to be uploading.
+
+Alternatives rejected:
+
+- Return an error and require the caller to cancel and join the writer first. Cascade deletion has no handle on a WAL writer in another goroutine, and the rollback would be indistinguishable from a real failure.
+- Delete the row and let the writer recreate it. The writer would then publish an object nobody is tracking.
+
+### Make `DeleteFile` mean every provider representation
+
+The provider contract remains one logical `fileName`, but every implementation must remove its own derived physical state:
+
+- local removes matching temporary and published paths, including a partial destination from cross-filesystem publication;
+- S3 removes the base object, manifest, exact numbered chunk objects and matching incomplete multipart uploads even when no manifest exists;
+- Azure removes the committed blob and any uncommitted blocks for its exact blob name. There is no API that drops staged blocks, so cleanup commits an empty block list first, which turns them into a zero-length blob the delete then removes. That step runs only when the blob does not exist, because a blob written with one Put Blob carries no committed blocks and an empty commit over it would truncate real content. Azurite does not reproduce this, since its delete already succeeds on a blob that holds only uncommitted blocks;
+- FTP, SFTP and NAS drop the stat probe that turned any lookup error into a successful deletion, and classify the removal reply instead. FTP has one irreducible gap: 550 is the only reply for a missing file and also the reply for a refused one, so it counts as absence;
+- Google Drive removes every matching incomplete or completed file created for the attempt, and reads a missing backups folder as absence rather than as a failure to retry forever;
+- rclone removes the exact remote object and reports lookup errors other than confirmed absence.
+
+S3 discovery matches the exact derived grammar rather than a bare prefix, so deleting `backup-1` cannot touch `backup-10`. Provider cleanup may retain its immediate best-effort rollback, but an ignored rollback result never replaces the durable row.
+
+The WAL retention capability needs no delta. Its requirement that each deletion transaction rechecks the FULL claim before removing a segment governs which segments may be chosen, and that recheck still happens inside the transaction that decides. Only the remote object removal moves after the commit.
+
+The connection check is unchanged. A credential that can write but not delete now fails at cleanup time, where the worker records the provider error on the row, keeps retrying and names the storage and file in its attempt log. Checking those permissions up front would reject storage configurations that work today, and it would rewrite `TestConnection` across every provider, call site and test double for a diagnosis that the deletion log already gives.
+
+Alternatives rejected:
+
+- Persist provider-specific part names in the table. That leaks representation details above providers and cannot describe incomplete multipart uploads after a hard stop.
+- Delete only the base object when no manifest exists. The current custom S3 layout can complete numbered objects before publishing its manifest.
+- Treat every stat error as not found. It lets the worker acknowledge deletion during an outage or permission failure.
+
+### Let storage deletion drain, then report what is left
+
+Storage deletion refuses while backup rows still reference the storage, which is the one question that has to leave the feature. It is answered through the existing registration mechanism at `backend/internal/features/storages/interfaces.go:48`, extended with a second method `GetStorageBackupReferences` rather than duplicated as a parallel registry, and implemented by the backup services that already register there. A counter that also reports backup references no longer matches the name `StorageDatabaseCounter`, so the interface, its registrar, the service field `storageDatabaseCounters` (`backend/internal/features/storages/service.go:22`) and the fixture `SetStorageDatabaseCountersForTest` (`backend/internal/features/storages/testing.go:13`) are renamed together.
+
+Pending deletions do not block anything. When a storage with no referencing backups is deleted, the service runs one drain pass over that storage's rows, bounded by both a row count and a wall-clock budget, then deletes the storage row, which cascades the rest away. The budget matters because the drain runs in the request path and the workspace listener repeats it for every storage in the workspace, so without it an unreachable provider would hold the user for the batch size times the provider timeout. The drain claims through the same path as the worker, so it excludes live writes for the same reason and leaves any file still being written to the cascade and the audit entry. Every pending deletion row still present for that storage after the drain goes into one audit-log entry through `AuditLogService.WriteAuditLog` (`backend/internal/features/audit_logs/service.go:24`) naming the storage, how many files are left behind and the first of their names up to a fixed limit, plus the same at warning level. The count tells the user the size of what was left, and the names give them something to act on, without putting thousands of lines into one audit entry. Reporting on what remains rather than on what the drain failed at also covers the rows the drain never saw, which are the ones the worker had already claimed and the ones its skip-locked claim passed over. Workspace deletion reaches storages through the listener at `backend/internal/features/storages/service.go:73` and takes the same path. Draining and writing an audit entry both need a `context.Context`, which `OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) error` (`backend/internal/features/workspaces/interfaces/interfaces.go:6`) does not carry, so the listener contract gains one and all three implementations move with it. Passing `context.Background()` instead would drop the request ID that every other line of the same deletion carries.
+
+The user-visible outcome matches what a retry-until-abandoned design would eventually produce, without making the user wait out an abandonment age for it, and without the fourth lifecycle state and the restrictive foreign key that design needs.
+
+Alternatives rejected:
+
+- Retry until a bounded age, then abandon. It leaves the storage undeletable for the length of that age and needs a fourth lifecycle state to end.
+- Delete the storage silently. The files are still in the user's storage, so the record of what was left has to survive somewhere the user can read.
+- Let storage deletion cascade the backup rows as it does today. Those rows are the only record of the object names, so the objects become unreachable at the moment the rows disappear.
+
+### Keep the existing suite honest once deletion is asynchronous
+
+Two groups of existing tests encode assumptions this change breaks.
+
+Object-absence assertions that run right after a catalog operation now have to drive the worker to completion before asserting. `Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects` in `backend/internal/features/backups/backups/core/physical/service/service_test.go` is one. `Test_WalUpload_DeleteCascadesIntoNullClaim_UploadCompletes_DeleteFileCleansOrphan` (`.../wal_upload_test.go:207`) is the sharper one, because it asserts at `:239` that the orphan is already gone, which is exactly the synchronous `deleteObject` call at `wal_upload.go:282` that this change replaces. Both get the single-pass seam, so they assert sequence rather than elapsed time.
+
+Tests that rebuild a WAL object key from its segment name break once the name carries a per-attempt UUID: `wal_upload_test.go:74`, `wal_upload_test.go:212`, `wal_watch_dir_test.go:51`, `wal_watch_dir_test.go:71` and `wal_stream_test.go:444`. They read the name from the catalog row instead, which is what production restore already does.
+
+The storage and workspace teardown fixtures need no change. `storages.RemoveTestStorage` and `workspaces_testing.RemoveTestWorkspaceDirect` delete rows that cascade into `storage_pending_deletions`, and `workspaces_testing.RemoveTestWorkspace` deletes through the API, which meets no new refusal. Only the storage-deletion controller tests move, because of the new backup-reference conflict. The database fixture does change: task 8.5 moves `DatabaseService.DeleteForTest` onto the transactional listener contract, because it runs the same chain as production deletion and would otherwise stop recording the references that chain owes.
+
+Alternatives rejected:
+
+- Rework these tests at the end of the change. Their assumptions are what tells the implementation whether the contract is right, so discovering them late would make each one look like a regression to explain away.
+- Give tests a synchronous store double that deletes inline. It would leave the production ordering untested exactly where the current bug lives.
+
+## Risks / Trade-offs
+
+- [A hung provider write blocks cleanup of its own file] -> The row stays in the live set until the call returns or the process stops, so the file is cleaned late rather than early. Nothing is deleted under a live writer, which is the property worth protecting.
+- [`attempt_count` carries two meanings] -> The migration comment, the repository method names and the design state that every event taking a file away from its writer increments it. A separate generation column would add a second number that always moves with the first.
+- [A published backup could lose its file if publication is slower than `commitWindow`] -> Confirmation is conditional on the receipt generation, so the transaction fails instead of publishing a file that cleanup already claimed. The backup is then failed rather than silently broken.
+- [Storage deletion now refuses while backup rows reference the storage] -> The refusal is actionable and the path out is the backup deletion flow this change makes reliable. The previous behavior removed the rows and made the objects unreachable forever.
+- [An obligation against a permanently broken target retries forever] -> Nothing bounds the number of attempts, so a file whose provider never accepts a deletion is retried once per capped retry delay for as long as its storage exists. No storage, workspace or backup operation waits on it, and the summary's oldest-obligation age keeps it visible. What it does cost is queue position: the claim orders by `not_before`, so a backlog of permanently overdue rows on one storage sorts ahead of a row that just came due on another, and the second row waits about backlog divided by batch size passes. There is no per-storage fairness rule, because expressing one would need a window function that PostgreSQL will not accept alongside `FOR UPDATE SKIP LOCKED`, and the delay it would prevent is bounded, visible through the oldest-obligation age and measured in passes rather than in anything a user waits on. This is the price of rejecting the alternative that stops retrying past a bounded age, which needs a fourth lifecycle state and an age threshold to end.
+- [Files remain after a storage is deleted with an unreachable provider] -> The drain attempt and the audit-log entry record exactly which files were left, in the same place the user reads other destructive events.
+- [A credential without delete permission fails only at cleanup time] -> Every failure is recorded on its row with the provider error and named in the attempt log, and the summary's oldest-obligation age shows that something is not draining. Rejecting such credentials at configuration time would break storages that work today for write-only workloads.
+- [A storage repointed at another location strands its old files] -> Per-attempt names mean the pending deletion targets a name that cannot collide with anything else, so the risk is a leftover file in the old location, which is what happens today.
+- [A second application process would defeat the live-write set] -> Writes in another process are invisible to this one's set, so its worker could delete a file mid-write. The `single-instance-runtime` capability makes that topology unsupported, and this change relies on that guarantee rather than re-deriving it.
+- [Provider cleanup discovery can delete a similarly named object] -> Providers match exact derived names, and MinIO coverage includes prefix-collision fixtures.
+- [Transactional database removal holds a transaction open across listener work] -> The listeners that talk to the source database keep doing best-effort work outside the transaction, and only catalog writes and deletion records run inside it. Production `DeleteDatabase` (`backend/internal/features/databases/service.go:252`) has no deadlock retry today; only the test path at `:299` retries `40P01`. A longer transaction makes a `40P01` more likely, so the deletion returns that error to its caller and the operation is repeated, which is safe because nothing was committed.
+- [Backups that failed before this change keep their files] -> They are cleaned when the backup or its database is deleted, which routes through the new contract. A migration that seeded them would run remote deletions derived from historical rows on the first upgraded start, against provider code whose complete-deletion behavior nothing has exercised yet.
+
+## Migration Plan
+
+1. Add `storage_pending_deletions`, its constraint, index and cascade. Nothing writes to it yet.
+2. Deploy the provider complete-deletion changes.
+3. Deploy the file store and migrate logical writes and terminal-state transactions, then physical FULL and INCR, WAL, history, sidecars, manifests, database deletion and retention deletion. Remove direct production write and deletion calls from backup packages in the same release.
+4. Start one `DeletionWorker` after dependency setup.
+5. Add the storage deletion reference check and the drain on delete.
+6. Verify queue drain, retry logs, provider absence and backup restore coverage before considering the rollout complete.
+
+Rollback first stops the worker, then reverts callers and provider changes. The down migration drops the table, which discards obligations recorded by the newer release; the release notes say so, because the alternative is a down migration that refuses to run.
diff --git a/openspec/changes/ensure-storage-file-cleanup/proposal.md b/openspec/changes/ensure-storage-file-cleanup/proposal.md
new file mode 100644
index 00000000..05c27f56
--- /dev/null
+++ b/openspec/changes/ensure-storage-file-cleanup/proposal.md
@@ -0,0 +1,54 @@
+## Why
+
+Logical and physical backup failures can leave partial or fully uploaded files behind after the catalog marks the backup failed or removes its row. Direct best-effort `DeleteFile` calls cannot survive a storage outage, a process restart or a database failure between uploading a file and publishing its catalog state.
+
+## What Changes
+
+- Add one storage-file lifecycle boundary that accepts a storage ID and a caller-chosen, unique file name. Backup code keeps ownership of file names and relationships; storage code owns provider selection and the cleanup obligation for one file.
+- Record a pending deletion for every file before the first byte reaches the provider, and let the PostgreSQL transaction that publishes the backup remove that record. A file is kept only because a committed catalog transaction claimed it, so every other outcome leaves the obligation in place.
+- Drain those obligations in one background worker with capped exponential backoff, persisted sanitized errors and a per-pass summary log.
+- Require every storage provider to remove the complete physical representation derived from a logical file name, including partial files, temporary files, chunk objects, manifests, uncommitted blocks and incomplete multipart uploads.
+- Give every object name a component unique to the write attempt that produced it. WAL and timeline history keys are currently derived from database, timeline and segment identity alone, and physical FULL and INCR keys are constant across the codec fallback that retries one backup through zstd, gzip and none, so in both cases a retry addresses the same object a pending cleanup is about to delete. Existing stored objects keep their names, which the catalog rows already carry.
+- Migrate logical backups, physical FULL and INCR backups, WAL and history uploads, sidecars, manifests, retention cleanup, cancellation and restart recovery to the new write and deletion contract.
+- Stop publishing a logical backup whose encryption metadata could not be written. Today that failure is logged and the backup is marked completed, which leaves a backup nobody can restore.
+- Add regression coverage for returned errors, process interruption, catalog commit failure, repeated requests, storage outages and confirmed eventual cleanup. MinIO tests cover the S3 representations that cannot be observed through a single object lookup.
+- Attempt a storage's outstanding cleanup when that storage is deleted, then record in the audit log every file the attempt could not remove. Outstanding cleanup never makes a storage or a workspace undeletable.
+- Rework the existing tests that assert synchronous object removal or derive a WAL object name from its segment name, so the suite proves eventual cleanup instead of breaking on it.
+- **BREAKING**: internal backup write and deletion callers stop receiving a raw `*storages.Storage` for file I/O and use the new file store instead. No compatibility adapter remains in backup packages.
+- **BREAKING**: deleting a storage configuration is rejected while backup rows still reference it, with one actionable conflict for both of the shapes this replaces. Logical backup rows currently cascade away and strand their objects forever, while physical full, incremental, WAL and history rows have no delete rule at all and already fail with a raw database constraint error. Either way the backups have to be removed through their own deletion path first.
+- **BREAKING**: the internal database-removal listener contract changes shape. `OnBeforeDatabaseRemove` receives the caller's transaction so its listeners can record deletions atomically, and the three existing implementations move to it. The old signature is deleted rather than kept alongside.
+- **BREAKING**: a logical backup whose encryption metadata cannot be written or validated now fails instead of completing. The sidecar was best effort before, so a backup could be published with no way to decrypt it.
+- **BREAKING**: provider deletion stops reporting success for an ambiguous result. FTP, SFTP and NAS return connection, permission and unknown stat errors instead of treating them as absence, and local deletion also removes the staging path. The `--test-storage` probe at `backend/cmd/storage_command.go:80` keeps calling the provider directly, so it starts failing on a lookup it used to pass.
+- **BREAKING**: the workspace-deletion listener contract gains a `context.Context`, because the storage listener now drains cleanup and writes an audit entry. All three implementations move with it.
+- **BREAKING**: the storage attachment-count registration contract changes shape. `StorageDatabaseCounter` gains a second method reporting backup references, so it, its registrar, the service field and the test fixture are renamed to say what they now report. Both registered implementations move with it.
+
+## Capabilities
+
+### New Capabilities
+
+- `storage-file-cleanup`: Defines pending deletion records, transactional publication, eventual deletion, retry behavior, provider cleanup guarantees and integration with logical and physical backup failures.
+
+### Modified Capabilities
+
+- `local-storage-file-lifecycle`: Extends idempotent local deletion to remove both published files and staging files left by interrupted writes.
+
+## Impact
+
+The backend gains a `backend/internal/features/storages/files` package, a PostgreSQL table for pending deletions and one background worker started by `backend/cmd/main.go`. Storage provider implementations under `backend/internal/features/storages/models` receive a stronger deletion contract. Logical and physical backup use cases, backupers, cleaners, schedulers, database deletion listeners and WAL upload code move their write and deletion paths to the new boundary. `backend/cmd/storage_command.go` keeps calling provider methods directly, because the `--test-storage` startup probe owns its own file and has no catalog state to publish.
+
+Deleting a storage first attempts its outstanding cleanup and then writes one audit-log entry for whatever it could not remove. The pending deletion row belongs to its storage row and disappears with it, so the storage and workspace teardown fixtures keep working unchanged; the database fixture moves onto the new listener contract with production deletion. Physical and WAL tests have to read object names from catalog rows rather than rebuilding them, and object-absence assertions have to run the deletion worker to completion first.
+
+The change adds schema and internal Go API changes but no new external dependency. Existing completed backup rows and their stored object names remain readable. The work answers to `AGENTS.md` and `backend/AGENTS.md`.
+
+## Out of scope
+
+- Keeping a permanent inventory of every successfully stored file.
+- Retrying failed uploads; the existing backup and WAL schedulers retain that policy.
+- Moving download and restore reads to the new package.
+- A UI or public API for inspecting and manually retrying file operations.
+- Discovering arbitrary orphaned objects that have no Databasus file name or pending deletion record.
+- Backfilling cleanup for backup rows that already failed before this change. Their files are removed when the backup or its database is deleted, which routes through the new contract.
+- Proving storage target identity, or restricting which storage configuration fields may change while cleanup is pending. Every write attempt produces a name no other attempt uses, so a repointed storage leaves its old files in the old location rather than deleting the wrong ones.
+- Verifying cleanup permissions during a storage connection check. A missing permission surfaces as a recorded deletion failure instead.
+- Changing how workspace deletion already refuses while the workspace still contains databases.
+- Changing retention selection, recovery-chain rules or point-in-time restore semantics.
diff --git a/openspec/changes/ensure-storage-file-cleanup/specs/local-storage-file-lifecycle/spec.md b/openspec/changes/ensure-storage-file-cleanup/specs/local-storage-file-lifecycle/spec.md
new file mode 100644
index 00000000..b52d6b3d
--- /dev/null
+++ b/openspec/changes/ensure-storage-file-cleanup/specs/local-storage-file-lifecycle/spec.md
@@ -0,0 +1,27 @@
+## MODIFIED Requirements
+
+### Requirement: Local storage deletion is idempotent
+
+Local storage SHALL remove both the requested path from the configured data folder and the matching staging path from the configured temporary folder. A missing path in either location SHALL be treated as successfully deleted only when the filesystem confirms that it does not exist.
+
+#### Scenario: Published file is deleted
+
+- **WHEN** a caller deletes an existing published file
+- **THEN** local storage removes it from the data folder
+- **AND** it removes a matching staging file when one exists
+
+#### Scenario: Interrupted staging file is deleted
+
+- **WHEN** an interrupted write leaves a file only in the temporary folder
+- **THEN** local storage removes that staging file
+
+#### Scenario: Missing file is deleted
+
+- **WHEN** neither the published path nor the staging path exists
+- **THEN** local storage returns success
+
+#### Scenario: Filesystem lookup fails
+
+- **WHEN** local storage cannot determine whether either path exists
+- **THEN** deletion returns the filesystem error
+
diff --git a/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md b/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md
new file mode 100644
index 00000000..41eb1ac7
--- /dev/null
+++ b/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md
@@ -0,0 +1,268 @@
+## Purpose
+
+Defines how Databasus publishes backup files and eventually removes every file that belongs to a failed, canceled, interrupted or deleted backup operation.
+
+## ADDED Requirements
+
+### Requirement: A stored file is kept only when a transaction claims it
+
+The system SHALL record a pending deletion for a unique file name before sending bytes to storage, and SHALL remove that record only as part of the transaction that publishes the file as part of a backup. Every other outcome SHALL leave the record in place, so the file is eventually removed.
+
+#### Scenario: File and backup are published together
+
+- **WHEN** storage accepts a file and the owning backup catalog change commits
+- **THEN** the system stops tracking the file for deletion
+- **AND** the file remains in storage
+
+#### Scenario: Catalog publication fails after upload
+
+- **WHEN** storage accepts a file but the owning catalog transaction fails
+- **THEN** the system keeps the catalog change uncommitted
+- **AND** it eventually removes the uploaded file
+
+#### Scenario: Application stops during a write
+
+- **WHEN** the application stops after recording a pending deletion but before publishing the file
+- **THEN** the record survives the restart
+- **AND** the system eventually removes every storage artifact created by that write
+
+#### Scenario: Provider rejects the write
+
+- **WHEN** a provider returns an error partway through a write
+- **THEN** the file becomes eligible for deletion immediately
+- **AND** the caller receives the provider error
+
+#### Scenario: Publication arrives too late
+
+- **WHEN** a caller tries to publish a file after cleanup has already taken it
+- **THEN** the publishing transaction fails instead of committing
+- **AND** the backup is not recorded as successful
+
+### Requirement: Every failed backup path discards its files
+
+The system SHALL leave the pending deletion in place for every file produced by an unsuccessful logical or physical backup operation, even when storage accepted the file before another stage failed. The owning backup flow SHALL identify the base file and each metadata, manifest, WAL or history sidecar that belongs to that operation.
+
+#### Scenario: Logical dump fails after producing output
+
+- **WHEN** a logical backup tool produces bytes and then exits with an error
+- **THEN** the backup is marked failed
+- **AND** the partial or completed storage file is eventually absent
+
+#### Scenario: Physical backup stream fails after producing output
+
+- **WHEN** a physical FULL or INCR stream writes bytes and then ends unsuccessfully
+- **THEN** the backup is not published as completed
+- **AND** its base file, manifest and metadata files are eventually absent
+
+#### Scenario: Sidecar creation fails
+
+- **WHEN** a base artifact is stored but its required metadata or manifest cannot be produced or published
+- **THEN** every file produced for that attempt is eventually absent
+
+#### Scenario: WAL or history catalog commit fails
+
+- **WHEN** a WAL segment or timeline history file reaches storage but its catalog transaction does not commit
+- **THEN** the artifact and its sidecar are eventually absent
+- **AND** a later retry uses a different per-attempt object name
+
+#### Scenario: Active backup is canceled
+
+- **WHEN** a user or scheduler cancels a backup that may still be writing
+- **THEN** every file from the canceled attempt is eventually absent
+- **AND** no file is removed while its write is still running
+
+### Requirement: Catalog removal and deletion requests are atomic
+
+The system SHALL commit the removal or terminal failure of catalog records together with durable deletion requests for all files those records own. It SHALL NOT perform remote storage deletion while holding that catalog transaction open.
+
+#### Scenario: Backup is removed while storage is unavailable
+
+- **WHEN** retention or a user removes a backup while its storage provider is unavailable
+- **THEN** the catalog removal and its deletion requests commit together
+- **AND** the system retries the remote deletion after storage becomes available
+
+#### Scenario: Deletion request cannot be persisted
+
+- **WHEN** the system cannot persist every required deletion request
+- **THEN** the related catalog removal or terminal state change rolls back
+- **AND** the catalog continues to identify the files that still exist
+
+#### Scenario: A cascade rolls back after part of it succeeded
+
+- **WHEN** a cascading removal fails partway and its transaction rolls back
+- **THEN** every catalog record it touched is restored
+- **AND** none of the files those records name have been removed from storage
+
+#### Scenario: A database with physical backups is deleted
+
+- **WHEN** database deletion cascades FULL, INCR, WAL, history, manifest or metadata catalog rows
+- **THEN** exact deletion requests for every referenced artifact commit in the same database transaction as the database removal
+- **AND** a failure to persist any request rolls back the database removal
+
+#### Scenario: Deletion is requested more than once
+
+- **WHEN** the same storage identifier and file name are requested for deletion repeatedly
+- **THEN** the system retains one cleanup obligation
+- **AND** each request succeeds without creating competing deletions
+
+### Requirement: A deletion request is always accepted
+
+Requesting deletion of a file SHALL succeed whatever state that file is in, including while its writer is still running. A request made during an active write SHALL take the file away from that write, so the write cannot be published afterwards.
+
+#### Scenario: Deletion is requested during an active write
+
+- **WHEN** a catalog transaction requests deletion for a file whose writer has not stopped
+- **THEN** the request succeeds and the transaction can commit
+- **AND** the file is eventually absent
+
+#### Scenario: A cascading removal meets an in-flight upload
+
+- **WHEN** a cascading catalog removal covers a WAL segment that is uploading at that moment
+- **THEN** the removal commits without waiting for the upload
+- **AND** the uploaded object is eventually absent
+
+#### Scenario: A write completes after its file was requested for deletion
+
+- **WHEN** a write finishes successfully after deletion was requested for the same file
+- **THEN** the write reports that the file was taken for deletion
+- **AND** its caller cannot publish that file as part of a backup
+
+### Requirement: Cleanup never removes a file that is still being written or already published
+
+The system SHALL use an immutable, unique object name for each independent write attempt. It SHALL NOT delete a file whose write is still running in the application process, and it SHALL NOT start a write for a file name that already carries a pending deletion. A deletion attempt that has taken a file SHALL invalidate any outstanding permission to publish it, and an attempt that outlived its own timeout SHALL NOT overwrite the result of a newer attempt.
+
+#### Scenario: A long write outlives the cleanup interval
+
+- **WHEN** a write runs for longer than the cleanup worker's scan interval
+- **THEN** the worker does not delete that file while the write is running
+- **AND** the write completes normally
+
+#### Scenario: Cleanup takes a file that was uploaded but never published
+
+- **WHEN** cleanup takes a file whose upload finished but whose catalog transaction never committed
+- **THEN** any later attempt to publish that file fails
+- **AND** the file is removed from storage
+
+#### Scenario: A file name already carries a pending deletion
+
+- **WHEN** a write starts for a file name whose deletion was already requested
+- **THEN** the write fails without sending bytes to the provider
+- **AND** the existing cleanup obligation keeps its own schedule
+
+#### Scenario: An attempt outlives its own timeout
+
+- **WHEN** a deletion attempt finishes after its timeout and the file was already picked up again
+- **THEN** the late attempt does not reschedule or overwrite the newer attempt's state
+
+#### Scenario: Cleanup worker stops after remote deletion
+
+- **WHEN** the worker stops after deleting the remote file but before recording completion
+- **THEN** a later worker repeats the idempotent deletion
+- **AND** the cleanup obligation is removed only after deletion is confirmed
+
+### Requirement: Deletion removes the complete logical file
+
+Every storage provider SHALL treat deletion as removal of all physical state derived from the requested logical file name. Missing state SHALL count as success only when the provider confirms that it is absent; connection, authorization and lookup failures SHALL remain errors, except where a protocol gives one reply for both absence and refusal.
+
+#### Scenario: Provider write leaves partial state
+
+- **WHEN** a write fails after creating a temporary file, partial remote file, staged block, chunk object, manifest or multipart upload
+- **THEN** deletion removes every piece derived from the logical file name
+
+#### Scenario: S3 manifest is missing
+
+- **WHEN** an interrupted chunked S3 write leaves completed chunk objects or incomplete multipart uploads without a manifest
+- **THEN** deletion finds and removes only the exact chunks and uploads derived from that logical file name
+- **AND** similarly prefixed files remain untouched
+
+#### Scenario: File is already absent
+
+- **WHEN** the provider confirms that every representation of the requested file is missing
+- **THEN** deletion succeeds
+
+#### Scenario: Provider lookup fails
+
+- **WHEN** a connection, permission or other lookup error prevents the provider from confirming absence
+- **THEN** deletion returns an error
+- **AND** the cleanup obligation remains pending
+
+#### Scenario: A protocol cannot separate absence from refusal
+
+- **WHEN** a provider's only reply for a missing file is also its reply for a refused one
+- **THEN** deletion treats that reply as absence
+- **AND** every other failure the protocol can distinguish is still an error
+
+### Requirement: Failed deletions remain retryable and observable
+
+The system SHALL retry failed deletions with exponential delay, a maximum delay and jitter. It SHALL persist the attempt count, the next attempt time and a sanitized bounded error message. Each worker pass SHALL have a stable `job_name` and a fresh `job_id`, and SHALL report how many obligations are pending, how many are overdue and the age of the oldest one. Every deletion attempt it logs SHALL identify the job, storage, file, attempt and next retry without exposing credentials.
+
+#### Scenario: Storage outage persists
+
+- **WHEN** repeated deletion attempts fail because storage is unavailable
+- **THEN** each failure remains recorded
+- **AND** retries continue at a bounded cadence
+
+#### Scenario: Storage credentials cannot delete
+
+- **WHEN** a provider rejects deletion because the configured credentials lack the permission
+- **THEN** the recorded error names the missing permission as the provider reported it
+- **AND** the obligation stays pending so it succeeds once the permission is granted
+
+#### Scenario: Storage recovers
+
+- **WHEN** a later deletion attempt succeeds after earlier failures
+- **THEN** the remote file is absent
+- **AND** the system removes the cleanup obligation
+
+#### Scenario: Provider error contains sensitive material
+
+- **WHEN** a provider returns an error containing credentials or a signed address
+- **THEN** persisted state and logs omit the sensitive value
+
+#### Scenario: A deletion attempt is logged
+
+- **WHEN** the worker attempts or reschedules a deletion
+- **THEN** its logs retain `job_name`, `job_id`, `storage_id`, `file_name`, attempt count and next retry time after centralized redaction
+
+#### Scenario: An operator asks what cleanup is outstanding
+
+- **WHEN** cleanup obligations remain
+- **THEN** each worker pass reports the pending count, the overdue count and the age of the oldest obligation
+- **AND** the failing storage and file are identifiable from the recorded state
+
+### Requirement: Deleting a storage reports the files it leaves behind
+
+The system SHALL reject deleting a storage configuration while backup records still reference it, because those records are the only place its file names exist. When no backup records reference it, deletion SHALL attempt the storage's outstanding cleanup first and SHALL then succeed. It SHALL record how many files it could not remove and their names up to a fixed limit. That record SHALL be somewhere the user can read. Outstanding cleanup SHALL NOT make a storage or a workspace permanently undeletable.
+
+#### Scenario: Storage is deleted while backups still reference it
+
+- **WHEN** a user deletes a storage that still has backup records
+- **THEN** deletion is rejected with an actionable conflict
+- **AND** the backups stay readable and removable through their own deletion path
+
+#### Scenario: Storage with reachable pending cleanup is deleted
+
+- **WHEN** a storage with no backup records and reachable pending cleanup is deleted
+- **THEN** the pending files are removed from the provider before the configuration disappears
+- **AND** deletion succeeds
+
+#### Scenario: Storage with an unreachable provider is deleted
+
+- **WHEN** a storage is deleted while its provider cannot be reached
+- **THEN** deletion still succeeds
+- **AND** the audit log names the storage, how many files were left behind and their names up to a fixed limit
+
+#### Scenario: Workspace containing such a storage is deleted
+
+- **WHEN** a workspace is deleted whose storages still hold pending cleanup
+- **THEN** deletion follows the same path as deleting each storage
+- **AND** the caller never sees a raw database constraint error
+
+### Requirement: Cleanup state contains no provider credentials
+
+Durable cleanup state SHALL contain file identity, scheduling data, retry data and sanitized errors only. It SHALL NOT duplicate provider credentials, tokens, private keys, signed addresses or complete provider configuration.
+
+#### Scenario: Cleanup state is inspected at rest
+
+- **WHEN** an operator or database tool reads durable cleanup state
+- **THEN** the record contains no provider authentication material or reusable signed address
diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.md
new file mode 100644
index 00000000..4d6a9633
--- /dev/null
+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md
@@ -0,0 +1,78 @@
+## 1. Establish the pending deletion model
+
+- [x] 1.1 Add the `storage_pending_deletions` migration with `not_before`, `attempt_count`, `last_error`, the unique `(storage_id, file_name)` constraint, the `ON DELETE CASCADE` storage foreign key and one index on `not_before`, commenting the dual role of `attempt_count` as retry counter and receipt generation; verify `make migration-up-test` applies it and `make migration-down` reverses it.
+- [x] 1.2 Add `StoredFileReference`, `WriteReceipt` and the narrow `FileWriter`, `FileRemover` and `StorageLocator` contracts plus `PendingDeletionRepository` under `backend/internal/features/storages/files` as `package storage_files`, split across `doc.go`, `store.go`, `repository.go`, `worker.go`, `model.go`, `dto.go`, `errors.go` and `timings.go` so the background loop and the transport types do not share a file with the store; verify with `make test` and confirm by inspection that the package imports neither its parent `storages` package nor any backup repository.
+- [x] 1.3 Implement the repository operations: insert-if-absent, insert-or-take, generation-conditional completion, generation-conditional reschedule and a skip-locked batch claim excluding a caller-supplied set of live IDs; verify rollback, an insert that conflicts with an existing row, duplicate request, stale-generation and empty-batch cases with repository tests, and that a claim skips rows locked by an uncommitted deletion request instead of waiting for it.
+- [x] 1.4 Add the package timing constants for the commit window, the attempt timeout, the retry base and maximum delay and the worker tick, with a test constructor that shrinks them; verify the shrunk values reach every consumer so no test waits on production timings.
+
+## 2. Route writes through the file store
+
+- [x] 2.1 Implement `storage_files.Store.WriteFile` so it joins the in-process live-write set before inserting the pending deletion row, inserts it ineligible by one commit window, calls the provider and then updates `not_before` conditionally on the unchanged generation before leaving the set. Verify that a write whose name already carries a pending deletion fails without sending bytes and reports that the name is already registered rather than guessing which caller registered it, that provider failure leaves the file eligible exactly when its writer is gone, that success returns a receipt carrying the generation, that a write whose file was taken for deletion returns an error instead of a receipt and that the row leaves the live set in every branch including panic.
+- [x] 2.2 Implement `ConfirmFileWrites(ctx, tx, receipts)` and `RequestFileDeletions(ctx, tx, references)` as transaction-bound state changes without remote calls, while the registration insert inside `WriteFile` commits on its own connection so a caller rollback cannot take the obligation with it, where confirmation fails unless every receipt matches exactly one row at its generation; verify commit, rollback, repeated calls, mixed receipt batches and a receipt invalidated by a concurrent deletion request.
+- [x] 2.3 Make `RequestFileDeletions` succeed against every state, inserting a missing row at `not_before = now()` and taking an existing one by setting `not_before` to now and incrementing the generation; verify it never returns an error for a running writer and that the writer's own completion cannot extend a file that was taken from it.
+- [x] 2.4 Wire `storage_files.Store`, its repository, storage locator and background worker through the existing storage dependency setup while keeping reads on the current `StorageService`; verify application construction and shutdown with the relevant dependency-injection tests.
+
+## 3. Make deletion durable and terminating
+
+- [x] 3.1 Implement one deletion-worker pass that takes the live-write mutex, reads the set under it and holds it until its claim statement returns, so no writer can join in between; that statement claims a bounded batch of due rows with skip-locked row selection ordered by `not_before`, excludes the set and increments the generation, commits the claim and only then performs remote I/O; verify a live write is never claimed even when it runs longer than the scan interval or when a cascading deletion request pulls its row forward to now, a file inherited from a stopped process is claimed on the first pass, a claim invalidates an outstanding receipt and an open deletion request does not block the batch.
+- [x] 3.2 Complete a successful deletion by removing the row and reschedule a failed one with capped exponential backoff, jitter, the incremented attempt count and a sanitized error, both conditional on the generation it claimed; verify transient failure, process restart, an attempt that outlived its timeout, an attempt whose row and storage were both removed by a storage deletion while it ran, and eventual success. Since `not_before` is compared against the database clock, drive these tests with the shrunk constants from task 1.4, a deterministic jitter source and fixtures that write `not_before` directly, not with a Go-side clock.
+- [x] 3.3 Persist only bounded, sanitized deletion errors and emit attempt, retry and success logs carrying `job_name`, a fresh `job_id`, `storage_id`, `file_name` and `error` as key-value pairs, with the attempt count and the next retry time in the message text and the per-pass pending count, overdue count and oldest-obligation age in the summary message, following the value-in-message rule in `backend/AGENTS.md`; verify centralized redaction and truncation preserve these fields while removing credentials, storage configuration and signed URLs.
+- [x] 3.4 Add a non-overlapping `Run` guard and a single-pass execution seam, then register graceful startup and shutdown with the application lifecycle in `backend/cmd/main.go`; verify a second `Run` panics, overlapping ticks are skipped and shutdown does not leave an unjoined in-process provider call.
+
+## 4. Enforce the provider cleanup contract
+
+- [x] 4.1 Make local storage deletion remove the exact final file and any staging or partial file for the same operation, while treating only confirmed absence as success; verify failure-after-write and repeated-deletion cases with filesystem tests.
+- [x] 4.2 Make S3-compatible deletion remove the base object, manifest, exact numbered chunks and incomplete multipart uploads even when the manifest was never published, without deleting prefix-neighbor objects; verify all cases against the repository MinIO test container, including repeated deletion and an injected mid-upload failure.
+- [x] 4.3 Make Azure deletion cover committed objects and uncommitted blocks for the exact operation. Guard the block-discarding step on blob existence rather than on a non-empty committed block list, because a blob written with one Put Blob has no committed blocks and committing an empty list over it would truncate real content. The step assumes Azure answers Get Block List for a blob that was never committed, which Azurite does; if Azure answers 404 instead, the step becomes a silent no-op that nothing in this repository can detect. Azurite cannot verify this end to end, because its `DeleteBlob` succeeds on a blob holding only uncommitted blocks and drops them, while Azure returns 404 and keeps charging for the blocks for a week; verify the block-discarding step directly instead, that a committed blob is not emptied by it and that `DeleteFile` still removes a committed blob and tolerates an absent one.
+- [x] 4.4 Make Google Drive treat a missing `databasus_backups` folder as a confirmed absence rather than an error, since cleanup would otherwise retry that obligation for as long as the storage exists, and leave rclone unchanged because it already resolves one exact object and returns any lookup error that is not a confirmed absence. `deleteByName` already lists every file with the exact name in the backups folder and deletes each, which covers a duplicate left by an interrupted attempt. Verify rclone against a local remote for the exact-path, similarly-prefixed, sidecar, absent and repeated cases. Google Drive has no test double or container in this repository and needs live OAuth, so it stays covered by inspection.
+- [x] 4.5 Tighten FTP, SFTP and NAS deletion so only an explicit not-found result is acknowledged as success and connectivity, permission and ambiguous failures are retried. Each one drops its stat probe and classifies the removal reply instead, because the probe was where any error became a silent success. Verify with provider tests that a stored file is gone after deletion and that an absent one still succeeds, and unit test the FTP reply classifier. Cover the refusal branch for SFTP and NAS by deleting a non-empty directory, in each provider's own package, where the test can reach the connection helper that creates it; `SaveFile` cannot, because it does not create parent directories. Absence is covered by the existing provider table, except on NAS, where the configured path is a directory that never exists and so also covers a removal whose parent is missing. FTP has no equivalent, because 550 is its answer to both a missing file and a refused one, so its classifier is unit tested instead.
+
+## 5. Integrate logical backups
+
+- [x] 5.1 Change logical database writers to accept the file store and return write receipts for every uploaded artifact, including metadata, without exposing a raw writable `Storage`; verify MySQL, PostgreSQL, MariaDB and MongoDB use-case tests compile and exercise the new contract.
+- [x] 5.2 Confirm all logical receipts in the same database transaction that publishes a successful backup, and leave the pending deletions in place on every failure and cancellation path; verify transaction rollback leaves files recoverable for cleanup and no remote deletion occurs inside the transaction.
+- [x] 5.3 Rename the PostgreSQL reproduction test `Test_MakeBackup_WhenUsecaseStoresArtifactThenFails_LeavesArtifact` to `Test_MakeBackup_WhenUsecaseStoresArtifactThenFails_ArtifactRemoved` and convert it, then cover cancellation after a stored artifact and metadata that cannot be restored from. These cases live at the backuper rather than once per engine, because every engine now reaches storage through the same file store and the discard logic they exercise is one code path; the engine use cases keep their own tests for producing bytes. Run the worker until idle and assert the catalog state plus actual absence of the base file and its metadata sidecar.
+- [x] 5.4 Cover a process restart after upload but before logical backup publication; verify the unpublished file is eventually deleted while a transaction-confirmed file is retained.
+
+## 6. Integrate physical full and incremental backups
+
+- [ ] 6.1 Route full and incremental stream uploads and their sidecars through the file store and return receipts only after writer goroutines are joined. Give `buildObjectName` (`.../executor.go:152`) an attempt UUID, since its four current inputs are constant across the codec fallback loop, and move its parameters into an `objectNameSpec` struct rather than adding a fifth positional value and a second adjacent `uuid.UUID`. Have `streamWithCodecFallback` (`.../executor.go:50`) mint each name through a `mintAndSaveAttemptName` callback its two callers supply, named for both of its effects, because the loop holds neither the naming inputs nor the repository that persists `file_name`, and move that function's stream inputs into a `streamAttemptSpec` struct rather than growing an already eight-value positional list. The callback becomes the only producer of object names: the pre-loop `buildObjectName` call and its `Save` (`create_full_uc.go:40` with its assignment and `Save` at `:42` and `:43`, `create_incremental_uc.go:76` with its own at `:78` and `:80`) stop minting the object key and keep only the stable pg_basebackup label, which `newPgBasebackupCommand` takes at `pg_basebackup.go:50` and which names the backup rather than the object. Return the settled attempt's name and take every consumer from that result rather than the caller's variable: `runStreamParams` (`.../executor.go:79`) so the upload-time manifest at `stream.go:317` follows it, the chain-broken removal and the reported `FileName` (`create_full_uc.go:84`, `:92`), `streamResult.FileName` (`create_full_uc.go:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`). Correct the two comments that state the old shape, `executor.go:45` ("every attempt streams to the same object key") and the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and move `executor_test.go` onto the new shape, renaming the table field `expected` (declared at `:19`, set at `:25` and `:31`, read at `:39`) to `objectName` and updating its values for the attempt UUID; verify stream success, a zstd rejection that downgrades to gzip and then completes without the second attempt being refused for a name already in use and with the manifest landing under the settled name, pg_basebackup failure after partial output, cancellation and provider failure with focused tests, renaming `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact` to `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_UploadedArtifactRemoved` and converting it into a cleanup assertion, because the old name states the leak this task removes.
+- [ ] 6.2 Confirm physical receipts in the same transaction that publishes the backup and leave pending deletions in place on terminal failure paths, including manifest, metadata and incremental-chain failures; verify rollback and restart behavior with service and scheduler tests.
+- [ ] 6.3 Convert the physical reproduction tests into passing cleanup tests on local storage and MinIO; run the worker until idle and assert that partial base files, manifests, exact chunks, multipart uploads and metadata are absent after ordinary errors and cancellation.
+- [ ] 6.4 Rewrite restart recovery so it no longer assumes a missing backup file name means no object exists, replacing the stale premise recorded at `scheduler.go:710`, and request deletion for a claimless `IN_PROGRESS` row that carries a file name in the same transaction that records its terminal state, gated on that conditional `IN_PROGRESS` transition affecting one row so a late completion that already wrote a terminal status keeps its file; verify with `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact` renamed to `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_ArtifactRemoved` and rewritten to assert eventual absence, and that recovery neither deletes a confirmed successful backup nor duplicates an existing pending deletion.
+
+## 7. Integrate WAL and backup history artifacts
+
+- [ ] 7.1 Give each WAL and history write attempt an immutable operation-specific object name while preserving the logical restore ordering metadata. Move `walSegmentObjectName` onto a `walObjectNameSpec` struct rather than taking it to four positional values with a second adjacent `uuid.UUID`, mint the WAL name once per upload attempt where the artifact name is built (`wal_upload.go:232`) and pass it to the sidecar, so `wal_upload.go:380` stops calling `walSegmentObjectName` a second time; do the same for history, whose key is built at `timeline.go:231` and whose sidecar appends `.metadata` to it at `:246`, using the `historyFileID` already minted at `timeline.go:230`. If either sidecar keeps rebuilding the key, artifact and sidecar get different UUIDs and every consumer that reads `file_name` off the row and appends `.metadata` misses. Replace the doc comment on `walSegmentObjectName` (`.../wal_upload.go:439`), which currently argues that the insert-first claim model makes a UUID unnecessary; verify two retries cannot address the same remote object and restore selection remains correct.
+- [ ] 7.2 Confirm WAL artifacts and their sidecars in the same transaction that marks the WAL row uploaded, and confirm history files in the transaction that publishes their catalog row, including the history insert failure that currently strands both artifact and sidecar; verify commit failure and restart leave unconfirmed files eligible for cleanup while the locally queued segment file remains available for retry.
+- [ ] 7.3 Replace the synchronous orphan cleanup inside WAL upload: the `!committed` branch at `wal_upload.go:275` deletes the object directly because the claim row is already gone, and its helper `deleteObject` (`wal_upload.go:391`) is reached from three other branches (`:247`, `:255`, `:262`) and twice inside this one, for the artifact at `:282` and its sidecar at `:283`. Route all of them through the file store, delete the comment at `:276` that explains why the cleaner cannot see those bytes, and confirm task 8.6's guard finds nothing left in this file; verify the cascade-versus-upload case still ends with no object and no row.
+- [ ] 7.4 Request deletion transactionally for failed, superseded and retention-expired WAL and history artifacts, and correct the doc comment on `PhysicalWalSegmentRepository.DeleteAbandonedClaims` (`core/physical/repositories/wal_segment_repository.go:122`), which claims a NULL `file_name` proves no object exists; task 6.4 owns the scheduler comment that cites it as shared reasoning, so this task changes only the repository file, by deleting both that premise and the sentences restating what the name and the signature already say, keeping only what a reader cannot derive from them; verify the worker removes the exact remote objects without affecting adjacent WAL segments or valid history files, and that an abandoned claim whose upload did reach the provider still loses its object.
+
+## 8. Replace synchronous cleanup paths
+
+- [ ] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.
+- [ ] 8.2 Change physical backup, WAL, history and metadata cleanup, `removeUploadedArtifactsAfterChainBroken` (`create_full_uc.go:192` and `:198`, whose manifest name also comes from the pre-loop file name, and whose doc comment at `:179` restates its name and promises that `DeleteFile` is idempotent on not-found, which stops being what the call does, so delete both claims and keep only what survives), `DeleteChainDependentsKeepFull`, `DeleteOrphanWalSegmentsInSpan`, `deleteOrphanedHistoryFiles` and retention paths to request exact references transactionally, removing both the fail-open and fail-closed remote deletion behavior and the remote calls that currently run inside the cascade transaction; verify cascade and partial-batch rollback cases, including that a rolled-back cascade leaves every object intact.
+- [ ] 8.3 Add clear-path tests that create real local and MinIO artifacts, invoke each cleaner, run the deletion worker until idle and assert both catalog removal and physical absence; include one transient deletion failure to prove the catalog may disappear before eventual remote cleanup succeeds.
+- [ ] 8.4 Make database removal and its backup listener use one PostgreSQL transaction that records logical, FULL, INCR, WAL, history, manifest and metadata references before cascading catalog rows, changing the `OnBeforeDatabaseRemove` contract and its three implementations together, and run the listener work that dials the source database, which is cancellation and replication-slot cleanup, before that transaction opens rather than inside it; verify controller-level commit, failure rollback, real local and MinIO cleanup through `DeleteDatabase`, that no catalog transaction stays open across a source-database dial timeout, and that a WAL upload for a name the cascade recorded waits for that transaction instead of publishing.
+- [ ] 8.5 Move `DatabaseService.DeleteForTest` onto the same transaction and listener contract, because it runs the same chain and is reached from `databases.RemoveTestDatabase` by 239 call sites; verify the suite still compiles and that a database removed through the fixture records the same references as one removed through the controller.
+- [ ] 8.6 Add a production-code guard that fails tests if backup workflows call provider `SaveFile` or `DeleteFile` directly, once sections 5 through 8 have removed the last of those calls. Scope it to non-test files under `backend/internal/features/backups`, so it flags neither the storages feature's own `Storage` methods, its provider tests and the `--test-storage` probe in `backend/cmd/storage_command.go`, nor the backup test fixtures that legitimately create and remove provider objects, such as the `SaveFile` call in `core/physical/service/service_test.go`; verify the guard by planting a `DeleteFile` call under `backend/internal/features/backups` and confirming the guard fails on it, then removing it and confirming the guard stays silent for the provider calls that live outside its scope.
+
+## 9. Guard and drain the storage lifecycle
+
+- [ ] 9.1 Refuse storage deletion while backup rows still reference the storage, returning an actionable conflict instead of a silent logical cascade or a raw foreign-key error, by adding a `GetStorageBackupReferences` method to the existing registration mechanism rather than adding a second registry, and renaming `StorageDatabaseCounter`, `AddStorageDatabaseCounter`, the service field `storageDatabaseCounters` and the fixture `SetStorageDatabaseCountersForTest` to `StorageReferenceReporter`, `AddStorageReferenceReporter`, `storageReferenceReporters` and `SetStorageReferenceReportersForTest`; verify controller responses for logical and physical references and that deletion succeeds once the backups are gone.
+- [ ] 9.2 Make storage deletion run one drain of that storage's pending deletions first, bounded by a row count and a wall-clock budget and claiming through the same live-write exclusion the worker uses, then write one audit-log entry and a warning naming the storage, how many pending deletion rows are still present for it after the drain and their names up to a fixed limit, counting rows the drain failed on, skipped as locked, never saw because the worker had claimed them or never reached before the budget expired; verify a reachable provider leaves nothing behind, an unreachable one still deletes the storage within the budget and reports each remaining file once, that the entry stays bounded for a storage with thousands of obligations and that the remaining rows disappear with the storage row.
+- [ ] 9.3 Give the storage workspace-deletion listener at `storages/service.go:73` the same drain and reporting path instead of deleting storage rows straight through its repository, adding a `context.Context` to `OnBeforeWorkspaceDeletion` and moving its three implementations onto the new signature so the drain and the audit entry keep the request ID; verify workspace deletion succeeds with an unreachable provider, records what it left behind and keeps the existing refusal for a workspace that still contains databases.
+
+## 10. Keep the existing suite and the module rules current
+
+- [ ] 10.1 Give object-absence assertions a worker drain seam and apply it to the existing cascade coverage, starting with `Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects`; verify no test waits on a ticker or a sleep to observe cleanup.
+- [ ] 10.2 Make WAL tests read object names from catalog rows instead of rebuilding them at `wal_upload_test.go:74`, `wal_upload_test.go:212`, `wal_watch_dir_test.go:51`, `wal_watch_dir_test.go:71` and `wal_stream_test.go:444`; verify they pass with per-attempt names. `Test_WalUpload_DeleteCascadesIntoNullClaim_UploadCompletes_DeleteFileCleansOrphan` also asserts the orphan is already absent, so it takes the worker drain seam from task 10.1 and a name that no longer promises synchronous removal.
+- [ ] 10.3 Re-check the storage and backup-config controller tests that delete a storage, including `Test_DeleteStorage_StorageNotReturnedViaGet`, `Test_DeleteStorageWithAttachedDatabases_CannotDelete` and `Test_DeleteStorageWithAttachedPhysicalConfig_CannotDelete`, against the new reference refusal; verify each one still asserts the outcome it was written for and confirm the shared teardown fixtures need no change because pending deletions cascade with their storage row.
+- [ ] 10.4 Record the new conventions in `backend/AGENTS.md`: backup code writes and deletes through the file store rather than a provider, a stored file is kept only when a committed transaction claims it, and a provider `DeleteFile` owes every representation derived from the logical name; verify each of the three rules appears in the storage section of that file and that the file store is named in the list of feature boundaries backup code goes through.
+
+## 11. Final verification
+
+- [ ] 11.1 Run focused logical, physical, WAL, cleaner, storage lifecycle and provider cleanup tests, including the MinIO integration suite, and confirm no test depends on timing sleeps instead of controllable worker passes.
+- [ ] 11.2 Run `make lint` and the repository backend test command (`make test` in the dev container or `make test-fedora` on supported Fedora), then resolve every failure without weakening security or cleanup assertions.
+- [ ] 11.3 Run `openspec validate ensure-storage-file-cleanup --strict` and resolve every validation error.
+- [ ] 11.4 Request the mandatory final reviewer audit against the root and backend agent rules, resolve every `CHANGES REQUIRED` finding and rerun the affected linters and tests.