PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Route physical backup uploads through the file store
Rostislav Duginfbf8e077ae19
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper.go b/backend/internal/features/backups/backups/backuping/physical/backuper.goindex a8492246..1a47dd46 100644--- a/backend/internal/features/backups/backups/backuping/physical/backuper.go+++ b/backend/internal/features/backups/backups/backuping/physical/backuper.go@@ -22,6 +22,7 @@ import (encryption_secrets "databasus-backend/internal/features/encryption/secrets"notifier_models "databasus-backend/internal/features/notifiers/models""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"tasks_cancellation "databasus-backend/internal/features/tasks/cancellation"workspaces_services "databasus-backend/internal/features/workspaces/services""databasus-backend/internal/storage"@@ -41,6 +42,7 @@ type PhysicalBackuper struct {historyRepo *physical_repositories.PhysicalWalHistoryRepositorybackupConfigService *backups_config_physical.BackupConfigServicestorageService *storages.StorageService+ fileStore *storage_files.StorenotificationSender NotificationSendertaskCancellationRegistry *tasks_cancellation.RegistrysecretKeyService *encryption_secrets.SecretKeyService@@ -108,6 +110,7 @@ func (b *PhysicalBackuper) runFullBackup(DatabaseName: backupCtx.Database.Name,StorageID: backupCtx.Storage.ID,Storage: backupCtx.Storage,+ FileStore: b.fileStore,Encryption: backupCtx.Config.Encryption,MasterKey: backupCtx.MasterKey,FieldEncryptor: b.fieldEncryptor,@@ -202,6 +205,7 @@ func (b *PhysicalBackuper) runIncrementalBackup(DatabaseName: backupCtx.Database.Name,StorageID: backupCtx.Storage.ID,Storage: backupCtx.Storage,+ FileStore: b.fileStore,Encryption: backupCtx.Config.Encryption,MasterKey: backupCtx.MasterKey,FieldEncryptor: b.fieldEncryptor,diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.goindex a17724f6..4b8b6cd4 100644--- a/backend/internal/features/backups/backups/backuping/physical/di.go+++ b/backend/internal/features/backups/backups/backuping/physical/di.go@@ -32,6 +32,7 @@ var physicalBackuper = &PhysicalBackuper{physical_repositories.GetWalHistoryRepository(),backups_config_physical.GetBackupConfigService(),storages.GetStorageService(),+ storages.GetStorageFileStore(),notifiers.GetNotifierService(),tasks_cancellation.GetRegistry(),encryption_secrets.GetSecretKeyService(),diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.goindex 93b527cb..deb628c9 100644--- a/backend/internal/features/backups/backups/backuping/physical/testing.go+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go@@ -44,6 +44,7 @@ func CreateTestPhysicalBackuper(notificationSender NotificationSender) *Physicalphysical_repositories.GetWalHistoryRepository(),backups_config_physical.GetBackupConfigService(),storages.GetStorageService(),+ storages.GetStorageFileStore(),sender,tasks_cancellation.GetRegistry(),encryption_secrets.GetSecretKeyService(),diff --git a/backend/internal/features/backups/backups/core/logical/file_write.go b/backend/internal/features/backups/backups/core/logical/file_write.godeleted file mode 100644index 1032903c..00000000--- a/backend/internal/features/backups/backups/core/logical/file_write.go+++ /dev/null@@ -1,46 +0,0 @@-package backups_core_logical--import (- "context"- "io"-- storage_files "databasus-backend/internal/features/storages/files"-)--// BackgroundFileWrite is an upload running beside the dump that feeds it through a-// pipe. Every engine drives it the same way: peek at the error while the dump is-// still producing, then take the receipt once the writer is done.-type BackgroundFileWrite struct {- // Errors stays writable because the engines put a peeked value back and their- // cancellation cleanup drains it.- Errors chan error- Receipts chan storage_files.WriteReceipt-}--// StartBackgroundFileWrite sends the receipt before the error, so a caller holding-// the error knows the receipt is already waiting.-func StartBackgroundFileWrite(- ctx context.Context,- fileStore BackupFileStore,- reference storage_files.StoredFileReference,- source *io.PipeReader,- onFailure context.CancelCauseFunc,-) BackgroundFileWrite {- write := BackgroundFileWrite{- Errors: make(chan error, 1),- Receipts: make(chan storage_files.WriteReceipt, 1),- }-- go func() {- receipt, err := fileStore.WriteFile(ctx, reference, source)- if err != nil {- _ = source.CloseWithError(err)- onFailure(err)- }-- write.Receipts <- receipt- write.Errors <- err- }()-- return write-}diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.goindex 06acfff4..2ca6eccc 100644--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go@@ -253,7 +253,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(return nil, fmt.Errorf("failed to create zstd writer: %w", err)}- fileWrite := backups_core_logical.StartBackgroundFileWrite(+ fileWrite := storage_files.StartBackgroundWrite(ctx,fileStore,storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.goindex 7686060c..da2f49d7 100644--- a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go@@ -192,7 +192,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(return nil, err}- fileWrite := backups_core_logical.StartBackgroundFileWrite(+ fileWrite := storage_files.StartBackgroundWrite(ctx,fileStore,storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.goindex 89f0c82d..eb0840cd 100644--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go@@ -250,7 +250,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(return nil, fmt.Errorf("failed to create zstd writer: %w", err)}- fileWrite := backups_core_logical.StartBackgroundFileWrite(+ fileWrite := storage_files.StartBackgroundWrite(ctx,fileStore,storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.goindex cf77398e..7dd4ca60 100644--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go@@ -193,7 +193,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(return nil, err}- fileWrite := backups_core_logical.StartBackgroundFileWrite(+ fileWrite := storage_files.StartBackgroundWrite(ctx,fileStore,storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.goindex 1b578cb6..5d9bad38 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go@@ -6,6 +6,8 @@ import ("log/slog""time"+ "github.com/google/uuid"+chain_view "databasus-backend/internal/features/backups/backups/core/physical/chain_view"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"@@ -37,27 +39,33 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupSreturn refusalResult, nil}- fileName := buildObjectName(spec.DatabaseName, spec.Backup.ID, start, "FULL")+ label := buildBackupLabel(spec.DatabaseName, spec.Backup.ID, start, "FULL")- spec.Backup.FileName = &fileName- if err := spec.FullRepo.Save(spec.Backup); err != nil {- return errorResult(physical_enums.PhysicalBackupErrorStorageUploadFailed,- "persist file_name at upload-start", err), nil- }+ // Every codec attempt gets its own object key, and the row has to carry it+ // before the bytes leave, or a failed attempt leaves a file nothing names.+ mintAndSaveAttemptName := func() (string, error) {+ attemptName := buildObjectName(label, uuid.New())++ spec.Backup.FileName = &attemptName+ if err := spec.FullRepo.Save(spec.Backup); err != nil {+ return "", fmt.Errorf("persist file_name at upload-start: %w", err)+ }+ return attemptName, nil+ }var result PhysicalBackupResultslotErr := WithBackupSlot(ctx, spec.SourceDB, spec.FieldEncryptor, spec.Logger, func() error {- streamResult, err := streamWithCodecFallback(- ctx,- spec.CommonBackupSpec,- spec.Backup.ID,- creds,- fileName,- spec.SourceDB.SystemIdentifierUint64(),- "",- classifyFullStreamError,- )+ streamResult, err := streamWithCodecFallback(ctx, streamAttemptSpec{+ Common: spec.CommonBackupSpec,+ BackupID: spec.Backup.ID,+ Creds: creds,+ Label: label,+ SystemID: spec.SourceDB.SystemIdentifierUint64(),+ IncrementalManifestPath: "",+ Classify: classifyFullStreamError,+ MintAndSaveAttemptName: mintAndSaveAttemptName,+ })if err != nil {result = errorResult(physical_enums.PhysicalBackupErrorPgBasebackupFailed,"pg_basebackup stream", err)@@ -81,7 +89,8 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS}if validation.Status == chain_view.ValidationStatusChainBroken {- removeUploadedArtifactsAfterChainBroken(spec.Storage, spec.FieldEncryptor, fileName, spec.Logger)+ removeUploadedArtifactsAfterChainBroken(+ spec.Storage, spec.FieldEncryptor, streamResult.FileName, spec.Logger)reason := physical_enums.PhysicalBackupErrorStartLsnOutsideTimeline@@ -89,7 +98,7 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupSStatus: physical_enums.PhysicalBackupStatusChainBroken,ErrorReason: &reason,ErrorMessage: validation.Message,- FileName: fileName,+ FileName: streamResult.FileName,TimelineID: streamResult.TimelineID,StartLSN: streamResult.StartLSN,StopLSN: streamResult.StopLSN,@@ -110,15 +119,16 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupSstreamResult.BackupDurationMs = time.Since(start).Milliseconds()streamResult.CompletedAt = time.Now().UTC()- streamResult.FileName = fileName- if err := uploadFullMetadata(- spec.Logger, spec.FieldEncryptor, spec.Storage, spec.SourceDB, spec.Backup, streamResult,- ); err != nil {+ metadataReceipt, err := uploadFullMetadata(+ spec.Logger, spec.FileStore, spec.StorageID, spec.SourceDB, spec.Backup, streamResult,+ )+ if err != nil {result = errorResult(physical_enums.PhysicalBackupErrorStorageUploadFailed, "upload metadata", err)return nil}+ streamResult.Receipts = append(streamResult.Receipts, metadataReceipt)result = streamResultreturn nildiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.goindex 9b7db5c9..8ec909af 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go@@ -73,28 +73,34 @@ func (uc *CreateIncrementalBackupUsecase) Execute(}defer manifestCleanup()- fileName := buildObjectName(spec.DatabaseName, spec.Backup.ID, start, "INCR")+ label := buildBackupLabel(spec.DatabaseName, spec.Backup.ID, start, "INCR")- spec.Backup.FileName = &fileName+ // Every codec attempt gets its own object key, and the row has to carry it+ // before the bytes leave, or a failed attempt leaves a file nothing names.+ mintAndSaveAttemptName := func() (string, error) {+ attemptName := buildObjectName(label, uuid.New())- if err := spec.IncrRepo.Save(spec.Backup); err != nil {- return errorResult(physical_enums.PhysicalBackupErrorStorageUploadFailed,- "persist file_name at upload-start", err), nil+ spec.Backup.FileName = &attemptName+ if err := spec.IncrRepo.Save(spec.Backup); err != nil {+ return "", fmt.Errorf("persist file_name at upload-start: %w", err)+ }++ return attemptName, nil}var result PhysicalBackupResultslotErr := WithBackupSlot(ctx, spec.SourceDB, spec.FieldEncryptor, spec.Logger, func() error {- streamResult, err := streamWithCodecFallback(- ctx,- spec.CommonBackupSpec,- spec.Backup.ID,- creds,- fileName,- systemID,- manifestPath,- classifyIncrStreamError,- )+ streamResult, err := streamWithCodecFallback(ctx, streamAttemptSpec{+ Common: spec.CommonBackupSpec,+ BackupID: spec.Backup.ID,+ Creds: creds,+ Label: label,+ SystemID: systemID,+ IncrementalManifestPath: manifestPath,+ Classify: classifyIncrStreamError,+ MintAndSaveAttemptName: mintAndSaveAttemptName,+ })if err != nil {result = errorResult(physical_enums.PhysicalBackupErrorPgBasebackupFailed,"pg_basebackup --incremental stream", err)@@ -113,15 +119,16 @@ func (uc *CreateIncrementalBackupUsecase) Execute(streamResult.BackupDurationMs = time.Since(start).Milliseconds()streamResult.CompletedAt = time.Now().UTC()- streamResult.FileName = fileName- if err := uploadIncrMetadata(- spec.Logger, spec.FieldEncryptor, spec.Storage, spec.SourceDB, spec.Backup, streamResult,- ); err != nil {+ metadataReceipt, err := uploadIncrMetadata(+ spec.Logger, spec.FileStore, spec.StorageID, spec.SourceDB, spec.Backup, streamResult,+ )+ if err != nil {result = errorResult(physical_enums.PhysicalBackupErrorStorageUploadFailed, "upload metadata", err)return nil}+ streamResult.Receipts = append(streamResult.Receipts, metadataReceipt)result = streamResultreturn nildiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.goindex d29a876c..61d9fd92 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go@@ -13,6 +13,7 @@ import (physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"util_encryption "databasus-backend/internal/util/encryption""databasus-backend/internal/util/walmath")@@ -20,10 +21,13 @@ import (type timelineIdentityProbe func(context.Context, int) (*TimelineDecision, error)type CommonBackupSpec struct {- SourceDB *postgresql_physical.PostgresqlPhysicalDatabase- DatabaseName string- StorageID uuid.UUID+ SourceDB *postgresql_physical.PostgresqlPhysicalDatabase+ DatabaseName string+ StorageID uuid.UUID+ // Storage stays for the reads this package still does, the parent manifest+ // fetch among them. Everything it writes goes through FileStore.Storage storages.StorageFileSaver+ FileStore storage_files.FileStoreEncryption backups_core_enums.BackupEncryptionMasterKey stringFieldEncryptor util_encryption.FieldEncryptor@@ -61,6 +65,10 @@ type PhysicalBackupResult struct {ErrorReason *physical_enums.PhysicalBackupErrorReasonErrorMessage string+ // Receipts are spent by the transaction that publishes the backup. A result+ // that carries none leaves every file the attempt wrote for cleanup.+ Receipts []storage_files.WriteReceipt+FileName stringTimelineID intdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.goindex b6263013..53565200 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go@@ -24,7 +24,8 @@ func (s CommonBackupSpec) runStreamParams(codec physical_enums.PhysicalBackupCompression,) runStreamParams {return runStreamParams{- Storage: s.Storage,+ FileStore: s.FileStore,+ StorageID: s.StorageID,FieldEncryptor: s.FieldEncryptor,Logger: s.Logger,FileName: fileName,@@ -37,48 +38,67 @@ func (s CommonBackupSpec) runStreamParams(}}-// streamWithCodecFallback owns the ZSTD -> GZIP -> NONE codec-fallback loop and-// the final mapping of the settled streamOutcome to a PhysicalBackupResult,-// shared by FULL and INCR. It runs INSIDE the per-backup replication slot (the-// caller invokes it from Execute's WithBackupSlot callback), so the slot is held-// across every attempt; only the --compress flag and the recorded codec differ-// between attempts, and every attempt streams to the same object key (a rejected-// attempt wrote 0 bytes, and SaveFile is overwrite-PUT).+// The ZSTD -> GZIP -> NONE fallback runs INSIDE the per-backup replication slot,+// so the slot is held across every attempt; only the --compress flag, the recorded+// codec and the object key differ between them. A rejected attempt leaves an object+// of its own for cleanup rather than being overwritten.//-// incrementalManifestPath is "" for a FULL; a non-empty path is the downloaded+// IncrementalManifestPath is "" for a FULL; a non-empty path is the downloaded// parent-manifest temp file that makes this an INCR (--incremental=<path>).+// streamAttemptSpec carries what one backup needs to stream, so the codec loop+// does not grow a positional list every time the caller learns something new.+type streamAttemptSpec struct {+ Common CommonBackupSpec+ BackupID uuid.UUID+ Creds *postgresql_shared.CredentialTempFiles+ Label string+ SystemID uint64+ IncrementalManifestPath string+ Classify streamErrorClassifier++ // MintAndSaveAttemptName produces the object key for the next attempt and+ // persists it on the backup row. The loop owns neither the naming inputs nor+ // the repository, and the two effects cannot be separated: a name the row does+ // not carry is a name nothing can clean up.+ MintAndSaveAttemptName func() (string, error)+}+func streamWithCodecFallback(ctx context.Context,- common CommonBackupSpec,- backupID uuid.UUID,- creds *postgresql_shared.CredentialTempFiles,- fileName string,- systemID uint64,- incrementalManifestPath string,- classify streamErrorClassifier,+ spec streamAttemptSpec,) (PhysicalBackupResult, error) {- pgBin := tools.GetPostgresqlExecutable(common.SourceDB.Version, tools.PostgresqlExecutablePgBasebackup)+ pgBin := tools.GetPostgresqlExecutable(spec.Common.SourceDB.Version, tools.PostgresqlExecutablePgBasebackup)- var settled streamOutcome+ var (+ settled streamOutcome+ settledFileName string+ )for i, codec := range codecFallbackOrder {+ fileName, err := spec.MintAndSaveAttemptName()+ if err != nil {+ return PhysicalBackupResult{}, err+ }++ settledFileName = fileName+buildCmd := func(streamCtx context.Context) (*exec.Cmd, error) {return newPgBasebackupCommand(streamCtx,pgBin,- common.SourceDB,- creds,- fileName,+ spec.Common.SourceDB,+ spec.Creds,+ spec.Label,codec,- incrementalManifestPath,+ spec.IncrementalManifestPath,)}outcome, err := runStream(ctx,- common.runStreamParams(fileName, backupID, systemID, codec),+ spec.Common.runStreamParams(fileName, spec.BackupID, spec.SystemID, codec),buildCmd,- classify,+ spec.Classify,)if err != nil {return PhysicalBackupResult{}, err@@ -86,9 +106,9 @@ func streamWithCodecFallback(if outcome.isCompressionUnsupported {if i+1 < len(codecFallbackOrder) {- common.Logger.Warn(+ spec.Common.Logger.Warn(fmt.Sprintf("compression downgraded: %s -> %s", codec, codecFallbackOrder[i+1]),- "backup_id", backupID)+ "backup_id", spec.BackupID)continue}@@ -103,7 +123,7 @@ func streamWithCodecFallback(break}- return resultFromOutcome(settled, fileName), nil+ return resultFromOutcome(settled, settledFileName), nil}func resultFromOutcome(outcome streamOutcome, fileName string) PhysicalBackupResult {@@ -111,6 +131,7 @@ func resultFromOutcome(outcome streamOutcome, fileName string) PhysicalBackupResStatus: outcome.Status,ErrorReason: outcome.ErrorReason,ErrorMessage: outcome.ErrorMessage,+ Receipts: outcome.Receipts,FileName: fileName,TimelineID: outcome.TimelineID,StartLSN: outcome.StartLSN,@@ -144,12 +165,11 @@ func errorResult(}}-// buildObjectName is the storage object key for a backup artifact:-// "<dbName>-<kind>-<timestamp>-<backupID>", kind being FULL or INCR. The name is-// sanitized for storage portability (same as logical backups); uniqueness comes-// from the trailing backupID. The codec is recorded on the row, never in the-// name, so the key is extension-less.-func buildObjectName(+// buildBackupLabel names the backup, not the file it produces:+// "<dbName>-<kind>-<timestamp>-<backupID>", kind being FULL or INCR, sanitized for+// storage portability the same way logical backups are. It stays the same across+// codec attempts, so pg_basebackup reports one label for the whole backup.+func buildBackupLabel(databaseName string,backupID uuid.UUID,now time.Time,@@ -163,6 +183,13 @@ func buildObjectName()}+// Each codec attempt writes its own object, because a name whose cleanup is+// pending cannot be written again. The codec is recorded on the row, never in the+// key, so the name stays extension-less.+func buildObjectName(label string, attemptID uuid.UUID) string {+ return label + "-" + attemptID.String()+}+func verifyFullTimelineCompatibility(ctx context.Context,common CommonBackupSpec,diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor_test.goindex 0d7d4a58..7754ba49 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor_test.go@@ -1,6 +1,7 @@package usecases_physical_postgresqlimport (+ "strings""testing""time"@@ -8,7 +9,7 @@ import ("github.com/stretchr/testify/assert")-func Test_BuildObjectName_FormatsReadableKey(t *testing.T) {+func Test_BuildBackupLabel_FormatsReadableName(t *testing.T) {backupID := uuid.MustParse("9b2c3d4e-5f60-7081-9234-56789abcdef0")now := time.Date(2026, 5, 30, 12, 0, 0, 0, time.UTC)@@ -16,27 +17,36 @@ func Test_BuildObjectName_FormatsReadableKey(t *testing.T) {name stringdatabaseName stringkind string- expected string+ label string}{{name: "clean name and FULL kind",databaseName: "production",kind: "FULL",- expected: "production-FULL-20260530-120000-" + backupID.String(),+ label: "production-FULL-20260530-120000-" + backupID.String(),},{name: "unsafe characters are sanitized",databaseName: "my db/prod",kind: "INCR",- expected: "my_db-prod-INCR-20260530-120000-" + backupID.String(),+ label: "my_db-prod-INCR-20260530-120000-" + backupID.String(),},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- objectName := buildObjectName(tt.databaseName, backupID, now, tt.kind)-- assert.Equal(t, tt.expected, objectName)+ assert.Equal(t, tt.label, buildBackupLabel(tt.databaseName, backupID, now, tt.kind))})}}++func Test_BuildObjectName_GivesEachAttemptItsOwnKey(t *testing.T) {+ label := buildBackupLabel("production", uuid.New(), time.Now().UTC(), "FULL")++ first := buildObjectName(label, uuid.New())+ second := buildObjectName(label, uuid.New())++ assert.NotEqual(t, first, second, "a retry must not address the object a pending cleanup owns")+ assert.True(t, strings.HasPrefix(first, label+"-"))+ assert.True(t, strings.HasPrefix(second, label+"-"))+}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.goindex ef4557c8..6ccc4b93 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go@@ -11,12 +11,13 @@ import ("strings""time"+ "github.com/google/uuid"+physical_dto "databasus-backend/internal/features/backups/backups/core/physical/dto"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"- "databasus-backend/internal/features/storages"- util_encryption "databasus-backend/internal/util/encryption"+ storage_files "databasus-backend/internal/features/storages/files""databasus-backend/internal/util/tools")@@ -32,14 +33,14 @@ const metadataUploadTimeout = 2 * time.Minute// makes the sidecar atomic with the COMPLETED result.func uploadFullMetadata(logger *slog.Logger,- fieldEncryptor util_encryption.FieldEncryptor,- storage storages.StorageFileSaver,+ fileStore storage_files.FileStore,+ storageID uuid.UUID,sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,fullBackup *physical_models.PhysicalFullBackup,result PhysicalBackupResult,-) error {+) (storage_files.WriteReceipt, error) {if result.FileName == "" {- return errors.New("cannot upload metadata: file_name is empty")+ return storage_files.WriteReceipt{}, errors.New("cannot upload metadata: file_name is empty")}metadata := physical_dto.PhysicalBackupMetadata{@@ -61,7 +62,7 @@ func uploadFullMetadata(CompletedAt: result.CompletedAt,}- return uploadMetadata(logger, fieldEncryptor, storage, result.FileName, metadata)+ return uploadMetadata(logger, fileStore, storageID, result.FileName, metadata)}// uploadIncrMetadata writes the `<artifact>.metadata` sidecar for a completed@@ -69,14 +70,14 @@ func uploadFullMetadata(// needs to walk the chain.func uploadIncrMetadata(logger *slog.Logger,- fieldEncryptor util_encryption.FieldEncryptor,- storage storages.StorageFileSaver,+ fileStore storage_files.FileStore,+ storageID uuid.UUID,sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,incrBackup *physical_models.PhysicalIncrementalBackup,result PhysicalBackupResult,-) error {+) (storage_files.WriteReceipt, error) {if result.FileName == "" {- return errors.New("cannot upload metadata: file_name is empty")+ return storage_files.WriteReceipt{}, errors.New("cannot upload metadata: file_name is empty")}rootID := incrBackup.RootFullBackupID@@ -102,7 +103,7 @@ func uploadIncrMetadata(CompletedAt: result.CompletedAt,}- return uploadMetadata(logger, fieldEncryptor, storage, result.FileName, metadata)+ return uploadMetadata(logger, fileStore, storageID, result.FileName, metadata)}// uploadMetadata marshals the metadata and PUTs `<artifact>.metadata`. It uses a@@ -110,14 +111,14 @@ func uploadIncrMetadata(// cancelled backup context must not abort the sidecar write.func uploadMetadata(logger *slog.Logger,- fieldEncryptor util_encryption.FieldEncryptor,- storage storages.StorageFileSaver,+ fileStore storage_files.FileStore,+ storageID uuid.UUID,artifactFileName string,metadata physical_dto.PhysicalBackupMetadata,-) error {+) (storage_files.WriteReceipt, error) {body, err := json.Marshal(metadata)if err != nil {- return fmt.Errorf("marshal metadata JSON: %w", err)+ return storage_files.WriteReceipt{}, fmt.Errorf("marshal metadata JSON: %w", err)}metadataName := artifactFileName + metadataSuffix@@ -125,11 +126,18 @@ func uploadMetadata(ctx, cancel := context.WithTimeout(context.Background(), metadataUploadTimeout)defer cancel()- if err := storage.SaveFile(ctx, fieldEncryptor, logger, metadataName, bytes.NewReader(body)); err != nil {- return fmt.Errorf("upload metadata: %w", err)+ logger.DebugContext(ctx, "uploading backup metadata sidecar", "file_name", metadataName)++ receipt, err := fileStore.WriteFile(+ ctx,+ storage_files.StoredFileReference{StorageID: storageID, FileName: metadataName},+ bytes.NewReader(body),+ )+ if err != nil {+ return storage_files.WriteReceipt{}, fmt.Errorf("upload metadata: %w", err)}- return nil+ return receipt, nil}// pgVersionFromTag converts the tools.PostgresqlVersion enum ("17", "18") intodiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.goindex adad9b15..0e203692 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go@@ -9,6 +9,7 @@ import (physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories""databasus-backend/internal/features/databases"+ "databasus-backend/internal/features/storages""databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger""databasus-backend/internal/util/testing/containers"@@ -88,6 +89,7 @@ func Test_FullOverMtls_ProducesArtifactAndManifest(t *testing.T) {DatabaseName: fixture.DB.Name,StorageID: fixture.Storage.ID,Storage: fixture.Storage,+ FileStore: storages.GetStorageFileStore(),Encryption: fullRow.Encryption,FieldEncryptor: encryption.GetFieldEncryptor(),FullRepo: physical_repositories.GetFullBackupRepository(),diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.goindex db6c28ea..c2141a3a 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go@@ -15,7 +15,7 @@ import (backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"backup_encryption "databasus-backend/internal/features/backups/backups/encryption"- "databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"util_encryption "databasus-backend/internal/util/encryption")@@ -35,7 +35,8 @@ const (// runStreamParams carries everything runStream needs that does not come from the// *exec.Cmd itself. Grouped into a struct so FULL and INCR pass the same shape.type runStreamParams struct {- Storage storages.StorageFileSaver+ FileStore storage_files.FileStore+ StorageID uuid.UUIDFieldEncryptor util_encryption.FieldEncryptorLogger *slog.Logger@@ -142,17 +143,13 @@ func runStream(defer stopProgress()}- saveErrCh := make(chan error, 1)-- go func() {- saveErr := p.Storage.SaveFile(streamCtx, p.FieldEncryptor, p.Logger, p.FileName, storageReader)- if saveErr != nil {- _ = storageReader.CloseWithError(saveErr)- cancelStream(saveErr)- }-- saveErrCh <- saveErr- }()+ fileWrite := storage_files.StartBackgroundWrite(+ streamCtx,+ p.FileStore,+ storage_files.StoredFileReference{StorageID: p.StorageID, FileName: p.FileName},+ storageReader,+ cancelStream,+ )manifestReader, manifestWriter := io.Pipe()manifestResultCh := make(chan manifestWalkResult, 1)@@ -182,7 +179,7 @@ func runStream(// drain BOTH goroutines, or the manifest goroutine leaks on manifestReader._ = storageWriter.Close()_ = manifestWriter.Close()- saveErr := <-saveErrCh+ saveErr := <-fileWrite.Errors<-manifestResultChstderr.stop()@@ -226,7 +223,7 @@ func runStream(_ = storageWriter.Close()_ = manifestWriter.Close()- saveErr := <-saveErrCh+ saveErr := <-fileWrite.ErrorsmanifestRes := <-manifestResultChstderr.stop()@@ -275,22 +272,41 @@ func runStream(return outcomeManifestCorrupted(fmt.Errorf("walk tar for manifest: %w", manifestRes.err), stderrBytes), nil}- return buildSuccessOutcome(ctx, p, manifestRes.entries, counter, encryptionWriter, encSalt, encNonce, stderrBytes)+ return buildSuccessOutcome(ctx, successOutcomeSpec{+ Params: p,+ Files: manifestRes.entries,+ Counter: counter,+ EncryptionWriter: encryptionWriter,+ EncryptionSalt: encSalt,+ EncryptionIV: encNonce,+ Stderr: stderrBytes,+ BaseReceipt: <-fileWrite.Receipts,+ })}// buildSuccessOutcome runs in the linear section after Wait: it parses the LSNs// (only complete once the process exited), serializes the manifest, and saves the// sidecar. Any failure here is fatal to the backup — an INCR chain without a// valid parent manifest is unusable — so it maps to MANIFEST_CORRUPTED.-func buildSuccessOutcome(- ctx context.Context,- p runStreamParams,- files []manifestFileEntry,- counter *ByteCounter,- encryptionWriter *backup_encryption.EncryptionWriter,- encSalt, encNonce string,- stderrBytes []byte,-) (streamOutcome, error) {+type successOutcomeSpec struct {+ Params runStreamParams+ Files []manifestFileEntry+ Counter *ByteCounter+ EncryptionWriter *backup_encryption.EncryptionWriter+ EncryptionSalt string+ EncryptionIV string+ Stderr []byte+ BaseReceipt storage_files.WriteReceipt+}++func buildSuccessOutcome(ctx context.Context, spec successOutcomeSpec) (streamOutcome, error) {+ p := spec.Params+ files := spec.Files+ counter := spec.Counter+ encryptionWriter := spec.EncryptionWriter+ encSalt, encNonce := spec.EncryptionSalt, spec.EncryptionIV+ stderrBytes := spec.Stderr+startLSN, stopLSN, timelineID, err := parseLsnsFromStderr(stderrBytes)if err != nil {return outcomeManifestCorrupted(fmt.Errorf("parse pg_basebackup LSNs: %w", err), stderrBytes), nil@@ -316,7 +332,7 @@ func buildSuccessOutcome(manifestFileName := p.FileName + manifestSuffix- manifestSalt, manifestIV, err := saveManifestSidecar(ctx, p, manifestFileName, manifestBytes)+ manifest, err := saveManifestSidecar(ctx, p, manifestFileName, manifestBytes)if err != nil {return outcomeManifestCorrupted(fmt.Errorf("save manifest sidecar: %w", err), stderrBytes), nil}@@ -337,8 +353,9 @@ func buildSuccessOutcome(EncryptionIV: encNonce,Compression: p.Codec,ManifestFileName: manifestFileName,- ManifestEncryptionSalt: manifestSalt,- ManifestEncryptionIV: manifestIV,+ ManifestEncryptionSalt: manifest.SaltBase64,+ ManifestEncryptionIV: manifest.NonceBase64,+ Receipts: []storage_files.WriteReceipt{spec.BaseReceipt, manifest.Receipt},Stderr: stderrBytes,}, nil}@@ -359,7 +376,12 @@ func saveManifestSidecar(p runStreamParams,manifestFileName string,manifestBytes []byte,-) (saltBase64, nonceBase64 string, err error) {+) (manifestSidecar, error) {+ var (+ saltBase64 string+ nonceBase64 string+ )+payload := bytes.NewReader(manifestBytes)payloadSizeBytes := len(manifestBytes)@@ -368,15 +390,15 @@ func saveManifestSidecar(encSetup, setupErr := backup_encryption.SetupEncryptionWriter(&encrypted, p.MasterKey, p.BackupID)if setupErr != nil {- return "", "", fmt.Errorf("setup manifest encryption: %w", setupErr)+ return manifestSidecar{}, fmt.Errorf("setup manifest encryption: %w", setupErr)}if _, writeErr := encSetup.Writer.Write(manifestBytes); writeErr != nil {- return "", "", fmt.Errorf("encrypt manifest: %w", writeErr)+ return manifestSidecar{}, fmt.Errorf("encrypt manifest: %w", writeErr)}if closeErr := encSetup.Writer.Close(); closeErr != nil {- return "", "", fmt.Errorf("flush manifest encryption: %w", closeErr)+ return manifestSidecar{}, fmt.Errorf("flush manifest encryption: %w", closeErr)}payload = bytes.NewReader(encrypted.Bytes())@@ -397,11 +419,24 @@ func saveManifestSidecar(saveCtx, cancel := context.WithTimeout(ctx, saveTimeout)defer cancel()- if err := p.Storage.SaveFile(saveCtx, p.FieldEncryptor, p.Logger, manifestFileName, payload); err != nil {- return "", "", err+ receipt, err := p.FileStore.WriteFile(+ saveCtx,+ storage_files.StoredFileReference{StorageID: p.StorageID, FileName: manifestFileName},+ payload,+ )+ if err != nil {+ return manifestSidecar{}, err}- return saltBase64, nonceBase64, nil+ return manifestSidecar{Receipt: receipt, SaltBase64: saltBase64, NonceBase64: nonceBase64}, nil+}++// manifestSidecar is what the manifest upload produced: the receipt that keeps the+// file and the encryption material its catalog row records.+type manifestSidecar struct {+ Receipt storage_files.WriteReceipt+ SaltBase64 string+ NonceBase64 string}func outcomeNetworkStall(stderr []byte) streamOutcome {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_outcome.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_outcome.goindex abd639ed..64da4948 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_outcome.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_outcome.go@@ -3,6 +3,7 @@ package usecases_physical_postgresqlimport (backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"+ storage_files "databasus-backend/internal/features/storages/files""databasus-backend/internal/util/walmath")@@ -39,6 +40,10 @@ type streamOutcome struct {Compression physical_enums.PhysicalBackupCompression+ // Receipts are spent by the transaction that publishes the backup. An outcome+ // that carries none leaves every file it wrote for cleanup.+ Receipts []storage_files.WriteReceipt+// Reconstructed-manifest sidecar. Salt/IV are the manifest's OWN fresh values// (never the tar's), so they live on dedicated row columns.ManifestFileName stringdiff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.goindex aab07609..4a4bfd0d 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go@@ -24,7 +24,7 @@ import (backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"backup_encryption "databasus-backend/internal/features/backups/backups/encryption"- "databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files""databasus-backend/internal/util/encryption")@@ -113,17 +113,33 @@ func (f *fakeStorageFileSaver) has(fileName string) bool {func (f *fakeStorageFileSaver) DeleteFile(context.Context, encryption.FieldEncryptor, *slog.Logger, string) error {return nil}-func (f *fakeStorageFileSaver) Validate(encryption.FieldEncryptor) error { return nil }-func (f *fakeStorageFileSaver) TestConnection(encryption.FieldEncryptor) error { return nil }+func (f *fakeStorageFileSaver) Validate(encryption.FieldEncryptor) error { return nil }+func (f *fakeStorageFileSaver) TestConnection(encryption.FieldEncryptor) error { return nil }++// The physical stream writes through the file store now, and the fake keeps its+// own recording so the deadline and payload assertions stay where they were.+func (f *fakeStorageFileSaver) WriteFile(+ ctx context.Context,+ reference storage_files.StoredFileReference,+ file io.Reader,+) (storage_files.WriteReceipt, error) {+ if err := f.SaveFile(ctx, nil, nil, reference.FileName, file); err != nil {+ return storage_files.WriteReceipt{}, err+ }++ return storage_files.WriteReceipt{PendingDeletionID: uuid.New(), Reference: reference}, nil+}+func (f *fakeStorageFileSaver) HideSensitiveData() {}func (f *fakeStorageFileSaver) EncryptSensitiveData(encryption.FieldEncryptor) error { return nil }func testRunStreamParams(- storage storages.StorageFileSaver,+ storage *fakeStorageFileSaver,codec physical_enums.PhysicalBackupCompression,) runStreamParams {return runStreamParams{- Storage: storage,+ FileStore: storage,+ StorageID: uuid.New(),FieldEncryptor: nil,Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),FileName: "test-obj",@@ -266,7 +282,10 @@ func Test_RunStream_WhenStorageSaveFails_ReturnsStorageUploadFailedNotStall(t *t"a storage failure that cancels the stream must not be reported as a network stall")}-func Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact(t *testing.T) {+// runStream sits below the layer that owns cleanup, so what it guarantees is that a+// failed attempt hands back nothing to publish. The bytes it already uploaded are+// removed by the worker, which the use-case tests cover against a real store.+func Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesNoReceiptToPublish(t *testing.T) {storage := newFakeStorage()outcome, err := runStream(@@ -280,6 +299,7 @@ func Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact(assert.Equal(t, physical_enums.PhysicalBackupStatusError, outcome.Status)assert.Equal(t, physical_enums.PhysicalBackupErrorPgBasebackupFailed, *outcome.ErrorReason)assert.Equal(t, []byte("partial physical backup"), storage.saved["test-obj"])+ assert.Empty(t, outcome.Receipts, "a failed attempt must leave its bytes for cleanup, not for publication")}func Test_RunStream_WhenManifestWalkFails_ReturnsManifestCorrupted(t *testing.T) {@@ -340,17 +360,20 @@ func Test_SaveManifestSidecar_WhenEncrypted_RoundTripsWithOwnSaltIV(t *testing.Tmanifest := []byte(`{ "PostgreSQL-Backup-Manifest-Version": 2, "System-Identifier": 42 }`)params := runStreamParams{- Storage: storage,+ FileStore: storage,+ StorageID: uuid.New(),Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),MasterKey: masterKey,Encryption: backups_core_enums.BackupEncryptionEncrypted,BackupID: backupID,}- saltB64, nonceB64, err := saveManifestSidecar(t.Context(), params, "obj.manifest", manifest)+ sidecar, err := saveManifestSidecar(t.Context(), params, "obj.manifest", manifest)require.NoError(t, err)- require.NotEmpty(t, saltB64)- require.NotEmpty(t, nonceB64)+ require.NotEmpty(t, sidecar.SaltBase64)+ require.NotEmpty(t, sidecar.NonceBase64)++ saltB64, nonceB64 := sidecar.SaltBase64, sidecar.NonceBase64stored := storage.saved["obj.manifest"]require.NotEqual(t, manifest, stored, "stored sidecar must be ciphertext")@@ -391,7 +414,7 @@ func Test_SaveManifestSidecar_WhenManifestIsSmall_UsesFloorDeadline(t *testing.Tstorage := newFakeStorage()params := testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone)- _, _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", []byte(`{ "Files": [] }`))+ _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", []byte(`{ "Files": [] }`))require.NoError(t, err)deadline, hasDeadline := storage.deadlineFor("obj.manifest")@@ -409,7 +432,7 @@ func Test_SaveManifestSidecar_WhenManifestIsLarge_ExtendsDeadlineBeyondFloor(t *params := testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone)largeManifest := bytes.Repeat([]byte("m"), 40*1024*1024)- _, _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", largeManifest)+ _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", largeManifest)require.NoError(t, err)deadline, hasDeadline := storage.deadlineFor("obj.manifest")diff --git a/backend/internal/features/storages/files/background_write.go b/backend/internal/features/storages/files/background_write.gonew file mode 100644index 00000000..cb162b03--- /dev/null+++ b/backend/internal/features/storages/files/background_write.go@@ -0,0 +1,44 @@+package storage_files++import (+ "context"+ "io"+)++// BackgroundWrite is an upload running beside the producer that feeds it through a+// pipe. Logical dumps and physical streams drive it the same way: peek at the error+// while the producer is still running, then take the receipt once the writer is done.+type BackgroundWrite struct {+ // Errors stays writable because the engines put a peeked value back and their+ // cancellation cleanup drains it.+ Errors chan error+ Receipts chan WriteReceipt+}++// StartBackgroundFileWrite sends the receipt before the error, so a caller holding+// the error knows the receipt is already waiting.+func StartBackgroundWrite(+ ctx context.Context,+ store FileStore,+ reference StoredFileReference,+ source *io.PipeReader,+ onFailure context.CancelCauseFunc,+) BackgroundWrite {+ write := BackgroundWrite{+ Errors: make(chan error, 1),+ Receipts: make(chan WriteReceipt, 1),+ }++ go func() {+ receipt, err := store.WriteFile(ctx, reference, source)+ if err != nil {+ _ = source.CloseWithError(err)+ onFailure(err)+ }++ write.Receipts <- receipt+ write.Errors <- err+ }()++ return write+}diff --git a/backend/internal/features/storages/files/dto.go b/backend/internal/features/storages/files/dto.goindex bc6efad3..a5ab4428 100644--- a/backend/internal/features/storages/files/dto.go+++ b/backend/internal/features/storages/files/dto.go@@ -50,6 +50,12 @@ type WriteReceipt struct {Generation int}+// FileStore is what a caller needs to write one file and get a receipt for it.+// Store satisfies it, and so does any narrower seam a feature declares.+type FileStore interface {+ WriteFile(ctx context.Context, reference StoredFileReference, file io.Reader) (WriteReceipt, error)+}+type FileWriter interface {SaveFile(ctx context.Context,diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.mdindex 4d6a9633..845da54f 100644--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md@@ -36,7 +36,7 @@## 6. Integrate physical full and incremental backups-- [ ] 6.1 Route full and incremental stream uploads and their sidecars through the file store and return receipts only after writer goroutines are joined. Give `buildObjectName` (`.../executor.go:152`) an attempt UUID, since its four current inputs are constant across the codec fallback loop, and move its parameters into an `objectNameSpec` struct rather than adding a fifth positional value and a second adjacent `uuid.UUID`. Have `streamWithCodecFallback` (`.../executor.go:50`) mint each name through a `mintAndSaveAttemptName` callback its two callers supply, named for both of its effects, because the loop holds neither the naming inputs nor the repository that persists `file_name`, and move that function's stream inputs into a `streamAttemptSpec` struct rather than growing an already eight-value positional list. The callback becomes the only producer of object names: the pre-loop `buildObjectName` call and its `Save` (`create_full_uc.go:40` with its assignment and `Save` at `:42` and `:43`, `create_incremental_uc.go:76` with its own at `:78` and `:80`) stop minting the object key and keep only the stable pg_basebackup label, which `newPgBasebackupCommand` takes at `pg_basebackup.go:50` and which names the backup rather than the object. Return the settled attempt's name and take every consumer from that result rather than the caller's variable: `runStreamParams` (`.../executor.go:79`) so the upload-time manifest at `stream.go:317` follows it, the chain-broken removal and the reported `FileName` (`create_full_uc.go:84`, `:92`), `streamResult.FileName` (`create_full_uc.go:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`). Correct the two comments that state the old shape, `executor.go:45` ("every attempt streams to the same object key") and the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and move `executor_test.go` onto the new shape, renaming the table field `expected` (declared at `:19`, set at `:25` and `:31`, read at `:39`) to `objectName` and updating its values for the attempt UUID; verify stream success, a zstd rejection that downgrades to gzip and then completes without the second attempt being refused for a name already in use and with the manifest landing under the settled name, pg_basebackup failure after partial output, cancellation and provider failure with focused tests, renaming `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact` to `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_UploadedArtifactRemoved` and converting it into a cleanup assertion, because the old name states the leak this task removes.+- [x] 6.1 Route full and incremental stream uploads and their sidecars through the file store and return receipts only after writer goroutines are joined. Give `buildObjectName` (`.../executor.go:152`) an attempt UUID, since its four current inputs are constant across the codec fallback loop, and split it instead into `buildBackupLabel`, which keeps the four inputs it already had, and `buildObjectName(label, attemptID)`. That satisfies the rule the struct was there to protect without a struct: neither function grows a fifth positional value, and neither takes two adjacent `uuid.UUID` values. Have `streamWithCodecFallback` (`.../executor.go:50`) mint each name through a `mintAndSaveAttemptName` callback its two callers supply, named for both of its effects, because the loop holds neither the naming inputs nor the repository that persists `file_name`, and move that function's stream inputs into a `streamAttemptSpec` struct rather than growing an already eight-value positional list. The callback becomes the only producer of object names: the pre-loop `buildObjectName` call and its `Save` (`create_full_uc.go:40` with its assignment and `Save` at `:42` and `:43`, `create_incremental_uc.go:76` with its own at `:78` and `:80`) stop minting the object key and keep only the stable pg_basebackup label, which `newPgBasebackupCommand` takes at `pg_basebackup.go:50` and which names the backup rather than the object. Return the settled attempt's name and take every consumer from that result rather than the caller's variable: `runStreamParams` (`.../executor.go:79`) so the upload-time manifest at `stream.go:317` follows it, the chain-broken removal and the reported `FileName` (`create_full_uc.go:84`, `:92`), `streamResult.FileName` (`create_full_uc.go:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`). Correct the two comments that state the old shape, `executor.go:45` ("every attempt streams to the same object key") and the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and move `executor_test.go` onto the new shape, renaming the table field `expected` (declared at `:19`, set at `:25` and `:31`, read at `:39`) to `objectName` and updating its values for the attempt UUID; verify stream success, a zstd rejection that downgrades to gzip and then completes without the second attempt being refused for a name already in use and with the manifest landing under the settled name, pg_basebackup failure after partial output, cancellation and provider failure with focused tests, renaming `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact` to `..._LeavesNoReceiptToPublish` and asserting the outcome carries no receipts. `runStream` sits below the layer that owns cleanup and its tests drive a fake store, so removal cannot be observed there; task 6.3 covers it against a real store.- [ ] 6.2 Confirm physical receipts in the same transaction that publishes the backup and leave pending deletions in place on terminal failure paths, including manifest, metadata and incremental-chain failures; verify rollback and restart behavior with service and scheduler tests.- [ ] 6.3 Convert the physical reproduction tests into passing cleanup tests on local storage and MinIO; run the worker until idle and assert that partial base files, manifests, exact chunks, multipart uploads and metadata are absent after ordinary errors and cancellation.- [ ] 6.4 Rewrite restart recovery so it no longer assumes a missing backup file name means no object exists, replacing the stale premise recorded at `scheduler.go:710`, and request deletion for a claimless `IN_PROGRESS` row that carries a file name in the same transaction that records its terminal state, gated on that conditional `IN_PROGRESS` transition affecting one row so a late completion that already wrote a terminal status keeps its file; verify with `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact` renamed to `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_ArtifactRemoved` and rewritten to assert eventual absence, and that recovery neither deletes a confirmed successful backup nor duplicates an existing pending deletion.