Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (backups): Route physical backup uploads through the file store

Rostislav Duginfbf8e077ae19
diff --git a/backend/internal/features/backups/backups/backuping/physical/backuper.go b/backend/internal/features/backups/backups/backuping/physical/backuper.go
index a8492246..1a47dd46 100644
--- a/backend/internal/features/backups/backups/backuping/physical/backuper.go
+++ b/backend/internal/features/backups/backups/backuping/physical/backuper.go
@@ -22,6 +22,7 @@ import (
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
notifier_models "databasus-backend/internal/features/notifiers/models"
"databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
tasks_cancellation "databasus-backend/internal/features/tasks/cancellation"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"databasus-backend/internal/storage"
@@ -41,6 +42,7 @@ type PhysicalBackuper struct {
historyRepo *physical_repositories.PhysicalWalHistoryRepository
backupConfigService *backups_config_physical.BackupConfigService
storageService *storages.StorageService
+ fileStore *storage_files.Store
notificationSender NotificationSender
taskCancellationRegistry *tasks_cancellation.Registry
secretKeyService *encryption_secrets.SecretKeyService
@@ -108,6 +110,7 @@ func (b *PhysicalBackuper) runFullBackup(
DatabaseName: backupCtx.Database.Name,
StorageID: backupCtx.Storage.ID,
Storage: backupCtx.Storage,
+ FileStore: b.fileStore,
Encryption: backupCtx.Config.Encryption,
MasterKey: backupCtx.MasterKey,
FieldEncryptor: b.fieldEncryptor,
@@ -202,6 +205,7 @@ func (b *PhysicalBackuper) runIncrementalBackup(
DatabaseName: backupCtx.Database.Name,
StorageID: backupCtx.Storage.ID,
Storage: backupCtx.Storage,
+ FileStore: b.fileStore,
Encryption: backupCtx.Config.Encryption,
MasterKey: backupCtx.MasterKey,
FieldEncryptor: b.fieldEncryptor,
diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.go
index a17724f6..4b8b6cd4 100644
--- a/backend/internal/features/backups/backups/backuping/physical/di.go
+++ b/backend/internal/features/backups/backups/backuping/physical/di.go
@@ -32,6 +32,7 @@ var physicalBackuper = &PhysicalBackuper{
physical_repositories.GetWalHistoryRepository(),
backups_config_physical.GetBackupConfigService(),
storages.GetStorageService(),
+ storages.GetStorageFileStore(),
notifiers.GetNotifierService(),
tasks_cancellation.GetRegistry(),
encryption_secrets.GetSecretKeyService(),
diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.go
index 93b527cb..deb628c9 100644
--- a/backend/internal/features/backups/backups/backuping/physical/testing.go
+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go
@@ -44,6 +44,7 @@ func CreateTestPhysicalBackuper(notificationSender NotificationSender) *Physical
physical_repositories.GetWalHistoryRepository(),
backups_config_physical.GetBackupConfigService(),
storages.GetStorageService(),
+ storages.GetStorageFileStore(),
sender,
tasks_cancellation.GetRegistry(),
encryption_secrets.GetSecretKeyService(),
diff --git a/backend/internal/features/backups/backups/core/logical/file_write.go b/backend/internal/features/backups/backups/core/logical/file_write.go
deleted file mode 100644
index 1032903c..00000000
--- a/backend/internal/features/backups/backups/core/logical/file_write.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package backups_core_logical
-
-import (
- "context"
- "io"
-
- storage_files "databasus-backend/internal/features/storages/files"
-)
-
-// BackgroundFileWrite is an upload running beside the dump that feeds it through a
-// pipe. Every engine drives it the same way: peek at the error while the dump is
-// still producing, then take the receipt once the writer is done.
-type BackgroundFileWrite struct {
- // Errors stays writable because the engines put a peeked value back and their
- // cancellation cleanup drains it.
- Errors chan error
- Receipts chan storage_files.WriteReceipt
-}
-
-// StartBackgroundFileWrite sends the receipt before the error, so a caller holding
-// the error knows the receipt is already waiting.
-func StartBackgroundFileWrite(
- ctx context.Context,
- fileStore BackupFileStore,
- reference storage_files.StoredFileReference,
- source *io.PipeReader,
- onFailure context.CancelCauseFunc,
-) BackgroundFileWrite {
- write := BackgroundFileWrite{
- Errors: make(chan error, 1),
- Receipts: make(chan storage_files.WriteReceipt, 1),
- }
-
- go func() {
- receipt, err := fileStore.WriteFile(ctx, reference, source)
- if err != nil {
- _ = source.CloseWithError(err)
- onFailure(err)
- }
-
- write.Receipts <- receipt
- write.Errors <- err
- }()
-
- return write
-}
diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
index 06acfff4..2ca6eccc 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go
@@ -253,7 +253,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
return nil, fmt.Errorf("failed to create zstd writer: %w", err)
}
- fileWrite := backups_core_logical.StartBackgroundFileWrite(
+ fileWrite := storage_files.StartBackgroundWrite(
ctx,
fileStore,
storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
diff --git a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
index 7686060c..da2f49d7 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mongodb/create_backup_uc.go
@@ -192,7 +192,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
return nil, err
}
- fileWrite := backups_core_logical.StartBackgroundFileWrite(
+ fileWrite := storage_files.StartBackgroundWrite(
ctx,
fileStore,
storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
index 89f0c82d..eb0840cd 100644
--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go
@@ -250,7 +250,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
return nil, fmt.Errorf("failed to create zstd writer: %w", err)
}
- fileWrite := backups_core_logical.StartBackgroundFileWrite(
+ fileWrite := storage_files.StartBackgroundWrite(
ctx,
fileStore,
storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
diff --git a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
index cf77398e..7dd4ca60 100644
--- a/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
+++ b/backend/internal/features/backups/backups/usecases/logical/postgresql/create_backup_uc.go
@@ -193,7 +193,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
return nil, err
}
- fileWrite := backups_core_logical.StartBackgroundFileWrite(
+ fileWrite := storage_files.StartBackgroundWrite(
ctx,
fileStore,
storage_files.StoredFileReference{StorageID: backup.StorageID, FileName: backup.FileName},
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
index 1b578cb6..5d9bad38 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_full_uc.go
@@ -6,6 +6,8 @@ import (
"log/slog"
"time"
+ "github.com/google/uuid"
+
chain_view "databasus-backend/internal/features/backups/backups/core/physical/chain_view"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
postgresql_shared "databasus-backend/internal/features/databases/databases/postgresql/shared"
@@ -37,27 +39,33 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS
return refusalResult, nil
}
- fileName := buildObjectName(spec.DatabaseName, spec.Backup.ID, start, "FULL")
+ label := buildBackupLabel(spec.DatabaseName, spec.Backup.ID, start, "FULL")
- spec.Backup.FileName = &fileName
- if err := spec.FullRepo.Save(spec.Backup); err != nil {
- return errorResult(physical_enums.PhysicalBackupErrorStorageUploadFailed,
- "persist file_name at upload-start", err), nil
- }
+ // Every codec attempt gets its own object key, and the row has to carry it
+ // before the bytes leave, or a failed attempt leaves a file nothing names.
+ mintAndSaveAttemptName := func() (string, error) {
+ attemptName := buildObjectName(label, uuid.New())
+
+ spec.Backup.FileName = &attemptName
+ if err := spec.FullRepo.Save(spec.Backup); err != nil {
+ return "", fmt.Errorf("persist file_name at upload-start: %w", err)
+ }
+ return attemptName, nil
+ }
var result PhysicalBackupResult
slotErr := WithBackupSlot(ctx, spec.SourceDB, spec.FieldEncryptor, spec.Logger, func() error {
- streamResult, err := streamWithCodecFallback(
- ctx,
- spec.CommonBackupSpec,
- spec.Backup.ID,
- creds,
- fileName,
- spec.SourceDB.SystemIdentifierUint64(),
- "",
- classifyFullStreamError,
- )
+ streamResult, err := streamWithCodecFallback(ctx, streamAttemptSpec{
+ Common: spec.CommonBackupSpec,
+ BackupID: spec.Backup.ID,
+ Creds: creds,
+ Label: label,
+ SystemID: spec.SourceDB.SystemIdentifierUint64(),
+ IncrementalManifestPath: "",
+ Classify: classifyFullStreamError,
+ MintAndSaveAttemptName: mintAndSaveAttemptName,
+ })
if err != nil {
result = errorResult(physical_enums.PhysicalBackupErrorPgBasebackupFailed,
"pg_basebackup stream", err)
@@ -81,7 +89,8 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS
}
if validation.Status == chain_view.ValidationStatusChainBroken {
- removeUploadedArtifactsAfterChainBroken(spec.Storage, spec.FieldEncryptor, fileName, spec.Logger)
+ removeUploadedArtifactsAfterChainBroken(
+ spec.Storage, spec.FieldEncryptor, streamResult.FileName, spec.Logger)
reason := physical_enums.PhysicalBackupErrorStartLsnOutsideTimeline
@@ -89,7 +98,7 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS
Status: physical_enums.PhysicalBackupStatusChainBroken,
ErrorReason: &reason,
ErrorMessage: validation.Message,
- FileName: fileName,
+ FileName: streamResult.FileName,
TimelineID: streamResult.TimelineID,
StartLSN: streamResult.StartLSN,
StopLSN: streamResult.StopLSN,
@@ -110,15 +119,16 @@ func (uc *CreateFullBackupUsecase) Execute(ctx context.Context, spec FullBackupS
streamResult.BackupDurationMs = time.Since(start).Milliseconds()
streamResult.CompletedAt = time.Now().UTC()
- streamResult.FileName = fileName
- if err := uploadFullMetadata(
- spec.Logger, spec.FieldEncryptor, spec.Storage, spec.SourceDB, spec.Backup, streamResult,
- ); err != nil {
+ metadataReceipt, err := uploadFullMetadata(
+ spec.Logger, spec.FileStore, spec.StorageID, spec.SourceDB, spec.Backup, streamResult,
+ )
+ if err != nil {
result = errorResult(physical_enums.PhysicalBackupErrorStorageUploadFailed, "upload metadata", err)
return nil
}
+ streamResult.Receipts = append(streamResult.Receipts, metadataReceipt)
result = streamResult
return nil
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
index 9b7db5c9..8ec909af 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/create_incremental_uc.go
@@ -73,28 +73,34 @@ func (uc *CreateIncrementalBackupUsecase) Execute(
}
defer manifestCleanup()
- fileName := buildObjectName(spec.DatabaseName, spec.Backup.ID, start, "INCR")
+ label := buildBackupLabel(spec.DatabaseName, spec.Backup.ID, start, "INCR")
- spec.Backup.FileName = &fileName
+ // Every codec attempt gets its own object key, and the row has to carry it
+ // before the bytes leave, or a failed attempt leaves a file nothing names.
+ mintAndSaveAttemptName := func() (string, error) {
+ attemptName := buildObjectName(label, uuid.New())
- if err := spec.IncrRepo.Save(spec.Backup); err != nil {
- return errorResult(physical_enums.PhysicalBackupErrorStorageUploadFailed,
- "persist file_name at upload-start", err), nil
+ spec.Backup.FileName = &attemptName
+ if err := spec.IncrRepo.Save(spec.Backup); err != nil {
+ return "", fmt.Errorf("persist file_name at upload-start: %w", err)
+ }
+
+ return attemptName, nil
}
var result PhysicalBackupResult
slotErr := WithBackupSlot(ctx, spec.SourceDB, spec.FieldEncryptor, spec.Logger, func() error {
- streamResult, err := streamWithCodecFallback(
- ctx,
- spec.CommonBackupSpec,
- spec.Backup.ID,
- creds,
- fileName,
- systemID,
- manifestPath,
- classifyIncrStreamError,
- )
+ streamResult, err := streamWithCodecFallback(ctx, streamAttemptSpec{
+ Common: spec.CommonBackupSpec,
+ BackupID: spec.Backup.ID,
+ Creds: creds,
+ Label: label,
+ SystemID: systemID,
+ IncrementalManifestPath: manifestPath,
+ Classify: classifyIncrStreamError,
+ MintAndSaveAttemptName: mintAndSaveAttemptName,
+ })
if err != nil {
result = errorResult(physical_enums.PhysicalBackupErrorPgBasebackupFailed,
"pg_basebackup --incremental stream", err)
@@ -113,15 +119,16 @@ func (uc *CreateIncrementalBackupUsecase) Execute(
streamResult.BackupDurationMs = time.Since(start).Milliseconds()
streamResult.CompletedAt = time.Now().UTC()
- streamResult.FileName = fileName
- if err := uploadIncrMetadata(
- spec.Logger, spec.FieldEncryptor, spec.Storage, spec.SourceDB, spec.Backup, streamResult,
- ); err != nil {
+ metadataReceipt, err := uploadIncrMetadata(
+ spec.Logger, spec.FileStore, spec.StorageID, spec.SourceDB, spec.Backup, streamResult,
+ )
+ if err != nil {
result = errorResult(physical_enums.PhysicalBackupErrorStorageUploadFailed, "upload metadata", err)
return nil
}
+ streamResult.Receipts = append(streamResult.Receipts, metadataReceipt)
result = streamResult
return nil
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
index d29a876c..61d9fd92 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/dto.go
@@ -13,6 +13,7 @@ import (
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
"databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
util_encryption "databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/walmath"
)
@@ -20,10 +21,13 @@ import (
type timelineIdentityProbe func(context.Context, int) (*TimelineDecision, error)
type CommonBackupSpec struct {
- SourceDB *postgresql_physical.PostgresqlPhysicalDatabase
- DatabaseName string
- StorageID uuid.UUID
+ SourceDB *postgresql_physical.PostgresqlPhysicalDatabase
+ DatabaseName string
+ StorageID uuid.UUID
+ // Storage stays for the reads this package still does, the parent manifest
+ // fetch among them. Everything it writes goes through FileStore.
Storage storages.StorageFileSaver
+ FileStore storage_files.FileStore
Encryption backups_core_enums.BackupEncryption
MasterKey string
FieldEncryptor util_encryption.FieldEncryptor
@@ -61,6 +65,10 @@ type PhysicalBackupResult struct {
ErrorReason *physical_enums.PhysicalBackupErrorReason
ErrorMessage string
+ // Receipts are spent by the transaction that publishes the backup. A result
+ // that carries none leaves every file the attempt wrote for cleanup.
+ Receipts []storage_files.WriteReceipt
+
FileName string
TimelineID int
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
index b6263013..53565200 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor.go
@@ -24,7 +24,8 @@ func (s CommonBackupSpec) runStreamParams(
codec physical_enums.PhysicalBackupCompression,
) runStreamParams {
return runStreamParams{
- Storage: s.Storage,
+ FileStore: s.FileStore,
+ StorageID: s.StorageID,
FieldEncryptor: s.FieldEncryptor,
Logger: s.Logger,
FileName: fileName,
@@ -37,48 +38,67 @@ func (s CommonBackupSpec) runStreamParams(
}
}
-// streamWithCodecFallback owns the ZSTD -> GZIP -> NONE codec-fallback loop and
-// the final mapping of the settled streamOutcome to a PhysicalBackupResult,
-// shared by FULL and INCR. It runs INSIDE the per-backup replication slot (the
-// caller invokes it from Execute's WithBackupSlot callback), so the slot is held
-// across every attempt; only the --compress flag and the recorded codec differ
-// between attempts, and every attempt streams to the same object key (a rejected
-// attempt wrote 0 bytes, and SaveFile is overwrite-PUT).
+// The ZSTD -> GZIP -> NONE fallback runs INSIDE the per-backup replication slot,
+// so the slot is held across every attempt; only the --compress flag, the recorded
+// codec and the object key differ between them. A rejected attempt leaves an object
+// of its own for cleanup rather than being overwritten.
//
-// incrementalManifestPath is "" for a FULL; a non-empty path is the downloaded
+// IncrementalManifestPath is "" for a FULL; a non-empty path is the downloaded
// parent-manifest temp file that makes this an INCR (--incremental=<path>).
+// streamAttemptSpec carries what one backup needs to stream, so the codec loop
+// does not grow a positional list every time the caller learns something new.
+type streamAttemptSpec struct {
+ Common CommonBackupSpec
+ BackupID uuid.UUID
+ Creds *postgresql_shared.CredentialTempFiles
+ Label string
+ SystemID uint64
+ IncrementalManifestPath string
+ Classify streamErrorClassifier
+
+ // MintAndSaveAttemptName produces the object key for the next attempt and
+ // persists it on the backup row. The loop owns neither the naming inputs nor
+ // the repository, and the two effects cannot be separated: a name the row does
+ // not carry is a name nothing can clean up.
+ MintAndSaveAttemptName func() (string, error)
+}
+
func streamWithCodecFallback(
ctx context.Context,
- common CommonBackupSpec,
- backupID uuid.UUID,
- creds *postgresql_shared.CredentialTempFiles,
- fileName string,
- systemID uint64,
- incrementalManifestPath string,
- classify streamErrorClassifier,
+ spec streamAttemptSpec,
) (PhysicalBackupResult, error) {
- pgBin := tools.GetPostgresqlExecutable(common.SourceDB.Version, tools.PostgresqlExecutablePgBasebackup)
+ pgBin := tools.GetPostgresqlExecutable(spec.Common.SourceDB.Version, tools.PostgresqlExecutablePgBasebackup)
- var settled streamOutcome
+ var (
+ settled streamOutcome
+ settledFileName string
+ )
for i, codec := range codecFallbackOrder {
+ fileName, err := spec.MintAndSaveAttemptName()
+ if err != nil {
+ return PhysicalBackupResult{}, err
+ }
+
+ settledFileName = fileName
+
buildCmd := func(streamCtx context.Context) (*exec.Cmd, error) {
return newPgBasebackupCommand(
streamCtx,
pgBin,
- common.SourceDB,
- creds,
- fileName,
+ spec.Common.SourceDB,
+ spec.Creds,
+ spec.Label,
codec,
- incrementalManifestPath,
+ spec.IncrementalManifestPath,
)
}
outcome, err := runStream(
ctx,
- common.runStreamParams(fileName, backupID, systemID, codec),
+ spec.Common.runStreamParams(fileName, spec.BackupID, spec.SystemID, codec),
buildCmd,
- classify,
+ spec.Classify,
)
if err != nil {
return PhysicalBackupResult{}, err
@@ -86,9 +106,9 @@ func streamWithCodecFallback(
if outcome.isCompressionUnsupported {
if i+1 < len(codecFallbackOrder) {
- common.Logger.Warn(
+ spec.Common.Logger.Warn(
fmt.Sprintf("compression downgraded: %s -> %s", codec, codecFallbackOrder[i+1]),
- "backup_id", backupID)
+ "backup_id", spec.BackupID)
continue
}
@@ -103,7 +123,7 @@ func streamWithCodecFallback(
break
}
- return resultFromOutcome(settled, fileName), nil
+ return resultFromOutcome(settled, settledFileName), nil
}
func resultFromOutcome(outcome streamOutcome, fileName string) PhysicalBackupResult {
@@ -111,6 +131,7 @@ func resultFromOutcome(outcome streamOutcome, fileName string) PhysicalBackupRes
Status: outcome.Status,
ErrorReason: outcome.ErrorReason,
ErrorMessage: outcome.ErrorMessage,
+ Receipts: outcome.Receipts,
FileName: fileName,
TimelineID: outcome.TimelineID,
StartLSN: outcome.StartLSN,
@@ -144,12 +165,11 @@ func errorResult(
}
}
-// buildObjectName is the storage object key for a backup artifact:
-// "<dbName>-<kind>-<timestamp>-<backupID>", kind being FULL or INCR. The name is
-// sanitized for storage portability (same as logical backups); uniqueness comes
-// from the trailing backupID. The codec is recorded on the row, never in the
-// name, so the key is extension-less.
-func buildObjectName(
+// buildBackupLabel names the backup, not the file it produces:
+// "<dbName>-<kind>-<timestamp>-<backupID>", kind being FULL or INCR, sanitized for
+// storage portability the same way logical backups are. It stays the same across
+// codec attempts, so pg_basebackup reports one label for the whole backup.
+func buildBackupLabel(
databaseName string,
backupID uuid.UUID,
now time.Time,
@@ -163,6 +183,13 @@ func buildObjectName(
)
}
+// Each codec attempt writes its own object, because a name whose cleanup is
+// pending cannot be written again. The codec is recorded on the row, never in the
+// key, so the name stays extension-less.
+func buildObjectName(label string, attemptID uuid.UUID) string {
+ return label + "-" + attemptID.String()
+}
+
func verifyFullTimelineCompatibility(
ctx context.Context,
common CommonBackupSpec,
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor_test.go
index 0d7d4a58..7754ba49 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/executor_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/executor_test.go
@@ -1,6 +1,7 @@
package usecases_physical_postgresql
import (
+ "strings"
"testing"
"time"
@@ -8,7 +9,7 @@ import (
"github.com/stretchr/testify/assert"
)
-func Test_BuildObjectName_FormatsReadableKey(t *testing.T) {
+func Test_BuildBackupLabel_FormatsReadableName(t *testing.T) {
backupID := uuid.MustParse("9b2c3d4e-5f60-7081-9234-56789abcdef0")
now := time.Date(2026, 5, 30, 12, 0, 0, 0, time.UTC)
@@ -16,27 +17,36 @@ func Test_BuildObjectName_FormatsReadableKey(t *testing.T) {
name string
databaseName string
kind string
- expected string
+ label string
}{
{
name: "clean name and FULL kind",
databaseName: "production",
kind: "FULL",
- expected: "production-FULL-20260530-120000-" + backupID.String(),
+ label: "production-FULL-20260530-120000-" + backupID.String(),
},
{
name: "unsafe characters are sanitized",
databaseName: "my db/prod",
kind: "INCR",
- expected: "my_db-prod-INCR-20260530-120000-" + backupID.String(),
+ label: "my_db-prod-INCR-20260530-120000-" + backupID.String(),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- objectName := buildObjectName(tt.databaseName, backupID, now, tt.kind)
-
- assert.Equal(t, tt.expected, objectName)
+ assert.Equal(t, tt.label, buildBackupLabel(tt.databaseName, backupID, now, tt.kind))
})
}
}
+
+func Test_BuildObjectName_GivesEachAttemptItsOwnKey(t *testing.T) {
+ label := buildBackupLabel("production", uuid.New(), time.Now().UTC(), "FULL")
+
+ first := buildObjectName(label, uuid.New())
+ second := buildObjectName(label, uuid.New())
+
+ assert.NotEqual(t, first, second, "a retry must not address the object a pending cleanup owns")
+ assert.True(t, strings.HasPrefix(first, label+"-"))
+ assert.True(t, strings.HasPrefix(second, label+"-"))
+}
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go
index ef4557c8..6ccc4b93 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/metadata.go
@@ -11,12 +11,13 @@ import (
"strings"
"time"
+ "github.com/google/uuid"
+
physical_dto "databasus-backend/internal/features/backups/backups/core/physical/dto"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"
postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical"
- "databasus-backend/internal/features/storages"
- util_encryption "databasus-backend/internal/util/encryption"
+ storage_files "databasus-backend/internal/features/storages/files"
"databasus-backend/internal/util/tools"
)
@@ -32,14 +33,14 @@ const metadataUploadTimeout = 2 * time.Minute
// makes the sidecar atomic with the COMPLETED result.
func uploadFullMetadata(
logger *slog.Logger,
- fieldEncryptor util_encryption.FieldEncryptor,
- storage storages.StorageFileSaver,
+ fileStore storage_files.FileStore,
+ storageID uuid.UUID,
sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,
fullBackup *physical_models.PhysicalFullBackup,
result PhysicalBackupResult,
-) error {
+) (storage_files.WriteReceipt, error) {
if result.FileName == "" {
- return errors.New("cannot upload metadata: file_name is empty")
+ return storage_files.WriteReceipt{}, errors.New("cannot upload metadata: file_name is empty")
}
metadata := physical_dto.PhysicalBackupMetadata{
@@ -61,7 +62,7 @@ func uploadFullMetadata(
CompletedAt: result.CompletedAt,
}
- return uploadMetadata(logger, fieldEncryptor, storage, result.FileName, metadata)
+ return uploadMetadata(logger, fileStore, storageID, result.FileName, metadata)
}
// uploadIncrMetadata writes the `<artifact>.metadata` sidecar for a completed
@@ -69,14 +70,14 @@ func uploadFullMetadata(
// needs to walk the chain.
func uploadIncrMetadata(
logger *slog.Logger,
- fieldEncryptor util_encryption.FieldEncryptor,
- storage storages.StorageFileSaver,
+ fileStore storage_files.FileStore,
+ storageID uuid.UUID,
sourceDB *postgresql_physical.PostgresqlPhysicalDatabase,
incrBackup *physical_models.PhysicalIncrementalBackup,
result PhysicalBackupResult,
-) error {
+) (storage_files.WriteReceipt, error) {
if result.FileName == "" {
- return errors.New("cannot upload metadata: file_name is empty")
+ return storage_files.WriteReceipt{}, errors.New("cannot upload metadata: file_name is empty")
}
rootID := incrBackup.RootFullBackupID
@@ -102,7 +103,7 @@ func uploadIncrMetadata(
CompletedAt: result.CompletedAt,
}
- return uploadMetadata(logger, fieldEncryptor, storage, result.FileName, metadata)
+ return uploadMetadata(logger, fileStore, storageID, result.FileName, metadata)
}
// uploadMetadata marshals the metadata and PUTs `<artifact>.metadata`. It uses a
@@ -110,14 +111,14 @@ func uploadIncrMetadata(
// cancelled backup context must not abort the sidecar write.
func uploadMetadata(
logger *slog.Logger,
- fieldEncryptor util_encryption.FieldEncryptor,
- storage storages.StorageFileSaver,
+ fileStore storage_files.FileStore,
+ storageID uuid.UUID,
artifactFileName string,
metadata physical_dto.PhysicalBackupMetadata,
-) error {
+) (storage_files.WriteReceipt, error) {
body, err := json.Marshal(metadata)
if err != nil {
- return fmt.Errorf("marshal metadata JSON: %w", err)
+ return storage_files.WriteReceipt{}, fmt.Errorf("marshal metadata JSON: %w", err)
}
metadataName := artifactFileName + metadataSuffix
@@ -125,11 +126,18 @@ func uploadMetadata(
ctx, cancel := context.WithTimeout(context.Background(), metadataUploadTimeout)
defer cancel()
- if err := storage.SaveFile(ctx, fieldEncryptor, logger, metadataName, bytes.NewReader(body)); err != nil {
- return fmt.Errorf("upload metadata: %w", err)
+ logger.DebugContext(ctx, "uploading backup metadata sidecar", "file_name", metadataName)
+
+ receipt, err := fileStore.WriteFile(
+ ctx,
+ storage_files.StoredFileReference{StorageID: storageID, FileName: metadataName},
+ bytes.NewReader(body),
+ )
+ if err != nil {
+ return storage_files.WriteReceipt{}, fmt.Errorf("upload metadata: %w", err)
}
- return nil
+ return receipt, nil
}
// pgVersionFromTag converts the tools.PostgresqlVersion enum ("17", "18") into
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
index adad9b15..0e203692 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mtls_test.go
@@ -9,6 +9,7 @@ import (
physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"
"databasus-backend/internal/features/databases"
+ "databasus-backend/internal/features/storages"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/testing/containers"
@@ -88,6 +89,7 @@ func Test_FullOverMtls_ProducesArtifactAndManifest(t *testing.T) {
DatabaseName: fixture.DB.Name,
StorageID: fixture.Storage.ID,
Storage: fixture.Storage,
+ FileStore: storages.GetStorageFileStore(),
Encryption: fullRow.Encryption,
FieldEncryptor: encryption.GetFieldEncryptor(),
FullRepo: physical_repositories.GetFullBackupRepository(),
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go
index db6c28ea..c2141a3a 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream.go
@@ -15,7 +15,7 @@ import (
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
util_encryption "databasus-backend/internal/util/encryption"
)
@@ -35,7 +35,8 @@ const (
// runStreamParams carries everything runStream needs that does not come from the
// *exec.Cmd itself. Grouped into a struct so FULL and INCR pass the same shape.
type runStreamParams struct {
- Storage storages.StorageFileSaver
+ FileStore storage_files.FileStore
+ StorageID uuid.UUID
FieldEncryptor util_encryption.FieldEncryptor
Logger *slog.Logger
@@ -142,17 +143,13 @@ func runStream(
defer stopProgress()
}
- saveErrCh := make(chan error, 1)
-
- go func() {
- saveErr := p.Storage.SaveFile(streamCtx, p.FieldEncryptor, p.Logger, p.FileName, storageReader)
- if saveErr != nil {
- _ = storageReader.CloseWithError(saveErr)
- cancelStream(saveErr)
- }
-
- saveErrCh <- saveErr
- }()
+ fileWrite := storage_files.StartBackgroundWrite(
+ streamCtx,
+ p.FileStore,
+ storage_files.StoredFileReference{StorageID: p.StorageID, FileName: p.FileName},
+ storageReader,
+ cancelStream,
+ )
manifestReader, manifestWriter := io.Pipe()
manifestResultCh := make(chan manifestWalkResult, 1)
@@ -182,7 +179,7 @@ func runStream(
// drain BOTH goroutines, or the manifest goroutine leaks on manifestReader.
_ = storageWriter.Close()
_ = manifestWriter.Close()
- saveErr := <-saveErrCh
+ saveErr := <-fileWrite.Errors
<-manifestResultCh
stderr.stop()
@@ -226,7 +223,7 @@ func runStream(
_ = storageWriter.Close()
_ = manifestWriter.Close()
- saveErr := <-saveErrCh
+ saveErr := <-fileWrite.Errors
manifestRes := <-manifestResultCh
stderr.stop()
@@ -275,22 +272,41 @@ func runStream(
return outcomeManifestCorrupted(fmt.Errorf("walk tar for manifest: %w", manifestRes.err), stderrBytes), nil
}
- return buildSuccessOutcome(ctx, p, manifestRes.entries, counter, encryptionWriter, encSalt, encNonce, stderrBytes)
+ return buildSuccessOutcome(ctx, successOutcomeSpec{
+ Params: p,
+ Files: manifestRes.entries,
+ Counter: counter,
+ EncryptionWriter: encryptionWriter,
+ EncryptionSalt: encSalt,
+ EncryptionIV: encNonce,
+ Stderr: stderrBytes,
+ BaseReceipt: <-fileWrite.Receipts,
+ })
}
// buildSuccessOutcome runs in the linear section after Wait: it parses the LSNs
// (only complete once the process exited), serializes the manifest, and saves the
// sidecar. Any failure here is fatal to the backup — an INCR chain without a
// valid parent manifest is unusable — so it maps to MANIFEST_CORRUPTED.
-func buildSuccessOutcome(
- ctx context.Context,
- p runStreamParams,
- files []manifestFileEntry,
- counter *ByteCounter,
- encryptionWriter *backup_encryption.EncryptionWriter,
- encSalt, encNonce string,
- stderrBytes []byte,
-) (streamOutcome, error) {
+type successOutcomeSpec struct {
+ Params runStreamParams
+ Files []manifestFileEntry
+ Counter *ByteCounter
+ EncryptionWriter *backup_encryption.EncryptionWriter
+ EncryptionSalt string
+ EncryptionIV string
+ Stderr []byte
+ BaseReceipt storage_files.WriteReceipt
+}
+
+func buildSuccessOutcome(ctx context.Context, spec successOutcomeSpec) (streamOutcome, error) {
+ p := spec.Params
+ files := spec.Files
+ counter := spec.Counter
+ encryptionWriter := spec.EncryptionWriter
+ encSalt, encNonce := spec.EncryptionSalt, spec.EncryptionIV
+ stderrBytes := spec.Stderr
+
startLSN, stopLSN, timelineID, err := parseLsnsFromStderr(stderrBytes)
if err != nil {
return outcomeManifestCorrupted(fmt.Errorf("parse pg_basebackup LSNs: %w", err), stderrBytes), nil
@@ -316,7 +332,7 @@ func buildSuccessOutcome(
manifestFileName := p.FileName + manifestSuffix
- manifestSalt, manifestIV, err := saveManifestSidecar(ctx, p, manifestFileName, manifestBytes)
+ manifest, err := saveManifestSidecar(ctx, p, manifestFileName, manifestBytes)
if err != nil {
return outcomeManifestCorrupted(fmt.Errorf("save manifest sidecar: %w", err), stderrBytes), nil
}
@@ -337,8 +353,9 @@ func buildSuccessOutcome(
EncryptionIV: encNonce,
Compression: p.Codec,
ManifestFileName: manifestFileName,
- ManifestEncryptionSalt: manifestSalt,
- ManifestEncryptionIV: manifestIV,
+ ManifestEncryptionSalt: manifest.SaltBase64,
+ ManifestEncryptionIV: manifest.NonceBase64,
+ Receipts: []storage_files.WriteReceipt{spec.BaseReceipt, manifest.Receipt},
Stderr: stderrBytes,
}, nil
}
@@ -359,7 +376,12 @@ func saveManifestSidecar(
p runStreamParams,
manifestFileName string,
manifestBytes []byte,
-) (saltBase64, nonceBase64 string, err error) {
+) (manifestSidecar, error) {
+ var (
+ saltBase64 string
+ nonceBase64 string
+ )
+
payload := bytes.NewReader(manifestBytes)
payloadSizeBytes := len(manifestBytes)
@@ -368,15 +390,15 @@ func saveManifestSidecar(
encSetup, setupErr := backup_encryption.SetupEncryptionWriter(&encrypted, p.MasterKey, p.BackupID)
if setupErr != nil {
- return "", "", fmt.Errorf("setup manifest encryption: %w", setupErr)
+ return manifestSidecar{}, fmt.Errorf("setup manifest encryption: %w", setupErr)
}
if _, writeErr := encSetup.Writer.Write(manifestBytes); writeErr != nil {
- return "", "", fmt.Errorf("encrypt manifest: %w", writeErr)
+ return manifestSidecar{}, fmt.Errorf("encrypt manifest: %w", writeErr)
}
if closeErr := encSetup.Writer.Close(); closeErr != nil {
- return "", "", fmt.Errorf("flush manifest encryption: %w", closeErr)
+ return manifestSidecar{}, fmt.Errorf("flush manifest encryption: %w", closeErr)
}
payload = bytes.NewReader(encrypted.Bytes())
@@ -397,11 +419,24 @@ func saveManifestSidecar(
saveCtx, cancel := context.WithTimeout(ctx, saveTimeout)
defer cancel()
- if err := p.Storage.SaveFile(saveCtx, p.FieldEncryptor, p.Logger, manifestFileName, payload); err != nil {
- return "", "", err
+ receipt, err := p.FileStore.WriteFile(
+ saveCtx,
+ storage_files.StoredFileReference{StorageID: p.StorageID, FileName: manifestFileName},
+ payload,
+ )
+ if err != nil {
+ return manifestSidecar{}, err
}
- return saltBase64, nonceBase64, nil
+ return manifestSidecar{Receipt: receipt, SaltBase64: saltBase64, NonceBase64: nonceBase64}, nil
+}
+
+// manifestSidecar is what the manifest upload produced: the receipt that keeps the
+// file and the encryption material its catalog row records.
+type manifestSidecar struct {
+ Receipt storage_files.WriteReceipt
+ SaltBase64 string
+ NonceBase64 string
}
func outcomeNetworkStall(stderr []byte) streamOutcome {
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_outcome.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_outcome.go
index abd639ed..64da4948 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_outcome.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_outcome.go
@@ -3,6 +3,7 @@ package usecases_physical_postgresql
import (
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
+ storage_files "databasus-backend/internal/features/storages/files"
"databasus-backend/internal/util/walmath"
)
@@ -39,6 +40,10 @@ type streamOutcome struct {
Compression physical_enums.PhysicalBackupCompression
+ // Receipts are spent by the transaction that publishes the backup. An outcome
+ // that carries none leaves every file it wrote for cleanup.
+ Receipts []storage_files.WriteReceipt
+
// Reconstructed-manifest sidecar. Salt/IV are the manifest's OWN fresh values
// (never the tar's), so they live on dedicated row columns.
ManifestFileName string
diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
index aab07609..4a4bfd0d 100644
--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/stream_test.go
@@ -24,7 +24,7 @@ import (
backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"
physical_enums "databasus-backend/internal/features/backups/backups/core/physical/enums"
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
- "databasus-backend/internal/features/storages"
+ storage_files "databasus-backend/internal/features/storages/files"
"databasus-backend/internal/util/encryption"
)
@@ -113,17 +113,33 @@ func (f *fakeStorageFileSaver) has(fileName string) bool {
func (f *fakeStorageFileSaver) DeleteFile(context.Context, encryption.FieldEncryptor, *slog.Logger, string) error {
return nil
}
-func (f *fakeStorageFileSaver) Validate(encryption.FieldEncryptor) error { return nil }
-func (f *fakeStorageFileSaver) TestConnection(encryption.FieldEncryptor) error { return nil }
+func (f *fakeStorageFileSaver) Validate(encryption.FieldEncryptor) error { return nil }
+func (f *fakeStorageFileSaver) TestConnection(encryption.FieldEncryptor) error { return nil }
+
+// The physical stream writes through the file store now, and the fake keeps its
+// own recording so the deadline and payload assertions stay where they were.
+func (f *fakeStorageFileSaver) WriteFile(
+ ctx context.Context,
+ reference storage_files.StoredFileReference,
+ file io.Reader,
+) (storage_files.WriteReceipt, error) {
+ if err := f.SaveFile(ctx, nil, nil, reference.FileName, file); err != nil {
+ return storage_files.WriteReceipt{}, err
+ }
+
+ return storage_files.WriteReceipt{PendingDeletionID: uuid.New(), Reference: reference}, nil
+}
+
func (f *fakeStorageFileSaver) HideSensitiveData() {}
func (f *fakeStorageFileSaver) EncryptSensitiveData(encryption.FieldEncryptor) error { return nil }
func testRunStreamParams(
- storage storages.StorageFileSaver,
+ storage *fakeStorageFileSaver,
codec physical_enums.PhysicalBackupCompression,
) runStreamParams {
return runStreamParams{
- Storage: storage,
+ FileStore: storage,
+ StorageID: uuid.New(),
FieldEncryptor: nil,
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
FileName: "test-obj",
@@ -266,7 +282,10 @@ func Test_RunStream_WhenStorageSaveFails_ReturnsStorageUploadFailedNotStall(t *t
"a storage failure that cancels the stream must not be reported as a network stall")
}
-func Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact(t *testing.T) {
+// runStream sits below the layer that owns cleanup, so what it guarantees is that a
+// failed attempt hands back nothing to publish. The bytes it already uploaded are
+// removed by the worker, which the use-case tests cover against a real store.
+func Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesNoReceiptToPublish(t *testing.T) {
storage := newFakeStorage()
outcome, err := runStream(
@@ -280,6 +299,7 @@ func Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact(
assert.Equal(t, physical_enums.PhysicalBackupStatusError, outcome.Status)
assert.Equal(t, physical_enums.PhysicalBackupErrorPgBasebackupFailed, *outcome.ErrorReason)
assert.Equal(t, []byte("partial physical backup"), storage.saved["test-obj"])
+ assert.Empty(t, outcome.Receipts, "a failed attempt must leave its bytes for cleanup, not for publication")
}
func Test_RunStream_WhenManifestWalkFails_ReturnsManifestCorrupted(t *testing.T) {
@@ -340,17 +360,20 @@ func Test_SaveManifestSidecar_WhenEncrypted_RoundTripsWithOwnSaltIV(t *testing.T
manifest := []byte(`{ "PostgreSQL-Backup-Manifest-Version": 2, "System-Identifier": 42 }`)
params := runStreamParams{
- Storage: storage,
+ FileStore: storage,
+ StorageID: uuid.New(),
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
MasterKey: masterKey,
Encryption: backups_core_enums.BackupEncryptionEncrypted,
BackupID: backupID,
}
- saltB64, nonceB64, err := saveManifestSidecar(t.Context(), params, "obj.manifest", manifest)
+ sidecar, err := saveManifestSidecar(t.Context(), params, "obj.manifest", manifest)
require.NoError(t, err)
- require.NotEmpty(t, saltB64)
- require.NotEmpty(t, nonceB64)
+ require.NotEmpty(t, sidecar.SaltBase64)
+ require.NotEmpty(t, sidecar.NonceBase64)
+
+ saltB64, nonceB64 := sidecar.SaltBase64, sidecar.NonceBase64
stored := storage.saved["obj.manifest"]
require.NotEqual(t, manifest, stored, "stored sidecar must be ciphertext")
@@ -391,7 +414,7 @@ func Test_SaveManifestSidecar_WhenManifestIsSmall_UsesFloorDeadline(t *testing.T
storage := newFakeStorage()
params := testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone)
- _, _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", []byte(`{ "Files": [] }`))
+ _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", []byte(`{ "Files": [] }`))
require.NoError(t, err)
deadline, hasDeadline := storage.deadlineFor("obj.manifest")
@@ -409,7 +432,7 @@ func Test_SaveManifestSidecar_WhenManifestIsLarge_ExtendsDeadlineBeyondFloor(t *
params := testRunStreamParams(storage, physical_enums.PhysicalBackupCompressionNone)
largeManifest := bytes.Repeat([]byte("m"), 40*1024*1024)
- _, _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", largeManifest)
+ _, err := saveManifestSidecar(t.Context(), params, "obj.manifest", largeManifest)
require.NoError(t, err)
deadline, hasDeadline := storage.deadlineFor("obj.manifest")
diff --git a/backend/internal/features/storages/files/background_write.go b/backend/internal/features/storages/files/background_write.go
new file mode 100644
index 00000000..cb162b03
--- /dev/null
+++ b/backend/internal/features/storages/files/background_write.go
@@ -0,0 +1,44 @@
+package storage_files
+
+import (
+ "context"
+ "io"
+)
+
+// BackgroundWrite is an upload running beside the producer that feeds it through a
+// pipe. Logical dumps and physical streams drive it the same way: peek at the error
+// while the producer is still running, then take the receipt once the writer is done.
+type BackgroundWrite struct {
+ // Errors stays writable because the engines put a peeked value back and their
+ // cancellation cleanup drains it.
+ Errors chan error
+ Receipts chan WriteReceipt
+}
+
+// StartBackgroundFileWrite sends the receipt before the error, so a caller holding
+// the error knows the receipt is already waiting.
+func StartBackgroundWrite(
+ ctx context.Context,
+ store FileStore,
+ reference StoredFileReference,
+ source *io.PipeReader,
+ onFailure context.CancelCauseFunc,
+) BackgroundWrite {
+ write := BackgroundWrite{
+ Errors: make(chan error, 1),
+ Receipts: make(chan WriteReceipt, 1),
+ }
+
+ go func() {
+ receipt, err := store.WriteFile(ctx, reference, source)
+ if err != nil {
+ _ = source.CloseWithError(err)
+ onFailure(err)
+ }
+
+ write.Receipts <- receipt
+ write.Errors <- err
+ }()
+
+ return write
+}
diff --git a/backend/internal/features/storages/files/dto.go b/backend/internal/features/storages/files/dto.go
index bc6efad3..a5ab4428 100644
--- a/backend/internal/features/storages/files/dto.go
+++ b/backend/internal/features/storages/files/dto.go
@@ -50,6 +50,12 @@ type WriteReceipt struct {
Generation int
}
+// FileStore is what a caller needs to write one file and get a receipt for it.
+// Store satisfies it, and so does any narrower seam a feature declares.
+type FileStore interface {
+ WriteFile(ctx context.Context, reference StoredFileReference, file io.Reader) (WriteReceipt, error)
+}
+
type FileWriter interface {
SaveFile(
ctx context.Context,
diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.md
index 4d6a9633..845da54f 100644
--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md
+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md
@@ -36,7 +36,7 @@
## 6. Integrate physical full and incremental backups
-- [ ] 6.1 Route full and incremental stream uploads and their sidecars through the file store and return receipts only after writer goroutines are joined. Give `buildObjectName` (`.../executor.go:152`) an attempt UUID, since its four current inputs are constant across the codec fallback loop, and move its parameters into an `objectNameSpec` struct rather than adding a fifth positional value and a second adjacent `uuid.UUID`. Have `streamWithCodecFallback` (`.../executor.go:50`) mint each name through a `mintAndSaveAttemptName` callback its two callers supply, named for both of its effects, because the loop holds neither the naming inputs nor the repository that persists `file_name`, and move that function's stream inputs into a `streamAttemptSpec` struct rather than growing an already eight-value positional list. The callback becomes the only producer of object names: the pre-loop `buildObjectName` call and its `Save` (`create_full_uc.go:40` with its assignment and `Save` at `:42` and `:43`, `create_incremental_uc.go:76` with its own at `:78` and `:80`) stop minting the object key and keep only the stable pg_basebackup label, which `newPgBasebackupCommand` takes at `pg_basebackup.go:50` and which names the backup rather than the object. Return the settled attempt's name and take every consumer from that result rather than the caller's variable: `runStreamParams` (`.../executor.go:79`) so the upload-time manifest at `stream.go:317` follows it, the chain-broken removal and the reported `FileName` (`create_full_uc.go:84`, `:92`), `streamResult.FileName` (`create_full_uc.go:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`). Correct the two comments that state the old shape, `executor.go:45` ("every attempt streams to the same object key") and the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and move `executor_test.go` onto the new shape, renaming the table field `expected` (declared at `:19`, set at `:25` and `:31`, read at `:39`) to `objectName` and updating its values for the attempt UUID; verify stream success, a zstd rejection that downgrades to gzip and then completes without the second attempt being refused for a name already in use and with the manifest landing under the settled name, pg_basebackup failure after partial output, cancellation and provider failure with focused tests, renaming `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact` to `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_UploadedArtifactRemoved` and converting it into a cleanup assertion, because the old name states the leak this task removes.
+- [x] 6.1 Route full and incremental stream uploads and their sidecars through the file store and return receipts only after writer goroutines are joined. Give `buildObjectName` (`.../executor.go:152`) an attempt UUID, since its four current inputs are constant across the codec fallback loop, and split it instead into `buildBackupLabel`, which keeps the four inputs it already had, and `buildObjectName(label, attemptID)`. That satisfies the rule the struct was there to protect without a struct: neither function grows a fifth positional value, and neither takes two adjacent `uuid.UUID` values. Have `streamWithCodecFallback` (`.../executor.go:50`) mint each name through a `mintAndSaveAttemptName` callback its two callers supply, named for both of its effects, because the loop holds neither the naming inputs nor the repository that persists `file_name`, and move that function's stream inputs into a `streamAttemptSpec` struct rather than growing an already eight-value positional list. The callback becomes the only producer of object names: the pre-loop `buildObjectName` call and its `Save` (`create_full_uc.go:40` with its assignment and `Save` at `:42` and `:43`, `create_incremental_uc.go:76` with its own at `:78` and `:80`) stop minting the object key and keep only the stable pg_basebackup label, which `newPgBasebackupCommand` takes at `pg_basebackup.go:50` and which names the backup rather than the object. Return the settled attempt's name and take every consumer from that result rather than the caller's variable: `runStreamParams` (`.../executor.go:79`) so the upload-time manifest at `stream.go:317` follows it, the chain-broken removal and the reported `FileName` (`create_full_uc.go:84`, `:92`), `streamResult.FileName` (`create_full_uc.go:113`), the incremental equivalent (`create_incremental_uc.go:116`) and the metadata sidecar (`metadata.go:123`). Correct the two comments that state the old shape, `executor.go:45` ("every attempt streams to the same object key") and the `buildObjectName` comment about uniqueness coming from the trailing backup ID, and move `executor_test.go` onto the new shape, renaming the table field `expected` (declared at `:19`, set at `:25` and `:31`, read at `:39`) to `objectName` and updating its values for the attempt UUID; verify stream success, a zstd rejection that downgrades to gzip and then completes without the second attempt being refused for a name already in use and with the manifest landing under the settled name, pg_basebackup failure after partial output, cancellation and provider failure with focused tests, renaming `Test_RunStream_WhenPgBasebackupWritesBytesThenFails_LeavesUploadedArtifact` to `..._LeavesNoReceiptToPublish` and asserting the outcome carries no receipts. `runStream` sits below the layer that owns cleanup and its tests drive a fake store, so removal cannot be observed there; task 6.3 covers it against a real store.
- [ ] 6.2 Confirm physical receipts in the same transaction that publishes the backup and leave pending deletions in place on terminal failure paths, including manifest, metadata and incremental-chain failures; verify rollback and restart behavior with service and scheduler tests.
- [ ] 6.3 Convert the physical reproduction tests into passing cleanup tests on local storage and MinIO; run the worker until idle and assert that partial base files, manifests, exact chunks, multipart uploads and metadata are absent after ordinary errors and cancellation.
- [ ] 6.4 Rewrite restart recovery so it no longer assumes a missing backup file name means no object exists, replacing the stale premise recorded at `scheduler.go:710`, and request deletion for a claimless `IN_PROGRESS` row that carries a file name in the same transaction that records its terminal state, gated on that conditional `IN_PROGRESS` transition affecting one row so a late completion that already wrote a terminal status keeps its file; verify with `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_LeavesArtifact` renamed to `Test_RecoverInFlightOnRestart_WhenFullHasNamedArtifact_ArtifactRemoved` and rewritten to assert eventual absence, and that recovery neither deletes a confirmed successful backup nor duplicates an existing pending deletion.