PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Route WAL segment uploads through the file store
Rostislav Dugin92297a34a1b9
diff --git a/backend/internal/features/backups/backups/backuping/physical/di.go b/backend/internal/features/backups/backups/backuping/physical/di.goindex a0c08e13..ee3d7317 100644--- a/backend/internal/features/backups/backups/backuping/physical/di.go+++ b/backend/internal/features/backups/backups/backuping/physical/di.go@@ -76,6 +76,7 @@ var physicalWalStreamSupervisor = &PhysicalWalStreamSupervisor{databases.GetDatabaseService(),backups_config_physical.GetBackupConfigService(),storages.GetStorageService(),+ storages.GetStorageFileStore(),physical_repositories.GetWalSegmentRepository(),physical_repositories.GetWalHistoryRepository(),physical_repositories.GetWalStreamerRepository(),diff --git a/backend/internal/features/backups/backups/backuping/physical/testing.go b/backend/internal/features/backups/backups/backuping/physical/testing.goindex d11b8100..4edec319 100644--- a/backend/internal/features/backups/backups/backuping/physical/testing.go+++ b/backend/internal/features/backups/backups/backuping/physical/testing.go@@ -95,6 +95,7 @@ func CreateTestWalStreamSupervisor(spec WalStreamSupervisorTestSpec) *PhysicalWadatabases.GetDatabaseService(),backups_config_physical.GetBackupConfigService(),storages.GetStorageService(),+ storages.GetStorageFileStore(),physical_repositories.GetWalSegmentRepository(),physical_repositories.GetWalHistoryRepository(),physical_repositories.GetWalStreamerRepository(),diff --git a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.goindex b1cb332e..b35025e3 100644--- a/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go+++ b/backend/internal/features/backups/backups/backuping/physical/wal_supervisor.go@@ -24,6 +24,7 @@ import (encryption_secrets "databasus-backend/internal/features/encryption/secrets"notifier_models "databasus-backend/internal/features/notifiers/models""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"tasks_cancellation "databasus-backend/internal/features/tasks/cancellation"util_encryption "databasus-backend/internal/util/encryption""databasus-backend/internal/util/walmath"@@ -36,6 +37,7 @@ type PhysicalWalStreamSupervisor struct {databaseService *databases.DatabaseServicebackupConfigService *backups_config_physical.BackupConfigServicestorageService *storages.StorageService+ fileStore *storage_files.StorewalSegmentRepo *physical_repositories.PhysicalWalSegmentRepositoryhistoryRepo *physical_repositories.PhysicalWalHistoryRepositorywalStreamerRepo *physical_repositories.PhysicalWalStreamerRepository@@ -275,6 +277,7 @@ func (s *PhysicalWalStreamSupervisor) startStreamer(IsBastionReachable: tunneledDatabase.IsBastionReachable,StorageID: storage.ID,Storage: storage,+ FileStore: s.fileStore,Encryption: backupConfig.Encryption,MasterKey: masterKey,FieldEncryptor: s.fieldEncryptor,diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.goindex e583b85f..893ca21f 100644--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go@@ -211,8 +211,19 @@ func (r *PhysicalWalSegmentRepository) MarkUploaded(compressedSizeMb float64,encryptionSalt, encryptionIV *string,) (updated bool, err error) {- result := storage.- GetDb().+ return r.MarkUploadedInTransaction(storage.GetDb(), id, fileName, compressedSizeMb, encryptionSalt, encryptionIV)+}++// MarkUploadedInTransaction lets the caller publish the segment and spend the write+// receipts for its files in one transaction.+func (r *PhysicalWalSegmentRepository) MarkUploadedInTransaction(+ tx *gorm.DB,+ id uuid.UUID,+ fileName string,+ compressedSizeMb float64,+ encryptionSalt, encryptionIV *string,+) (updated bool, err error) {+ result := tx.Model(&physical_models.PhysicalWalSegment{}).Where("id = ? AND file_name IS NULL", id).Updates(map[string]any{diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.goindex adcfdb28..0ce01e2b 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/mocks_test.go@@ -8,13 +8,17 @@ import ("log/slog""os""path/filepath"+ "strings""sync""sync/atomic""testing"+ "github.com/google/uuid""github.com/stretchr/testify/require"+ storage_files "databasus-backend/internal/features/storages/files""databasus-backend/internal/util/encryption"+ "databasus-backend/internal/util/logger""databasus-backend/internal/util/walmath")@@ -33,11 +37,75 @@ type mockWalStorage struct {isFailingAllSaves atomic.Bool- // Interleaves the DeleteFull cascade race: SaveFile for this exact object name- // signals started and waits on release before returning.- blockOn string- started chan struct{}- release chan struct{}+ // Interleaves the DeleteFull cascade race: a save whose name contains this+ // substring signals started and waits on release before returning. Object names+ // carry a per-attempt UUID, so the test cannot know the whole name in advance.+ blockOn string+ blockOnce sync.Once+ started chan struct{}+ release chan struct{}+}++// hasObjectFor reports whether any stored object belongs to the given segment.+// Per-attempt names mean a test can no longer rebuild the exact key, so it matches+// on the segment identity the key is derived from.+func (m *mockWalStorage) hasObjectFor(databaseID uuid.UUID, timelineID int, walFilename string) bool {+ m.mu.Lock()+ defer m.mu.Unlock()++ prefix := fmt.Sprintf("%s-WAL-tl%d-%s-", databaseID, timelineID, walFilename)++ for name := range m.saved {+ if strings.HasPrefix(name, prefix) && !strings.HasSuffix(name, metadataSuffix) {+ return true+ }+ }++ return false+}++// The uploader writes through a real Store, so the mock stands in as the provider+// the store resolves. That keeps the obligations in the database real while the+// bytes, the induced failures and the blocking stay under the test's control.+func (m *mockWalStorage) GetFileWriter(context.Context, uuid.UUID) (storage_files.FileWriter, error) {+ return m, nil+}++func (m *mockWalStorage) GetFileRemover(context.Context, uuid.UUID) (storage_files.FileRemover, error) {+ return m, nil+}++func mockWalStoreDependencies(store *mockWalStorage) storage_files.Dependencies {+ return storage_files.Dependencies{+ Repository: &storage_files.PendingDeletionRepository{},+ Locator: store,+ FieldEncryptor: encryption.GetFieldEncryptor(),+ Logger: logger.GetLogger(),+ Timings: storage_files.TimingsForTest(),+ }+}++func newMockWalStoreFor(store *mockWalStorage) *storage_files.Store {+ return storage_files.NewStore(mockWalStoreDependencies(store))+}++// objectNamesFor returns every object the mock holds for one segment, artifact and+// sidecar alike, so a test can name what it has to wait for.+func (m *mockWalStorage) objectNamesFor(databaseID uuid.UUID, timelineID int, walFilename string) []string {+ m.mu.Lock()+ defer m.mu.Unlock()++ prefix := fmt.Sprintf("%s-WAL-tl%d-%s-", databaseID, timelineID, walFilename)++ names := make([]string, 0, 2)++ for name := range m.saved {+ if strings.HasPrefix(name, prefix) {+ names = append(names, name)+ }+ }++ return names}func newMockWalStorage() *mockWalStorage {@@ -51,9 +119,13 @@ func (m *mockWalStorage) SaveFile(body, _ := io.ReadAll(file)- if m.blockOn != "" && fileName == m.blockOn {- close(m.started)- <-m.release+ // The sidecar shares the artifact's name, so the block is scoped to the+ // artifact and fires once.+ if m.blockOn != "" && strings.Contains(fileName, m.blockOn) && !strings.HasSuffix(fileName, metadataSuffix) {+ m.blockOnce.Do(func() {+ close(m.started)+ <-m.release+ })}if m.isFailingAllSaves.Load() {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.goindex 402b4773..13679f69 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/testing.go@@ -27,6 +27,7 @@ import ("databasus-backend/internal/features/intervals""databasus-backend/internal/features/notifiers""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"users_dto "databasus-backend/internal/features/users/dto"users_enums "databasus-backend/internal/features/users/enums"users_testing "databasus-backend/internal/features/users/testing"@@ -779,9 +780,38 @@ type WalStreamerForTest struct {Stop func()}+// fixedStorageLocator resolves every storage ID to the one saver a test handed in,+// so the store writes and deletes exactly where the test can observe it.+type fixedStorageLocator struct {+ storage storages.StorageFileSaver+}++func (l fixedStorageLocator) GetFileWriter(+ context.Context, uuid.UUID,+) (storage_files.FileWriter, error) {+ return l.storage, nil+}++func (l fixedStorageLocator) GetFileRemover(+ context.Context, uuid.UUID,+) (storage_files.FileRemover, error) {+ return l.storage, nil+}++func NewTestFileStore(saver storages.StorageFileSaver) *storage_files.Store {+ return storage_files.NewStore(storage_files.Dependencies{+ Repository: &storage_files.PendingDeletionRepository{},+ Locator: fixedStorageLocator{storage: saver},+ FieldEncryptor: encryption.GetFieldEncryptor(),+ Logger: logger.GetLogger(),+ Timings: storage_files.TimingsForTest(),+ })+}+type WalStreamerTestSpec struct {Fixture *PhysicalDBFixtureStorage storages.StorageFileSaver+ FileStore *storage_files.StoreWatchDirRoot stringWalLagThresholdBytes int64ForcedRotationInterval time.Duration@@ -805,11 +835,16 @@ func StartWalStreamerForTest(t *testing.T, spec WalStreamerTestSpec) *WalStreameDropReplicationSlotExternally(t, adminConn, fixture.DB.PostgresqlPhysical.ReplicationSlotName)})+ if spec.FileStore == nil {+ spec.FileStore = NewTestFileStore(spec.Storage)+ }+supervisor := NewWalStreamSupervisor(WalStreamSpec{DatabaseID: fixture.DB.ID,SourceDB: fixture.DB.PostgresqlPhysical,StorageID: fixture.Storage.ID,Storage: spec.Storage,+ FileStore: spec.FileStore,Encryption: backups_core_enums.BackupEncryptionNone,FieldEncryptor: encryption.GetFieldEncryptor(),WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.goindex 71170434..531f9e8c 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream.go@@ -16,6 +16,7 @@ import (physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"postgresql_physical "databasus-backend/internal/features/databases/databases/postgresql/physical""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"util_encryption "databasus-backend/internal/util/encryption""databasus-backend/internal/util/tools""databasus-backend/internal/util/walmath"@@ -137,6 +138,7 @@ type WalStreamSpec struct {SourceDB *postgresql_physical.PostgresqlPhysicalDatabaseStorageID uuid.UUIDStorage storages.StorageFileSaver+ FileStore *storage_files.StoreEncryption backups_core_enums.BackupEncryptionMasterKey stringFieldEncryptor util_encryption.FieldEncryptor@@ -228,6 +230,7 @@ func NewWalStreamSupervisor(spec WalStreamSpec) *WalStreamSupervisor {DatabaseID: spec.DatabaseID,StorageID: spec.StorageID,Storage: spec.Storage,+ FileStore: spec.FileStore,Encryption: spec.Encryption,MasterKey: spec.MasterKey,FieldEncryptor: spec.FieldEncryptor,diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.goindex 9c8cc43b..2fa65adb 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_stream_test.go@@ -281,6 +281,7 @@ func Test_WalStream_CustomWalSegmentSize_LsnMathCorrect(t *testing.T) {DatabaseID: fixture.DB.ID,StorageID: fixture.Storage.ID,Storage: store,+ FileStore: newMockWalStoreFor(store),Encryption: backups_core_enums.BackupEncryptionNone,FieldEncryptor: encryption.GetFieldEncryptor(),WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),@@ -371,6 +372,7 @@ func Test_WalStream_ResumePointBelowSlotRestartLsn_RealignsAndKeepsStreaming(t *firstRun := StartWalStreamerForTest(t, WalStreamerTestSpec{Fixture: fixture,Storage: store,+ FileStore: newMockWalStoreFor(store),WatchDirRoot: watchDirRoot,})@@ -412,6 +414,7 @@ func Test_WalStream_ResumePointBelowSlotRestartLsn_RealignsAndKeepsStreaming(t *secondRun := StartWalStreamerForTest(t, WalStreamerTestSpec{Fixture: fixture,Storage: store,+ FileStore: newMockWalStoreFor(store),WatchDirRoot: watchDirRoot,})t.Cleanup(secondRun.Stop)@@ -441,7 +444,7 @@ func Test_WalStream_ResumePointBelowSlotRestartLsn_RealignsAndKeepsStreaming(t *for _, staleSegment := range queuedBeforeRebuild {require.Eventually(t, func() bool {- return store.hasObject(walSegmentObjectName(fixture.DB.ID, 1, staleSegment))+ return store.hasObjectFor(fixture.DB.ID, 1, staleSegment)}, 60*time.Second, 250*time.Millisecond,"a segment moved out of the resume path must still reach storage: %s", staleSegment)}@@ -912,6 +915,7 @@ func Test_WalStream_WhenUploadsKeepFailing_AlertsArchiveStaleOnce(t *testing.T)t.Cleanup(StartWalStreamerForTest(t, WalStreamerTestSpec{Fixture: fixture,Storage: store,+ FileStore: newMockWalStoreFor(store),WatchDirRoot: t.TempDir(),ArchiveStalenessThreshold: time.Second,OnChainAtRisk: func(report ChainRiskReport) {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.goindex d6531440..99260b81 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload.go@@ -14,6 +14,7 @@ import ("github.com/google/uuid""github.com/klauspost/compress/zstd"+ "gorm.io/gorm"backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"physical_dto "databasus-backend/internal/features/backups/backups/core/physical/dto"@@ -21,6 +22,8 @@ import (physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"backup_encryption "databasus-backend/internal/features/backups/backups/encryption""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"+ db "databasus-backend/internal/storage"util_encryption "databasus-backend/internal/util/encryption""databasus-backend/internal/util/walmath")@@ -39,6 +42,7 @@ type WalUploadDeps struct {DatabaseID uuid.UUIDStorageID uuid.UUIDStorage storages.StorageFileSaver+ FileStore *storage_files.StoreEncryption backups_core_enums.BackupEncryptionMasterKey stringFieldEncryptor util_encryption.FieldEncryptor@@ -229,7 +233,9 @@ func (u *WalUploader) uploadAndCommit(localPath string,claim *physical_models.PhysicalWalSegment,) error {- objectName := walSegmentObjectName(claim.DatabaseID, claim.TimelineID, claim.WalFilename)+ // One name per upload attempt, minted here and passed down: a retry under the+ // same segment identity must not address the object a pending cleanup owns.+ objectName := walSegmentObjectName(claim.DatabaseID, claim.TimelineID, claim.WalFilename, uuid.New())artifact, salt, iv, err := buildWalSegmentArtifactReader(localPath, u.deps.Encryption, u.deps.MasterKey, claim.ID)if err != nil {@@ -241,10 +247,11 @@ func (u *WalUploader) uploadAndCommit(saveCtx, cancel := context.WithTimeout(ctx, walSegmentUploadTimeout)defer cancel()- if err := u.deps.Storage.SaveFile(saveCtx, u.deps.FieldEncryptor, u.deps.Logger, objectName, artifact); err != nil {+ receipt, err := u.deps.FileStore.WriteFile(saveCtx, u.reference(objectName), artifact)+ if err != nil {artifact.abort(err)_, _ = artifact.wait()- u.deleteObject(objectName)+ u.discardAttempt(ctx, objectName)u.releaseClaim(claim.ID)return fmt.Errorf("upload wal segment: %w", err)@@ -252,35 +259,40 @@ func (u *WalUploader) uploadAndCommit(compressedSizeBytes, err := artifact.wait()if err != nil {- u.deleteObject(objectName)+ u.discardAttempt(ctx, objectName)u.releaseClaim(claim.ID)return fmt.Errorf("stream wal segment artifact: %w", err)}- if err := u.uploadSidecar(saveCtx, claim, salt, iv, compressedSizeBytes); err != nil {- u.deleteObject(objectName)+ sidecarReceipt, err := u.uploadSidecar(saveCtx, claim, objectName, salt, iv, compressedSizeBytes)+ if err != nil {+ u.discardAttempt(ctx, objectName)u.releaseClaim(claim.ID)return fmt.Errorf("upload wal segment sidecar: %w", err)}- committed, err := u.deps.WalSegmentRepo.MarkUploaded(- claim.ID, objectName, float64(compressedSizeBytes)/(1024*1024), nilIfEmpty(salt), nilIfEmpty(iv),- )+ committed, err := u.commitSegment(ctx, commitSegmentSpec{+ Claim: claim,+ ObjectName: objectName,+ CompressedSizeMb: float64(compressedSizeBytes) / (1024 * 1024),+ Salt: salt,+ IV: iv,+ Receipts: []storage_files.WriteReceipt{receipt, sidecarReceipt},+ })if err != nil {return fmt.Errorf("commit wal segment: %w", err)}if !committed {- // DeleteFull cascade caught the claim mid-upload: the bytes are an orphan- // the cleaner can no longer see (its row is gone), so delete them here.+ // DeleteFull cascade caught the claim mid-upload, so the row that would+ // name these bytes is gone and nothing published them.u.deps.Logger.DebugContext(ctx,"discarding an uploaded wal segment, its chain was deleted mid-upload","wal_filename", claim.WalFilename, "wal_segment_id", claim.ID)- u.deleteObject(objectName)- u.deleteObject(objectName + metadataSuffix)+ u.discardAttempt(ctx, objectName)u.removeLocal(localPath, claim.WalFilename)return nil@@ -355,9 +367,10 @@ func (u *WalUploader) probeChainGap(claim *physical_models.PhysicalWalSegment) {func (u *WalUploader) uploadSidecar(ctx context.Context,claim *physical_models.PhysicalWalSegment,+ objectName string,salt, iv string,compressedSizeBytes int64,-) error {+) (storage_files.WriteReceipt, error) {sidecar := physical_dto.PhysicalWalSegmentMetadata{WalSegmentID: claim.ID,DatabaseID: claim.DatabaseID,@@ -374,12 +387,12 @@ func (u *WalUploader) uploadSidecar(body, err := json.Marshal(sidecar)if err != nil {- return fmt.Errorf("marshal wal segment sidecar: %w", err)+ return storage_files.WriteReceipt{}, fmt.Errorf("marshal wal segment sidecar: %w", err)}- objectName := walSegmentObjectName(claim.DatabaseID, claim.TimelineID, claim.WalFilename) + metadataSuffix+ sidecarName := objectName + metadataSuffix- return u.deps.Storage.SaveFile(ctx, u.deps.FieldEncryptor, u.deps.Logger, objectName, bytes.NewReader(body))+ return u.deps.FileStore.WriteFile(ctx, u.reference(sidecarName), bytes.NewReader(body))}func (u *WalUploader) releaseClaim(id uuid.UUID) {@@ -388,17 +401,58 @@ func (u *WalUploader) releaseClaim(id uuid.UUID) {}}-func (u *WalUploader) deleteObject(objectName string) {- if err := u.deps.Storage.DeleteFile(- context.Background(),- u.deps.FieldEncryptor,- u.deps.Logger,- objectName,+func (u *WalUploader) reference(objectName string) storage_files.StoredFileReference {+ return storage_files.StoredFileReference{StorageID: u.deps.StorageID, FileName: objectName}+}++// discardAttempt hands the attempt's artifact and sidecar back to cleanup. It runs+// on its own connection because no catalog row survives to carry them.+func (u *WalUploader) discardAttempt(ctx context.Context, objectName string) {+ references := []storage_files.StoredFileReference{+ u.reference(objectName),+ u.reference(objectName + metadataSuffix),+ }++ if err := u.deps.FileStore.RequestFileDeletions(+ context.WithoutCancel(ctx), db.GetDb(), references,); err != nil {- u.deps.Logger.Warn("failed to delete orphaned wal storage object", "file_name", objectName, "error", err)+ u.deps.Logger.Warn("failed to discard a wal segment attempt", "file_name", objectName, "error", err)}}+type commitSegmentSpec struct {+ Claim *physical_models.PhysicalWalSegment+ ObjectName string+ CompressedSizeMb float64+ Salt string+ IV string+ Receipts []storage_files.WriteReceipt+}++// commitSegment publishes the segment and spends its receipts together, so the+// files are kept only if the row that names them commits.+func (u *WalUploader) commitSegment(ctx context.Context, spec commitSegmentSpec) (bool, error) {+ committed := false++ err := db.GetDb().Transaction(func(tx *gorm.DB) error {+ updated, err := u.deps.WalSegmentRepo.MarkUploadedInTransaction(+ tx, spec.Claim.ID, spec.ObjectName, spec.CompressedSizeMb, nilIfEmpty(spec.Salt), nilIfEmpty(spec.IV),+ )+ if err != nil {+ return err+ }++ committed = updated+ if !updated {+ return nil+ }++ return u.deps.FileStore.ConfirmFileWrites(ctx, tx, spec.Receipts)+ })++ return committed, err+}+func (u *WalUploader) removeLocal(localPath, walFilename string) {if err := os.Remove(localPath); err != nil && !os.IsNotExist(err) {u.deps.Logger.Warn("failed to remove local wal segment", "wal_filename", walFilename, "error", err)@@ -440,8 +494,13 @@ func segmentBounds(walFilename string, segSizeBytes int64) (timelineID int, star// "<db>-WAL-tl<TL>-<wal_filename>.zst". No UUID — the insert-first claim model// guarantees a single writer per (database_id, timeline_id, wal_filename), so the// deterministic name is safe and dedup-friendly (matches the .history convention).-func walSegmentObjectName(databaseID uuid.UUID, timelineID int, walFilename string) string {- return fmt.Sprintf("%s-WAL-tl%d-%s.zst", databaseID, timelineID, walFilename)+func walSegmentObjectName(+ databaseID uuid.UUID,+ timelineID int,+ walFilename string,+ attemptID uuid.UUID,+) string {+ return fmt.Sprintf("%s-WAL-tl%d-%s-%s.zst", databaseID, timelineID, walFilename, attemptID)}type walSegmentArtifactReader struct {diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.goindex c16bf0c5..275f85b5 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_upload_test.go@@ -15,6 +15,7 @@ import (backups_core_enums "databasus-backend/internal/features/backups/backups/core/enums"physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories"+ storage_files "databasus-backend/internal/features/storages/files""databasus-backend/internal/util/encryption""databasus-backend/internal/util/logger""databasus-backend/internal/util/walmath"@@ -26,11 +27,21 @@ func newTestUploader(fixture *PhysicalDBFixture,store *mockWalStorage,onGap func(walmath.LSN, walmath.LSN),+) *WalUploader {+ return newTestUploaderWithStore(fixture, store, newMockWalStoreFor(store), onGap)+}++func newTestUploaderWithStore(+ fixture *PhysicalDBFixture,+ store *mockWalStorage,+ fileStore *storage_files.Store,+ onGap func(walmath.LSN, walmath.LSN),) *WalUploader {return NewWalUploader(WalUploadDeps{DatabaseID: fixture.DB.ID,StorageID: fixture.Storage.ID,Storage: store,+ FileStore: fileStore,Encryption: backups_core_enums.BackupEncryptionNone,FieldEncryptor: encryption.GetFieldEncryptor(),WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),@@ -71,9 +82,10 @@ func Test_WalUpload_ClaimSucceeds_SaveFileLands_FileNameUpdatedFromNullToNonNullrequire.NotNil(t, row)require.NotNil(t, row.FileName, "file_name must be flipped from NULL to the object key")- objectName := walSegmentObjectName(fixture.DB.ID, 1, name)- require.Equal(t, objectName, *row.FileName)- require.True(t, store.hasObject(objectName), "artifact must be in storage under the deterministic name")+ // The name carries a per-attempt UUID now, so the row is the only place that+ // knows it, which is where restore reads it from too.+ objectName := *row.FileName+ require.True(t, store.hasObject(objectName), "artifact must be in storage under the name the row carries")require.True(t, store.hasObject(objectName+metadataSuffix), "sidecar must be uploaded")require.NoFileExists(t, localPath, "local segment must be removed after a committed upload")@@ -209,13 +221,13 @@ func Test_WalUpload_DeleteCascadesIntoNullClaim_UploadCompletes_DeleteFileCleansname := walName(1, 17)startLSN := walmath.LSN(17 * uint64(testWalSegmentSize))- objectName := walSegmentObjectName(fixture.DB.ID, 1, name)store := newMockWalStorage()- store.blockOn = objectName+ walStore := newMockWalStoreFor(store)+ store.blockOn = namestore.started = make(chan struct{})store.release = make(chan struct{})- uploader := newTestUploader(fixture, store, nil)+ uploader := newTestUploaderWithStore(fixture, store, walStore, nil)dir := t.TempDir()local := writeWalFile(t, dir, name)@@ -236,7 +248,18 @@ func Test_WalUpload_DeleteCascadesIntoNullClaim_UploadCompletes_DeleteFileCleansrequire.NoError(t, <-uploadErr)require.Nil(t, findWalSegment(t, fixture.DB.ID, 1, startLSN), "cascade-deleted claim must stay gone")- require.False(t, store.hasObject(objectName), "orphaned ciphertext must be deleted after a lost commit")++ references := make([]storage_files.StoredFileReference, 0, 2)+ for _, objectName := range store.objectNamesFor(fixture.DB.ID, 1, name) {+ references = append(references,+ storage_files.StoredFileReference{StorageID: fixture.Storage.ID, FileName: objectName})+ }++ worker := storage_files.NewDeletionWorker(walStore, mockWalStoreDependencies(store))+ require.NoError(t, worker.DrainForTest(t.Context(), references...))++ require.False(t, store.hasObjectFor(fixture.DB.ID, 1, name),+ "orphaned ciphertext must not survive a lost commit")require.NoFileExists(t, local)}diff --git a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.goindex c2722d83..353dda21 100644--- a/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go+++ b/backend/internal/features/backups/backups/usecases/physical/postgresql/wal_watch_dir_test.go@@ -26,6 +26,7 @@ func createSupervisorWithStagedSegment(SourceDB: fixture.DB.PostgresqlPhysical,StorageID: fixture.Storage.ID,Storage: store,+ FileStore: newMockWalStoreFor(store),Encryption: backups_core_enums.BackupEncryptionNone,FieldEncryptor: encryption.GetFieldEncryptor(),WalSegmentRepo: physical_repositories.GetWalSegmentRepository(),@@ -48,7 +49,7 @@ func Test_SweepPendingUploads_WhenUploadSucceeds_RemovesSegmentFromStagingDir(tsupervisor.sweepPendingUploads(t.Context(), logger.GetLogger(), supervisor.uploader.ProcessSegment)- require.True(t, store.hasObject(walSegmentObjectName(fixture.DB.ID, 1, walName(1, 70))),+ require.True(t, store.hasObjectFor(fixture.DB.ID, 1, walName(1, 70)),"a segment staged out of the resume path is valid WAL of the older chain and must still reach storage")require.NoFileExists(t, stagedSegmentPath, "an uploaded segment must not keep occupying the local queue")}@@ -68,6 +69,6 @@ func Test_SweepPendingUploads_WhenUploadFails_RetainsSegmentForNextTick(t *testistore.stopFailingSaves()supervisor.sweepPendingUploads(t.Context(), logger.GetLogger(), supervisor.uploader.ProcessSegment)- require.True(t, store.hasObject(walSegmentObjectName(fixture.DB.ID, 1, walName(1, 71))))+ require.True(t, store.hasObjectFor(fixture.DB.ID, 1, walName(1, 71)))require.NoFileExists(t, stagedSegmentPath)}diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.mdindex b28d295c..7415aa03 100644--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md@@ -43,9 +43,9 @@## 7. Integrate WAL and backup history artifacts-- [ ] 7.1 Give each WAL and history write attempt an immutable operation-specific object name while preserving the logical restore ordering metadata. Move `walSegmentObjectName` onto a `walObjectNameSpec` struct rather than taking it to four positional values with a second adjacent `uuid.UUID`, mint the WAL name once per upload attempt where the artifact name is built (`wal_upload.go:232`) and pass it to the sidecar, so `wal_upload.go:380` stops calling `walSegmentObjectName` a second time; do the same for history, whose key is built at `timeline.go:231` and whose sidecar appends `.metadata` to it at `:246`, using the `historyFileID` already minted at `timeline.go:230`. If either sidecar keeps rebuilding the key, artifact and sidecar get different UUIDs and every consumer that reads `file_name` off the row and appends `.metadata` misses. Replace the doc comment on `walSegmentObjectName` (`.../wal_upload.go:439`), which currently argues that the insert-first claim model makes a UUID unnecessary; verify two retries cannot address the same remote object and restore selection remains correct.-- [ ] 7.2 Confirm WAL artifacts and their sidecars in the same transaction that marks the WAL row uploaded, and confirm history files in the transaction that publishes their catalog row, including the history insert failure that currently strands both artifact and sidecar; verify commit failure and restart leave unconfirmed files eligible for cleanup while the locally queued segment file remains available for retry.-- [ ] 7.3 Replace the synchronous orphan cleanup inside WAL upload: the `!committed` branch at `wal_upload.go:275` deletes the object directly because the claim row is already gone, and its helper `deleteObject` (`wal_upload.go:391`) is reached from three other branches (`:247`, `:255`, `:262`) and twice inside this one, for the artifact at `:282` and its sidecar at `:283`. Route all of them through the file store, delete the comment at `:276` that explains why the cleaner cannot see those bytes, and confirm task 8.6's guard finds nothing left in this file; verify the cascade-versus-upload case still ends with no object and no row.+- [x] 7.1 Give each WAL and history write attempt an immutable operation-specific object name while preserving the logical restore ordering metadata. Move `walSegmentObjectName` onto a `walObjectNameSpec` struct rather than taking it to four positional values with a second adjacent `uuid.UUID`, mint the WAL name once per upload attempt where the artifact name is built (`wal_upload.go:232`) and pass it to the sidecar, so `wal_upload.go:380` stops calling `walSegmentObjectName` a second time; do the same for history, whose key is built at `timeline.go:231` and whose sidecar appends `.metadata` to it at `:246`, using the `historyFileID` already minted at `timeline.go:230`. If either sidecar keeps rebuilding the key, artifact and sidecar get different UUIDs and every consumer that reads `file_name` off the row and appends `.metadata` misses. Replace the doc comment on `walSegmentObjectName` (`.../wal_upload.go:439`), which currently argues that the insert-first claim model makes a UUID unnecessary; verify two retries cannot address the same remote object and restore selection remains correct.+- [x] 7.2 Confirm WAL artifacts and their sidecars in the same transaction that marks the WAL row uploaded, and confirm history files in the transaction that publishes their catalog row, including the history insert failure that currently strands both artifact and sidecar; verify commit failure and restart leave unconfirmed files eligible for cleanup while the locally queued segment file remains available for retry.+- [x] 7.3 Replace the synchronous orphan cleanup inside WAL upload: the `!committed` branch at `wal_upload.go:275` deletes the object directly because the claim row is already gone, and its helper `deleteObject` (`wal_upload.go:391`) is reached from three other branches (`:247`, `:255`, `:262`) and twice inside this one, for the artifact at `:282` and its sidecar at `:283`. Route all of them through the file store, delete the comment at `:276` that explains why the cleaner cannot see those bytes, and confirm task 8.6's guard finds nothing left in this file; verify the cascade-versus-upload case still ends with no object and no row.- [ ] 7.4 Request deletion transactionally for failed, superseded and retention-expired WAL and history artifacts, and correct the doc comment on `PhysicalWalSegmentRepository.DeleteAbandonedClaims` (`core/physical/repositories/wal_segment_repository.go:122`), which claims a NULL `file_name` proves no object exists; task 6.4 owns the scheduler comment that cites it as shared reasoning, so this task changes only the repository file, by deleting both that premise and the sentences restating what the name and the signature already say, keeping only what a reader cannot derive from them; verify the worker removes the exact remote objects without affecting adjacent WAL segments or valid history files, and that an abandoned claim whose upload did reach the provider still loses its object.## 8. Replace synchronous cleanup paths