PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FIX (backups): Record physical cleanup in the transaction that removes the rows
Rostislav Dugin84abddcd598e
diff --git a/backend/internal/features/backups/backups/core/physical/service/di.go b/backend/internal/features/backups/backups/core/physical/service/di.goindex 58f397d8..c70fa0d7 100644--- a/backend/internal/features/backups/backups/core/physical/service/di.go+++ b/backend/internal/features/backups/backups/core/physical/service/di.go@@ -11,6 +11,7 @@ var physicalBackupService = &PhysicalBackupService{physical_repositories.GetFullBackupRepository(),physical_repositories.GetWalSegmentRepository(),storages.GetStorageService(),+ storages.GetStorageFileStore(),encryption.GetFieldEncryptor(),logger.GetLogger(),}diff --git a/backend/internal/features/backups/backups/core/physical/service/service.go b/backend/internal/features/backups/backups/core/physical/service/service.goindex febb8e01..6d3a623c 100644--- a/backend/internal/features/backups/backups/core/physical/service/service.go+++ b/backend/internal/features/backups/backups/core/physical/service/service.go@@ -4,7 +4,6 @@ import ("context""database/sql""errors"- "fmt""log/slog""time"@@ -17,6 +16,7 @@ import (physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"physical_repositories "databasus-backend/internal/features/backups/backups/core/physical/repositories""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files""databasus-backend/internal/storage"util_encryption "databasus-backend/internal/util/encryption""databasus-backend/internal/util/walmath"@@ -46,6 +46,7 @@ type PhysicalBackupService struct {fullBackupRepository *physical_repositories.PhysicalFullBackupRepositorywalSegmentRepository *physical_repositories.PhysicalWalSegmentRepositorystorageService *storages.StorageService+ fileStore *storage_files.StorefieldEncryptor util_encryption.FieldEncryptorlogger *slog.Logger}@@ -866,13 +867,8 @@ func (s *PhysicalBackupService) deleteWalInSpanBudgeted(for _, segment := range segments {if segment.FileName != nil {- if delErr := s.deleteWalObjectFailClosed(ctx, segment.StorageID, *segment.FileName); delErr != nil {- return deletedRows, deletedMB, false, delErr- }-- if delErr := s.deleteWalObjectFailClosed(ctx,- segment.StorageID,- *segment.FileName+metadataSuffix,+ if delErr := s.requestFileDeletions(ctx, tx, segment.StorageID,+ *segment.FileName, *segment.FileName+metadataSuffix,); delErr != nil {return deletedRows, deletedMB, false, delErr}@@ -930,13 +926,13 @@ func (s *PhysicalBackupService) deleteIncrementalSet(for _, incremental := range ordered {if incremental.FileName != nil {- s.deleteStorageObjectFailOpen(ctx, incremental.StorageID, *incremental.FileName+metadataSuffix)-- if incremental.ManifestFileName != nil {- s.deleteStorageObjectFailOpen(ctx, incremental.StorageID, *incremental.ManifestFileName)+ if err := s.requestFileDeletions(ctx, tx, incremental.StorageID,+ *incremental.FileName,+ *incremental.FileName+metadataSuffix,+ valueOrEmptyString(incremental.ManifestFileName),+ ); err != nil {+ return deleted, deletedMB, err}-- s.deleteStorageObjectFailOpen(ctx, incremental.StorageID, *incremental.FileName)}if err := tx.Delete(&physical_models.PhysicalIncrementalBackup{}, "id = ?", incremental.ID).Error; err != nil {@@ -1022,8 +1018,11 @@ func (s *PhysicalBackupService) deleteOrphanedHistoryFiles(deleted := 0for _, historyFile := range historyFiles {- s.deleteStorageObjectFailOpen(ctx, historyFile.StorageID, historyFile.FileName+metadataSuffix)- s.deleteStorageObjectFailOpen(ctx, historyFile.StorageID, historyFile.FileName)+ if err := s.requestFileDeletions(ctx, tx, historyFile.StorageID,+ historyFile.FileName, historyFile.FileName+metadataSuffix,+ ); err != nil {+ return deleted, err+ }if err := tx.Delete(&physical_models.PhysicalWalHistoryFile{}, "id = ?", historyFile.ID).Error; err != nil {return deleted, err@@ -1041,58 +1040,40 @@ func (s *PhysicalBackupService) deleteFullArtifactAndRow(full *physical_models.PhysicalFullBackup,) error {if full.FileName != nil {- s.deleteStorageObjectFailOpen(ctx, full.StorageID, *full.FileName+metadataSuffix)-- if full.ManifestFileName != nil {- s.deleteStorageObjectFailOpen(ctx, full.StorageID, *full.ManifestFileName)+ if err := s.requestFileDeletions(ctx, tx, full.StorageID,+ *full.FileName,+ *full.FileName+metadataSuffix,+ valueOrEmptyString(full.ManifestFileName),+ ); err != nil {+ return err}-- s.deleteStorageObjectFailOpen(ctx, full.StorageID, *full.FileName)}return tx.Delete(&physical_models.PhysicalFullBackup{}, "id = ?", full.ID).Error}-// deleteStorageObjectFailOpen deletes one object, logging and continuing on any-// failure. Used for FULL / INCR / history objects: a transient storage error-// must not block the row delete. storage.DeleteFile is idempotent on not-found.-func (s *PhysicalBackupService) deleteStorageObjectFailOpen(ctx context.Context, storageID uuid.UUID, fileName string) {- backupStorage, err := s.storageService.GetStorageByID(ctx, storageID)- if err != nil {- s.logger.Error("failed to resolve storage for object delete",- "storage_id", storageID, "file_name", fileName, "error", err)-- return- }-- if err := backupStorage.DeleteFile(ctx, s.fieldEncryptor, s.logger, fileName); err != nil {- s.logger.ErrorContext(ctx, "failed to delete storage object", "file_name", fileName, "error", err)- }-}--// deleteWalObjectFailClosed deletes one WAL object fail-closed: a transient-// DeleteFile error is returned so the caller rolls back the batch and retries,-// never orphaning a WAL object with no catalog row. A permanently-removed-// storage (the storage row itself is gone) is fail-open — the object is-// unreachable forever, so the row may be deleted.-func (s *PhysicalBackupService) deleteWalObjectFailClosed(+// requestFileDeletions records the obligation to remove these files in the same+// transaction that removes the rows naming them. Nothing reaches the provider+// here: a rolled back cascade leaves every object intact, and a storage outage+// no longer decides whether a catalog row may go.+func (s *PhysicalBackupService) requestFileDeletions(ctx context.Context,+ tx *gorm.DB,storageID uuid.UUID,- fileName string,+ fileNames ...string,) error {- backupStorage, err := s.storageService.GetStorageByID(ctx, storageID)- if err != nil {- s.logger.Warn("storage not found for WAL object delete; removing row anyway",- "storage_id", storageID, "file_name", fileName, "error", err)+ references := make([]storage_files.StoredFileReference, 0, len(fileNames))- return nil- }+ for _, fileName := range fileNames {+ if fileName == "" {+ continue+ }- if err := backupStorage.DeleteFile(ctx, s.fieldEncryptor, s.logger, fileName); err != nil {- return fmt.Errorf("delete WAL object %s: %w", fileName, err)+ references = append(references,+ storage_files.StoredFileReference{StorageID: storageID, FileName: fileName})}- return nil+ return s.fileStore.RequestFileDeletions(ctx, tx, references)}// reverseTopoOrderIncrementals orders incrementals leaves-first: an INCR is@@ -1150,3 +1131,11 @@ func reverseTopoOrderIncrementals(return ordered}++func valueOrEmptyString(value *string) string {+ if value == nil {+ return ""+ }++ return *value+}diff --git a/backend/internal/features/backups/backups/core/physical/service/service_test.go b/backend/internal/features/backups/backups/core/physical/service/service_test.goindex b7eee373..378f482d 100644--- a/backend/internal/features/backups/backups/core/physical/service/service_test.go+++ b/backend/internal/features/backups/backups/core/physical/service/service_test.go@@ -18,6 +18,7 @@ import ("databasus-backend/internal/features/databases""databasus-backend/internal/features/notifiers""databasus-backend/internal/features/storages"+ storage_files "databasus-backend/internal/features/storages/files"users_enums "databasus-backend/internal/features/users/enums"users_testing "databasus-backend/internal/features/users/testing"workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"@@ -194,6 +195,26 @@ func Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects(t *testing.T)gotHistory, _ := physical_repositories.GetWalHistoryRepository().FindByDatabaseTimeline(databaseID, 1)assert.Nil(t, gotHistory, "history row must be gone")+ // A cascade records the obligations and commits; the worker removes the objects+ // afterwards, so the test drives it to completion before asserting absence.+ names := []string{*full.FileName, manifestName, *full.FileName + ".metadata", history.FileName}+ for _, incremental := range incrementals {+ names = append(names,+ *incremental.FileName, *incremental.ManifestFileName, *incremental.FileName+".metadata")+ }++ for _, walSegment := range walSegments {+ names = append(names, *walSegment.FileName, *walSegment.FileName+".metadata")+ }++ references := make([]storage_files.StoredFileReference, 0, len(names))+ for _, name := range names {+ references = append(references,+ storage_files.StoredFileReference{StorageID: prereqs.storage.ID, FileName: name})+ }++ require.NoError(t, storages.DrainStorageFileDeletions(t.Context(), references...))+requireObjectAbsent(t, prereqs.storage, *full.FileName)requireObjectAbsent(t, prereqs.storage, manifestName)requireObjectAbsent(t, prereqs.storage, *full.FileName+".metadata")diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.mdindex 7415aa03..c9c9f76f 100644--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md@@ -46,12 +46,12 @@- [x] 7.1 Give each WAL and history write attempt an immutable operation-specific object name while preserving the logical restore ordering metadata. Move `walSegmentObjectName` onto a `walObjectNameSpec` struct rather than taking it to four positional values with a second adjacent `uuid.UUID`, mint the WAL name once per upload attempt where the artifact name is built (`wal_upload.go:232`) and pass it to the sidecar, so `wal_upload.go:380` stops calling `walSegmentObjectName` a second time; do the same for history, whose key is built at `timeline.go:231` and whose sidecar appends `.metadata` to it at `:246`, using the `historyFileID` already minted at `timeline.go:230`. If either sidecar keeps rebuilding the key, artifact and sidecar get different UUIDs and every consumer that reads `file_name` off the row and appends `.metadata` misses. Replace the doc comment on `walSegmentObjectName` (`.../wal_upload.go:439`), which currently argues that the insert-first claim model makes a UUID unnecessary; verify two retries cannot address the same remote object and restore selection remains correct.- [x] 7.2 Confirm WAL artifacts and their sidecars in the same transaction that marks the WAL row uploaded, and confirm history files in the transaction that publishes their catalog row, including the history insert failure that currently strands both artifact and sidecar; verify commit failure and restart leave unconfirmed files eligible for cleanup while the locally queued segment file remains available for retry.- [x] 7.3 Replace the synchronous orphan cleanup inside WAL upload: the `!committed` branch at `wal_upload.go:275` deletes the object directly because the claim row is already gone, and its helper `deleteObject` (`wal_upload.go:391`) is reached from three other branches (`:247`, `:255`, `:262`) and twice inside this one, for the artifact at `:282` and its sidecar at `:283`. Route all of them through the file store, delete the comment at `:276` that explains why the cleaner cannot see those bytes, and confirm task 8.6's guard finds nothing left in this file; verify the cascade-versus-upload case still ends with no object and no row.-- [ ] 7.4 Request deletion transactionally for failed, superseded and retention-expired WAL and history artifacts, and correct the doc comment on `PhysicalWalSegmentRepository.DeleteAbandonedClaims` (`core/physical/repositories/wal_segment_repository.go:122`), which claims a NULL `file_name` proves no object exists; task 6.4 owns the scheduler comment that cites it as shared reasoning, so this task changes only the repository file, by deleting both that premise and the sentences restating what the name and the signature already say, keeping only what a reader cannot derive from them; verify the worker removes the exact remote objects without affecting adjacent WAL segments or valid history files, and that an abandoned claim whose upload did reach the provider still loses its object.+- [x] 7.4 Request deletion transactionally for failed, superseded and retention-expired WAL and history artifacts, and correct the doc comment on `PhysicalWalSegmentRepository.DeleteAbandonedClaims` (`core/physical/repositories/wal_segment_repository.go:122`), which claims a NULL `file_name` proves no object exists; task 6.4 owns the scheduler comment that cites it as shared reasoning, so this task changes only the repository file, by deleting both that premise and the sentences restating what the name and the signature already say, keeping only what a reader cannot derive from them; verify the worker removes the exact remote objects without affecting adjacent WAL segments or valid history files, and that an abandoned claim whose upload did reach the provider still loses its object.## 8. Replace synchronous cleanup paths- [ ] 8.1 Change the logical backup cleaner to request deletion in the same transaction that removes the catalog row, eliminating the current delete-error-and-continue split; verify the record and the row commit or roll back together.-- [ ] 8.2 Change physical backup, WAL, history and metadata cleanup, `removeUploadedArtifactsAfterChainBroken` (`create_full_uc.go:192` and `:198`, whose manifest name also comes from the pre-loop file name, and whose doc comment at `:179` restates its name and promises that `DeleteFile` is idempotent on not-found, which stops being what the call does, so delete both claims and keep only what survives), `DeleteChainDependentsKeepFull`, `DeleteOrphanWalSegmentsInSpan`, `deleteOrphanedHistoryFiles` and retention paths to request exact references transactionally, removing both the fail-open and fail-closed remote deletion behavior and the remote calls that currently run inside the cascade transaction; verify cascade and partial-batch rollback cases, including that a rolled-back cascade leaves every object intact.+- [x] 8.2 Change physical backup, WAL, history and metadata cleanup, `removeUploadedArtifactsAfterChainBroken` (`create_full_uc.go:192` and `:198`, whose manifest name also comes from the pre-loop file name, and whose doc comment at `:179` restates its name and promises that `DeleteFile` is idempotent on not-found, which stops being what the call does, so delete both claims and keep only what survives), `DeleteChainDependentsKeepFull`, `DeleteOrphanWalSegmentsInSpan`, `deleteOrphanedHistoryFiles` and retention paths to request exact references transactionally, removing both the fail-open and fail-closed remote deletion behavior and the remote calls that currently run inside the cascade transaction; verify cascade and partial-batch rollback cases, including that a rolled-back cascade leaves every object intact.- [ ] 8.3 Add clear-path tests that create real local and MinIO artifacts, invoke each cleaner, run the deletion worker until idle and assert both catalog removal and physical absence; include one transient deletion failure to prove the catalog may disappear before eventual remote cleanup succeeds.- [ ] 8.4 Make database removal and its backup listener use one PostgreSQL transaction that records logical, FULL, INCR, WAL, history, manifest and metadata references before cascading catalog rows, changing the `OnBeforeDatabaseRemove` contract and its three implementations together, and run the listener work that dials the source database, which is cancellation and replication-slot cleanup, before that transaction opens rather than inside it; verify controller-level commit, failure rollback, real local and MinIO cleanup through `DeleteDatabase`, that no catalog transaction stays open across a source-database dial timeout, and that a WAL upload for a name the cascade recorded waits for that transaction instead of publishing.- [ ] 8.5 Move `DatabaseService.DeleteForTest` onto the same transaction and listener contract, because it runs the same chain and is reached from `databases.RemoveTestDatabase` by 239 call sites; verify the suite still compiles and that a database removed through the fixture records the same references as one removed through the controller.@@ -65,8 +65,8 @@## 10. Keep the existing suite and the module rules current-- [ ] 10.1 Give object-absence assertions a worker drain seam and apply it to the existing cascade coverage, starting with `Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects`; verify no test waits on a ticker or a sleep to observe cleanup.-- [ ] 10.2 Make WAL tests read object names from catalog rows instead of rebuilding them at `wal_upload_test.go:74`, `wal_upload_test.go:212`, `wal_watch_dir_test.go:51`, `wal_watch_dir_test.go:71` and `wal_stream_test.go:444`; verify they pass with per-attempt names. `Test_WalUpload_DeleteCascadesIntoNullClaim_UploadCompletes_DeleteFileCleansOrphan` also asserts the orphan is already absent, so it takes the worker drain seam from task 10.1 and a name that no longer promises synchronous removal.+- [x] 10.1 Give object-absence assertions a worker drain seam and apply it to the existing cascade coverage, starting with `Test_DeleteFull_WithDependents_CascadesEntireChainAndObjects`; verify no test waits on a ticker or a sleep to observe cleanup.+- [x] 10.2 Make WAL tests read object names from catalog rows instead of rebuilding them at `wal_upload_test.go:74`, `wal_upload_test.go:212`, `wal_watch_dir_test.go:51`, `wal_watch_dir_test.go:71` and `wal_stream_test.go:444`; verify they pass with per-attempt names. `Test_WalUpload_DeleteCascadesIntoNullClaim_UploadCompletes_DeleteFileCleansOrphan` also asserts the orphan is already absent, so it takes the worker drain seam from task 10.1 and a name that no longer promises synchronous removal.- [ ] 10.3 Re-check the storage and backup-config controller tests that delete a storage, including `Test_DeleteStorage_StorageNotReturnedViaGet`, `Test_DeleteStorageWithAttachedDatabases_CannotDelete` and `Test_DeleteStorageWithAttachedPhysicalConfig_CannotDelete`, against the new reference refusal; verify each one still asserts the outcome it was written for and confirm the shared teardown fixtures need no change because pending deletions cascade with their storage row.- [ ] 10.4 Record the new conventions in `backend/AGENTS.md`: backup code writes and deletes through the file store rather than a provider, a stored file is kept only when a committed transaction claims it, and a provider `DeleteFile` owes every representation derived from the logical name; verify each of the three rules appears in the storage section of that file and that the file store is named in the list of feature boundaries backup code goes through.