PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
REFACTOR (backups): Put each comment on what it describes
Rostislav Dugin27cd667c650f
diff --git a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go b/backend/internal/features/backups/backups/backuping/logical/backuper_test.goindex 393b0655..81012eda 100644--- a/backend/internal/features/backups/backups/backuping/logical/backuper_test.go+++ b/backend/internal/features/backups/backups/backuping/logical/backuper_test.go@@ -366,8 +366,6 @@ func Test_MakeBackup_WhenMetadataIsUnusable_BackupFailsAndArtifactRemoved(t *tesassertBackupFilesRemoved(t, fixture, backup.FileName)}-// The caller names the files its scenario actually produced: asserting the absence-// of a file nothing ever wrote would pass with no cleanup at all.// A receipt goes stale when something claims the file between the upload and the// publishing transaction. Bumping the generation by hand reproduces that without a// second goroutine racing the test.diff --git a/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go b/backend/internal/features/backups/backups/backuping/logical/cleaner_test.goindex 46399dd2..418293b3 100644--- a/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go+++ b/backend/internal/features/backups/backups/backuping/logical/cleaner_test.go@@ -612,8 +612,6 @@ func createTestInterval() intervals.Interval {}}-// The catalog row goes while the provider is refusing, and the file follows once-// it stops.func Test_DeleteBackup_WhenProviderRefusesThenRecovers_RowGoesFirstAndFileFollows(t *testing.T) {fixture := CreateBackupTestFixture(t, "Flaky Cleanup Workspace")backup := SeedInProgressTestBackup(t, fixture.Database.ID, fixture.Storage.ID)diff --git a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.goindex 5644748c..8dd0b4e6 100644--- a/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go+++ b/backend/internal/features/backups/backups/core/physical/repositories/wal_segment_repository.go@@ -199,11 +199,6 @@ func (r *PhysicalWalSegmentRepository) FindByChainKey(return &segment, nil}-// MarkUploaded flips file_name from NULL to the durable object key, guarded by-// file_name IS NULL so a DeleteFull cascade that removed the claim mid-upload-// makes this a no-op. updated=true means the upload is durably committed;-// updated=false means the NULL claim no longer exists (cascade caught it) and the-// caller must DeleteFile the now-orphaned storage object.type WalSegmentUpload struct {SegmentID uuid.UUIDFileName string@@ -212,10 +207,9 @@ type WalSegmentUpload struct {EncryptionIV *string}-func (r *PhysicalWalSegmentRepository) MarkUploaded(upload WalSegmentUpload) (updated bool, err error) {- return r.MarkUploadedInTransaction(storage.GetDb(), upload)-}-+// The file_name IS NULL guard makes this a no-op when a DeleteFull cascade removed+// the claim mid-upload. An updated=false result therefore means nothing published+// the object, and the caller hands it back through the file store.func (r *PhysicalWalSegmentRepository) MarkUploadedInTransaction(tx *gorm.DB,upload WalSegmentUpload,diff --git a/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go b/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.goindex ee4d5f47..bc358bd8 100644--- a/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go+++ b/backend/internal/features/backups/backups/core/physical/service/database_removal_listener.go@@ -26,29 +26,6 @@ func (s *PhysicalBackupService) OnBeforeDatabaseRemove(ctx context.Context, data})}-// The physical rows are the only record of the file names in a storage, so the-// storage cannot go while they exist.-func (s *PhysicalBackupService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {- var total int64-- for _, model := range []any{- &physical_models.PhysicalFullBackup{},- &physical_models.PhysicalIncrementalBackup{},- &physical_models.PhysicalWalSegment{},- &physical_models.PhysicalWalHistoryFile{},- } {- var count int64-- if err := db.GetDb().Model(model).Where("storage_id = ?", storageID).Count(&count).Error; err != nil {- return 0, fmt.Errorf("count physical rows of a storage: %w", err)- }-- total += count- }-- return total, nil-}-func (s *PhysicalBackupService) collectDatabaseFileReferences(tx *gorm.DB,databaseID uuid.UUID,diff --git a/backend/internal/features/backups/backups/core/physical/service/storage_backup_counter.go b/backend/internal/features/backups/backups/core/physical/service/storage_backup_counter.gonew file mode 100644index 00000000..6e3bcc5d--- /dev/null+++ b/backend/internal/features/backups/backups/core/physical/service/storage_backup_counter.go@@ -0,0 +1,33 @@+package physical_service++import (+ "fmt"++ "github.com/google/uuid"++ physical_models "databasus-backend/internal/features/backups/backups/core/physical/models"+ db "databasus-backend/internal/storage"+)++// The physical rows are the only record of the file names in a storage, so the+// storage cannot go while they exist.+func (s *PhysicalBackupService) GetStorageBackupReferenceCount(storageID uuid.UUID) (int64, error) {+ var total int64++ for _, model := range []any{+ &physical_models.PhysicalFullBackup{},+ &physical_models.PhysicalIncrementalBackup{},+ &physical_models.PhysicalWalSegment{},+ &physical_models.PhysicalWalHistoryFile{},+ } {+ var count int64++ if err := db.GetDb().Model(model).Where("storage_id = ?", storageID).Count(&count).Error; err != nil {+ return 0, fmt.Errorf("count physical rows of a storage: %w", err)+ }++ total += count+ }++ return total, nil+}diff --git a/backend/internal/features/backups/backups/services/logical_service.go b/backend/internal/features/backups/backups/services/logical_service.goindex a82467e8..b5f55131 100644--- a/backend/internal/features/backups/backups/services/logical_service.go+++ b/backend/internal/features/backups/backups/services/logical_service.go@@ -83,7 +83,7 @@ func (s *LogicalBackupService) OnBeforeBackupsStorageChange(ctx context.Context,// The logical_backups rows are the only record of the file names in a storage, so// the storage cannot go while they exist.-func (s *LogicalBackupService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {+func (s *LogicalBackupService) GetStorageBackupReferenceCount(storageID uuid.UUID) (int64, error) {return s.backupRepository.CountByStorageID(storageID)}diff --git a/backend/internal/features/storages/controller_test.go b/backend/internal/features/storages/controller_test.goindex 20b5281c..db5be76a 100644--- a/backend/internal/features/storages/controller_test.go+++ b/backend/internal/features/storages/controller_test.go@@ -33,9 +33,9 @@ import (test_utils "databasus-backend/internal/util/testing")-type mockStorageReferenceReporter struct{}+type mockStorageDatabaseCounter struct{}-func (m *mockStorageReferenceReporter) GetStorageAttachedDatabasesIDs(+func (m *mockStorageDatabaseCounter) GetStorageAttachedDatabasesIDs(storageID uuid.UUID,) ([]uuid.UUID, error) {return []uuid.UUID{}, nil@@ -420,7 +420,7 @@ func Test_WorkspaceRolePermissions(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {router := createRouter()- SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})owner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)workspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Test Workspace", owner, router)@@ -1136,7 +1136,7 @@ func Test_TransferStorage_PermissionsEnforced(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {router := createRouter()- SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})sourceOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)targetOwner := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)@@ -1228,7 +1228,7 @@ func Test_TransferStorage_PermissionsEnforced(t *testing.T) {func Test_TransferStorageNotManagableWorkspace_TransferFailed(t *testing.T) {router := createRouter()- SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})userA := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)userB := users_testing.CreateTestUser(t.Context(), users_enums.UserRoleMember)@@ -1287,7 +1287,7 @@ func createRouter() *gin.Engine {audit_logs.SetupDependencies()SetupDependencies()- SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})+ SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})return router}@@ -1359,7 +1359,9 @@ func Test_DeleteStorage_WhenCleanupIsPending_DrainsItBeforeTheCredentialsGo(t *tworkspace := workspaces_testing.CreateTestWorkspace(t.Context(), "Draining Storage Workspace", owner, router)storage := CreateTestStorage(workspace.ID)- SetStorageDatabaseCountersForTest(&mockStorageReferenceReporter{})+ t.Cleanup(func() { workspaces_testing.RemoveTestWorkspace(t.Context(), workspace, router) })++ SetStorageDatabaseCountersForTest(&mockStorageDatabaseCounter{})fileName := "drained-" + storage.ID.String()@@ -1389,6 +1391,6 @@ type countingBackupCounter struct {backupReferences int64}-func (c *countingBackupCounter) GetStorageBackupReferences(uuid.UUID) (int64, error) {+func (c *countingBackupCounter) GetStorageBackupReferenceCount(uuid.UUID) (int64, error) {return c.backupReferences, nil}diff --git a/backend/internal/features/storages/interfaces.go b/backend/internal/features/storages/interfaces.goindex 09d97bd3..9a6d788a 100644--- a/backend/internal/features/storages/interfaces.go+++ b/backend/internal/features/storages/interfaces.go@@ -53,5 +53,5 @@ type StorageDatabaseCounter interface {// storage while they exist would make the objects unreachable. Each backup feature// counts its own rows, because the tables belong to it.type StorageBackupCounter interface {- GetStorageBackupReferences(storageID uuid.UUID) (int64, error)+ GetStorageBackupReferenceCount(storageID uuid.UUID) (int64, error)}diff --git a/backend/internal/features/storages/service.go b/backend/internal/features/storages/service.goindex a316955c..2fae7fa6 100644--- a/backend/internal/features/storages/service.go+++ b/backend/internal/features/storages/service.go@@ -31,11 +31,11 @@ func (s *StorageService) AddStorageBackupCounter(counter StorageBackupCounter) {s.storageBackupCounters = append(s.storageBackupCounters, counter)}-func (s *StorageService) GetStorageBackupReferences(storageID uuid.UUID) (int64, error) {+func (s *StorageService) GetStorageBackupReferenceCount(storageID uuid.UUID) (int64, error) {var total int64for _, counter := range s.storageBackupCounters {- count, err := counter.GetStorageBackupReferences(storageID)+ count, err := counter.GetStorageBackupReferenceCount(storageID)if err != nil {return 0, err}@@ -221,7 +221,7 @@ func (s *StorageService) DeleteStorage(return ErrStorageHasAttachedDatabases}- backupReferences, err := s.GetStorageBackupReferences(storage.ID)+ backupReferences, err := s.GetStorageBackupReferenceCount(storage.ID)if err != nil {return err}