PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FEATURE (limits): Add max backup size limit and total backups size limit
Rostislav Dugin594a3294c65e
diff --git a/backend/cmd/main.go b/backend/cmd/main.goindex bec1b388..d56730eb 100644--- a/backend/cmd/main.go+++ b/backend/cmd/main.go@@ -272,6 +272,10 @@ func runBackgroundTasks(log *slog.Logger) {backuping.GetBackupsScheduler().Run(ctx)})+ go runWithPanicLogging(log, "backup cleaner background service", func() {+ backuping.GetBackupCleaner().Run(ctx)+ })+go runWithPanicLogging(log, "restore background service", func() {restoring.GetRestoresScheduler().Run(ctx)})diff --git a/backend/internal/features/backups/backups/backuping/backuper.go b/backend/internal/features/backups/backups/backuping/backuper.goindex 5a17e164..2495c4c8 100644--- a/backend/internal/features/backups/backups/backuping/backuper.go+++ b/backend/internal/features/backups/backups/backuping/backuper.go@@ -157,21 +157,41 @@ func (n *BackuperNode) MakeBackup(backupID uuid.UUID, isCallNotifier bool) {start := time.Now().UTC()+ ctx, cancel := context.WithCancel(context.Background())+ n.backupCancelManager.RegisterTask(backup.ID, cancel)+ defer n.backupCancelManager.UnregisterTask(backup.ID)+backupProgressListener := func(completedMBs float64,) {backup.BackupSizeMb = completedMBsbackup.BackupDurationMs = time.Since(start).Milliseconds()+ // Check size limit (0 = unlimited)+ if backupConfig.MaxBackupSizeMB > 0 &&+ completedMBs > float64(backupConfig.MaxBackupSizeMB) {+ errMsg := fmt.Sprintf(+ "backup size (%.2f MB) exceeded maximum allowed size (%d MB)",+ completedMBs,+ backupConfig.MaxBackupSizeMB,+ )++ backup.Status = backups_core.BackupStatusFailed+ backup.IsSkipRetry = true+ backup.FailMessage = &errMsg+ if err := n.backupRepository.Save(backup); err != nil {+ n.logger.Error("Failed to save backup with size exceeded error", "error", err)+ }+ cancel() // Cancel the backup context++ return+ }+if err := n.backupRepository.Save(backup); err != nil {n.logger.Error("Failed to update backup progress", "error", err)}}- ctx, cancel := context.WithCancel(context.Background())- n.backupCancelManager.RegisterTask(backup.ID, cancel)- defer n.backupCancelManager.UnregisterTask(backup.ID)-backupMetadata, err := n.createBackupUseCase.Execute(ctx,backup.ID,@@ -181,6 +201,29 @@ func (n *BackuperNode) MakeBackup(backupID uuid.UUID, isCallNotifier bool) {backupProgressListener,)if err != nil {+ // Check if backup was already marked as failed by progress listener (e.g., size limit exceeded)+ // If so, skip error handling to avoid overwriting the status+ currentBackup, fetchErr := n.backupRepository.FindByID(backup.ID)+ if fetchErr == nil && currentBackup.Status == backups_core.BackupStatusFailed {+ n.logger.Warn(+ "Backup already marked as failed by progress listener, skipping error handling",+ "backupId",+ backup.ID,+ "failMessage",+ *currentBackup.FailMessage,+ )++ // Still call notification for size limit failures+ n.SendBackupNotification(+ backupConfig,+ currentBackup,+ backups_config.NotificationBackupFailed,+ currentBackup.FailMessage,+ )++ return+ }+errMsg := err.Error()// Log detailed error information for debuggingdiff --git a/backend/internal/features/backups/backups/backuping/backuper_test.go b/backend/internal/features/backups/backups/backuping/backuper_test.goindex 12751202..45b9ed6f 100644--- a/backend/internal/features/backups/backups/backuping/backuper_test.go+++ b/backend/internal/features/backups/backups/backuping/backuper_test.go@@ -1,13 +1,10 @@package backupingimport (- "context"- "errors""strings""testing""time"- common "databasus-backend/internal/features/backups/backups/common"backups_core "databasus-backend/internal/features/backups/backups/core"backups_config "databasus-backend/internal/features/backups/config""databasus-backend/internal/features/databases"@@ -18,7 +15,6 @@ import (workspaces_testing "databasus-backend/internal/features/workspaces/testing"cache_utils "databasus-backend/internal/util/cache"- "github.com/google/uuid""github.com/stretchr/testify/assert""github.com/stretchr/testify/mock")@@ -158,35 +154,120 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {})}-type CreateFailedBackupUsecase struct {-}+func Test_BackupSizeLimits(t *testing.T) {+ cache_utils.ClearAllCache()+ user := users_testing.CreateTestUser(users_enums.UserRoleAdmin)+ router := CreateTestRouter()+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)-func (uc *CreateFailedBackupUsecase) Execute(- ctx context.Context,- backupID uuid.UUID,- backupConfig *backups_config.BackupConfig,- database *databases.Database,- storage *storages.Storage,- backupProgressListener func(completedMBs float64),-) (*common.BackupMetadata, error) {- backupProgressListener(10)- return nil, errors.New("backup failed")-}+ defer func() {+ // cleanup backups first+ backups, _ := backupRepository.FindByDatabaseID(database.ID)+ for _, backup := range backups {+ backupRepository.DeleteByID(backup.ID)+ }++ databases.RemoveTestDatabase(database)+ time.Sleep(50 * time.Millisecond) // Wait for cascading deletes+ notifiers.RemoveTestNotifier(notifier)+ storages.RemoveTestStorage(storage.ID)+ workspaces_testing.RemoveTestWorkspace(workspace, router)+ }()++ t.Run("UnlimitedSize_MaxBackupSizeMBIsZero_BackupCompletes", func(t *testing.T) {+ // Enable backups with unlimited size (0)+ backupConfig := backups_config.EnableBackupsForTestDatabase(database.ID, storage)+ backupConfig.MaxBackupSizeMB = 0 // unlimited+ backupConfig, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ backuperNode := CreateTestBackuperNode()+ backuperNode.createBackupUseCase = &CreateLargeBackupUsecase{}++ // Create a backup record+ backup := &backups_core.Backup{+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusInProgress,+ CreatedAt: time.Now().UTC(),+ }+ err = backupRepository.Save(backup)+ assert.NoError(t, err)++ backuperNode.MakeBackup(backup.ID, false)-type CreateSuccessBackupUsecase struct{}--func (uc *CreateSuccessBackupUsecase) Execute(- ctx context.Context,- backupID uuid.UUID,- backupConfig *backups_config.BackupConfig,- database *databases.Database,- storage *storages.Storage,- backupProgressListener func(completedMBs float64),-) (*common.BackupMetadata, error) {- backupProgressListener(10)- return &common.BackupMetadata{- EncryptionSalt: nil,- EncryptionIV: nil,- Encryption: backups_config.BackupEncryptionNone,- }, nil+ // Verify backup completed successfully even with large size+ updatedBackup, err := backupRepository.FindByID(backup.ID)+ assert.NoError(t, err)+ assert.Equal(t, backups_core.BackupStatusCompleted, updatedBackup.Status)+ assert.Equal(t, float64(10000), updatedBackup.BackupSizeMb)+ assert.Nil(t, updatedBackup.FailMessage)+ })++ t.Run("SizeExceeded_BackupFailedWithIsSkipRetry", func(t *testing.T) {+ // Enable backups with 5 MB limit+ backupConfig := backups_config.EnableBackupsForTestDatabase(database.ID, storage)+ backupConfig.MaxBackupSizeMB = 5+ backupConfig, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ backuperNode := CreateTestBackuperNode()+ backuperNode.createBackupUseCase = &CreateProgressiveBackupUsecase{}++ // Create a backup record+ backup := &backups_core.Backup{+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusInProgress,+ CreatedAt: time.Now().UTC(),+ }+ err = backupRepository.Save(backup)+ assert.NoError(t, err)++ backuperNode.MakeBackup(backup.ID, false)++ // Verify backup was marked as failed with IsSkipRetry=true+ updatedBackup, err := backupRepository.FindByID(backup.ID)+ assert.NoError(t, err)+ assert.Equal(t, backups_core.BackupStatusFailed, updatedBackup.Status)+ assert.True(t, updatedBackup.IsSkipRetry)+ assert.NotNil(t, updatedBackup.FailMessage)+ assert.Contains(t, *updatedBackup.FailMessage, "exceeded maximum allowed size")+ assert.Contains(t, *updatedBackup.FailMessage, "10.00 MB")+ assert.Contains(t, *updatedBackup.FailMessage, "5 MB")+ assert.Greater(t, updatedBackup.BackupSizeMb, float64(5))+ })++ t.Run("SizeWithinLimit_BackupCompletes", func(t *testing.T) {+ // Enable backups with 100 MB limit+ backupConfig := backups_config.EnableBackupsForTestDatabase(database.ID, storage)+ backupConfig.MaxBackupSizeMB = 100+ backupConfig, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ backuperNode := CreateTestBackuperNode()+ backuperNode.createBackupUseCase = &CreateMediumBackupUsecase{}++ // Create a backup record+ backup := &backups_core.Backup{+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusInProgress,+ CreatedAt: time.Now().UTC(),+ }+ err = backupRepository.Save(backup)+ assert.NoError(t, err)++ backuperNode.MakeBackup(backup.ID, false)++ // Verify backup completed successfully+ updatedBackup, err := backupRepository.FindByID(backup.ID)+ assert.NoError(t, err)+ assert.Equal(t, backups_core.BackupStatusCompleted, updatedBackup.Status)+ assert.Equal(t, float64(50), updatedBackup.BackupSizeMb)+ assert.Nil(t, updatedBackup.FailMessage)+ })}diff --git a/backend/internal/features/backups/backups/backuping/cleaner.go b/backend/internal/features/backups/backups/backuping/cleaner.gonew file mode 100644index 00000000..7f0de96f--- /dev/null+++ b/backend/internal/features/backups/backups/backuping/cleaner.go@@ -0,0 +1,242 @@+package backuping++import (+ "context"+ "fmt"+ "log/slog"+ "sync"+ "sync/atomic"+ "time"++ "github.com/google/uuid"++ backups_core "databasus-backend/internal/features/backups/backups/core"+ backups_config "databasus-backend/internal/features/backups/config"+ "databasus-backend/internal/features/storages"+ util_encryption "databasus-backend/internal/util/encryption"+ "databasus-backend/internal/util/period"+)++const (+ cleanerTickerInterval = 1 * time.Minute+)++type BackupCleaner struct {+ backupRepository *backups_core.BackupRepository+ storageService *storages.StorageService+ backupConfigService *backups_config.BackupConfigService+ fieldEncryptor util_encryption.FieldEncryptor+ logger *slog.Logger+ backupRemoveListeners []backups_core.BackupRemoveListener++ runOnce sync.Once+ hasRun atomic.Bool+}++func (c *BackupCleaner) Run(ctx context.Context) {+ wasAlreadyRun := c.hasRun.Load()++ c.runOnce.Do(func() {+ c.hasRun.Store(true)++ if ctx.Err() != nil {+ return+ }++ ticker := time.NewTicker(cleanerTickerInterval)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ if err := c.cleanOldBackups(); err != nil {+ c.logger.Error("Failed to clean old backups", "error", err)+ }++ if err := c.cleanExceededBackups(); err != nil {+ c.logger.Error("Failed to clean exceeded backups", "error", err)+ }+ }+ }+ })++ if wasAlreadyRun {+ panic(fmt.Sprintf("%T.Run() called multiple times", c))+ }+}++func (c *BackupCleaner) DeleteBackup(backup *backups_core.Backup) error {+ for _, listener := range c.backupRemoveListeners {+ if err := listener.OnBeforeBackupRemove(backup); err != nil {+ return err+ }+ }++ storage, err := c.storageService.GetStorageByID(backup.StorageID)+ if err != nil {+ return err+ }++ err = storage.DeleteFile(c.fieldEncryptor, backup.ID)+ if err != nil {+ // we do not return error here, because sometimes clean up performed+ // before unavailable storage removal or change - therefore we should+ // proceed even in case of error. It's possible that some S3 or+ // storage is not available yet, it should not block us+ c.logger.Error("Failed to delete backup file", "error", err)+ }++ return c.backupRepository.DeleteByID(backup.ID)+}++func (c *BackupCleaner) AddBackupRemoveListener(listener backups_core.BackupRemoveListener) {+ c.backupRemoveListeners = append(c.backupRemoveListeners, listener)+}++func (c *BackupCleaner) cleanOldBackups() error {+ enabledBackupConfigs, err := c.backupConfigService.GetBackupConfigsWithEnabledBackups()+ if err != nil {+ return err+ }++ for _, backupConfig := range enabledBackupConfigs {+ backupStorePeriod := backupConfig.StorePeriod++ if backupStorePeriod == period.PeriodForever {+ continue+ }++ storeDuration := backupStorePeriod.ToDuration()+ dateBeforeBackupsShouldBeDeleted := time.Now().UTC().Add(-storeDuration)++ oldBackups, err := c.backupRepository.FindBackupsBeforeDate(+ backupConfig.DatabaseID,+ dateBeforeBackupsShouldBeDeleted,+ )+ if err != nil {+ c.logger.Error(+ "Failed to find old backups for database",+ "databaseId",+ backupConfig.DatabaseID,+ "error",+ err,+ )+ continue+ }++ for _, backup := range oldBackups {+ if err := c.DeleteBackup(backup); err != nil {+ c.logger.Error("Failed to delete old backup", "backupId", backup.ID, "error", err)+ continue+ }++ c.logger.Info(+ "Deleted old backup",+ "backupId",+ backup.ID,+ "databaseId",+ backupConfig.DatabaseID,+ )+ }+ }++ return nil+}++func (c *BackupCleaner) cleanExceededBackups() error {+ enabledBackupConfigs, err := c.backupConfigService.GetBackupConfigsWithEnabledBackups()+ if err != nil {+ return err+ }++ for _, backupConfig := range enabledBackupConfigs {+ if backupConfig.MaxBackupsTotalSizeMB <= 0 {+ continue+ }++ if err := c.cleanExceededBackupsForDatabase(+ backupConfig.DatabaseID,+ backupConfig.MaxBackupsTotalSizeMB,+ ); err != nil {+ c.logger.Error(+ "Failed to clean exceeded backups for database",+ "databaseId",+ backupConfig.DatabaseID,+ "error",+ err,+ )+ continue+ }+ }++ return nil+}++func (c *BackupCleaner) cleanExceededBackupsForDatabase(+ databaseID uuid.UUID,+ limitperDbMB int64,+) error {+ for {+ backupsTotalSizeMB, err := c.backupRepository.GetTotalSizeByDatabase(databaseID)+ if err != nil {+ return err+ }++ if backupsTotalSizeMB <= float64(limitperDbMB) {+ break+ }++ oldestBackups, err := c.backupRepository.FindOldestByDatabaseExcludingInProgress(+ databaseID,+ 1,+ )+ if err != nil {+ return err+ }++ if len(oldestBackups) == 0 {+ c.logger.Warn(+ "No backups to delete but still over limit",+ "databaseId",+ databaseID,+ "totalSizeMB",+ backupsTotalSizeMB,+ "limitMB",+ limitperDbMB,+ )+ break+ }++ backup := oldestBackups[0]+ if err := c.DeleteBackup(backup); err != nil {+ c.logger.Error(+ "Failed to delete exceeded backup",+ "backupId",+ backup.ID,+ "databaseId",+ databaseID,+ "error",+ err,+ )+ return err+ }++ c.logger.Info(+ "Deleted exceeded backup",+ "backupId",+ backup.ID,+ "databaseId",+ databaseID,+ "backupSizeMB",+ backup.BackupSizeMb,+ "totalSizeMB",+ backupsTotalSizeMB,+ "limitMB",+ limitperDbMB,+ )+ }++ return nil+}diff --git a/backend/internal/features/backups/backups/backuping/cleaner_test.go b/backend/internal/features/backups/backups/backuping/cleaner_test.gonew file mode 100644index 00000000..942f6c8c--- /dev/null+++ b/backend/internal/features/backups/backups/backuping/cleaner_test.go@@ -0,0 +1,491 @@+package backuping++import (+ "testing"+ "time"++ backups_core "databasus-backend/internal/features/backups/backups/core"+ backups_config "databasus-backend/internal/features/backups/config"+ "databasus-backend/internal/features/databases"+ "databasus-backend/internal/features/intervals"+ "databasus-backend/internal/features/notifiers"+ "databasus-backend/internal/features/storages"+ users_enums "databasus-backend/internal/features/users/enums"+ users_testing "databasus-backend/internal/features/users/testing"+ workspaces_testing "databasus-backend/internal/features/workspaces/testing"+ "databasus-backend/internal/storage"+ "databasus-backend/internal/util/period"++ "github.com/google/uuid"+ "github.com/stretchr/testify/assert"+)++func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {+ router := CreateTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)++ // Create backup interval+ interval := createTestInterval()++ backupConfig := &backups_config.BackupConfig{+ DatabaseID: database.ID,+ IsBackupsEnabled: true,+ StorePeriod: period.PeriodWeek,+ StorageID: &storage.ID,+ BackupIntervalID: interval.ID,+ BackupInterval: interval,+ }+ _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // Create backups with different ages+ now := time.Now().UTC()+ oldBackup1 := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 10,+ CreatedAt: now.Add(-10 * 24 * time.Hour), // 10 days old+ }+ oldBackup2 := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 10,+ CreatedAt: now.Add(-8 * 24 * time.Hour), // 8 days old+ }+ recentBackup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 10,+ CreatedAt: now.Add(-3 * 24 * time.Hour), // 3 days old+ }++ err = backupRepository.Save(oldBackup1)+ assert.NoError(t, err)+ err = backupRepository.Save(oldBackup2)+ assert.NoError(t, err)+ err = backupRepository.Save(recentBackup)+ assert.NoError(t, err)++ // Run cleanup+ cleaner := GetBackupCleaner()+ err = cleaner.cleanOldBackups()+ assert.NoError(t, err)++ // Verify old backups deleted, recent backup remains+ remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Equal(t, 1, len(remainingBackups))+ assert.Equal(t, recentBackup.ID, remainingBackups[0].ID)+}++func Test_CleanOldBackups_SkipsDatabaseWithForeverStorePeriod(t *testing.T) {+ router := CreateTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)++ // Create backup interval+ interval := createTestInterval()++ backupConfig := &backups_config.BackupConfig{+ DatabaseID: database.ID,+ IsBackupsEnabled: true,+ StorePeriod: period.PeriodForever,+ StorageID: &storage.ID,+ BackupIntervalID: interval.ID,+ BackupInterval: interval,+ }+ _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // Create very old backup+ oldBackup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 10,+ CreatedAt: time.Now().UTC().Add(-365 * 24 * time.Hour), // 1 year old+ }+ err = backupRepository.Save(oldBackup)+ assert.NoError(t, err)++ // Run cleanup+ cleaner := GetBackupCleaner()+ err = cleaner.cleanOldBackups()+ assert.NoError(t, err)++ // Verify backup still exists+ remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Equal(t, 1, len(remainingBackups))+ assert.Equal(t, oldBackup.ID, remainingBackups[0].ID)+}++func Test_CleanExceededBackups_WhenUnderLimit_NoBackupsDeleted(t *testing.T) {+ router := CreateTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)++ // Create backup interval+ interval := createTestInterval()++ backupConfig := &backups_config.BackupConfig{+ DatabaseID: database.ID,+ IsBackupsEnabled: true,+ StorePeriod: period.PeriodForever,+ StorageID: &storage.ID,+ MaxBackupsTotalSizeMB: 100, // 100 MB limit+ BackupIntervalID: interval.ID,+ BackupInterval: interval,+ }+ _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // Create 3 backups totaling 50MB (under limit)+ for i := 0; i < 3; i++ {+ backup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 16.67,+ CreatedAt: time.Now().UTC().Add(-time.Duration(i) * time.Hour),+ }+ err = backupRepository.Save(backup)+ assert.NoError(t, err)+ }++ // Run cleanup+ cleaner := GetBackupCleaner()+ err = cleaner.cleanExceededBackups()+ assert.NoError(t, err)++ // Verify all backups remain+ remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Equal(t, 3, len(remainingBackups))+}++func Test_CleanExceededBackups_WhenOverLimit_DeletesOldestBackups(t *testing.T) {+ router := CreateTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)++ // Create backup interval+ interval := createTestInterval()++ backupConfig := &backups_config.BackupConfig{+ DatabaseID: database.ID,+ IsBackupsEnabled: true,+ StorePeriod: period.PeriodForever,+ StorageID: &storage.ID,+ MaxBackupsTotalSizeMB: 30, // 30 MB limit+ BackupIntervalID: interval.ID,+ BackupInterval: interval,+ }+ _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // Create 5 backups of 10MB each (total 50MB, over 30MB limit)+ now := time.Now().UTC()+ var backupIDs []uuid.UUID+ for i := 0; i < 5; i++ {+ backup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 10,+ CreatedAt: now.Add(-time.Duration(4-i) * time.Hour), // Oldest first+ }+ err = backupRepository.Save(backup)+ assert.NoError(t, err)+ backupIDs = append(backupIDs, backup.ID)+ }++ // Run cleanup+ cleaner := GetBackupCleaner()+ err = cleaner.cleanExceededBackups()+ assert.NoError(t, err)++ // Verify 2 oldest backups deleted, 3 newest remain+ remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Equal(t, 3, len(remainingBackups))++ // Check that the newest 3 backups remain+ remainingIDs := make(map[uuid.UUID]bool)+ for _, backup := range remainingBackups {+ remainingIDs[backup.ID] = true+ }+ assert.False(t, remainingIDs[backupIDs[0]]) // Oldest deleted+ assert.False(t, remainingIDs[backupIDs[1]]) // 2nd oldest deleted+ assert.True(t, remainingIDs[backupIDs[2]]) // 3rd remains+ assert.True(t, remainingIDs[backupIDs[3]]) // 4th remains+ assert.True(t, remainingIDs[backupIDs[4]]) // Newest remains+}++func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {+ router := CreateTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)++ // Create backup interval+ interval := createTestInterval()++ backupConfig := &backups_config.BackupConfig{+ DatabaseID: database.ID,+ IsBackupsEnabled: true,+ StorePeriod: period.PeriodForever,+ StorageID: &storage.ID,+ MaxBackupsTotalSizeMB: 50, // 50 MB limit+ BackupIntervalID: interval.ID,+ BackupInterval: interval,+ }+ _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ now := time.Now().UTC()++ // Create 3 completed backups of 30MB each+ completedBackups := make([]*backups_core.Backup, 3)+ for i := 0; i < 3; i++ {+ backup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 30,+ CreatedAt: now.Add(-time.Duration(3-i) * time.Hour),+ }+ err = backupRepository.Save(backup)+ assert.NoError(t, err)+ completedBackups[i] = backup+ }++ // Create 1 in-progress backup (should be excluded from size calculation and deletion)+ inProgressBackup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusInProgress,+ BackupSizeMb: 10,+ CreatedAt: now,+ }+ err = backupRepository.Save(inProgressBackup)+ assert.NoError(t, err)++ // Run cleanup+ cleaner := GetBackupCleaner()+ err = cleaner.cleanExceededBackups()+ assert.NoError(t, err)++ // Verify: only completed backups deleted, in-progress remains+ remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)++ // Should have in-progress + 1 completed (total 40MB completed + 10MB in-progress)+ assert.GreaterOrEqual(t, len(remainingBackups), 2)++ // Verify in-progress backup still exists+ var inProgressFound bool+ for _, backup := range remainingBackups {+ if backup.ID == inProgressBackup.ID {+ inProgressFound = true+ assert.Equal(t, backups_core.BackupStatusInProgress, backup.Status)+ }+ }+ assert.True(t, inProgressFound, "In-progress backup should not be deleted")+}++func Test_CleanExceededBackups_WithZeroLimit_SkipsDatabase(t *testing.T) {+ router := CreateTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)++ // Create backup interval+ interval := createTestInterval()++ backupConfig := &backups_config.BackupConfig{+ DatabaseID: database.ID,+ IsBackupsEnabled: true,+ StorePeriod: period.PeriodForever,+ StorageID: &storage.ID,+ MaxBackupsTotalSizeMB: 0, // No size limit+ BackupIntervalID: interval.ID,+ BackupInterval: interval,+ }+ _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // Create large backups+ for i := 0; i < 10; i++ {+ backup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 100,+ CreatedAt: time.Now().UTC().Add(-time.Duration(i) * time.Hour),+ }+ err = backupRepository.Save(backup)+ assert.NoError(t, err)+ }++ // Run cleanup+ cleaner := GetBackupCleaner()+ err = cleaner.cleanExceededBackups()+ assert.NoError(t, err)++ // Verify all backups remain+ remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Equal(t, 10, len(remainingBackups))+}++func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {+ router := CreateTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)++ // Create completed backups+ completedBackup1 := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 10.5,+ CreatedAt: time.Now().UTC(),+ }+ completedBackup2 := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 20.3,+ CreatedAt: time.Now().UTC(),+ }+ // Create failed backup (should be included)+ failedBackup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusFailed,+ BackupSizeMb: 5.2,+ CreatedAt: time.Now().UTC(),+ }+ // Create in-progress backup (should be excluded)+ inProgressBackup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: storage.ID,+ Status: backups_core.BackupStatusInProgress,+ BackupSizeMb: 100,+ CreatedAt: time.Now().UTC(),+ }++ err := backupRepository.Save(completedBackup1)+ assert.NoError(t, err)+ err = backupRepository.Save(completedBackup2)+ assert.NoError(t, err)+ err = backupRepository.Save(failedBackup)+ assert.NoError(t, err)+ err = backupRepository.Save(inProgressBackup)+ assert.NoError(t, err)++ // Calculate total size+ totalSize, err := backupRepository.GetTotalSizeByDatabase(database.ID)+ assert.NoError(t, err)++ // Should be 10.5 + 20.3 + 5.2 = 36.0 (excluding in-progress 100)+ assert.InDelta(t, 36.0, totalSize, 0.1)+}++// Mock listener for testing+type mockBackupRemoveListener struct {+ onBeforeBackupRemove func(*backups_core.Backup) error+}++func (m *mockBackupRemoveListener) OnBeforeBackupRemove(backup *backups_core.Backup) error {+ if m.onBeforeBackupRemove != nil {+ return m.onBeforeBackupRemove(backup)+ }++ return nil+}++// Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase verifies resilience+// when storage becomes unavailable. Even if storage.DeleteFile fails (e.g., storage is offline,+// credentials changed, or storage was deleted), the backup record should still be removed from+// the database. This prevents orphaned backup records when storage is no longer accessible.+func Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase(t *testing.T) {+ router := CreateTestRouter()+ owner := users_testing.CreateTestUser(users_enums.UserRoleMember)+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)+ testStorage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier)++ backup := &backups_core.Backup{+ ID: uuid.New(),+ DatabaseID: database.ID,+ StorageID: testStorage.ID,+ Status: backups_core.BackupStatusCompleted,+ BackupSizeMb: 10,+ CreatedAt: time.Now().UTC(),+ }+ err := backupRepository.Save(backup)+ assert.NoError(t, err)++ cleaner := GetBackupCleaner()++ err = cleaner.DeleteBackup(backup)+ assert.NoError(t, err, "DeleteBackup should succeed even when storage file doesn't exist")++ deletedBackup, err := backupRepository.FindByID(backup.ID)+ assert.Error(t, err, "Backup should not exist in database")+ assert.Nil(t, deletedBackup)+}++func createTestInterval() *intervals.Interval {+ timeOfDay := "04:00"+ interval := &intervals.Interval{+ Interval: intervals.IntervalDaily,+ TimeOfDay: &timeOfDay,+ }++ err := storage.GetDb().Create(interval).Error+ if err != nil {+ panic(err)+ }++ return interval+}diff --git a/backend/internal/features/backups/backups/backuping/di.go b/backend/internal/features/backups/backups/backuping/di.goindex 86b3893f..b9235def 100644--- a/backend/internal/features/backups/backups/backuping/di.go+++ b/backend/internal/features/backups/backups/backuping/di.go@@ -24,6 +24,17 @@ var backupRepository = &backups_core.BackupRepository{}var taskCancelManager = tasks_cancellation.GetTaskCancelManager()+var backupCleaner = &BackupCleaner{+ backupRepository: backupRepository,+ storageService: storages.GetStorageService(),+ backupConfigService: backups_config.GetBackupConfigService(),+ fieldEncryptor: encryption.GetFieldEncryptor(),+ logger: logger.GetLogger(),+ backupRemoveListeners: []backups_core.BackupRemoveListener{},+ runOnce: sync.Once{},+ hasRun: atomic.Bool{},+}+var backupNodesRegistry = &BackupNodesRegistry{client: cache_utils.GetValkeyClient(),logger: logger.GetLogger(),@@ -59,7 +70,6 @@ var backuperNode = &BackuperNode{var backupsScheduler = &BackupsScheduler{backupRepository: backupRepository,backupConfigService: backups_config.GetBackupConfigService(),- storageService: storages.GetStorageService(),taskCancelManager: taskCancelManager,backupNodesRegistry: backupNodesRegistry,lastBackupTime: time.Now().UTC(),@@ -81,3 +91,7 @@ func GetBackuperNode() *BackuperNode {func GetBackupNodesRegistry() *BackupNodesRegistry {return backupNodesRegistry}++func GetBackupCleaner() *BackupCleaner {+ return backupCleaner+}diff --git a/backend/internal/features/backups/backups/backuping/mocks.go b/backend/internal/features/backups/backups/backuping/mocks.goindex 5bd84a4b..7807f06e 100644--- a/backend/internal/features/backups/backups/backuping/mocks.go+++ b/backend/internal/features/backups/backups/backuping/mocks.go@@ -1,8 +1,16 @@package backupingimport (+ "context"+ "errors"++ common "databasus-backend/internal/features/backups/backups/common"+ backups_config "databasus-backend/internal/features/backups/config"+ "databasus-backend/internal/features/databases""databasus-backend/internal/features/notifiers"+ "databasus-backend/internal/features/storages"+ "github.com/google/uuid""github.com/stretchr/testify/mock")@@ -17,3 +25,113 @@ func (m *MockNotificationSender) SendNotification() {m.Called(notifier, title, message)}++type CreateFailedBackupUsecase struct{}++func (uc *CreateFailedBackupUsecase) Execute(+ ctx context.Context,+ backupID uuid.UUID,+ backupConfig *backups_config.BackupConfig,+ database *databases.Database,+ storage *storages.Storage,+ backupProgressListener func(completedMBs float64),+) (*common.BackupMetadata, error) {+ backupProgressListener(10)+ return nil, errors.New("backup failed")+}++type CreateSuccessBackupUsecase struct{}++func (uc *CreateSuccessBackupUsecase) Execute(+ ctx context.Context,+ backupID uuid.UUID,+ backupConfig *backups_config.BackupConfig,+ database *databases.Database,+ storage *storages.Storage,+ backupProgressListener func(completedMBs float64),+) (*common.BackupMetadata, error) {+ backupProgressListener(10)+ return &common.BackupMetadata{+ EncryptionSalt: nil,+ EncryptionIV: nil,+ Encryption: backups_config.BackupEncryptionNone,+ }, nil+}++// CreateLargeBackupUsecase simulates a large backup (10000 MB)+type CreateLargeBackupUsecase struct{}++func (uc *CreateLargeBackupUsecase) Execute(+ ctx context.Context,+ backupID uuid.UUID,+ backupConfig *backups_config.BackupConfig,+ database *databases.Database,+ storage *storages.Storage,+ backupProgressListener func(completedMBs float64),+) (*common.BackupMetadata, error) {+ backupProgressListener(10000)+ return &common.BackupMetadata{+ EncryptionSalt: nil,+ EncryptionIV: nil,+ Encryption: backups_config.BackupEncryptionNone,+ }, nil+}++// CreateProgressiveBackupUsecase simulates progressive size updates that exceed limit+type CreateProgressiveBackupUsecase struct{}++func (uc *CreateProgressiveBackupUsecase) Execute(+ ctx context.Context,+ backupID uuid.UUID,+ backupConfig *backups_config.BackupConfig,+ database *databases.Database,+ storage *storages.Storage,+ backupProgressListener func(completedMBs float64),+) (*common.BackupMetadata, error) {+ // Simulate progressive backup that grows beyond limit+ backupProgressListener(1)+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }++ backupProgressListener(3)+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }++ backupProgressListener(5)+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }++ backupProgressListener(10) // This exceeds the 5 MB limit+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }++ // Should not reach here due to cancellation+ return &common.BackupMetadata{+ EncryptionSalt: nil,+ EncryptionIV: nil,+ Encryption: backups_config.BackupEncryptionNone,+ }, nil+}++// CreateMediumBackupUsecase simulates a 50 MB backup+type CreateMediumBackupUsecase struct{}++func (uc *CreateMediumBackupUsecase) Execute(+ ctx context.Context,+ backupID uuid.UUID,+ backupConfig *backups_config.BackupConfig,+ database *databases.Database,+ storage *storages.Storage,+ backupProgressListener func(completedMBs float64),+) (*common.BackupMetadata, error) {+ backupProgressListener(50)+ return &common.BackupMetadata{+ EncryptionSalt: nil,+ EncryptionIV: nil,+ Encryption: backups_config.BackupEncryptionNone,+ }, nil+}diff --git a/backend/internal/features/backups/backups/backuping/scheduler.go b/backend/internal/features/backups/backups/backuping/scheduler.goindex deb9f789..f774fec9 100644--- a/backend/internal/features/backups/backups/backuping/scheduler.go+++ b/backend/internal/features/backups/backups/backuping/scheduler.go@@ -13,10 +13,7 @@ import ("databasus-backend/internal/config"backups_core "databasus-backend/internal/features/backups/backups/core"backups_config "databasus-backend/internal/features/backups/config"- "databasus-backend/internal/features/storages"task_cancellation "databasus-backend/internal/features/tasks/cancellation"- "databasus-backend/internal/util/encryption"- "databasus-backend/internal/util/period")const (@@ -28,7 +25,6 @@ const (type BackupsScheduler struct {backupRepository *backups_core.BackupRepositorybackupConfigService *backups_config.BackupConfigService- storageService *storages.StorageServicetaskCancelManager *task_cancellation.TaskCancelManagerbackupNodesRegistry *BackupNodesRegistry@@ -84,10 +80,6 @@ func (s *BackupsScheduler) Run(ctx context.Context) {case <-ctx.Done():returncase <-ticker.C:- if err := s.cleanOldBackups(); err != nil {- s.logger.Error("Failed to clean old backups", "error", err)- }-if err := s.checkDeadNodesAndFailBackups(); err != nil {s.logger.Error("Failed to check dead nodes and fail backups", "error", err)}@@ -293,6 +285,10 @@ func (s *BackupsScheduler) GetRemainedBackupTryCount(lastBackup *backups_core.Bareturn 0}+ if lastBackup.IsSkipRetry {+ return 0+ }+backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(lastBackup.DatabaseID)if err != nil {s.logger.Error("Failed to get backup config by database ID", "error", err)@@ -325,74 +321,6 @@ func (s *BackupsScheduler) GetRemainedBackupTryCount(lastBackup *backups_core.Bareturn maxFailedTriesCount - len(lastFailedBackups)}-func (s *BackupsScheduler) cleanOldBackups() error {- enabledBackupConfigs, err := s.backupConfigService.GetBackupConfigsWithEnabledBackups()- if err != nil {- return err- }-- for _, backupConfig := range enabledBackupConfigs {- backupStorePeriod := backupConfig.StorePeriod-- if backupStorePeriod == period.PeriodForever {- continue- }-- storeDuration := backupStorePeriod.ToDuration()- dateBeforeBackupsShouldBeDeleted := time.Now().UTC().Add(-storeDuration)-- oldBackups, err := s.backupRepository.FindBackupsBeforeDate(- backupConfig.DatabaseID,- dateBeforeBackupsShouldBeDeleted,- )- if err != nil {- s.logger.Error(- "Failed to find old backups for database",- "databaseId",- backupConfig.DatabaseID,- "error",- err,- )- continue- }-- for _, backup := range oldBackups {- storage, err := s.storageService.GetStorageByID(backup.StorageID)- if err != nil {- s.logger.Error(- "Failed to get storage by ID",- "storageId",- backup.StorageID,- "error",- err,- )- continue- }-- encryptor := encryption.GetFieldEncryptor()- err = storage.DeleteFile(encryptor, backup.ID)- if err != nil {- s.logger.Error("Failed to delete backup file", "backupId", backup.ID, "error", err)- }-- if err := s.backupRepository.DeleteByID(backup.ID); err != nil {- s.logger.Error("Failed to delete old backup", "backupId", backup.ID, "error", err)- continue- }-- s.logger.Info(- "Deleted old backup",- "backupId",- backup.ID,- "databaseId",- backupConfig.DatabaseID,- )- }- }-- return nil-}-func (s *BackupsScheduler) runPendingBackups() error {enabledBackupConfigs, err := s.backupConfigService.GetBackupConfigsWithEnabledBackups()if err != nil {diff --git a/backend/internal/features/backups/backups/backuping/scheduler_test.go b/backend/internal/features/backups/backups/backuping/scheduler_test.goindex b3e8b6f3..a0d6cc7d 100644--- a/backend/internal/features/backups/backups/backuping/scheduler_test.go+++ b/backend/internal/features/backups/backups/backuping/scheduler_test.go@@ -1101,3 +1101,84 @@ func Test_StartBackup_WhenBackupAlreadyInProgress_SkipsNewBackup(t *testing.T) {time.Sleep(200 * time.Millisecond)}++func Test_RunPendingBackups_WhenLastBackupFailedWithIsSkipRetry_SkipsBackupEvenWithRetriesEnabled(+ t *testing.T,+) {+ cache_utils.ClearAllCache()+ backuperNode := CreateTestBackuperNode()+ cancel := StartBackuperNodeForTest(t, backuperNode)+ defer StopBackuperNodeForTest(t, cancel, backuperNode)++ user := users_testing.CreateTestUser(users_enums.UserRoleAdmin)+ router := CreateTestRouter()+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)+ storage := storages.CreateTestStorage(workspace.ID)+ notifier := notifiers.CreateTestNotifier(workspace.ID)+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)++ defer func() {+ backups, _ := backupRepository.FindByDatabaseID(database.ID)+ for _, backup := range backups {+ backupRepository.DeleteByID(backup.ID)+ }++ databases.RemoveTestDatabase(database)+ time.Sleep(50 * time.Millisecond)+ storages.RemoveTestStorage(storage.ID)+ notifiers.RemoveTestNotifier(notifier)+ workspaces_testing.RemoveTestWorkspace(workspace, router)+ }()++ // Enable backups with retries enabled and high retry count+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)+ assert.NoError(t, err)++ timeOfDay := "04:00"+ backupConfig.BackupInterval = &intervals.Interval{+ Interval: intervals.IntervalDaily,+ TimeOfDay: &timeOfDay,+ }+ backupConfig.IsBackupsEnabled = true+ backupConfig.StorePeriod = period.PeriodWeek+ backupConfig.Storage = storage+ backupConfig.StorageID = &storage.ID+ backupConfig.IsRetryIfFailed = true+ backupConfig.MaxFailedTriesCount = 5++ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // Create a failed backup with IsSkipRetry set to true+ failMessage := "backup failed due to size limit exceeded"+ backupRepository.Save(&backups_core.Backup{+ DatabaseID: database.ID,+ StorageID: storage.ID,++ Status: backups_core.BackupStatusFailed,+ FailMessage: &failMessage,+ IsSkipRetry: true,++ CreatedAt: time.Now().UTC().Add(-1 * time.Hour),+ })++ // Verify GetRemainedBackupTryCount returns 0 even though retries are enabled+ lastBackup, err := backupRepository.FindLastByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.NotNil(t, lastBackup)++ remainedTries := GetBackupsScheduler().GetRemainedBackupTryCount(lastBackup)+ assert.Equal(t, 0, remainedTries, "Should return 0 tries when IsSkipRetry is true")++ // Run the scheduler+ GetBackupsScheduler().runPendingBackups()++ time.Sleep(100 * time.Millisecond)++ // Verify no new backup was created (still only 1 backup exists)+ backups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Len(t, backups, 1, "No retry should be attempted when IsSkipRetry is true")++ time.Sleep(200 * time.Millisecond)+}diff --git a/backend/internal/features/backups/backups/backuping/testing.go b/backend/internal/features/backups/backups/backuping/testing.goindex fa06770c..22aa39e4 100644--- a/backend/internal/features/backups/backups/backuping/testing.go+++ b/backend/internal/features/backups/backups/backuping/testing.go@@ -57,17 +57,16 @@ func CreateTestBackuperNode() *BackuperNode {func CreateTestScheduler() *BackupsScheduler {return &BackupsScheduler{- backupRepository,- backups_config.GetBackupConfigService(),- storages.GetStorageService(),- taskCancelManager,- backupNodesRegistry,- time.Now().UTC(),- logger.GetLogger(),- make(map[uuid.UUID]BackupToNodeRelation),- CreateTestBackuperNode(),- sync.Once{},- atomic.Bool{},+ backupRepository: backupRepository,+ backupConfigService: backups_config.GetBackupConfigService(),+ taskCancelManager: taskCancelManager,+ backupNodesRegistry: backupNodesRegistry,+ lastBackupTime: time.Now().UTC(),+ logger: logger.GetLogger(),+ backupToNodeRelations: make(map[uuid.UUID]BackupToNodeRelation),+ backuperNode: CreateTestBackuperNode(),+ runOnce: sync.Once{},+ hasRun: atomic.Bool{},}}diff --git a/backend/internal/features/backups/backups/core/model.go b/backend/internal/features/backups/backups/core/model.goindex afff3105..f060ab2c 100644--- a/backend/internal/features/backups/backups/core/model.go+++ b/backend/internal/features/backups/backups/core/model.go@@ -15,6 +15,7 @@ type Backup struct {Status BackupStatus `json:"status" gorm:"column:status;not null"`FailMessage *string `json:"failMessage" gorm:"column:fail_message"`+ IsSkipRetry bool `json:"isSkipRetry" gorm:"column:is_skip_retry;type:boolean;not null"`BackupSizeMb float64 `json:"backupSizeMb" gorm:"column:backup_size_mb;default:0"`diff --git a/backend/internal/features/backups/backups/core/repository.go b/backend/internal/features/backups/backups/core/repository.goindex 82823483..dfa81f0c 100644--- a/backend/internal/features/backups/backups/core/repository.go+++ b/backend/internal/features/backups/backups/core/repository.go@@ -212,3 +212,36 @@ func (r *BackupRepository) CountByDatabaseID(databaseID uuid.UUID) (int64, errorreturn count, nil}++func (r *BackupRepository) GetTotalSizeByDatabase(databaseID uuid.UUID) (float64, error) {+ var totalSize float64++ if err := storage.+ GetDb().+ Model(&Backup{}).+ Select("COALESCE(SUM(backup_size_mb), 0)").+ Where("database_id = ? AND status != ?", databaseID, BackupStatusInProgress).+ Scan(&totalSize).Error; err != nil {+ return 0, err+ }++ return totalSize, nil+}++func (r *BackupRepository) FindOldestByDatabaseExcludingInProgress(+ databaseID uuid.UUID,+ limit int,+) ([]*Backup, error) {+ var backups []*Backup++ if err := storage.+ GetDb().+ Where("database_id = ? AND status != ?", databaseID, BackupStatusInProgress).+ Order("created_at ASC").+ Limit(limit).+ Find(&backups).Error; err != nil {+ return nil, err+ }++ return backups, nil+}diff --git a/backend/internal/features/backups/backups/di.go b/backend/internal/features/backups/backups/di.goindex cddb1590..d79dc6af 100644--- a/backend/internal/features/backups/backups/di.go+++ b/backend/internal/features/backups/backups/di.go@@ -25,22 +25,23 @@ var backupRepository = &backups_core.BackupRepository{}var taskCancelManager = task_cancellation.GetTaskCancelManager()var backupService = &BackupService{- databaseService: databases.GetDatabaseService(),- storageService: storages.GetStorageService(),- backupRepository: backupRepository,- notifierService: notifiers.GetNotifierService(),- notificationSender: notifiers.GetNotifierService(),- backupConfigService: backups_config.GetBackupConfigService(),- secretKeyService: encryption_secrets.GetSecretKeyService(),- fieldEncryptor: encryption.GetFieldEncryptor(),- createBackupUseCase: usecases.GetCreateBackupUsecase(),- logger: logger.GetLogger(),- backupRemoveListeners: []backups_core.BackupRemoveListener{},- workspaceService: workspaces_services.GetWorkspaceService(),- auditLogService: audit_logs.GetAuditLogService(),- taskCancelManager: taskCancelManager,- downloadTokenService: backups_download.GetDownloadTokenService(),- backupSchedulerService: backuping.GetBackupsScheduler(),+ databases.GetDatabaseService(),+ storages.GetStorageService(),+ backupRepository,+ notifiers.GetNotifierService(),+ notifiers.GetNotifierService(),+ backups_config.GetBackupConfigService(),+ encryption_secrets.GetSecretKeyService(),+ encryption.GetFieldEncryptor(),+ usecases.GetCreateBackupUsecase(),+ logger.GetLogger(),+ []backups_core.BackupRemoveListener{},+ workspaces_services.GetWorkspaceService(),+ audit_logs.GetAuditLogService(),+ taskCancelManager,+ backups_download.GetDownloadTokenService(),+ backuping.GetBackupsScheduler(),+ backuping.GetBackupCleaner(),}var backupController = &BackupController{diff --git a/backend/internal/features/backups/backups/service.go b/backend/internal/features/backups/backups/service.goindex 527797cc..101305a5 100644--- a/backend/internal/features/backups/backups/service.go+++ b/backend/internal/features/backups/backups/service.go@@ -46,6 +46,7 @@ type BackupService struct {taskCancelManager *task_cancellation.TaskCancelManagerdownloadTokenService *backups_download.DownloadTokenServicebackupSchedulerService *backuping.BackupsScheduler+ backupCleaner *backuping.BackupCleaner}func (s *BackupService) AddBackupRemoveListener(listener backups_core.BackupRemoveListener) {@@ -189,7 +190,7 @@ func (s *BackupService) DeleteBackup(database.WorkspaceID,)- return s.deleteBackup(backup)+ return s.backupCleaner.DeleteBackup(backup)}func (s *BackupService) GetBackup(backupID uuid.UUID) (*backups_core.Backup, error) {@@ -292,29 +293,6 @@ func (s *BackupService) GetBackupFile(return reader, backup, database, nil}-func (s *BackupService) deleteBackup(backup *backups_core.Backup) error {- for _, listener := range s.backupRemoveListeners {- if err := listener.OnBeforeBackupRemove(backup); err != nil {- return err- }- }-- storage, err := s.storageService.GetStorageByID(backup.StorageID)- if err != nil {- return err- }-- err = storage.DeleteFile(s.fieldEncryptor, backup.ID)- if err != nil {- // we do not return error here, because sometimes clean up performed- // before unavailable storage removal or change - therefore we should- // proceed even in case of error- s.logger.Error("Failed to delete backup file", "error", err)- }-- return s.backupRepository.DeleteByID(backup.ID)-}-func (s *BackupService) deleteDbBackups(databaseID uuid.UUID) error {dbBackupsInProgress, err := s.backupRepository.FindByDatabaseIdAndStatus(databaseID,@@ -336,7 +314,7 @@ func (s *BackupService) deleteDbBackups(databaseID uuid.UUID) error {}for _, dbBackup := range dbBackups {- err := s.deleteBackup(dbBackup)+ err := s.backupCleaner.DeleteBackup(dbBackup)if err != nil {return err}diff --git a/backend/internal/features/backups/config/model.go b/backend/internal/features/backups/config/model.goindex 82b425cd..7e63b2bf 100644--- a/backend/internal/features/backups/config/model.go+++ b/backend/internal/features/backups/config/model.go@@ -31,6 +31,11 @@ type BackupConfig struct {MaxFailedTriesCount int `json:"maxFailedTriesCount" gorm:"column:max_failed_tries_count;type:int;not null"`Encryption BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`++ // MaxBackupSizeMB limits individual backup size. 0 = unlimited.+ MaxBackupSizeMB int64 `json:"maxBackupSizeMb" gorm:"column:max_backup_size_mb;type:int;not null"`+ // MaxBackupsTotalSizeMB limits total size of all backups. 0 = unlimited.+ MaxBackupsTotalSizeMB int64 `json:"maxBackupsTotalSizeMb" gorm:"column:max_backups_total_size_mb;type:int;not null"`}func (h *BackupConfig) TableName() string {@@ -89,20 +94,30 @@ func (b *BackupConfig) Validate() error {return errors.New("encryption must be NONE or ENCRYPTED")}+ if b.MaxBackupSizeMB < 0 {+ return errors.New("max backup size must be non-negative")+ }++ if b.MaxBackupsTotalSizeMB < 0 {+ return errors.New("max backups total size must be non-negative")+ }+return nil}func (b *BackupConfig) Copy(newDatabaseID uuid.UUID) *BackupConfig {return &BackupConfig{- DatabaseID: newDatabaseID,- IsBackupsEnabled: b.IsBackupsEnabled,- StorePeriod: b.StorePeriod,- BackupIntervalID: uuid.Nil,- BackupInterval: b.BackupInterval.Copy(),- StorageID: b.StorageID,- SendNotificationsOn: b.SendNotificationsOn,- IsRetryIfFailed: b.IsRetryIfFailed,- MaxFailedTriesCount: b.MaxFailedTriesCount,- Encryption: b.Encryption,+ DatabaseID: newDatabaseID,+ IsBackupsEnabled: b.IsBackupsEnabled,+ StorePeriod: b.StorePeriod,+ BackupIntervalID: uuid.Nil,+ BackupInterval: b.BackupInterval.Copy(),+ StorageID: b.StorageID,+ SendNotificationsOn: b.SendNotificationsOn,+ IsRetryIfFailed: b.IsRetryIfFailed,+ MaxFailedTriesCount: b.MaxFailedTriesCount,+ Encryption: b.Encryption,+ MaxBackupSizeMB: b.MaxBackupSizeMB,+ MaxBackupsTotalSizeMB: b.MaxBackupsTotalSizeMB,}}diff --git a/backend/internal/features/restores/di.go b/backend/internal/features/restores/di.goindex 86705830..483549e8 100644--- a/backend/internal/features/restores/di.go+++ b/backend/internal/features/restores/di.go@@ -6,6 +6,7 @@ import (audit_logs "databasus-backend/internal/features/audit_logs""databasus-backend/internal/features/backups/backups"+ "databasus-backend/internal/features/backups/backups/backuping"backups_config "databasus-backend/internal/features/backups/config""databasus-backend/internal/features/databases""databasus-backend/internal/features/disk"@@ -51,6 +52,7 @@ func SetupDependencies() {setupOnce.Do(func() {backups.GetBackupService().AddBackupRemoveListener(restoreService)+ backuping.GetBackupCleaner().AddBackupRemoveListener(restoreService)isSetup.Store(true)})diff --git a/backend/migrations/20260119124334_add_backup_size_limits.sql b/backend/migrations/20260119124334_add_backup_size_limits.sqlnew file mode 100644index 00000000..3d5ac22e--- /dev/null+++ b/backend/migrations/20260119124334_add_backup_size_limits.sql@@ -0,0 +1,23 @@+-- +goose Up+-- +goose StatementBegin++ALTER TABLE backup_configs+ ADD COLUMN max_backup_size_mb BIGINT NOT NULL DEFAULT 0,+ ADD COLUMN max_backups_total_size_mb BIGINT NOT NULL DEFAULT 0;++ALTER TABLE backups+ ADD COLUMN is_skip_retry BOOLEAN NOT NULL DEFAULT FALSE;++-- +goose StatementEnd++-- +goose Down+-- +goose StatementBegin++ALTER TABLE backup_configs+ DROP COLUMN IF EXISTS max_backups_total_size_mb,+ DROP COLUMN IF EXISTS max_backup_size_mb;++ALTER TABLE backups+ DROP COLUMN IF EXISTS is_skip_retry;++-- +goose StatementEnddiff --git a/frontend/src/entity/backups/model/BackupConfig.ts b/frontend/src/entity/backups/model/BackupConfig.tsindex 77fb00ed..8fcb1f47 100644--- a/frontend/src/entity/backups/model/BackupConfig.ts+++ b/frontend/src/entity/backups/model/BackupConfig.ts@@ -15,4 +15,7 @@ export interface BackupConfig {isRetryIfFailed: boolean;maxFailedTriesCount: number;encryption: BackupEncryption;++ maxBackupSizeMb: number;+ maxBackupsTotalSizeMb: number;}diff --git a/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx b/frontend/src/features/backups/ui/EditBackupConfigComponent.tsxindex 63dec884..bac77f6b 100644--- a/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx+++ b/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx@@ -79,7 +79,10 @@ export const EditBackupConfigComponent = ({const [isShowWarn, setIsShowWarn] = useState(false);- const hasAdvancedValues = !!backupConfig?.isRetryIfFailed;+ const hasAdvancedValues =+ !!backupConfig?.isRetryIfFailed ||+ (backupConfig?.maxBackupSizeMb ?? 0) > 0 ||+ (backupConfig?.maxBackupsTotalSizeMb ?? 0) > 0;const [isShowAdvanced, setShowAdvanced] = useState(hasAdvancedValues);const timeFormat = useMemo(() => {@@ -162,6 +165,9 @@ export const EditBackupConfigComponent = ({isRetryIfFailed: true,maxFailedTriesCount: 3,encryption: BackupEncryption.ENCRYPTED,++ maxBackupSizeMb: 0,+ maxBackupsTotalSizeMb: 0,});}loadStorages();@@ -565,6 +571,89 @@ export const EditBackupConfigComponent = ({</div></div>)}++ <div className="mt-5 mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">+ <div className="mb-1 min-w-[150px] sm:mb-0">Max backup size limit</div>+ <div className="flex items-center">+ <Checkbox+ checked={backupConfig.maxBackupSizeMb > 0}+ onChange={(e) => {+ updateBackupConfig({+ maxBackupSizeMb: e.target.checked ? backupConfig.maxBackupSizeMb || 1000 : 0,+ });+ }}+ >+ Enable+ </Checkbox>++ <Tooltip+ className="cursor-pointer"+ title="Limits the size of each individual backup. Note that backups are typically 15× smaller than the database size. For example, a 100 MB backup represents approximately 1.5 GB database."+ >+ <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />+ </Tooltip>+ </div>+ </div>++ {backupConfig.maxBackupSizeMb > 0 && (+ <div className="mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">+ <div className="mb-1 min-w-[150px] sm:mb-0">Max backup size (MB)</div>++ <InputNumber+ min={1}+ value={backupConfig.maxBackupSizeMb}+ onChange={(value) => updateBackupConfig({ maxBackupSizeMb: value || 1 })}+ size="small"+ className="w-full max-w-[100px] grow"+ />++ <div className="ml-2 text-xs text-gray-600 dark:text-gray-400">+ {(backupConfig.maxBackupSizeMb / 1024).toFixed(2)} GB+ </div>+ </div>+ )}++ <div className="mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">+ <div className="mb-1 min-w-[150px] sm:mb-0">Limit total backups size</div>+ <div className="flex items-center">+ <Checkbox+ checked={backupConfig.maxBackupsTotalSizeMb > 0}+ onChange={(e) => {+ updateBackupConfig({+ maxBackupsTotalSizeMb: e.target.checked+ ? backupConfig.maxBackupsTotalSizeMb || 1_000_000+ : 0,+ });+ }}+ >+ Enable+ </Checkbox>++ <Tooltip+ className="cursor-pointer"+ title="Limits the total size of all backups in storage. Once this limit is exceeded, the oldest backups are automatically removed until the total size is within the limit again."+ >+ <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />+ </Tooltip>+ </div>+ </div>++ {backupConfig.maxBackupsTotalSizeMb > 0 && (+ <div className="mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">+ <div className="mb-1 min-w-[150px] sm:mb-0">Max total size (MB)</div>+ <InputNumber+ min={1}+ value={backupConfig.maxBackupsTotalSizeMb}+ onChange={(value) => updateBackupConfig({ maxBackupsTotalSizeMb: value || 1 })}+ size="small"+ className="w-full max-w-[100px] grow"+ />++ <div className="ml-2 text-xs text-gray-600 dark:text-gray-400">+ {(backupConfig.maxBackupsTotalSizeMb / 1024).toFixed(2)} GB+ </div>+ </div>+ )}</>)}