PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FEATURE (backups): Add retries in case of failed backups
Rostislav Dugin56b5e8ac50b6
diff --git a/backend/internal/features/backups/backups/background_service.go b/backend/internal/features/backups/backups/background_service.goindex 548554df..f4937c2b 100644--- a/backend/internal/features/backups/backups/background_service.go+++ b/backend/internal/features/backups/backups/background_service.go@@ -4,7 +4,6 @@ import ("log/slog""postgresus-backend/internal/config"backups_config "postgresus-backend/internal/features/backups/config"- "postgresus-backend/internal/features/databases""postgresus-backend/internal/features/storages""postgresus-backend/internal/util/period""time"@@ -13,9 +12,8 @@ import (type BackupBackgroundService struct {backupService *BackupServicebackupRepository *BackupRepository- databaseService *databases.DatabaseService- storageService *storages.StorageServicebackupConfigService *backups_config.BackupConfigService+ storageService *storages.StorageServicelastBackupTime time.Timelogger *slog.Logger@@ -51,7 +49,7 @@ func (s *BackupBackgroundService) Run() {}}-func (s *BackupBackgroundService) IsBackupsRunning() bool {+func (s *BackupBackgroundService) IsBackupsWorkerRunning() bool {// if last backup time is more than 5 minutes ago, return falsereturn s.lastBackupTime.After(time.Now().UTC().Add(-5 * time.Minute))}@@ -90,18 +88,12 @@ func (s *BackupBackgroundService) failBackupsInProgress() error {}func (s *BackupBackgroundService) cleanOldBackups() error {- allDatabases, err := s.databaseService.GetAllDatabases()+ enabledBackupConfigs, err := s.backupConfigService.GetBackupConfigsWithEnabledBackups()if err != nil {return err}- for _, database := range allDatabases {- backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)- if err != nil {- s.logger.Error("Failed to get backup config by database ID", "error", err)- continue- }-+ for _, backupConfig := range enabledBackupConfigs {backupStorePeriod := backupConfig.StorePeriodif backupStorePeriod == period.PeriodForever {@@ -112,14 +104,14 @@ func (s *BackupBackgroundService) cleanOldBackups() error {dateBeforeBackupsShouldBeDeleted := time.Now().UTC().Add(-storeDuration)oldBackups, err := s.backupRepository.FindBackupsBeforeDate(- database.ID,+ backupConfig.DatabaseID,dateBeforeBackupsShouldBeDeleted,)if err != nil {s.logger.Error("Failed to find old backups for database","databaseId",- database.ID,+ backupConfig.DatabaseID,"error",err,)@@ -149,7 +141,13 @@ func (s *BackupBackgroundService) cleanOldBackups() error {continue}- s.logger.Info("Deleted old backup", "backupId", backup.ID, "databaseId", database.ID)+ s.logger.Info(+ "Deleted old backup",+ "backupId",+ backup.ID,+ "databaseId",+ backupConfig.DatabaseID,+ )}}@@ -157,28 +155,22 @@ func (s *BackupBackgroundService) cleanOldBackups() error {}func (s *BackupBackgroundService) runPendingBackups() error {- allDatabases, err := s.databaseService.GetAllDatabases()+ enabledBackupConfigs, err := s.backupConfigService.GetBackupConfigsWithEnabledBackups()if err != nil {return err}- for _, database := range allDatabases {- backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)- if err != nil {- s.logger.Error("Failed to get backup config by database ID", "error", err)- continue- }-+ for _, backupConfig := range enabledBackupConfigs {if backupConfig.BackupInterval == nil {continue}- lastBackup, err := s.backupRepository.FindLastByDatabaseID(database.ID)+ lastBackup, err := s.backupRepository.FindLastByDatabaseID(backupConfig.DatabaseID)if err != nil {s.logger.Error("Failed to get last backup for database","databaseId",- database.ID,+ backupConfig.DatabaseID,"error",err,)@@ -190,19 +182,71 @@ func (s *BackupBackgroundService) runPendingBackups() error {lastBackupTime = &lastBackup.CreatedAt}- if backupConfig.BackupInterval.ShouldTriggerBackup(time.Now().UTC(), lastBackupTime) {+ remainedBackupTryCount := s.GetRemainedBackupTryCount(lastBackup)++ if backupConfig.BackupInterval.ShouldTriggerBackup(time.Now().UTC(), lastBackupTime) ||+ remainedBackupTryCount > 0 {s.logger.Info("Triggering scheduled backup","databaseId",- database.ID,+ backupConfig.DatabaseID,"intervalType",backupConfig.BackupInterval.Interval,)- go s.backupService.MakeBackup(database.ID)- s.logger.Info("Successfully triggered scheduled backup", "databaseId", database.ID)+ go s.backupService.MakeBackup(backupConfig.DatabaseID, remainedBackupTryCount == 1)+ s.logger.Info(+ "Successfully triggered scheduled backup",+ "databaseId",+ backupConfig.DatabaseID,+ )}}return nil}++// GetRemainedBackupTryCount returns the number of remaining backup tries for a given backup.+// If the backup is not failed or the backup config does not allow retries, it returns 0.+// If the backup is failed and the backup config allows retries, it returns the number of remaining tries.+// If the backup is failed and the backup config does not allow retries, it returns 0.+func (s *BackupBackgroundService) GetRemainedBackupTryCount(lastBackup *Backup) int {+ if lastBackup == nil {+ return 0+ }++ if lastBackup.Status != BackupStatusFailed {+ return 0+ }++ backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(lastBackup.DatabaseID)+ if err != nil {+ s.logger.Error("Failed to get backup config by database ID", "error", err)+ return 0+ }++ if !backupConfig.IsRetryIfFailed {+ return 0+ }++ maxFailedTriesCount := backupConfig.MaxFailedTriesCount++ lastBackups, err := s.backupRepository.FindByDatabaseIDWithLimit(+ lastBackup.DatabaseID,+ maxFailedTriesCount,+ )+ if err != nil {+ s.logger.Error("Failed to find last backups by database ID", "error", err)+ return 0+ }++ lastFailedBackups := make([]*Backup, 0)++ for _, backup := range lastBackups {+ if backup.Status == BackupStatusFailed {+ lastFailedBackups = append(lastFailedBackups, backup)+ }+ }++ return maxFailedTriesCount - len(lastFailedBackups)+}diff --git a/backend/internal/features/backups/backups/background_service_test.go b/backend/internal/features/backups/backups/background_service_test.gonew file mode 100644index 00000000..93754719--- /dev/null+++ b/backend/internal/features/backups/backups/background_service_test.go@@ -0,0 +1,314 @@+package backups++import (+ backups_config "postgresus-backend/internal/features/backups/config"+ "postgresus-backend/internal/features/databases"+ "postgresus-backend/internal/features/intervals"+ "postgresus-backend/internal/features/notifiers"+ "postgresus-backend/internal/features/storages"+ "postgresus-backend/internal/features/users"+ "postgresus-backend/internal/util/period"+ "testing"+ "time"++ "github.com/stretchr/testify/assert"+)++func Test_MakeBackupForDbHavingBackupDayAgo_BackupCreated(t *testing.T) {+ // setup data+ user := users.GetTestUser()+ storage := storages.CreateTestStorage(user.UserID)+ notifier := notifiers.CreateTestNotifier(user.UserID)+ database := databases.CreateTestDatabase(user.UserID, storage, notifier)++ // Enable backups for the database+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)+ assert.NoError(t, err)++ timeOfDay := "04:00"+ backupConfig.BackupInterval = &intervals.Interval{+ Interval: intervals.IntervalDaily,+ TimeOfDay: &timeOfDay,+ }+ backupConfig.IsBackupsEnabled = true+ backupConfig.StorePeriod = period.PeriodWeek+ backupConfig.Storage = storage+ backupConfig.StorageID = &storage.ID++ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // add old backup+ backupRepository.Save(&Backup{+ Database: database,+ DatabaseID: database.ID,++ Storage: storage,+ StorageID: storage.ID,++ Status: BackupStatusCompleted,++ CreatedAt: time.Now().UTC().Add(-24 * time.Hour),+ })++ GetBackupBackgroundService().runPendingBackups()++ time.Sleep(100 * time.Millisecond)++ // assertions+ backups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Len(t, backups, 2)++ // cleanup+ for _, backup := range backups {+ err := backupRepository.DeleteByID(backup.ID)+ assert.NoError(t, err)+ }++ databases.RemoveTestDatabase(database)+ storages.RemoveTestStorage(storage.ID)+ notifiers.RemoveTestNotifier(notifier)+}++func Test_MakeBackupForDbHavingHourAgoBackup_BackupSkipped(t *testing.T) {+ // setup data+ user := users.GetTestUser()+ storage := storages.CreateTestStorage(user.UserID)+ notifier := notifiers.CreateTestNotifier(user.UserID)+ database := databases.CreateTestDatabase(user.UserID, storage, notifier)++ // Enable backups for the database+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)+ assert.NoError(t, err)++ timeOfDay := "04:00"+ backupConfig.BackupInterval = &intervals.Interval{+ Interval: intervals.IntervalDaily,+ TimeOfDay: &timeOfDay,+ }+ backupConfig.IsBackupsEnabled = true+ backupConfig.StorePeriod = period.PeriodWeek+ backupConfig.Storage = storage+ backupConfig.StorageID = &storage.ID++ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // add recent backup (1 hour ago)+ backupRepository.Save(&Backup{+ Database: database,+ DatabaseID: database.ID,++ Storage: storage,+ StorageID: storage.ID,++ Status: BackupStatusCompleted,++ CreatedAt: time.Now().UTC().Add(-1 * time.Hour),+ })++ GetBackupBackgroundService().runPendingBackups()++ time.Sleep(100 * time.Millisecond)++ // assertions+ backups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Len(t, backups, 1) // Should still be 1 backup, no new backup created++ // cleanup+ for _, backup := range backups {+ err := backupRepository.DeleteByID(backup.ID)+ assert.NoError(t, err)+ }++ databases.RemoveTestDatabase(database)+ storages.RemoveTestStorage(storage.ID)+ notifiers.RemoveTestNotifier(notifier)+}++func Test_MakeBackupHavingFailedBackupWithoutRetries_BackupSkipped(t *testing.T) {+ // setup data+ user := users.GetTestUser()+ storage := storages.CreateTestStorage(user.UserID)+ notifier := notifiers.CreateTestNotifier(user.UserID)+ database := databases.CreateTestDatabase(user.UserID, storage, notifier)++ // Enable backups for the database with retries disabled+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)+ assert.NoError(t, err)++ timeOfDay := "04:00"+ backupConfig.BackupInterval = &intervals.Interval{+ Interval: intervals.IntervalDaily,+ TimeOfDay: &timeOfDay,+ }+ backupConfig.IsBackupsEnabled = true+ backupConfig.StorePeriod = period.PeriodWeek+ backupConfig.Storage = storage+ backupConfig.StorageID = &storage.ID+ backupConfig.IsRetryIfFailed = false+ backupConfig.MaxFailedTriesCount = 0++ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // add failed backup+ failMessage := "backup failed"+ backupRepository.Save(&Backup{+ Database: database,+ DatabaseID: database.ID,++ Storage: storage,+ StorageID: storage.ID,++ Status: BackupStatusFailed,+ FailMessage: &failMessage,++ CreatedAt: time.Now().UTC().Add(-1 * time.Hour),+ })++ GetBackupBackgroundService().runPendingBackups()++ time.Sleep(100 * time.Millisecond)++ // assertions+ backups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Len(t, backups, 1) // Should still be 1 backup, no retry attempted++ // cleanup+ for _, backup := range backups {+ err := backupRepository.DeleteByID(backup.ID)+ assert.NoError(t, err)+ }++ databases.RemoveTestDatabase(database)+ storages.RemoveTestStorage(storage.ID)+ notifiers.RemoveTestNotifier(notifier)+}++func Test_MakeBackupHavingFailedBackupWithRetries_BackupCreated(t *testing.T) {+ // setup data+ user := users.GetTestUser()+ storage := storages.CreateTestStorage(user.UserID)+ notifier := notifiers.CreateTestNotifier(user.UserID)+ database := databases.CreateTestDatabase(user.UserID, storage, notifier)++ // Enable backups for the database with retries enabled+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)+ assert.NoError(t, err)++ timeOfDay := "04:00"+ backupConfig.BackupInterval = &intervals.Interval{+ Interval: intervals.IntervalDaily,+ TimeOfDay: &timeOfDay,+ }+ backupConfig.IsBackupsEnabled = true+ backupConfig.StorePeriod = period.PeriodWeek+ backupConfig.Storage = storage+ backupConfig.StorageID = &storage.ID+ backupConfig.IsRetryIfFailed = true+ backupConfig.MaxFailedTriesCount = 3++ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ // add failed backup+ failMessage := "backup failed"+ backupRepository.Save(&Backup{+ Database: database,+ DatabaseID: database.ID,++ Storage: storage,+ StorageID: storage.ID,++ Status: BackupStatusFailed,+ FailMessage: &failMessage,++ CreatedAt: time.Now().UTC().Add(-1 * time.Hour),+ })++ GetBackupBackgroundService().runPendingBackups()++ time.Sleep(100 * time.Millisecond)++ // assertions+ backups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Len(t, backups, 2) // Should have 2 backups, retry was attempted++ // cleanup+ for _, backup := range backups {+ err := backupRepository.DeleteByID(backup.ID)+ assert.NoError(t, err)+ }++ databases.RemoveTestDatabase(database)+ storages.RemoveTestStorage(storage.ID)+ notifiers.RemoveTestNotifier(notifier)+}++func Test_MakeBackupHavingFailedBackupWithRetries_RetriesCountNotExceeded(t *testing.T) {+ // setup data+ user := users.GetTestUser()+ storage := storages.CreateTestStorage(user.UserID)+ notifier := notifiers.CreateTestNotifier(user.UserID)+ database := databases.CreateTestDatabase(user.UserID, storage, notifier)++ // Enable backups for the database with retries enabled+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)+ assert.NoError(t, err)++ timeOfDay := "04:00"+ backupConfig.BackupInterval = &intervals.Interval{+ Interval: intervals.IntervalDaily,+ TimeOfDay: &timeOfDay,+ }+ backupConfig.IsBackupsEnabled = true+ backupConfig.StorePeriod = period.PeriodWeek+ backupConfig.Storage = storage+ backupConfig.StorageID = &storage.ID+ backupConfig.IsRetryIfFailed = true+ backupConfig.MaxFailedTriesCount = 3++ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)+ assert.NoError(t, err)++ failMessage := "backup failed"++ for i := 0; i < 3; i++ {+ backupRepository.Save(&Backup{+ Database: database,+ DatabaseID: database.ID,++ Storage: storage,+ StorageID: storage.ID,++ Status: BackupStatusFailed,+ FailMessage: &failMessage,++ CreatedAt: time.Now().UTC().Add(-1 * time.Hour),+ })+ }++ GetBackupBackgroundService().runPendingBackups()++ time.Sleep(100 * time.Millisecond)++ // assertions+ backups, err := backupRepository.FindByDatabaseID(database.ID)+ assert.NoError(t, err)+ assert.Len(t, backups, 3) // Should have 3 backups, not more than max++ // cleanup+ for _, backup := range backups {+ err := backupRepository.DeleteByID(backup.ID)+ assert.NoError(t, err)+ }++ databases.RemoveTestDatabase(database)+ storages.RemoveTestStorage(storage.ID)+ notifiers.RemoveTestNotifier(notifier)+}diff --git a/backend/internal/features/backups/backups/di.go b/backend/internal/features/backups/backups/di.goindex d6074e78..44b0ac00 100644--- a/backend/internal/features/backups/backups/di.go+++ b/backend/internal/features/backups/backups/di.go@@ -27,9 +27,8 @@ var backupService = &BackupService{var backupBackgroundService = &BackupBackgroundService{backupService,backupRepository,- databases.GetDatabaseService(),- storages.GetStorageService(),backups_config.GetBackupConfigService(),+ storages.GetStorageService(),time.Now().UTC(),logger.GetLogger(),}diff --git a/backend/internal/features/backups/backups/repository.go b/backend/internal/features/backups/backups/repository.goindex dc751135..2cc8a746 100644--- a/backend/internal/features/backups/backups/repository.go+++ b/backend/internal/features/backups/backups/repository.go@@ -1,6 +1,7 @@package backupsimport (+ "errors""postgresus-backend/internal/storage""time"@@ -43,6 +44,30 @@ func (r *BackupRepository) FindByDatabaseID(databaseID uuid.UUID) ([]*Backup, erreturn backups, nil}+func (r *BackupRepository) FindByDatabaseIDWithLimit(+ databaseID uuid.UUID,+ limit int,+) ([]*Backup, error) {+ if limit <= 0 {+ return nil, errors.New("limit must be greater than 0")+ }++ var backups []*Backup++ if err := storage.+ GetDb().+ Preload("Database").+ Preload("Storage").+ Where("database_id = ?", databaseID).+ Order("created_at DESC").+ Limit(limit).+ Find(&backups).Error; err != nil {+ return nil, err+ }++ return backups, nil+}+func (r *BackupRepository) FindByStorageID(storageID uuid.UUID) ([]*Backup, error) {var backups []*Backupdiff --git a/backend/internal/features/backups/backups/service.go b/backend/internal/features/backups/backups/service.goindex 37f6ebe1..b3eb8f65 100644--- a/backend/internal/features/backups/backups/service.go+++ b/backend/internal/features/backups/backups/service.go@@ -68,7 +68,7 @@ func (s *BackupService) MakeBackupWithAuth(return errors.New("user does not have access to this database")}- go s.MakeBackup(databaseID)+ go s.MakeBackup(databaseID, true)return nil}@@ -114,7 +114,7 @@ func (s *BackupService) DeleteBackup(return s.deleteBackup(backup)}-func (s *BackupService) MakeBackup(databaseID uuid.UUID) {+func (s *BackupService) MakeBackup(databaseID uuid.UUID, isLastTry bool) {database, err := s.databaseService.GetDatabaseByID(databaseID)if err != nil {s.logger.Error("Failed to get database by ID", "error", err)@@ -244,6 +244,10 @@ func (s *BackupService) MakeBackup(databaseID uuid.UUID) {)}+ if !isLastTry {+ return+ }+s.SendBackupNotification(backupConfig,backup,diff --git a/backend/internal/features/backups/backups/service_test.go b/backend/internal/features/backups/backups/service_test.goindex b0d02056..d08ce7d5 100644--- a/backend/internal/features/backups/backups/service_test.go+++ b/backend/internal/features/backups/backups/service_test.go@@ -52,7 +52,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {}),).Once()- backupService.MakeBackup(database.ID)+ backupService.MakeBackup(database.ID, true)// Verify all expectations were metmockNotificationSender.AssertExpectations(t)@@ -84,7 +84,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {[]BackupRemoveListener{},}- backupService.MakeBackup(database.ID)+ backupService.MakeBackup(database.ID, true)// Verify all expectations were metmockNotificationSender.AssertExpectations(t)@@ -119,7 +119,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {capturedMessage = args.Get(2).(string)}).Once()- backupService.MakeBackup(database.ID)+ backupService.MakeBackup(database.ID, true)// Verify expectations were metmockNotificationSender.AssertExpectations(t)diff --git a/backend/internal/features/backups/config/model.go b/backend/internal/features/backups/config/model.goindex bb66d7c0..491526ea 100644--- a/backend/internal/features/backups/config/model.go+++ b/backend/internal/features/backups/config/model.go@@ -27,6 +27,9 @@ type BackupConfig struct {SendNotificationsOn []BackupNotificationType `json:"sendNotificationsOn" gorm:"-"`SendNotificationsOnString string `json:"-" gorm:"column:send_notifications_on;type:text;not null"`+ IsRetryIfFailed bool `json:"isRetryIfFailed" gorm:"column:is_retry_if_failed;type:boolean;not null"`+ MaxFailedTriesCount int `json:"maxFailedTriesCount" gorm:"column:max_failed_tries_count;type:int;not null"`+CpuCount int `json:"cpuCount" gorm:"type:int;not null"`}@@ -81,5 +84,9 @@ func (b *BackupConfig) Validate() error {return errors.New("cpu count is required")}+ if b.IsRetryIfFailed && b.MaxFailedTriesCount <= 0 {+ return errors.New("max failed tries count must be greater than 0")+ }+return nil}diff --git a/backend/internal/features/backups/config/repository.go b/backend/internal/features/backups/config/repository.goindex 3d7c4870..7c7f6025 100644--- a/backend/internal/features/backups/config/repository.go+++ b/backend/internal/features/backups/config/repository.go@@ -74,7 +74,7 @@ func (r *BackupConfigRepository) FindByDatabaseID(databaseID uuid.UUID) (*Backupreturn &backupConfig, nil}-func (r *BackupConfigRepository) FindWithEnabledBackups() ([]*BackupConfig, error) {+func (r *BackupConfigRepository) GetWithEnabledBackups() ([]*BackupConfig, error) {var backupConfigs []*BackupConfigif err := storage.diff --git a/backend/internal/features/backups/config/service.go b/backend/internal/features/backups/config/service.goindex e2670ee2..1a4163df 100644--- a/backend/internal/features/backups/config/service.go+++ b/backend/internal/features/backups/config/service.go@@ -133,6 +133,10 @@ func (s *BackupConfigService) IsStorageUsing(return s.backupConfigRepository.IsStorageUsing(storageID)}+func (s *BackupConfigService) GetBackupConfigsWithEnabledBackups() ([]*BackupConfig, error) {+ return s.backupConfigRepository.GetWithEnabledBackups()+}+func (s *BackupConfigService) initializeDefaultConfig(databaseID uuid.UUID,) error {@@ -150,7 +154,9 @@ func (s *BackupConfigService) initializeDefaultConfig(NotificationBackupFailed,NotificationBackupSuccess,},- CpuCount: 1,+ CpuCount: 1,+ IsRetryIfFailed: true,+ MaxFailedTriesCount: 3,})return errdiff --git a/backend/internal/features/healthcheck/attempt/check_pg_health_uc_test.go b/backend/internal/features/healthcheck/attempt/check_pg_health_uc_test.goindex 66af447c..93a8cbb7 100644--- a/backend/internal/features/healthcheck/attempt/check_pg_health_uc_test.go+++ b/backend/internal/features/healthcheck/attempt/check_pg_health_uc_test.go@@ -18,6 +18,7 @@ import (func Test_CheckPgHealthUseCase(t *testing.T) {user := users.GetTestUser()+storage := storages.CreateTestStorage(user.UserID)notifier := notifiers.CreateTestNotifier(user.UserID)diff --git a/backend/internal/features/system/healthcheck/service.go b/backend/internal/features/system/healthcheck/service.goindex 5f4baf22..a9e28360 100644--- a/backend/internal/features/system/healthcheck/service.go+++ b/backend/internal/features/system/healthcheck/service.go@@ -29,7 +29,7 @@ func (s *HealthcheckService) IsHealthy() error {return errors.New("cannot connect to the database")}- if !s.backupBackgroundService.IsBackupsRunning() {+ if !s.backupBackgroundService.IsBackupsWorkerRunning() {return errors.New("backups are not running for more than 5 minutes")}diff --git a/backend/migrations/20250713160827_add_backup_retries.sql b/backend/migrations/20250713160827_add_backup_retries.sqlnew file mode 100644index 00000000..94a98e84--- /dev/null+++ b/backend/migrations/20250713160827_add_backup_retries.sql@@ -0,0 +1,13 @@+-- +goose Up+-- +goose StatementBegin+ALTER TABLE backup_configs+ ADD COLUMN is_retry_if_failed BOOLEAN NOT NULL DEFAULT FALSE,+ ADD COLUMN max_failed_tries_count INT NOT NULL DEFAULT 0;+-- +goose StatementEnd++-- +goose Down+-- +goose StatementBegin+ALTER TABLE backup_configs+ DROP COLUMN is_retry_if_failed,+ DROP COLUMN max_failed_tries_count;+-- +goose StatementEnddiff --git a/frontend/src/entity/backups/model/BackupConfig.ts b/frontend/src/entity/backups/model/BackupConfig.tsindex b40a79d2..76165aad 100644--- a/frontend/src/entity/backups/model/BackupConfig.ts+++ b/frontend/src/entity/backups/model/BackupConfig.ts@@ -12,4 +12,6 @@ export interface BackupConfig {storage?: Storage;sendNotificationsOn: BackupNotificationType[];cpuCount: number;+ isRetryIfFailed: boolean;+ maxFailedTriesCount: number;}diff --git a/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx b/frontend/src/features/backups/ui/EditBackupConfigComponent.tsxindex 5274a40f..0ee0d357 100644--- a/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx+++ b/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx@@ -151,6 +151,8 @@ export const EditBackupConfigComponent = ({cpuCount: 1,storePeriod: Period.WEEK,sendNotificationsOn: [],+ isRetryIfFailed: true,+ maxFailedTriesCount: 3,});}loadStorages();@@ -289,6 +291,43 @@ export const EditBackupConfigComponent = ({</div>)}+ <div className="mt-4 mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Retry backup if failed</div>+ <Switch+ size="small"+ checked={backupConfig.isRetryIfFailed}+ onChange={(checked) => updateBackupConfig({ isRetryIfFailed: checked })}+ />++ <Tooltip+ className="cursor-pointer"+ title="Automatically retry failed backups. Backups can fail due to network failures, storage issues or temporary database unavailability."+ >+ <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />+ </Tooltip>+ </div>++ {backupConfig.isRetryIfFailed && (+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Max failed tries count</div>+ <InputNumber+ min={1}+ max={10}+ value={backupConfig.maxFailedTriesCount}+ onChange={(value) => updateBackupConfig({ maxFailedTriesCount: value || 1 })}+ size="small"+ className="max-w-[200px] grow"+ />++ <Tooltip+ className="cursor-pointer"+ title="Maximum number of retry attempts for failed backups. You will receive a notification when all tries have failed."+ >+ <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />+ </Tooltip>+ </div>+ )}+<div className="mt-5 mb-1 flex w-full items-center"><div className="min-w-[150px]">CPU count</div><InputNumberdiff --git a/frontend/src/features/backups/ui/ShowBackupConfigComponent.tsx b/frontend/src/features/backups/ui/ShowBackupConfigComponent.tsxindex 03a61bb8..fb18571d 100644--- a/frontend/src/features/backups/ui/ShowBackupConfigComponent.tsx+++ b/frontend/src/features/backups/ui/ShowBackupConfigComponent.tsx@@ -134,6 +134,18 @@ export const ShowBackupConfigComponent = ({ database }: Props) => {</div>)}+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Retry if failed</div>+ <div>{backupConfig.isRetryIfFailed ? 'Yes' : 'No'}</div>+ </div>++ {backupConfig.isRetryIfFailed && (+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Max failed tries count</div>+ <div>{backupConfig.maxFailedTriesCount}</div>+ </div>+ )}+<div className="mb-1 flex w-full items-center"><div className="min-w-[150px]">Store period</div><div>{backupConfig.storePeriod ? periodLabels[backupConfig.storePeriod] : ''}</div>diff --git a/frontend/src/features/healthcheck/ui/HealthckeckAttemptsComponent.tsx b/frontend/src/features/healthcheck/ui/HealthckeckAttemptsComponent.tsxindex dca84a3b..57ea938c 100644--- a/frontend/src/features/healthcheck/ui/HealthckeckAttemptsComponent.tsx+++ b/frontend/src/features/healthcheck/ui/HealthckeckAttemptsComponent.tsx@@ -78,29 +78,31 @@ export const HealthckeckAttemptsComponent = ({ database }: Props) => {};useEffect(() => {- let isHealthcheckEnabled = false;+ let interval: number | null = null;setIsHealthcheckConfigLoading(true);healthcheckConfigApi.getHealthcheckConfig(database.id).then((healthcheckConfig) => {setIsHealthcheckConfigLoading(false);if (healthcheckConfig.isHealthcheckEnabled) {- isHealthcheckEnabled = true;setIsShowHealthcheckConfig(true);-loadHealthcheckAttempts();++ // Set up interval only if healthcheck+ // is enabled and period is 'today'+ if (period === 'today') {+ interval = setInterval(() => {+ loadHealthcheckAttempts(false);+ }, 60_000); // 5 seconds+ }}});- if (period === 'today') {- if (isHealthcheckEnabled) {- const interval = setInterval(() => {- loadHealthcheckAttempts(false);- }, 60_000); // 1 minute-- return () => clearInterval(interval);+ return () => {+ if (interval) {+ clearInterval(interval);}- }+ };}, [period]);if (isHealthcheckConfigLoading) {