Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
feat: add auto-heal job to restart unhealthy containers (#1780)
Garrett Edwardsaf1e274b2476
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex cdf080317..b555a70e3 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -44,6 +44,9 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedulvulnerabilityScanJob := pkg_scheduler.NewVulnerabilityScanJob(appServices.Vulnerability, appServices.Settings)newScheduler.RegisterJob(vulnerabilityScanJob)+ autoHealJob := pkg_scheduler.NewAutoHealJob(appServices.Docker, appServices.Settings, appServices.Event, appServices.Notification)+ newScheduler.RegisterJob(autoHealJob)+setupJobScheduleCallbacks(appCtx,appServices,@@ -57,8 +60,9 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedulscheduledPruneJob,gitOpsSyncJob,vulnerabilityScanJob,+ autoHealJob,)- setupSettingsCallbacks(appCtx, appServices, appConfig, newScheduler, imagePollingJob, autoUpdateJob, environmentHealthJob, fsWatcherJob, scheduledPruneJob, vulnerabilityScanJob)+ setupSettingsCallbacks(appCtx, appServices, appConfig, newScheduler, imagePollingJob, autoUpdateJob, environmentHealthJob, fsWatcherJob, scheduledPruneJob, vulnerabilityScanJob, autoHealJob)}func setupJobScheduleCallbacks(@@ -74,6 +78,7 @@ func setupJobScheduleCallbacks(scheduledPruneJob *pkg_scheduler.ScheduledPruneJob,gitOpsSyncJob *pkg_scheduler.GitOpsSyncJob,vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob,+ autoHealJob *pkg_scheduler.AutoHealJob,) {if appServices.JobSchedule == nil {return@@ -95,6 +100,7 @@ func setupJobScheduleCallbacks(scheduledPruneJob,gitOpsSyncJob,vulnerabilityScanJob,+ autoHealJob,)}}@@ -113,6 +119,7 @@ func handleJobScheduleChangeInternal(scheduledPruneJob *pkg_scheduler.ScheduledPruneJob,gitOpsSyncJob *pkg_scheduler.GitOpsSyncJob,vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob,+ autoHealJob *pkg_scheduler.AutoHealJob,) {switch key {case "pollingInterval":@@ -150,10 +157,14 @@ func handleJobScheduleChangeInternal(if err := newScheduler.RescheduleJob(ctx, vulnerabilityScanJob); err != nil {slog.WarnContext(ctx, "Failed to reschedule vulnerability-scan job", "error", err)}+ case "autoHealInterval":+ if err := newScheduler.RescheduleJob(ctx, autoHealJob); err != nil {+ slog.WarnContext(ctx, "Failed to reschedule auto-heal job", "error", err)+ }}}-func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services, appConfig *config.Config, newScheduler *pkg_scheduler.JobScheduler, imagePollingJob *pkg_scheduler.ImagePollingJob, autoUpdateJob *pkg_scheduler.AutoUpdateJob, environmentHealthJob *pkg_scheduler.EnvironmentHealthJob, fsWatcherJob *pkg_scheduler.FilesystemWatcherJob, scheduledPruneJob *pkg_scheduler.ScheduledPruneJob, vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob) {+func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services, appConfig *config.Config, newScheduler *pkg_scheduler.JobScheduler, imagePollingJob *pkg_scheduler.ImagePollingJob, autoUpdateJob *pkg_scheduler.AutoUpdateJob, environmentHealthJob *pkg_scheduler.EnvironmentHealthJob, fsWatcherJob *pkg_scheduler.FilesystemWatcherJob, scheduledPruneJob *pkg_scheduler.ScheduledPruneJob, vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob, autoHealJob *pkg_scheduler.AutoHealJob) {appServices.Settings.OnImagePollingSettingsChanged = func(_ context.Context) {if err := newScheduler.RescheduleJob(lifecycleCtx, imagePollingJob); err != nil {slog.WarnContext(lifecycleCtx, "Failed to reschedule image-polling job", "error", err)@@ -190,6 +201,11 @@ func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services,slog.WarnContext(lifecycleCtx, "Failed to reschedule vulnerability-scan job", "error", err)}}+ appServices.Settings.OnAutoHealSettingsChanged = func(ctx context.Context) {+ if err := newScheduler.RescheduleJob(ctx, autoHealJob); err != nil {+ slog.WarnContext(ctx, "Failed to reschedule auto-heal job", "error", err)+ }+ }// Only set up timeout sync callback on main instance (not in agent mode)if !appConfig.AgentMode {diff --git a/backend/internal/huma/handlers/notifications.go b/backend/internal/huma/handlers/notifications.goindex 21515cd43..3685ed1ae 100644--- a/backend/internal/huma/handlers/notifications.go+++ b/backend/internal/huma/handlers/notifications.go@@ -95,6 +95,7 @@ var supportedNotificationTestTypes = map[string]struct{}{"batch-image-update": {},"vulnerability-found": {},"prune-report": {},+ "auto-heal": {},}func normalizeNotificationTestType(testType string) string {diff --git a/backend/internal/huma/handlers/notifications_test.go b/backend/internal/huma/handlers/notifications_test.gonew file mode 100644index 000000000..a08d7124f--- /dev/null+++ b/backend/internal/huma/handlers/notifications_test.go@@ -0,0 +1,32 @@+package handlers++import (+ "testing"++ "github.com/stretchr/testify/assert"+)++func TestIsSupportedNotificationTestType(t *testing.T) {+ expected := []string{+ "simple",+ "image-update",+ "batch-image-update",+ "vulnerability-found",+ "prune-report",+ "auto-heal",+ }++ for _, tt := range expected {+ assert.True(t, isSupportedNotificationTestType(tt), "expected %q to be supported", tt)+ }++ assert.False(t, isSupportedNotificationTestType("bogus"))+ assert.False(t, isSupportedNotificationTestType(""))+}++func TestNormalizeNotificationTestType(t *testing.T) {+ assert.Equal(t, "simple", normalizeNotificationTestType(""))+ assert.Equal(t, "simple", normalizeNotificationTestType(" "))+ assert.Equal(t, "auto-heal", normalizeNotificationTestType("auto-heal"))+ assert.Equal(t, "auto-heal", normalizeNotificationTestType(" auto-heal "))+}diff --git a/backend/internal/models/notification.go b/backend/internal/models/notification.goindex e8fc8b8ae..911d6d1f1 100644--- a/backend/internal/models/notification.go+++ b/backend/internal/models/notification.go@@ -44,6 +44,7 @@ const (NotificationEventContainerUpdate NotificationEventType = "container_update"NotificationEventVulnerabilityFound NotificationEventType = "vulnerability_found"NotificationEventPruneReport NotificationEventType = "prune_report"+ NotificationEventAutoHeal NotificationEventType = "auto_heal")type EmailTLSMode stringdiff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.goindex cda7149ee..35ba7e163 100644--- a/backend/internal/models/settings.go+++ b/backend/internal/models/settings.go@@ -70,6 +70,11 @@ type Settings struct {ScheduledPruneVolumes SettingVariable `key:"scheduledPruneVolumes" meta:"label=Scheduled Prune Volumes;type=boolean;keywords=prune,volumes,cleanup,maintenance;category=internal;description=Remove unused volumes during scheduled prune"`ScheduledPruneNetworks SettingVariable `key:"scheduledPruneNetworks" meta:"label=Scheduled Prune Networks;type=boolean;keywords=prune,networks,cleanup,maintenance;category=internal;description=Remove unused networks during scheduled prune"`ScheduledPruneBuildCache SettingVariable `key:"scheduledPruneBuildCache" meta:"label=Scheduled Prune Build Cache;type=boolean;keywords=prune,build cache,cleanup,maintenance;category=internal;description=Remove Docker build cache during scheduled prune"`+ AutoHealEnabled SettingVariable `key:"autoHealEnabled" meta:"label=Auto Heal;type=boolean;keywords=auto,heal,health,restart,unhealthy,recovery,container,healthcheck;category=internal;description=Automatically restart containers that become unhealthy"`+ AutoHealInterval SettingVariable `key:"autoHealInterval" meta:"label=Auto Heal Interval;type=cron;keywords=auto,heal,interval,frequency,schedule,health,jobs;description=How often to check container health (cron expression)" catmeta:"id=jobschedule"`+ AutoHealExcludedContainers SettingVariable `key:"autoHealExcludedContainers" meta:"label=Auto Heal Excluded Containers;type=text;keywords=auto,heal,exclude,containers,ignore,skip,health;category=internal;description=Comma-separated list of containers to exclude from auto-heal"`+ AutoHealMaxRestarts SettingVariable `key:"autoHealMaxRestarts" meta:"label=Auto Heal Max Restarts;type=number;keywords=auto,heal,max,restarts,limit,loop,protection;category=internal;description=Maximum auto-heal restarts per container within the restart window (default: 5)"`+ AutoHealRestartWindow SettingVariable `key:"autoHealRestartWindow" meta:"label=Auto Heal Restart Window;type=number;keywords=auto,heal,restart,window,minutes,cooldown,protection;category=internal;description=Time window in minutes for counting auto-heal restarts (default: 30)"`MaxImageUploadSize SettingVariable `key:"maxImageUploadSize" meta:"label=Max Image Upload Size;type=number;keywords=upload,size,limit,maximum,image,tar,file,megabytes,mb,storage;category=internal;description=Maximum size in MB for image archive uploads (default: 500)"`DockerHost SettingVariable `key:"dockerHost,public,envOverride" meta:"label=Docker Host;type=text;keywords=docker,host,daemon,socket,unix,remote;category=internal;description=URI for Docker daemon"`diff --git a/backend/internal/services/apprise_service.go b/backend/internal/services/apprise_service.goindex dce38e852..27f4048e6 100644--- a/backend/internal/services/apprise_service.go+++ b/backend/internal/services/apprise_service.go@@ -113,6 +113,9 @@ func (s *AppriseService) SendNotification(ctx context.Context, title, body, formcase models.NotificationEventVulnerabilityFound:// No dedicated tag in AppriseSettings; notification is sent without a tag++ case models.NotificationEventAutoHeal:+ // No dedicated tag in AppriseSettings; notification is sent without a tag}payload := AppriseNotificationPayload{diff --git a/backend/internal/services/job_service.go b/backend/internal/services/job_service.goindex f387695fb..78fad813e 100644--- a/backend/internal/services/job_service.go+++ b/backend/internal/services/job_service.go@@ -62,6 +62,7 @@ func (s *JobService) GetJobSchedules(ctx context.Context) jobschedule.Config {ScheduledPruneInterval: s.settings.GetStringSetting(ctx, "scheduledPruneInterval", "0 0 0 * * *"),GitopsSyncInterval: s.settings.GetStringSetting(ctx, "gitopsSyncInterval", "0 */1 * * * *"),VulnerabilityScanInterval: s.settings.GetStringSetting(ctx, "vulnerabilityScanInterval", "0 0 0 * * *"),+ AutoHealInterval: s.settings.GetStringSetting(ctx, "autoHealInterval", "*/30 * * * * *"),}}@@ -88,6 +89,7 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule{key: "scheduledPruneInterval", current: current.ScheduledPruneInterval, update: updates.ScheduledPruneInterval},{key: "gitopsSyncInterval", current: current.GitopsSyncInterval, update: updates.GitopsSyncInterval},{key: "vulnerabilityScanInterval", current: current.VulnerabilityScanInterval, update: updates.VulnerabilityScanInterval},+ {key: "autoHealInterval", current: current.AutoHealInterval, update: updates.AutoHealInterval},}// Validate inputs (cron expressions)@@ -228,6 +230,7 @@ func (s *JobService) getJobScheduleInternal(ctx context.Context, meta meta.JobMe"scheduledPruneInterval": "0 0 0 * * *","gitopsSyncInterval": "0 */1 * * * *","vulnerabilityScanInterval": "0 0 0 * * *",+ "autoHealInterval": "*/30 * * * * *",}defaultSchedule := defaultSchedules[meta.SettingsKey]diff --git a/backend/internal/services/notification_service.go b/backend/internal/services/notification_service.goindex a299f52bb..97e193979 100644--- a/backend/internal/services/notification_service.go+++ b/backend/internal/services/notification_service.go@@ -32,6 +32,7 @@ const (notificationTestTypeBatchImageUpdate = "batch-image-update"notificationTestTypeVulnerability = "vulnerability-found"notificationTestTypePruneReport = "prune-report"+ notificationTestTypeAutoHeal = "auto-heal")var supportedNotificationTestTypes = map[string]struct{}{@@ -40,6 +41,7 @@ var supportedNotificationTestTypes = map[string]struct{}{notificationTestTypeBatchImageUpdate: {},notificationTestTypeVulnerability: {},notificationTestTypePruneReport: {},+ notificationTestTypeAutoHeal: {},}// VulnerabilityNotificationPayload is the data sent to all providers for vulnerability_found events.@@ -903,6 +905,34 @@ func (s *NotificationService) TestNotification(ctx context.Context, provider mod}}+ if testType == notificationTestTypeAutoHeal {+ testContainerName := "test-container"+ switch provider {+ case models.NotificationProviderDiscord:+ return s.sendDiscordAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderEmail:+ return s.sendEmailAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderTelegram:+ return s.sendTelegramAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderSignal:+ return s.sendSignalAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderSlack:+ return s.sendSlackAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderNtfy:+ return s.sendNtfyAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderPushover:+ return s.sendPushoverAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderGotify:+ return s.sendGotifyAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderMatrix:+ return s.sendMatrixAutoHealNotification(ctx, testContainerName, setting.Config)+ case models.NotificationProviderGeneric:+ return s.sendGenericAutoHealNotification(ctx, testContainerName, setting.Config)+ default:+ return fmt.Errorf("unknown provider: %s", provider)+ }+ }+if testType == notificationTestTypePruneReport {result := &system.PruneAllResult{Success: true,@@ -3193,6 +3223,184 @@ func (s *NotificationService) sendGenericPruneNotification(ctx context.Context,return notifications.SendGenericWithTitle(ctx, genericConfig, "System Prune Report", message)}+// SendAutoHealNotification sends a notification when a container is auto-healed.+func (s *NotificationService) SendAutoHealNotification(ctx context.Context, containerName, containerID string) error {+ settings, err := s.GetAllSettings(ctx)+ if err != nil {+ return fmt.Errorf("failed to get notification settings: %w", err)+ }++ var errs []string+ for _, setting := range settings {+ if !setting.Enabled {+ continue+ }++ if !s.isEventEnabled(setting.Config, models.NotificationEventAutoHeal) {+ continue+ }++ var sendErr error+ switch setting.Provider {+ case models.NotificationProviderDiscord:+ sendErr = s.sendDiscordAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderEmail:+ sendErr = s.sendEmailAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderTelegram:+ sendErr = s.sendTelegramAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderSignal:+ sendErr = s.sendSignalAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderSlack:+ sendErr = s.sendSlackAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderNtfy:+ sendErr = s.sendNtfyAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderPushover:+ sendErr = s.sendPushoverAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderGotify:+ sendErr = s.sendGotifyAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderMatrix:+ sendErr = s.sendMatrixAutoHealNotification(ctx, containerName, setting.Config)+ case models.NotificationProviderGeneric:+ sendErr = s.sendGenericAutoHealNotification(ctx, containerName, setting.Config)+ default:+ slog.WarnContext(ctx, "Unknown notification provider", "provider", setting.Provider)+ continue+ }++ status := "success"+ var errMsg *string+ if sendErr != nil {+ status = "failed"+ msg := sendErr.Error()+ errMsg = new(msg)+ errs = append(errs, fmt.Sprintf("%s: %s", setting.Provider, msg))+ }++ s.logNotification(ctx, setting.Provider, containerName, status, errMsg, models.JSON{+ "containerID": containerID,+ "eventType": string(models.NotificationEventAutoHeal),+ })+ }++ if len(errs) > 0 {+ return fmt.Errorf("notification errors: %s", strings.Join(errs, "; "))+ }++ return nil+}++func (s *NotificationService) sendDiscordAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var discordConfig models.DiscordConfig+ if err := s.unmarshalConfigInternal(config, &discordConfig); err != nil {+ return err+ }+ if discordConfig.WebhookID == "" || discordConfig.Token == "" {+ return fmt.Errorf("discord webhook ID or token not configured")+ }+ s.decryptDiscordTokenInternal(&discordConfig)+ message := fmt.Sprintf("**Container '%s' was automatically restarted because it was unhealthy**", containerName)+ return notifications.SendDiscord(ctx, discordConfig, message)+}++func (s *NotificationService) sendEmailAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var emailConfig models.EmailConfig+ if err := s.unmarshalConfigInternal(config, &emailConfig); err != nil {+ return err+ }+ if err := s.validateEmailConfigInternal(&emailConfig); err != nil {+ return err+ }+ s.decryptEmailPasswordInternal(&emailConfig)+ subject := fmt.Sprintf("Auto Heal: Container '%s' Restarted", containerName)+ body := fmt.Sprintf("<p>Container <strong>%s</strong> was automatically restarted because it was unhealthy.</p>", containerName)+ return notifications.SendEmail(ctx, emailConfig, subject, body)+}++func (s *NotificationService) sendTelegramAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var telegramConfig models.TelegramConfig+ if err := s.unmarshalConfigInternal(config, &telegramConfig); err != nil {+ return err+ }+ if telegramConfig.BotToken == "" || len(telegramConfig.ChatIDs) == 0 {+ return fmt.Errorf("telegram bot token or chat IDs not configured")+ }+ s.decryptTelegramTokenInternal(&telegramConfig)+ if telegramConfig.ParseMode == "" {+ telegramConfig.ParseMode = "HTML"+ }+ message := fmt.Sprintf("<b>Auto Heal:</b> Container '%s' was automatically restarted because it was unhealthy", containerName)+ return notifications.SendTelegram(ctx, telegramConfig, message)+}++func (s *NotificationService) sendSignalAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var signalConfig models.SignalConfig+ if err := s.unmarshalConfigInternal(config, &signalConfig); err != nil {+ return err+ }+ message := fmt.Sprintf("Auto Heal: Container '%s' was automatically restarted because it was unhealthy", containerName)+ return notifications.SendSignal(ctx, signalConfig, message)+}++func (s *NotificationService) sendSlackAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var slackConfig models.SlackConfig+ if err := s.unmarshalConfigInternal(config, &slackConfig); err != nil {+ return err+ }+ message := fmt.Sprintf("*Auto Heal:* Container '%s' was automatically restarted because it was unhealthy", containerName)+ return notifications.SendSlack(ctx, slackConfig, message)+}++func (s *NotificationService) sendNtfyAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var ntfyConfig models.NtfyConfig+ if err := s.unmarshalConfigInternal(config, &ntfyConfig); err != nil {+ return err+ }+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)+ return notifications.SendNtfy(ctx, ntfyConfig, message)+}++func (s *NotificationService) sendPushoverAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var pushoverConfig models.PushoverConfig+ if err := s.unmarshalConfigInternal(config, &pushoverConfig); err != nil {+ return err+ }+ if pushoverConfig.Title == "" {+ pushoverConfig.Title = "Auto Heal"+ }+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)+ return notifications.SendPushover(ctx, pushoverConfig, message)+}++func (s *NotificationService) sendGotifyAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var gotifyConfig models.GotifyConfig+ if err := s.unmarshalConfigInternal(config, &gotifyConfig); err != nil {+ return err+ }+ if gotifyConfig.Title == "" {+ gotifyConfig.Title = "Auto Heal"+ }+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)+ return notifications.SendGotify(ctx, gotifyConfig, message)+}++func (s *NotificationService) sendMatrixAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var matrixConfig models.MatrixConfig+ if err := s.unmarshalConfigInternal(config, &matrixConfig); err != nil {+ return err+ }+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)+ return notifications.SendMatrix(ctx, matrixConfig, message)+}++func (s *NotificationService) sendGenericAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {+ var genericConfig models.GenericConfig+ if err := s.unmarshalConfigInternal(config, &genericConfig); err != nil {+ return err+ }+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)+ return notifications.SendGenericWithTitle(ctx, genericConfig, "Auto Heal", message)+}+// Helper methods to reduce code duplicationfunc (s *NotificationService) unmarshalConfigInternal(config models.JSON, dest any) error {configBytes, err := json.Marshal(config)diff --git a/backend/internal/services/notification_service_test.go b/backend/internal/services/notification_service_test.goindex d0f4657ae..c2e696c5f 100644--- a/backend/internal/services/notification_service_test.go+++ b/backend/internal/services/notification_service_test.go@@ -299,3 +299,22 @@ func TestNotificationService_MigrateDiscordWebhookUrlToFields_PreservesAllFieldsrequire.False(t, discordConfig.Events[models.NotificationEventImageUpdate])require.True(t, discordConfig.Events[models.NotificationEventContainerUpdate])}++func TestSupportedNotificationTestTypes_IncludesAutoHeal(t *testing.T) {+ expected := []string{+ notificationTestTypeSimple,+ notificationTestTypeImageUpdate,+ notificationTestTypeBatchImageUpdate,+ notificationTestTypeVulnerability,+ notificationTestTypePruneReport,+ notificationTestTypeAutoHeal,+ }++ for _, tt := range expected {+ _, ok := supportedNotificationTestTypes[tt]+ require.True(t, ok, "expected %q to be in supportedNotificationTestTypes", tt)+ }++ require.Equal(t, len(expected), len(supportedNotificationTestTypes),+ "supportedNotificationTestTypes has unexpected entries")+}diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.goindex 2cc2ecda0..a85c43891 100644--- a/backend/internal/services/settings_service.go+++ b/backend/internal/services/settings_service.go@@ -38,6 +38,7 @@ type SettingsService struct {OnProjectsDirectoryChanged func(ctx context.Context)OnScheduledPruneSettingsChanged func(ctx context.Context)OnVulnerabilityScanSettingsChanged func(ctx context.Context)+ OnAutoHealSettingsChanged func(ctx context.Context)OnTimeoutSettingsChanged func(ctx context.Context, timeoutSettings map[string]string)}@@ -103,6 +104,11 @@ func (s *SettingsService) getDefaultSettings() *models.Settings {ScheduledPruneVolumes: models.SettingVariable{Value: "false"},ScheduledPruneNetworks: models.SettingVariable{Value: "true"},ScheduledPruneBuildCache: models.SettingVariable{Value: "false"},+ AutoHealEnabled: models.SettingVariable{Value: "false"},+ AutoHealInterval: models.SettingVariable{Value: "*/30 * * * * *"},+ AutoHealExcludedContainers: models.SettingVariable{Value: ""},+ AutoHealMaxRestarts: models.SettingVariable{Value: "5"},+ AutoHealRestartWindow: models.SettingVariable{Value: "30"},GitopsSyncInterval: models.SettingVariable{Value: "0 */1 * * * *"},BaseServerURL: models.SettingVariable{Value: "http://localhost"},EnableGravatar: models.SettingVariable{Value: "true"},@@ -457,7 +463,7 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Ureturn nil, fmt.Errorf("failed to load current settings: %w", err)}- valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedTimeouts, err := s.prepareUpdateValues(updates, cfg, defaultCfg)+ valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, err := s.prepareUpdateValues(updates, cfg, defaultCfg)if err != nil {return nil, err}@@ -491,6 +497,9 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Uif changedVulnerabilityScan && s.OnVulnerabilityScanSettingsChanged != nil {s.OnVulnerabilityScanSettingsChanged(ctx)}+ if changedAutoHeal && s.OnAutoHealSettingsChanged != nil {+ s.OnAutoHealSettingsChanged(ctx)+ }if slices.ContainsFunc(valuesToUpdate, func(sv models.SettingVariable) bool { return sv.Key == "projectsDirectory" }) && s.OnProjectsDirectoryChanged != nil {s.OnProjectsDirectoryChanged(ctx)}@@ -517,7 +526,7 @@ var timeoutSettingKeys = []string{"trivyConcurrentScanContainers",}-func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *models.Settings) ([]models.SettingVariable, bool, bool, bool, bool, map[string]string, error) {+func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *models.Settings) ([]models.SettingVariable, bool, bool, bool, bool, bool, map[string]string, error) {rt := reflect.TypeFor[settings.Update]()rv := reflect.ValueOf(updates)valuesToUpdate := make([]models.SettingVariable, 0)@@ -526,6 +535,7 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defachangedAutoUpdate := falsechangedScheduledPrune := falsechangedVulnerabilityScan := false+ changedAutoHeal := falsechangedTimeouts := make(map[string]string)for i := 0; i < rt.NumField(); i++ {@@ -543,10 +553,10 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa}// Validate cron settings- cronFields := []string{"scheduledPruneInterval", "autoUpdateInterval", "pollingInterval", "environmentHealthInterval", "eventCleanupInterval", "analyticsHeartbeatInterval", "vulnerabilityScanInterval", "gitopsSyncInterval"}+ cronFields := []string{"scheduledPruneInterval", "autoUpdateInterval", "pollingInterval", "environmentHealthInterval", "eventCleanupInterval", "analyticsHeartbeatInterval", "vulnerabilityScanInterval", "gitopsSyncInterval", "autoHealInterval"}if slices.Contains(cronFields, key) && value != "" {if _, err := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow).Parse(value); err != nil {- return nil, false, false, false, false, nil, fmt.Errorf("invalid cron expression for %s: %w", key, err)+ return nil, false, false, false, false, false, nil, fmt.Errorf("invalid cron expression for %s: %w", key, err)}}@@ -565,7 +575,7 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaif errors.Is(err, models.SettingSensitiveForbiddenError{}) {continue} else if err != nil {- return nil, false, false, false, false, nil, fmt.Errorf("failed to update in-memory config for key '%s': %w", key, err)+ return nil, false, false, false, false, false, nil, fmt.Errorf("failed to update in-memory config for key '%s': %w", key, err)}valuesToUpdate = append(valuesToUpdate, models.SettingVariable{@@ -582,6 +592,8 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defachangedScheduledPrune = truecase "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers":changedVulnerabilityScan = true+ case "autoHealEnabled", "autoHealInterval", "autoHealExcludedContainers", "autoHealMaxRestarts", "autoHealRestartWindow":+ changedAutoHeal = true}// Track timeout setting changes@@ -590,7 +602,7 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa}}- return valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedTimeouts, nil+ return valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, nil}func (s *SettingsService) persistSettings(ctx context.Context, values []models.SettingVariable) error {diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.gonew file mode 100644index 000000000..c03c98af5--- /dev/null+++ b/backend/pkg/scheduler/auto_heal_job.go@@ -0,0 +1,267 @@+package scheduler++import (+ "context"+ "log/slog"+ "strings"+ "sync"+ "time"++ "github.com/docker/docker/api/types/container"+ "github.com/getarcaneapp/arcane/backend/internal/models"+ "github.com/getarcaneapp/arcane/backend/internal/services"+ "github.com/getarcaneapp/arcane/backend/pkg/libarcane"+ "github.com/robfig/cron/v3"+)++const AutoHealJobName = "auto-heal"++// restartRecord tracks restart timestamps for a single container.+type restartRecord struct {+ timestamps []time.Time+}++type AutoHealJob struct {+ dockerClientService *services.DockerClientService+ settingsService *services.SettingsService+ eventService *services.EventService+ notificationService *services.NotificationService++ mu sync.Mutex+ restarts map[string]*restartRecord+}++func NewAutoHealJob(+ dockerClientService *services.DockerClientService,+ settingsService *services.SettingsService,+ eventService *services.EventService,+ notificationService *services.NotificationService,+) *AutoHealJob {+ return &AutoHealJob{+ dockerClientService: dockerClientService,+ settingsService: settingsService,+ eventService: eventService,+ notificationService: notificationService,+ restarts: make(map[string]*restartRecord),+ }+}++func (j *AutoHealJob) Name() string {+ return AutoHealJobName+}++func (j *AutoHealJob) Schedule(ctx context.Context) string {+ schedule := j.settingsService.GetStringSetting(ctx, "autoHealInterval", "*/30 * * * * *")+ if schedule == "" {+ schedule = "*/30 * * * * *"+ }++ parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)+ if _, err := parser.Parse(schedule); err != nil {+ slog.WarnContext(ctx, "Invalid cron expression for auto-heal, using default", "invalid_schedule", schedule, "error", err)+ return "*/30 * * * * *"+ }++ return schedule+}++func (j *AutoHealJob) Run(ctx context.Context) {+ enabled := j.settingsService.GetBoolSetting(ctx, "autoHealEnabled", false)+ if !enabled {+ slog.DebugContext(ctx, "auto-heal disabled; skipping run")+ return+ }++ dockerClient, err := j.dockerClientService.GetClient()+ if err != nil {+ slog.ErrorContext(ctx, "auto-heal failed to get Docker client", "error", err)+ return+ }++ containers, err := dockerClient.ContainerList(ctx, container.ListOptions{All: false})+ if err != nil {+ slog.ErrorContext(ctx, "auto-heal failed to list containers", "error", err)+ return+ }++ excludedContainers := j.parseExcludedContainers(ctx)+ maxRestarts := j.settingsService.GetIntSetting(ctx, "autoHealMaxRestarts", 5)+ restartWindowMinutes := j.settingsService.GetIntSetting(ctx, "autoHealRestartWindow", 30)+ restartWindow := time.Duration(restartWindowMinutes) * time.Minute++ for _, c := range containers {+ // Skip Arcane internal containers+ if libarcane.IsInternalContainer(c.Labels) {+ continue+ }++ containerName := j.getContainerName(c.Names)++ // Skip excluded containers+ if j.isExcluded(containerName, excludedContainers) {+ continue+ }++ // Inspect to get health status+ inspect, err := dockerClient.ContainerInspect(ctx, c.ID)+ if err != nil {+ slog.WarnContext(ctx, "auto-heal failed to inspect container", "container", containerName, "error", err)+ continue+ }++ // Skip if no healthcheck configured or not unhealthy+ if inspect.State == nil || inspect.State.Health == nil {+ continue+ }+ if inspect.State.Health.Status != "unhealthy" {+ continue+ }++ // Check restart-loop protection+ if !j.canRestart(c.ID, maxRestarts, restartWindow) {+ slog.WarnContext(ctx, "auto-heal restart-loop protection: skipping container",+ "container", containerName,+ "max_restarts", maxRestarts,+ "window_minutes", restartWindowMinutes,+ )+ continue+ }++ // Restart the container+ slog.InfoContext(ctx, "auto-heal restarting unhealthy container", "container", containerName, "container_id", c.ID)+ if err := dockerClient.ContainerRestart(ctx, c.ID, container.StopOptions{}); err != nil {+ slog.ErrorContext(ctx, "auto-heal failed to restart container", "container", containerName, "error", err)+ continue+ }++ // Record the restart+ j.recordRestart(c.ID)++ // Log event+ if err := j.eventService.LogContainerEvent(+ ctx,+ models.EventTypeContainerRestart,+ c.ID,+ containerName,+ "", // no user - system action+ "system",+ "",+ models.JSON{"action": "auto-heal", "reason": "unhealthy"},+ ); err != nil {+ slog.WarnContext(ctx, "auto-heal failed to log event", "container", containerName, "error", err)+ }++ // Send notification+ if err := j.notificationService.SendAutoHealNotification(ctx, containerName, c.ID); err != nil {+ slog.WarnContext(ctx, "auto-heal failed to send notification", "container", containerName, "error", err)+ }++ slog.InfoContext(ctx, "auto-heal successfully restarted container", "container", containerName)+ }+}++func (j *AutoHealJob) Reschedule(ctx context.Context) error {+ slog.InfoContext(ctx, "rescheduling auto-heal job in new scheduler; currently requires restart")+ return nil+}++// canRestart checks if a container can be restarted within the rate limit.+func (j *AutoHealJob) canRestart(containerID string, maxRestarts int, window time.Duration) bool {+ j.mu.Lock()+ defer j.mu.Unlock()++ record, exists := j.restarts[containerID]+ if !exists {+ return true+ }++ cutoff := time.Now().Add(-window)+ recent := j.pruneTimestamps(record.timestamps, cutoff)+ record.timestamps = recent++ return len(recent) < maxRestarts+}++// recordRestart records a restart timestamp for a container.+func (j *AutoHealJob) recordRestart(containerID string) {+ j.mu.Lock()+ defer j.mu.Unlock()++ record, exists := j.restarts[containerID]+ if !exists {+ record = &restartRecord{}+ j.restarts[containerID] = record+ }++ record.timestamps = append(record.timestamps, time.Now())+}++// pruneTimestamps removes timestamps older than the cutoff.+func (j *AutoHealJob) pruneTimestamps(timestamps []time.Time, cutoff time.Time) []time.Time {+ result := make([]time.Time, 0, len(timestamps))+ for _, ts := range timestamps {+ if ts.After(cutoff) {+ result = append(result, ts)+ }+ }+ return result+}++func (j *AutoHealJob) parseExcludedContainers(ctx context.Context) map[string]struct{} {+ raw := j.settingsService.GetStringSetting(ctx, "autoHealExcludedContainers", "")+ excluded := make(map[string]struct{})+ if raw == "" {+ return excluded+ }+ for _, name := range strings.Split(raw, ",") {+ trimmed := strings.TrimSpace(name)+ if trimmed != "" {+ excluded[trimmed] = struct{}{}+ }+ }+ return excluded+}++func (j *AutoHealJob) getContainerName(names []string) string {+ if len(names) == 0 {+ return ""+ }+ // Docker container names are prefixed with "/"+ return strings.TrimPrefix(names[0], "/")+}++func (j *AutoHealJob) isExcluded(name string, excluded map[string]struct{}) bool {+ _, ok := excluded[name]+ return ok+}++// ResetRestartTracking clears all restart records (exported for testing).+func (j *AutoHealJob) ResetRestartTracking() {+ j.mu.Lock()+ defer j.mu.Unlock()+ j.restarts = make(map[string]*restartRecord)+}++// CanRestartExported exposes canRestart for testing.+func (j *AutoHealJob) CanRestartExported(containerID string, maxRestarts int, window time.Duration) bool {+ return j.canRestart(containerID, maxRestarts, window)+}++// RecordRestartExported exposes recordRestart for testing.+func (j *AutoHealJob) RecordRestartExported(containerID string) {+ j.recordRestart(containerID)+}++// RecordRestartAtExported records a restart at a specific time for testing.+func (j *AutoHealJob) RecordRestartAtExported(containerID string, t time.Time) {+ j.mu.Lock()+ defer j.mu.Unlock()++ record, exists := j.restarts[containerID]+ if !exists {+ record = &restartRecord{}+ j.restarts[containerID] = record+ }++ record.timestamps = append(record.timestamps, t)+}diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.gonew file mode 100644index 000000000..9b3c05176--- /dev/null+++ b/backend/pkg/scheduler/auto_heal_job_test.go@@ -0,0 +1,114 @@+package scheduler++import (+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++func newTestAutoHealJob() *AutoHealJob {+ return &AutoHealJob{+ restarts: make(map[string]*restartRecord),+ }+}++func TestAutoHeal_CanRestart_UnderLimit(t *testing.T) {+ job := newTestAutoHealJob()++ // No restarts recorded yet — should be allowed+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))++ // Record 4 restarts (under limit of 5)+ for i := 0; i < 4; i++ {+ job.RecordRestartExported("container-1")+ }++ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))+}++func TestAutoHeal_CanRestart_AtLimit(t *testing.T) {+ job := newTestAutoHealJob()++ // Record exactly 5 restarts (at limit)+ for i := 0; i < 5; i++ {+ job.RecordRestartExported("container-1")+ }++ require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))+}++func TestAutoHeal_CanRestart_WindowExpiry(t *testing.T) {+ job := newTestAutoHealJob()++ // Record 5 restarts 31 minutes ago (outside window)+ oldTime := time.Now().Add(-31 * time.Minute)+ for i := 0; i < 5; i++ {+ job.RecordRestartAtExported("container-1", oldTime)+ }++ // Should be allowed because all timestamps are outside the 30-minute window+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))+}++func TestAutoHeal_CanRestart_MixedTimestamps(t *testing.T) {+ job := newTestAutoHealJob()++ // Record 3 old restarts (outside window)+ oldTime := time.Now().Add(-31 * time.Minute)+ for i := 0; i < 3; i++ {+ job.RecordRestartAtExported("container-1", oldTime)+ }++ // Record 4 recent restarts (inside window)+ for i := 0; i < 4; i++ {+ job.RecordRestartExported("container-1")+ }++ // Should still be allowed (only 4 recent, limit is 5)+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))++ // Add one more recent restart+ job.RecordRestartExported("container-1")++ // Now should be blocked (5 recent)+ require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))+}++func TestAutoHeal_CanRestart_DifferentContainers(t *testing.T) {+ job := newTestAutoHealJob()++ // Fill up container-1+ for i := 0; i < 5; i++ {+ job.RecordRestartExported("container-1")+ }++ // container-1 should be blocked+ require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))++ // container-2 should still be allowed+ require.True(t, job.CanRestartExported("container-2", 5, 30*time.Minute))+}++func TestAutoHeal_Schedule_Default(t *testing.T) {+ job := newTestAutoHealJob()+ // Without a settings service, Schedule would panic.+ // We test the Name() method directly.+ require.Equal(t, "auto-heal", job.Name())+}++func TestAutoHeal_ResetRestartTracking(t *testing.T) {+ job := newTestAutoHealJob()++ // Fill up container-1+ for i := 0; i < 5; i++ {+ job.RecordRestartExported("container-1")+ }+ require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))++ // Reset tracking+ job.ResetRestartTracking()++ // Should be allowed again+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))+}diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 29af18f68..14e8c4f51 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -185,6 +185,12 @@"auto_update_containers_excluded_many": "{count} containers excluded","auto_update_exclude_description": "Select containers to exclude from automatic updates.","auto_update_excluded_containers": "Excluded Containers",+ "auto_heal_excluded_containers": "Excluded Containers",+ "auto_heal_exclude_description": "Select containers to exclude from auto-heal monitoring.",+ "auto_heal_max_restarts_label": "Max Restarts",+ "auto_heal_max_restarts_description": "Maximum number of automatic restarts per container within the restart window.",+ "auto_heal_restart_window_label": "Restart Window (minutes)",+ "auto_heal_restart_window_description": "Time window in minutes for counting automatic restarts. Resets after this period.","_comment_common_actions": "=== COMMON - ACTION STATES ===","common_action_creating": "Creating…","common_action_deleting": "Deleting…",@@ -1705,6 +1711,7 @@"notifications_email_test_vulnerability_found": "Vulnerability Found Notification","notifications_test_vulnerability_notification": "Test Vulnerability Notification","notifications_test_prune_report_notification": "Test Prune Report Notification",+ "notifications_test_auto_heal_notification": "Test Auto-Heal Notification","notifications_apprise_title": "Apprise Integration","notifications_apprise_description": "Configure Apprise API for unified notifications across multiple services","notifications_apprise_enabled_label": "Enable Apprise",@@ -2004,5 +2011,7 @@"project_includes": "Includes","notifications_event_prune_report_label": "System Prune Report","notifications_event_prune_report_description": "Receive a summary report when a scheduled prune operation completes",+ "notifications_event_auto_heal_label": "Auto-Heal Restart",+ "notifications_event_auto_heal_description": "Notify when an unhealthy container is automatically restarted","version_info_build_features": "Build Features"}diff --git a/frontend/src/lib/types/job-schedule.type.ts b/frontend/src/lib/types/job-schedule.type.tsindex 42c713078..172053fd0 100644--- a/frontend/src/lib/types/job-schedule.type.ts+++ b/frontend/src/lib/types/job-schedule.type.ts@@ -7,6 +7,7 @@ export type JobSchedules = {scheduledPruneInterval: string;gitopsSyncInterval: string;vulnerabilityScanInterval: string;+ autoHealInterval: string;};export type JobSchedulesUpdate = Partial<JobSchedules>;diff --git a/frontend/src/lib/types/notification-providers.ts b/frontend/src/lib/types/notification-providers.tsindex 716e0912d..452ddec79 100644--- a/frontend/src/lib/types/notification-providers.ts+++ b/frontend/src/lib/types/notification-providers.ts@@ -22,6 +22,7 @@ export interface BaseProviderFormValues {eventContainerUpdate: boolean;eventVulnerabilityFound: boolean;eventPruneReport: boolean;+ eventAutoHeal: boolean;}// Provider-specific form value types@@ -177,7 +178,8 @@ export function discordSettingsToFormValues(settings?: NotificationSettings): DieventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -196,7 +198,8 @@ export function emailSettingsToFormValues(settings?: NotificationSettings): EmaieventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -213,7 +216,8 @@ export function telegramSettingsToFormValues(settings?: NotificationSettings): TeventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -233,7 +237,8 @@ export function signalSettingsToFormValues(settings?: NotificationSettings): SigeventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -252,7 +257,8 @@ export function slackSettingsToFormValues(settings?: NotificationSettings): SlaceventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -279,7 +285,8 @@ export function discordFormValuesToSettings(values: DiscordFormValues): Notificaimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -304,7 +311,8 @@ export function emailFormValuesToSettings(values: EmailFormValues): Notificationimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -327,7 +335,8 @@ export function telegramFormValuesToSettings(values: TelegramFormValues): Notifiimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -353,7 +362,8 @@ export function signalFormValuesToSettings(values: SignalFormValues): Notificatiimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -375,7 +385,8 @@ export function slackFormValuesToSettings(values: SlackFormValues): Notificationimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -400,7 +411,8 @@ export function ntfySettingsToFormValues(settings?: NotificationSettings): NtfyFeventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -417,7 +429,8 @@ export function pushoverSettingsToFormValues(settings?: NotificationSettings): PeventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -436,7 +449,8 @@ export function gotifySettingsToFormValues(settings?: NotificationSettings): GoteventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -454,7 +468,8 @@ export function matrixSettingsToFormValues(settings?: NotificationSettings): MateventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -479,7 +494,8 @@ export function genericSettingsToFormValues(settings?: NotificationSettings): GeeventImageUpdate: events?.image_update ?? true,eventContainerUpdate: events?.container_update ?? true,eventVulnerabilityFound: events?.vulnerability_found ?? true,- eventPruneReport: events?.prune_report ?? true+ eventPruneReport: events?.prune_report ?? true,+ eventAutoHeal: events?.auto_heal ?? true};}@@ -506,7 +522,8 @@ export function ntfyFormValuesToSettings(values: NtfyFormValues): NotificationSeimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -529,7 +546,8 @@ export function pushoverFormValuesToSettings(values: PushoverFormValues): Notifiimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -551,7 +569,8 @@ export function gotifyFormValuesToSettings(values: GotifyFormValues): Notificatiimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -572,7 +591,8 @@ export function matrixFormValuesToSettings(values: MatrixFormValues): Notificatiimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};@@ -608,7 +628,8 @@ export function genericFormValuesToSettings(values: GenericFormValues): Notificaimage_update: values.eventImageUpdate,container_update: values.eventContainerUpdate,vulnerability_found: values.eventVulnerabilityFound,- prune_report: values.eventPruneReport+ prune_report: values.eventPruneReport,+ auto_heal: values.eventAutoHeal}}};diff --git a/frontend/src/lib/types/settings.type.ts b/frontend/src/lib/types/settings.type.tsindex 62c2c373b..510ecf2d2 100644--- a/frontend/src/lib/types/settings.type.ts+++ b/frontend/src/lib/types/settings.type.ts@@ -19,6 +19,10 @@ export type Settings = {scheduledPruneBuildCache?: boolean;vulnerabilityScanEnabled?: boolean;vulnerabilityScanInterval?: number;+ autoHealEnabled?: boolean;+ autoHealExcludedContainers?: string;+ autoHealMaxRestarts?: number;+ autoHealRestartWindow?: number;maxImageUploadSize: number;baseServerUrl: string;enableGravatar: boolean;diff --git a/frontend/src/routes/(app)/environments/[id]/+page.svelte b/frontend/src/routes/(app)/environments/[id]/+page.svelteindex 14f51d2d6..239b439d3 100644--- a/frontend/src/routes/(app)/environments/[id]/+page.svelte+++ b/frontend/src/routes/(app)/environments/[id]/+page.svelte@@ -149,7 +149,11 @@scheduledPruneNetworks: z.boolean(),scheduledPruneBuildCache: z.boolean(),vulnerabilityScanEnabled: z.boolean(),- autoUpdateExcludedContainers: z.string()+ autoUpdateExcludedContainers: z.string(),+ autoHealEnabled: z.boolean(),+ autoHealExcludedContainers: z.string(),+ autoHealMaxRestarts: z.coerce.number().int().min(1),+ autoHealRestartWindow: z.coerce.number().int().min(1)});// Build current settings object from environment and settings data@@ -173,7 +177,11 @@scheduledPruneNetworks: settings?.scheduledPruneNetworks ?? true,scheduledPruneBuildCache: settings?.scheduledPruneBuildCache ?? false,vulnerabilityScanEnabled: settings?.vulnerabilityScanEnabled ?? false,- autoUpdateExcludedContainers: settings?.autoUpdateExcludedContainers || ''+ autoUpdateExcludedContainers: settings?.autoUpdateExcludedContainers || '',+ autoHealEnabled: settings?.autoHealEnabled ?? false,+ autoHealExcludedContainers: settings?.autoHealExcludedContainers || '',+ autoHealMaxRestarts: settings?.autoHealMaxRestarts ?? 5,+ autoHealRestartWindow: settings?.autoHealRestartWindow ?? 30});// Custom save handler for environment-specific settings@@ -204,7 +212,11 @@scheduledPruneNetworks: formData.scheduledPruneNetworks,scheduledPruneBuildCache: formData.scheduledPruneBuildCache,vulnerabilityScanEnabled: formData.vulnerabilityScanEnabled,- autoUpdateExcludedContainers: formData.autoUpdateExcludedContainers+ autoUpdateExcludedContainers: formData.autoUpdateExcludedContainers,+ autoHealEnabled: formData.autoHealEnabled,+ autoHealExcludedContainers: formData.autoHealExcludedContainers,+ autoHealMaxRestarts: formData.autoHealMaxRestarts,+ autoHealRestartWindow: formData.autoHealRestartWindow});}diff --git a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelteindex ad484b330..c528b0910 100644--- a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte+++ b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte@@ -43,7 +43,7 @@});const containersPromise = $derived.by(async () => {- if (!$formInputs.autoUpdate.value) return [];+ if (!$formInputs.autoUpdate.value && !$formInputs.autoHealEnabled.value) return [];const result = await tryCatch(containerService.getContainers({ pagination: { page: 1, limit: 100 } }));if (result.error) throw result.error;return result.data.data;@@ -83,6 +83,8 @@return undefined;case 'vulnerabilityScanEnabled':return undefined;+ case 'autoHealEnabled':+ return `${envBase}?tab=jobs`;default:return prereq.settingsUrl;}@@ -106,6 +108,46 @@}}+ const autoHealExcludedContainers = new SvelteSet<string>();+ let autoHealSearchTerm = $state('');++ $effect(() => {+ const savedValue = $formInputs.autoHealExcludedContainers?.value || '';+ const names = savedValue+ .split(',')+ .map((s: string) => normalizeContainerName(s.trim()))+ .filter(Boolean);++ const currentNames = Array.from(autoHealExcludedContainers);+ if (names.length !== currentNames.length || names.some((n: string) => !autoHealExcludedContainers.has(n))) {+ autoHealExcludedContainers.clear();+ names.forEach((n: string) => autoHealExcludedContainers.add(n));+ }+ });++ function toggleAutoHealContainerExclusion(containerName: string) {+ const normalizedName = normalizeContainerName(containerName);+ if (autoHealExcludedContainers.has(normalizedName)) {+ autoHealExcludedContainers.delete(normalizedName);+ } else {+ autoHealExcludedContainers.add(normalizedName);+ }++ const newValue = Array.from(autoHealExcludedContainers).join(',');+ if ($formInputs.autoHealExcludedContainers) {+ $formInputs.autoHealExcludedContainers.value = newValue;+ }+ }++ function mapContainerToAutoHealItem(container: ContainerSummaryDto) {+ const name = getContainerName(container);+ return {+ value: name,+ label: name,+ selected: autoHealExcludedContainers.has(name)+ };+ }+const categories = [{ id: 'monitoring', label: m.jobs_monitoring_heading() },{ id: 'maintenance', label: m.jobs_maintenance_heading() },@@ -137,6 +179,8 @@return $formInputs.pollingEnabled.value;case 'vulnerability-scan':return $formInputs.vulnerabilityScanEnabled.value;+ case 'auto-heal':+ return $formInputs.autoHealEnabled.value;default:return undefined;}@@ -217,6 +261,8 @@<Switch bind:checked={$formInputs.scheduledPruneEnabled.value} />{:else if job.id === 'vulnerability-scan'}<Switch bind:checked={$formInputs.vulnerabilityScanEnabled.value} />+ {:else if job.id === 'auto-heal'}+ <Switch bind:checked={$formInputs.autoHealEnabled.value} />{/if}{/snippet}@@ -284,6 +330,99 @@</div>{/if}+ {#if job.id === 'auto-heal' && $formInputs.autoHealEnabled.value}+ <div class="border-border/20 space-y-3 border-t pt-3">+ <div class="grid gap-3 sm:grid-cols-2">+ <div class="space-y-1">+ <Label for="auto-heal-max-restarts" class="text-sm font-medium"+ >{m.auto_heal_max_restarts_label()}</Label+ >+ <p class="text-muted-foreground text-xs">{m.auto_heal_max_restarts_description()}</p>+ <Input+ id="auto-heal-max-restarts"+ type="number"+ min="1"+ class="h-8 w-full"+ bind:value={$formInputs.autoHealMaxRestarts.value}+ />+ </div>+ <div class="space-y-1">+ <Label for="auto-heal-restart-window" class="text-sm font-medium"+ >{m.auto_heal_restart_window_label()}</Label+ >+ <p class="text-muted-foreground text-xs">{m.auto_heal_restart_window_description()}</p>+ <Input+ id="auto-heal-restart-window"+ type="number"+ min="1"+ class="h-8 w-full"+ bind:value={$formInputs.autoHealRestartWindow.value}+ />+ </div>+ </div>++ <div class="space-y-1">+ <Label class="text-sm font-medium">+ {m.auto_heal_excluded_containers()}+ {#await containersPromise then containers}+ <span class="text-muted-foreground ml-1 font-normal">+ ({containers.filter((c) => autoHealExcludedContainers.has(getContainerName(c))).length})+ </span>+ {/await}+ </Label>+ <p class="text-muted-foreground text-xs">{m.auto_heal_exclude_description()}</p>+ </div>++ <div class="rounded-md border p-2">+ <Input+ type="search"+ placeholder="Search containers..."+ class="mb-2 h-8"+ bind:value={autoHealSearchTerm}+ />+ <ScrollArea.Root class="h-64 w-full rounded-md border p-2">+ <div class="space-y-2">+ {#await containersPromise}+ <div class="flex items-center justify-center p-4">+ <Spinner class="size-4" />+ </div>+ {:then containers}+ {@const allItems = containers.map(mapContainerToAutoHealItem)}+ {@const filteredItems = autoHealSearchTerm+ ? allItems.filter((item) =>+ item.label.toLowerCase().includes(autoHealSearchTerm.toLowerCase())+ )+ : allItems}++ {#if filteredItems.length === 0}+ <p class="text-muted-foreground py-4 text-center text-sm">+ {m.common_no_results_found()}+ </p>+ {:else}+ {#each filteredItems as container (container.value)}+ <div class="flex items-center space-x-2">+ <Checkbox+ id="auto-heal-container-{container.value}"+ checked={container.selected}+ onCheckedChange={() => toggleAutoHealContainerExclusion(container.value)}+ />+ <Label for="auto-heal-container-{container.value}" class="text-sm font-normal">+ {container.label}+ </Label>+ </div>+ {/each}+ {/if}+ {:catch error}+ <div class="text-destructive p-2 text-sm">+ {error.message || 'Failed to load containers'}+ </div>+ {/await}+ </div>+ </ScrollArea.Root>+ </div>+ </div>+ {/if}+{#if job.id === 'scheduled-prune'}{#if $formInputs.scheduledPruneEnabled.value}<div class="border-border/20 space-y-4 border-t pt-3">diff --git a/frontend/src/routes/(app)/settings/notifications/providers/BuiltInProviderForm.svelte b/frontend/src/routes/(app)/settings/notifications/providers/BuiltInProviderForm.svelteindex 05d517101..582c1d7ae 100644--- a/frontend/src/routes/(app)/settings/notifications/providers/BuiltInProviderForm.svelte+++ b/frontend/src/routes/(app)/settings/notifications/providers/BuiltInProviderForm.svelte@@ -99,7 +99,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -123,7 +124,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -174,7 +176,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -203,7 +206,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -256,7 +260,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -281,7 +286,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -303,7 +309,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -327,7 +334,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -350,7 +358,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -370,7 +379,8 @@eventImageUpdate: z.boolean(),eventContainerUpdate: z.boolean(),eventVulnerabilityFound: z.boolean(),- eventPruneReport: z.boolean()+ eventPruneReport: z.boolean(),+ eventAutoHeal: z.boolean()}).superRefine((d, ctx) => {if (!d.enabled) return;@@ -1057,7 +1067,8 @@{ label: m.notifications_email_test_image_update(), testType: 'image-update' },{ label: m.notifications_email_test_batch_image_update(), testType: 'batch-image-update' },{ label: m.notifications_test_vulnerability_notification(), testType: 'vulnerability-found' },- { label: m.notifications_test_prune_report_notification(), testType: 'prune-report' }+ { label: m.notifications_test_prune_report_notification(), testType: 'prune-report' },+ { label: m.notifications_test_auto_heal_notification(), testType: 'auto-heal' }];const validation = $derived.by(() => providerSchemas[provider].safeParse(values));@@ -1088,6 +1099,7 @@bind:eventContainerUpdate={values.eventContainerUpdate}bind:eventVulnerabilityFound={values.eventVulnerabilityFound}bind:eventPruneReport={values.eventPruneReport}+ bind:eventAutoHeal={values.eventAutoHeal}{disabled}/>diff --git a/frontend/src/routes/(app)/settings/notifications/providers/EventSubscriptions.svelte b/frontend/src/routes/(app)/settings/notifications/providers/EventSubscriptions.svelteindex a09776ba7..3cc2c5cc2 100644--- a/frontend/src/routes/(app)/settings/notifications/providers/EventSubscriptions.svelte+++ b/frontend/src/routes/(app)/settings/notifications/providers/EventSubscriptions.svelte@@ -9,6 +9,7 @@eventContainerUpdate: boolean;eventVulnerabilityFound: boolean;eventPruneReport: boolean;+ eventAutoHeal: boolean;disabled?: boolean;}@@ -18,6 +19,7 @@eventContainerUpdate = $bindable(),eventVulnerabilityFound = $bindable(),eventPruneReport = $bindable(),+ eventAutoHeal = $bindable(),disabled = false}: Props = $props();</script>@@ -54,5 +56,12 @@label={m.notifications_event_vulnerability_found_label()}description={m.notifications_event_vulnerability_found_description()}/>+ <SwitchWithLabel+ id="{providerId}-event-auto-heal"+ bind:checked={eventAutoHeal}+ {disabled}+ label={m.notifications_event_auto_heal_label()}+ description={m.notifications_event_auto_heal_description()}+ /></div></div>diff --git a/frontend/src/routes/(app)/settings/notifications/providers/NotificationProviderTestMenu.svelte b/frontend/src/routes/(app)/settings/notifications/providers/NotificationProviderTestMenu.svelteindex 330621852..0bd210952 100644--- a/frontend/src/routes/(app)/settings/notifications/providers/NotificationProviderTestMenu.svelte+++ b/frontend/src/routes/(app)/settings/notifications/providers/NotificationProviderTestMenu.svelte@@ -25,7 +25,8 @@{ label: m.notifications_email_test_image_update(), testType: 'image-update' },{ label: m.notifications_email_test_batch_image_update(), testType: 'batch-image-update' },{ label: m.notifications_test_vulnerability_notification(), testType: 'vulnerability-found' },- { label: m.notifications_test_prune_report_notification(), testType: 'prune-report' }+ { label: m.notifications_test_prune_report_notification(), testType: 'prune-report' },+ { label: m.notifications_test_auto_heal_notification(), testType: 'auto-heal' }]}: Props = $props();</script>diff --git a/types/jobschedule/jobschedule.go b/types/jobschedule/jobschedule.goindex d9438e20d..251a208d3 100644--- a/types/jobschedule/jobschedule.go+++ b/types/jobschedule/jobschedule.go@@ -15,6 +15,7 @@ type Config struct {ScheduledPruneInterval string `json:"scheduledPruneInterval"`GitopsSyncInterval string `json:"gitopsSyncInterval"`VulnerabilityScanInterval string `json:"vulnerabilityScanInterval"`+ AutoHealInterval string `json:"autoHealInterval"`}// Update is used to update job schedule intervals (in minutes).@@ -29,6 +30,7 @@ type Update struct {ScheduledPruneInterval *string `json:"scheduledPruneInterval,omitempty"`GitopsSyncInterval *string `json:"gitopsSyncInterval,omitempty"`VulnerabilityScanInterval *string `json:"vulnerabilityScanInterval,omitempty"`+ AutoHealInterval *string `json:"autoHealInterval,omitempty"`}// JobStatus represents the current status and metadata for a background job.diff --git a/types/meta/job_metadata.go b/types/meta/job_metadata.goindex 30296e456..c1afcdcbd 100644--- a/types/meta/job_metadata.go+++ b/types/meta/job_metadata.go@@ -171,6 +171,24 @@ var jobMetadataRegistry = map[string]JobMetadata{},},},+ "auto-heal": {+ ID: "auto-heal",+ Name: "Auto Heal",+ Description: "Automatically restarts containers that become unhealthy",+ Category: "monitoring",+ SettingsKey: "autoHealInterval",+ EnabledKey: "autoHealEnabled",+ ManagerOnly: false,+ IsContinuous: false,+ CanRunManually: true,+ Prerequisites: []JobPrerequisiteMetadata{+ {+ SettingKey: "autoHealEnabled",+ Label: "Auto heal enabled",+ SettingsURL: "/settings/general",+ },+ },+ },}func GetJobMetadata(jobID string) (JobMetadata, bool) {diff --git a/types/settings/settings.go b/types/settings/settings.goindex d962c21ae..1d0052d46 100644--- a/types/settings/settings.go+++ b/types/settings/settings.go@@ -319,4 +319,29 @@ type Update struct {//// Required: falseAutoUpdateExcludedContainers *string `json:"autoUpdateExcludedContainers,omitempty"`++ // AutoHealEnabled indicates if automatic container healing is enabled.+ //+ // Required: false+ AutoHealEnabled *string `json:"autoHealEnabled,omitempty"`++ // AutoHealInterval is the cron expression for how often to check container health.+ //+ // Required: false+ AutoHealInterval *string `json:"autoHealInterval,omitempty"`++ // AutoHealExcludedContainers is a comma-separated list of container names to exclude from auto-heal.+ //+ // Required: false+ AutoHealExcludedContainers *string `json:"autoHealExcludedContainers,omitempty"`++ // AutoHealMaxRestarts is the maximum number of auto-heal restarts per container within the restart window.+ //+ // Required: false+ AutoHealMaxRestarts *string `json:"autoHealMaxRestarts,omitempty"`++ // AutoHealRestartWindow is the time window in minutes for counting auto-heal restarts.+ //+ // Required: false+ AutoHealRestartWindow *string `json:"autoHealRestartWindow,omitempty"`}