Skip to content

Modern Docker Management, Designed for Everyone

feat: add auto-heal job to restart unhealthy containers (#1780)

Garrett Edwardsaf1e274b2476
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index cdf080317..b555a70e3 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -44,6 +44,9 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
vulnerabilityScanJob := pkg_scheduler.NewVulnerabilityScanJob(appServices.Vulnerability, appServices.Settings)
newScheduler.RegisterJob(vulnerabilityScanJob)
+ autoHealJob := pkg_scheduler.NewAutoHealJob(appServices.Docker, appServices.Settings, appServices.Event, appServices.Notification)
+ newScheduler.RegisterJob(autoHealJob)
+
setupJobScheduleCallbacks(
appCtx,
appServices,
@@ -57,8 +60,9 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
scheduledPruneJob,
gitOpsSyncJob,
vulnerabilityScanJob,
+ autoHealJob,
)
- setupSettingsCallbacks(appCtx, appServices, appConfig, newScheduler, imagePollingJob, autoUpdateJob, environmentHealthJob, fsWatcherJob, scheduledPruneJob, vulnerabilityScanJob)
+ setupSettingsCallbacks(appCtx, appServices, appConfig, newScheduler, imagePollingJob, autoUpdateJob, environmentHealthJob, fsWatcherJob, scheduledPruneJob, vulnerabilityScanJob, autoHealJob)
}
func setupJobScheduleCallbacks(
@@ -74,6 +78,7 @@ func setupJobScheduleCallbacks(
scheduledPruneJob *pkg_scheduler.ScheduledPruneJob,
gitOpsSyncJob *pkg_scheduler.GitOpsSyncJob,
vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob,
+ autoHealJob *pkg_scheduler.AutoHealJob,
) {
if appServices.JobSchedule == nil {
return
@@ -95,6 +100,7 @@ func setupJobScheduleCallbacks(
scheduledPruneJob,
gitOpsSyncJob,
vulnerabilityScanJob,
+ autoHealJob,
)
}
}
@@ -113,6 +119,7 @@ func handleJobScheduleChangeInternal(
scheduledPruneJob *pkg_scheduler.ScheduledPruneJob,
gitOpsSyncJob *pkg_scheduler.GitOpsSyncJob,
vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob,
+ autoHealJob *pkg_scheduler.AutoHealJob,
) {
switch key {
case "pollingInterval":
@@ -150,10 +157,14 @@ func handleJobScheduleChangeInternal(
if err := newScheduler.RescheduleJob(ctx, vulnerabilityScanJob); err != nil {
slog.WarnContext(ctx, "Failed to reschedule vulnerability-scan job", "error", err)
}
+ case "autoHealInterval":
+ if err := newScheduler.RescheduleJob(ctx, autoHealJob); err != nil {
+ slog.WarnContext(ctx, "Failed to reschedule auto-heal job", "error", err)
+ }
}
}
-func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services, appConfig *config.Config, newScheduler *pkg_scheduler.JobScheduler, imagePollingJob *pkg_scheduler.ImagePollingJob, autoUpdateJob *pkg_scheduler.AutoUpdateJob, environmentHealthJob *pkg_scheduler.EnvironmentHealthJob, fsWatcherJob *pkg_scheduler.FilesystemWatcherJob, scheduledPruneJob *pkg_scheduler.ScheduledPruneJob, vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob) {
+func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services, appConfig *config.Config, newScheduler *pkg_scheduler.JobScheduler, imagePollingJob *pkg_scheduler.ImagePollingJob, autoUpdateJob *pkg_scheduler.AutoUpdateJob, environmentHealthJob *pkg_scheduler.EnvironmentHealthJob, fsWatcherJob *pkg_scheduler.FilesystemWatcherJob, scheduledPruneJob *pkg_scheduler.ScheduledPruneJob, vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob, autoHealJob *pkg_scheduler.AutoHealJob) {
appServices.Settings.OnImagePollingSettingsChanged = func(_ context.Context) {
if err := newScheduler.RescheduleJob(lifecycleCtx, imagePollingJob); err != nil {
slog.WarnContext(lifecycleCtx, "Failed to reschedule image-polling job", "error", err)
@@ -190,6 +201,11 @@ func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services,
slog.WarnContext(lifecycleCtx, "Failed to reschedule vulnerability-scan job", "error", err)
}
}
+ appServices.Settings.OnAutoHealSettingsChanged = func(ctx context.Context) {
+ if err := newScheduler.RescheduleJob(ctx, autoHealJob); err != nil {
+ slog.WarnContext(ctx, "Failed to reschedule auto-heal job", "error", err)
+ }
+ }
// Only set up timeout sync callback on main instance (not in agent mode)
if !appConfig.AgentMode {
diff --git a/backend/internal/huma/handlers/notifications.go b/backend/internal/huma/handlers/notifications.go
index 21515cd43..3685ed1ae 100644
--- a/backend/internal/huma/handlers/notifications.go
+++ b/backend/internal/huma/handlers/notifications.go
@@ -95,6 +95,7 @@ var supportedNotificationTestTypes = map[string]struct{}{
"batch-image-update": {},
"vulnerability-found": {},
"prune-report": {},
+ "auto-heal": {},
}
func normalizeNotificationTestType(testType string) string {
diff --git a/backend/internal/huma/handlers/notifications_test.go b/backend/internal/huma/handlers/notifications_test.go
new file mode 100644
index 000000000..a08d7124f
--- /dev/null
+++ b/backend/internal/huma/handlers/notifications_test.go
@@ -0,0 +1,32 @@
+package handlers
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestIsSupportedNotificationTestType(t *testing.T) {
+ expected := []string{
+ "simple",
+ "image-update",
+ "batch-image-update",
+ "vulnerability-found",
+ "prune-report",
+ "auto-heal",
+ }
+
+ for _, tt := range expected {
+ assert.True(t, isSupportedNotificationTestType(tt), "expected %q to be supported", tt)
+ }
+
+ assert.False(t, isSupportedNotificationTestType("bogus"))
+ assert.False(t, isSupportedNotificationTestType(""))
+}
+
+func TestNormalizeNotificationTestType(t *testing.T) {
+ assert.Equal(t, "simple", normalizeNotificationTestType(""))
+ assert.Equal(t, "simple", normalizeNotificationTestType(" "))
+ assert.Equal(t, "auto-heal", normalizeNotificationTestType("auto-heal"))
+ assert.Equal(t, "auto-heal", normalizeNotificationTestType(" auto-heal "))
+}
diff --git a/backend/internal/models/notification.go b/backend/internal/models/notification.go
index e8fc8b8ae..911d6d1f1 100644
--- a/backend/internal/models/notification.go
+++ b/backend/internal/models/notification.go
@@ -44,6 +44,7 @@ const (
NotificationEventContainerUpdate NotificationEventType = "container_update"
NotificationEventVulnerabilityFound NotificationEventType = "vulnerability_found"
NotificationEventPruneReport NotificationEventType = "prune_report"
+ NotificationEventAutoHeal NotificationEventType = "auto_heal"
)
type EmailTLSMode string
diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.go
index cda7149ee..35ba7e163 100644
--- a/backend/internal/models/settings.go
+++ b/backend/internal/models/settings.go
@@ -70,6 +70,11 @@ type Settings struct {
ScheduledPruneVolumes SettingVariable `key:"scheduledPruneVolumes" meta:"label=Scheduled Prune Volumes;type=boolean;keywords=prune,volumes,cleanup,maintenance;category=internal;description=Remove unused volumes during scheduled prune"`
ScheduledPruneNetworks SettingVariable `key:"scheduledPruneNetworks" meta:"label=Scheduled Prune Networks;type=boolean;keywords=prune,networks,cleanup,maintenance;category=internal;description=Remove unused networks during scheduled prune"`
ScheduledPruneBuildCache SettingVariable `key:"scheduledPruneBuildCache" meta:"label=Scheduled Prune Build Cache;type=boolean;keywords=prune,build cache,cleanup,maintenance;category=internal;description=Remove Docker build cache during scheduled prune"`
+ AutoHealEnabled SettingVariable `key:"autoHealEnabled" meta:"label=Auto Heal;type=boolean;keywords=auto,heal,health,restart,unhealthy,recovery,container,healthcheck;category=internal;description=Automatically restart containers that become unhealthy"`
+ AutoHealInterval SettingVariable `key:"autoHealInterval" meta:"label=Auto Heal Interval;type=cron;keywords=auto,heal,interval,frequency,schedule,health,jobs;description=How often to check container health (cron expression)" catmeta:"id=jobschedule"`
+ AutoHealExcludedContainers SettingVariable `key:"autoHealExcludedContainers" meta:"label=Auto Heal Excluded Containers;type=text;keywords=auto,heal,exclude,containers,ignore,skip,health;category=internal;description=Comma-separated list of containers to exclude from auto-heal"`
+ AutoHealMaxRestarts SettingVariable `key:"autoHealMaxRestarts" meta:"label=Auto Heal Max Restarts;type=number;keywords=auto,heal,max,restarts,limit,loop,protection;category=internal;description=Maximum auto-heal restarts per container within the restart window (default: 5)"`
+ AutoHealRestartWindow SettingVariable `key:"autoHealRestartWindow" meta:"label=Auto Heal Restart Window;type=number;keywords=auto,heal,restart,window,minutes,cooldown,protection;category=internal;description=Time window in minutes for counting auto-heal restarts (default: 30)"`
MaxImageUploadSize SettingVariable `key:"maxImageUploadSize" meta:"label=Max Image Upload Size;type=number;keywords=upload,size,limit,maximum,image,tar,file,megabytes,mb,storage;category=internal;description=Maximum size in MB for image archive uploads (default: 500)"`
DockerHost SettingVariable `key:"dockerHost,public,envOverride" meta:"label=Docker Host;type=text;keywords=docker,host,daemon,socket,unix,remote;category=internal;description=URI for Docker daemon"`
diff --git a/backend/internal/services/apprise_service.go b/backend/internal/services/apprise_service.go
index dce38e852..27f4048e6 100644
--- a/backend/internal/services/apprise_service.go
+++ b/backend/internal/services/apprise_service.go
@@ -113,6 +113,9 @@ func (s *AppriseService) SendNotification(ctx context.Context, title, body, form
case models.NotificationEventVulnerabilityFound:
// No dedicated tag in AppriseSettings; notification is sent without a tag
+
+ case models.NotificationEventAutoHeal:
+ // No dedicated tag in AppriseSettings; notification is sent without a tag
}
payload := AppriseNotificationPayload{
diff --git a/backend/internal/services/job_service.go b/backend/internal/services/job_service.go
index f387695fb..78fad813e 100644
--- a/backend/internal/services/job_service.go
+++ b/backend/internal/services/job_service.go
@@ -62,6 +62,7 @@ func (s *JobService) GetJobSchedules(ctx context.Context) jobschedule.Config {
ScheduledPruneInterval: s.settings.GetStringSetting(ctx, "scheduledPruneInterval", "0 0 0 * * *"),
GitopsSyncInterval: s.settings.GetStringSetting(ctx, "gitopsSyncInterval", "0 */1 * * * *"),
VulnerabilityScanInterval: s.settings.GetStringSetting(ctx, "vulnerabilityScanInterval", "0 0 0 * * *"),
+ AutoHealInterval: s.settings.GetStringSetting(ctx, "autoHealInterval", "*/30 * * * * *"),
}
}
@@ -88,6 +89,7 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule
{key: "scheduledPruneInterval", current: current.ScheduledPruneInterval, update: updates.ScheduledPruneInterval},
{key: "gitopsSyncInterval", current: current.GitopsSyncInterval, update: updates.GitopsSyncInterval},
{key: "vulnerabilityScanInterval", current: current.VulnerabilityScanInterval, update: updates.VulnerabilityScanInterval},
+ {key: "autoHealInterval", current: current.AutoHealInterval, update: updates.AutoHealInterval},
}
// Validate inputs (cron expressions)
@@ -228,6 +230,7 @@ func (s *JobService) getJobScheduleInternal(ctx context.Context, meta meta.JobMe
"scheduledPruneInterval": "0 0 0 * * *",
"gitopsSyncInterval": "0 */1 * * * *",
"vulnerabilityScanInterval": "0 0 0 * * *",
+ "autoHealInterval": "*/30 * * * * *",
}
defaultSchedule := defaultSchedules[meta.SettingsKey]
diff --git a/backend/internal/services/notification_service.go b/backend/internal/services/notification_service.go
index a299f52bb..97e193979 100644
--- a/backend/internal/services/notification_service.go
+++ b/backend/internal/services/notification_service.go
@@ -32,6 +32,7 @@ const (
notificationTestTypeBatchImageUpdate = "batch-image-update"
notificationTestTypeVulnerability = "vulnerability-found"
notificationTestTypePruneReport = "prune-report"
+ notificationTestTypeAutoHeal = "auto-heal"
)
var supportedNotificationTestTypes = map[string]struct{}{
@@ -40,6 +41,7 @@ var supportedNotificationTestTypes = map[string]struct{}{
notificationTestTypeBatchImageUpdate: {},
notificationTestTypeVulnerability: {},
notificationTestTypePruneReport: {},
+ notificationTestTypeAutoHeal: {},
}
// VulnerabilityNotificationPayload is the data sent to all providers for vulnerability_found events.
@@ -903,6 +905,34 @@ func (s *NotificationService) TestNotification(ctx context.Context, provider mod
}
}
+ if testType == notificationTestTypeAutoHeal {
+ testContainerName := "test-container"
+ switch provider {
+ case models.NotificationProviderDiscord:
+ return s.sendDiscordAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderEmail:
+ return s.sendEmailAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderTelegram:
+ return s.sendTelegramAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderSignal:
+ return s.sendSignalAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderSlack:
+ return s.sendSlackAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderNtfy:
+ return s.sendNtfyAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderPushover:
+ return s.sendPushoverAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderGotify:
+ return s.sendGotifyAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderMatrix:
+ return s.sendMatrixAutoHealNotification(ctx, testContainerName, setting.Config)
+ case models.NotificationProviderGeneric:
+ return s.sendGenericAutoHealNotification(ctx, testContainerName, setting.Config)
+ default:
+ return fmt.Errorf("unknown provider: %s", provider)
+ }
+ }
+
if testType == notificationTestTypePruneReport {
result := &system.PruneAllResult{
Success: true,
@@ -3193,6 +3223,184 @@ func (s *NotificationService) sendGenericPruneNotification(ctx context.Context,
return notifications.SendGenericWithTitle(ctx, genericConfig, "System Prune Report", message)
}
+// SendAutoHealNotification sends a notification when a container is auto-healed.
+func (s *NotificationService) SendAutoHealNotification(ctx context.Context, containerName, containerID string) error {
+ settings, err := s.GetAllSettings(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to get notification settings: %w", err)
+ }
+
+ var errs []string
+ for _, setting := range settings {
+ if !setting.Enabled {
+ continue
+ }
+
+ if !s.isEventEnabled(setting.Config, models.NotificationEventAutoHeal) {
+ continue
+ }
+
+ var sendErr error
+ switch setting.Provider {
+ case models.NotificationProviderDiscord:
+ sendErr = s.sendDiscordAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderEmail:
+ sendErr = s.sendEmailAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderTelegram:
+ sendErr = s.sendTelegramAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderSignal:
+ sendErr = s.sendSignalAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderSlack:
+ sendErr = s.sendSlackAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderNtfy:
+ sendErr = s.sendNtfyAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderPushover:
+ sendErr = s.sendPushoverAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderGotify:
+ sendErr = s.sendGotifyAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderMatrix:
+ sendErr = s.sendMatrixAutoHealNotification(ctx, containerName, setting.Config)
+ case models.NotificationProviderGeneric:
+ sendErr = s.sendGenericAutoHealNotification(ctx, containerName, setting.Config)
+ default:
+ slog.WarnContext(ctx, "Unknown notification provider", "provider", setting.Provider)
+ continue
+ }
+
+ status := "success"
+ var errMsg *string
+ if sendErr != nil {
+ status = "failed"
+ msg := sendErr.Error()
+ errMsg = new(msg)
+ errs = append(errs, fmt.Sprintf("%s: %s", setting.Provider, msg))
+ }
+
+ s.logNotification(ctx, setting.Provider, containerName, status, errMsg, models.JSON{
+ "containerID": containerID,
+ "eventType": string(models.NotificationEventAutoHeal),
+ })
+ }
+
+ if len(errs) > 0 {
+ return fmt.Errorf("notification errors: %s", strings.Join(errs, "; "))
+ }
+
+ return nil
+}
+
+func (s *NotificationService) sendDiscordAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var discordConfig models.DiscordConfig
+ if err := s.unmarshalConfigInternal(config, &discordConfig); err != nil {
+ return err
+ }
+ if discordConfig.WebhookID == "" || discordConfig.Token == "" {
+ return fmt.Errorf("discord webhook ID or token not configured")
+ }
+ s.decryptDiscordTokenInternal(&discordConfig)
+ message := fmt.Sprintf("**Container '%s' was automatically restarted because it was unhealthy**", containerName)
+ return notifications.SendDiscord(ctx, discordConfig, message)
+}
+
+func (s *NotificationService) sendEmailAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var emailConfig models.EmailConfig
+ if err := s.unmarshalConfigInternal(config, &emailConfig); err != nil {
+ return err
+ }
+ if err := s.validateEmailConfigInternal(&emailConfig); err != nil {
+ return err
+ }
+ s.decryptEmailPasswordInternal(&emailConfig)
+ subject := fmt.Sprintf("Auto Heal: Container '%s' Restarted", containerName)
+ body := fmt.Sprintf("<p>Container <strong>%s</strong> was automatically restarted because it was unhealthy.</p>", containerName)
+ return notifications.SendEmail(ctx, emailConfig, subject, body)
+}
+
+func (s *NotificationService) sendTelegramAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var telegramConfig models.TelegramConfig
+ if err := s.unmarshalConfigInternal(config, &telegramConfig); err != nil {
+ return err
+ }
+ if telegramConfig.BotToken == "" || len(telegramConfig.ChatIDs) == 0 {
+ return fmt.Errorf("telegram bot token or chat IDs not configured")
+ }
+ s.decryptTelegramTokenInternal(&telegramConfig)
+ if telegramConfig.ParseMode == "" {
+ telegramConfig.ParseMode = "HTML"
+ }
+ message := fmt.Sprintf("<b>Auto Heal:</b> Container '%s' was automatically restarted because it was unhealthy", containerName)
+ return notifications.SendTelegram(ctx, telegramConfig, message)
+}
+
+func (s *NotificationService) sendSignalAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var signalConfig models.SignalConfig
+ if err := s.unmarshalConfigInternal(config, &signalConfig); err != nil {
+ return err
+ }
+ message := fmt.Sprintf("Auto Heal: Container '%s' was automatically restarted because it was unhealthy", containerName)
+ return notifications.SendSignal(ctx, signalConfig, message)
+}
+
+func (s *NotificationService) sendSlackAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var slackConfig models.SlackConfig
+ if err := s.unmarshalConfigInternal(config, &slackConfig); err != nil {
+ return err
+ }
+ message := fmt.Sprintf("*Auto Heal:* Container '%s' was automatically restarted because it was unhealthy", containerName)
+ return notifications.SendSlack(ctx, slackConfig, message)
+}
+
+func (s *NotificationService) sendNtfyAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var ntfyConfig models.NtfyConfig
+ if err := s.unmarshalConfigInternal(config, &ntfyConfig); err != nil {
+ return err
+ }
+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)
+ return notifications.SendNtfy(ctx, ntfyConfig, message)
+}
+
+func (s *NotificationService) sendPushoverAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var pushoverConfig models.PushoverConfig
+ if err := s.unmarshalConfigInternal(config, &pushoverConfig); err != nil {
+ return err
+ }
+ if pushoverConfig.Title == "" {
+ pushoverConfig.Title = "Auto Heal"
+ }
+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)
+ return notifications.SendPushover(ctx, pushoverConfig, message)
+}
+
+func (s *NotificationService) sendGotifyAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var gotifyConfig models.GotifyConfig
+ if err := s.unmarshalConfigInternal(config, &gotifyConfig); err != nil {
+ return err
+ }
+ if gotifyConfig.Title == "" {
+ gotifyConfig.Title = "Auto Heal"
+ }
+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)
+ return notifications.SendGotify(ctx, gotifyConfig, message)
+}
+
+func (s *NotificationService) sendMatrixAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var matrixConfig models.MatrixConfig
+ if err := s.unmarshalConfigInternal(config, &matrixConfig); err != nil {
+ return err
+ }
+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)
+ return notifications.SendMatrix(ctx, matrixConfig, message)
+}
+
+func (s *NotificationService) sendGenericAutoHealNotification(ctx context.Context, containerName string, config models.JSON) error {
+ var genericConfig models.GenericConfig
+ if err := s.unmarshalConfigInternal(config, &genericConfig); err != nil {
+ return err
+ }
+ message := fmt.Sprintf("Container '%s' was automatically restarted because it was unhealthy", containerName)
+ return notifications.SendGenericWithTitle(ctx, genericConfig, "Auto Heal", message)
+}
+
// Helper methods to reduce code duplication
func (s *NotificationService) unmarshalConfigInternal(config models.JSON, dest any) error {
configBytes, err := json.Marshal(config)
diff --git a/backend/internal/services/notification_service_test.go b/backend/internal/services/notification_service_test.go
index d0f4657ae..c2e696c5f 100644
--- a/backend/internal/services/notification_service_test.go
+++ b/backend/internal/services/notification_service_test.go
@@ -299,3 +299,22 @@ func TestNotificationService_MigrateDiscordWebhookUrlToFields_PreservesAllFields
require.False(t, discordConfig.Events[models.NotificationEventImageUpdate])
require.True(t, discordConfig.Events[models.NotificationEventContainerUpdate])
}
+
+func TestSupportedNotificationTestTypes_IncludesAutoHeal(t *testing.T) {
+ expected := []string{
+ notificationTestTypeSimple,
+ notificationTestTypeImageUpdate,
+ notificationTestTypeBatchImageUpdate,
+ notificationTestTypeVulnerability,
+ notificationTestTypePruneReport,
+ notificationTestTypeAutoHeal,
+ }
+
+ for _, tt := range expected {
+ _, ok := supportedNotificationTestTypes[tt]
+ require.True(t, ok, "expected %q to be in supportedNotificationTestTypes", tt)
+ }
+
+ require.Equal(t, len(expected), len(supportedNotificationTestTypes),
+ "supportedNotificationTestTypes has unexpected entries")
+}
diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.go
index 2cc2ecda0..a85c43891 100644
--- a/backend/internal/services/settings_service.go
+++ b/backend/internal/services/settings_service.go
@@ -38,6 +38,7 @@ type SettingsService struct {
OnProjectsDirectoryChanged func(ctx context.Context)
OnScheduledPruneSettingsChanged func(ctx context.Context)
OnVulnerabilityScanSettingsChanged func(ctx context.Context)
+ OnAutoHealSettingsChanged func(ctx context.Context)
OnTimeoutSettingsChanged func(ctx context.Context, timeoutSettings map[string]string)
}
@@ -103,6 +104,11 @@ func (s *SettingsService) getDefaultSettings() *models.Settings {
ScheduledPruneVolumes: models.SettingVariable{Value: "false"},
ScheduledPruneNetworks: models.SettingVariable{Value: "true"},
ScheduledPruneBuildCache: models.SettingVariable{Value: "false"},
+ AutoHealEnabled: models.SettingVariable{Value: "false"},
+ AutoHealInterval: models.SettingVariable{Value: "*/30 * * * * *"},
+ AutoHealExcludedContainers: models.SettingVariable{Value: ""},
+ AutoHealMaxRestarts: models.SettingVariable{Value: "5"},
+ AutoHealRestartWindow: models.SettingVariable{Value: "30"},
GitopsSyncInterval: models.SettingVariable{Value: "0 */1 * * * *"},
BaseServerURL: models.SettingVariable{Value: "http://localhost"},
EnableGravatar: models.SettingVariable{Value: "true"},
@@ -457,7 +463,7 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U
return nil, fmt.Errorf("failed to load current settings: %w", err)
}
- valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedTimeouts, err := s.prepareUpdateValues(updates, cfg, defaultCfg)
+ valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, err := s.prepareUpdateValues(updates, cfg, defaultCfg)
if err != nil {
return nil, err
}
@@ -491,6 +497,9 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U
if changedVulnerabilityScan && s.OnVulnerabilityScanSettingsChanged != nil {
s.OnVulnerabilityScanSettingsChanged(ctx)
}
+ if changedAutoHeal && s.OnAutoHealSettingsChanged != nil {
+ s.OnAutoHealSettingsChanged(ctx)
+ }
if slices.ContainsFunc(valuesToUpdate, func(sv models.SettingVariable) bool { return sv.Key == "projectsDirectory" }) && s.OnProjectsDirectoryChanged != nil {
s.OnProjectsDirectoryChanged(ctx)
}
@@ -517,7 +526,7 @@ var timeoutSettingKeys = []string{
"trivyConcurrentScanContainers",
}
-func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *models.Settings) ([]models.SettingVariable, bool, bool, bool, bool, map[string]string, error) {
+func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *models.Settings) ([]models.SettingVariable, bool, bool, bool, bool, bool, map[string]string, error) {
rt := reflect.TypeFor[settings.Update]()
rv := reflect.ValueOf(updates)
valuesToUpdate := make([]models.SettingVariable, 0)
@@ -526,6 +535,7 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
changedAutoUpdate := false
changedScheduledPrune := false
changedVulnerabilityScan := false
+ changedAutoHeal := false
changedTimeouts := make(map[string]string)
for i := 0; i < rt.NumField(); i++ {
@@ -543,10 +553,10 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
}
// Validate cron settings
- cronFields := []string{"scheduledPruneInterval", "autoUpdateInterval", "pollingInterval", "environmentHealthInterval", "eventCleanupInterval", "analyticsHeartbeatInterval", "vulnerabilityScanInterval", "gitopsSyncInterval"}
+ cronFields := []string{"scheduledPruneInterval", "autoUpdateInterval", "pollingInterval", "environmentHealthInterval", "eventCleanupInterval", "analyticsHeartbeatInterval", "vulnerabilityScanInterval", "gitopsSyncInterval", "autoHealInterval"}
if slices.Contains(cronFields, key) && value != "" {
if _, err := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow).Parse(value); err != nil {
- return nil, false, false, false, false, nil, fmt.Errorf("invalid cron expression for %s: %w", key, err)
+ return nil, false, false, false, false, false, nil, fmt.Errorf("invalid cron expression for %s: %w", key, err)
}
}
@@ -565,7 +575,7 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
if errors.Is(err, models.SettingSensitiveForbiddenError{}) {
continue
} else if err != nil {
- return nil, false, false, false, false, nil, fmt.Errorf("failed to update in-memory config for key '%s': %w", key, err)
+ return nil, false, false, false, false, false, nil, fmt.Errorf("failed to update in-memory config for key '%s': %w", key, err)
}
valuesToUpdate = append(valuesToUpdate, models.SettingVariable{
@@ -582,6 +592,8 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
changedScheduledPrune = true
case "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers":
changedVulnerabilityScan = true
+ case "autoHealEnabled", "autoHealInterval", "autoHealExcludedContainers", "autoHealMaxRestarts", "autoHealRestartWindow":
+ changedAutoHeal = true
}
// Track timeout setting changes
@@ -590,7 +602,7 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
}
}
- return valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedTimeouts, nil
+ return valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, nil
}
func (s *SettingsService) persistSettings(ctx context.Context, values []models.SettingVariable) error {
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
new file mode 100644
index 000000000..c03c98af5
--- /dev/null
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -0,0 +1,267 @@
+package scheduler
+
+import (
+ "context"
+ "log/slog"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/docker/docker/api/types/container"
+ "github.com/getarcaneapp/arcane/backend/internal/models"
+ "github.com/getarcaneapp/arcane/backend/internal/services"
+ "github.com/getarcaneapp/arcane/backend/pkg/libarcane"
+ "github.com/robfig/cron/v3"
+)
+
+const AutoHealJobName = "auto-heal"
+
+// restartRecord tracks restart timestamps for a single container.
+type restartRecord struct {
+ timestamps []time.Time
+}
+
+type AutoHealJob struct {
+ dockerClientService *services.DockerClientService
+ settingsService *services.SettingsService
+ eventService *services.EventService
+ notificationService *services.NotificationService
+
+ mu sync.Mutex
+ restarts map[string]*restartRecord
+}
+
+func NewAutoHealJob(
+ dockerClientService *services.DockerClientService,
+ settingsService *services.SettingsService,
+ eventService *services.EventService,
+ notificationService *services.NotificationService,
+) *AutoHealJob {
+ return &AutoHealJob{
+ dockerClientService: dockerClientService,
+ settingsService: settingsService,
+ eventService: eventService,
+ notificationService: notificationService,
+ restarts: make(map[string]*restartRecord),
+ }
+}
+
+func (j *AutoHealJob) Name() string {
+ return AutoHealJobName
+}
+
+func (j *AutoHealJob) Schedule(ctx context.Context) string {
+ schedule := j.settingsService.GetStringSetting(ctx, "autoHealInterval", "*/30 * * * * *")
+ if schedule == "" {
+ schedule = "*/30 * * * * *"
+ }
+
+ parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ if _, err := parser.Parse(schedule); err != nil {
+ slog.WarnContext(ctx, "Invalid cron expression for auto-heal, using default", "invalid_schedule", schedule, "error", err)
+ return "*/30 * * * * *"
+ }
+
+ return schedule
+}
+
+func (j *AutoHealJob) Run(ctx context.Context) {
+ enabled := j.settingsService.GetBoolSetting(ctx, "autoHealEnabled", false)
+ if !enabled {
+ slog.DebugContext(ctx, "auto-heal disabled; skipping run")
+ return
+ }
+
+ dockerClient, err := j.dockerClientService.GetClient()
+ if err != nil {
+ slog.ErrorContext(ctx, "auto-heal failed to get Docker client", "error", err)
+ return
+ }
+
+ containers, err := dockerClient.ContainerList(ctx, container.ListOptions{All: false})
+ if err != nil {
+ slog.ErrorContext(ctx, "auto-heal failed to list containers", "error", err)
+ return
+ }
+
+ excludedContainers := j.parseExcludedContainers(ctx)
+ maxRestarts := j.settingsService.GetIntSetting(ctx, "autoHealMaxRestarts", 5)
+ restartWindowMinutes := j.settingsService.GetIntSetting(ctx, "autoHealRestartWindow", 30)
+ restartWindow := time.Duration(restartWindowMinutes) * time.Minute
+
+ for _, c := range containers {
+ // Skip Arcane internal containers
+ if libarcane.IsInternalContainer(c.Labels) {
+ continue
+ }
+
+ containerName := j.getContainerName(c.Names)
+
+ // Skip excluded containers
+ if j.isExcluded(containerName, excludedContainers) {
+ continue
+ }
+
+ // Inspect to get health status
+ inspect, err := dockerClient.ContainerInspect(ctx, c.ID)
+ if err != nil {
+ slog.WarnContext(ctx, "auto-heal failed to inspect container", "container", containerName, "error", err)
+ continue
+ }
+
+ // Skip if no healthcheck configured or not unhealthy
+ if inspect.State == nil || inspect.State.Health == nil {
+ continue
+ }
+ if inspect.State.Health.Status != "unhealthy" {
+ continue
+ }
+
+ // Check restart-loop protection
+ if !j.canRestart(c.ID, maxRestarts, restartWindow) {
+ slog.WarnContext(ctx, "auto-heal restart-loop protection: skipping container",
+ "container", containerName,
+ "max_restarts", maxRestarts,
+ "window_minutes", restartWindowMinutes,
+ )
+ continue
+ }
+
+ // Restart the container
+ slog.InfoContext(ctx, "auto-heal restarting unhealthy container", "container", containerName, "container_id", c.ID)
+ if err := dockerClient.ContainerRestart(ctx, c.ID, container.StopOptions{}); err != nil {
+ slog.ErrorContext(ctx, "auto-heal failed to restart container", "container", containerName, "error", err)
+ continue
+ }
+
+ // Record the restart
+ j.recordRestart(c.ID)
+
+ // Log event
+ if err := j.eventService.LogContainerEvent(
+ ctx,
+ models.EventTypeContainerRestart,
+ c.ID,
+ containerName,
+ "", // no user - system action
+ "system",
+ "",
+ models.JSON{"action": "auto-heal", "reason": "unhealthy"},
+ ); err != nil {
+ slog.WarnContext(ctx, "auto-heal failed to log event", "container", containerName, "error", err)
+ }
+
+ // Send notification
+ if err := j.notificationService.SendAutoHealNotification(ctx, containerName, c.ID); err != nil {
+ slog.WarnContext(ctx, "auto-heal failed to send notification", "container", containerName, "error", err)
+ }
+
+ slog.InfoContext(ctx, "auto-heal successfully restarted container", "container", containerName)
+ }
+}
+
+func (j *AutoHealJob) Reschedule(ctx context.Context) error {
+ slog.InfoContext(ctx, "rescheduling auto-heal job in new scheduler; currently requires restart")
+ return nil
+}
+
+// canRestart checks if a container can be restarted within the rate limit.
+func (j *AutoHealJob) canRestart(containerID string, maxRestarts int, window time.Duration) bool {
+ j.mu.Lock()
+ defer j.mu.Unlock()
+
+ record, exists := j.restarts[containerID]
+ if !exists {
+ return true
+ }
+
+ cutoff := time.Now().Add(-window)
+ recent := j.pruneTimestamps(record.timestamps, cutoff)
+ record.timestamps = recent
+
+ return len(recent) < maxRestarts
+}
+
+// recordRestart records a restart timestamp for a container.
+func (j *AutoHealJob) recordRestart(containerID string) {
+ j.mu.Lock()
+ defer j.mu.Unlock()
+
+ record, exists := j.restarts[containerID]
+ if !exists {
+ record = &restartRecord{}
+ j.restarts[containerID] = record
+ }
+
+ record.timestamps = append(record.timestamps, time.Now())
+}
+
+// pruneTimestamps removes timestamps older than the cutoff.
+func (j *AutoHealJob) pruneTimestamps(timestamps []time.Time, cutoff time.Time) []time.Time {
+ result := make([]time.Time, 0, len(timestamps))
+ for _, ts := range timestamps {
+ if ts.After(cutoff) {
+ result = append(result, ts)
+ }
+ }
+ return result
+}
+
+func (j *AutoHealJob) parseExcludedContainers(ctx context.Context) map[string]struct{} {
+ raw := j.settingsService.GetStringSetting(ctx, "autoHealExcludedContainers", "")
+ excluded := make(map[string]struct{})
+ if raw == "" {
+ return excluded
+ }
+ for _, name := range strings.Split(raw, ",") {
+ trimmed := strings.TrimSpace(name)
+ if trimmed != "" {
+ excluded[trimmed] = struct{}{}
+ }
+ }
+ return excluded
+}
+
+func (j *AutoHealJob) getContainerName(names []string) string {
+ if len(names) == 0 {
+ return ""
+ }
+ // Docker container names are prefixed with "/"
+ return strings.TrimPrefix(names[0], "/")
+}
+
+func (j *AutoHealJob) isExcluded(name string, excluded map[string]struct{}) bool {
+ _, ok := excluded[name]
+ return ok
+}
+
+// ResetRestartTracking clears all restart records (exported for testing).
+func (j *AutoHealJob) ResetRestartTracking() {
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ j.restarts = make(map[string]*restartRecord)
+}
+
+// CanRestartExported exposes canRestart for testing.
+func (j *AutoHealJob) CanRestartExported(containerID string, maxRestarts int, window time.Duration) bool {
+ return j.canRestart(containerID, maxRestarts, window)
+}
+
+// RecordRestartExported exposes recordRestart for testing.
+func (j *AutoHealJob) RecordRestartExported(containerID string) {
+ j.recordRestart(containerID)
+}
+
+// RecordRestartAtExported records a restart at a specific time for testing.
+func (j *AutoHealJob) RecordRestartAtExported(containerID string, t time.Time) {
+ j.mu.Lock()
+ defer j.mu.Unlock()
+
+ record, exists := j.restarts[containerID]
+ if !exists {
+ record = &restartRecord{}
+ j.restarts[containerID] = record
+ }
+
+ record.timestamps = append(record.timestamps, t)
+}
diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.go
new file mode 100644
index 000000000..9b3c05176
--- /dev/null
+++ b/backend/pkg/scheduler/auto_heal_job_test.go
@@ -0,0 +1,114 @@
+package scheduler
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func newTestAutoHealJob() *AutoHealJob {
+ return &AutoHealJob{
+ restarts: make(map[string]*restartRecord),
+ }
+}
+
+func TestAutoHeal_CanRestart_UnderLimit(t *testing.T) {
+ job := newTestAutoHealJob()
+
+ // No restarts recorded yet — should be allowed
+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+
+ // Record 4 restarts (under limit of 5)
+ for i := 0; i < 4; i++ {
+ job.RecordRestartExported("container-1")
+ }
+
+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+}
+
+func TestAutoHeal_CanRestart_AtLimit(t *testing.T) {
+ job := newTestAutoHealJob()
+
+ // Record exactly 5 restarts (at limit)
+ for i := 0; i < 5; i++ {
+ job.RecordRestartExported("container-1")
+ }
+
+ require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+}
+
+func TestAutoHeal_CanRestart_WindowExpiry(t *testing.T) {
+ job := newTestAutoHealJob()
+
+ // Record 5 restarts 31 minutes ago (outside window)
+ oldTime := time.Now().Add(-31 * time.Minute)
+ for i := 0; i < 5; i++ {
+ job.RecordRestartAtExported("container-1", oldTime)
+ }
+
+ // Should be allowed because all timestamps are outside the 30-minute window
+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+}
+
+func TestAutoHeal_CanRestart_MixedTimestamps(t *testing.T) {
+ job := newTestAutoHealJob()
+
+ // Record 3 old restarts (outside window)
+ oldTime := time.Now().Add(-31 * time.Minute)
+ for i := 0; i < 3; i++ {
+ job.RecordRestartAtExported("container-1", oldTime)
+ }
+
+ // Record 4 recent restarts (inside window)
+ for i := 0; i < 4; i++ {
+ job.RecordRestartExported("container-1")
+ }
+
+ // Should still be allowed (only 4 recent, limit is 5)
+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+
+ // Add one more recent restart
+ job.RecordRestartExported("container-1")
+
+ // Now should be blocked (5 recent)
+ require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+}
+
+func TestAutoHeal_CanRestart_DifferentContainers(t *testing.T) {
+ job := newTestAutoHealJob()
+
+ // Fill up container-1
+ for i := 0; i < 5; i++ {
+ job.RecordRestartExported("container-1")
+ }
+
+ // container-1 should be blocked
+ require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+
+ // container-2 should still be allowed
+ require.True(t, job.CanRestartExported("container-2", 5, 30*time.Minute))
+}
+
+func TestAutoHeal_Schedule_Default(t *testing.T) {
+ job := newTestAutoHealJob()
+ // Without a settings service, Schedule would panic.
+ // We test the Name() method directly.
+ require.Equal(t, "auto-heal", job.Name())
+}
+
+func TestAutoHeal_ResetRestartTracking(t *testing.T) {
+ job := newTestAutoHealJob()
+
+ // Fill up container-1
+ for i := 0; i < 5; i++ {
+ job.RecordRestartExported("container-1")
+ }
+ require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+
+ // Reset tracking
+ job.ResetRestartTracking()
+
+ // Should be allowed again
+ require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
+}
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 29af18f68..14e8c4f51 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -185,6 +185,12 @@
"auto_update_containers_excluded_many": "{count} containers excluded",
"auto_update_exclude_description": "Select containers to exclude from automatic updates.",
"auto_update_excluded_containers": "Excluded Containers",
+ "auto_heal_excluded_containers": "Excluded Containers",
+ "auto_heal_exclude_description": "Select containers to exclude from auto-heal monitoring.",
+ "auto_heal_max_restarts_label": "Max Restarts",
+ "auto_heal_max_restarts_description": "Maximum number of automatic restarts per container within the restart window.",
+ "auto_heal_restart_window_label": "Restart Window (minutes)",
+ "auto_heal_restart_window_description": "Time window in minutes for counting automatic restarts. Resets after this period.",
"_comment_common_actions": "=== COMMON - ACTION STATES ===",
"common_action_creating": "Creating…",
"common_action_deleting": "Deleting…",
@@ -1705,6 +1711,7 @@
"notifications_email_test_vulnerability_found": "Vulnerability Found Notification",
"notifications_test_vulnerability_notification": "Test Vulnerability Notification",
"notifications_test_prune_report_notification": "Test Prune Report Notification",
+ "notifications_test_auto_heal_notification": "Test Auto-Heal Notification",
"notifications_apprise_title": "Apprise Integration",
"notifications_apprise_description": "Configure Apprise API for unified notifications across multiple services",
"notifications_apprise_enabled_label": "Enable Apprise",
@@ -2004,5 +2011,7 @@
"project_includes": "Includes",
"notifications_event_prune_report_label": "System Prune Report",
"notifications_event_prune_report_description": "Receive a summary report when a scheduled prune operation completes",
+ "notifications_event_auto_heal_label": "Auto-Heal Restart",
+ "notifications_event_auto_heal_description": "Notify when an unhealthy container is automatically restarted",
"version_info_build_features": "Build Features"
}
diff --git a/frontend/src/lib/types/job-schedule.type.ts b/frontend/src/lib/types/job-schedule.type.ts
index 42c713078..172053fd0 100644
--- a/frontend/src/lib/types/job-schedule.type.ts
+++ b/frontend/src/lib/types/job-schedule.type.ts
@@ -7,6 +7,7 @@ export type JobSchedules = {
scheduledPruneInterval: string;
gitopsSyncInterval: string;
vulnerabilityScanInterval: string;
+ autoHealInterval: string;
};
export type JobSchedulesUpdate = Partial<JobSchedules>;
diff --git a/frontend/src/lib/types/notification-providers.ts b/frontend/src/lib/types/notification-providers.ts
index 716e0912d..452ddec79 100644
--- a/frontend/src/lib/types/notification-providers.ts
+++ b/frontend/src/lib/types/notification-providers.ts
@@ -22,6 +22,7 @@ export interface BaseProviderFormValues {
eventContainerUpdate: boolean;
eventVulnerabilityFound: boolean;
eventPruneReport: boolean;
+ eventAutoHeal: boolean;
}
// Provider-specific form value types
@@ -177,7 +178,8 @@ export function discordSettingsToFormValues(settings?: NotificationSettings): Di
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -196,7 +198,8 @@ export function emailSettingsToFormValues(settings?: NotificationSettings): Emai
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -213,7 +216,8 @@ export function telegramSettingsToFormValues(settings?: NotificationSettings): T
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -233,7 +237,8 @@ export function signalSettingsToFormValues(settings?: NotificationSettings): Sig
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -252,7 +257,8 @@ export function slackSettingsToFormValues(settings?: NotificationSettings): Slac
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -279,7 +285,8 @@ export function discordFormValuesToSettings(values: DiscordFormValues): Notifica
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -304,7 +311,8 @@ export function emailFormValuesToSettings(values: EmailFormValues): Notification
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -327,7 +335,8 @@ export function telegramFormValuesToSettings(values: TelegramFormValues): Notifi
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -353,7 +362,8 @@ export function signalFormValuesToSettings(values: SignalFormValues): Notificati
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -375,7 +385,8 @@ export function slackFormValuesToSettings(values: SlackFormValues): Notification
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -400,7 +411,8 @@ export function ntfySettingsToFormValues(settings?: NotificationSettings): NtfyF
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -417,7 +429,8 @@ export function pushoverSettingsToFormValues(settings?: NotificationSettings): P
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -436,7 +449,8 @@ export function gotifySettingsToFormValues(settings?: NotificationSettings): Got
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -454,7 +468,8 @@ export function matrixSettingsToFormValues(settings?: NotificationSettings): Mat
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -479,7 +494,8 @@ export function genericSettingsToFormValues(settings?: NotificationSettings): Ge
eventImageUpdate: events?.image_update ?? true,
eventContainerUpdate: events?.container_update ?? true,
eventVulnerabilityFound: events?.vulnerability_found ?? true,
- eventPruneReport: events?.prune_report ?? true
+ eventPruneReport: events?.prune_report ?? true,
+ eventAutoHeal: events?.auto_heal ?? true
};
}
@@ -506,7 +522,8 @@ export function ntfyFormValuesToSettings(values: NtfyFormValues): NotificationSe
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -529,7 +546,8 @@ export function pushoverFormValuesToSettings(values: PushoverFormValues): Notifi
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -551,7 +569,8 @@ export function gotifyFormValuesToSettings(values: GotifyFormValues): Notificati
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -572,7 +591,8 @@ export function matrixFormValuesToSettings(values: MatrixFormValues): Notificati
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
@@ -608,7 +628,8 @@ export function genericFormValuesToSettings(values: GenericFormValues): Notifica
image_update: values.eventImageUpdate,
container_update: values.eventContainerUpdate,
vulnerability_found: values.eventVulnerabilityFound,
- prune_report: values.eventPruneReport
+ prune_report: values.eventPruneReport,
+ auto_heal: values.eventAutoHeal
}
}
};
diff --git a/frontend/src/lib/types/settings.type.ts b/frontend/src/lib/types/settings.type.ts
index 62c2c373b..510ecf2d2 100644
--- a/frontend/src/lib/types/settings.type.ts
+++ b/frontend/src/lib/types/settings.type.ts
@@ -19,6 +19,10 @@ export type Settings = {
scheduledPruneBuildCache?: boolean;
vulnerabilityScanEnabled?: boolean;
vulnerabilityScanInterval?: number;
+ autoHealEnabled?: boolean;
+ autoHealExcludedContainers?: string;
+ autoHealMaxRestarts?: number;
+ autoHealRestartWindow?: number;
maxImageUploadSize: number;
baseServerUrl: string;
enableGravatar: boolean;
diff --git a/frontend/src/routes/(app)/environments/[id]/+page.svelte b/frontend/src/routes/(app)/environments/[id]/+page.svelte
index 14f51d2d6..239b439d3 100644
--- a/frontend/src/routes/(app)/environments/[id]/+page.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/+page.svelte
@@ -149,7 +149,11 @@
scheduledPruneNetworks: z.boolean(),
scheduledPruneBuildCache: z.boolean(),
vulnerabilityScanEnabled: z.boolean(),
- autoUpdateExcludedContainers: z.string()
+ autoUpdateExcludedContainers: z.string(),
+ autoHealEnabled: z.boolean(),
+ autoHealExcludedContainers: z.string(),
+ autoHealMaxRestarts: z.coerce.number().int().min(1),
+ autoHealRestartWindow: z.coerce.number().int().min(1)
});
// Build current settings object from environment and settings data
@@ -173,7 +177,11 @@
scheduledPruneNetworks: settings?.scheduledPruneNetworks ?? true,
scheduledPruneBuildCache: settings?.scheduledPruneBuildCache ?? false,
vulnerabilityScanEnabled: settings?.vulnerabilityScanEnabled ?? false,
- autoUpdateExcludedContainers: settings?.autoUpdateExcludedContainers || ''
+ autoUpdateExcludedContainers: settings?.autoUpdateExcludedContainers || '',
+ autoHealEnabled: settings?.autoHealEnabled ?? false,
+ autoHealExcludedContainers: settings?.autoHealExcludedContainers || '',
+ autoHealMaxRestarts: settings?.autoHealMaxRestarts ?? 5,
+ autoHealRestartWindow: settings?.autoHealRestartWindow ?? 30
});
// Custom save handler for environment-specific settings
@@ -204,7 +212,11 @@
scheduledPruneNetworks: formData.scheduledPruneNetworks,
scheduledPruneBuildCache: formData.scheduledPruneBuildCache,
vulnerabilityScanEnabled: formData.vulnerabilityScanEnabled,
- autoUpdateExcludedContainers: formData.autoUpdateExcludedContainers
+ autoUpdateExcludedContainers: formData.autoUpdateExcludedContainers,
+ autoHealEnabled: formData.autoHealEnabled,
+ autoHealExcludedContainers: formData.autoHealExcludedContainers,
+ autoHealMaxRestarts: formData.autoHealMaxRestarts,
+ autoHealRestartWindow: formData.autoHealRestartWindow
});
}
diff --git a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
index ad484b330..c528b0910 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
@@ -43,7 +43,7 @@
});
const containersPromise = $derived.by(async () => {
- if (!$formInputs.autoUpdate.value) return [];
+ if (!$formInputs.autoUpdate.value && !$formInputs.autoHealEnabled.value) return [];
const result = await tryCatch(containerService.getContainers({ pagination: { page: 1, limit: 100 } }));
if (result.error) throw result.error;
return result.data.data;
@@ -83,6 +83,8 @@
return undefined;
case 'vulnerabilityScanEnabled':
return undefined;
+ case 'autoHealEnabled':
+ return `${envBase}?tab=jobs`;
default:
return prereq.settingsUrl;
}
@@ -106,6 +108,46 @@
}
}
+ const autoHealExcludedContainers = new SvelteSet<string>();
+ let autoHealSearchTerm = $state('');
+
+ $effect(() => {
+ const savedValue = $formInputs.autoHealExcludedContainers?.value || '';
+ const names = savedValue
+ .split(',')
+ .map((s: string) => normalizeContainerName(s.trim()))
+ .filter(Boolean);
+
+ const currentNames = Array.from(autoHealExcludedContainers);
+ if (names.length !== currentNames.length || names.some((n: string) => !autoHealExcludedContainers.has(n))) {
+ autoHealExcludedContainers.clear();
+ names.forEach((n: string) => autoHealExcludedContainers.add(n));
+ }
+ });
+
+ function toggleAutoHealContainerExclusion(containerName: string) {
+ const normalizedName = normalizeContainerName(containerName);
+ if (autoHealExcludedContainers.has(normalizedName)) {
+ autoHealExcludedContainers.delete(normalizedName);
+ } else {
+ autoHealExcludedContainers.add(normalizedName);
+ }
+
+ const newValue = Array.from(autoHealExcludedContainers).join(',');
+ if ($formInputs.autoHealExcludedContainers) {
+ $formInputs.autoHealExcludedContainers.value = newValue;
+ }
+ }
+
+ function mapContainerToAutoHealItem(container: ContainerSummaryDto) {
+ const name = getContainerName(container);
+ return {
+ value: name,
+ label: name,
+ selected: autoHealExcludedContainers.has(name)
+ };
+ }
+
const categories = [
{ id: 'monitoring', label: m.jobs_monitoring_heading() },
{ id: 'maintenance', label: m.jobs_maintenance_heading() },
@@ -137,6 +179,8 @@
return $formInputs.pollingEnabled.value;
case 'vulnerability-scan':
return $formInputs.vulnerabilityScanEnabled.value;
+ case 'auto-heal':
+ return $formInputs.autoHealEnabled.value;
default:
return undefined;
}
@@ -217,6 +261,8 @@
<Switch bind:checked={$formInputs.scheduledPruneEnabled.value} />
{:else if job.id === 'vulnerability-scan'}
<Switch bind:checked={$formInputs.vulnerabilityScanEnabled.value} />
+ {:else if job.id === 'auto-heal'}
+ <Switch bind:checked={$formInputs.autoHealEnabled.value} />
{/if}
{/snippet}
@@ -284,6 +330,99 @@
</div>
{/if}
+ {#if job.id === 'auto-heal' && $formInputs.autoHealEnabled.value}
+ <div class="border-border/20 space-y-3 border-t pt-3">
+ <div class="grid gap-3 sm:grid-cols-2">
+ <div class="space-y-1">
+ <Label for="auto-heal-max-restarts" class="text-sm font-medium"
+ >{m.auto_heal_max_restarts_label()}</Label
+ >
+ <p class="text-muted-foreground text-xs">{m.auto_heal_max_restarts_description()}</p>
+ <Input
+ id="auto-heal-max-restarts"
+ type="number"
+ min="1"
+ class="h-8 w-full"
+ bind:value={$formInputs.autoHealMaxRestarts.value}
+ />
+ </div>
+ <div class="space-y-1">
+ <Label for="auto-heal-restart-window" class="text-sm font-medium"
+ >{m.auto_heal_restart_window_label()}</Label
+ >
+ <p class="text-muted-foreground text-xs">{m.auto_heal_restart_window_description()}</p>
+ <Input
+ id="auto-heal-restart-window"
+ type="number"
+ min="1"
+ class="h-8 w-full"
+ bind:value={$formInputs.autoHealRestartWindow.value}
+ />
+ </div>
+ </div>
+
+ <div class="space-y-1">
+ <Label class="text-sm font-medium">
+ {m.auto_heal_excluded_containers()}
+ {#await containersPromise then containers}
+ <span class="text-muted-foreground ml-1 font-normal">
+ ({containers.filter((c) => autoHealExcludedContainers.has(getContainerName(c))).length})
+ </span>
+ {/await}
+ </Label>
+ <p class="text-muted-foreground text-xs">{m.auto_heal_exclude_description()}</p>
+ </div>
+
+ <div class="rounded-md border p-2">
+ <Input
+ type="search"
+ placeholder="Search containers..."
+ class="mb-2 h-8"
+ bind:value={autoHealSearchTerm}
+ />
+ <ScrollArea.Root class="h-64 w-full rounded-md border p-2">
+ <div class="space-y-2">
+ {#await containersPromise}
+ <div class="flex items-center justify-center p-4">
+ <Spinner class="size-4" />
+ </div>
+ {:then containers}
+ {@const allItems = containers.map(mapContainerToAutoHealItem)}
+ {@const filteredItems = autoHealSearchTerm
+ ? allItems.filter((item) =>
+ item.label.toLowerCase().includes(autoHealSearchTerm.toLowerCase())
+ )
+ : allItems}
+
+ {#if filteredItems.length === 0}
+ <p class="text-muted-foreground py-4 text-center text-sm">
+ {m.common_no_results_found()}
+ </p>
+ {:else}
+ {#each filteredItems as container (container.value)}
+ <div class="flex items-center space-x-2">
+ <Checkbox
+ id="auto-heal-container-{container.value}"
+ checked={container.selected}
+ onCheckedChange={() => toggleAutoHealContainerExclusion(container.value)}
+ />
+ <Label for="auto-heal-container-{container.value}" class="text-sm font-normal">
+ {container.label}
+ </Label>
+ </div>
+ {/each}
+ {/if}
+ {:catch error}
+ <div class="text-destructive p-2 text-sm">
+ {error.message || 'Failed to load containers'}
+ </div>
+ {/await}
+ </div>
+ </ScrollArea.Root>
+ </div>
+ </div>
+ {/if}
+
{#if job.id === 'scheduled-prune'}
{#if $formInputs.scheduledPruneEnabled.value}
<div class="border-border/20 space-y-4 border-t pt-3">
diff --git a/frontend/src/routes/(app)/settings/notifications/providers/BuiltInProviderForm.svelte b/frontend/src/routes/(app)/settings/notifications/providers/BuiltInProviderForm.svelte
index 05d517101..582c1d7ae 100644
--- a/frontend/src/routes/(app)/settings/notifications/providers/BuiltInProviderForm.svelte
+++ b/frontend/src/routes/(app)/settings/notifications/providers/BuiltInProviderForm.svelte
@@ -99,7 +99,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -123,7 +124,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -174,7 +176,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -203,7 +206,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -256,7 +260,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -281,7 +286,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -303,7 +309,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -327,7 +334,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -350,7 +358,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -370,7 +379,8 @@
eventImageUpdate: z.boolean(),
eventContainerUpdate: z.boolean(),
eventVulnerabilityFound: z.boolean(),
- eventPruneReport: z.boolean()
+ eventPruneReport: z.boolean(),
+ eventAutoHeal: z.boolean()
})
.superRefine((d, ctx) => {
if (!d.enabled) return;
@@ -1057,7 +1067,8 @@
{ label: m.notifications_email_test_image_update(), testType: 'image-update' },
{ label: m.notifications_email_test_batch_image_update(), testType: 'batch-image-update' },
{ label: m.notifications_test_vulnerability_notification(), testType: 'vulnerability-found' },
- { label: m.notifications_test_prune_report_notification(), testType: 'prune-report' }
+ { label: m.notifications_test_prune_report_notification(), testType: 'prune-report' },
+ { label: m.notifications_test_auto_heal_notification(), testType: 'auto-heal' }
];
const validation = $derived.by(() => providerSchemas[provider].safeParse(values));
@@ -1088,6 +1099,7 @@
bind:eventContainerUpdate={values.eventContainerUpdate}
bind:eventVulnerabilityFound={values.eventVulnerabilityFound}
bind:eventPruneReport={values.eventPruneReport}
+ bind:eventAutoHeal={values.eventAutoHeal}
{disabled}
/>
diff --git a/frontend/src/routes/(app)/settings/notifications/providers/EventSubscriptions.svelte b/frontend/src/routes/(app)/settings/notifications/providers/EventSubscriptions.svelte
index a09776ba7..3cc2c5cc2 100644
--- a/frontend/src/routes/(app)/settings/notifications/providers/EventSubscriptions.svelte
+++ b/frontend/src/routes/(app)/settings/notifications/providers/EventSubscriptions.svelte
@@ -9,6 +9,7 @@
eventContainerUpdate: boolean;
eventVulnerabilityFound: boolean;
eventPruneReport: boolean;
+ eventAutoHeal: boolean;
disabled?: boolean;
}
@@ -18,6 +19,7 @@
eventContainerUpdate = $bindable(),
eventVulnerabilityFound = $bindable(),
eventPruneReport = $bindable(),
+ eventAutoHeal = $bindable(),
disabled = false
}: Props = $props();
</script>
@@ -54,5 +56,12 @@
label={m.notifications_event_vulnerability_found_label()}
description={m.notifications_event_vulnerability_found_description()}
/>
+ <SwitchWithLabel
+ id="{providerId}-event-auto-heal"
+ bind:checked={eventAutoHeal}
+ {disabled}
+ label={m.notifications_event_auto_heal_label()}
+ description={m.notifications_event_auto_heal_description()}
+ />
</div>
</div>
diff --git a/frontend/src/routes/(app)/settings/notifications/providers/NotificationProviderTestMenu.svelte b/frontend/src/routes/(app)/settings/notifications/providers/NotificationProviderTestMenu.svelte
index 330621852..0bd210952 100644
--- a/frontend/src/routes/(app)/settings/notifications/providers/NotificationProviderTestMenu.svelte
+++ b/frontend/src/routes/(app)/settings/notifications/providers/NotificationProviderTestMenu.svelte
@@ -25,7 +25,8 @@
{ label: m.notifications_email_test_image_update(), testType: 'image-update' },
{ label: m.notifications_email_test_batch_image_update(), testType: 'batch-image-update' },
{ label: m.notifications_test_vulnerability_notification(), testType: 'vulnerability-found' },
- { label: m.notifications_test_prune_report_notification(), testType: 'prune-report' }
+ { label: m.notifications_test_prune_report_notification(), testType: 'prune-report' },
+ { label: m.notifications_test_auto_heal_notification(), testType: 'auto-heal' }
]
}: Props = $props();
</script>
diff --git a/types/jobschedule/jobschedule.go b/types/jobschedule/jobschedule.go
index d9438e20d..251a208d3 100644
--- a/types/jobschedule/jobschedule.go
+++ b/types/jobschedule/jobschedule.go
@@ -15,6 +15,7 @@ type Config struct {
ScheduledPruneInterval string `json:"scheduledPruneInterval"`
GitopsSyncInterval string `json:"gitopsSyncInterval"`
VulnerabilityScanInterval string `json:"vulnerabilityScanInterval"`
+ AutoHealInterval string `json:"autoHealInterval"`
}
// Update is used to update job schedule intervals (in minutes).
@@ -29,6 +30,7 @@ type Update struct {
ScheduledPruneInterval *string `json:"scheduledPruneInterval,omitempty"`
GitopsSyncInterval *string `json:"gitopsSyncInterval,omitempty"`
VulnerabilityScanInterval *string `json:"vulnerabilityScanInterval,omitempty"`
+ AutoHealInterval *string `json:"autoHealInterval,omitempty"`
}
// JobStatus represents the current status and metadata for a background job.
diff --git a/types/meta/job_metadata.go b/types/meta/job_metadata.go
index 30296e456..c1afcdcbd 100644
--- a/types/meta/job_metadata.go
+++ b/types/meta/job_metadata.go
@@ -171,6 +171,24 @@ var jobMetadataRegistry = map[string]JobMetadata{
},
},
},
+ "auto-heal": {
+ ID: "auto-heal",
+ Name: "Auto Heal",
+ Description: "Automatically restarts containers that become unhealthy",
+ Category: "monitoring",
+ SettingsKey: "autoHealInterval",
+ EnabledKey: "autoHealEnabled",
+ ManagerOnly: false,
+ IsContinuous: false,
+ CanRunManually: true,
+ Prerequisites: []JobPrerequisiteMetadata{
+ {
+ SettingKey: "autoHealEnabled",
+ Label: "Auto heal enabled",
+ SettingsURL: "/settings/general",
+ },
+ },
+ },
}
func GetJobMetadata(jobID string) (JobMetadata, bool) {
diff --git a/types/settings/settings.go b/types/settings/settings.go
index d962c21ae..1d0052d46 100644
--- a/types/settings/settings.go
+++ b/types/settings/settings.go
@@ -319,4 +319,29 @@ type Update struct {
//
// Required: false
AutoUpdateExcludedContainers *string `json:"autoUpdateExcludedContainers,omitempty"`
+
+ // AutoHealEnabled indicates if automatic container healing is enabled.
+ //
+ // Required: false
+ AutoHealEnabled *string `json:"autoHealEnabled,omitempty"`
+
+ // AutoHealInterval is the cron expression for how often to check container health.
+ //
+ // Required: false
+ AutoHealInterval *string `json:"autoHealInterval,omitempty"`
+
+ // AutoHealExcludedContainers is a comma-separated list of container names to exclude from auto-heal.
+ //
+ // Required: false
+ AutoHealExcludedContainers *string `json:"autoHealExcludedContainers,omitempty"`
+
+ // AutoHealMaxRestarts is the maximum number of auto-heal restarts per container within the restart window.
+ //
+ // Required: false
+ AutoHealMaxRestarts *string `json:"autoHealMaxRestarts,omitempty"`
+
+ // AutoHealRestartWindow is the time window in minutes for counting auto-heal restarts.
+ //
+ // Required: false
+ AutoHealRestartWindow *string `json:"autoHealRestartWindow,omitempty"`
}