Skip to content

Modern Docker Management, Designed for Everyone

fix: use non-http context for jobs (#1770)

Kyle Mendell49b2d4e155fb
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index aca78ca34..cdf080317 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -45,6 +45,7 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
newScheduler.RegisterJob(vulnerabilityScanJob)
setupJobScheduleCallbacks(
+ appCtx,
appServices,
appConfig,
newScheduler,
@@ -57,10 +58,11 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
gitOpsSyncJob,
vulnerabilityScanJob,
)
- setupSettingsCallbacks(appServices, appConfig, newScheduler, imagePollingJob, autoUpdateJob, environmentHealthJob, fsWatcherJob, scheduledPruneJob, vulnerabilityScanJob)
+ setupSettingsCallbacks(appCtx, appServices, appConfig, newScheduler, imagePollingJob, autoUpdateJob, environmentHealthJob, fsWatcherJob, scheduledPruneJob, vulnerabilityScanJob)
}
func setupJobScheduleCallbacks(
+ lifecycleCtx context.Context,
appServices *Services,
appConfig *config.Config,
newScheduler *pkg_scheduler.JobScheduler,
@@ -79,8 +81,9 @@ func setupJobScheduleCallbacks(
appServices.JobSchedule.OnJobSchedulesChanged = func(ctx context.Context, changedKeys []string) {
for _, key := range changedKeys {
+ slog.DebugContext(lifecycleCtx, "Processing job schedule change", "key", key, "triggerContextCanceled", ctx.Err() != nil)
handleJobScheduleChangeInternal(
- ctx,
+ lifecycleCtx,
key,
appConfig,
newScheduler,
@@ -150,41 +153,41 @@ func handleJobScheduleChangeInternal(
}
}
-func setupSettingsCallbacks(appServices *Services, appConfig *config.Config, newScheduler *pkg_scheduler.JobScheduler, imagePollingJob *pkg_scheduler.ImagePollingJob, autoUpdateJob *pkg_scheduler.AutoUpdateJob, environmentHealthJob *pkg_scheduler.EnvironmentHealthJob, fsWatcherJob *pkg_scheduler.FilesystemWatcherJob, scheduledPruneJob *pkg_scheduler.ScheduledPruneJob, vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob) {
- appServices.Settings.OnImagePollingSettingsChanged = func(ctx context.Context) {
- if err := newScheduler.RescheduleJob(ctx, imagePollingJob); err != nil {
- slog.WarnContext(ctx, "Failed to reschedule image-polling job", "error", err)
+func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services, appConfig *config.Config, newScheduler *pkg_scheduler.JobScheduler, imagePollingJob *pkg_scheduler.ImagePollingJob, autoUpdateJob *pkg_scheduler.AutoUpdateJob, environmentHealthJob *pkg_scheduler.EnvironmentHealthJob, fsWatcherJob *pkg_scheduler.FilesystemWatcherJob, scheduledPruneJob *pkg_scheduler.ScheduledPruneJob, vulnerabilityScanJob *pkg_scheduler.VulnerabilityScanJob) {
+ appServices.Settings.OnImagePollingSettingsChanged = func(_ context.Context) {
+ if err := newScheduler.RescheduleJob(lifecycleCtx, imagePollingJob); err != nil {
+ slog.WarnContext(lifecycleCtx, "Failed to reschedule image-polling job", "error", err)
}
- if err := newScheduler.RescheduleJob(ctx, autoUpdateJob); err != nil {
- slog.WarnContext(ctx, "Failed to reschedule auto-update job", "error", err)
+ if err := newScheduler.RescheduleJob(lifecycleCtx, autoUpdateJob); err != nil {
+ slog.WarnContext(lifecycleCtx, "Failed to reschedule auto-update job", "error", err)
}
if !appConfig.AgentMode {
- if err := newScheduler.RescheduleJob(ctx, environmentHealthJob); err != nil {
- slog.WarnContext(ctx, "Failed to reschedule environment-health job", "error", err)
+ if err := newScheduler.RescheduleJob(lifecycleCtx, environmentHealthJob); err != nil {
+ slog.WarnContext(lifecycleCtx, "Failed to reschedule environment-health job", "error", err)
}
}
}
appServices.Settings.OnAutoUpdateSettingsChanged = func(ctx context.Context) {
- slog.DebugContext(ctx, "AutoUpdateSettingsChanged callback triggered")
- if err := newScheduler.RescheduleJob(ctx, autoUpdateJob); err != nil {
- slog.WarnContext(ctx, "Failed to reschedule auto-update job", "error", err)
+ slog.DebugContext(lifecycleCtx, "AutoUpdateSettingsChanged callback triggered", "triggerContextCanceled", ctx.Err() != nil)
+ if err := newScheduler.RescheduleJob(lifecycleCtx, autoUpdateJob); err != nil {
+ slog.WarnContext(lifecycleCtx, "Failed to reschedule auto-update job", "error", err)
}
}
- appServices.Settings.OnProjectsDirectoryChanged = func(ctx context.Context) {
+ appServices.Settings.OnProjectsDirectoryChanged = func(_ context.Context) {
if fsWatcherJob != nil {
- if err := fsWatcherJob.RestartProjectsWatcher(ctx); err != nil {
- slog.WarnContext(ctx, "Failed to restart projects filesystem watcher", "error", err)
+ if err := fsWatcherJob.RestartProjectsWatcher(lifecycleCtx); err != nil {
+ slog.WarnContext(lifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)
}
}
}
- appServices.Settings.OnScheduledPruneSettingsChanged = func(ctx context.Context) {
- if err := newScheduler.RescheduleJob(ctx, scheduledPruneJob); err != nil {
- slog.WarnContext(ctx, "Failed to reschedule scheduled-prune job", "error", err)
+ appServices.Settings.OnScheduledPruneSettingsChanged = func(_ context.Context) {
+ if err := newScheduler.RescheduleJob(lifecycleCtx, scheduledPruneJob); err != nil {
+ slog.WarnContext(lifecycleCtx, "Failed to reschedule scheduled-prune job", "error", err)
}
}
- appServices.Settings.OnVulnerabilityScanSettingsChanged = func(ctx context.Context) {
- if err := newScheduler.RescheduleJob(ctx, vulnerabilityScanJob); err != nil {
- slog.WarnContext(ctx, "Failed to reschedule vulnerability-scan job", "error", err)
+ appServices.Settings.OnVulnerabilityScanSettingsChanged = func(_ context.Context) {
+ if err := newScheduler.RescheduleJob(lifecycleCtx, vulnerabilityScanJob); err != nil {
+ slog.WarnContext(lifecycleCtx, "Failed to reschedule vulnerability-scan job", "error", err)
}
}
diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.go
index 6aa1dd193..9fcff2515 100644
--- a/backend/internal/models/settings.go
+++ b/backend/internal/models/settings.go
@@ -64,6 +64,7 @@ type Settings struct {
PruneMode SettingVariable `key:"dockerPruneMode" meta:"label=Docker Prune Action;type=select;keywords=prune,cleanup,clean,remove,delete,unused,dangling,space,disk;category=internal;description=Configure how unused Docker images are cleaned up"`
ScheduledPruneEnabled SettingVariable `key:"scheduledPruneEnabled" meta:"label=Scheduled Prune Enabled;type=boolean;keywords=prune,cleanup,maintenance,schedule,automatic;category=internal;description=Enable scheduled pruning of unused Docker resources"`
ScheduledPruneInterval SettingVariable `key:"scheduledPruneInterval" meta:"label=Scheduled Prune Interval;type=cron;keywords=prune,cleanup,interval,minutes,schedule;category=internal;description=How often to run scheduled prunes (cron expression)"`
+ GitopsSyncInterval SettingVariable `key:"gitopsSyncInterval" meta:"label=GitOps Sync Interval;type=cron;keywords=gitops,sync,interval,frequency,schedule,repository;category=internal;description=How often to run GitOps synchronization checks (cron expression)"`
ScheduledPruneContainers SettingVariable `key:"scheduledPruneContainers" meta:"label=Scheduled Prune Containers;type=boolean;keywords=prune,containers,cleanup,maintenance;category=internal;description=Remove stopped containers during scheduled prune"`
ScheduledPruneImages SettingVariable `key:"scheduledPruneImages" meta:"label=Scheduled Prune Images;type=boolean;keywords=prune,images,cleanup,maintenance;category=internal;description=Remove unused images during scheduled prune"`
ScheduledPruneVolumes SettingVariable `key:"scheduledPruneVolumes" meta:"label=Scheduled Prune Volumes;type=boolean;keywords=prune,volumes,cleanup,maintenance;category=internal;description=Remove unused volumes during scheduled prune"`
diff --git a/backend/internal/services/job_service.go b/backend/internal/services/job_service.go
index ccd5ded3f..f387695fb 100644
--- a/backend/internal/services/job_service.go
+++ b/backend/internal/services/job_service.go
@@ -60,7 +60,7 @@ func (s *JobService) GetJobSchedules(ctx context.Context) jobschedule.Config {
AutoUpdateInterval: s.settings.GetStringSetting(ctx, "autoUpdateInterval", "0 0 0 * * *"),
PollingInterval: s.settings.GetStringSetting(ctx, "pollingInterval", "0 */15 * * * *"),
ScheduledPruneInterval: s.settings.GetStringSetting(ctx, "scheduledPruneInterval", "0 0 0 * * *"),
- GitopsSyncInterval: s.settings.GetStringSetting(ctx, "gitopsSyncInterval", "0 */5 * * * *"),
+ GitopsSyncInterval: s.settings.GetStringSetting(ctx, "gitopsSyncInterval", "0 */1 * * * *"),
VulnerabilityScanInterval: s.settings.GetStringSetting(ctx, "vulnerabilityScanInterval", "0 0 0 * * *"),
}
}
@@ -226,7 +226,7 @@ func (s *JobService) getJobScheduleInternal(ctx context.Context, meta meta.JobMe
"autoUpdateInterval": "0 0 0 * * *",
"pollingInterval": "0 */15 * * * *",
"scheduledPruneInterval": "0 0 0 * * *",
- "gitopsSyncInterval": "0 */5 * * * *",
+ "gitopsSyncInterval": "0 */1 * * * *",
"vulnerabilityScanInterval": "0 0 0 * * *",
}
diff --git a/backend/internal/services/job_service_test.go b/backend/internal/services/job_service_test.go
new file mode 100644
index 000000000..a12e34284
--- /dev/null
+++ b/backend/internal/services/job_service_test.go
@@ -0,0 +1,22 @@
+package services
+
+import (
+ "context"
+ "testing"
+
+ "github.com/getarcaneapp/arcane/backend/internal/config"
+ "github.com/stretchr/testify/require"
+)
+
+func TestJobService_GetJobSchedules_DefaultGitOpsInterval(t *testing.T) {
+ ctx := context.Background()
+ db := setupSettingsTestDB(t)
+
+ settingsSvc, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ cfg := jobSvc.GetJobSchedules(ctx)
+
+ require.Equal(t, "0 */1 * * * *", cfg.GitopsSyncInterval)
+}
diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.go
index d82c8f7a5..9ed7aa25a 100644
--- a/backend/internal/services/settings_service.go
+++ b/backend/internal/services/settings_service.go
@@ -103,6 +103,7 @@ func (s *SettingsService) getDefaultSettings() *models.Settings {
ScheduledPruneVolumes: models.SettingVariable{Value: "false"},
ScheduledPruneNetworks: models.SettingVariable{Value: "true"},
ScheduledPruneBuildCache: models.SettingVariable{Value: "false"},
+ GitopsSyncInterval: models.SettingVariable{Value: "0 */1 * * * *"},
BaseServerURL: models.SettingVariable{Value: "http://localhost"},
EnableGravatar: models.SettingVariable{Value: "true"},
DefaultShell: models.SettingVariable{Value: "/bin/sh"},
@@ -553,7 +554,7 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
}
// Validate cron settings
- cronFields := []string{"scheduledPruneInterval", "autoUpdateInterval", "pollingInterval", "environmentHealthInterval", "eventCleanupInterval", "analyticsHeartbeatInterval", "vulnerabilityScanInterval"}
+ cronFields := []string{"scheduledPruneInterval", "autoUpdateInterval", "pollingInterval", "environmentHealthInterval", "eventCleanupInterval", "analyticsHeartbeatInterval", "vulnerabilityScanInterval", "gitopsSyncInterval"}
if slices.Contains(cronFields, key) && value != "" {
if _, err := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow).Parse(value); err != nil {
return nil, false, false, false, false, nil, fmt.Errorf("invalid cron expression for %s: %w", key, err)
diff --git a/backend/pkg/scheduler/gitops_sync_job.go b/backend/pkg/scheduler/gitops_sync_job.go
index b8c242908..7de1cb41b 100644
--- a/backend/pkg/scheduler/gitops_sync_job.go
+++ b/backend/pkg/scheduler/gitops_sync_job.go
@@ -2,9 +2,12 @@ package scheduler
import (
"context"
+ "fmt"
"log/slog"
+ "strconv"
"github.com/getarcaneapp/arcane/backend/internal/services"
+ "github.com/robfig/cron/v3"
)
type GitOpsSyncJob struct {
@@ -24,8 +27,30 @@ func (j *GitOpsSyncJob) Name() string {
}
func (j *GitOpsSyncJob) Schedule(ctx context.Context) string {
- // Default interval: 1 minute to check for due syncs
- return "0 */1 * * * *"
+ schedule := j.settingsService.GetStringSetting(ctx, "gitopsSyncInterval", "0 */1 * * * *")
+ if schedule == "" {
+ schedule = "0 */1 * * * *"
+ }
+
+ // Handle legacy straight int if it somehow didn't get migrated
+ if i, err := strconv.Atoi(schedule); err == nil {
+ if i <= 0 {
+ i = 1
+ }
+ if i%60 == 0 {
+ schedule = fmt.Sprintf("0 0 */%d * * *", i/60)
+ } else {
+ schedule = fmt.Sprintf("0 */%d * * * *", i)
+ }
+ }
+
+ parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ if _, err := parser.Parse(schedule); err != nil {
+ slog.WarnContext(ctx, "Invalid cron expression for gitops-sync, using default", "invalid_schedule", schedule, "error", err)
+ return "0 */1 * * * *"
+ }
+
+ return schedule
}
func (j *GitOpsSyncJob) Run(ctx context.Context) {
diff --git a/backend/pkg/scheduler/gitops_sync_job_test.go b/backend/pkg/scheduler/gitops_sync_job_test.go
new file mode 100644
index 000000000..fea09ddf9
--- /dev/null
+++ b/backend/pkg/scheduler/gitops_sync_job_test.go
@@ -0,0 +1,47 @@
+package scheduler
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGitOpsSyncJobSchedule_Default(t *testing.T) {
+ ctx := context.Background()
+ settingsSvc := setupAnalyticsSettingsService(t)
+ job := NewGitOpsSyncJob(nil, settingsSvc)
+
+ got := job.Schedule(ctx)
+ require.Equal(t, "0 */1 * * * *", got)
+}
+
+func TestGitOpsSyncJobSchedule_UsesConfiguredCron(t *testing.T) {
+ ctx := context.Background()
+ settingsSvc := setupAnalyticsSettingsService(t)
+ require.NoError(t, settingsSvc.SetStringSetting(ctx, "gitopsSyncInterval", "0 */7 * * * *"))
+ job := NewGitOpsSyncJob(nil, settingsSvc)
+
+ got := job.Schedule(ctx)
+ require.Equal(t, "0 */7 * * * *", got)
+}
+
+func TestGitOpsSyncJobSchedule_LegacyIntegerMinutes(t *testing.T) {
+ ctx := context.Background()
+ settingsSvc := setupAnalyticsSettingsService(t)
+ require.NoError(t, settingsSvc.SetStringSetting(ctx, "gitopsSyncInterval", "120"))
+ job := NewGitOpsSyncJob(nil, settingsSvc)
+
+ got := job.Schedule(ctx)
+ require.Equal(t, "0 0 */2 * * *", got)
+}
+
+func TestGitOpsSyncJobSchedule_InvalidCronFallsBackToDefault(t *testing.T) {
+ ctx := context.Background()
+ settingsSvc := setupAnalyticsSettingsService(t)
+ require.NoError(t, settingsSvc.SetStringSetting(ctx, "gitopsSyncInterval", "not-a-cron"))
+ job := NewGitOpsSyncJob(nil, settingsSvc)
+
+ got := job.Schedule(ctx)
+ require.Equal(t, "0 */1 * * * *", got)
+}
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index 685e43ef8..9e105a843 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -85,6 +85,7 @@ func (js *JobScheduler) RescheduleJob(ctx context.Context, job schedulertypes.Jo
}
js.entryIDs[job.Name()] = entryID
+ slog.DebugContext(ctx, "Job rescheduled", "name", job.Name(), "schedule", schedule, "contextCanceled", ctx.Err() != nil)
return nil
}
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
new file mode 100644
index 000000000..cb3d1b069
--- /dev/null
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -0,0 +1,95 @@
+package scheduler
+
+import (
+ "context"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+type testSchedulerJob struct {
+ name string
+ schedule string
+ run func(context.Context)
+}
+
+func (j *testSchedulerJob) Name() string { return j.name }
+
+func (j *testSchedulerJob) Schedule(context.Context) string { return j.schedule }
+
+func (j *testSchedulerJob) Run(ctx context.Context) {
+ if j.run != nil {
+ j.run(ctx)
+ }
+}
+
+func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {
+ js := NewJobScheduler(context.Background(), nil)
+
+ var once sync.Once
+ runErrCh := make(chan error, 1)
+ runCtx, cancelRunCtx := context.WithCancel(context.Background())
+ defer cancelRunCtx()
+
+ job := &testSchedulerJob{
+ name: "test-reschedule-provided-context",
+ schedule: "*/1 * * * * *",
+ run: func(ctx context.Context) {
+ once.Do(func() { runErrCh <- ctx.Err() })
+ },
+ }
+
+ require.NoError(t, js.RescheduleJob(runCtx, job))
+ js.cron.Start()
+ defer js.cron.Stop()
+
+ select {
+ case err := <-runErrCh:
+ require.NoError(t, err)
+ case <-time.After(2500 * time.Millisecond):
+ t.Fatal("timed out waiting for scheduled run")
+ }
+}
+
+func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T) {
+ lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())
+ js := NewJobScheduler(lifecycleCtx, nil)
+
+ startedCh := make(chan struct{}, 1)
+ stoppedCh := make(chan struct{}, 1)
+ job := &testSchedulerJob{
+ name: "test-reschedule-lifecycle-shutdown",
+ schedule: "*/1 * * * * *",
+ run: func(ctx context.Context) {
+ select {
+ case startedCh <- struct{}{}:
+ default:
+ }
+ <-ctx.Done()
+ select {
+ case stoppedCh <- struct{}{}:
+ default:
+ }
+ },
+ }
+
+ require.NoError(t, js.RescheduleJob(lifecycleCtx, job))
+ js.cron.Start()
+ defer js.cron.Stop()
+
+ select {
+ case <-startedCh:
+ case <-time.After(2500 * time.Millisecond):
+ t.Fatal("timed out waiting for scheduled run")
+ }
+
+ cancelLifecycle()
+
+ select {
+ case <-stoppedCh:
+ case <-time.After(1500 * time.Millisecond):
+ t.Fatal("scheduled job did not observe lifecycle cancellation")
+ }
+}