Skip to content

Modern Docker Management, Designed for Everyone

refactor: register each gitsync job individually (#2792)

Kyle Mendellf1e9f44fe79a
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 10b95a737..7188b6b28 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -19,9 +19,6 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
newScheduler.RegisterJob(jobs.AutoUpdate)
newScheduler.RegisterJob(jobs.ImagePolling)
- if !appConfig.AgentMode {
- newScheduler.RegisterJob(jobs.EnvironmentHealth)
- }
newScheduler.RegisterJob(jobs.DockerClientRefresh)
newScheduler.RegisterJob(jobs.Analytics)
// Send initial heartbeat on startup without blocking bootstrap.
@@ -32,13 +29,41 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
newScheduler.RegisterJob(jobs.ScheduledPrune)
// FilesystemWatcher is intentionally not scheduler-registered; it watches inline
// and is only rebound on settings changes below.
- newScheduler.RegisterJob(jobs.GitOpsSync)
newScheduler.RegisterJob(jobs.VulnerabilityScan)
newScheduler.RegisterJob(jobs.AutoHeal)
+ // GitOps sync and environment health are no longer single global jobs; each
+ // entity registers its own dynamic job.
+ registerDynamicJobs(appCtx, newScheduler, appServices, appConfig)
+
setupSettingsCallbacks(appCtx, appServices, appConfig, newScheduler, jobs)
}
+// registerDynamicJobs injects the scheduler into the services that own per-entity
+// jobs and registers the jobs for already-existing entities at startup. AddJob is
+// an idempotent upsert, so these run safely before the scheduler is started.
+func registerDynamicJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobScheduler, appServices *di.Services, appConfig *config.Config) {
+ // GitOps: one job per auto-sync-enabled sync (runs on manager and agents).
+ if appServices.GitOpsSync != nil {
+ appServices.GitOpsSync.SetScheduler(appCtx, newScheduler)
+ appServices.GitOpsSync.RegisterAutoSyncJobsOnStartup(appCtx)
+ }
+
+ // Environment health: one job per enabled environment (manager only). The Jobs
+ // UI still addresses "environment-health" by ID, so bridge its reschedule and
+ // run-now back to EnvironmentService.
+ if !appConfig.AgentMode && appServices.Environment != nil {
+ appServices.Environment.SetScheduler(appCtx, newScheduler)
+ appServices.JobSchedule.OnEnvironmentHealthReschedule = func(ctx context.Context) {
+ appServices.Environment.RescheduleHealthJobs(ctx)
+ }
+ appServices.JobSchedule.RunEnvironmentHealthNow = func(ctx context.Context) error {
+ return appServices.Environment.RunHealthChecksNow(ctx)
+ }
+ appServices.Environment.RegisterHealthJobsOnStartup(appCtx)
+ }
+}
+
func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *di.Services, appConfig *config.Config, newScheduler *pkg_scheduler.JobScheduler, jobs *di.Jobs) {
appServices.Settings.OnImagePollingSettingsChanged = func(_ context.Context) {
if err := newScheduler.RescheduleJob(lifecycleCtx, jobs.ImagePolling); err != nil {
@@ -47,11 +72,6 @@ func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *di.Servic
if err := newScheduler.RescheduleJob(lifecycleCtx, jobs.AutoUpdate); err != nil {
slog.WarnContext(lifecycleCtx, "Failed to reschedule auto-update job", "error", err)
}
- if !appConfig.AgentMode {
- if err := newScheduler.RescheduleJob(lifecycleCtx, jobs.EnvironmentHealth); err != nil {
- slog.WarnContext(lifecycleCtx, "Failed to reschedule environment-health job", "error", err)
- }
- }
}
appServices.Settings.OnAutoUpdateSettingsChanged = func(ctx context.Context) {
slog.DebugContext(lifecycleCtx, "AutoUpdateSettingsChanged callback triggered", "triggerContextCanceled", ctx.Err() != nil)
diff --git a/backend/internal/configschema/schema_test.go b/backend/internal/configschema/schema_test.go
index 5528af07c..c000f4567 100644
--- a/backend/internal/configschema/schema_test.go
+++ b/backend/internal/configschema/schema_test.go
@@ -301,7 +301,6 @@ var expectedSettingOverrideKeys = []string{
"gitSyncMaxBinarySizeMb",
"gitSyncMaxFiles",
"gitSyncMaxTotalSizeMb",
- "gitopsSyncInterval",
"httpClientTimeout",
"keyboardShortcutsEnabled",
"maxImageUploadSize",
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index c24d48c4f..817da2950 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -152,7 +152,6 @@ func provideAuthMiddlewareInternal(auth *services.AuthService, apiKey *services.
type Jobs struct {
AutoUpdate *pkg_scheduler.AutoUpdateJob
ImagePolling *pkg_scheduler.ImagePollingJob
- EnvironmentHealth *pkg_scheduler.EnvironmentHealthJob
DockerClientRefresh *pkg_scheduler.DockerClientRefreshJob
Analytics *pkg_scheduler.AnalyticsJob
EventCleanup *pkg_scheduler.EventCleanupJob
@@ -160,7 +159,6 @@ type Jobs struct {
ExpiredSessionsCleanup *pkg_scheduler.ExpiredSessionsCleanupJob
ScheduledPrune *pkg_scheduler.ScheduledPruneJob
FilesystemWatcher *pkg_scheduler.FilesystemWatcherJob
- GitOpsSync *pkg_scheduler.GitOpsSyncJob
VulnerabilityScan *pkg_scheduler.VulnerabilityScanJob
AutoHeal *pkg_scheduler.AutoHealJob
}
diff --git a/backend/internal/di/wire.go b/backend/internal/di/wire.go
index 301e47ea6..dfc8398d5 100644
--- a/backend/internal/di/wire.go
+++ b/backend/internal/di/wire.go
@@ -91,11 +91,10 @@ var JobSet = wire.NewSet(
wire.FieldsOf(new(*Services),
"Updater", "Settings", "ImageUpdate", "Environment", "Docker", "KV",
"Event", "Activity", "Session", "System", "Notification", "Project",
- "Template", "GitOpsSync", "Vulnerability", "Volume",
+ "Template", "Vulnerability", "Volume",
),
pkg_scheduler.NewAutoUpdateJob,
pkg_scheduler.NewImagePollingJob,
- pkg_scheduler.NewEnvironmentHealthJob,
pkg_scheduler.NewDockerClientRefreshJob,
provideAnalyticsJobInternal,
pkg_scheduler.NewEventCleanupJob,
@@ -103,7 +102,6 @@ var JobSet = wire.NewSet(
pkg_scheduler.NewExpiredSessionsCleanupJob,
pkg_scheduler.NewScheduledPruneJob,
provideFilesystemWatcherJobInternal,
- pkg_scheduler.NewGitOpsSyncJob,
pkg_scheduler.NewVulnerabilityScanJob,
pkg_scheduler.NewAutoHealJob,
wire.Struct(new(Jobs), "*"),
diff --git a/backend/internal/di/wire_gen.go b/backend/internal/di/wire_gen.go
index 64f99f7da..75f5f6e73 100644
--- a/backend/internal/di/wire_gen.go
+++ b/backend/internal/di/wire_gen.go
@@ -122,7 +122,6 @@ func InitializeJobs(ctx context.Context, cfg *config.Config, svcs *Services) *Jo
imageUpdateService := svcs.ImageUpdate
environmentService := svcs.Environment
imagePollingJob := scheduler.NewImagePollingJob(imageUpdateService, settingsService, environmentService)
- environmentHealthJob := scheduler.NewEnvironmentHealthJob(environmentService, settingsService)
dockerClientService := svcs.Docker
dockerClientRefreshJob := scheduler.NewDockerClientRefreshJob(dockerClientService, settingsService)
kvService := svcs.KV
@@ -140,15 +139,12 @@ func InitializeJobs(ctx context.Context, cfg *config.Config, svcs *Services) *Jo
projectService := svcs.Project
templateService := svcs.Template
filesystemWatcherJob := provideFilesystemWatcherJobInternal(ctx, projectService, templateService, settingsService, cfg)
- gitOpsSyncService := svcs.GitOpsSync
- gitOpsSyncJob := scheduler.NewGitOpsSyncJob(gitOpsSyncService, settingsService)
vulnerabilityService := svcs.Vulnerability
vulnerabilityScanJob := scheduler.NewVulnerabilityScanJob(vulnerabilityService, settingsService)
autoHealJob := scheduler.NewAutoHealJob(dockerClientService, settingsService, eventService, notificationService)
jobs := &Jobs{
AutoUpdate: autoUpdateJob,
ImagePolling: imagePollingJob,
- EnvironmentHealth: environmentHealthJob,
DockerClientRefresh: dockerClientRefreshJob,
Analytics: analyticsJob,
EventCleanup: eventCleanupJob,
@@ -156,7 +152,6 @@ func InitializeJobs(ctx context.Context, cfg *config.Config, svcs *Services) *Jo
ExpiredSessionsCleanup: expiredSessionsCleanupJob,
ScheduledPrune: scheduledPruneJob,
FilesystemWatcher: filesystemWatcherJob,
- GitOpsSync: gitOpsSyncJob,
VulnerabilityScan: vulnerabilityScanJob,
AutoHeal: autoHealJob,
}
@@ -189,6 +184,6 @@ var ServiceSet = wire.NewSet(
var JobSet = wire.NewSet(wire.FieldsOf(new(*Services),
"Updater", "Settings", "ImageUpdate", "Environment", "Docker", "KV",
"Event", "Activity", "Session", "System", "Notification", "Project",
- "Template", "GitOpsSync", "Vulnerability", "Volume",
-), scheduler.NewAutoUpdateJob, scheduler.NewImagePollingJob, scheduler.NewEnvironmentHealthJob, scheduler.NewDockerClientRefreshJob, provideAnalyticsJobInternal, scheduler.NewEventCleanupJob, scheduler.NewPruningVolumeHelperJob, scheduler.NewExpiredSessionsCleanupJob, scheduler.NewScheduledPruneJob, provideFilesystemWatcherJobInternal, scheduler.NewGitOpsSyncJob, scheduler.NewVulnerabilityScanJob, scheduler.NewAutoHealJob, wire.Struct(new(Jobs), "*"),
+ "Template", "Vulnerability", "Volume",
+), scheduler.NewAutoUpdateJob, scheduler.NewImagePollingJob, scheduler.NewDockerClientRefreshJob, provideAnalyticsJobInternal, scheduler.NewEventCleanupJob, scheduler.NewPruningVolumeHelperJob, scheduler.NewExpiredSessionsCleanupJob, scheduler.NewScheduledPruneJob, provideFilesystemWatcherJobInternal, scheduler.NewVulnerabilityScanJob, scheduler.NewAutoHealJob, wire.Struct(new(Jobs), "*"),
)
diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.go
index ede6e7e3c..a6a86c666 100644
--- a/backend/internal/models/settings.go
+++ b/backend/internal/models/settings.go
@@ -96,7 +96,6 @@ type Settings struct {
DefaultDeployPullPolicy SettingVariable `key:"defaultDeployPullPolicy" meta:"label=Default Deploy Pull Policy;type=select;keywords=deploy,pull,policy,compose,up,missing,always;category=internal;description=Default image pull policy when deploying projects"`
ScheduledPruneEnabled SettingVariable `key:"scheduledPruneEnabled" meta:"label=Scheduled Prune Enabled;type=boolean;keywords=prune,cleanup,maintenance,schedule,automatic;category=internal;description=Enable scheduled pruning of unused Docker resources"`
ScheduledPruneInterval SettingVariable `key:"scheduledPruneInterval" meta:"label=Scheduled Prune Interval;type=cron;keywords=prune,cleanup,interval,minutes,schedule;category=internal;description=How often to run scheduled prunes (cron expression)"`
- GitopsSyncInterval SettingVariable `key:"gitopsSyncInterval" meta:"label=GitOps Sync Interval;type=cron;keywords=gitops,sync,interval,frequency,schedule,repository;category=internal;description=How often to run GitOps synchronization checks (cron expression)"`
PruneContainerMode SettingVariable `key:"pruneContainerMode" meta:"label=Prune Containers;type=select;keywords=prune,containers,cleanup,maintenance,mode,older,stopped;category=internal;description=Select how containers should be pruned when the scheduled prune job runs"`
PruneContainerUntil SettingVariable `key:"pruneContainerUntil" meta:"label=Container Age Filter;type=text;keywords=prune,containers,cleanup,maintenance,until,older,duration;category=internal;description=Duration threshold for scheduled container prune when mode is olderThan"`
PruneImageMode SettingVariable `key:"pruneImageMode" meta:"label=Prune Images;type=select;keywords=prune,images,cleanup,maintenance,mode,dangling,all,older;category=internal;description=Select how images should be pruned when the scheduled prune job runs"`
diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.go
index 4993fec26..569f58124 100644
--- a/backend/internal/services/environment_service.go
+++ b/backend/internal/services/environment_service.go
@@ -24,6 +24,7 @@ import (
"github.com/getarcaneapp/arcane/types/containerregistry"
"github.com/getarcaneapp/arcane/types/environment"
"github.com/getarcaneapp/arcane/types/gitops"
+ schedulertypes "github.com/getarcaneapp/arcane/types/scheduler"
"github.com/google/uuid"
"github.com/moby/moby/client"
"gorm.io/gorm"
@@ -40,8 +41,22 @@ type EnvironmentService struct {
tokenCacheMu sync.RWMutex
tokenCache map[string]edgeTokenCacheEntry
tokenByEnvID map[string]string
+
+ // scheduler and lifecycleCtx are injected post-construction via SetScheduler
+ // (manager-only). Each enabled environment gets its own health-check job; this
+ // replaces the single global environment-health job.
+ scheduler DynamicScheduler
+ lifecycleCtx context.Context
+ // runningHealthChecks guards against a per-environment health check overlapping
+ // with itself (replaces the old single job's atomic "running" guard).
+ runningHealthChecks sync.Map
}
+const (
+ defaultEnvironmentHealthInterval = "0 */2 * * * *"
+ environmentHealthCheckTimeout = 90 * time.Second
+)
+
type edgeTokenCacheEntry struct {
EnvironmentID string
ExpiresAt time.Time
@@ -74,6 +89,164 @@ func NewEnvironmentService(db *database.DB, httpClient *http.Client, dockerServi
}
}
+const environmentHealthJobPrefix = "environment-health:"
+
+func environmentHealthJobNameInternal(envID string) string { return environmentHealthJobPrefix + envID }
+
+// SetScheduler injects the job scheduler and app lifecycle context. Called during
+// bootstrap on the manager only (agent mode leaves scheduler nil, so all health-job
+// registration becomes a no-op).
+func (s *EnvironmentService) SetScheduler(ctx context.Context, scheduler DynamicScheduler) { //nolint:contextcheck // health-check jobs must capture the app lifecycle context, not request contexts
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ s.lifecycleCtx = ctx
+ s.scheduler = scheduler
+}
+
+func (s *EnvironmentService) schedulerCtxInternal(ctx context.Context) context.Context {
+ if s.lifecycleCtx != nil {
+ return s.lifecycleCtx
+ }
+ if ctx != nil {
+ return context.WithoutCancel(ctx)
+ }
+ return context.Background()
+}
+
+// buildHealthJobInternal builds the dynamic job for one environment. The schedule is
+// the global environmentHealthInterval (environment health has no per-entity
+// interval); the run body re-reads nothing it can't and self-cancels cleanly when
+// the environment is gone.
+func (s *EnvironmentService) buildHealthJobInternal(envID string) *schedulertypes.GenericJob {
+ return &schedulertypes.GenericJob{
+ JobName: environmentHealthJobNameInternal(envID),
+ ScheduleFn: func(ctx context.Context) string {
+ sched := s.settingsService.GetStringSetting(ctx, "environmentHealthInterval", defaultEnvironmentHealthInterval)
+ if sched == "" {
+ return defaultEnvironmentHealthInterval
+ }
+ return sched
+ },
+ RunFn: func(ctx context.Context) {
+ s.runHealthCheckInternal(ctx, envID)
+ },
+ }
+}
+
+func (s *EnvironmentService) registerHealthJobInternal(ctx context.Context, envID string) {
+ if s.scheduler == nil {
+ return
+ }
+ schedulerCtx := s.schedulerCtxInternal(ctx)
+ if err := s.scheduler.AddJob(schedulerCtx, s.buildHealthJobInternal(envID)); err != nil {
+ slog.ErrorContext(schedulerCtx, "Failed to register environment health job", "environment_id", envID, "error", err)
+ }
+}
+
+func (s *EnvironmentService) removeHealthJobInternal(ctx context.Context, envID string) {
+ if s.scheduler == nil {
+ return
+ }
+ s.scheduler.RemoveJob(s.schedulerCtxInternal(ctx), environmentHealthJobNameInternal(envID))
+}
+
+func (s *EnvironmentService) listEnabledEnvironmentIDsInternal(ctx context.Context) ([]string, error) {
+ var ids []string
+ if err := s.db.WithContext(ctx).
+ Table("environments").
+ Where("enabled = ?", true).
+ Pluck("id", &ids).Error; err != nil {
+ return nil, fmt.Errorf("failed to list enabled environments: %w", err)
+ }
+ return ids, nil
+}
+
+func (s *EnvironmentService) registerAllEnabledHealthJobsInternal(ctx context.Context) int {
+ ids, err := s.listEnabledEnvironmentIDsInternal(ctx)
+ if err != nil {
+ slog.ErrorContext(ctx, "failed to list environments for health jobs", "error", err)
+ return 0
+ }
+ for _, id := range ids {
+ s.registerHealthJobInternal(ctx, id)
+ }
+ return len(ids)
+}
+
+// RegisterHealthJobsOnStartup registers a health-check job for every enabled
+// environment. Replaces the old global environment-health job.
+func (s *EnvironmentService) RegisterHealthJobsOnStartup(ctx context.Context) {
+ if s.scheduler == nil {
+ return
+ }
+ n := s.registerAllEnabledHealthJobsInternal(ctx)
+ slog.InfoContext(ctx, "Registered environment health jobs on startup", "count", n)
+}
+
+// RescheduleHealthJobs re-registers all enabled environments' health jobs, picking
+// up a changed global interval. Wired from the Jobs UI via JobService.
+func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {
+ if s.scheduler == nil {
+ return
+ }
+ s.registerAllEnabledHealthJobsInternal(ctx)
+}
+
+// RunHealthChecksNow runs every enabled environment's health check synchronously.
+// Backs the "run now" button for the environment-health job in the Jobs UI.
+func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {
+ ids, err := s.listEnabledEnvironmentIDsInternal(ctx)
+ if err != nil {
+ return err
+ }
+ for _, id := range ids {
+ s.runHealthCheckInternal(ctx, id)
+ }
+ return nil
+}
+
+func (s *EnvironmentService) acquireHealthLockInternal(envID string) (func(), bool) {
+ if _, loaded := s.runningHealthChecks.LoadOrStore(envID, struct{}{}); loaded {
+ return nil, false
+ }
+ return func() { s.runningHealthChecks.Delete(envID) }, true
+}
+
+// runHealthCheckInternal tests one environment's connection (updating its DB status)
+// and, for online remotes, syncs registries and repositories to it.
+func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) {
+ release, ok := s.acquireHealthLockInternal(envID)
+ if !ok {
+ slog.WarnContext(ctx, "environment health check skipped; previous run still in progress", "environment_id", envID)
+ return
+ }
+ defer release()
+
+ status, err := s.TestConnection(ctx, envID, nil)
+ switch {
+ case err != nil:
+ slog.WarnContext(ctx, "environment health check failed", "environment_id", envID, "status", status, "error", err)
+ return
+ case status != "online":
+ return
+ }
+
+ // Local environment (ID "0") has no registries/repositories to push.
+ if envID == "0" {
+ return
+ }
+
+ syncCtx, cancel := context.WithTimeout(ctx, environmentHealthCheckTimeout)
+ defer cancel()
+ if err := s.SyncRegistriesToEnvironment(syncCtx, envID); err != nil {
+ slog.WarnContext(syncCtx, "failed to sync registries during health check", "environment_id", envID, "error", err)
+ }
+ if err := s.SyncRepositoriesToEnvironment(syncCtx, envID); err != nil {
+ slog.WarnContext(syncCtx, "failed to sync git repositories during health check", "environment_id", envID, "error", err)
+ }
+}
+
func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context, token string) (string, error) {
token = strings.TrimSpace(token)
if token == "" {
@@ -285,6 +458,10 @@ func (s *EnvironmentService) CreateEnvironment(ctx context.Context, environment
// Create event in background
go s.createEnvironmentEvent(context.WithoutCancel(ctx), environment.ID, environment.Name, models.EventTypeEnvironmentCreate, "Environment Created", fmt.Sprintf("Environment '%s' was created", environment.Name), models.EventSeveritySuccess, userID, username)
+ if environment.Enabled {
+ s.registerHealthJobInternal(ctx, environment.ID)
+ }
+
return environment, nil
}
@@ -696,6 +873,17 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, u
s.syncEnvironmentTokenCacheInternal(id, accessToken)
}
+ // Reconcile the per-environment health job when the enabled flag is toggled.
+ if rawEnabled, ok := updates["enabled"]; ok {
+ if enabled, isBool := rawEnabled.(bool); isBool {
+ if enabled {
+ s.registerHealthJobInternal(ctx, id)
+ } else {
+ s.removeHealthJobInternal(ctx, id)
+ }
+ }
+ }
+
// Create event in background (skip for local environment)
if id != "0" {
go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, updated.Name, models.EventTypeEnvironmentUpdate, "Environment Updated", fmt.Sprintf("Environment '%s' was updated", updated.Name), models.EventSeverityInfo, userID, username)
@@ -711,7 +899,13 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u
return err
}
+ // Stop the per-environment health job before the row is removed.
+ s.removeHealthJobInternal(ctx, id)
+
if err := s.db.WithContext(ctx).Delete(&models.Environment{}, "id = ?", id).Error; err != nil {
+ if env.Enabled {
+ s.registerHealthJobInternal(ctx, env.ID)
+ }
return fmt.Errorf("failed to delete environment: %w", err)
}
diff --git a/backend/internal/services/gitops_sync_service.go b/backend/internal/services/gitops_sync_service.go
index bdaf091e9..2ec306faa 100644
--- a/backend/internal/services/gitops_sync_service.go
+++ b/backend/internal/services/gitops_sync_service.go
@@ -10,6 +10,7 @@ import (
"os"
"path/filepath"
"strings"
+ "sync"
"time"
"github.com/getarcaneapp/arcane/backend/internal/common"
@@ -21,10 +22,23 @@ import (
"github.com/getarcaneapp/arcane/backend/pkg/utils/mapper"
"github.com/getarcaneapp/arcane/types/gitops"
projecttypes "github.com/getarcaneapp/arcane/types/project"
+ schedulertypes "github.com/getarcaneapp/arcane/types/scheduler"
"github.com/getarcaneapp/arcane/types/swarm"
"gorm.io/gorm"
)
+// DynamicScheduler is the subset of the job scheduler used by services that
+// register per-entity jobs at runtime (GitOps syncs, environment health). It is a
+// consumer-side interface satisfied by *pkg/scheduler.JobScheduler; the scheduler
+// is injected post-construction via SetScheduler because it is created after the
+// service graph is built (and pkg/scheduler imports this package, so it cannot be
+// a wire input here).
+type DynamicScheduler interface {
+ AddJob(ctx context.Context, job schedulertypes.Job) error
+ RemoveJob(ctx context.Context, name string)
+ HasJob(name string) bool
+}
+
type GitOpsSyncService struct {
db *database.DB
repoService *GitRepositoryService
@@ -32,6 +46,15 @@ type GitOpsSyncService struct {
swarmService *SwarmService
eventService *EventService
settingsService *SettingsService
+
+ // scheduler and lifecycleCtx are injected post-construction via SetScheduler.
+ scheduler DynamicScheduler
+ lifecycleCtx context.Context
+ // runningSyncs guards against overlapping PerformSync runs for the same sync.
+ // The previous single global job serialized syncs implicitly; with independent
+ // per-sync jobs (plus manual/webhook/startup triggers) a sync could otherwise
+ // run concurrently with itself and race the clone/redeploy of one project.
+ runningSyncs sync.Map
}
const defaultGitSyncTimeout = 5 * time.Minute
@@ -44,13 +67,6 @@ const (
defaultMaxSyncBinarySize = defaultMaxSyncBinarySizeMB * 1024 * 1024
)
-type scheduledGitOpsSync struct {
- ID string
- EnvironmentID string
- SyncInterval int
- LastSyncAt *time.Time
-}
-
// preparedSyncSource captures the repository data needed by the sync execution
// paths after the source repository has been cloned and validated.
type preparedSyncSource struct {
@@ -106,6 +122,131 @@ func NewGitOpsSyncService(db *database.DB, repoService *GitRepositoryService, pr
}
}
+const gitOpsSyncJobPrefix = "gitops-sync:"
+
+func gitOpsSyncJobNameInternal(syncID string) string { return gitOpsSyncJobPrefix + syncID }
+
+// SetScheduler injects the job scheduler and the app lifecycle context. It must be
+// called during bootstrap (after the service graph is built) before any per-sync
+// jobs are registered. The lifecycle context is used for background sync kicks so
+// they outlive the request/bootstrap goroutine that triggered them.
+func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler DynamicScheduler) { //nolint:contextcheck // background sync kicks must capture the app lifecycle context, not request contexts
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ s.lifecycleCtx = ctx
+ s.scheduler = scheduler
+}
+
+func (s *GitOpsSyncService) schedulerCtxInternal(ctx context.Context) context.Context {
+ if s.lifecycleCtx != nil {
+ return s.lifecycleCtx
+ }
+ if ctx != nil {
+ return context.WithoutCancel(ctx)
+ }
+ return context.Background()
+}
+
+// buildSyncJobInternal returns the dynamic job for a single sync. The schedule is a
+// fixed "@every Nm" interval; the run body re-reads the sync each fire (so a row
+// deleted or toggled to AutoSync=false self-cancels cleanly) and delegates to
+// PerformSync, which owns its own timeout and the per-sync in-flight guard.
+func (s *GitOpsSyncService) buildSyncJobInternal(syncID, environmentID string, intervalMinutes int) *schedulertypes.GenericJob {
+ interval := max(intervalMinutes, 1)
+ schedule := fmt.Sprintf("@every %dm", interval)
+ return &schedulertypes.GenericJob{
+ JobName: gitOpsSyncJobNameInternal(syncID),
+ ScheduleFn: func(_ context.Context) string {
+ return schedule
+ },
+ RunFn: func(ctx context.Context) {
+ sync, err := s.GetSyncByID(ctx, environmentID, syncID)
+ if err != nil {
+ slog.DebugContext(ctx, "gitops auto-sync skipped; sync not found", "syncId", syncID, "error", err)
+ return
+ }
+ if !sync.AutoSync {
+ return
+ }
+ if _, err := s.PerformSync(ctx, environmentID, syncID, systemUser); err != nil {
+ slog.ErrorContext(ctx, "gitops auto-sync run failed", "syncId", syncID, "error", err)
+ }
+ },
+ }
+}
+
+func (s *GitOpsSyncService) registerSyncJobInternal(ctx context.Context, syncID, environmentID string, intervalMinutes int) {
+ if s.scheduler == nil {
+ return
+ }
+ job := s.buildSyncJobInternal(syncID, environmentID, intervalMinutes)
+ schedulerCtx := s.schedulerCtxInternal(ctx)
+ if err := s.scheduler.AddJob(schedulerCtx, job); err != nil {
+ slog.ErrorContext(schedulerCtx, "Failed to register gitops sync job", "syncId", syncID, "error", err)
+ }
+}
+
+func (s *GitOpsSyncService) unregisterSyncJobInternal(ctx context.Context, syncID string) {
+ if s.scheduler == nil {
+ return
+ }
+ s.scheduler.RemoveJob(s.schedulerCtxInternal(ctx), gitOpsSyncJobNameInternal(syncID))
+}
+
+// kickSyncInternal runs a sync once in the background on the app lifecycle context.
+// Used when auto-sync is freshly enabled or when a sync is overdue at startup, so
+// the first run does not wait a full interval.
+func (s *GitOpsSyncService) kickSyncInternal(ctx context.Context, syncID, environmentID string) {
+ ctx = s.schedulerCtxInternal(ctx)
+ go func() {
+ if _, err := s.PerformSync(ctx, environmentID, syncID, systemUser); err != nil {
+ slog.ErrorContext(ctx, "gitops immediate sync kick failed", "syncId", syncID, "error", err)
+ }
+ }()
+}
+
+// RegisterAutoSyncJobsOnStartup registers a dynamic job for every auto-sync-enabled
+// sync and kicks an immediate run for any that are overdue. This replaces the old
+// global polling job so existing syncs keep running after upgrade.
+func (s *GitOpsSyncService) RegisterAutoSyncJobsOnStartup(ctx context.Context) {
+ if s.scheduler == nil {
+ return
+ }
+ var syncs []models.GitOpsSync
+ if err := s.db.WithContext(ctx).
+ Where("auto_sync = ?", true).
+ Find(&syncs).Error; err != nil {
+ slog.ErrorContext(ctx, "Failed to load auto-sync jobs on startup", "error", err)
+ return
+ }
+ for i := range syncs {
+ sync := syncs[i]
+ s.registerSyncJobInternal(ctx, sync.ID, sync.EnvironmentID, sync.SyncInterval)
+ if isGitOpsSyncOverdueInternal(&sync) {
+ s.kickSyncInternal(ctx, sync.ID, sync.EnvironmentID)
+ }
+ }
+ slog.InfoContext(ctx, "Registered gitops auto-sync jobs on startup", "count", len(syncs))
+}
+
+func isGitOpsSyncOverdueInternal(sync *models.GitOpsSync) bool {
+ if sync.LastSyncAt == nil {
+ return true
+ }
+ interval := max(sync.SyncInterval, 1)
+ return time.Now().After(sync.LastSyncAt.Add(time.Duration(interval) * time.Minute))
+}
+
+// acquireSyncLockInternal returns a release func and true when no sync is currently
+// running for the given id; otherwise it returns false and the caller should skip.
+func (s *GitOpsSyncService) acquireSyncLockInternal(syncID string) (func(), bool) {
+ if _, loaded := s.runningSyncs.LoadOrStore(syncID, struct{}{}); loaded {
+ return nil, false
+ }
+ return func() { s.runningSyncs.Delete(syncID) }, true
+}
+
func (s *GitOpsSyncService) getEnvironmentSyncLimits(ctx context.Context) (int, int64, int64) {
if s.settingsService == nil {
return defaultMaxSyncFiles, defaultMaxSyncTotalSize, defaultMaxSyncBinarySize
@@ -305,6 +446,12 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string
// Don't fail the entire creation - the sync config exists and can be retried
}
+ // Register the recurring job if auto-sync is on. The initial sync above already
+ // ran once, so no extra kick is needed here.
+ if sync.AutoSync {
+ s.registerSyncJobInternal(ctx, sync.ID, sync.EnvironmentID, sync.SyncInterval)
+ }
+
return s.GetSyncByID(ctx, "", sync.ID)
}
@@ -314,6 +461,17 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
return nil, err
}
+ // Capture state needed to reconcile the dynamic job after the update.
+ oldAutoSync := sync.AutoSync
+ newAutoSync := sync.AutoSync
+ if req.AutoSync != nil {
+ newAutoSync = *req.AutoSync
+ }
+ newInterval := sync.SyncInterval
+ if req.SyncInterval != nil {
+ newInterval = *req.SyncInterval
+ }
+
updates := make(map[string]any)
if req.Name != nil {
@@ -381,6 +539,18 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
})
}
+ // Reconcile the dynamic job to match the new state.
+ switch {
+ case newAutoSync:
+ s.registerSyncJobInternal(ctx, sync.ID, sync.EnvironmentID, newInterval)
+ if !oldAutoSync {
+ // Freshly enabled — kick a run now so it doesn't wait a full interval.
+ s.kickSyncInternal(ctx, sync.ID, sync.EnvironmentID)
+ }
+ default:
+ s.unregisterSyncJobInternal(ctx, sync.ID)
+ }
+
return s.GetSyncByID(ctx, environmentID, id)
}
@@ -391,6 +561,10 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id st
return err
}
+ // Stop the recurring job before the row goes away. Any in-flight run re-reads
+ // the sync and self-cancels once the row is gone.
+ s.unregisterSyncJobInternal(ctx, id)
+
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
// Clear gitops_managed_by for the associated project, if any.
if sync.ProjectID != nil && *sync.ProjectID != "" {
@@ -406,6 +580,9 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id st
}
return nil
}); err != nil {
+ if sync.AutoSync {
+ s.registerSyncJobInternal(ctx, sync.ID, sync.EnvironmentID, sync.SyncInterval)
+ }
return err
}
@@ -427,6 +604,15 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id st
}
func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id string, actor models.User) (*gitops.SyncResult, error) {
+ // Coalesce overlapping runs for the same sync (scheduled fire, startup/enable
+ // kick, manual trigger, webhook) so they don't race the clone/redeploy.
+ release, ok := s.acquireSyncLockInternal(id)
+ if !ok {
+ slog.InfoContext(ctx, "GitOps sync already in progress; skipping", "syncId", id)
+ return &gitops.SyncResult{Success: false, Message: "sync already in progress", SyncedAt: time.Now()}, nil
+ }
+ defer release()
+
syncCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
defer cancel()
@@ -712,41 +898,6 @@ func (s *GitOpsSyncService) GetSyncStatus(ctx context.Context, environmentID, id
return status, nil
}
-func (s *GitOpsSyncService) SyncAllEnabled(ctx context.Context) error {
- var syncs []scheduledGitOpsSync
- if err := s.db.WithContext(ctx).
- Table("gitops_syncs").
- Select("id", "environment_id", "sync_interval", "last_sync_at").
- Where("auto_sync = ?", true).
- Find(&syncs).Error; err != nil {
- return fmt.Errorf("failed to get auto-sync enabled syncs: %w", err)
- }
-
- for _, sync := range syncs {
- // Check if sync is due
- if sync.LastSyncAt != nil {
- nextSync := sync.LastSyncAt.Add(time.Duration(sync.SyncInterval) * time.Minute)
- // Use a 30-second buffer to account for execution time drift
- if time.Now().Add(30 * time.Second).Before(nextSync) {
- continue
- }
- }
-
- // Perform sync
- result, err := s.PerformSync(ctx, sync.EnvironmentID, sync.ID, systemUser)
- if err != nil {
- slog.ErrorContext(ctx, "Failed to sync", "syncId", sync.ID, "error", err)
- continue
- }
-
- if result.Success {
- slog.InfoContext(ctx, "Sync completed", "syncId", sync.ID, "message", result.Message)
- }
- }
-
- return nil
-}
-
func (s *GitOpsSyncService) ReconcileDirectorySyncProjectsOnStartup(ctx context.Context) error {
var syncs []models.GitOpsSync
if err := s.db.WithContext(ctx).
diff --git a/backend/internal/services/job_service.go b/backend/internal/services/job_service.go
index 6adcace01..0eb61e094 100644
--- a/backend/internal/services/job_service.go
+++ b/backend/internal/services/job_service.go
@@ -38,6 +38,13 @@ type JobService struct {
scheduler JobRunner
lifecycleCtx context.Context
location *time.Location // Timezone for cron schedule calculations
+
+ // environment-health is no longer a single scheduler job — it fans out to one
+ // dynamic job per environment owned by EnvironmentService. These bridge the Jobs
+ // UI (which addresses jobs by ID) back to that service. Set during bootstrap on
+ // the manager only.
+ OnEnvironmentHealthReschedule func(ctx context.Context)
+ RunEnvironmentHealthNow func(ctx context.Context) error
}
func NewJobService(db *database.DB, settings *SettingsService, cfg *config.Config) *JobService {
@@ -67,7 +74,6 @@ func (s *JobService) GetJobSchedules(ctx context.Context) jobschedule.Config {
DockerClientRefreshInterval: s.settings.GetStringSetting(ctx, "dockerClientRefreshInterval", "*/30 * * * * *"),
PollingInterval: s.settings.GetStringSetting(ctx, "pollingInterval", "0 */15 * * * *"),
ScheduledPruneInterval: s.settings.GetStringSetting(ctx, "scheduledPruneInterval", "0 0 0 * * *"),
- GitopsSyncInterval: s.settings.GetStringSetting(ctx, "gitopsSyncInterval", "0 */1 * * * *"),
VulnerabilityScanInterval: s.settings.GetStringSetting(ctx, "vulnerabilityScanInterval", "0 0 0 * * *"),
AutoHealInterval: s.settings.GetStringSetting(ctx, "autoHealInterval", "*/30 * * * * *"),
}
@@ -95,7 +101,6 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule
{key: "dockerClientRefreshInterval", current: current.DockerClientRefreshInterval, update: updates.DockerClientRefreshInterval},
{key: "pollingInterval", current: current.PollingInterval, update: updates.PollingInterval},
{key: "scheduledPruneInterval", current: current.ScheduledPruneInterval, update: updates.ScheduledPruneInterval},
- {key: "gitopsSyncInterval", current: current.GitopsSyncInterval, update: updates.GitopsSyncInterval},
{key: "vulnerabilityScanInterval", current: current.VulnerabilityScanInterval, update: updates.VulnerabilityScanInterval},
{key: "autoHealInterval", current: current.AutoHealInterval, update: updates.AutoHealInterval},
}
@@ -168,6 +173,19 @@ func (s *JobService) RescheduleJobsForSettingKeys(ctx context.Context, changedKe
continue
}
+ // environment-health fans out to per-environment dynamic jobs; delegate the
+ // reschedule to EnvironmentService instead of looking up a single job.
+ if jobID == "environment-health" {
+ if s.OnEnvironmentHealthReschedule != nil {
+ reschedCtx := ctx //nolint:contextcheck // lifecycle context preferred so jobs outlive the request
+ if s.lifecycleCtx != nil {
+ reschedCtx = s.lifecycleCtx
+ }
+ s.OnEnvironmentHealthReschedule(reschedCtx)
+ }
+ continue
+ }
+
job, ok := s.scheduler.GetJob(jobID)
if !ok {
slog.DebugContext(ctx, "Skipping reschedule for unregistered job", "job", jobID)
@@ -231,6 +249,12 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponse
}
func (s *JobService) RunJobNowInline(ctx context.Context, jobID string) error {
+ // environment-health runs as per-environment dynamic jobs; "run now" fans out
+ // through EnvironmentService rather than a single registered job.
+ if jobID == "environment-health" && s.RunEnvironmentHealthNow != nil {
+ return s.RunEnvironmentHealthNow(context.WithoutCancel(ctx))
+ }
+
job, err := s.getRunnableJobInternal(jobID)
if err != nil {
return err
@@ -289,7 +313,6 @@ func (s *JobService) getJobScheduleInternal(ctx context.Context, meta meta.JobMe
"dockerClientRefreshInterval": "*/30 * * * * *",
"pollingInterval": "0 */15 * * * *",
"scheduledPruneInterval": "0 0 0 * * *",
- "gitopsSyncInterval": "0 */1 * * * *",
"vulnerabilityScanInterval": "0 0 0 * * *",
"autoHealInterval": "*/30 * * * * *",
}
diff --git a/backend/internal/services/job_service_test.go b/backend/internal/services/job_service_test.go
index b4facc479..59f8b93b0 100644
--- a/backend/internal/services/job_service_test.go
+++ b/backend/internal/services/job_service_test.go
@@ -10,19 +10,6 @@ import (
"github.com/stretchr/testify/require"
)
-func TestJobService_GetJobSchedules_DefaultGitOpsInterval(t *testing.T) {
- ctx := context.Background()
- db := setupSettingsTestDB(t)
-
- settingsSvc, err := NewSettingsService(ctx, db)
- require.NoError(t, err)
-
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
- cfg := jobSvc.GetJobSchedules(ctx)
-
- require.Equal(t, "0 */1 * * * *", cfg.GitopsSyncInterval)
-}
-
func TestJobService_GetJobSchedules_DefaultDockerClientRefreshInterval(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.go
index e59baa699..9496b00fb 100644
--- a/backend/internal/services/settings_service.go
+++ b/backend/internal/services/settings_service.go
@@ -142,7 +142,6 @@ func DefaultSettingsConfig() *models.Settings {
AutoHealMaxRestarts: models.SettingVariable{Value: "5"},
AutoHealRestartWindow: models.SettingVariable{Value: "30"},
VolumeBrowserHelperIdleTimeout: models.SettingVariable{Value: "10"},
- GitopsSyncInterval: models.SettingVariable{Value: "0 */1 * * * *"},
BaseServerURL: models.SettingVariable{Value: "http://localhost"},
EnableGravatar: models.SettingVariable{Value: "true"},
DefaultShell: models.SettingVariable{Value: "/bin/sh"},
diff --git a/backend/pkg/scheduler/environment_health_job.go b/backend/pkg/scheduler/environment_health_job.go
deleted file mode 100644
index ffb39be33..000000000
--- a/backend/pkg/scheduler/environment_health_job.go
+++ /dev/null
@@ -1,160 +0,0 @@
-package scheduler
-
-import (
- "context"
- "log/slog"
- "sync/atomic"
- "time"
-
- "github.com/getarcaneapp/arcane/backend/internal/services"
- "golang.org/x/sync/errgroup"
-)
-
-const (
- defaultEnvironmentSyncConcurrency = 4
- defaultEnvironmentSyncTimeout = 90 * time.Second
-)
-
-type healthEnvironment struct {
- ID string
- Name string
-}
-
-type EnvironmentHealthJob struct {
- environmentService *services.EnvironmentService
- settingsService *services.SettingsService
- syncConcurrency int
- syncTimeout time.Duration
- running atomic.Bool
-}
-
-func NewEnvironmentHealthJob(environmentService *services.EnvironmentService, settingsService *services.SettingsService) *EnvironmentHealthJob {
- return &EnvironmentHealthJob{
- environmentService: environmentService,
- settingsService: settingsService,
- syncConcurrency: defaultEnvironmentSyncConcurrency,
- syncTimeout: defaultEnvironmentSyncTimeout,
- }
-}
-
-func (j *EnvironmentHealthJob) Name() string {
- return "environment-health"
-}
-
-func (j *EnvironmentHealthJob) Schedule(ctx context.Context) string {
- s := j.settingsService.GetStringSetting(ctx, "environmentHealthInterval", "0 */2 * * * *")
- if s == "" {
- return "0 */2 * * * *"
- }
- return s
-}
-
-func (j *EnvironmentHealthJob) Run(ctx context.Context) {
- if !j.running.CompareAndSwap(false, true) {
- slog.WarnContext(ctx, "environment health check skipped; previous run still in progress")
- return
- }
- defer j.running.Store(false)
-
- slog.InfoContext(ctx, "environment health check started")
-
- // Get all environments using the DB directly
- db := j.environmentService.GetDB()
- var environments []healthEnvironment
-
- if err := db.WithContext(ctx).
- Model(&healthEnvironment{}).
- Table("environments").
- Select("id", "name").
- Where("enabled = ?", true).
- Find(&environments).Error; err != nil {
- slog.ErrorContext(ctx, "failed to list environments for health check", "error", err)
- return
- }
-
- checkedCount := 0
- onlineCount := 0
- offlineCount := 0
- syncedRemoteCount := 0
- var onlineRemote []healthEnvironment
-
- for _, env := range environments {
- checkedCount++
-
- // Test connection without custom URL (will update DB status)
- status, err := j.environmentService.TestConnection(ctx, env.ID, nil)
- switch {
- case err != nil:
- slog.WarnContext(ctx, "environment health check failed", "environment_id", env.ID, "environment_name", env.Name, "status", status, "error", err)
- offlineCount++
- case status == "online":
- onlineCount++
- // Queue sync for online remote environments (skip local environment ID "0")
- if env.ID != "0" {
- onlineRemote = append(onlineRemote, env)
- syncedRemoteCount++
- }
- default:
- offlineCount++
- }
- }
-
- j.syncOnlineRemoteEnvironments(ctx, onlineRemote)
- slog.InfoContext(ctx, "environment health check completed", "checked", checkedCount, "online", onlineCount, "offline", offlineCount, "remote_sync_queued", syncedRemoteCount)
-}
-
-func (j *EnvironmentHealthJob) Reschedule(ctx context.Context) error {
- slog.InfoContext(ctx, "rescheduling environment health job in new scheduler; currently requires restart")
- return nil
-}
-
-func (j *EnvironmentHealthJob) syncOnlineRemoteEnvironments(ctx context.Context, environments []healthEnvironment) {
- if len(environments) == 0 {
- return
- }
-
- syncCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), j.syncTimeout)
- defer cancel()
-
- g, groupCtx := errgroup.WithContext(syncCtx)
- if j.syncConcurrency > 0 {
- g.SetLimit(j.syncConcurrency)
- }
-
- for _, env := range environments {
- g.Go(func() error {
- if err := j.environmentService.SyncRegistriesToEnvironment(groupCtx, env.ID); err != nil {
- slog.WarnContext(groupCtx, "failed to sync registries during health check",
- "environment_id", env.ID,
- "environment_name", env.Name,
- "error", err)
- return nil
- }
-
- slog.DebugContext(groupCtx, "successfully synced registries during health check",
- "environment_id", env.ID,
- "environment_name", env.Name)
- return nil
- })
-
- g.Go(func() error {
- if err := j.environmentService.SyncRepositoriesToEnvironment(groupCtx, env.ID); err != nil {
- slog.WarnContext(groupCtx, "failed to sync git repositories during health check",
- "environment_id", env.ID,
- "environment_name", env.Name,
- "error", err)
- return nil
- }
-
- slog.DebugContext(groupCtx, "successfully synced git repositories during health check",
- "environment_id", env.ID,
- "environment_name", env.Name)
- return nil
- })
- }
-
- _ = g.Wait()
- if syncCtx.Err() != nil {
- slog.WarnContext(ctx, "environment health sync phase timed out or canceled", "error", syncCtx.Err())
- }
-}
diff --git a/backend/pkg/scheduler/environment_health_job_test.go b/backend/pkg/scheduler/environment_health_job_test.go
deleted file mode 100644
index 4e83a7f22..000000000
--- a/backend/pkg/scheduler/environment_health_job_test.go
+++ /dev/null
@@ -1,25 +0,0 @@
-package scheduler
-
-import (
- "testing"
-
- "github.com/stretchr/testify/require"
-)
-
-func TestNewEnvironmentHealthJob_Defaults(t *testing.T) {
- job := NewEnvironmentHealthJob(nil, nil)
-
- require.Equal(t, defaultEnvironmentSyncConcurrency, job.syncConcurrency)
- require.Equal(t, defaultEnvironmentSyncTimeout, job.syncTimeout)
- require.False(t, job.running.Load())
-}
-
-func TestEnvironmentHealthJob_RunGuardAtomic(t *testing.T) {
- job := &EnvironmentHealthJob{}
-
- require.True(t, job.running.CompareAndSwap(false, true))
- require.False(t, job.running.CompareAndSwap(false, true))
-
- job.running.Store(false)
- require.True(t, job.running.CompareAndSwap(false, true))
-}
diff --git a/backend/pkg/scheduler/gitops_sync_job.go b/backend/pkg/scheduler/gitops_sync_job.go
deleted file mode 100644
index b1ac03e95..000000000
--- a/backend/pkg/scheduler/gitops_sync_job.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package scheduler
-
-import (
- "context"
- "log/slog"
-
- "github.com/getarcaneapp/arcane/backend/internal/services"
- "github.com/robfig/cron/v3"
-)
-
-type GitOpsSyncJob struct {
- syncService *services.GitOpsSyncService
- settingsService *services.SettingsService
-}
-
-func NewGitOpsSyncJob(syncService *services.GitOpsSyncService, settingsService *services.SettingsService) *GitOpsSyncJob {
- return &GitOpsSyncJob{
- syncService: syncService,
- settingsService: settingsService,
- }
-}
-
-func (j *GitOpsSyncJob) Name() string {
- return "gitops-sync"
-}
-
-func (j *GitOpsSyncJob) ShouldSchedule(ctx context.Context) bool {
- return j.settingsService.GetBoolSetting(ctx, "gitopsSyncEnabled", true)
-}
-
-func (j *GitOpsSyncJob) Schedule(ctx context.Context) string {
- schedule := j.settingsService.GetStringSetting(ctx, "gitopsSyncInterval", "0 */1 * * * *")
- if schedule == "" {
- schedule = "0 */1 * * * *"
- }
-
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
- if _, err := parser.Parse(schedule); err != nil {
- slog.WarnContext(ctx, "Invalid cron expression for gitops-sync, using default", "invalid_schedule", schedule, "error", err)
- return "0 */1 * * * *"
- }
-
- return schedule
-}
-
-func (j *GitOpsSyncJob) Run(ctx context.Context) {
- enabled := j.settingsService.GetBoolSetting(ctx, "gitopsSyncEnabled", true)
- if !enabled {
- slog.DebugContext(ctx, "GitOps sync disabled; skipping run")
- return
- }
-
- slog.InfoContext(ctx, "GitOps sync run started")
-
- if err := j.syncService.SyncAllEnabled(ctx); err != nil {
- slog.ErrorContext(ctx, "GitOps sync run failed", "err", err)
- return
- }
-
- slog.InfoContext(ctx, "GitOps sync run completed")
-}
diff --git a/backend/pkg/scheduler/gitops_sync_job_test.go b/backend/pkg/scheduler/gitops_sync_job_test.go
deleted file mode 100644
index fe13620b1..000000000
--- a/backend/pkg/scheduler/gitops_sync_job_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package scheduler
-
-import (
- "context"
- "testing"
-
- "github.com/stretchr/testify/require"
-)
-
-func TestGitOpsSyncJobSchedule_Default(t *testing.T) {
- ctx := context.Background()
- _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
- job := NewGitOpsSyncJob(nil, settingsSvc)
-
- got := job.Schedule(ctx)
- require.Equal(t, "0 */1 * * * *", got)
-}
-
-func TestGitOpsSyncJobSchedule_UsesConfiguredCron(t *testing.T) {
- ctx := context.Background()
- _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
- require.NoError(t, settingsSvc.SetStringSetting(ctx, "gitopsSyncInterval", "0 */7 * * * *"))
- job := NewGitOpsSyncJob(nil, settingsSvc)
-
- got := job.Schedule(ctx)
- require.Equal(t, "0 */7 * * * *", got)
-}
-
-func TestGitOpsSyncJobSchedule_InvalidCronFallsBackToDefault(t *testing.T) {
- ctx := context.Background()
- _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
- require.NoError(t, settingsSvc.SetStringSetting(ctx, "gitopsSyncInterval", "not-a-cron"))
- job := NewGitOpsSyncJob(nil, settingsSvc)
-
- got := job.Schedule(ctx)
- require.Equal(t, "0 */1 * * * *", got)
-}
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index 996df263b..7d9099f58 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -3,6 +3,7 @@ package scheduler
import (
"context"
"log/slog"
+ "sync"
"time"
schedulertypes "github.com/getarcaneapp/arcane/types/scheduler"
@@ -10,6 +11,11 @@ import (
)
type JobScheduler struct {
+ // mu guards jobs, jobsByID and entryIDs. It is held across the cron
+ // add/remove calls (which are themselves quick and never block on job
+ // execution) but never across job.Run — Run executes on cron's own
+ // goroutine, so a running job must not call back into a locking method here.
+ mu sync.Mutex
cron *cron.Cron
jobs []schedulertypes.Job
jobsByID map[string]schedulertypes.Job
@@ -36,37 +42,75 @@ func NewJobScheduler(ctx context.Context, location *time.Location) *JobScheduler
}
}
+// RegisterJob records a static job to be scheduled when StartScheduler runs. Use
+// AddJob for jobs added dynamically at runtime.
func (js *JobScheduler) RegisterJob(job schedulertypes.Job) {
+ js.mu.Lock()
+ defer js.mu.Unlock()
js.jobs = append(js.jobs, job)
js.jobsByID[job.Name()] = job
}
func (js *JobScheduler) GetJob(jobID string) (schedulertypes.Job, bool) {
+ js.mu.Lock()
+ defer js.mu.Unlock()
job, ok := js.jobsByID[jobID]
return job, ok
}
+// HasJob reports whether a job with the given name is currently registered.
+func (js *JobScheduler) HasJob(jobID string) bool {
+ js.mu.Lock()
+ defer js.mu.Unlock()
+ _, ok := js.jobsByID[jobID]
+ return ok
+}
+
func (js *JobScheduler) StartScheduler() {
+ js.mu.Lock()
for _, job := range js.jobs {
if err := js.scheduleJobInternal(js.context, job); err != nil {
slog.ErrorContext(js.context, "Failed to schedule job", "name", job.Name(), "error", err)
}
}
+ js.mu.Unlock()
js.cron.Start()
}
-func (js *JobScheduler) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {
- if entryID, ok := js.entryIDs[job.Name()]; ok {
+// AddJob registers and schedules a job at runtime. It is an idempotent upsert: if
+// a job with the same name is already scheduled, its existing cron entry is
+// removed first (preventing a leaked, forever-firing entry when the schedule
+// changes). Safe to call before or after StartScheduler.
+func (js *JobScheduler) AddJob(ctx context.Context, job schedulertypes.Job) error {
+ js.mu.Lock()
+ defer js.mu.Unlock()
+ return js.upsertJobInternal(ctx, job)
+}
+
+// RemoveJob unschedules and forgets a job by name. It is a no-op (not an error)
+// when no job with that name is registered.
+func (js *JobScheduler) RemoveJob(ctx context.Context, jobName string) {
+ js.mu.Lock()
+ defer js.mu.Unlock()
+
+ if entryID, ok := js.entryIDs[jobName]; ok {
js.cron.Remove(entryID)
- delete(js.entryIDs, job.Name())
+ delete(js.entryIDs, jobName)
}
-
- if err := js.scheduleJobInternal(ctx, job); err != nil {
- return err
+ delete(js.jobsByID, jobName)
+ for i, j := range js.jobs {
+ if j.Name() == jobName {
+ js.jobs = append(js.jobs[:i], js.jobs[i+1:]...)
+ break
+ }
}
+ slog.DebugContext(ctx, "Job removed", "name", jobName)
+}
- slog.DebugContext(ctx, "Job rescheduled", "name", job.Name(), "scheduled", js.isJobScheduledInternal(job.Name()), "contextCanceled", ctx.Err() != nil)
- return nil
+func (js *JobScheduler) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {
+ js.mu.Lock()
+ defer js.mu.Unlock()
+ return js.upsertJobInternal(ctx, job)
}
// GetLocation returns the timezone location used by the scheduler for cron expressions.
@@ -81,6 +125,29 @@ func (js *JobScheduler) Run(ctx context.Context) error {
return nil
}
+// upsertJobInternal records the job and (re)schedules it. Callers must hold js.mu.
+func (js *JobScheduler) upsertJobInternal(ctx context.Context, job schedulertypes.Job) error {
+ jobName := job.Name()
+ if entryID, ok := js.entryIDs[jobName]; ok {
+ js.cron.Remove(entryID)
+ delete(js.entryIDs, jobName)
+ }
+ delete(js.jobsByID, jobName)
+
+ if err := js.scheduleJobInternal(ctx, job); err != nil {
+ return err
+ }
+
+ js.jobsByID[jobName] = job
+
+ slog.DebugContext(ctx, "Job scheduled", "name", jobName, "scheduled", js.isJobScheduledInternal(jobName), "contextCanceled", ctx.Err() != nil)
+ return nil
+}
+
+// scheduleJobInternal adds the job to cron. Callers must hold js.mu. The cron
+// closure runs the job with the scheduler's lifecycle context (js.context), never
+// the per-call ctx — the per-call ctx may be a request context that is canceled
+// once the originating handler returns, which would silently kill future fires.
func (js *JobScheduler) scheduleJobInternal(ctx context.Context, job schedulertypes.Job) error {
if conditionalJob, ok := job.(schedulertypes.ConditionalJob); ok && !conditionalJob.ShouldSchedule(ctx) {
slog.DebugContext(ctx, "Job disabled; not scheduling", "name", job.Name())
@@ -91,9 +158,9 @@ func (js *JobScheduler) scheduleJobInternal(ctx context.Context, job schedulerty
slog.InfoContext(ctx, "Starting Job", "name", job.Name(), "schedule", schedule)
entryID, err := js.cron.AddFunc(schedule, func() {
- slog.InfoContext(ctx, "Job starting", "name", job.Name(), "schedule", schedule)
- job.Run(ctx)
- slog.InfoContext(ctx, "Job finished", "name", job.Name())
+ slog.InfoContext(js.context, "Job starting", "name", job.Name(), "schedule", schedule)
+ job.Run(js.context)
+ slog.InfoContext(js.context, "Job finished", "name", job.Name())
})
if err != nil {
return err
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
index 5e1c6cda5..854af182b 100644
--- a/backend/pkg/scheduler/scheduler_test.go
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"time"
+ schedulertypes "github.com/getarcaneapp/arcane/types/scheduler"
"github.com/stretchr/testify/require"
)
@@ -184,3 +185,66 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T
t.Fatal("scheduled job did not observe lifecycle cancellation")
}
}
+
+func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {
+ js := NewJobScheduler(context.Background(), nil)
+
+ job := &testSchedulerJob{name: "dyn-upsert", schedule: "*/5 * * * * *"}
+ require.NoError(t, js.AddJob(context.Background(), job))
+ require.True(t, js.HasJob(job.Name()))
+ require.Len(t, js.cron.Entries(), 1)
+ firstEntry := js.entryIDs[job.Name()]
+
+ // Re-adding with a changed schedule (e.g. a new sync interval) must replace the
+ // existing cron entry, not leak a second one that keeps firing forever.
+ job.schedule = "*/10 * * * * *"
+ require.NoError(t, js.AddJob(context.Background(), job))
+ require.Len(t, js.cron.Entries(), 1)
+ require.NotEqual(t, firstEntry, js.entryIDs[job.Name()])
+}
+
+func TestJobScheduler_AddJob_InvalidRescheduleClearsRegisteredState(t *testing.T) {
+ js := NewJobScheduler(context.Background(), nil)
+
+ job := &testSchedulerJob{name: "dyn-invalid-reschedule", schedule: "*/5 * * * * *"}
+ require.NoError(t, js.AddJob(context.Background(), job))
+ require.True(t, js.HasJob(job.Name()))
+ require.Len(t, js.cron.Entries(), 1)
+
+ job.schedule = "not a cron schedule"
+ require.Error(t, js.AddJob(context.Background(), job))
+ require.False(t, js.HasJob(job.Name()))
+ require.NotContains(t, js.entryIDs, job.Name())
+ require.Empty(t, js.cron.Entries())
+}
+
+func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {
+ js := NewJobScheduler(context.Background(), nil)
+
+ // Removing an unknown job must be a safe no-op (e.g. deleting a sync that never
+ // had auto-sync enabled).
+ js.RemoveJob(context.Background(), "never-registered")
+
+ job := &testSchedulerJob{name: "dyn-remove", schedule: "*/5 * * * * *"}
+ require.NoError(t, js.AddJob(context.Background(), job))
+ require.True(t, js.HasJob(job.Name()))
+ require.Len(t, js.cron.Entries(), 1)
+
+ js.RemoveJob(context.Background(), job.Name())
+ require.False(t, js.HasJob(job.Name()))
+ require.NotContains(t, js.entryIDs, job.Name())
+ require.Empty(t, js.cron.Entries())
+}
+
+func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T) {
+ js := NewJobScheduler(context.Background(), nil)
+
+ job := &schedulertypes.GenericJob{
+ JobName: "generic-dyn",
+ ScheduleFn: func(context.Context) string { return "@every 1m" },
+ RunFn: func(context.Context) {},
+ }
+ require.NoError(t, js.AddJob(context.Background(), job))
+ require.True(t, js.HasJob(job.Name()))
+ require.Len(t, js.cron.Entries(), 1)
+}
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 2b11fe281..7f0a49e23 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -1532,8 +1532,6 @@
"jobs_image_polling_description": "Checks container registries for new image versions",
"jobs_scheduled_prune_name": "Scheduled Prune",
"jobs_scheduled_prune_description": "Removes unused containers, images, volumes, and networks",
- "jobs_gitops_sync_name": "GitOps Sync",
- "jobs_gitops_sync_description": "Synchronizes project state with configured git repositories",
"jobs_filesystem_watcher_name": "Filesystem Watcher",
"jobs_filesystem_watcher_description": "Monitors project directory for changes and syncs automatically",
"jobs_cron_expression": "Cron Expression",
diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.ts
index 5e4ad5349..8ab407b44 100644
--- a/frontend/src/lib/types/settings.ts
+++ b/frontend/src/lib/types/settings.ts
@@ -151,7 +151,6 @@ export type JobSchedules = {
dockerClientRefreshInterval: string;
pollingInterval: string;
scheduledPruneInterval: string;
- gitopsSyncInterval: string;
vulnerabilityScanInterval: string;
autoHealInterval: string;
};
diff --git a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
index 19da64703..e51954b9f 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
@@ -75,8 +75,6 @@
case 'pollingEnabled':
case 'autoUpdate':
return `${envBase}?tab=docker`;
- case 'gitopsSyncEnabled':
- return `${envBase}?tab=gitops`;
case 'scheduledPruneEnabled':
return `${envBase}?tab=jobs`;
case 'vulnerabilityScanEnabled':
@@ -148,7 +146,7 @@
{ id: 'telemetry', label: m.jobs_telemetry_heading() }
];
- const hiddenJobIds = new Set(['analytics-heartbeat', 'gitops-sync', 'filesystem-watcher']);
+ const hiddenJobIds = new Set(['analytics-heartbeat', 'filesystem-watcher']);
function getJobsByCategory(categoryId: string, jobs: JobStatus[]): JobStatus[] {
return jobs.filter((j) => {
diff --git a/types/jobschedule/jobschedule.go b/types/jobschedule/jobschedule.go
index 03b9f5fca..5889cfcf1 100644
--- a/types/jobschedule/jobschedule.go
+++ b/types/jobschedule/jobschedule.go
@@ -14,7 +14,6 @@ type Config struct {
DockerClientRefreshInterval string `json:"dockerClientRefreshInterval"`
PollingInterval string `json:"pollingInterval"`
ScheduledPruneInterval string `json:"scheduledPruneInterval"`
- GitopsSyncInterval string `json:"gitopsSyncInterval"`
VulnerabilityScanInterval string `json:"vulnerabilityScanInterval"`
AutoHealInterval string `json:"autoHealInterval"`
}
@@ -30,7 +29,6 @@ type Update struct {
DockerClientRefreshInterval *string `json:"dockerClientRefreshInterval,omitempty"`
PollingInterval *string `json:"pollingInterval,omitempty"`
ScheduledPruneInterval *string `json:"scheduledPruneInterval,omitempty"`
- GitopsSyncInterval *string `json:"gitopsSyncInterval,omitempty"`
VulnerabilityScanInterval *string `json:"vulnerabilityScanInterval,omitempty"`
AutoHealInterval *string `json:"autoHealInterval,omitempty"`
}
diff --git a/types/meta/job_metadata.go b/types/meta/job_metadata.go
index 1715ab322..d333839a7 100644
--- a/types/meta/job_metadata.go
+++ b/types/meta/job_metadata.go
@@ -146,24 +146,6 @@ var jobMetadataRegistry = map[string]JobMetadata{
},
},
},
- "gitops-sync": {
- ID: "gitops-sync",
- Name: "GitOps Sync",
- Description: "Synchronizes project state with configured git repositories",
- Category: "sync",
- SettingsKey: "gitopsSyncInterval",
- EnabledKey: "gitopsSyncEnabled",
- ManagerOnly: false,
- IsContinuous: false,
- CanRunManually: true,
- Prerequisites: []JobPrerequisiteMetadata{
- {
- SettingKey: "gitopsSyncEnabled",
- Label: "GitOps sync enabled",
- SettingsURL: "/settings/gitops",
- },
- },
- },
"filesystem-watcher": {
ID: "filesystem-watcher",
Name: "Filesystem Watcher",
diff --git a/types/scheduler/job.go b/types/scheduler/job.go
index 372ef3123..f35fe5541 100644
--- a/types/scheduler/job.go
+++ b/types/scheduler/job.go
@@ -13,3 +13,35 @@ type Job interface {
type ConditionalJob interface {
ShouldSchedule(ctx context.Context) bool
}
+
+// GenericJob is a reusable Job built from closures. It lets a service register a
+// per-entity dynamic job (e.g. one per GitOps sync or one per environment) without
+// importing the scheduler package: the service constructs a GenericJob and hands it
+// to the scheduler through the types/scheduler.Job interface.
+//
+// JobName must be unique per logical job; per-entity jobs use a "<subsystem>:<entityID>"
+// scheme (e.g. "gitops-sync:abc123"). ShouldRunFn is optional — when nil the job is
+// always scheduled, matching the behavior of a Job that does not implement
+// ConditionalJob.
+type GenericJob struct {
+ JobName string
+ ScheduleFn func(ctx context.Context) string
+ RunFn func(ctx context.Context)
+ ShouldRunFn func(ctx context.Context) bool
+}
+
+func (g *GenericJob) Name() string { return g.JobName }
+
+func (g *GenericJob) Schedule(ctx context.Context) string { return g.ScheduleFn(ctx) }
+
+func (g *GenericJob) Run(ctx context.Context) { g.RunFn(ctx) }
+
+// ShouldSchedule satisfies ConditionalJob. A GenericJob without a ShouldRunFn is
+// always scheduled; the scheduler treats a ConditionalJob returning false as "do
+// not schedule", so nil must map to true rather than a nil-func panic.
+func (g *GenericJob) ShouldSchedule(ctx context.Context) bool {
+ if g.ShouldRunFn == nil {
+ return true
+ }
+ return g.ShouldRunFn(ctx)
+}