Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix(scheduler): bound environment health sync concurrency and prevent overlapping runs (#1813)
Kyle Mendell90fb163163e6
diff --git a/backend/pkg/scheduler/environment_health_job.go b/backend/pkg/scheduler/environment_health_job.goindex 06cd0046d..b992227f5 100644--- a/backend/pkg/scheduler/environment_health_job.go+++ b/backend/pkg/scheduler/environment_health_job.go@@ -5,19 +5,38 @@ import ("fmt""log/slog""strconv"+ "sync/atomic"+ "time""github.com/getarcaneapp/arcane/backend/internal/services"+ "golang.org/x/sync/errgroup")+const (+ defaultEnvironmentSyncConcurrency = 4+ defaultEnvironmentSyncTimeout = 90 * time.Second+)++type healthEnvironment struct {+ ID string+ Name string+ Enabled bool+}+type EnvironmentHealthJob struct {environmentService *services.EnvironmentServicesettingsService *services.SettingsService+ syncConcurrency int+ syncTimeout time.Duration+ running atomic.Bool}func NewEnvironmentHealthJob(environmentService *services.EnvironmentService, settingsService *services.SettingsService) *EnvironmentHealthJob {return &EnvironmentHealthJob{environmentService: environmentService,settingsService: settingsService,+ syncConcurrency: defaultEnvironmentSyncConcurrency,+ syncTimeout: defaultEnvironmentSyncTimeout,}}@@ -46,22 +65,20 @@ func (j *EnvironmentHealthJob) Schedule(ctx context.Context) string {}func (j *EnvironmentHealthJob) Run(ctx context.Context) {+ if !j.running.CompareAndSwap(false, true) {+ slog.WarnContext(ctx, "environment health check skipped; previous run still in progress")+ return+ }+ defer j.running.Store(false)+slog.InfoContext(ctx, "environment health check started")// Get all environments using the DB directlydb := j.environmentService.GetDB()- var environments []struct {- ID string- Name string- Enabled bool- }+ var environments []healthEnvironmentif err := db.WithContext(ctx).- Model(&struct {- ID string `gorm:"column:id"`- Name string `gorm:"column:name"`- Enabled bool `gorm:"column:enabled"`- }{}).+ Model(&healthEnvironment{}).Table("environments").Where("enabled = ?", true).Find(&environments).Error; err != nil {@@ -72,6 +89,8 @@ func (j *EnvironmentHealthJob) Run(ctx context.Context) {checkedCount := 0onlineCount := 0offlineCount := 0+ syncedRemoteCount := 0+ var onlineRemote []healthEnvironmentfor _, env := range environments {checkedCount++@@ -84,44 +103,72 @@ func (j *EnvironmentHealthJob) Run(ctx context.Context) {offlineCount++case status == "online":onlineCount++- // Sync registries and git repositories to online remote environments (skip local environment ID "0")+ // Queue sync for online remote environments (skip local environment ID "0")if env.ID != "0" {- go func(envID, envName string) {- syncCtx := context.WithoutCancel(ctx)- if err := j.environmentService.SyncRegistriesToEnvironment(syncCtx, envID); err != nil {- slog.WarnContext(syncCtx, "failed to sync registries during health check",- "environment_id", envID,- "environment_name", envName,- "error", err)- } else {- slog.DebugContext(syncCtx, "successfully synced registries during health check",- "environment_id", envID,- "environment_name", envName)- }- }(env.ID, env.Name)- go func(envID, envName string) {- syncCtx := context.WithoutCancel(ctx)- if err := j.environmentService.SyncRepositoriesToEnvironment(syncCtx, envID); err != nil {- slog.WarnContext(syncCtx, "failed to sync git repositories during health check",- "environment_id", envID,- "environment_name", envName,- "error", err)- } else {- slog.DebugContext(syncCtx, "successfully synced git repositories during health check",- "environment_id", envID,- "environment_name", envName)- }- }(env.ID, env.Name)+ onlineRemote = append(onlineRemote, env)+ syncedRemoteCount++}default:offlineCount++}}- slog.InfoContext(ctx, "environment health check completed", "checked", checkedCount, "online", onlineCount, "offline", offlineCount)+ j.syncOnlineRemoteEnvironments(ctx, onlineRemote)+ slog.InfoContext(ctx, "environment health check completed", "checked", checkedCount, "online", onlineCount, "offline", offlineCount, "remote_sync_queued", syncedRemoteCount)}func (j *EnvironmentHealthJob) Reschedule(ctx context.Context) error {slog.InfoContext(ctx, "rescheduling environment health job in new scheduler; currently requires restart")return nil}++func (j *EnvironmentHealthJob) syncOnlineRemoteEnvironments(ctx context.Context, environments []healthEnvironment) {+ if len(environments) == 0 {+ return+ }++ syncCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), j.syncTimeout)+ defer cancel()++ g, groupCtx := errgroup.WithContext(syncCtx)+ if j.syncConcurrency > 0 {+ g.SetLimit(j.syncConcurrency)+ }++ for _, env := range environments {+ g.Go(func() error {+ if err := j.environmentService.SyncRegistriesToEnvironment(groupCtx, env.ID); err != nil {+ slog.WarnContext(groupCtx, "failed to sync registries during health check",+ "environment_id", env.ID,+ "environment_name", env.Name,+ "error", err)+ return nil+ }++ slog.DebugContext(groupCtx, "successfully synced registries during health check",+ "environment_id", env.ID,+ "environment_name", env.Name)+ return nil+ })++ g.Go(func() error {+ if err := j.environmentService.SyncRepositoriesToEnvironment(groupCtx, env.ID); err != nil {+ slog.WarnContext(groupCtx, "failed to sync git repositories during health check",+ "environment_id", env.ID,+ "environment_name", env.Name,+ "error", err)+ return nil+ }++ slog.DebugContext(groupCtx, "successfully synced git repositories during health check",+ "environment_id", env.ID,+ "environment_name", env.Name)+ return nil+ })+ }++ _ = g.Wait()+ if syncCtx.Err() != nil {+ slog.WarnContext(ctx, "environment health sync phase timed out or canceled", "error", syncCtx.Err())+ }+}diff --git a/backend/pkg/scheduler/environment_health_job_test.go b/backend/pkg/scheduler/environment_health_job_test.gonew file mode 100644index 000000000..4e83a7f22--- /dev/null+++ b/backend/pkg/scheduler/environment_health_job_test.go@@ -0,0 +1,25 @@+package scheduler++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestNewEnvironmentHealthJob_Defaults(t *testing.T) {+ job := NewEnvironmentHealthJob(nil, nil)++ require.Equal(t, defaultEnvironmentSyncConcurrency, job.syncConcurrency)+ require.Equal(t, defaultEnvironmentSyncTimeout, job.syncTimeout)+ require.False(t, job.running.Load())+}++func TestEnvironmentHealthJob_RunGuardAtomic(t *testing.T) {+ job := &EnvironmentHealthJob{}++ require.True(t, job.running.CompareAndSwap(false, true))+ require.False(t, job.running.CompareAndSwap(false, true))++ job.running.Store(false)+ require.True(t, job.running.CompareAndSwap(false, true))+}