Skip to content

Modern Docker Management, Designed for Everyone

22.3 KB
Raw Blame
package bootstrap

import (
	"cmp"
	"context"
	"encoding/json/v2"
	"errors"
	"fmt"
	"log/slog"
	"maps"
	"net/http"
	"slices"
	"sync"

	"github.com/getarcaneapp/arcane/types/v2/features"
	schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
	"github.com/italypaleale/francis/actor"
	"go.uber.org/fx"

	"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
	"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
	"github.com/getarcaneapp/arcane/backend/v2/internal/config"
	"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
	"github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
	"github.com/getarcaneapp/arcane/backend/v2/internal/job"
	"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
	"github.com/getarcaneapp/arcane/backend/v2/internal/system"
	"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)

func newJobScheduler(
	appCtx context.Context,
	lc fx.Lifecycle,
	cfg *config.Config,
	runtime *runs.Coordinator,
	_ *runs.Admission,
	imageUpdateWatcher *scheduler.ImageUpdateWatcher,
	analytics *scheduler.AnalyticsJob,
	systemService *system.SystemService,
	jobService *job.JobService,
	workflows *flow.Engine,
	volumes *volume.VolumeService,
	gitopsSync *gitops.GitOpsSyncService,
	actors *francis.Runtime,
) (
	schedulertypes.JobScheduler,
	error,
) {
	schedulerCtx, cancelScheduler := context.WithCancel(appCtx)
	jobScheduler, err := scheduler.NewJobScheduler(schedulerCtx, jobService.Coordinator(), cfg.GetLocation())
	if err != nil {
		cancelScheduler()
		return nil, err
	}
	jobService.SetScheduler(schedulerCtx, jobScheduler)
	var resumeDone chan struct{}
	lc.Append(fx.Hook{
		OnStart: func(ctx context.Context) error {
			started := false
			defer func() {
				if !started {
					cancelScheduler()
				}
			}()
			slog.InfoContext(appCtx, "Starting scheduler")
			if startErr := jobService.Coordinator().Start(ctx, schedulerCtx); startErr != nil {
				return fmt.Errorf("initialize scheduler: %w", startErr)
			}
			if reconcileCoordinatorStartupErr := reconcileCoordinatorStartup(ctx, jobService, workflows, volumes, systemService, gitopsSync); reconcileCoordinatorStartupErr != nil {
				return fmt.Errorf("reconcile scheduler startup: %w", reconcileCoordinatorStartupErr)
			}
			settleLegacyActorWork(ctx, actors.Service())
			if gitopsSync != nil {
				gitopsSync.RegisterAutoSyncJobsOnStartup(ctx)
				gitopsSync.SubscribeProjectFileChanges(schedulerCtx)
			}
			if imageUpdateWatcher != nil {
				imageUpdateWatcher.SetCoordinator(jobService.Coordinator())
				if registerBusWatcherErr := jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true); registerBusWatcherErr != nil {
					return registerBusWatcherErr
				}
			}
			if startSchedulerErr := jobScheduler.StartScheduler(ctx); startSchedulerErr != nil {
				return fmt.Errorf("start scheduler: %w", startSchedulerErr)
			}
			jobService.Coordinator().Activate()
			if analytics != nil {
				if _, submitErr := jobService.Coordinator().Submit(ctx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); submitErr != nil {
					return fmt.Errorf("submit startup job %q: %w", analytics.Name(), submitErr)
				}
			}
			// Standalone tasks wait until update-all recovery has read its job, so a resumed step's new checkpoint
			// is never mistaken for one the previous process left.
			if !cfg.AgentMode && systemService != nil {
				resumeDone = make(chan struct{})
				go func() {
					defer close(resumeDone)
					systemService.ResumeUpdateAllOnStartup(schedulerCtx)
					workflows.Ready()
				}()
			} else {
				workflows.Ready()
			}
			if startupCancellationErr := ctx.Err(); startupCancellationErr != nil {
				return fmt.Errorf("complete scheduler startup: %w", startupCancellationErr)
			}
			started = true
			return nil
		},
		OnStop: func(ctx context.Context) error {
			cancelScheduler()
			resumeErr := func() error {
				if resumeDone == nil {
					return nil
				}
				select {
				case <-resumeDone:
					return nil
				case <-ctx.Done():
					return ctx.Err()
				}
			}()
			if stopErr := errors.Join(resumeErr, jobService.Coordinator().Stop(ctx), jobScheduler.Stop(ctx)); stopErr != nil {
				slog.ErrorContext(ctx, "Job scheduler exited with error", "error", stopErr)
				return stopErr
			}
			slog.InfoContext(ctx, "Scheduler stopped")
			return nil
		},
	})
	return jobScheduler, nil
}

// legacyActorWork names the state and job actor types of the actors workflows replaced.
// TODO(v3): remove once no supported upgrade path can carry their saved work.
var legacyActorWork = []struct{ stateType, jobType, jobActorID string }{
	{stateType: "backup-run", jobType: "backup-run"},
	{stateType: "single-container-update-state", jobType: "single-container-update", jobActorID: "single-container"},
}

// settleLegacyActorWork removes the saved commands and pending jobs of the replaced backup and container-update
// actors. Nothing delivers them any more; the startup reconcile above has already failed their records and activities.
func settleLegacyActorWork(ctx context.Context, service *actor.Service) {
	for _, legacy := range legacyActorWork {
		settled := 0
		for cursor := ""; ; {
			page, err := service.ListStates(ctx, legacy.stateType, &actor.ListStatesOpts{After: cursor, Limit: 100})
			if err != nil {
				slog.WarnContext(ctx, "Failed to list work saved by a replaced actor", "actorType", legacy.stateType, "error", err)
				break
			}
			for _, state := range page.States {
				jobs, listErr := service.ListJobs(ctx, legacy.jobType, cmp.Or(legacy.jobActorID, state.ActorID))
				for _, job := range jobs {
					listErr = errors.Join(listErr, service.DeleteJob(ctx, legacy.jobType, cmp.Or(legacy.jobActorID, state.ActorID), job.JobID))
				}
				if settleErr := errors.Join(listErr, service.DeleteState(ctx, legacy.stateType, state.ActorID)); settleErr != nil && !errors.Is(settleErr, actor.ErrStateNotFound) {
					slog.WarnContext(ctx, "Failed to settle work saved by a replaced actor", "actorType", legacy.stateType, "actorId", state.ActorID, "error", settleErr)
					continue
				}
				settled++
			}
			if cursor = page.AfterID(); cursor == "" {
				break
			}
		}
		if settled > 0 {
			slog.InfoContext(ctx, "Settled work saved by a replaced actor; interrupted backups and updates were marked failed", "actorType", legacy.stateType, "count", settled)
		}
	}
}

// reconcileCoordinatorStartup fails backups and activities the previous process left running,
// except those that unfinished runs and standalone workflows still own.
func reconcileCoordinatorStartup(
	ctx context.Context,
	jobs *job.JobService,
	workflows *flow.Engine,
	volumes *volume.VolumeService,
	systemService *system.SystemService,
	syncService *gitops.GitOpsSyncService,
) error {
	records, err := jobs.Coordinator().Records(ctx)
	if err != nil {
		return err
	}
	standalone, err := workflows.Standalone(ctx)
	if err != nil {
		return err
	}
	activities, backups := []string{}, []string{}
	for _, record := range append(records, schedulertypes.QueueRecord{Runs: standalone}) {
		for _, run := range record.Runs {
			if run.Status.Terminal() {
				continue
			}
			for _, target := range run.Outcome.Targets {
				var checkpoint struct {
					BackupID string `json:"backupId"`
				}
				if len(target.RecoveryData) == 0 {
					continue
				}
				if decodeErr := json.Unmarshal(target.RecoveryData, &checkpoint); decodeErr != nil {
					return fmt.Errorf("decode startup recovery evidence: %w", decodeErr)
				}
				if checkpoint.BackupID != "" {
					backups = append(backups, checkpoint.BackupID)
				}
			}
		}
	}
	for _, run := range standalone {
		if run.ActivityID != "" {
			activities = append(activities, run.ActivityID)
		}
	}
	if volumes != nil {
		if reconcileInterruptedBackupsErr := volumes.ReconcileInterruptedBackups(ctx, backups...); reconcileInterruptedBackupsErr != nil {
			return reconcileInterruptedBackupsErr
		}
	}
	if systemService != nil {
		if reconcileSystemBackupsErr := systemService.ReconcileInterruptedBackups(ctx, backups...); reconcileSystemBackupsErr != nil {
			return reconcileSystemBackupsErr
		}
	}
	if syncService != nil {
		if reconcileInterruptedBackupsOnStartupErr := syncService.ReconcileInterruptedBackupsOnStartup(ctx, backups...); reconcileInterruptedBackupsOnStartupErr != nil {
			return reconcileInterruptedBackupsOnStartupErr
		}
	}
	return jobs.ReconcileStartupActivities(ctx, activities...)
}

type registerJobsParams struct {
	fx.In

	Lifecycle fx.Lifecycle
	AppCtx    context.Context
	Config    *config.Config
	Scheduler schedulertypes.JobScheduler

	Activity    *activity.ActivityService
	GitOpsSync  *gitops.GitOpsSyncService
	Environment *environment.EnvironmentService
	JobSchedule *job.JobService
	Settings    *settings.SettingsService
	Volume      *volume.VolumeService
	System      *system.SystemService
	Admission   *runs.Admission
	Apns        *apns.ApnsService

	AutoUpdate             *flow.Job `name:"auto-update"`
	ImageUpdateWatcher     *scheduler.ImageUpdateWatcher
	DockerClientRefresh    *scheduler.DockerClientRefreshJob
	Analytics              *scheduler.AnalyticsJob
	UpgradeLogCleanup      schedulertypes.Job `name:"upgrade-log-cleanup"`
	EventCleanup           *scheduler.EventCleanupJob
	PruningVolumeHelper    *scheduler.PruningVolumeHelperJob
	ExpiredSessionsCleanup *scheduler.ExpiredSessionsCleanupJob
	ScheduledPrune         *scheduler.ScheduledPruneJob
	FilesystemWatcher      *scheduler.FilesystemWatcherJob
	VulnerabilityScan      *flow.Job `name:"vulnerability-scan"`
	VulnerabilityRisk      *scheduler.VulnerabilityRiskJob
	AutoPatch              *flow.Job `name:"auto-patch"`
	AutoHeal               *scheduler.AutoHealJob
	ActivitySweep          *scheduler.ActivitySweepJob
	UploadSessionsCleanup  *scheduler.UploadSessionsCleanupJob
	GitCloneCleanup        *scheduler.GitCloneCleanupJob
	BackupRepositoryPrune  *scheduler.BackupRepositoryPruneJob
	ApnsOutbox             *scheduler.ApnsOutboxJob
}

func registerJobs(params registerJobsParams) error {
	params.JobSchedule.SetScheduler(params.AppCtx, params.Scheduler)

	for _, job := range []schedulertypes.Job{
		params.AutoUpdate,
		params.DockerClientRefresh,
		params.Analytics,
		params.EventCleanup,
		params.PruningVolumeHelper,
		params.ExpiredSessionsCleanup,
		params.ScheduledPrune,
		params.VulnerabilityScan,
		params.VulnerabilityRisk,
		params.AutoPatch,
		params.AutoHeal,
		params.ActivitySweep,
		params.UploadSessionsCleanup,
		params.GitCloneCleanup,
		params.BackupRepositoryPrune,
		params.UpgradeLogCleanup,
		params.ApnsOutbox,
	} {
		if err := params.Scheduler.RegisterJob(job); err != nil {
			return err
		}
	}
	// FilesystemWatcher is intentionally not scheduler-registered; it watches inline
	// and is only rebound on settings changes below.
	// Internal self-healing sweep (managers and agents alike); intentionally
	// absent from job_metadata so it stays out of the Jobs UI.

	// GitOps sync and environment health are no longer single global jobs; each
	// entity registers its own dynamic job.
	if err := registerDynamicJobs(dynamicJobsParams{
		AppCtx:      params.AppCtx,
		Config:      params.Config,
		Scheduler:   params.Scheduler,
		GitOpsSync:  params.GitOpsSync,
		Environment: params.Environment,
		JobSchedule: params.JobSchedule,
		Volume:      params.Volume,
		System:      params.System,
		Admission:   params.Admission,
	}); err != nil {
		return err
	}

	if err := setupSettingsSubscriptions(settingsSubscriptionsParams{
		Lifecycle:          params.Lifecycle,
		LifecycleCtx:       params.AppCtx,
		Config:             params.Config,
		Scheduler:          params.Scheduler,
		Settings:           params.Settings,
		Environment:        params.Environment,
		JobSchedule:        params.JobSchedule,
		AutoUpdate:         params.AutoUpdate,
		ImageUpdateWatcher: params.ImageUpdateWatcher,
		FilesystemWatcher:  params.FilesystemWatcher,
		ScheduledPrune:     params.ScheduledPrune,
		VulnerabilityScan:  params.VulnerabilityScan,
		VulnerabilityRisk:  params.VulnerabilityRisk,
		AutoPatch:          params.AutoPatch,
		AutoHeal:           params.AutoHeal,
		Apns:               params.Apns,
		ApnsOutbox:         params.ApnsOutbox,
	}); err != nil {
		return err
	}
	return nil
}

type dynamicJobsParams struct {
	AppCtx      context.Context
	Config      *config.Config
	Scheduler   schedulertypes.JobScheduler
	GitOpsSync  *gitops.GitOpsSyncService
	Environment *environment.EnvironmentService
	JobSchedule *job.JobService
	Volume      *volume.VolumeService
	System      *system.SystemService
	Admission   *runs.Admission
}

// registerDynamicJobs injects the scheduler into the services that own per-entity
// jobs and registers the jobs for already-existing entities at startup. AddJob is
// an idempotent upsert, so these run safely before the scheduler is started.
func registerDynamicJobs(params dynamicJobsParams) error {
	// Volume backups run on the environment that owns the Docker volume. This is
	// registered on managers and agents; environment proxying persists each policy
	// in the correct Arcane database.
	if params.Volume != nil {
		if err := params.Volume.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
			return err
		}
		params.Volume.RegisterBackupJobsOnStartup(params.AppCtx)
	}
	if !params.Config.AgentMode && params.System != nil {
		if err := params.System.SetBackupScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
			return err
		}
		params.System.RegisterBackupJobOnStartup(params.AppCtx)
	}
	// GitOps startup submissions wait for Francis in the scheduler start hook.
	if params.GitOpsSync != nil {
		if err := params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
			return err
		}
	}

	// Environment health: one job per enabled environment (manager only). The Jobs
	// UI still addresses "environment-health" by ID, so bridge its reschedule and
	// run-now back to EnvironmentService.
	if !params.Config.AgentMode && params.Environment != nil {
		if err := params.Environment.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
			return err
		}
		params.JobSchedule.OnEnvironmentHealthReschedule = func(ctx context.Context) {
			params.Environment.RescheduleHealthJobs(ctx)
		}
		params.JobSchedule.RunEnvironmentHealthNow = func(ctx context.Context) error {
			return params.Environment.RunHealthChecksNow(ctx)
		}
		params.Environment.RescheduleHealthJobs(params.AppCtx)
	}
	return nil
}

type settingsSubscriptionsParams struct {
	Lifecycle    fx.Lifecycle
	LifecycleCtx context.Context
	Config       *config.Config
	Scheduler    settingsEffectsScheduler
	Settings     settingsChangeSubscriber
	Environment  timeoutSettingsEnvironment
	JobSchedule  *job.JobService

	AutoUpdate         *flow.Job
	ImageUpdateWatcher *scheduler.ImageUpdateWatcher
	FilesystemWatcher  *scheduler.FilesystemWatcherJob
	ScheduledPrune     *scheduler.ScheduledPruneJob
	VulnerabilityScan  *flow.Job
	VulnerabilityRisk  *scheduler.VulnerabilityRiskJob
	AutoPatch          *flow.Job
	AutoHeal           *scheduler.AutoHealJob
	Apns               *apns.ApnsService
	ApnsOutbox         *scheduler.ApnsOutboxJob
}

type settingsChangeSubscriber interface {
	SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func()
}

type settingsEffectsScheduler interface {
	RescheduleJob(ctx context.Context, job schedulertypes.Job) error
}

type timeoutSettingsEnvironment interface {
	ListRemoteEnvironments(ctx context.Context) ([]environment.Environment, error)
	ProxyRequest(ctx context.Context, envID, method, path string, body []byte) ([]byte, int, error)
}

func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {
	unsubscribes := make([]func(), 0, 8)
	subscribe := func(keys []string, callback func([]libarcane.SettingUpdate)) {
		unsubscribes = append(unsubscribes, params.Settings.SubscribeSettingsChanges(keys, callback))
	}
	rescheduleOn := func(keys []string, job schedulertypes.Job) {
		subscribe(keys, func(_ []libarcane.SettingUpdate) {
			if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, job); err != nil {
				slog.WarnContext(params.LifecycleCtx, "Failed to reschedule job", "job", job.Name(), "error", err)
			}
		})
	}
	timeoutSyncExecutor, cancelTimeoutSync := setupTimeoutSettingsSubscription(params, subscribe)

	subscribe([]string{"pollingEnabled", "pollingInterval"}, func(_ []libarcane.SettingUpdate) {
		params.ImageUpdateWatcher.RefreshSchedule()
		params.ImageUpdateWatcher.Trigger()
		if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoUpdate); err != nil {
			slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-update job", "error", err)
		}
	})

	subscribe([]string{"apnsEnabled"}, func(updates []libarcane.SettingUpdate) {
		params.Apns.ApplyEnabledUpdates(params.LifecycleCtx, updates, func() error {
			return params.Scheduler.RescheduleJob(params.LifecycleCtx, params.ApnsOutbox)
		})
	})

	rescheduleOn([]string{"autoUpdate", "autoUpdateInterval"}, params.AutoUpdate)

	subscribe([]string{"dockerClientRefreshInterval", "environmentHealthInterval"}, func(updates []libarcane.SettingUpdate) {
		changedKeys := make([]string, 0, len(updates))
		for _, update := range updates {
			changedKeys = append(changedKeys, update.Key)
		}
		if err := params.JobSchedule.RescheduleJobsForSettingKeys(params.LifecycleCtx, changedKeys); err != nil {
			slog.WarnContext(params.LifecycleCtx, "Failed to reschedule jobs after settings change", "keys", changedKeys, "error", err)
		}
	})

	subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {
		if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {
			slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)
			params.FilesystemWatcher.RequestRestart()
		}
	})

	subscribe([]string{"templatesDirectory"}, func(_ []libarcane.SettingUpdate) {
		if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {
			slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)
			params.FilesystemWatcher.RequestRestart()
		}
	})

	rescheduleOn([]string{"scheduledPruneEnabled", "scheduledPruneInterval"}, params.ScheduledPrune)
	rescheduleOn([]string{
		features.VulnerabilityManagementSettingKey, "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyNetwork", "trivySecurityOpts", "trivyPrivileged",
		"trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers",
	}, params.VulnerabilityScan)
	rescheduleOn([]string{features.VulnerabilityManagementSettingKey}, params.VulnerabilityRisk)
	rescheduleOn([]string{features.VulnerabilityManagementSettingKey, "imageAutoPatchEnabled", "imageAutoPatchInterval"}, params.AutoPatch)
	rescheduleOn([]string{"autoHealEnabled", "autoHealInterval", "autoHealExcludedContainers", "autoHealMaxRestarts", "autoHealRestartWindow"}, params.AutoHeal)

	params.Lifecycle.Append(fx.Hook{
		OnStop: func(ctx context.Context) error {
			for _, unsubscribe := range slices.Backward(unsubscribes) {
				unsubscribe()
			}
			if cancelTimeoutSync != nil {
				cancelTimeoutSync()
			}
			return timeoutSyncExecutor.Stop(ctx)
		},
	})
	return nil
}

type timeoutSyncWorker struct {
	mu      sync.Mutex
	updates map[string]string
	wake    chan struct{}
	done    chan struct{}
	ctx     context.Context
	cancel  context.CancelFunc
	closed  bool
}

func (w *timeoutSyncWorker) Stop(ctx context.Context) error {
	if w == nil {
		return nil
	}
	w.mu.Lock()
	w.closed = true
	w.cancel()
	w.mu.Unlock()
	select {
	case <-w.done:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*timeoutSyncWorker, context.CancelFunc) {
	if params.Config.AgentMode {
		return nil, nil
	}
	ctx, cancel := context.WithCancel(params.LifecycleCtx)
	worker := &timeoutSyncWorker{wake: make(chan struct{}, 1), done: make(chan struct{}), ctx: ctx, cancel: cancel}
	go func() {
		defer close(worker.done)
		for {
			select {
			case <-ctx.Done():
				return
			case <-worker.wake:
			}
			worker.mu.Lock()
			updates := worker.updates
			worker.updates = nil
			worker.mu.Unlock()
			if len(updates) > 0 {
				syncTimeoutSettingsToAgents(ctx, params.Environment, updates)
			}
		}
	}()
	subscribe(libarcane.TimeoutSettingKeys(), func(updates []libarcane.SettingUpdate) {
		worker.mu.Lock()
		defer worker.mu.Unlock()
		if worker.closed {
			return
		}
		if worker.updates == nil {
			worker.updates = make(map[string]string, len(updates))
		}
		for _, update := range updates {
			worker.updates[update.Key] = update.Value
		}
		select {
		case worker.wake <- struct{}{}:
		default:
		}
	})
	return worker, cancel
}

// syncTimeoutSettingsToAgents syncs timeout settings to all connected remote environments
func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSettingsEnvironment, settingsMap map[string]string) {
	envs, err := localEnvironment.ListRemoteEnvironments(ctx)
	if err != nil {
		slog.WarnContext(ctx, "Failed to list remote environments for timeout sync", "error", err)
		return
	}

	if len(envs) == 0 {
		return
	}

	body, err := json.Marshal(settingsMap)
	if err != nil {
		slog.WarnContext(ctx, "Failed to marshal timeout settings for sync", "error", err)
		return
	}

	slog.InfoContext(ctx, "Syncing environment settings to remote environments", "count", len(envs), "keys", slices.Sorted(maps.Keys(settingsMap)))

	for _, env := range envs {
		responseBody, statusCode, proxyRequestErr := localEnvironment.ProxyRequest(ctx, env.ID, http.MethodPut, "/api/environments/0/settings", body)
		if proxyRequestErr != nil {
			slog.WarnContext(ctx, "Failed to sync timeout settings to environment", "environmentId", env.ID, "environmentName", env.Name, "error", proxyRequestErr)
			continue
		}
		if statusCode < http.StatusOK || statusCode >= http.StatusMultipleChoices {
			slog.WarnContext(ctx, "Environment returned non-OK status for timeout sync", "environmentId", env.ID, "environmentName", env.Name, "statusCode", statusCode, "response", string(responseBody))
			continue
		}
		slog.DebugContext(ctx, "Successfully synced timeout settings to environment", "environmentId", env.ID, "environmentName", env.Name)
	}
}