Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix(bootstrap): stop the app when the Francis host fails (#4371)
Kyle Mendelle9b8ee1bf045
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex c0b8c530d..f15fb9402 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -67,17 +67,17 @@ func Bootstrap(ctx context.Context) error {startup.ApplyMemoryLimit(ctx)slog.InfoContext(ctx, "Arcane Identity Configuration", "puid", os.Getuid(), "pgid", os.Getgid())- appCtx, cancelApp := context.WithCancel(ctx)+ appCtx, cancelApp := context.WithCancelCause(ctx)appCtx = utils.WithAppLifecycleContext(appCtx)db, err := database.Initialize(appCtx, cfg.DatabaseURL, database.MigrationOptions{AllowDowngrade: cfg.AllowDowngrade})if err != nil {- cancelApp()+ cancelApp(nil)return fmt.Errorf("failed to initialize database: %w", err)}slog.InfoContext(appCtx, "Database initialized successfully")defer func() {- cancelApp()+ cancelApp(nil)if closeErr := db.Close(); closeErr != nil {slog.ErrorContext(ctx, "Error closing database", "error", closeErr)}@@ -110,30 +110,36 @@ func Bootstrap(ctx context.Context) error {app := fx.New(applicationOptions(appCtx, cfg, db, cancelApp))- startCtx, cancelStart := context.WithTimeout(ctx, 5*time.Minute)+ startCtx, cancelStart := context.WithTimeout(ctx, app.StartTimeout())defer cancelStart()if startErr := app.Start(startCtx); startErr != nil {return fmt.Errorf("start application: %w", startErr)}+ // appCtx also ends when a component stops the app itself; its cause is then the run error.+ var runErr errorselect {- case <-ctx.Done():- slog.InfoContext(appCtx, "Context canceled")+ case <-appCtx.Done():+ if ctx.Err() == nil {+ runErr = context.Cause(appCtx)+ } else {+ slog.InfoContext(appCtx, "Context canceled")+ }case signal := <-app.Done():slog.InfoContext(appCtx, "Received shutdown signal", "signal", signal)}- stopCtx, cancelStop := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)+ stopCtx, cancelStop := context.WithTimeout(context.WithoutCancel(ctx), app.StopTimeout())defer cancelStop()if stopErr := app.Stop(stopCtx); stopErr != nil {- return fmt.Errorf("stop application: %w", stopErr)+ return errors.Join(runErr, fmt.Errorf("stop application: %w", stopErr))}slog.InfoContext(context.WithoutCancel(appCtx), "Arcane shutdown complete")- return nil+ return runErr}-func applicationOptions(appCtx context.Context, cfg *config.Config, db *database.DB, cancelApp context.CancelFunc) fx.Option {+func applicationOptions(appCtx context.Context, cfg *config.Config, db *database.DB, cancelApp context.CancelCauseFunc) fx.Option {return fx.Options(fx.Supply(cfg, db, cancelApp),fx.Provide(@@ -209,9 +215,7 @@ func initializeStartupState(p initializeStartupStateParams) {appCtx := p.AppCtxcfg := p.Config- if p.Volume != nil {- startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)- }+ startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)runtimeCfg := &startup.RuntimeConfig{AgentMode: cfg.AgentMode,@@ -253,26 +257,22 @@ func initializeStartupState(p initializeStartupStateParams) {if err := p.Environment.EnsureLocalEnvironment(appCtx, cfg.AppUrl); err != nil {slog.WarnContext(appCtx, "Failed to ensure local environment", "error", err)}- if p.GitOpsSync != nil {- if err := p.GitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)- }- if err := p.GitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)- }- if err := p.GitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)- }- if err := p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)- }+ if err := p.GitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)+ }+ if err := p.GitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)+ }+ if err := p.GitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)+ }+ if err := p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)}p.Vulnerability.ImportLegacyReportFiles(appCtx)p.Vulnerability.BackfillScanItems(appCtx)- if p.Project != nil {- if err := p.Project.RecoverProjectRenameJournals(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)- }+ if err := p.Project.RecoverProjectRenameJournals(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)}if !cfg.AgentMode {@@ -304,10 +304,8 @@ func initializeStartupState(p initializeStartupStateParams) {slog.InfoContext(ctx, "Docker API versions detected", "clientApiVersion", dockerClient.ClientVersion(), "serverApiVersion", version.APIVersion, "effectiveApiVersion", effectiveAPIVersion)return nil})- if p.Swarm != nil {- if err := p.Swarm.SyncSwarmEnabledState(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)- }+ if err := p.Swarm.SyncSwarmEnabledState(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)}startup.InitializeNonAgentFeatures(appCtx, runtimeCfg,@@ -396,10 +394,10 @@ func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.})}-func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {+func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelCauseFunc) {lc.Append(fx.Hook{OnStop: func(context.Context) error {- cancelApp()+ cancelApp(nil)return nil},})diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 257cea7f0..87fe666c5 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -361,7 +361,7 @@ func TestNewHTTPServerRejectsInvalidTLSCertificate(t *testing.T) {}func TestRegisterAppCancelHookRunsAfterLaterStopHooks(t *testing.T) {- appCtx, cancelApp := context.WithCancel(t.Context())+ appCtx, cancelApp := context.WithCancelCause(t.Context())lifecycle := fxtest.NewLifecycle(t)registerAppCancelHook(lifecycle, cancelApp)appActiveDuringDependencyStop := false@@ -449,8 +449,8 @@ func TestJobSchedulerStopCancelsItsPrivateContext(t *testing.T) {}func TestApplicationOptionsValidate(t *testing.T) {- appCtx, cancelApp := context.WithCancel(t.Context())- defer cancelApp()+ appCtx, cancelApp := context.WithCancelCause(t.Context())+ defer cancelApp(nil)err := fx.ValidateApp(applicationOptions(appCtx,diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex b1fe7e37b..21ea08179 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -406,13 +406,13 @@ type settingsSubscriptionsParams struct {Settings settingsChangeSubscriberEnvironment timeoutSettingsEnvironment- AutoUpdate *flow.Job `name:"auto-update"`+ AutoUpdate *flow.JobImageUpdateWatcher *scheduler.ImageUpdateWatcherFilesystemWatcher *scheduler.FilesystemWatcherJobScheduledPrune *scheduler.ScheduledPruneJob- VulnerabilityScan *flow.Job `name:"vulnerability-scan"`+ VulnerabilityScan *flow.JobVulnerabilityRisk *scheduler.VulnerabilityRiskJob- AutoPatch *flow.Job `name:"auto-patch"`+ AutoPatch *flow.JobAutoHeal *scheduler.AutoHealJobApns *apns.ApnsServiceApnsOutbox *scheduler.ApnsOutboxJob@@ -446,10 +446,8 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {timeoutSyncExecutor, cancelTimeoutSync := setupTimeoutSettingsSubscription(params, subscribe)subscribe([]string{"pollingEnabled", "pollingInterval"}, func(_ []libarcane.SettingUpdate) {- if params.ImageUpdateWatcher != nil {- params.ImageUpdateWatcher.RefreshSchedule()- params.ImageUpdateWatcher.Trigger()- }+ params.ImageUpdateWatcher.RefreshSchedule()+ params.ImageUpdateWatcher.Trigger()if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoUpdate); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-update job", "error", err)}@@ -464,18 +462,14 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {rescheduleOn([]string{"autoUpdate", "autoUpdateInterval"}, params.AutoUpdate)subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {- if params.FilesystemWatcher != nil {- if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {- slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)- }+ if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {+ slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)}})subscribe([]string{"templatesDirectory"}, func(_ []libarcane.SettingUpdate) {- if params.FilesystemWatcher != nil {- if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {- slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)- }+ if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {+ slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)}})diff --git a/backend/internal/bootstrap/jobs_bootstrap_test.go b/backend/internal/bootstrap/jobs_bootstrap_test.goindex eb2d2df26..9c6022276 100644--- a/backend/internal/bootstrap/jobs_bootstrap_test.go+++ b/backend/internal/bootstrap/jobs_bootstrap_test.go@@ -84,13 +84,14 @@ func TestSettingsTimeoutSyncDoesNotBlockOtherEffects(t *testing.T) {localScheduler := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}localEnvironment := &timeoutSyncEnvironmentStubInternal{started: make(chan struct{})}require.NoError(t, setupSettingsSubscriptions(settingsSubscriptionsParams{- Lifecycle: lifecycle,- LifecycleCtx: t.Context(),- Config: &config.Config{},- Scheduler: localScheduler,- Settings: settings,- Environment: localEnvironment,- AutoUpdate: &flow.Job{JobName: "auto-update"},+ Lifecycle: lifecycle,+ LifecycleCtx: t.Context(),+ Config: &config.Config{},+ Scheduler: localScheduler,+ Settings: settings,+ Environment: localEnvironment,+ AutoUpdate: &flow.Job{JobName: "auto-update"},+ ImageUpdateWatcher: &scheduler.ImageUpdateWatcher{},}))timeoutCallbackDone := make(chan struct{})diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.goindex 62067af9a..0da1a084e 100644--- a/backend/internal/di/di_test.go+++ b/backend/internal/di/di_test.go@@ -125,8 +125,8 @@ type graphParams struct {}func TestOptionsValidate(t *testing.T) {- _, cancel := context.WithCancel(t.Context())- defer cancel()+ _, cancel := context.WithCancelCause(t.Context())+ defer cancel(nil)err := fx.ValidateApp(fx.Supply(cancel,diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex 332473f86..46e3fe233 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -3,6 +3,7 @@ package diimport ("context""errors"+ "fmt""log/slog""net/http""time"@@ -104,7 +105,7 @@ func provideActorRuntimeInternal(db *database.DB,settingsService *settings.SettingsService,lc fx.Lifecycle,- cancelApp context.CancelFunc,+ cancelApp context.CancelCauseFunc,) (*francis.Runtime, error) {runtime, err := francis.New(cfg.DatabaseURL, "", "", cfg.ActorPort)if err != nil {@@ -121,7 +122,10 @@ func provideActorRuntimeInternal(if configureIdentityErr := runtime.ConfigureIdentity(cfg.EncryptionKey, settingsService.GetSettingsConfig().InstanceID.Value); configureIdentityErr != nil {return configureIdentityErr}- return runtime.Start(ctx, appCtx, func(err error) { slog.ErrorContext(appCtx, "Francis host failed", "error", err); cancelApp() })+ return runtime.Start(ctx, appCtx, func(err error) {+ slog.ErrorContext(appCtx, "Francis host failed", "error", err)+ cancelApp(fmt.Errorf("francis host failed: %w", err))+ })}, OnStop: runtime.Stop})return runtime, nil}