Skip to content

Modern Docker Management, Designed for Everyone

fix(bootstrap): stop the app when the Francis host fails (#4371)

Kyle Mendelle9b8ee1bf045
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index c0b8c530d..f15fb9402 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -67,17 +67,17 @@ func Bootstrap(ctx context.Context) error {
startup.ApplyMemoryLimit(ctx)
slog.InfoContext(ctx, "Arcane Identity Configuration", "puid", os.Getuid(), "pgid", os.Getgid())
- appCtx, cancelApp := context.WithCancel(ctx)
+ appCtx, cancelApp := context.WithCancelCause(ctx)
appCtx = utils.WithAppLifecycleContext(appCtx)
db, err := database.Initialize(appCtx, cfg.DatabaseURL, database.MigrationOptions{AllowDowngrade: cfg.AllowDowngrade})
if err != nil {
- cancelApp()
+ cancelApp(nil)
return fmt.Errorf("failed to initialize database: %w", err)
}
slog.InfoContext(appCtx, "Database initialized successfully")
defer func() {
- cancelApp()
+ cancelApp(nil)
if closeErr := db.Close(); closeErr != nil {
slog.ErrorContext(ctx, "Error closing database", "error", closeErr)
}
@@ -110,30 +110,36 @@ func Bootstrap(ctx context.Context) error {
app := fx.New(applicationOptions(appCtx, cfg, db, cancelApp))
- startCtx, cancelStart := context.WithTimeout(ctx, 5*time.Minute)
+ startCtx, cancelStart := context.WithTimeout(ctx, app.StartTimeout())
defer cancelStart()
if startErr := app.Start(startCtx); startErr != nil {
return fmt.Errorf("start application: %w", startErr)
}
+ // appCtx also ends when a component stops the app itself; its cause is then the run error.
+ var runErr error
select {
- case <-ctx.Done():
- slog.InfoContext(appCtx, "Context canceled")
+ case <-appCtx.Done():
+ if ctx.Err() == nil {
+ runErr = context.Cause(appCtx)
+ } else {
+ slog.InfoContext(appCtx, "Context canceled")
+ }
case signal := <-app.Done():
slog.InfoContext(appCtx, "Received shutdown signal", "signal", signal)
}
- stopCtx, cancelStop := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ stopCtx, cancelStop := context.WithTimeout(context.WithoutCancel(ctx), app.StopTimeout())
defer cancelStop()
if stopErr := app.Stop(stopCtx); stopErr != nil {
- return fmt.Errorf("stop application: %w", stopErr)
+ return errors.Join(runErr, fmt.Errorf("stop application: %w", stopErr))
}
slog.InfoContext(context.WithoutCancel(appCtx), "Arcane shutdown complete")
- return nil
+ return runErr
}
-func applicationOptions(appCtx context.Context, cfg *config.Config, db *database.DB, cancelApp context.CancelFunc) fx.Option {
+func applicationOptions(appCtx context.Context, cfg *config.Config, db *database.DB, cancelApp context.CancelCauseFunc) fx.Option {
return fx.Options(
fx.Supply(cfg, db, cancelApp),
fx.Provide(
@@ -209,9 +215,7 @@ func initializeStartupState(p initializeStartupStateParams) {
appCtx := p.AppCtx
cfg := p.Config
- if p.Volume != nil {
- startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)
- }
+ startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)
runtimeCfg := &startup.RuntimeConfig{
AgentMode: cfg.AgentMode,
@@ -253,26 +257,22 @@ func initializeStartupState(p initializeStartupStateParams) {
if err := p.Environment.EnsureLocalEnvironment(appCtx, cfg.AppUrl); err != nil {
slog.WarnContext(appCtx, "Failed to ensure local environment", "error", err)
}
- if p.GitOpsSync != nil {
- if err := p.GitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)
- }
- if err := p.GitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)
- }
- if err := p.GitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)
- }
- if err := p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)
- }
+ if err := p.GitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)
+ }
+ if err := p.GitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)
+ }
+ if err := p.GitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)
+ }
+ if err := p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)
}
p.Vulnerability.ImportLegacyReportFiles(appCtx)
p.Vulnerability.BackfillScanItems(appCtx)
- if p.Project != nil {
- if err := p.Project.RecoverProjectRenameJournals(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)
- }
+ if err := p.Project.RecoverProjectRenameJournals(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)
}
if !cfg.AgentMode {
@@ -304,10 +304,8 @@ func initializeStartupState(p initializeStartupStateParams) {
slog.InfoContext(ctx, "Docker API versions detected", "clientApiVersion", dockerClient.ClientVersion(), "serverApiVersion", version.APIVersion, "effectiveApiVersion", effectiveAPIVersion)
return nil
})
- if p.Swarm != nil {
- if err := p.Swarm.SyncSwarmEnabledState(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)
- }
+ if err := p.Swarm.SyncSwarmEnabledState(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)
}
startup.InitializeNonAgentFeatures(appCtx, runtimeCfg,
@@ -396,10 +394,10 @@ func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.
})
}
-func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {
+func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelCauseFunc) {
lc.Append(fx.Hook{
OnStop: func(context.Context) error {
- cancelApp()
+ cancelApp(nil)
return nil
},
})
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 257cea7f0..87fe666c5 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -361,7 +361,7 @@ func TestNewHTTPServerRejectsInvalidTLSCertificate(t *testing.T) {
}
func TestRegisterAppCancelHookRunsAfterLaterStopHooks(t *testing.T) {
- appCtx, cancelApp := context.WithCancel(t.Context())
+ appCtx, cancelApp := context.WithCancelCause(t.Context())
lifecycle := fxtest.NewLifecycle(t)
registerAppCancelHook(lifecycle, cancelApp)
appActiveDuringDependencyStop := false
@@ -449,8 +449,8 @@ func TestJobSchedulerStopCancelsItsPrivateContext(t *testing.T) {
}
func TestApplicationOptionsValidate(t *testing.T) {
- appCtx, cancelApp := context.WithCancel(t.Context())
- defer cancelApp()
+ appCtx, cancelApp := context.WithCancelCause(t.Context())
+ defer cancelApp(nil)
err := fx.ValidateApp(applicationOptions(
appCtx,
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index b1fe7e37b..21ea08179 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -406,13 +406,13 @@ type settingsSubscriptionsParams struct {
Settings settingsChangeSubscriber
Environment timeoutSettingsEnvironment
- AutoUpdate *flow.Job `name:"auto-update"`
+ AutoUpdate *flow.Job
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
FilesystemWatcher *scheduler.FilesystemWatcherJob
ScheduledPrune *scheduler.ScheduledPruneJob
- VulnerabilityScan *flow.Job `name:"vulnerability-scan"`
+ VulnerabilityScan *flow.Job
VulnerabilityRisk *scheduler.VulnerabilityRiskJob
- AutoPatch *flow.Job `name:"auto-patch"`
+ AutoPatch *flow.Job
AutoHeal *scheduler.AutoHealJob
Apns *apns.ApnsService
ApnsOutbox *scheduler.ApnsOutboxJob
@@ -446,10 +446,8 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {
timeoutSyncExecutor, cancelTimeoutSync := setupTimeoutSettingsSubscription(params, subscribe)
subscribe([]string{"pollingEnabled", "pollingInterval"}, func(_ []libarcane.SettingUpdate) {
- if params.ImageUpdateWatcher != nil {
- params.ImageUpdateWatcher.RefreshSchedule()
- params.ImageUpdateWatcher.Trigger()
- }
+ params.ImageUpdateWatcher.RefreshSchedule()
+ params.ImageUpdateWatcher.Trigger()
if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoUpdate); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-update job", "error", err)
}
@@ -464,18 +462,14 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {
rescheduleOn([]string{"autoUpdate", "autoUpdateInterval"}, params.AutoUpdate)
subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {
- if params.FilesystemWatcher != nil {
- if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {
- slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)
- }
+ if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {
+ slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)
}
})
subscribe([]string{"templatesDirectory"}, func(_ []libarcane.SettingUpdate) {
- if params.FilesystemWatcher != nil {
- if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {
- slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)
- }
+ if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {
+ slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)
}
})
diff --git a/backend/internal/bootstrap/jobs_bootstrap_test.go b/backend/internal/bootstrap/jobs_bootstrap_test.go
index eb2d2df26..9c6022276 100644
--- a/backend/internal/bootstrap/jobs_bootstrap_test.go
+++ b/backend/internal/bootstrap/jobs_bootstrap_test.go
@@ -84,13 +84,14 @@ func TestSettingsTimeoutSyncDoesNotBlockOtherEffects(t *testing.T) {
localScheduler := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}
localEnvironment := &timeoutSyncEnvironmentStubInternal{started: make(chan struct{})}
require.NoError(t, setupSettingsSubscriptions(settingsSubscriptionsParams{
- Lifecycle: lifecycle,
- LifecycleCtx: t.Context(),
- Config: &config.Config{},
- Scheduler: localScheduler,
- Settings: settings,
- Environment: localEnvironment,
- AutoUpdate: &flow.Job{JobName: "auto-update"},
+ Lifecycle: lifecycle,
+ LifecycleCtx: t.Context(),
+ Config: &config.Config{},
+ Scheduler: localScheduler,
+ Settings: settings,
+ Environment: localEnvironment,
+ AutoUpdate: &flow.Job{JobName: "auto-update"},
+ ImageUpdateWatcher: &scheduler.ImageUpdateWatcher{},
}))
timeoutCallbackDone := make(chan struct{})
diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.go
index 62067af9a..0da1a084e 100644
--- a/backend/internal/di/di_test.go
+++ b/backend/internal/di/di_test.go
@@ -125,8 +125,8 @@ type graphParams struct {
}
func TestOptionsValidate(t *testing.T) {
- _, cancel := context.WithCancel(t.Context())
- defer cancel()
+ _, cancel := context.WithCancelCause(t.Context())
+ defer cancel(nil)
err := fx.ValidateApp(
fx.Supply(
cancel,
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 332473f86..46e3fe233 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -3,6 +3,7 @@ package di
import (
"context"
"errors"
+ "fmt"
"log/slog"
"net/http"
"time"
@@ -104,7 +105,7 @@ func provideActorRuntimeInternal(
db *database.DB,
settingsService *settings.SettingsService,
lc fx.Lifecycle,
- cancelApp context.CancelFunc,
+ cancelApp context.CancelCauseFunc,
) (*francis.Runtime, error) {
runtime, err := francis.New(cfg.DatabaseURL, "", "", cfg.ActorPort)
if err != nil {
@@ -121,7 +122,10 @@ func provideActorRuntimeInternal(
if configureIdentityErr := runtime.ConfigureIdentity(cfg.EncryptionKey, settingsService.GetSettingsConfig().InstanceID.Value); configureIdentityErr != nil {
return configureIdentityErr
}
- return runtime.Start(ctx, appCtx, func(err error) { slog.ErrorContext(appCtx, "Francis host failed", "error", err); cancelApp() })
+ return runtime.Start(ctx, appCtx, func(err error) {
+ slog.ErrorContext(appCtx, "Francis host failed", "error", err)
+ cancelApp(fmt.Errorf("francis host failed: %w", err))
+ })
}, OnStop: runtime.Stop})
return runtime, nil
}