Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: wait for schedulers during shutdown (#3230)
Kyle Mendellc5bbf1a21006
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex 1a1679dd1..5fe62c147 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -11,6 +11,7 @@ import ("os""os/signal""strings"+ "sync""syscall""time"@@ -60,18 +61,28 @@ func Bootstrap(ctx context.Context) error {appCtx, cancelApp := context.WithCancel(ctx)appCtx = utils.WithAppLifecycleContext(appCtx)- defer cancelApp()db, err := initializeDBAndMigrate(appCtx, cfg)if err != nil {+ cancelApp()return fmt.Errorf("failed to initialize database: %w", err)}+ var appServices *di.Servicesdefer func(ctx context.Context) {+ cancelApp()// appCtx is already canceled here, so derive the shutdown deadline from a// non-canceled copy of it.baseCtx := context.WithoutCancel(ctx)shutdownCtx, shutdownCancel := context.WithTimeout(baseCtx, 10*time.Second)defer shutdownCancel()+ if appServices != nil {+ if appServices.Volume != nil {+ appServices.Volume.CleanupHelperContainers(shutdownCtx)+ }+ if appServices.Docker != nil {+ appServices.Docker.Close()+ }+ }if err := db.Close(); err != nil {slog.ErrorContext(shutdownCtx, "Error closing database", "error", err)}@@ -79,20 +90,11 @@ func Bootstrap(ctx context.Context) error {httpClient := newConfiguredHTTPClient(cfg)- appServices, err := di.InitializeServices(appCtx, db, cfg, httpClient)+ appServices, err = di.InitializeServices(appCtx, db, cfg, httpClient)if err != nil {return fmt.Errorf("failed to initialize services: %w", err)}dockerClientService := appServices.Docker- defer dockerClientService.Close()- defer func(ctx context.Context) {- baseCtx := context.WithoutCancel(ctx)- shutdownCtx, shutdownCancel := context.WithTimeout(baseCtx, 10*time.Second)- defer shutdownCancel()- if appServices.Volume != nil {- appServices.Volume.CleanupHelperContainers(shutdownCtx)- }- }(appCtx)initializeStartupState(appCtx, cfg, appServices, dockerClientService, httpClient)@@ -105,7 +107,7 @@ func Bootstrap(ctx context.Context) error {startEdgeTunnelClientIfConfigured(appCtx, cfg, router)- err = runServicesInternal(appCtx, cfg, router, tunnelServer, scheduler)+ err = runServicesInternal(appCtx, cancelApp, cfg, router, tunnelServer, scheduler)if err != nil {return fmt.Errorf("failed to run services: %w", err)}@@ -388,21 +390,14 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpCl}}-func runServicesInternal(appCtx context.Context, cfg *config.Config, router http.Handler, tunnelServer *edge.TunnelServer, schedulers ...interface {+func runServicesInternal(appCtx context.Context, cancelApp context.CancelFunc, cfg *config.Config, router http.Handler, tunnelServer *edge.TunnelServer, schedulers ...interface {Run(ctx context.Context) error}) error {- for _, s := range schedulers {- scheduler := s- go func() {- slog.InfoContext(appCtx, "Starting scheduler")- if err := scheduler.Run(appCtx); err != nil {- if !errors.Is(err, context.Canceled) {- slog.ErrorContext(appCtx, "Job scheduler exited with error", "error", err)- }- }- slog.InfoContext(appCtx, "Scheduler stopped")- }()- }+ var schedulerWaitGroup sync.WaitGroup+ defer func() {+ cancelApp()+ schedulerWaitGroup.Wait()+ }()listenAddr := cfg.ListenAddr()useTLS, tlsCertFile, tlsKeyFile, edgeCfg, err := prepareServerTLSInternal(appCtx, cfg)@@ -427,6 +422,18 @@ func runServicesInternal(appCtx context.Context, cfg *config.Config, router httpif err != nil {return err}+ for _, serviceScheduler := range schedulers {+ scheduler := serviceScheduler+ schedulerWaitGroup.Go(func() {+ slog.InfoContext(appCtx, "Starting scheduler")+ if err := scheduler.Run(appCtx); err != nil {+ if !errors.Is(err, context.Canceled) {+ slog.ErrorContext(appCtx, "Job scheduler exited with error", "error", err)+ }+ }+ slog.InfoContext(appCtx, "Scheduler stopped")+ })+ }go func() {slog.InfoContext(appCtx, "Starting HTTP server", "addr", listenAddr, "listen", cfg.Listen, "port", cfg.Port, "tls_enabled", useTLS)@@ -445,10 +452,12 @@ func runServicesInternal(appCtx context.Context, cfg *config.Config, router httpquit := make(chan os.Signal, 1)signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)+ defer signal.Stop(quit)select {case <-quit:slog.InfoContext(appCtx, "Received shutdown signal")+ cancelApp()case <-appCtx.Done():slog.InfoContext(appCtx, "Context canceled")}diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 6ee6dc155..9975e862c 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -8,6 +8,7 @@ import ("net/http""net/http/httptest""strconv"+ "sync/atomic""testing""time"@@ -21,6 +22,28 @@ import ("golang.org/x/net/http2")+type blockingServiceSchedulerInternal struct {+ runCalls atomic.Int32+ started chan struct{}+ cancellationObserved chan struct{}+ release <-chan struct{}+}++func (s *blockingServiceSchedulerInternal) Run(ctx context.Context) error {+ s.runCalls.Add(1)+ if s.started != nil {+ s.started <- struct{}{}+ }+ <-ctx.Done()+ if s.cancellationObserved != nil {+ s.cancellationObserved <- struct{}{}+ }+ if s.release != nil {+ <-s.release+ }+ return nil+}+func TestNormalizeTunnelGRPCRequestPathInternal(t *testing.T) {fullMethodPath := tunnelpb.TunnelService_Connect_FullMethodName@@ -253,6 +276,84 @@ func TestShutdownCancelsStreamingRequestContextsInternal(t *testing.T) {require.ErrorIs(t, <-errCh, http.ErrServerClosed)}+func TestRunServicesInternalDoesNotStartSchedulersBeforeServerInitialization(t *testing.T) {+ appCtx, cancelApp := context.WithCancel(context.Background())+ scheduler := &blockingServiceSchedulerInternal{}+ cfg := &config.Config{+ AgentMode: true,+ TLSEnabled: true,+ }++ err := runServicesInternal(appCtx, cancelApp, cfg, http.NewServeMux(), nil, scheduler)+ require.ErrorContains(t, err, "TLS_ENABLED requires both TLS_CERT_FILE and TLS_KEY_FILE")+ require.Zero(t, scheduler.runCalls.Load())+ require.ErrorIs(t, appCtx.Err(), context.Canceled)+}++func TestRunServicesInternalWaitsForSchedulersBeforeReleasingDependencies(t *testing.T) {+ appCtx, cancelApp := context.WithCancel(context.Background())+ started := make(chan struct{}, 1)+ cancellationObserved := make(chan struct{}, 1)+ releaseScheduler := make(chan struct{}, 1)+ scheduler := &blockingServiceSchedulerInternal{+ started: started,+ cancellationObserved: cancellationObserved,+ release: releaseScheduler,+ }+ cfg := &config.Config{+ AgentMode: true,+ Environment: config.AppEnvironmentTest,+ Listen: "127.0.0.1",+ Port: "0",+ }++ runDone := make(chan error, 1)+ dependenciesReleased := make(chan struct{})+ go func() {+ err := runServicesInternal(appCtx, cancelApp, cfg, http.NewServeMux(), nil, scheduler)+ close(dependenciesReleased)+ runDone <- err+ }()+ t.Cleanup(func() {+ cancelApp()+ select {+ case releaseScheduler <- struct{}{}:+ default:+ }+ })++ select {+ case <-started:+ case <-time.After(time.Second):+ t.Fatal("scheduler did not start")+ }+ cancelApp()+ select {+ case <-cancellationObserved:+ case <-time.After(time.Second):+ t.Fatal("scheduler did not observe lifecycle cancellation")+ }++ select {+ case <-dependenciesReleased:+ t.Fatal("service runner released dependencies before the scheduler stopped")+ case <-time.After(100 * time.Millisecond):+ }++ releaseScheduler <- struct{}{}+ select {+ case <-dependenciesReleased:+ case <-time.After(time.Second):+ t.Fatal("service runner did not release dependencies after the scheduler stopped")+ }+ select {+ case err := <-runDone:+ require.NoError(t, err)+ case <-time.After(time.Second):+ t.Fatal("service runner did not return")+ }+}+func TestPrepareServerTLSInternal_AgentModeSkipsManagerMTLSValidation(t *testing.T) {cfg := &config.Config{AgentMode: true,diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.goindex 2fb92dcab..0ec57a571 100644--- a/backend/pkg/scheduler/scheduler.go+++ b/backend/pkg/scheduler/scheduler.go@@ -159,7 +159,10 @@ func (js *JobScheduler) GetLocation() *time.Location {func (js *JobScheduler) Run(ctx context.Context) error {js.StartScheduler()<-ctx.Done()- js.cron.Stop()+ // Running jobs may still own Docker or database resources. Wait for them here+ // so Bootstrap cannot close shared services underneath them; the process-level+ // signal handler owns the hard shutdown deadline for non-cooperative jobs.+ <-js.cron.Stop().Done()return nil}diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.goindex a6a405dd5..d579704e6 100644--- a/backend/pkg/scheduler/scheduler_test.go+++ b/backend/pkg/scheduler/scheduler_test.go@@ -260,3 +260,69 @@ func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T)require.True(t, js.HasJob(job.Name()))require.Len(t, js.cron.Entries(), 1)}++func TestJobScheduler_RunWaitsForCanceledJobToFinish(t *testing.T) {+ lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())+ jobStarted := make(chan struct{}, 1)+ cancellationObserved := make(chan struct{}, 1)+ releaseJob := make(chan struct{}, 1)+ jobFinished := make(chan struct{}, 1)+ var runOnce sync.Once++ js := NewJobScheduler(lifecycleCtx, nil)+ js.RegisterJob(&testSchedulerJob{+ name: "test-shutdown-waits",+ schedule: "*/1 * * * * *",+ run: func(ctx context.Context) {+ runOnce.Do(func() {+ jobStarted <- struct{}{}+ <-ctx.Done()+ cancellationObserved <- struct{}{}+ <-releaseJob+ jobFinished <- struct{}{}+ })+ },+ })++ runDone := make(chan error, 1)+ go func() { runDone <- js.Run(lifecycleCtx) }()+ t.Cleanup(func() {+ cancelLifecycle()+ select {+ case releaseJob <- struct{}{}:+ default:+ }+ })++ select {+ case <-jobStarted:+ case <-time.After(2500 * time.Millisecond):+ t.Fatal("timed out waiting for scheduled job to start")+ }++ cancelLifecycle()+ select {+ case <-cancellationObserved:+ case <-time.After(time.Second):+ t.Fatal("scheduled job did not observe lifecycle cancellation")+ }++ select {+ case err := <-runDone:+ t.Fatalf("scheduler returned before the running job finished: %v", err)+ case <-time.After(100 * time.Millisecond):+ }++ releaseJob <- struct{}{}+ select {+ case <-jobFinished:+ case <-time.After(time.Second):+ t.Fatal("scheduled job did not finish after release")+ }+ select {+ case err := <-runDone:+ require.NoError(t, err)+ case <-time.After(time.Second):+ t.Fatal("scheduler did not return after the running job finished")+ }+}diff --git a/backend/pkg/utils/signals/signals.go b/backend/pkg/utils/signals/signals.goindex a6aabd528..0cd8cb80d 100644--- a/backend/pkg/utils/signals/signals.go+++ b/backend/pkg/utils/signals/signals.go@@ -6,6 +6,7 @@ import ("os""os/signal""syscall"+ "time")/*@@ -19,8 +20,10 @@ Also referenced from pocket-id/pocket-idvar onlyOneSignalHandler = make(chan struct{})+const forcedShutdownTimeout = 30 * time.Second+// SignalContext returns a context that is canceled when the application receives an interrupt signal.-// A second signal forces an immediate shutdown.+// A second signal or an expired graceful-shutdown deadline forces the process to exit.func SignalContext(parentCtx context.Context) context.Context {close(onlyOneSignalHandler) // Panics when called twice@@ -33,8 +36,17 @@ func SignalContext(parentCtx context.Context) context.Context {slog.Info("Received interrupt signal. Shutting down…")cancel()- <-sigCh- slog.Warn("Received a second interrupt signal. Forcing an immediate shutdown.")+ shutdownTimer := time.NewTimer(forcedShutdownTimeout)+ defer shutdownTimer.Stop()++ select {+ case <-sigCh:+ slog.Warn("Received a second interrupt signal. Forcing an immediate shutdown.")+ case <-shutdownTimer.C:+ slog.Error("Graceful shutdown timed out. Forcing process exit.", "timeout", forcedShutdownTimeout)+ }+ // Go cannot safely terminate a hung goroutine. Exit the process instead of+ // allowing Bootstrap to close dependencies that goroutine may still use.os.Exit(1)}()