Skip to content

Modern Docker Management, Designed for Everyone

fix: wait for schedulers during shutdown (#3230)

Kyle Mendellc5bbf1a21006
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index 1a1679dd1..5fe62c147 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -11,6 +11,7 @@ import (
"os"
"os/signal"
"strings"
+ "sync"
"syscall"
"time"
@@ -60,18 +61,28 @@ func Bootstrap(ctx context.Context) error {
appCtx, cancelApp := context.WithCancel(ctx)
appCtx = utils.WithAppLifecycleContext(appCtx)
- defer cancelApp()
db, err := initializeDBAndMigrate(appCtx, cfg)
if err != nil {
+ cancelApp()
return fmt.Errorf("failed to initialize database: %w", err)
}
+ var appServices *di.Services
defer func(ctx context.Context) {
+ cancelApp()
// appCtx is already canceled here, so derive the shutdown deadline from a
// non-canceled copy of it.
baseCtx := context.WithoutCancel(ctx)
shutdownCtx, shutdownCancel := context.WithTimeout(baseCtx, 10*time.Second)
defer shutdownCancel()
+ if appServices != nil {
+ if appServices.Volume != nil {
+ appServices.Volume.CleanupHelperContainers(shutdownCtx)
+ }
+ if appServices.Docker != nil {
+ appServices.Docker.Close()
+ }
+ }
if err := db.Close(); err != nil {
slog.ErrorContext(shutdownCtx, "Error closing database", "error", err)
}
@@ -79,20 +90,11 @@ func Bootstrap(ctx context.Context) error {
httpClient := newConfiguredHTTPClient(cfg)
- appServices, err := di.InitializeServices(appCtx, db, cfg, httpClient)
+ appServices, err = di.InitializeServices(appCtx, db, cfg, httpClient)
if err != nil {
return fmt.Errorf("failed to initialize services: %w", err)
}
dockerClientService := appServices.Docker
- defer dockerClientService.Close()
- defer func(ctx context.Context) {
- baseCtx := context.WithoutCancel(ctx)
- shutdownCtx, shutdownCancel := context.WithTimeout(baseCtx, 10*time.Second)
- defer shutdownCancel()
- if appServices.Volume != nil {
- appServices.Volume.CleanupHelperContainers(shutdownCtx)
- }
- }(appCtx)
initializeStartupState(appCtx, cfg, appServices, dockerClientService, httpClient)
@@ -105,7 +107,7 @@ func Bootstrap(ctx context.Context) error {
startEdgeTunnelClientIfConfigured(appCtx, cfg, router)
- err = runServicesInternal(appCtx, cfg, router, tunnelServer, scheduler)
+ err = runServicesInternal(appCtx, cancelApp, cfg, router, tunnelServer, scheduler)
if err != nil {
return fmt.Errorf("failed to run services: %w", err)
}
@@ -388,21 +390,14 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpCl
}
}
-func runServicesInternal(appCtx context.Context, cfg *config.Config, router http.Handler, tunnelServer *edge.TunnelServer, schedulers ...interface {
+func runServicesInternal(appCtx context.Context, cancelApp context.CancelFunc, cfg *config.Config, router http.Handler, tunnelServer *edge.TunnelServer, schedulers ...interface {
Run(ctx context.Context) error
}) error {
- for _, s := range schedulers {
- scheduler := s
- go func() {
- slog.InfoContext(appCtx, "Starting scheduler")
- if err := scheduler.Run(appCtx); err != nil {
- if !errors.Is(err, context.Canceled) {
- slog.ErrorContext(appCtx, "Job scheduler exited with error", "error", err)
- }
- }
- slog.InfoContext(appCtx, "Scheduler stopped")
- }()
- }
+ var schedulerWaitGroup sync.WaitGroup
+ defer func() {
+ cancelApp()
+ schedulerWaitGroup.Wait()
+ }()
listenAddr := cfg.ListenAddr()
useTLS, tlsCertFile, tlsKeyFile, edgeCfg, err := prepareServerTLSInternal(appCtx, cfg)
@@ -427,6 +422,18 @@ func runServicesInternal(appCtx context.Context, cfg *config.Config, router http
if err != nil {
return err
}
+ for _, serviceScheduler := range schedulers {
+ scheduler := serviceScheduler
+ schedulerWaitGroup.Go(func() {
+ slog.InfoContext(appCtx, "Starting scheduler")
+ if err := scheduler.Run(appCtx); err != nil {
+ if !errors.Is(err, context.Canceled) {
+ slog.ErrorContext(appCtx, "Job scheduler exited with error", "error", err)
+ }
+ }
+ slog.InfoContext(appCtx, "Scheduler stopped")
+ })
+ }
go func() {
slog.InfoContext(appCtx, "Starting HTTP server", "addr", listenAddr, "listen", cfg.Listen, "port", cfg.Port, "tls_enabled", useTLS)
@@ -445,10 +452,12 @@ func runServicesInternal(appCtx context.Context, cfg *config.Config, router http
quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+ defer signal.Stop(quit)
select {
case <-quit:
slog.InfoContext(appCtx, "Received shutdown signal")
+ cancelApp()
case <-appCtx.Done():
slog.InfoContext(appCtx, "Context canceled")
}
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 6ee6dc155..9975e862c 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -8,6 +8,7 @@ import (
"net/http"
"net/http/httptest"
"strconv"
+ "sync/atomic"
"testing"
"time"
@@ -21,6 +22,28 @@ import (
"golang.org/x/net/http2"
)
+type blockingServiceSchedulerInternal struct {
+ runCalls atomic.Int32
+ started chan struct{}
+ cancellationObserved chan struct{}
+ release <-chan struct{}
+}
+
+func (s *blockingServiceSchedulerInternal) Run(ctx context.Context) error {
+ s.runCalls.Add(1)
+ if s.started != nil {
+ s.started <- struct{}{}
+ }
+ <-ctx.Done()
+ if s.cancellationObserved != nil {
+ s.cancellationObserved <- struct{}{}
+ }
+ if s.release != nil {
+ <-s.release
+ }
+ return nil
+}
+
func TestNormalizeTunnelGRPCRequestPathInternal(t *testing.T) {
fullMethodPath := tunnelpb.TunnelService_Connect_FullMethodName
@@ -253,6 +276,84 @@ func TestShutdownCancelsStreamingRequestContextsInternal(t *testing.T) {
require.ErrorIs(t, <-errCh, http.ErrServerClosed)
}
+func TestRunServicesInternalDoesNotStartSchedulersBeforeServerInitialization(t *testing.T) {
+ appCtx, cancelApp := context.WithCancel(context.Background())
+ scheduler := &blockingServiceSchedulerInternal{}
+ cfg := &config.Config{
+ AgentMode: true,
+ TLSEnabled: true,
+ }
+
+ err := runServicesInternal(appCtx, cancelApp, cfg, http.NewServeMux(), nil, scheduler)
+ require.ErrorContains(t, err, "TLS_ENABLED requires both TLS_CERT_FILE and TLS_KEY_FILE")
+ require.Zero(t, scheduler.runCalls.Load())
+ require.ErrorIs(t, appCtx.Err(), context.Canceled)
+}
+
+func TestRunServicesInternalWaitsForSchedulersBeforeReleasingDependencies(t *testing.T) {
+ appCtx, cancelApp := context.WithCancel(context.Background())
+ started := make(chan struct{}, 1)
+ cancellationObserved := make(chan struct{}, 1)
+ releaseScheduler := make(chan struct{}, 1)
+ scheduler := &blockingServiceSchedulerInternal{
+ started: started,
+ cancellationObserved: cancellationObserved,
+ release: releaseScheduler,
+ }
+ cfg := &config.Config{
+ AgentMode: true,
+ Environment: config.AppEnvironmentTest,
+ Listen: "127.0.0.1",
+ Port: "0",
+ }
+
+ runDone := make(chan error, 1)
+ dependenciesReleased := make(chan struct{})
+ go func() {
+ err := runServicesInternal(appCtx, cancelApp, cfg, http.NewServeMux(), nil, scheduler)
+ close(dependenciesReleased)
+ runDone <- err
+ }()
+ t.Cleanup(func() {
+ cancelApp()
+ select {
+ case releaseScheduler <- struct{}{}:
+ default:
+ }
+ })
+
+ select {
+ case <-started:
+ case <-time.After(time.Second):
+ t.Fatal("scheduler did not start")
+ }
+ cancelApp()
+ select {
+ case <-cancellationObserved:
+ case <-time.After(time.Second):
+ t.Fatal("scheduler did not observe lifecycle cancellation")
+ }
+
+ select {
+ case <-dependenciesReleased:
+ t.Fatal("service runner released dependencies before the scheduler stopped")
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ releaseScheduler <- struct{}{}
+ select {
+ case <-dependenciesReleased:
+ case <-time.After(time.Second):
+ t.Fatal("service runner did not release dependencies after the scheduler stopped")
+ }
+ select {
+ case err := <-runDone:
+ require.NoError(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("service runner did not return")
+ }
+}
+
func TestPrepareServerTLSInternal_AgentModeSkipsManagerMTLSValidation(t *testing.T) {
cfg := &config.Config{
AgentMode: true,
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index 2fb92dcab..0ec57a571 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -159,7 +159,10 @@ func (js *JobScheduler) GetLocation() *time.Location {
func (js *JobScheduler) Run(ctx context.Context) error {
js.StartScheduler()
<-ctx.Done()
- js.cron.Stop()
+ // Running jobs may still own Docker or database resources. Wait for them here
+ // so Bootstrap cannot close shared services underneath them; the process-level
+ // signal handler owns the hard shutdown deadline for non-cooperative jobs.
+ <-js.cron.Stop().Done()
return nil
}
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
index a6a405dd5..d579704e6 100644
--- a/backend/pkg/scheduler/scheduler_test.go
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -260,3 +260,69 @@ func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T)
require.True(t, js.HasJob(job.Name()))
require.Len(t, js.cron.Entries(), 1)
}
+
+func TestJobScheduler_RunWaitsForCanceledJobToFinish(t *testing.T) {
+ lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())
+ jobStarted := make(chan struct{}, 1)
+ cancellationObserved := make(chan struct{}, 1)
+ releaseJob := make(chan struct{}, 1)
+ jobFinished := make(chan struct{}, 1)
+ var runOnce sync.Once
+
+ js := NewJobScheduler(lifecycleCtx, nil)
+ js.RegisterJob(&testSchedulerJob{
+ name: "test-shutdown-waits",
+ schedule: "*/1 * * * * *",
+ run: func(ctx context.Context) {
+ runOnce.Do(func() {
+ jobStarted <- struct{}{}
+ <-ctx.Done()
+ cancellationObserved <- struct{}{}
+ <-releaseJob
+ jobFinished <- struct{}{}
+ })
+ },
+ })
+
+ runDone := make(chan error, 1)
+ go func() { runDone <- js.Run(lifecycleCtx) }()
+ t.Cleanup(func() {
+ cancelLifecycle()
+ select {
+ case releaseJob <- struct{}{}:
+ default:
+ }
+ })
+
+ select {
+ case <-jobStarted:
+ case <-time.After(2500 * time.Millisecond):
+ t.Fatal("timed out waiting for scheduled job to start")
+ }
+
+ cancelLifecycle()
+ select {
+ case <-cancellationObserved:
+ case <-time.After(time.Second):
+ t.Fatal("scheduled job did not observe lifecycle cancellation")
+ }
+
+ select {
+ case err := <-runDone:
+ t.Fatalf("scheduler returned before the running job finished: %v", err)
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ releaseJob <- struct{}{}
+ select {
+ case <-jobFinished:
+ case <-time.After(time.Second):
+ t.Fatal("scheduled job did not finish after release")
+ }
+ select {
+ case err := <-runDone:
+ require.NoError(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("scheduler did not return after the running job finished")
+ }
+}
diff --git a/backend/pkg/utils/signals/signals.go b/backend/pkg/utils/signals/signals.go
index a6aabd528..0cd8cb80d 100644
--- a/backend/pkg/utils/signals/signals.go
+++ b/backend/pkg/utils/signals/signals.go
@@ -6,6 +6,7 @@ import (
"os"
"os/signal"
"syscall"
+ "time"
)
/*
@@ -19,8 +20,10 @@ Also referenced from pocket-id/pocket-id
var onlyOneSignalHandler = make(chan struct{})
+const forcedShutdownTimeout = 30 * time.Second
+
// SignalContext returns a context that is canceled when the application receives an interrupt signal.
-// A second signal forces an immediate shutdown.
+// A second signal or an expired graceful-shutdown deadline forces the process to exit.
func SignalContext(parentCtx context.Context) context.Context {
close(onlyOneSignalHandler) // Panics when called twice
@@ -33,8 +36,17 @@ func SignalContext(parentCtx context.Context) context.Context {
slog.Info("Received interrupt signal. Shutting down…")
cancel()
- <-sigCh
- slog.Warn("Received a second interrupt signal. Forcing an immediate shutdown.")
+ shutdownTimer := time.NewTimer(forcedShutdownTimeout)
+ defer shutdownTimer.Stop()
+
+ select {
+ case <-sigCh:
+ slog.Warn("Received a second interrupt signal. Forcing an immediate shutdown.")
+ case <-shutdownTimer.C:
+ slog.Error("Graceful shutdown timed out. Forcing process exit.", "timeout", forcedShutdownTimeout)
+ }
+ // Go cannot safely terminate a hung goroutine. Exit the process instead of
+ // allowing Bootstrap to close dependencies that goroutine may still use.
os.Exit(1)
}()