Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Join Router-owned limiter workers during shutdown

pulse-triage[bot]5b560521e26d
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md
index 90f36f2f4b..1b1ee43136 100644
--- a/docs/release-control/v6/internal/subsystems/api-contracts.md
+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md
@@ -11221,3 +11221,20 @@ Pinning the Patrol attention detail header on phones
layout-only change. It reads no new attention fields, sends no new request,
and leaves the attention projection, lifecycle, and queue ordering contracts
above unchanged.
+
+### Router-owned worker shutdown
+
+Each Router owns its five route-specific limiters, eight endpoint-category
+limiters and, in hosted mode, its tenant limiter. `ShutdownBackgroundWorkers`
+seals lifecycle-worker admission, cancels and joins the constructor-owned
+workers exactly once; the server's existing `ShutdownAIIntelligence` calls
+that same lifecycle. HTTP admission must stop before shutdown. Resource and
+RBAC stores retain their separate shutdown owners. Integration fixtures use
+this lifecycle before stopping the monitor and removing temporary data.
+
+`RateLimiter.Stop` is concurrent-safe and joins its cleanup worker. Stopping
+cleanup does not clear request history, bypass rate enforcement or stop another
+Router's or the package-global limiters. Limits, windows, tenant selection,
+revocation and response payloads are unchanged. The connected lifetime and
+ownership controls are in `router_limiter_lifecycle_test.go`; the integration
+fixture's cleanup also runs on a failed listener/request assertion.
diff --git a/internal/api/rate_limit_config.go b/internal/api/rate_limit_config.go
index 575364fc53..1ddbc3f55d 100644
--- a/internal/api/rate_limit_config.go
+++ b/internal/api/rate_limit_config.go
@@ -57,6 +57,21 @@ func newEndpointRateLimitConfig() *EndpointRateLimitConfig {
}
}
+// Stop joins only the workers owned by this endpoint configuration. A Router
+// must not stop the package-global configuration or another Router's budgets.
+func (cfg *EndpointRateLimitConfig) Stop() {
+ if cfg == nil {
+ return
+ }
+ for _, limiter := range []*RateLimiter{
+ cfg.AuthEndpoints, cfg.ConfigEndpoints, cfg.ExportEndpoints,
+ cfg.RecoveryEndpoints, cfg.UpdateEndpoints, cfg.WebSocketEndpoints,
+ cfg.GeneralAPI, cfg.PublicEndpoints,
+ } {
+ limiter.Stop()
+ }
+}
+
func generalAPIRateLimit() int {
if os.Getenv("PULSE_DEV") != "true" {
return defaultGeneralAPIRateLimit
diff --git a/internal/api/ratelimit.go b/internal/api/ratelimit.go
index cb597e0bf4..9d207c4b3f 100644
--- a/internal/api/ratelimit.go
+++ b/internal/api/ratelimit.go
@@ -19,6 +19,8 @@ type RateLimiter struct {
limit int
window time.Duration
stopCleanup chan struct{}
+ cleanupDone chan struct{}
+ stopOnce sync.Once
}
// NewRateLimiter creates a rate limiter that allows limit requests per window duration.
@@ -44,10 +46,12 @@ func NewRateLimiter(limit int, window time.Duration) *RateLimiter {
limit: limit,
window: window,
stopCleanup: make(chan struct{}),
+ cleanupDone: make(chan struct{}),
}
// Clean up old entries periodically
go func() {
+ defer close(rl.cleanupDone)
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
@@ -64,13 +68,19 @@ func NewRateLimiter(limit int, window time.Duration) *RateLimiter {
return rl
}
-// Stop stops the cleanup routine
+// Stop stops and joins the cleanup routine. Concurrent owners may call it
+// repeatedly without racing a channel close. Request budgets remain intact.
func (rl *RateLimiter) Stop() {
- select {
- case <-rl.stopCleanup:
+ if rl == nil {
return
- default:
- close(rl.stopCleanup)
+ }
+ rl.stopOnce.Do(func() {
+ if rl.stopCleanup != nil {
+ close(rl.stopCleanup)
+ }
+ })
+ if rl.cleanupDone != nil {
+ <-rl.cleanupDone
}
}
diff --git a/internal/api/router.go b/internal/api/router.go
index 44e96339d8..082251c472 100644
--- a/internal/api/router.go
+++ b/internal/api/router.go
@@ -132,6 +132,7 @@ type Router struct {
handoffExchangeRateLimiter *RateLimiter
bootstrapTokenValidationLimiter *RateLimiter
tenantRateLimiter *TenantRateLimiter
+ endpointRateLimitConfig *EndpointRateLimitConfig
persistence *config.ConfigPersistence
multiTenant *config.MultiTenantPersistence
oidcMu sync.Mutex
@@ -166,6 +167,9 @@ type Router struct {
lifecycleCancel context.CancelFunc
lifecycleWG sync.WaitGroup
backgroundWorkersOnce sync.Once
+ backgroundShutdownOnce sync.Once
+ lifecycleMu sync.Mutex
+ lifecycleStopped bool
hostedMode bool
stripeWebhookHandlers *StripeWebhookHandlers
patrolLifecycleMu sync.Mutex
@@ -234,6 +238,7 @@ func NewRouter(cfg *config.Config, monitor *monitoring.Monitor, mtMonitor *monit
wsHub: wsHub,
reloadFunc: reloadFunc,
updateManager: updateManager,
+ endpointRateLimitConfig: newEndpointRateLimitConfig(),
updateHistory: updateHistory,
exportLimiter: NewRateLimiter(5, 1*time.Minute), // 5 attempts per minute
downloadLimiter: NewRateLimiter(60, 1*time.Minute), // downloads/installers per minute per IP
@@ -350,7 +355,7 @@ func NewRouter(cfg *config.Config, monitor *monitoring.Monitor, mtMonitor *monit
// Auth context middleware extracts user/token info BEFORE tenant middleware
handler = AuthContextMiddleware(cfg, r.mtMonitor, handler)
- handler = UniversalRateLimitMiddlewareWithConfig(newEndpointRateLimitConfig(), handler)
+ handler = UniversalRateLimitMiddlewareWithConfig(r.endpointRateLimitConfig, handler)
r.wrapped = handler
return r
}
@@ -1562,6 +1567,11 @@ func (r *Router) startLifecycleWorker(worker func()) {
if r == nil || worker == nil {
return
}
+ r.lifecycleMu.Lock()
+ defer r.lifecycleMu.Unlock()
+ if r.lifecycleStopped {
+ return
+ }
r.lifecycleWG.Add(1)
go func() {
defer r.lifecycleWG.Done()
@@ -2978,40 +2988,64 @@ func (r *Router) ShutdownRBAC() {
}
}
+// ShutdownBackgroundWorkers seals worker admission and joins the workers
+// created by this Router. Call it after HTTP admission is stopped, before
+// closing the monitor and removing its data directory. It does not close
+// resource or RBAC stores; their existing owner shutdown methods still apply.
+func (r *Router) ShutdownBackgroundWorkers() {
+ r.shutdownBackgroundWorkers()
+}
+
func (r *Router) shutdownBackgroundWorkers() {
- if r.lifecycleCancel != nil {
- r.lifecycleCancel()
- }
- if r.updateManager != nil {
- r.updateManager.Close()
- }
- if r.sessionStore != nil {
- r.sessionStore.Shutdown()
- }
- if r.csrfStore != nil {
- r.csrfStore.Shutdown()
- }
- if r.recoveryTokenStore != nil {
- r.recoveryTokenStore.Shutdown()
- }
- if r.aiSettingsHandler != nil {
- r.aiSettingsHandler.StopServices()
- }
- if r.aiHandler != nil {
- r.aiHandler.clearApprovalStore()
- }
- if r.trueNASPoller != nil {
- r.trueNASPoller.Stop()
- }
- if r.vmwarePoller != nil {
- r.vmwarePoller.Stop()
+ if r == nil {
+ return
}
- if r.deployStore != nil {
- if err := r.deployStore.Close(); err != nil {
- log.Error().Err(err).Msg("Failed to close deploy store")
+ r.backgroundShutdownOnce.Do(func() {
+ r.lifecycleMu.Lock()
+ r.lifecycleStopped = true
+ if r.lifecycleCancel != nil {
+ r.lifecycleCancel()
}
- }
- r.lifecycleWG.Wait()
+ r.lifecycleMu.Unlock()
+ for _, limiter := range []*RateLimiter{
+ r.exportLimiter, r.downloadLimiter, r.signupRateLimiter,
+ r.handoffExchangeRateLimiter, r.bootstrapTokenValidationLimiter,
+ } {
+ limiter.Stop()
+ }
+ r.endpointRateLimitConfig.Stop()
+ r.tenantRateLimiter.Stop()
+ if r.updateManager != nil {
+ r.updateManager.Close()
+ }
+ if r.sessionStore != nil {
+ r.sessionStore.Shutdown()
+ }
+ if r.csrfStore != nil {
+ r.csrfStore.Shutdown()
+ }
+ if r.recoveryTokenStore != nil {
+ r.recoveryTokenStore.Shutdown()
+ }
+ if r.aiSettingsHandler != nil {
+ r.aiSettingsHandler.StopServices()
+ }
+ if r.aiHandler != nil {
+ r.aiHandler.clearApprovalStore()
+ }
+ if r.trueNASPoller != nil {
+ r.trueNASPoller.Stop()
+ }
+ if r.vmwarePoller != nil {
+ r.vmwarePoller.Stop()
+ }
+ if r.deployStore != nil {
+ if err := r.deployStore.Close(); err != nil {
+ log.Error().Err(err).Msg("Failed to close deploy store")
+ }
+ }
+ r.lifecycleWG.Wait()
+ })
}
// StartAIChat starts the AI chat service
diff --git a/internal/api/router_integration_test.go b/internal/api/router_integration_test.go
index 8835f1febe..e5ac444e01 100644
--- a/internal/api/router_integration_test.go
+++ b/internal/api/router_integration_test.go
@@ -201,9 +201,17 @@ func newIntegrationServerWithRuntimeMode(
return nil
}, version)
+ // Register constructor cleanup before a listener or request assertion can
+ // fail. The fixture owns the same lifecycle used by the running server.
+ t.Cleanup(func() {
+ router.ShutdownBackgroundWorkers()
+ router.ShutdownResourceStores()
+ router.ShutdownRBAC()
+ })
srv := newIPv4HTTPServer(t, router.Handler())
t.Cleanup(func() {
srv.Close()
+ router.ShutdownBackgroundWorkers()
if monitor != nil {
monitor.StopDiscoveryService()
monitor.Stop()
diff --git a/internal/api/router_limiter_lifecycle_test.go b/internal/api/router_limiter_lifecycle_test.go
new file mode 100644
index 0000000000..b649e9ea39
--- /dev/null
+++ b/internal/api/router_limiter_lifecycle_test.go
@@ -0,0 +1,143 @@
+package api
+
+import (
+ "net/http"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+)
+
+func routerOwnedLimiters(r *Router) []*RateLimiter {
+ limiters := []*RateLimiter{r.exportLimiter, r.downloadLimiter, r.signupRateLimiter,
+ r.handoffExchangeRateLimiter, r.bootstrapTokenValidationLimiter}
+ limiters = append(limiters, rateLimitFixtureWorkers(r.endpointRateLimitConfig)...)
+ if r.tenantRateLimiter != nil {
+ limiters = append(limiters, r.tenantRateLimiter.limiter)
+ }
+ return limiters
+}
+
+func requireLimiterJoined(t *testing.T, limiter *RateLimiter) {
+ t.Helper()
+ select {
+ case <-limiter.cleanupDone:
+ default:
+ t.Error("Stop returned with a cleanup worker still running")
+ }
+}
+
+func TestRateLimiterConcurrentStopJoinsWithoutResettingBudget(t *testing.T) {
+ limiter := NewRateLimiter(1, time.Minute)
+ t.Cleanup(limiter.Stop)
+ if !limiter.Allow("192.0.2.1") || limiter.Allow("192.0.2.1") {
+ t.Fatal("fixture did not consume its budget")
+ }
+ var wg sync.WaitGroup
+ for i := 0; i < 32; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ limiter.Stop()
+ limiter.Allow("192.0.2.2") // In-flight requests retain normal accounting.
+ }()
+ }
+ wg.Wait()
+ requireLimiterJoined(t, limiter)
+ if limiter.Allow("192.0.2.1") {
+ t.Error("stopping cleanup reset an exhausted request budget")
+ }
+ (*RateLimiter)(nil).Stop()
+ (&RateLimiter{}).Stop()
+}
+
+func TestRouterShutdownJoinsEveryOwnedLimiter(t *testing.T) {
+ for _, hosted := range []string{"false", "true"} {
+ t.Run(hosted, func(t *testing.T) {
+ t.Setenv("PULSE_HOSTED_MODE", hosted)
+ for i := 0; i < 3; i++ {
+ r := NewRouter(&config.Config{DataPath: t.TempDir()}, nil, nil, nil, nil, "test")
+ cleanupTestRouter(t, r)
+ owned := routerOwnedLimiters(r)
+ want := 13
+ if hosted == "true" {
+ want++
+ }
+ if len(owned) != want {
+ t.Fatalf("owned limiters = %d, want %d", len(owned), want)
+ }
+ r.ShutdownBackgroundWorkers()
+ r.ShutdownBackgroundWorkers()
+ for _, limiter := range owned {
+ requireLimiterJoined(t, limiter)
+ }
+ }
+ })
+ }
+}
+
+func TestRouterShutdownPreservesOtherRouterAndGlobalBudgets(t *testing.T) {
+ isolateRateLimitFixture(t)
+ InitializeRateLimiters()
+ global := globalRateLimitConfig
+ first := NewRouter(&config.Config{DataPath: t.TempDir()}, nil, nil, nil, nil, "test")
+ cleanupTestRouter(t, first)
+ second := NewRouter(&config.Config{DataPath: t.TempDir()}, nil, nil, nil, nil, "test")
+ cleanupTestRouter(t, second)
+ login := getRateLimiterForEndpoint(second.endpointRateLimitConfig, "/api/login", http.MethodPost)
+ for i := 0; i < login.limit; i++ {
+ if !login.Allow("192.0.2.1") {
+ t.Fatal("other router budget exhausted early")
+ }
+ }
+ first.ShutdownBackgroundWorkers()
+ if login.Allow("192.0.2.1") {
+ t.Error("another router's shutdown reset login enforcement")
+ }
+ for _, limiter := range append(routerOwnedLimiters(second), rateLimitFixtureWorkers(global)...) {
+ select {
+ case <-limiter.stopCleanup:
+ t.Error("shutdown stopped a limiter owned by another router/global configuration")
+ default:
+ }
+ }
+ if !second.exportLimiter.Allow("192.0.2.2") {
+ t.Error("other router no longer admits its independent export budget")
+ }
+}
+
+func TestRouterShutdownSealsAndJoinsConcurrentWorkerAdmission(t *testing.T) {
+ r := NewRouter(&config.Config{DataPath: t.TempDir()}, nil, nil, nil, nil, "test")
+ cleanupTestRouter(t, r)
+ var started, exited atomic.Int64
+ var callers sync.WaitGroup
+ for i := 0; i < 32; i++ {
+ callers.Add(2)
+ go func() {
+ defer callers.Done()
+ r.startLifecycleWorker(func() {
+ started.Add(1)
+ <-r.lifecycleCtx.Done()
+ exited.Add(1)
+ })
+ }()
+ go func() {
+ defer callers.Done()
+ r.ShutdownBackgroundWorkers()
+ }()
+ }
+ callers.Wait()
+ if started.Load() != exited.Load() {
+ t.Fatalf("shutdown returned before joining admitted workers: started %d, exited %d", started.Load(), exited.Load())
+ }
+ r.startLifecycleWorker(func() { t.Error("worker admitted after shutdown") })
+ r.StartBackgroundWorkers() // Late application start cannot reopen admission.
+ r.ShutdownBackgroundWorkers()
+ for _, limiter := range routerOwnedLimiters(r) {
+ requireLimiterJoined(t, limiter)
+ }
+ (*Router)(nil).ShutdownBackgroundWorkers()
+ (&Router{}).ShutdownBackgroundWorkers()
+}