Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Join Router-owned limiter workers during shutdown
pulse-triage[bot]5b560521e26d
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.mdindex 90f36f2f4b..1b1ee43136 100644--- a/docs/release-control/v6/internal/subsystems/api-contracts.md+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md@@ -11221,3 +11221,20 @@ Pinning the Patrol attention detail header on phoneslayout-only change. It reads no new attention fields, sends no new request,and leaves the attention projection, lifecycle, and queue ordering contractsabove unchanged.++### Router-owned worker shutdown++Each Router owns its five route-specific limiters, eight endpoint-category+limiters and, in hosted mode, its tenant limiter. `ShutdownBackgroundWorkers`+seals lifecycle-worker admission, cancels and joins the constructor-owned+workers exactly once; the server's existing `ShutdownAIIntelligence` calls+that same lifecycle. HTTP admission must stop before shutdown. Resource and+RBAC stores retain their separate shutdown owners. Integration fixtures use+this lifecycle before stopping the monitor and removing temporary data.++`RateLimiter.Stop` is concurrent-safe and joins its cleanup worker. Stopping+cleanup does not clear request history, bypass rate enforcement or stop another+Router's or the package-global limiters. Limits, windows, tenant selection,+revocation and response payloads are unchanged. The connected lifetime and+ownership controls are in `router_limiter_lifecycle_test.go`; the integration+fixture's cleanup also runs on a failed listener/request assertion.diff --git a/internal/api/rate_limit_config.go b/internal/api/rate_limit_config.goindex 575364fc53..1ddbc3f55d 100644--- a/internal/api/rate_limit_config.go+++ b/internal/api/rate_limit_config.go@@ -57,6 +57,21 @@ func newEndpointRateLimitConfig() *EndpointRateLimitConfig {}}+// Stop joins only the workers owned by this endpoint configuration. A Router+// must not stop the package-global configuration or another Router's budgets.+func (cfg *EndpointRateLimitConfig) Stop() {+ if cfg == nil {+ return+ }+ for _, limiter := range []*RateLimiter{+ cfg.AuthEndpoints, cfg.ConfigEndpoints, cfg.ExportEndpoints,+ cfg.RecoveryEndpoints, cfg.UpdateEndpoints, cfg.WebSocketEndpoints,+ cfg.GeneralAPI, cfg.PublicEndpoints,+ } {+ limiter.Stop()+ }+}+func generalAPIRateLimit() int {if os.Getenv("PULSE_DEV") != "true" {return defaultGeneralAPIRateLimitdiff --git a/internal/api/ratelimit.go b/internal/api/ratelimit.goindex cb597e0bf4..9d207c4b3f 100644--- a/internal/api/ratelimit.go+++ b/internal/api/ratelimit.go@@ -19,6 +19,8 @@ type RateLimiter struct {limit intwindow time.DurationstopCleanup chan struct{}+ cleanupDone chan struct{}+ stopOnce sync.Once}// NewRateLimiter creates a rate limiter that allows limit requests per window duration.@@ -44,10 +46,12 @@ func NewRateLimiter(limit int, window time.Duration) *RateLimiter {limit: limit,window: window,stopCleanup: make(chan struct{}),+ cleanupDone: make(chan struct{}),}// Clean up old entries periodicallygo func() {+ defer close(rl.cleanupDone)ticker := time.NewTicker(1 * time.Minute)defer ticker.Stop()@@ -64,13 +68,19 @@ func NewRateLimiter(limit int, window time.Duration) *RateLimiter {return rl}-// Stop stops the cleanup routine+// Stop stops and joins the cleanup routine. Concurrent owners may call it+// repeatedly without racing a channel close. Request budgets remain intact.func (rl *RateLimiter) Stop() {- select {- case <-rl.stopCleanup:+ if rl == nil {return- default:- close(rl.stopCleanup)+ }+ rl.stopOnce.Do(func() {+ if rl.stopCleanup != nil {+ close(rl.stopCleanup)+ }+ })+ if rl.cleanupDone != nil {+ <-rl.cleanupDone}}diff --git a/internal/api/router.go b/internal/api/router.goindex 44e96339d8..082251c472 100644--- a/internal/api/router.go+++ b/internal/api/router.go@@ -132,6 +132,7 @@ type Router struct {handoffExchangeRateLimiter *RateLimiterbootstrapTokenValidationLimiter *RateLimitertenantRateLimiter *TenantRateLimiter+ endpointRateLimitConfig *EndpointRateLimitConfigpersistence *config.ConfigPersistencemultiTenant *config.MultiTenantPersistenceoidcMu sync.Mutex@@ -166,6 +167,9 @@ type Router struct {lifecycleCancel context.CancelFunclifecycleWG sync.WaitGroupbackgroundWorkersOnce sync.Once+ backgroundShutdownOnce sync.Once+ lifecycleMu sync.Mutex+ lifecycleStopped boolhostedMode boolstripeWebhookHandlers *StripeWebhookHandlerspatrolLifecycleMu sync.Mutex@@ -234,6 +238,7 @@ func NewRouter(cfg *config.Config, monitor *monitoring.Monitor, mtMonitor *monitwsHub: wsHub,reloadFunc: reloadFunc,updateManager: updateManager,+ endpointRateLimitConfig: newEndpointRateLimitConfig(),updateHistory: updateHistory,exportLimiter: NewRateLimiter(5, 1*time.Minute), // 5 attempts per minutedownloadLimiter: NewRateLimiter(60, 1*time.Minute), // downloads/installers per minute per IP@@ -350,7 +355,7 @@ func NewRouter(cfg *config.Config, monitor *monitoring.Monitor, mtMonitor *monit// Auth context middleware extracts user/token info BEFORE tenant middlewarehandler = AuthContextMiddleware(cfg, r.mtMonitor, handler)- handler = UniversalRateLimitMiddlewareWithConfig(newEndpointRateLimitConfig(), handler)+ handler = UniversalRateLimitMiddlewareWithConfig(r.endpointRateLimitConfig, handler)r.wrapped = handlerreturn r}@@ -1562,6 +1567,11 @@ func (r *Router) startLifecycleWorker(worker func()) {if r == nil || worker == nil {return}+ r.lifecycleMu.Lock()+ defer r.lifecycleMu.Unlock()+ if r.lifecycleStopped {+ return+ }r.lifecycleWG.Add(1)go func() {defer r.lifecycleWG.Done()@@ -2978,40 +2988,64 @@ func (r *Router) ShutdownRBAC() {}}+// ShutdownBackgroundWorkers seals worker admission and joins the workers+// created by this Router. Call it after HTTP admission is stopped, before+// closing the monitor and removing its data directory. It does not close+// resource or RBAC stores; their existing owner shutdown methods still apply.+func (r *Router) ShutdownBackgroundWorkers() {+ r.shutdownBackgroundWorkers()+}+func (r *Router) shutdownBackgroundWorkers() {- if r.lifecycleCancel != nil {- r.lifecycleCancel()- }- if r.updateManager != nil {- r.updateManager.Close()- }- if r.sessionStore != nil {- r.sessionStore.Shutdown()- }- if r.csrfStore != nil {- r.csrfStore.Shutdown()- }- if r.recoveryTokenStore != nil {- r.recoveryTokenStore.Shutdown()- }- if r.aiSettingsHandler != nil {- r.aiSettingsHandler.StopServices()- }- if r.aiHandler != nil {- r.aiHandler.clearApprovalStore()- }- if r.trueNASPoller != nil {- r.trueNASPoller.Stop()- }- if r.vmwarePoller != nil {- r.vmwarePoller.Stop()+ if r == nil {+ return}- if r.deployStore != nil {- if err := r.deployStore.Close(); err != nil {- log.Error().Err(err).Msg("Failed to close deploy store")+ r.backgroundShutdownOnce.Do(func() {+ r.lifecycleMu.Lock()+ r.lifecycleStopped = true+ if r.lifecycleCancel != nil {+ r.lifecycleCancel()}- }- r.lifecycleWG.Wait()+ r.lifecycleMu.Unlock()+ for _, limiter := range []*RateLimiter{+ r.exportLimiter, r.downloadLimiter, r.signupRateLimiter,+ r.handoffExchangeRateLimiter, r.bootstrapTokenValidationLimiter,+ } {+ limiter.Stop()+ }+ r.endpointRateLimitConfig.Stop()+ r.tenantRateLimiter.Stop()+ if r.updateManager != nil {+ r.updateManager.Close()+ }+ if r.sessionStore != nil {+ r.sessionStore.Shutdown()+ }+ if r.csrfStore != nil {+ r.csrfStore.Shutdown()+ }+ if r.recoveryTokenStore != nil {+ r.recoveryTokenStore.Shutdown()+ }+ if r.aiSettingsHandler != nil {+ r.aiSettingsHandler.StopServices()+ }+ if r.aiHandler != nil {+ r.aiHandler.clearApprovalStore()+ }+ if r.trueNASPoller != nil {+ r.trueNASPoller.Stop()+ }+ if r.vmwarePoller != nil {+ r.vmwarePoller.Stop()+ }+ if r.deployStore != nil {+ if err := r.deployStore.Close(); err != nil {+ log.Error().Err(err).Msg("Failed to close deploy store")+ }+ }+ r.lifecycleWG.Wait()+ })}// StartAIChat starts the AI chat servicediff --git a/internal/api/router_integration_test.go b/internal/api/router_integration_test.goindex 8835f1febe..e5ac444e01 100644--- a/internal/api/router_integration_test.go+++ b/internal/api/router_integration_test.go@@ -201,9 +201,17 @@ func newIntegrationServerWithRuntimeMode(return nil}, version)+ // Register constructor cleanup before a listener or request assertion can+ // fail. The fixture owns the same lifecycle used by the running server.+ t.Cleanup(func() {+ router.ShutdownBackgroundWorkers()+ router.ShutdownResourceStores()+ router.ShutdownRBAC()+ })srv := newIPv4HTTPServer(t, router.Handler())t.Cleanup(func() {srv.Close()+ router.ShutdownBackgroundWorkers()if monitor != nil {monitor.StopDiscoveryService()monitor.Stop()diff --git a/internal/api/router_limiter_lifecycle_test.go b/internal/api/router_limiter_lifecycle_test.gonew file mode 100644index 0000000000..b649e9ea39--- /dev/null+++ b/internal/api/router_limiter_lifecycle_test.go@@ -0,0 +1,143 @@+package api++import (+ "net/http"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+)++func routerOwnedLimiters(r *Router) []*RateLimiter {+ limiters := []*RateLimiter{r.exportLimiter, r.downloadLimiter, r.signupRateLimiter,+ r.handoffExchangeRateLimiter, r.bootstrapTokenValidationLimiter}+ limiters = append(limiters, rateLimitFixtureWorkers(r.endpointRateLimitConfig)...)+ if r.tenantRateLimiter != nil {+ limiters = append(limiters, r.tenantRateLimiter.limiter)+ }+ return limiters+}++func requireLimiterJoined(t *testing.T, limiter *RateLimiter) {+ t.Helper()+ select {+ case <-limiter.cleanupDone:+ default:+ t.Error("Stop returned with a cleanup worker still running")+ }+}++func TestRateLimiterConcurrentStopJoinsWithoutResettingBudget(t *testing.T) {+ limiter := NewRateLimiter(1, time.Minute)+ t.Cleanup(limiter.Stop)+ if !limiter.Allow("192.0.2.1") || limiter.Allow("192.0.2.1") {+ t.Fatal("fixture did not consume its budget")+ }+ var wg sync.WaitGroup+ for i := 0; i < 32; i++ {+ wg.Add(1)+ go func() {+ defer wg.Done()+ limiter.Stop()+ limiter.Allow("192.0.2.2") // In-flight requests retain normal accounting.+ }()+ }+ wg.Wait()+ requireLimiterJoined(t, limiter)+ if limiter.Allow("192.0.2.1") {+ t.Error("stopping cleanup reset an exhausted request budget")+ }+ (*RateLimiter)(nil).Stop()+ (&RateLimiter{}).Stop()+}++func TestRouterShutdownJoinsEveryOwnedLimiter(t *testing.T) {+ for _, hosted := range []string{"false", "true"} {+ t.Run(hosted, func(t *testing.T) {+ t.Setenv("PULSE_HOSTED_MODE", hosted)+ for i := 0; i < 3; i++ {+ r := NewRouter(&config.Config{DataPath: t.TempDir()}, nil, nil, nil, nil, "test")+ cleanupTestRouter(t, r)+ owned := routerOwnedLimiters(r)+ want := 13+ if hosted == "true" {+ want+++ }+ if len(owned) != want {+ t.Fatalf("owned limiters = %d, want %d", len(owned), want)+ }+ r.ShutdownBackgroundWorkers()+ r.ShutdownBackgroundWorkers()+ for _, limiter := range owned {+ requireLimiterJoined(t, limiter)+ }+ }+ })+ }+}++func TestRouterShutdownPreservesOtherRouterAndGlobalBudgets(t *testing.T) {+ isolateRateLimitFixture(t)+ InitializeRateLimiters()+ global := globalRateLimitConfig+ first := NewRouter(&config.Config{DataPath: t.TempDir()}, nil, nil, nil, nil, "test")+ cleanupTestRouter(t, first)+ second := NewRouter(&config.Config{DataPath: t.TempDir()}, nil, nil, nil, nil, "test")+ cleanupTestRouter(t, second)+ login := getRateLimiterForEndpoint(second.endpointRateLimitConfig, "/api/login", http.MethodPost)+ for i := 0; i < login.limit; i++ {+ if !login.Allow("192.0.2.1") {+ t.Fatal("other router budget exhausted early")+ }+ }+ first.ShutdownBackgroundWorkers()+ if login.Allow("192.0.2.1") {+ t.Error("another router's shutdown reset login enforcement")+ }+ for _, limiter := range append(routerOwnedLimiters(second), rateLimitFixtureWorkers(global)...) {+ select {+ case <-limiter.stopCleanup:+ t.Error("shutdown stopped a limiter owned by another router/global configuration")+ default:+ }+ }+ if !second.exportLimiter.Allow("192.0.2.2") {+ t.Error("other router no longer admits its independent export budget")+ }+}++func TestRouterShutdownSealsAndJoinsConcurrentWorkerAdmission(t *testing.T) {+ r := NewRouter(&config.Config{DataPath: t.TempDir()}, nil, nil, nil, nil, "test")+ cleanupTestRouter(t, r)+ var started, exited atomic.Int64+ var callers sync.WaitGroup+ for i := 0; i < 32; i++ {+ callers.Add(2)+ go func() {+ defer callers.Done()+ r.startLifecycleWorker(func() {+ started.Add(1)+ <-r.lifecycleCtx.Done()+ exited.Add(1)+ })+ }()+ go func() {+ defer callers.Done()+ r.ShutdownBackgroundWorkers()+ }()+ }+ callers.Wait()+ if started.Load() != exited.Load() {+ t.Fatalf("shutdown returned before joining admitted workers: started %d, exited %d", started.Load(), exited.Load())+ }+ r.startLifecycleWorker(func() { t.Error("worker admitted after shutdown") })+ r.StartBackgroundWorkers() // Late application start cannot reopen admission.+ r.ShutdownBackgroundWorkers()+ for _, limiter := range routerOwnedLimiters(r) {+ requireLimiterJoined(t, limiter)+ }+ (*Router)(nil).ShutdownBackgroundWorkers()+ (&Router{}).ShutdownBackgroundWorkers()+}