Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Close retired tenant manager admission before shutdown
pulse-triage[bot]b010e9a0f7c7
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 94ae194f2b..d4db747e21 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,28 @@## Purpose+### Retired tenant managers cannot create replacement runtimes++`MultiTenantMonitor.Stop` closes admission under the same lock as lazy+initialization, before cancelling or draining any tenant. Both cached and new+`GetMonitor` lookups reject a retired manager; `PeekMonitor` does not expose its+retained runtimes. The initialization path rechecks the fence after acquiring+its write lock. An incomplete shutdown keeps its existing runtime/data owner,+not a usable read owner. Repeated Stop can finish cleanup but never reopens+admission. A separately constructed manager owns replacement runtimes.++`multi_tenant_monitor_stop_test.go` covers unused/default/scoped lookups,+canonical-config and saved-file preservation, concurrent cached reads during+drain, incomplete cleanup and explicit cleanup completion.+`TestReloadableMonitorRetiredManagerRejectsLookups` exercises the real reload+handover while preserving its canonical credential owner.+`TestTenantMonitorRetirementFailsClosedAndReplacementRecovers` connects the+retirement fence to the existing tenant HTTP 503 guard and configuration+handler: stale consumers cannot recreate or borrow default state, while a+new manager recovers the saved tenant inventory. This does not prove detached+worker quiescence, the whole removal/startup journey or installed recovery.+Authentication, tenant selection, TLS and the HTTP error shape are unchanged.+### Datastore discovery cannot turn unread history into absencePBS datastore enumeration uses the same bounded, fully read and closed HTTP 200diff --git a/internal/api/tenant_monitor_retirement_test.go b/internal/api/tenant_monitor_retirement_test.gonew file mode 100644index 0000000000..717d053514--- /dev/null+++ b/internal/api/tenant_monitor_retirement_test.go@@ -0,0 +1,84 @@+package api++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+)++// A stale router/handler reference can outlive a manager swap. It must report+// unavailability, not recreate the retired org's runtime or fall back to the+// default org. A separately owned replacement manager can still serve it.+func TestTenantMonitorRetirementFailsClosedAndReplacementRecovers(t *testing.T) {+ dir := t.TempDir()+ mtp := config.NewMultiTenantPersistence(dir)+ if err := mtp.SaveOrganization(&models.Organization{ID: "tenant-a", DisplayName: "Tenant A"}); err != nil {+ t.Fatal(err)+ }+ persistence, err := mtp.GetPersistence("tenant-a")+ if err != nil {+ t.Fatal(err)+ }+ if err := persistence.SaveNodesConfig([]config.PVEInstance{{Name: "Tenant saved PVE", Host: "https://tenant.invalid:8006", Disabled: true}}, nil, nil); err != nil {+ t.Fatal(err)+ }+ cfg := &config.Config{DataPath: dir, ConfigPath: dir}+ retired := monitoring.NewMultiTenantMonitor(cfg, mtp, nil)+ t.Cleanup(retired.Stop)+ var retiredInitializations atomic.Int32+ retired.SetMonitorInitializer(func(*monitoring.Monitor) { retiredInitializations.Add(1) })+ retired.Stop()++ router := &Router{mtMonitor: retired, monitor: &monitoring.Monitor{}}+ handlers := NewConfigHandlers(mtp, retired, nil, nil, nil, nil, false)+ guarded := router.tenantMonitorGuardMiddleware(http.HandlerFunc(handlers.HandleGetNodes))+ request := func() *httptest.ResponseRecorder {+ req := httptest.NewRequest(http.MethodGet, "/api/config/nodes", nil)+ req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "tenant-a"))+ rec := httptest.NewRecorder()+ guarded.ServeHTTP(rec, req)+ return rec+ }+ rec := request()+ if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), `"tenant_unavailable"`) {+ t.Fatalf("retired manager returned %d: %s", rec.Code, rec.Body.String())+ }+ if retiredInitializations.Load() != 0 {+ t.Fatal("HTTP guard reinitialized the retired tenant")+ }+ ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-a")+ if cfg, persistence, monitor := handlers.ContextState(ctx); cfg != nil || persistence != nil || monitor != nil {+ t.Fatal("stale config handler exposed tenant or default state")+ }+ if router.getTenantMonitor(ctx) != nil {+ t.Fatal("stale router fell back to the default runtime")+ }++ replacement := monitoring.NewMultiTenantMonitor(cfg, mtp, nil)+ t.Cleanup(replacement.Stop)+ // Rebind only these consumers, without altering production swap order.+ router.mtMonitor = replacement+ handlers.SetMultiTenantMonitor(replacement)+ rec = request()+ if rec.Code != http.StatusOK {+ t.Fatalf("replacement manager returned %d: %s", rec.Code, rec.Body.String())+ }+ var nodes []NodeResponse+ if err := json.Unmarshal(rec.Body.Bytes(), &nodes); err != nil {+ t.Fatal(err)+ }+ if len(nodes) != 1 || nodes[0].Name != "Tenant saved PVE" || nodes[0].Host != "https://tenant.invalid:8006" {+ t.Fatal("replacement did not recover the saved tenant inventory")+ }+ if retiredInitializations.Load() != 0 {+ t.Fatal("replacement read consulted the retired owner")+ }+}diff --git a/internal/monitoring/multi_tenant_monitor.go b/internal/monitoring/multi_tenant_monitor.goindex ecd5c9a0f7..65a7b9c3ed 100644--- a/internal/monitoring/multi_tenant_monitor.go+++ b/internal/monitoring/multi_tenant_monitor.go@@ -28,6 +28,7 @@ type MultiTenantMonitor struct {initializer func(*Monitor)globalCtx context.ContextglobalCancel context.CancelFunc+ stopped bool // Terminal admission fence, protected by mu.}// NewMultiTenantMonitor creates a new multi-tenant monitor manager.@@ -152,6 +153,10 @@ func (mtm *MultiTenantMonitor) GetMonitor(orgID string) (*Monitor, error) {}mtm.mu.RLock()+ if mtm.stopped {+ mtm.mu.RUnlock()+ return nil, fmt.Errorf("tenant monitor manager is stopped")+ }monitor, exists := mtm.monitors[orgID]_, deleting := mtm.tenantDeleting[orgID]mtm.mu.RUnlock()@@ -166,6 +171,12 @@ func (mtm *MultiTenantMonitor) GetMonitor(orgID string) (*Monitor, error) {mtm.mu.Lock()defer mtm.mu.Unlock()+ // Stop can win between the cache lookup and this initialization lock.+ // A retired manager must never reopen stores or mutate the shared config.+ if mtm.stopped {+ return nil, fmt.Errorf("tenant monitor manager is stopped")+ }+if mtm.monitors == nil {mtm.monitors = make(map[string]*Monitor)}@@ -300,15 +311,20 @@ func (mtm *MultiTenantMonitor) PeekMonitor(orgID string) (*Monitor, bool) {mtm.mu.RLock()defer mtm.mu.RUnlock()+ if mtm.stopped {+ return nil, false+ }monitor, exists := mtm.monitors[orgID]return monitor, exists}-// Stop stops all tenant monitors.+// Stop permanently closes admission and stops all tenant monitors. A new+// manager, rather than a lookup on this retired one, owns replacement runtimes.func (mtm *MultiTenantMonitor) Stop() {log.Info().Msg("stopping MultiTenantMonitor and all tenant instances")mtm.mu.Lock()+ mtm.stopped = trueif mtm.globalCancel != nil {mtm.globalCancel()}diff --git a/internal/monitoring/multi_tenant_monitor_stop_test.go b/internal/monitoring/multi_tenant_monitor_stop_test.gonew file mode 100644index 0000000000..007524e04c--- /dev/null+++ b/internal/monitoring/multi_tenant_monitor_stop_test.go@@ -0,0 +1,153 @@+package monitoring++import (+ "crypto/sha256"+ "io/fs"+ "os"+ "path/filepath"+ "reflect"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+)++// Stop retires the owner, not just the currently materialized tenant loops.+// In particular, a handler holding the old manager through a reload must not+// initialize a previously unused tenant or reload the default config into it.+func TestMultiTenantMonitorStopRejectsLazyRecreation(t *testing.T) {+ mtp, persistence := newTestTenantPersistence(t)+ if err := mtp.SaveOrganization(&models.Organization{ID: "tenant-a", DisplayName: "Tenant A"}); err != nil {+ t.Fatal(err)+ }+ cfg := &config.Config{+ DataPath: persistence.GetConfigDir(),+ PVEInstances: []config.PVEInstance{{Name: "saved-primary", Host: "https://primary.invalid:8006", Disabled: true}},+ APITokens: []config.APITokenRecord{{ID: "primary-token", Hash: "synthetic-primary-hash"}},+ }+ before := cfg.DeepCopy()+ snapshot := func() map[string][32]byte {+ result := make(map[string][32]byte)+ err := filepath.WalkDir(mtp.BaseDataDir(), func(path string, entry fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ if entry.IsDir() {+ result[path] = [32]byte{}+ return nil+ }+ data, err := os.ReadFile(path)+ if err != nil {+ return err+ }+ result[path] = sha256.Sum256(data)+ return nil+ })+ if err != nil {+ t.Fatal(err)+ }+ return result+ }+ filesBefore := snapshot()+ mtm := NewMultiTenantMonitor(cfg, mtp, nil)+ t.Cleanup(mtm.Stop)+ var initialized atomic.Int32+ mtm.SetMonitorInitializer(func(*Monitor) { initialized.Add(1) })+ mtm.Stop()+ mtm.Stop() // Repeated shutdown must not reopen admission.++ for _, orgID := range []string{"default", "tenant-a", "not-provisioned", " tenant-a "} {+ if monitor, err := mtm.GetMonitor(orgID); monitor != nil || err == nil {+ t.Errorf("retired manager admitted %q: monitor present=%v, err=%v", orgID, monitor != nil, err)+ }+ if monitor, ok := mtm.PeekMonitor(orgID); monitor != nil || ok {+ t.Errorf("retired manager exposed cached monitor for %q", orgID)+ }+ }+ if initialized.Load() != 0 {+ t.Errorf("retired manager initialized %d runtimes", initialized.Load())+ }+ if !reflect.DeepEqual(cfg.DeepCopy(), before) {+ t.Error("retired lookup changed the canonical config")+ }+ // A rejected lookup must not create stores/writers or change saved files.+ if !reflect.DeepEqual(snapshot(), filesBefore) {+ t.Error("retired lookup created or changed persisted runtime material")+ }++}++func TestMultiTenantMonitorStopFencesRetainedRuntimes(t *testing.T) {+ mtm := NewMultiTenantMonitor(&config.Config{}, nil, nil)+ // Retain a loop deliberately, without starting stores or provider work.+ // The global cancellation signal proves Stop has acquired the same lock+ // as admission; the per-tenant done signal still prevents cleanup.+ done := make(chan struct{})+ mtm.monitors["tenant-a"] = &Monitor{}+ mtm.monitors["tenant-b"] = &Monitor{}+ mtm.tenantDone["tenant-a"] = done+ mtm.tenantDone["tenant-b"] = done+ stopped := make(chan struct{})+ go func() {+ mtm.Stop()+ close(stopped)+ }()+ t.Cleanup(func() { close(done); <-stopped })+ select {+ case <-mtm.globalCtx.Done():+ case <-time.After(5 * time.Second):+ t.Fatal("Stop did not close admission")+ }+ var readers sync.WaitGroup+ for range 32 {+ readers.Go(func() {+ for _, orgID := range []string{"tenant-a", "tenant-b", "default", "new-tenant"} {+ if monitor, err := mtm.GetMonitor(orgID); monitor != nil || err == nil {+ t.Errorf("lookup admitted %q after shutdown began", orgID)+ }+ if monitor, ok := mtm.PeekMonitor(orgID); monitor != nil || ok {+ t.Errorf("peek exposed %q after shutdown began", orgID)+ }+ }+ })+ }+ readers.Wait()+ mtm.mu.RLock()+ retained := len(mtm.monitors)+ mtm.mu.RUnlock()+ if retained != 2 {+ t.Errorf("shutdown lost runtime ownership before loop completion: retained=%d", retained)+ }+}++func TestMultiTenantMonitorStopRetainsFenceAfterIncompleteCleanup(t *testing.T) {+ previous := tenantMonitorShutdownTimeout+ tenantMonitorShutdownTimeout = time.Millisecond+ t.Cleanup(func() { tenantMonitorShutdownTimeout = previous })+ mtm := NewMultiTenantMonitor(&config.Config{}, nil, nil)+ done := make(chan struct{})+ mtm.monitors["tenant-a"] = &Monitor{}+ mtm.tenantDone["tenant-a"] = done+ mtm.Stop()+ if len(mtm.monitors) != 1 {+ t.Fatal("incomplete shutdown lost its runtime owner")+ }+ if monitor, ok := mtm.PeekMonitor("tenant-a"); monitor != nil || ok {+ t.Error("incomplete shutdown exposed its retained runtime")+ }+ mtm.FinishTenantDeletion("tenant-a")+ if monitor, err := mtm.GetMonitor("tenant-a"); monitor != nil || err == nil {+ t.Error("deletion cleanup reopened the retired manager")+ }+ close(done)+ mtm.Stop() // The admission fence must not prevent finishing owned cleanup.+ if len(mtm.monitors) != 0 {+ t.Error("completed loop was not cleaned up on repeated Stop")+ }+ if monitor, err := mtm.GetMonitor("default"); monitor != nil || err == nil {+ t.Error("completed cleanup reopened the retired manager")+ }+}diff --git a/internal/monitoring/reload_test.go b/internal/monitoring/reload_test.goindex 41b9fb811c..90a844c662 100644--- a/internal/monitoring/reload_test.go+++ b/internal/monitoring/reload_test.go@@ -15,6 +15,30 @@ import ("github.com/stretchr/testify/require")+func TestReloadableMonitorRetiredManagerRejectsLookups(t *testing.T) {+ t.Setenv("PULSE_DATA_DIR", t.TempDir())+ cfg, err := config.LoadWithoutLoggingInit()+ require.NoError(t, err)+ rm, err := NewReloadableMonitor(cfg, config.NewMultiTenantPersistence(cfg.DataPath), nil)+ require.NoError(t, err)+ ctx, cancel := context.WithCancel(context.Background())+ t.Cleanup(cancel)+ rm.Start(ctx)+ t.Cleanup(rm.Stop)+ retired := rm.GetMultiTenantMonitor()+ require.NoError(t, rm.Reload())+ replacement := rm.GetMultiTenantMonitor()+ require.NotSame(t, retired, replacement)+ monitor, err := retired.GetMonitor("default")+ require.Error(t, err, "stale handler must not recreate the retired default runtime")+ require.Nil(t, monitor)+ monitor, exists := retired.PeekMonitor("default")+ require.False(t, exists)+ require.Nil(t, monitor)+ require.NoError(t, replacement.globalCtx.Err(), "replacement must have its own live lifetime")+ require.Same(t, cfg, rm.GetConfig(), "retirement must not replace the canonical credential owner")+}+func TestReloadableMonitor_Lifecycle_Coverage(t *testing.T) {// Create minimal configcfg := &config.Config{