Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Close retired tenant manager admission before shutdown

pulse-triage[bot]b010e9a0f7c7
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 94ae194f2b..d4db747e21 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,28 @@
## Purpose
+### Retired tenant managers cannot create replacement runtimes
+
+`MultiTenantMonitor.Stop` closes admission under the same lock as lazy
+initialization, before cancelling or draining any tenant. Both cached and new
+`GetMonitor` lookups reject a retired manager; `PeekMonitor` does not expose its
+retained runtimes. The initialization path rechecks the fence after acquiring
+its write lock. An incomplete shutdown keeps its existing runtime/data owner,
+not a usable read owner. Repeated Stop can finish cleanup but never reopens
+admission. A separately constructed manager owns replacement runtimes.
+
+`multi_tenant_monitor_stop_test.go` covers unused/default/scoped lookups,
+canonical-config and saved-file preservation, concurrent cached reads during
+drain, incomplete cleanup and explicit cleanup completion.
+`TestReloadableMonitorRetiredManagerRejectsLookups` exercises the real reload
+handover while preserving its canonical credential owner.
+`TestTenantMonitorRetirementFailsClosedAndReplacementRecovers` connects the
+retirement fence to the existing tenant HTTP 503 guard and configuration
+handler: stale consumers cannot recreate or borrow default state, while a
+new manager recovers the saved tenant inventory. This does not prove detached
+worker quiescence, the whole removal/startup journey or installed recovery.
+Authentication, tenant selection, TLS and the HTTP error shape are unchanged.
+
### Datastore discovery cannot turn unread history into absence
PBS datastore enumeration uses the same bounded, fully read and closed HTTP 200
diff --git a/internal/api/tenant_monitor_retirement_test.go b/internal/api/tenant_monitor_retirement_test.go
new file mode 100644
index 0000000000..717d053514
--- /dev/null
+++ b/internal/api/tenant_monitor_retirement_test.go
@@ -0,0 +1,84 @@
+package api
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+)
+
+// A stale router/handler reference can outlive a manager swap. It must report
+// unavailability, not recreate the retired org's runtime or fall back to the
+// default org. A separately owned replacement manager can still serve it.
+func TestTenantMonitorRetirementFailsClosedAndReplacementRecovers(t *testing.T) {
+ dir := t.TempDir()
+ mtp := config.NewMultiTenantPersistence(dir)
+ if err := mtp.SaveOrganization(&models.Organization{ID: "tenant-a", DisplayName: "Tenant A"}); err != nil {
+ t.Fatal(err)
+ }
+ persistence, err := mtp.GetPersistence("tenant-a")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := persistence.SaveNodesConfig([]config.PVEInstance{{Name: "Tenant saved PVE", Host: "https://tenant.invalid:8006", Disabled: true}}, nil, nil); err != nil {
+ t.Fatal(err)
+ }
+ cfg := &config.Config{DataPath: dir, ConfigPath: dir}
+ retired := monitoring.NewMultiTenantMonitor(cfg, mtp, nil)
+ t.Cleanup(retired.Stop)
+ var retiredInitializations atomic.Int32
+ retired.SetMonitorInitializer(func(*monitoring.Monitor) { retiredInitializations.Add(1) })
+ retired.Stop()
+
+ router := &Router{mtMonitor: retired, monitor: &monitoring.Monitor{}}
+ handlers := NewConfigHandlers(mtp, retired, nil, nil, nil, nil, false)
+ guarded := router.tenantMonitorGuardMiddleware(http.HandlerFunc(handlers.HandleGetNodes))
+ request := func() *httptest.ResponseRecorder {
+ req := httptest.NewRequest(http.MethodGet, "/api/config/nodes", nil)
+ req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "tenant-a"))
+ rec := httptest.NewRecorder()
+ guarded.ServeHTTP(rec, req)
+ return rec
+ }
+ rec := request()
+ if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), `"tenant_unavailable"`) {
+ t.Fatalf("retired manager returned %d: %s", rec.Code, rec.Body.String())
+ }
+ if retiredInitializations.Load() != 0 {
+ t.Fatal("HTTP guard reinitialized the retired tenant")
+ }
+ ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-a")
+ if cfg, persistence, monitor := handlers.ContextState(ctx); cfg != nil || persistence != nil || monitor != nil {
+ t.Fatal("stale config handler exposed tenant or default state")
+ }
+ if router.getTenantMonitor(ctx) != nil {
+ t.Fatal("stale router fell back to the default runtime")
+ }
+
+ replacement := monitoring.NewMultiTenantMonitor(cfg, mtp, nil)
+ t.Cleanup(replacement.Stop)
+ // Rebind only these consumers, without altering production swap order.
+ router.mtMonitor = replacement
+ handlers.SetMultiTenantMonitor(replacement)
+ rec = request()
+ if rec.Code != http.StatusOK {
+ t.Fatalf("replacement manager returned %d: %s", rec.Code, rec.Body.String())
+ }
+ var nodes []NodeResponse
+ if err := json.Unmarshal(rec.Body.Bytes(), &nodes); err != nil {
+ t.Fatal(err)
+ }
+ if len(nodes) != 1 || nodes[0].Name != "Tenant saved PVE" || nodes[0].Host != "https://tenant.invalid:8006" {
+ t.Fatal("replacement did not recover the saved tenant inventory")
+ }
+ if retiredInitializations.Load() != 0 {
+ t.Fatal("replacement read consulted the retired owner")
+ }
+}
diff --git a/internal/monitoring/multi_tenant_monitor.go b/internal/monitoring/multi_tenant_monitor.go
index ecd5c9a0f7..65a7b9c3ed 100644
--- a/internal/monitoring/multi_tenant_monitor.go
+++ b/internal/monitoring/multi_tenant_monitor.go
@@ -28,6 +28,7 @@ type MultiTenantMonitor struct {
initializer func(*Monitor)
globalCtx context.Context
globalCancel context.CancelFunc
+ stopped bool // Terminal admission fence, protected by mu.
}
// NewMultiTenantMonitor creates a new multi-tenant monitor manager.
@@ -152,6 +153,10 @@ func (mtm *MultiTenantMonitor) GetMonitor(orgID string) (*Monitor, error) {
}
mtm.mu.RLock()
+ if mtm.stopped {
+ mtm.mu.RUnlock()
+ return nil, fmt.Errorf("tenant monitor manager is stopped")
+ }
monitor, exists := mtm.monitors[orgID]
_, deleting := mtm.tenantDeleting[orgID]
mtm.mu.RUnlock()
@@ -166,6 +171,12 @@ func (mtm *MultiTenantMonitor) GetMonitor(orgID string) (*Monitor, error) {
mtm.mu.Lock()
defer mtm.mu.Unlock()
+ // Stop can win between the cache lookup and this initialization lock.
+ // A retired manager must never reopen stores or mutate the shared config.
+ if mtm.stopped {
+ return nil, fmt.Errorf("tenant monitor manager is stopped")
+ }
+
if mtm.monitors == nil {
mtm.monitors = make(map[string]*Monitor)
}
@@ -300,15 +311,20 @@ func (mtm *MultiTenantMonitor) PeekMonitor(orgID string) (*Monitor, bool) {
mtm.mu.RLock()
defer mtm.mu.RUnlock()
+ if mtm.stopped {
+ return nil, false
+ }
monitor, exists := mtm.monitors[orgID]
return monitor, exists
}
-// Stop stops all tenant monitors.
+// Stop permanently closes admission and stops all tenant monitors. A new
+// manager, rather than a lookup on this retired one, owns replacement runtimes.
func (mtm *MultiTenantMonitor) Stop() {
log.Info().Msg("stopping MultiTenantMonitor and all tenant instances")
mtm.mu.Lock()
+ mtm.stopped = true
if mtm.globalCancel != nil {
mtm.globalCancel()
}
diff --git a/internal/monitoring/multi_tenant_monitor_stop_test.go b/internal/monitoring/multi_tenant_monitor_stop_test.go
new file mode 100644
index 0000000000..007524e04c
--- /dev/null
+++ b/internal/monitoring/multi_tenant_monitor_stop_test.go
@@ -0,0 +1,153 @@
+package monitoring
+
+import (
+ "crypto/sha256"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "reflect"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+// Stop retires the owner, not just the currently materialized tenant loops.
+// In particular, a handler holding the old manager through a reload must not
+// initialize a previously unused tenant or reload the default config into it.
+func TestMultiTenantMonitorStopRejectsLazyRecreation(t *testing.T) {
+ mtp, persistence := newTestTenantPersistence(t)
+ if err := mtp.SaveOrganization(&models.Organization{ID: "tenant-a", DisplayName: "Tenant A"}); err != nil {
+ t.Fatal(err)
+ }
+ cfg := &config.Config{
+ DataPath: persistence.GetConfigDir(),
+ PVEInstances: []config.PVEInstance{{Name: "saved-primary", Host: "https://primary.invalid:8006", Disabled: true}},
+ APITokens: []config.APITokenRecord{{ID: "primary-token", Hash: "synthetic-primary-hash"}},
+ }
+ before := cfg.DeepCopy()
+ snapshot := func() map[string][32]byte {
+ result := make(map[string][32]byte)
+ err := filepath.WalkDir(mtp.BaseDataDir(), func(path string, entry fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if entry.IsDir() {
+ result[path] = [32]byte{}
+ return nil
+ }
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ result[path] = sha256.Sum256(data)
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return result
+ }
+ filesBefore := snapshot()
+ mtm := NewMultiTenantMonitor(cfg, mtp, nil)
+ t.Cleanup(mtm.Stop)
+ var initialized atomic.Int32
+ mtm.SetMonitorInitializer(func(*Monitor) { initialized.Add(1) })
+ mtm.Stop()
+ mtm.Stop() // Repeated shutdown must not reopen admission.
+
+ for _, orgID := range []string{"default", "tenant-a", "not-provisioned", " tenant-a "} {
+ if monitor, err := mtm.GetMonitor(orgID); monitor != nil || err == nil {
+ t.Errorf("retired manager admitted %q: monitor present=%v, err=%v", orgID, monitor != nil, err)
+ }
+ if monitor, ok := mtm.PeekMonitor(orgID); monitor != nil || ok {
+ t.Errorf("retired manager exposed cached monitor for %q", orgID)
+ }
+ }
+ if initialized.Load() != 0 {
+ t.Errorf("retired manager initialized %d runtimes", initialized.Load())
+ }
+ if !reflect.DeepEqual(cfg.DeepCopy(), before) {
+ t.Error("retired lookup changed the canonical config")
+ }
+ // A rejected lookup must not create stores/writers or change saved files.
+ if !reflect.DeepEqual(snapshot(), filesBefore) {
+ t.Error("retired lookup created or changed persisted runtime material")
+ }
+
+}
+
+func TestMultiTenantMonitorStopFencesRetainedRuntimes(t *testing.T) {
+ mtm := NewMultiTenantMonitor(&config.Config{}, nil, nil)
+ // Retain a loop deliberately, without starting stores or provider work.
+ // The global cancellation signal proves Stop has acquired the same lock
+ // as admission; the per-tenant done signal still prevents cleanup.
+ done := make(chan struct{})
+ mtm.monitors["tenant-a"] = &Monitor{}
+ mtm.monitors["tenant-b"] = &Monitor{}
+ mtm.tenantDone["tenant-a"] = done
+ mtm.tenantDone["tenant-b"] = done
+ stopped := make(chan struct{})
+ go func() {
+ mtm.Stop()
+ close(stopped)
+ }()
+ t.Cleanup(func() { close(done); <-stopped })
+ select {
+ case <-mtm.globalCtx.Done():
+ case <-time.After(5 * time.Second):
+ t.Fatal("Stop did not close admission")
+ }
+ var readers sync.WaitGroup
+ for range 32 {
+ readers.Go(func() {
+ for _, orgID := range []string{"tenant-a", "tenant-b", "default", "new-tenant"} {
+ if monitor, err := mtm.GetMonitor(orgID); monitor != nil || err == nil {
+ t.Errorf("lookup admitted %q after shutdown began", orgID)
+ }
+ if monitor, ok := mtm.PeekMonitor(orgID); monitor != nil || ok {
+ t.Errorf("peek exposed %q after shutdown began", orgID)
+ }
+ }
+ })
+ }
+ readers.Wait()
+ mtm.mu.RLock()
+ retained := len(mtm.monitors)
+ mtm.mu.RUnlock()
+ if retained != 2 {
+ t.Errorf("shutdown lost runtime ownership before loop completion: retained=%d", retained)
+ }
+}
+
+func TestMultiTenantMonitorStopRetainsFenceAfterIncompleteCleanup(t *testing.T) {
+ previous := tenantMonitorShutdownTimeout
+ tenantMonitorShutdownTimeout = time.Millisecond
+ t.Cleanup(func() { tenantMonitorShutdownTimeout = previous })
+ mtm := NewMultiTenantMonitor(&config.Config{}, nil, nil)
+ done := make(chan struct{})
+ mtm.monitors["tenant-a"] = &Monitor{}
+ mtm.tenantDone["tenant-a"] = done
+ mtm.Stop()
+ if len(mtm.monitors) != 1 {
+ t.Fatal("incomplete shutdown lost its runtime owner")
+ }
+ if monitor, ok := mtm.PeekMonitor("tenant-a"); monitor != nil || ok {
+ t.Error("incomplete shutdown exposed its retained runtime")
+ }
+ mtm.FinishTenantDeletion("tenant-a")
+ if monitor, err := mtm.GetMonitor("tenant-a"); monitor != nil || err == nil {
+ t.Error("deletion cleanup reopened the retired manager")
+ }
+ close(done)
+ mtm.Stop() // The admission fence must not prevent finishing owned cleanup.
+ if len(mtm.monitors) != 0 {
+ t.Error("completed loop was not cleaned up on repeated Stop")
+ }
+ if monitor, err := mtm.GetMonitor("default"); monitor != nil || err == nil {
+ t.Error("completed cleanup reopened the retired manager")
+ }
+}
diff --git a/internal/monitoring/reload_test.go b/internal/monitoring/reload_test.go
index 41b9fb811c..90a844c662 100644
--- a/internal/monitoring/reload_test.go
+++ b/internal/monitoring/reload_test.go
@@ -15,6 +15,30 @@ import (
"github.com/stretchr/testify/require"
)
+func TestReloadableMonitorRetiredManagerRejectsLookups(t *testing.T) {
+ t.Setenv("PULSE_DATA_DIR", t.TempDir())
+ cfg, err := config.LoadWithoutLoggingInit()
+ require.NoError(t, err)
+ rm, err := NewReloadableMonitor(cfg, config.NewMultiTenantPersistence(cfg.DataPath), nil)
+ require.NoError(t, err)
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+ rm.Start(ctx)
+ t.Cleanup(rm.Stop)
+ retired := rm.GetMultiTenantMonitor()
+ require.NoError(t, rm.Reload())
+ replacement := rm.GetMultiTenantMonitor()
+ require.NotSame(t, retired, replacement)
+ monitor, err := retired.GetMonitor("default")
+ require.Error(t, err, "stale handler must not recreate the retired default runtime")
+ require.Nil(t, monitor)
+ monitor, exists := retired.PeekMonitor("default")
+ require.False(t, exists)
+ require.Nil(t, monitor)
+ require.NoError(t, replacement.globalCtx.Err(), "replacement must have its own live lifetime")
+ require.Same(t, cfg, rm.GetConfig(), "retirement must not replace the canonical credential owner")
+}
+
func TestReloadableMonitor_Lifecycle_Coverage(t *testing.T) {
// Create minimal config
cfg := &config.Config{