Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Join runtime alert checks before tenant shutdown completes

pulse-triage[bot]a61efb9cb3b0
diff --git a/internal/api/org_handlers.go b/internal/api/org_handlers.go
index fbae571366..ca80eca417 100644
--- a/internal/api/org_handlers.go
+++ b/internal/api/org_handlers.go
@@ -400,6 +400,7 @@ func (h *OrgHandlers) HandleDeleteOrg(w http.ResponseWriter, r *http.Request) {
writeErrorResponse(w, http.StatusNotFound, "not_found", "Organization not found", nil)
return
}
+ log.Error().Err(err).Str("org_id", orgID).Msg("Failed to delete organization persistence")
writeErrorResponse(w, http.StatusInternalServerError, "delete_failed", "Failed to delete organization", nil)
return
}
diff --git a/internal/monitoring/alert_checks_shutdown_test.go b/internal/monitoring/alert_checks_shutdown_test.go
new file mode 100644
index 0000000000..f9d5386986
--- /dev/null
+++ b/internal/monitoring/alert_checks_shutdown_test.go
@@ -0,0 +1,229 @@
+package monitoring
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/mock"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+func configureAlertShutdownFixture(t *testing.T) {
+ t.Helper()
+ t.Setenv("PULSE_DEV", "true")
+ t.Setenv("HOME", t.TempDir())
+ t.Setenv(mockKeepRealPollingEnv, "false")
+ t.Setenv("PULSE_MOCK_TRENDS_SEED_DURATION", "5m")
+ t.Setenv("PULSE_MOCK_TRENDS_SAMPLE_INTERVAL", "1m")
+ t.Setenv("PULSE_MOCK_SEED_METRICS_STORE", "false")
+ previousEnabled, previousConfig := mock.IsMockEnabled(), mock.GetConfig()
+ t.Cleanup(func() {
+ if err := mock.SetEnabled(false); err != nil {
+ t.Error(err)
+ }
+ mock.SetMockConfig(previousConfig)
+ if err := mock.SetEnabled(previousEnabled); err != nil {
+ t.Error(err)
+ }
+ })
+ if err := mock.SetEnabled(false); err != nil {
+ t.Fatal(err)
+ }
+ fixture := mock.DefaultConfig
+ fixture.NodeCount = 1
+ fixture.VMsPerNode, fixture.LXCsPerNode = 0, 0
+ fixture.DockerHostCount, fixture.GenericHostCount, fixture.K8sClusterCount = 0, 0, 0
+ fixture.RandomMetrics = false
+ mock.SetMockConfig(fixture)
+ if err := mock.SetEnabled(true); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// A synchronous lifecycle consumer holds an actual mock alert pass in flight.
+// No sleeps choose the overlap; the channel proves the pass has reached the
+// writer boundary before cancellation or deletion starts.
+func blockFirstShutdownAlert(t *testing.T, monitor *Monitor) (<-chan struct{}, func()) {
+ t.Helper()
+ manager := monitor.GetAlertManager()
+ cfg := manager.GetConfig()
+ cfg.Enabled = true
+ cfg.ActivationState = alerts.ActivationPending
+ cfg.TimeThresholds = map[string]int{}
+ cfg.MetricTimeThresholds = nil
+ cfg.NodeDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5}
+ manager.UpdateConfig(cfg)
+ entered, release := make(chan struct{}), make(chan struct{})
+ var enterOnce, releaseOnce sync.Once
+ unsubscribe := manager.SubscribeLifecycleCallback(func(alerts.LifecycleEvent) {
+ enterOnce.Do(func() {
+ close(entered)
+ <-release
+ })
+ })
+ unblock := func() { releaseOnce.Do(func() { close(release) }) }
+ t.Cleanup(unsubscribe)
+ t.Cleanup(unblock)
+ return entered, unblock
+}
+
+func TestMonitorStartJoinsMockAlertChecksBeforeReturning(t *testing.T) {
+ configureAlertShutdownFixture(t)
+ monitor, err := New(&config.Config{DataPath: t.TempDir()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ entered, unblock := blockFirstShutdownAlert(t, monitor)
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ monitor.Start(ctx, nil)
+ }()
+ t.Cleanup(func() {
+ cancel()
+ unblock()
+ select {
+ case <-done:
+ monitor.Stop()
+ case <-time.After(5 * time.Second):
+ t.Error("monitor did not finish after releasing the alert pass")
+ }
+ })
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("mock pass did not reach the lifecycle writer")
+ }
+ cancel()
+ select {
+ case <-done:
+ t.Fatal("monitor reported completion while its alert pass could still write")
+ case <-time.After(100 * time.Millisecond):
+ }
+ unblock()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("monitor failed to join its completed alert pass")
+ }
+ if len(monitor.GetAlertManager().GetActiveAlerts()) == 0 {
+ t.Fatal("shutdown silently dropped the admitted alert pass")
+ }
+}
+
+func TestMonitorStartJoinsConnectionAlertChecksBeforeReturning(t *testing.T) {
+ configureAlertShutdownFixture(t)
+ monitor, err := New(&config.Config{DataPath: t.TempDir(), PVEPollingInterval: 10 * time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ entered, release := make(chan struct{}), make(chan struct{})
+ var enterOnce, releaseOnce sync.Once
+ unblock := func() { releaseOnce.Do(func() { close(release) }) }
+ monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {
+ enterOnce.Do(func() { close(entered) })
+ <-release
+ return nil
+ })
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ monitor.Start(ctx, nil)
+ }()
+ t.Cleanup(func() {
+ cancel()
+ unblock()
+ select {
+ case <-done:
+ monitor.Stop()
+ case <-time.After(5 * time.Second):
+ t.Error("monitor did not finish after releasing the connection lister")
+ }
+ })
+ select {
+ case <-entered:
+ case <-time.After(15 * time.Second):
+ t.Fatal("connection check did not reach its registered lister")
+ }
+ cancel()
+ select {
+ case <-done:
+ t.Fatal("monitor reported completion while a connection alert check remained in flight")
+ case <-time.After(100 * time.Millisecond):
+ }
+ unblock()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("monitor failed to join its completed connection check")
+ }
+}
+
+func TestTenantDeletionRetainsBlockedAlertPassThenRemovesCompletedRuntime(t *testing.T) {
+ configureAlertShutdownFixture(t)
+ previousTimeout := tenantDeletionShutdownTimeout
+ tenantDeletionShutdownTimeout = 100 * time.Millisecond
+ t.Cleanup(func() { tenantDeletionShutdownTimeout = previousTimeout })
+ root := t.TempDir()
+ persistence := config.NewMultiTenantPersistence(root)
+ if err := persistence.SaveOrganization(&models.Organization{ID: "alert-owner", DisplayName: "Alert owner"}); err != nil {
+ t.Fatal(err)
+ }
+ manager := NewMultiTenantMonitor(&config.Config{DataPath: root}, persistence, nil)
+ var entered <-chan struct{}
+ var unblock func()
+ manager.SetMonitorInitializer(func(monitor *Monitor) {
+ entered, unblock = blockFirstShutdownAlert(t, monitor)
+ })
+ monitor, err := manager.GetMonitor("alert-owner")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { unblock(); manager.Stop() })
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("tenant mock pass did not reach the lifecycle writer")
+ }
+ // The deletion budget still refuses a live writer. A timeout must retain
+ // its original owner and organization bytes, not close stores underneath it.
+ if err := manager.BeginTenantDeletion("alert-owner"); err == nil {
+ t.Fatal("deletion accepted an unfinished alert pass")
+ }
+ if got, ok := manager.PeekMonitor("alert-owner"); !ok || got != monitor {
+ t.Fatal("blocked alert writer lost its runtime owner")
+ }
+ manager.FinishTenantDeletion("alert-owner")
+ if _, err := manager.GetMonitor("alert-owner"); err == nil {
+ t.Fatal("deferred cleanup released the unfinished writer's guard")
+ }
+ orgPath := filepath.Join(root, "orgs", "alert-owner")
+ if _, err := os.Stat(filepath.Join(orgPath, "org.json")); err != nil {
+ t.Fatalf("blocked runtime lost its organization data: %v", err)
+ }
+ unblock()
+ if err := manager.BeginTenantDeletion("alert-owner"); err != nil {
+ t.Fatalf("completed alert pass prevented safe removal: %v", err)
+ }
+ if err := persistence.DeleteOrganization("alert-owner"); err != nil {
+ t.Fatal(err)
+ }
+ manager.FinishTenantDeletion("alert-owner")
+ if _, ok := manager.PeekMonitor("alert-owner"); ok {
+ t.Fatal("completed tenant runtime was retained after deletion")
+ }
+ if _, err := manager.GetMonitor("alert-owner"); err == nil {
+ t.Fatal("deleted organization can recreate a runtime")
+ }
+ if _, err := os.Stat(orgPath); !os.IsNotExist(err) {
+ t.Fatalf("safe deletion left an organization directory: %v", err)
+ }
+}
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 33505f2c4d..6e76f913f9 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -2265,6 +2265,23 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
})
defer leaveMockModeSwitches()
+ // Tenant deletion waits for Start to return before closing stores and
+ // removing the data directory. Alert passes must belong to that same
+ // lifetime: detached mock checks can otherwise keep writing alert files
+ // after the monitoring loop has reported completion.
+ var alertChecks sync.WaitGroup
+ defer alertChecks.Wait()
+ startAlertCheck := func(check func()) {
+ if ctx.Err() != nil {
+ return
+ }
+ alertChecks.Go(func() {
+ if ctx.Err() == nil {
+ check()
+ }
+ })
+ }
+
// Set up alert callbacks. Projection replay is deliberately absent here:
// the canonical resource store is not attached yet, so a replay now could
// not advance the durable watermark and would walk the same events again
@@ -2320,11 +2337,11 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
if mock.IsMockEnabled() {
if keepRealPolling {
log.Info().Msg("mock mode enabled - running mock alerts and real metric polling")
- go m.checkMockAlerts()
+ startAlertCheck(m.checkMockAlerts)
go m.poll(ctx, wsHub)
} else {
log.Info().Msg("mock mode enabled - skipping real node polling")
- go m.checkMockAlerts()
+ startAlertCheck(m.checkMockAlerts)
}
} else {
go m.poll(ctx, wsHub)
@@ -2354,7 +2371,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
m.evaluateNotificationDelivery(now)
if mock.IsMockEnabled() {
// In mock mode, keep synthetic alerts fresh
- go m.checkMockAlerts()
+ startAlertCheck(m.checkMockAlerts)
if keepRealPolling {
// Keep real metrics flowing while mock UI mode is active.
go m.poll(ctx, wsHub)
@@ -2367,7 +2384,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
// ledger, which lives behind the api layer. We invoke the
// registered lister here so a wedged PVE/PBS/PMG/VMware/TrueNAS
// connection escalates into the top-nav alert stream.
- go m.checkConnectionAlerts()
+ startAlertCheck(m.checkConnectionAlerts)
case <-broadcastTicker.C:
// Broadcast current state regardless of polling status