Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Join runtime alert checks before tenant shutdown completes
pulse-triage[bot]a61efb9cb3b0
diff --git a/internal/api/org_handlers.go b/internal/api/org_handlers.goindex fbae571366..ca80eca417 100644--- a/internal/api/org_handlers.go+++ b/internal/api/org_handlers.go@@ -400,6 +400,7 @@ func (h *OrgHandlers) HandleDeleteOrg(w http.ResponseWriter, r *http.Request) {writeErrorResponse(w, http.StatusNotFound, "not_found", "Organization not found", nil)return}+ log.Error().Err(err).Str("org_id", orgID).Msg("Failed to delete organization persistence")writeErrorResponse(w, http.StatusInternalServerError, "delete_failed", "Failed to delete organization", nil)return}diff --git a/internal/monitoring/alert_checks_shutdown_test.go b/internal/monitoring/alert_checks_shutdown_test.gonew file mode 100644index 0000000000..f9d5386986--- /dev/null+++ b/internal/monitoring/alert_checks_shutdown_test.go@@ -0,0 +1,229 @@+package monitoring++import (+ "context"+ "os"+ "path/filepath"+ "sync"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/mock"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+)++func configureAlertShutdownFixture(t *testing.T) {+ t.Helper()+ t.Setenv("PULSE_DEV", "true")+ t.Setenv("HOME", t.TempDir())+ t.Setenv(mockKeepRealPollingEnv, "false")+ t.Setenv("PULSE_MOCK_TRENDS_SEED_DURATION", "5m")+ t.Setenv("PULSE_MOCK_TRENDS_SAMPLE_INTERVAL", "1m")+ t.Setenv("PULSE_MOCK_SEED_METRICS_STORE", "false")+ previousEnabled, previousConfig := mock.IsMockEnabled(), mock.GetConfig()+ t.Cleanup(func() {+ if err := mock.SetEnabled(false); err != nil {+ t.Error(err)+ }+ mock.SetMockConfig(previousConfig)+ if err := mock.SetEnabled(previousEnabled); err != nil {+ t.Error(err)+ }+ })+ if err := mock.SetEnabled(false); err != nil {+ t.Fatal(err)+ }+ fixture := mock.DefaultConfig+ fixture.NodeCount = 1+ fixture.VMsPerNode, fixture.LXCsPerNode = 0, 0+ fixture.DockerHostCount, fixture.GenericHostCount, fixture.K8sClusterCount = 0, 0, 0+ fixture.RandomMetrics = false+ mock.SetMockConfig(fixture)+ if err := mock.SetEnabled(true); err != nil {+ t.Fatal(err)+ }+}++// A synchronous lifecycle consumer holds an actual mock alert pass in flight.+// No sleeps choose the overlap; the channel proves the pass has reached the+// writer boundary before cancellation or deletion starts.+func blockFirstShutdownAlert(t *testing.T, monitor *Monitor) (<-chan struct{}, func()) {+ t.Helper()+ manager := monitor.GetAlertManager()+ cfg := manager.GetConfig()+ cfg.Enabled = true+ cfg.ActivationState = alerts.ActivationPending+ cfg.TimeThresholds = map[string]int{}+ cfg.MetricTimeThresholds = nil+ cfg.NodeDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5}+ manager.UpdateConfig(cfg)+ entered, release := make(chan struct{}), make(chan struct{})+ var enterOnce, releaseOnce sync.Once+ unsubscribe := manager.SubscribeLifecycleCallback(func(alerts.LifecycleEvent) {+ enterOnce.Do(func() {+ close(entered)+ <-release+ })+ })+ unblock := func() { releaseOnce.Do(func() { close(release) }) }+ t.Cleanup(unsubscribe)+ t.Cleanup(unblock)+ return entered, unblock+}++func TestMonitorStartJoinsMockAlertChecksBeforeReturning(t *testing.T) {+ configureAlertShutdownFixture(t)+ monitor, err := New(&config.Config{DataPath: t.TempDir()})+ if err != nil {+ t.Fatal(err)+ }+ entered, unblock := blockFirstShutdownAlert(t, monitor)+ ctx, cancel := context.WithCancel(context.Background())+ done := make(chan struct{})+ go func() {+ defer close(done)+ monitor.Start(ctx, nil)+ }()+ t.Cleanup(func() {+ cancel()+ unblock()+ select {+ case <-done:+ monitor.Stop()+ case <-time.After(5 * time.Second):+ t.Error("monitor did not finish after releasing the alert pass")+ }+ })+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ t.Fatal("mock pass did not reach the lifecycle writer")+ }+ cancel()+ select {+ case <-done:+ t.Fatal("monitor reported completion while its alert pass could still write")+ case <-time.After(100 * time.Millisecond):+ }+ unblock()+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("monitor failed to join its completed alert pass")+ }+ if len(monitor.GetAlertManager().GetActiveAlerts()) == 0 {+ t.Fatal("shutdown silently dropped the admitted alert pass")+ }+}++func TestMonitorStartJoinsConnectionAlertChecksBeforeReturning(t *testing.T) {+ configureAlertShutdownFixture(t)+ monitor, err := New(&config.Config{DataPath: t.TempDir(), PVEPollingInterval: 10 * time.Second})+ if err != nil {+ t.Fatal(err)+ }+ entered, release := make(chan struct{}), make(chan struct{})+ var enterOnce, releaseOnce sync.Once+ unblock := func() { releaseOnce.Do(func() { close(release) }) }+ monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {+ enterOnce.Do(func() { close(entered) })+ <-release+ return nil+ })+ ctx, cancel := context.WithCancel(context.Background())+ done := make(chan struct{})+ go func() {+ defer close(done)+ monitor.Start(ctx, nil)+ }()+ t.Cleanup(func() {+ cancel()+ unblock()+ select {+ case <-done:+ monitor.Stop()+ case <-time.After(5 * time.Second):+ t.Error("monitor did not finish after releasing the connection lister")+ }+ })+ select {+ case <-entered:+ case <-time.After(15 * time.Second):+ t.Fatal("connection check did not reach its registered lister")+ }+ cancel()+ select {+ case <-done:+ t.Fatal("monitor reported completion while a connection alert check remained in flight")+ case <-time.After(100 * time.Millisecond):+ }+ unblock()+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("monitor failed to join its completed connection check")+ }+}++func TestTenantDeletionRetainsBlockedAlertPassThenRemovesCompletedRuntime(t *testing.T) {+ configureAlertShutdownFixture(t)+ previousTimeout := tenantDeletionShutdownTimeout+ tenantDeletionShutdownTimeout = 100 * time.Millisecond+ t.Cleanup(func() { tenantDeletionShutdownTimeout = previousTimeout })+ root := t.TempDir()+ persistence := config.NewMultiTenantPersistence(root)+ if err := persistence.SaveOrganization(&models.Organization{ID: "alert-owner", DisplayName: "Alert owner"}); err != nil {+ t.Fatal(err)+ }+ manager := NewMultiTenantMonitor(&config.Config{DataPath: root}, persistence, nil)+ var entered <-chan struct{}+ var unblock func()+ manager.SetMonitorInitializer(func(monitor *Monitor) {+ entered, unblock = blockFirstShutdownAlert(t, monitor)+ })+ monitor, err := manager.GetMonitor("alert-owner")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { unblock(); manager.Stop() })+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ t.Fatal("tenant mock pass did not reach the lifecycle writer")+ }+ // The deletion budget still refuses a live writer. A timeout must retain+ // its original owner and organization bytes, not close stores underneath it.+ if err := manager.BeginTenantDeletion("alert-owner"); err == nil {+ t.Fatal("deletion accepted an unfinished alert pass")+ }+ if got, ok := manager.PeekMonitor("alert-owner"); !ok || got != monitor {+ t.Fatal("blocked alert writer lost its runtime owner")+ }+ manager.FinishTenantDeletion("alert-owner")+ if _, err := manager.GetMonitor("alert-owner"); err == nil {+ t.Fatal("deferred cleanup released the unfinished writer's guard")+ }+ orgPath := filepath.Join(root, "orgs", "alert-owner")+ if _, err := os.Stat(filepath.Join(orgPath, "org.json")); err != nil {+ t.Fatalf("blocked runtime lost its organization data: %v", err)+ }+ unblock()+ if err := manager.BeginTenantDeletion("alert-owner"); err != nil {+ t.Fatalf("completed alert pass prevented safe removal: %v", err)+ }+ if err := persistence.DeleteOrganization("alert-owner"); err != nil {+ t.Fatal(err)+ }+ manager.FinishTenantDeletion("alert-owner")+ if _, ok := manager.PeekMonitor("alert-owner"); ok {+ t.Fatal("completed tenant runtime was retained after deletion")+ }+ if _, err := manager.GetMonitor("alert-owner"); err == nil {+ t.Fatal("deleted organization can recreate a runtime")+ }+ if _, err := os.Stat(orgPath); !os.IsNotExist(err) {+ t.Fatalf("safe deletion left an organization directory: %v", err)+ }+}diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 33505f2c4d..6e76f913f9 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -2265,6 +2265,23 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {})defer leaveMockModeSwitches()+ // Tenant deletion waits for Start to return before closing stores and+ // removing the data directory. Alert passes must belong to that same+ // lifetime: detached mock checks can otherwise keep writing alert files+ // after the monitoring loop has reported completion.+ var alertChecks sync.WaitGroup+ defer alertChecks.Wait()+ startAlertCheck := func(check func()) {+ if ctx.Err() != nil {+ return+ }+ alertChecks.Go(func() {+ if ctx.Err() == nil {+ check()+ }+ })+ }+// Set up alert callbacks. Projection replay is deliberately absent here:// the canonical resource store is not attached yet, so a replay now could// not advance the durable watermark and would walk the same events again@@ -2320,11 +2337,11 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {if mock.IsMockEnabled() {if keepRealPolling {log.Info().Msg("mock mode enabled - running mock alerts and real metric polling")- go m.checkMockAlerts()+ startAlertCheck(m.checkMockAlerts)go m.poll(ctx, wsHub)} else {log.Info().Msg("mock mode enabled - skipping real node polling")- go m.checkMockAlerts()+ startAlertCheck(m.checkMockAlerts)}} else {go m.poll(ctx, wsHub)@@ -2354,7 +2371,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {m.evaluateNotificationDelivery(now)if mock.IsMockEnabled() {// In mock mode, keep synthetic alerts fresh- go m.checkMockAlerts()+ startAlertCheck(m.checkMockAlerts)if keepRealPolling {// Keep real metrics flowing while mock UI mode is active.go m.poll(ctx, wsHub)@@ -2367,7 +2384,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {// ledger, which lives behind the api layer. We invoke the// registered lister here so a wedged PVE/PBS/PMG/VMware/TrueNAS// connection escalates into the top-nav alert stream.- go m.checkConnectionAlerts()+ startAlertCheck(m.checkConnectionAlerts)case <-broadcastTicker.C:// Broadcast current state regardless of polling status