Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep connection alerts running after a monitor reload
rcourtman9c58cffd4dce
diff --git a/internal/api/connections_alerts.go b/internal/api/connections_alerts.goindex acd8c948d3..16c0a4553d 100644--- a/internal/api/connections_alerts.go+++ b/internal/api/connections_alerts.go@@ -158,6 +158,39 @@ func snapshotConnectionsForAlerts(connections []Connection) []alerts.ConnectionSreturn out}+// installConnectionAlertLister gives monitor the closure its poll tick uses to+// raise and resolve connection-degraded alerts, bound to that monitor. Every+// path that attaches a monitor to the router calls it through+// configureMonitorDependencies: startup, SetMonitor after a full reload+// (node auto-registration replaces the default monitor), and the+// multi-tenant initializer that lazily creates the default monitor. A lister+// installed once at startup stays on the discarded monitor, so the+// replacement would never evaluate connection alerts again until restart.+//+// Only the default org's monitor gets a lister: the closure reads the+// default org's connection config, and per-org connection alerts are not+// wired yet. Until the platform pollers exist the monitor keeps whatever+// lister it had, so a running monitor never sees nil runtime owners.+func (r *Router) installConnectionAlertLister(monitor *monitoring.Monitor) {+ if r == nil || monitor == nil || r.configHandlers == nil || r.trueNASPoller == nil || r.vmwarePoller == nil {+ return+ }+ if orgID := monitor.GetOrgID(); orgID != "" && orgID != "default" {+ return+ }+ getCfg := r.configHandlers.Config+ getPersist := r.configHandlers.Persistence+ runtime := aggregatorRuntimeSources{+ orgID: "default",+ truenasPoller: r.trueNASPoller,+ vmwarePoller: r.vmwarePoller,+ }+ monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {+ ctx := context.Background()+ return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, runtime)+ })+}+func buildAlertConnectionSnapshotsWithRuntimeSources(ctx context.Context,cfg *config.Config,diff --git a/internal/api/connections_alerts_monitor_swap_test.go b/internal/api/connections_alerts_monitor_swap_test.gonew file mode 100644index 0000000000..f1ea2d0342--- /dev/null+++ b/internal/api/connections_alerts_monitor_swap_test.go@@ -0,0 +1,83 @@+package api++import (+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"+)++// A full monitor reload (node auto-registration, settings that need one)+// replaces the default monitor and reattaches it with SetMonitor and+// SetMultiTenantMonitor, in that order, exactly as pkg/server does. The+// replacement must keep raising connection-degraded alerts from its own poll+// loop: a lister installed only on the startup monitor left the new one with+// none, so wedged connections stayed silent until restart.+func TestConnectionAlertsSurviveMonitorReload(t *testing.T) {+ setMockModeForTest(t, false)+ t.Setenv(truenas.FeatureTrueNAS, "true")+ setTrueNASFeatureForTest(t, true)++ nas := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api/v2.0/system/info":+ _, _ = w.Write([]byte(`{"hostname":"fixture-nas","version":"TrueNAS-SCALE-24.10.2","uptime_seconds":100}`))+ case "/api/v2.0/pool":+ w.WriteHeader(http.StatusUnauthorized)+ _, _ = w.Write([]byte(`{"error":"fixture required inventory denied"}`))+ case "/api/v2.0/pool/dataset", "/api/v2.0/disk", "/api/v2.0/alert/list":+ _, _ = w.Write([]byte(`[]`))+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(nas.Close)++ dataDir := t.TempDir()+ cfg := &config.Config{DataPath: dataDir, ConfigPath: dataDir, PVEPollingInterval: 10 * time.Second}+ persistence := config.NewConfigPersistence(dataDir)+ tn := trueNASInstanceFromRawURL(t, "fixture-nas", nas.URL, true)+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{tn}); err != nil {+ t.Fatal(err)+ }++ startup, err := monitoring.New(cfg)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(startup.Stop)+ router := NewRouter(cfg, startup, nil, nil, nil, "fixture")+ cleanupTestRouter(t, router)++ // The reload sequence from pkg/server refreshRouterAfterMonitorReload.+ reloaded := monitoring.NewMultiTenantMonitor(cfg, config.NewMultiTenantPersistence(dataDir), nil)+ t.Cleanup(reloaded.Stop)+ replacement, err := reloaded.GetMonitor("default")+ if err != nil {+ t.Fatal(err)+ }+ if replacement == startup {+ t.Fatal("reload fixture must produce a new default monitor")+ }+ router.SetMonitor(replacement)+ router.SetMultiTenantMonitor(reloaded)++ nasID := "truenas:" + tn.ID+ manager := replacement.GetAlertManager()+ deadline := time.Now().Add(40 * time.Second)+ for time.Now().Before(deadline) {+ for _, alert := range manager.GetActiveAlerts() {+ if alert.Type == "connection-degraded" && alert.ResourceID == nasID {+ t.Logf("reloaded monitor raised %s for the wedged connection", alert.Level)+ return+ }+ }+ time.Sleep(50 * time.Millisecond)+ }+ t.Fatal("the reloaded default monitor never evaluated connection-degraded alerts")+}diff --git a/internal/api/router.go b/internal/api/router.goindex 98e2b784a6..26648d29fd 100644--- a/internal/api/router.go+++ b/internal/api/router.go@@ -466,28 +466,9 @@ func (r *Router) setupRoutes() {r.trueNASPoller.Start(r.lifecycleCtx)r.vmwarePoller = monitoring.NewVMwarePoller(r.multiTenant, 0)r.vmwarePoller.Start(r.lifecycleCtx)- if r.monitor != nil {- // Drive the connection-degraded alert off the same aggregator the- // HTTP handler uses, so the active-notification stream stays in- // lockstep with the Settings → Infrastructure badges. Single-tenant- // only for now; multi-tenant per-org wiring is a follow-up.- // Capture the constructed runtime owners, not the nil fields that- // existed earlier while handlers were being registered. The monitor- // may already be running, so publish only this complete snapshot source.- getCfg := r.configHandlers.Config- getPersist := r.configHandlers.Persistence- monitor := r.monitor- trueNASPoller := r.trueNASPoller- vmwarePoller := r.vmwarePoller- r.monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {- ctx := context.Background()- return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, aggregatorRuntimeSources{- orgID: "default",- truenasPoller: trueNASPoller,- vmwarePoller: vmwarePoller,- })- })- }+ // The connection-degraded lister is installed by+ // configureMonitorDependencies, which runs below for the startup monitor+ // and again for every monitor a reload or tenant initializer attaches.updateHandlers := NewUpdateHandlersWithContext(r.updateManager, r.updateHistory, r.lifecycleCtx)updateHandlers.SetUpdateReadinessSources(r.updateReadinessConfigSnapshot,@@ -1905,6 +1886,8 @@ func (r *Router) configureMonitorDependenciesLocked(m *monitoring.Monitor) {}}+ r.installConnectionAlertLister(m)+// Apply saved notification policy before the first publication, which can// evaluate alerts. The combined boundary avoids one refresh per source.if adapter := r.monitorAdapterForMonitor(m); adapter != nil {