Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep connection alerts running after a monitor reload

rcourtman9c58cffd4dce
diff --git a/internal/api/connections_alerts.go b/internal/api/connections_alerts.go
index acd8c948d3..16c0a4553d 100644
--- a/internal/api/connections_alerts.go
+++ b/internal/api/connections_alerts.go
@@ -158,6 +158,39 @@ func snapshotConnectionsForAlerts(connections []Connection) []alerts.ConnectionS
return out
}
+// installConnectionAlertLister gives monitor the closure its poll tick uses to
+// raise and resolve connection-degraded alerts, bound to that monitor. Every
+// path that attaches a monitor to the router calls it through
+// configureMonitorDependencies: startup, SetMonitor after a full reload
+// (node auto-registration replaces the default monitor), and the
+// multi-tenant initializer that lazily creates the default monitor. A lister
+// installed once at startup stays on the discarded monitor, so the
+// replacement would never evaluate connection alerts again until restart.
+//
+// Only the default org's monitor gets a lister: the closure reads the
+// default org's connection config, and per-org connection alerts are not
+// wired yet. Until the platform pollers exist the monitor keeps whatever
+// lister it had, so a running monitor never sees nil runtime owners.
+func (r *Router) installConnectionAlertLister(monitor *monitoring.Monitor) {
+ if r == nil || monitor == nil || r.configHandlers == nil || r.trueNASPoller == nil || r.vmwarePoller == nil {
+ return
+ }
+ if orgID := monitor.GetOrgID(); orgID != "" && orgID != "default" {
+ return
+ }
+ getCfg := r.configHandlers.Config
+ getPersist := r.configHandlers.Persistence
+ runtime := aggregatorRuntimeSources{
+ orgID: "default",
+ truenasPoller: r.trueNASPoller,
+ vmwarePoller: r.vmwarePoller,
+ }
+ monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {
+ ctx := context.Background()
+ return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, runtime)
+ })
+}
+
func buildAlertConnectionSnapshotsWithRuntimeSources(
ctx context.Context,
cfg *config.Config,
diff --git a/internal/api/connections_alerts_monitor_swap_test.go b/internal/api/connections_alerts_monitor_swap_test.go
new file mode 100644
index 0000000000..f1ea2d0342
--- /dev/null
+++ b/internal/api/connections_alerts_monitor_swap_test.go
@@ -0,0 +1,83 @@
+package api
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"
+)
+
+// A full monitor reload (node auto-registration, settings that need one)
+// replaces the default monitor and reattaches it with SetMonitor and
+// SetMultiTenantMonitor, in that order, exactly as pkg/server does. The
+// replacement must keep raising connection-degraded alerts from its own poll
+// loop: a lister installed only on the startup monitor left the new one with
+// none, so wedged connections stayed silent until restart.
+func TestConnectionAlertsSurviveMonitorReload(t *testing.T) {
+ setMockModeForTest(t, false)
+ t.Setenv(truenas.FeatureTrueNAS, "true")
+ setTrueNASFeatureForTest(t, true)
+
+ nas := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api/v2.0/system/info":
+ _, _ = w.Write([]byte(`{"hostname":"fixture-nas","version":"TrueNAS-SCALE-24.10.2","uptime_seconds":100}`))
+ case "/api/v2.0/pool":
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"error":"fixture required inventory denied"}`))
+ case "/api/v2.0/pool/dataset", "/api/v2.0/disk", "/api/v2.0/alert/list":
+ _, _ = w.Write([]byte(`[]`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(nas.Close)
+
+ dataDir := t.TempDir()
+ cfg := &config.Config{DataPath: dataDir, ConfigPath: dataDir, PVEPollingInterval: 10 * time.Second}
+ persistence := config.NewConfigPersistence(dataDir)
+ tn := trueNASInstanceFromRawURL(t, "fixture-nas", nas.URL, true)
+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{tn}); err != nil {
+ t.Fatal(err)
+ }
+
+ startup, err := monitoring.New(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(startup.Stop)
+ router := NewRouter(cfg, startup, nil, nil, nil, "fixture")
+ cleanupTestRouter(t, router)
+
+ // The reload sequence from pkg/server refreshRouterAfterMonitorReload.
+ reloaded := monitoring.NewMultiTenantMonitor(cfg, config.NewMultiTenantPersistence(dataDir), nil)
+ t.Cleanup(reloaded.Stop)
+ replacement, err := reloaded.GetMonitor("default")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if replacement == startup {
+ t.Fatal("reload fixture must produce a new default monitor")
+ }
+ router.SetMonitor(replacement)
+ router.SetMultiTenantMonitor(reloaded)
+
+ nasID := "truenas:" + tn.ID
+ manager := replacement.GetAlertManager()
+ deadline := time.Now().Add(40 * time.Second)
+ for time.Now().Before(deadline) {
+ for _, alert := range manager.GetActiveAlerts() {
+ if alert.Type == "connection-degraded" && alert.ResourceID == nasID {
+ t.Logf("reloaded monitor raised %s for the wedged connection", alert.Level)
+ return
+ }
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+ t.Fatal("the reloaded default monitor never evaluated connection-degraded alerts")
+}
diff --git a/internal/api/router.go b/internal/api/router.go
index 98e2b784a6..26648d29fd 100644
--- a/internal/api/router.go
+++ b/internal/api/router.go
@@ -466,28 +466,9 @@ func (r *Router) setupRoutes() {
r.trueNASPoller.Start(r.lifecycleCtx)
r.vmwarePoller = monitoring.NewVMwarePoller(r.multiTenant, 0)
r.vmwarePoller.Start(r.lifecycleCtx)
- if r.monitor != nil {
- // Drive the connection-degraded alert off the same aggregator the
- // HTTP handler uses, so the active-notification stream stays in
- // lockstep with the Settings → Infrastructure badges. Single-tenant
- // only for now; multi-tenant per-org wiring is a follow-up.
- // Capture the constructed runtime owners, not the nil fields that
- // existed earlier while handlers were being registered. The monitor
- // may already be running, so publish only this complete snapshot source.
- getCfg := r.configHandlers.Config
- getPersist := r.configHandlers.Persistence
- monitor := r.monitor
- trueNASPoller := r.trueNASPoller
- vmwarePoller := r.vmwarePoller
- r.monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {
- ctx := context.Background()
- return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, aggregatorRuntimeSources{
- orgID: "default",
- truenasPoller: trueNASPoller,
- vmwarePoller: vmwarePoller,
- })
- })
- }
+ // The connection-degraded lister is installed by
+ // configureMonitorDependencies, which runs below for the startup monitor
+ // and again for every monitor a reload or tenant initializer attaches.
updateHandlers := NewUpdateHandlersWithContext(r.updateManager, r.updateHistory, r.lifecycleCtx)
updateHandlers.SetUpdateReadinessSources(
r.updateReadinessConfigSnapshot,
@@ -1905,6 +1886,8 @@ func (r *Router) configureMonitorDependenciesLocked(m *monitoring.Monitor) {
}
}
+ r.installConnectionAlertLister(m)
+
// Apply saved notification policy before the first publication, which can
// evaluate alerts. The combined boundary avoids one refresh per source.
if adapter := r.monitorAdapterForMonitor(m); adapter != nil {