Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Restore runtime-backed TrueNAS and VMware connection alerts

pulse-triage[bot]7f5c8a0cf14c
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md
index f28f290b1a..cba4da2477 100644
--- a/docs/release-control/v6/internal/subsystems/api-contracts.md
+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md
@@ -20,6 +20,27 @@
## Purpose
+### Platform connection alerts share the constructed runtime owners
+
+The default monitor's connection-alert snapshot callback is installed only after
+TrueNAS and VMware pollers have been constructed. It captures those exact
+router-owned instances, as used by the request-time connections ledger, rather
+than startup's nil fields. A runtime inventory or login failure can therefore
+reach the existing connection-degraded detector with the same state, error
+category and observation time shown by `GET /api/connections`.
+
+Existing three-observation firing, healthy recovery confirmation, disabled and
+paused suppression, tenant separation and source-specific policy are unchanged.
+This preserves the existing default-monitor scope; it does not add per-tenant
+alert wiring, a probe, route, permission, wire field or new detector. Saved Tests
+and configured scope still cannot prove a TrueNAS runtime poll.
+`TestRouterPlatformConnectionAlertsFollowRuntime` in `internal/api/contract_test.go`
+uses the actual constructor, loopback appliance clients, saved config, ledger,
+normal monitor ticker and alert manager. It covers both observed failures,
+paused peers, independent authoritative disable and required-inventory recovery.
+Those controls prove source wiring, not #2382's cause, native appliance recovery
+or release availability.
+
### Metrics History responses are bound to the issued selection
The shared History client admits HTTP200 JSON only when the response resource
diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go
index fdf0d3e677..560758dae7 100644
--- a/internal/api/contract_test.go
+++ b/internal/api/contract_test.go
@@ -25195,3 +25195,168 @@ func TestTrueNASSavedProbePreservesRuntimeEvidence(t *testing.T) {
})
}
}
+
+// Exercise the real router constructor and monitor ticker: a handler-only
+// summary test cannot detect a closure capturing nil pollers during startup.
+// The appliances are loopback analogues, not native acceptance of #2382.
+func TestRouterPlatformConnectionAlertsFollowRuntime(t *testing.T) {
+ setMockModeForTest(t, false)
+ t.Setenv(truenas.FeatureTrueNAS, "true")
+ t.Setenv(vmware.FeatureVMware, "true")
+ setTrueNASFeatureForTest(t, true)
+ oldVMware := vmware.IsFeatureEnabled()
+ vmware.SetFeatureEnabled(true)
+ t.Cleanup(func() { vmware.SetFeatureEnabled(oldVMware) })
+
+ var failPools atomic.Bool
+ failPools.Store(true)
+ nas := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api/v2.0/system/info":
+ _, _ = w.Write([]byte(`{"hostname":"fixture-nas","version":"TrueNAS-SCALE-24.10.2","uptime_seconds":100}`))
+ case "/api/v2.0/pool":
+ if failPools.Load() {
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"error":"fixture required inventory denied"}`))
+ return
+ }
+ _, _ = w.Write([]byte(`[{"id":1,"name":"fixture-pool","status":"ONLINE","size":1000,"allocated":400,"free":600}]`))
+ case "/api/v2.0/pool/dataset", "/api/v2.0/disk", "/api/v2.0/alert/list":
+ _, _ = w.Write([]byte(`[]`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(nas.Close)
+ vc := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "fixture vCenter login denied", http.StatusUnauthorized)
+ }))
+ t.Cleanup(vc.Close)
+
+ dataDir := t.TempDir()
+ cfg := &config.Config{DataPath: dataDir, ConfigPath: dataDir, PVEPollingInterval: 10 * time.Second}
+ persistence := config.NewConfigPersistence(dataDir)
+ tn := trueNASInstanceFromRawURL(t, "fixture-nas", nas.URL, true)
+ pausedTN := tn
+ pausedTN.ID, pausedTN.Enabled = "paused-nas", false
+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{tn, pausedTN}); err != nil {
+ t.Fatal(err)
+ }
+ vw := config.NewVMwareVCenterInstance()
+ vw.ID, vw.Name, vw.Host = "fixture-vc", "fixture-vc", vc.URL
+ vw.Username, vw.Password, vw.InsecureSkipVerify = "fixture-user", "fixture-password", true
+ pausedVW := vw
+ pausedVW.ID, pausedVW.Enabled = "paused-vc", false
+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{vw, pausedVW}); err != nil {
+ t.Fatal(err)
+ }
+
+ monitor, err := monitoring.New(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(monitor.Stop)
+ router := NewRouter(cfg, monitor, nil, nil, nil, "fixture")
+ cleanupTestRouter(t, router)
+ manager := monitor.GetAlertManager()
+
+ rows := func() map[string]Connection {
+ t.Helper()
+ rec := httptest.NewRecorder()
+ router.connectionsHandlers.HandleList(rec, httptest.NewRequest(http.MethodGet, "/api/connections", nil))
+ if rec.Code != http.StatusOK {
+ t.Fatalf("ledger status=%d", rec.Code)
+ }
+ var response ConnectionsListResponse
+ if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
+ t.Fatal(err)
+ }
+ result := make(map[string]Connection, len(response.Connections))
+ for _, row := range response.Connections {
+ result[row.ID] = row
+ }
+ return result
+ }
+ active := func(id string) *alerts.Alert {
+ for _, alert := range manager.GetActiveAlerts() {
+ if alert.Type == "connection-degraded" && alert.ResourceID == id {
+ return &alert
+ }
+ }
+ return nil
+ }
+ wait := func(timeout time.Duration, predicate func() bool, failure string) {
+ t.Helper()
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ if predicate() {
+ return
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Fatal(failure)
+ }
+ nasID, vcID := "truenas:"+tn.ID, "vmware:"+vw.ID
+ wait(5*time.Second, func() bool {
+ got := rows()
+ return got[nasID].State == ConnectionStateUnauthorized && got[vcID].State == ConnectionStateUnauthorized
+ }, "actual platform failures did not reach the ledger")
+ if active(nasID) != nil || active(vcID) != nil {
+ t.Fatal("alerts bypassed the monitor confirmation ticks")
+ }
+ t.Log("actual TrueNAS required-method and VMware login failures reach the ledger before alerts")
+
+ ctx, cancel := context.WithCancel(context.Background())
+ stopped := make(chan struct{})
+ go func() { defer close(stopped); monitor.Start(ctx, nil) }()
+ t.Cleanup(func() {
+ cancel()
+ select {
+ case <-stopped:
+ case <-time.After(5 * time.Second):
+ t.Error("monitor loop did not stop")
+ }
+ })
+ wait(36*time.Second, func() bool { return active(nasID) != nil && active(vcID) != nil },
+ "ledger failures stayed silent in the real monitor alert loop")
+ for _, id := range []string{nasID, vcID} {
+ alert := active(id)
+ row := rows()[id]
+ if alert.Level != alerts.AlertLevelCritical || alert.Metadata["state"] != string(row.State) || alert.Metadata["lastErrorCategory"] != row.LastError.Category {
+ t.Fatalf("alert did not carry the ledger's observed failure: %+v", alert)
+ }
+ }
+ for _, id := range []string{"truenas:" + pausedTN.ID, "vmware:" + pausedVW.ID} {
+ if rows()[id].State != ConnectionStatePaused || active(id) != nil {
+ t.Fatalf("paused connection %s alerted", id)
+ }
+ }
+ t.Log("both runtime failures raised confirmed critical alerts; paused peers remained silent")
+
+ vw.Enabled = false
+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{vw, pausedVW}); err != nil {
+ t.Fatal(err)
+ }
+ wait(13*time.Second, func() bool { return rows()[vcID].State == ConnectionStatePaused && active(vcID) == nil },
+ "authoritative disable did not suppress the existing VMware alert")
+ if active(nasID) == nil {
+ t.Fatal("disabling VMware suppressed the independent TrueNAS failure")
+ }
+ t.Log("disabling VMware clears only its alert, without claiming a successful poll")
+
+ // Recovery comes from the next ordinary saved-connection poll, not a
+ // separate Test or a restart. Keep the real cadence and confirmation policy.
+ failPools.Store(false)
+ wait(65*time.Second, func() bool { return rows()[nasID].State == ConnectionStateActive },
+ "completed required-inventory recovery did not reach the ledger")
+ if active(nasID) == nil {
+ t.Fatal("one successful poll bypassed recovery confirmation")
+ }
+ wait(36*time.Second, func() bool { return active(nasID) == nil },
+ "healthy runtime confirmation did not resolve the TrueNAS alert")
+ if rows()[nasID].LastSeen == nil {
+ t.Fatal("recovery omitted actual observation time")
+ }
+ t.Log("real TrueNAS inventory recovery reaches the ledger and clears only after monitor confirmations")
+}
diff --git a/internal/api/router.go b/internal/api/router.go
index f4a4bfb22a..44e96339d8 100644
--- a/internal/api/router.go
+++ b/internal/api/router.go
@@ -432,25 +432,6 @@ func (r *Router) setupRoutes() {
func(context.Context) *monitoring.TrueNASPoller { return r.trueNASPoller },
func(context.Context) *monitoring.VMwarePoller { return r.vmwarePoller },
)
- if r.monitor != nil {
- // Drive the connection-degraded alert off the same aggregator the
- // HTTP handler uses, so the active-notification stream stays in
- // lockstep with the Settings → Infrastructure badges. Single-tenant
- // only for now; multi-tenant per-org wiring is a follow-up.
- getCfg := r.configHandlers.Config
- getPersist := r.configHandlers.Persistence
- monitor := r.monitor
- trueNASPoller := r.trueNASPoller
- vmwarePoller := r.vmwarePoller
- r.monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {
- ctx := context.Background()
- return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, aggregatorRuntimeSources{
- orgID: "default",
- truenasPoller: trueNASPoller,
- vmwarePoller: vmwarePoller,
- })
- })
- }
r.availabilityHandlers = NewAvailabilityHandlers(
r.configHandlers.Persistence,
r.configHandlers.Monitor,
@@ -481,6 +462,28 @@ func (r *Router) setupRoutes() {
r.trueNASPoller.Start(r.lifecycleCtx)
r.vmwarePoller = monitoring.NewVMwarePoller(r.multiTenant, 0)
r.vmwarePoller.Start(r.lifecycleCtx)
+ if r.monitor != nil {
+ // Drive the connection-degraded alert off the same aggregator the
+ // HTTP handler uses, so the active-notification stream stays in
+ // lockstep with the Settings → Infrastructure badges. Single-tenant
+ // only for now; multi-tenant per-org wiring is a follow-up.
+ // Capture the constructed runtime owners, not the nil fields that
+ // existed earlier while handlers were being registered. The monitor
+ // may already be running, so publish only this complete snapshot source.
+ getCfg := r.configHandlers.Config
+ getPersist := r.configHandlers.Persistence
+ monitor := r.monitor
+ trueNASPoller := r.trueNASPoller
+ vmwarePoller := r.vmwarePoller
+ r.monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {
+ ctx := context.Background()
+ return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, aggregatorRuntimeSources{
+ orgID: "default",
+ truenasPoller: trueNASPoller,
+ vmwarePoller: vmwarePoller,
+ })
+ })
+ }
updateHandlers := NewUpdateHandlersWithContext(r.updateManager, r.updateHistory, r.lifecycleCtx)
updateHandlers.SetUpdateReadinessSources(
r.updateReadinessConfigSnapshot,