Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Restore runtime-backed TrueNAS and VMware connection alerts
pulse-triage[bot]7f5c8a0cf14c
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.mdindex f28f290b1a..cba4da2477 100644--- a/docs/release-control/v6/internal/subsystems/api-contracts.md+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md@@ -20,6 +20,27 @@## Purpose+### Platform connection alerts share the constructed runtime owners++The default monitor's connection-alert snapshot callback is installed only after+TrueNAS and VMware pollers have been constructed. It captures those exact+router-owned instances, as used by the request-time connections ledger, rather+than startup's nil fields. A runtime inventory or login failure can therefore+reach the existing connection-degraded detector with the same state, error+category and observation time shown by `GET /api/connections`.++Existing three-observation firing, healthy recovery confirmation, disabled and+paused suppression, tenant separation and source-specific policy are unchanged.+This preserves the existing default-monitor scope; it does not add per-tenant+alert wiring, a probe, route, permission, wire field or new detector. Saved Tests+and configured scope still cannot prove a TrueNAS runtime poll.+`TestRouterPlatformConnectionAlertsFollowRuntime` in `internal/api/contract_test.go`+uses the actual constructor, loopback appliance clients, saved config, ledger,+normal monitor ticker and alert manager. It covers both observed failures,+paused peers, independent authoritative disable and required-inventory recovery.+Those controls prove source wiring, not #2382's cause, native appliance recovery+or release availability.+### Metrics History responses are bound to the issued selectionThe shared History client admits HTTP200 JSON only when the response resourcediff --git a/internal/api/contract_test.go b/internal/api/contract_test.goindex fdf0d3e677..560758dae7 100644--- a/internal/api/contract_test.go+++ b/internal/api/contract_test.go@@ -25195,3 +25195,168 @@ func TestTrueNASSavedProbePreservesRuntimeEvidence(t *testing.T) {})}}++// Exercise the real router constructor and monitor ticker: a handler-only+// summary test cannot detect a closure capturing nil pollers during startup.+// The appliances are loopback analogues, not native acceptance of #2382.+func TestRouterPlatformConnectionAlertsFollowRuntime(t *testing.T) {+ setMockModeForTest(t, false)+ t.Setenv(truenas.FeatureTrueNAS, "true")+ t.Setenv(vmware.FeatureVMware, "true")+ setTrueNASFeatureForTest(t, true)+ oldVMware := vmware.IsFeatureEnabled()+ vmware.SetFeatureEnabled(true)+ t.Cleanup(func() { vmware.SetFeatureEnabled(oldVMware) })++ var failPools atomic.Bool+ failPools.Store(true)+ nas := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api/v2.0/system/info":+ _, _ = w.Write([]byte(`{"hostname":"fixture-nas","version":"TrueNAS-SCALE-24.10.2","uptime_seconds":100}`))+ case "/api/v2.0/pool":+ if failPools.Load() {+ w.WriteHeader(http.StatusUnauthorized)+ _, _ = w.Write([]byte(`{"error":"fixture required inventory denied"}`))+ return+ }+ _, _ = w.Write([]byte(`[{"id":1,"name":"fixture-pool","status":"ONLINE","size":1000,"allocated":400,"free":600}]`))+ case "/api/v2.0/pool/dataset", "/api/v2.0/disk", "/api/v2.0/alert/list":+ _, _ = w.Write([]byte(`[]`))+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(nas.Close)+ vc := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "fixture vCenter login denied", http.StatusUnauthorized)+ }))+ t.Cleanup(vc.Close)++ dataDir := t.TempDir()+ cfg := &config.Config{DataPath: dataDir, ConfigPath: dataDir, PVEPollingInterval: 10 * time.Second}+ persistence := config.NewConfigPersistence(dataDir)+ tn := trueNASInstanceFromRawURL(t, "fixture-nas", nas.URL, true)+ pausedTN := tn+ pausedTN.ID, pausedTN.Enabled = "paused-nas", false+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{tn, pausedTN}); err != nil {+ t.Fatal(err)+ }+ vw := config.NewVMwareVCenterInstance()+ vw.ID, vw.Name, vw.Host = "fixture-vc", "fixture-vc", vc.URL+ vw.Username, vw.Password, vw.InsecureSkipVerify = "fixture-user", "fixture-password", true+ pausedVW := vw+ pausedVW.ID, pausedVW.Enabled = "paused-vc", false+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{vw, pausedVW}); err != nil {+ t.Fatal(err)+ }++ monitor, err := monitoring.New(cfg)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(monitor.Stop)+ router := NewRouter(cfg, monitor, nil, nil, nil, "fixture")+ cleanupTestRouter(t, router)+ manager := monitor.GetAlertManager()++ rows := func() map[string]Connection {+ t.Helper()+ rec := httptest.NewRecorder()+ router.connectionsHandlers.HandleList(rec, httptest.NewRequest(http.MethodGet, "/api/connections", nil))+ if rec.Code != http.StatusOK {+ t.Fatalf("ledger status=%d", rec.Code)+ }+ var response ConnectionsListResponse+ if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {+ t.Fatal(err)+ }+ result := make(map[string]Connection, len(response.Connections))+ for _, row := range response.Connections {+ result[row.ID] = row+ }+ return result+ }+ active := func(id string) *alerts.Alert {+ for _, alert := range manager.GetActiveAlerts() {+ if alert.Type == "connection-degraded" && alert.ResourceID == id {+ return &alert+ }+ }+ return nil+ }+ wait := func(timeout time.Duration, predicate func() bool, failure string) {+ t.Helper()+ deadline := time.Now().Add(timeout)+ for time.Now().Before(deadline) {+ if predicate() {+ return+ }+ time.Sleep(20 * time.Millisecond)+ }+ t.Fatal(failure)+ }+ nasID, vcID := "truenas:"+tn.ID, "vmware:"+vw.ID+ wait(5*time.Second, func() bool {+ got := rows()+ return got[nasID].State == ConnectionStateUnauthorized && got[vcID].State == ConnectionStateUnauthorized+ }, "actual platform failures did not reach the ledger")+ if active(nasID) != nil || active(vcID) != nil {+ t.Fatal("alerts bypassed the monitor confirmation ticks")+ }+ t.Log("actual TrueNAS required-method and VMware login failures reach the ledger before alerts")++ ctx, cancel := context.WithCancel(context.Background())+ stopped := make(chan struct{})+ go func() { defer close(stopped); monitor.Start(ctx, nil) }()+ t.Cleanup(func() {+ cancel()+ select {+ case <-stopped:+ case <-time.After(5 * time.Second):+ t.Error("monitor loop did not stop")+ }+ })+ wait(36*time.Second, func() bool { return active(nasID) != nil && active(vcID) != nil },+ "ledger failures stayed silent in the real monitor alert loop")+ for _, id := range []string{nasID, vcID} {+ alert := active(id)+ row := rows()[id]+ if alert.Level != alerts.AlertLevelCritical || alert.Metadata["state"] != string(row.State) || alert.Metadata["lastErrorCategory"] != row.LastError.Category {+ t.Fatalf("alert did not carry the ledger's observed failure: %+v", alert)+ }+ }+ for _, id := range []string{"truenas:" + pausedTN.ID, "vmware:" + pausedVW.ID} {+ if rows()[id].State != ConnectionStatePaused || active(id) != nil {+ t.Fatalf("paused connection %s alerted", id)+ }+ }+ t.Log("both runtime failures raised confirmed critical alerts; paused peers remained silent")++ vw.Enabled = false+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{vw, pausedVW}); err != nil {+ t.Fatal(err)+ }+ wait(13*time.Second, func() bool { return rows()[vcID].State == ConnectionStatePaused && active(vcID) == nil },+ "authoritative disable did not suppress the existing VMware alert")+ if active(nasID) == nil {+ t.Fatal("disabling VMware suppressed the independent TrueNAS failure")+ }+ t.Log("disabling VMware clears only its alert, without claiming a successful poll")++ // Recovery comes from the next ordinary saved-connection poll, not a+ // separate Test or a restart. Keep the real cadence and confirmation policy.+ failPools.Store(false)+ wait(65*time.Second, func() bool { return rows()[nasID].State == ConnectionStateActive },+ "completed required-inventory recovery did not reach the ledger")+ if active(nasID) == nil {+ t.Fatal("one successful poll bypassed recovery confirmation")+ }+ wait(36*time.Second, func() bool { return active(nasID) == nil },+ "healthy runtime confirmation did not resolve the TrueNAS alert")+ if rows()[nasID].LastSeen == nil {+ t.Fatal("recovery omitted actual observation time")+ }+ t.Log("real TrueNAS inventory recovery reaches the ledger and clears only after monitor confirmations")+}diff --git a/internal/api/router.go b/internal/api/router.goindex f4a4bfb22a..44e96339d8 100644--- a/internal/api/router.go+++ b/internal/api/router.go@@ -432,25 +432,6 @@ func (r *Router) setupRoutes() {func(context.Context) *monitoring.TrueNASPoller { return r.trueNASPoller },func(context.Context) *monitoring.VMwarePoller { return r.vmwarePoller },)- if r.monitor != nil {- // Drive the connection-degraded alert off the same aggregator the- // HTTP handler uses, so the active-notification stream stays in- // lockstep with the Settings → Infrastructure badges. Single-tenant- // only for now; multi-tenant per-org wiring is a follow-up.- getCfg := r.configHandlers.Config- getPersist := r.configHandlers.Persistence- monitor := r.monitor- trueNASPoller := r.trueNASPoller- vmwarePoller := r.vmwarePoller- r.monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {- ctx := context.Background()- return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, aggregatorRuntimeSources{- orgID: "default",- truenasPoller: trueNASPoller,- vmwarePoller: vmwarePoller,- })- })- }r.availabilityHandlers = NewAvailabilityHandlers(r.configHandlers.Persistence,r.configHandlers.Monitor,@@ -481,6 +462,28 @@ func (r *Router) setupRoutes() {r.trueNASPoller.Start(r.lifecycleCtx)r.vmwarePoller = monitoring.NewVMwarePoller(r.multiTenant, 0)r.vmwarePoller.Start(r.lifecycleCtx)+ if r.monitor != nil {+ // Drive the connection-degraded alert off the same aggregator the+ // HTTP handler uses, so the active-notification stream stays in+ // lockstep with the Settings → Infrastructure badges. Single-tenant+ // only for now; multi-tenant per-org wiring is a follow-up.+ // Capture the constructed runtime owners, not the nil fields that+ // existed earlier while handlers were being registered. The monitor+ // may already be running, so publish only this complete snapshot source.+ getCfg := r.configHandlers.Config+ getPersist := r.configHandlers.Persistence+ monitor := r.monitor+ trueNASPoller := r.trueNASPoller+ vmwarePoller := r.vmwarePoller+ r.monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {+ ctx := context.Background()+ return buildAlertConnectionSnapshotsWithRuntimeSources(ctx, getCfg(ctx), getPersist(ctx), monitor, aggregatorRuntimeSources{+ orgID: "default",+ truenasPoller: trueNASPoller,+ vmwarePoller: vmwarePoller,+ })+ })+ }updateHandlers := NewUpdateHandlersWithContext(r.updateManager, r.updateHistory, r.lifecycleCtx)updateHandlers.SetUpdateReadinessSources(r.updateReadinessConfigSnapshot,