Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Restart PBS metric timing after node-status gaps
pulse-triage[bot]667906c5f21e
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex 8fa70e7f3e..169e2ba659 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -15,6 +15,21 @@## Purpose+### Missing PBS node-status polls restart metric timing++When PBS node-status collection fails, absent CPU and memory observations pass+through the shared metric evaluator. Pending activation, explicit intent grace+and memory recovery timing restart without changing an open occurrence, its+acknowledgement, trusted readings or lifecycle history. A fresh poll must satisfy+the existing full delay; missing time cannot establish continuous evidence.+Measured zero retains normal recovery, including immediate CPU recovery.++`pbs_observation_continuity_test.go` covers both metrics, legacy and explicit+grace, recovery boundaries, retained low/high values, exact-PBS isolation and+live/shadow agreement. Global pause, disabled policy and full-outage precedence+remain unchanged. No collector, connectivity, schema or transport changes;+synthetic manager/browser proof is not installed PBS or recipient recovery.+### Storage capacity warnings require observed byte countersdiff --git a/internal/alerts/pbs.go b/internal/alerts/pbs.goindex b7575126e5..ab8e30d985 100644--- a/internal/alerts/pbs.go+++ b/internal/alerts/pbs.go@@ -125,21 +125,22 @@ func (m *Manager) CheckPBS(pbs models.PBSInstance) {return}- // Endpoint failure is not evidence of metric recovery. Connectivity above- // remains independent, as do explicit policy suppression and full outages.- if pbs.NodeMetricsUnavailable {- return- }-- m.evaluateUnifiedMetrics(&UnifiedResourceInput{+ input := &UnifiedResourceInput{ID: pbs.ID,Type: "pbs",Name: pbs.Name,Node: pbs.Host,Instance: pbs.Name,- CPU: &UnifiedResourceMetric{Percent: pbs.CPU},- Memory: &UnifiedResourceMetric{Percent: pbs.Memory},- }, thresholds, nil)+ }+ // Endpoint failure is a missing observation, not merely a pause in metric+ // evaluation. Let the shared evaluator interrupt activation/recovery runs+ // and intent grace without changing an open alert's trusted evidence.+ // Connectivity above and full-outage precedence remain independent.+ if !pbs.NodeMetricsUnavailable {+ input.CPU = &UnifiedResourceMetric{Percent: pbs.CPU}+ input.Memory = &UnifiedResourceMetric{Percent: pbs.Memory}+ }+ m.evaluateUnifiedMetrics(input, thresholds, nil)}// checkPBSOffline creates an alert for offline PBS instancesdiff --git a/internal/alerts/pbs_observation_continuity_test.go b/internal/alerts/pbs_observation_continuity_test.gonew file mode 100644index 0000000000..f670b62fc8--- /dev/null+++ b/internal/alerts/pbs_observation_continuity_test.go@@ -0,0 +1,184 @@+package alerts++import (+ "bytes"+ "encoding/json"+ "fmt"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+)++func pbsContinuityManager(t *testing.T, explicit bool) (*Manager, *atomic.Int64) {+ t.Helper()+ m, clock := continuityManager(t, explicit)+ m.mu.Lock()+ m.config.PBSDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 70}+ m.config.PBSDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70}+ m.mu.Unlock()+ return m, clock+}++func pbsContinuitySample(id, metric string, value float64, missing bool) models.PBSInstance {+ pbs := models.PBSInstance{ID: id, Name: "backup " + id, Host: id + ".example.invalid",+ Status: "online", ConnectionHealth: "healthy", NodeMetricsUnavailable: missing}+ if metric == "cpu" {+ pbs.CPU = value+ } else {+ pbs.Memory = value+ }+ return pbs+}++func pbsContinuityAlerts(t *testing.T, m *Manager) []byte {+ t.Helper()+ data, err := json.Marshal(m.GetActiveAlerts())+ if err != nil {+ t.Fatal(err)+ }+ return data+}++func TestPBSMetricGapRestartsActivation(t *testing.T) {+ for _, explicit := range []bool{false, true} {+ for _, metric := range []string{"cpu", "memory"} {+ for _, retained := range []float64{0, 85, 99} {+ t.Run(fmt.Sprintf("%s/intent=%v/retained=%.0f", metric, explicit, retained), func(t *testing.T) {+ m, clock := pbsContinuityManager(t, explicit)+ const id, peer = "pbs-primary", "pbs-primary-other"+ m.CheckPBS(pbsContinuitySample(id, metric, 85, false))+ m.CheckPBS(pbsContinuitySample(peer, metric, 85, false))+ if incident, ok := continuityIncident(m, id, metric); !ok || incident.State != reducer.StatePending {+ t.Fatalf("fresh warning did not start a pending run: %+v", incident)+ }+ clock.Store(int64(2 * time.Minute))+ for range 3 {+ m.CheckPBS(pbsContinuitySample(id, metric, retained, true))+ }+ if _, ok := continuityIncident(m, id, metric); ok {+ t.Error("PBS_GAP_ACTIVATION: missing node status retained the pending run")+ }+ if _, ok := continuityIncident(m, peer, metric); !ok {+ t.Error("missing node status interrupted another PBS")+ }+ m.mu.RLock()+ for _, pending := range m.intentPending {+ if pending.ResourceID == id {+ t.Error("PBS_GAP_INTENT: missing node status retained durable intent grace")+ }+ }+ m.mu.RUnlock()+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{})) != 0 {+ t.Fatal("missing node status manufactured a lifecycle transition")+ }+ m.CheckPBS(pbsContinuitySample(id, metric, 85, false))+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("PBS_GAP_FRESH: first fresh poll completed grace across missing node status")+ }+ clock.Store(int64(2*time.Minute + 59*time.Second))+ m.CheckPBS(pbsContinuitySample(id, metric, 85, false))+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("fresh activation completed before the whole configured delay")+ }+ clock.Store(int64(3 * time.Minute))+ m.CheckPBS(pbsContinuitySample(id, metric, 85, false))+ active := m.GetActiveAlerts()+ if len(active) != 1 || active[0].ResourceID != id || active[0].Type != metric ||+ len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired}})) != 1 {+ t.Fatalf("fresh continuous evidence did not fire exactly once: %+v", active)+ }+ if m.ShadowDivergences() != 0 {+ t.Fatal("PBS interruption diverged from the shadow reducer")+ }+ })+ }+ }+ }+}++func TestPBSMetricGapRestartsRecovery(t *testing.T) {+ for _, retained := range []float64{0, 85, 99} {+ t.Run(fmt.Sprintf("retained=%.0f", retained), func(t *testing.T) {+ m, clock := pbsContinuityManager(t, false)+ const id = "pbs-primary"+ m.CheckPBS(pbsContinuitySample(id, "memory", 95, false))+ initial := m.GetActiveAlerts()+ if len(initial) != 1 {+ t.Fatal("critical memory control must fire immediately")+ }+ if err := m.AcknowledgeAlert(initial[0].ID, "operator"); err != nil {+ t.Fatal(err)+ }+ clock.Store(int64(10 * time.Second))+ m.CheckPBS(pbsContinuitySample(id, "memory", 0, false))+ if incident, _ := continuityIncident(m, id, "memory"); incident.RecoverySince.IsZero() {+ t.Fatal("fresh zero must begin the existing memory recovery delay")+ }+ before := pbsContinuityAlerts(t, m)+ events := len(queryAlertEvents(t, m, eventlog.Filter{}))+ clock.Store(int64(2 * time.Minute))+ for range 3 {+ m.CheckPBS(pbsContinuitySample(id, "memory", retained, true))+ }+ if incident, _ := continuityIncident(m, id, "memory"); !incident.RecoverySince.IsZero() {+ t.Error("PBS_GAP_RECOVERY: missing node status retained the recovery run")+ }+ if !bytes.Equal(before, pbsContinuityAlerts(t, m)) || len(queryAlertEvents(t, m, eventlog.Filter{})) != events {+ t.Fatal("missing node status changed the occurrence, acknowledgement, trusted evidence or ledger")+ }+ m.CheckPBS(pbsContinuitySample(id, "memory", 0, false))+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("PBS_GAP_RESOLVED: first fresh zero falsely recovered across the missing interval")+ }+ clock.Store(int64(2*time.Minute + 59*time.Second))+ m.CheckPBS(pbsContinuitySample(id, "memory", 0, false))+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("fresh recovery completed before the whole configured delay")+ }+ clock.Store(int64(3 * time.Minute))+ m.CheckPBS(pbsContinuitySample(id, "memory", 0, false))+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeResolved}})) != 1 {+ t.Fatal("fresh continuous evidence must resolve the original exactly once")+ }+ if m.ShadowDivergences() != 0 {+ t.Fatal("PBS recovery interruption diverged from the shadow reducer")+ }+ })+ }+}++func TestPBSMetricGapKeepsImmediateCPURecoveryAndGlobalPause(t *testing.T) {+ m, _ := pbsContinuityManager(t, false)+ m.mu.Lock()+ m.config.MetricTimeThresholds["all"]["cpu"] = 0+ m.mu.Unlock()+ pbs := pbsContinuitySample("pbs-primary", "cpu", 95, false)+ m.CheckPBS(pbs)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("CPU control did not fire")+ }+ pbs.CPU, pbs.NodeMetricsUnavailable = 0, true+ m.CheckPBS(pbs)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("missing CPU observation resolved the alert")+ }+ m.mu.Lock()+ m.config.Enabled = false+ m.mu.Unlock()+ pbs.NodeMetricsUnavailable = false+ m.CheckPBS(pbs)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("global pause must not be treated as metric recovery")+ }+ m.mu.Lock()+ m.config.Enabled = true+ m.mu.Unlock()+ m.CheckPBS(pbs)+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeResolved}})) != 1 {+ t.Fatal("measured zero must retain immediate CPU recovery, not gain a new delay")+ }+}