Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Restart PBS metric timing after node-status gaps

pulse-triage[bot]667906c5f21e
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 8fa70e7f3e..169e2ba659 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -15,6 +15,21 @@
## Purpose
+### Missing PBS node-status polls restart metric timing
+
+When PBS node-status collection fails, absent CPU and memory observations pass
+through the shared metric evaluator. Pending activation, explicit intent grace
+and memory recovery timing restart without changing an open occurrence, its
+acknowledgement, trusted readings or lifecycle history. A fresh poll must satisfy
+the existing full delay; missing time cannot establish continuous evidence.
+Measured zero retains normal recovery, including immediate CPU recovery.
+
+`pbs_observation_continuity_test.go` covers both metrics, legacy and explicit
+grace, recovery boundaries, retained low/high values, exact-PBS isolation and
+live/shadow agreement. Global pause, disabled policy and full-outage precedence
+remain unchanged. No collector, connectivity, schema or transport changes;
+synthetic manager/browser proof is not installed PBS or recipient recovery.
+
### Storage capacity warnings require observed byte counters
diff --git a/internal/alerts/pbs.go b/internal/alerts/pbs.go
index b7575126e5..ab8e30d985 100644
--- a/internal/alerts/pbs.go
+++ b/internal/alerts/pbs.go
@@ -125,21 +125,22 @@ func (m *Manager) CheckPBS(pbs models.PBSInstance) {
return
}
- // Endpoint failure is not evidence of metric recovery. Connectivity above
- // remains independent, as do explicit policy suppression and full outages.
- if pbs.NodeMetricsUnavailable {
- return
- }
-
- m.evaluateUnifiedMetrics(&UnifiedResourceInput{
+ input := &UnifiedResourceInput{
ID: pbs.ID,
Type: "pbs",
Name: pbs.Name,
Node: pbs.Host,
Instance: pbs.Name,
- CPU: &UnifiedResourceMetric{Percent: pbs.CPU},
- Memory: &UnifiedResourceMetric{Percent: pbs.Memory},
- }, thresholds, nil)
+ }
+ // Endpoint failure is a missing observation, not merely a pause in metric
+ // evaluation. Let the shared evaluator interrupt activation/recovery runs
+ // and intent grace without changing an open alert's trusted evidence.
+ // Connectivity above and full-outage precedence remain independent.
+ if !pbs.NodeMetricsUnavailable {
+ input.CPU = &UnifiedResourceMetric{Percent: pbs.CPU}
+ input.Memory = &UnifiedResourceMetric{Percent: pbs.Memory}
+ }
+ m.evaluateUnifiedMetrics(input, thresholds, nil)
}
// checkPBSOffline creates an alert for offline PBS instances
diff --git a/internal/alerts/pbs_observation_continuity_test.go b/internal/alerts/pbs_observation_continuity_test.go
new file mode 100644
index 0000000000..f670b62fc8
--- /dev/null
+++ b/internal/alerts/pbs_observation_continuity_test.go
@@ -0,0 +1,184 @@
+package alerts
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+func pbsContinuityManager(t *testing.T, explicit bool) (*Manager, *atomic.Int64) {
+ t.Helper()
+ m, clock := continuityManager(t, explicit)
+ m.mu.Lock()
+ m.config.PBSDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ m.config.PBSDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ m.mu.Unlock()
+ return m, clock
+}
+
+func pbsContinuitySample(id, metric string, value float64, missing bool) models.PBSInstance {
+ pbs := models.PBSInstance{ID: id, Name: "backup " + id, Host: id + ".example.invalid",
+ Status: "online", ConnectionHealth: "healthy", NodeMetricsUnavailable: missing}
+ if metric == "cpu" {
+ pbs.CPU = value
+ } else {
+ pbs.Memory = value
+ }
+ return pbs
+}
+
+func pbsContinuityAlerts(t *testing.T, m *Manager) []byte {
+ t.Helper()
+ data, err := json.Marshal(m.GetActiveAlerts())
+ if err != nil {
+ t.Fatal(err)
+ }
+ return data
+}
+
+func TestPBSMetricGapRestartsActivation(t *testing.T) {
+ for _, explicit := range []bool{false, true} {
+ for _, metric := range []string{"cpu", "memory"} {
+ for _, retained := range []float64{0, 85, 99} {
+ t.Run(fmt.Sprintf("%s/intent=%v/retained=%.0f", metric, explicit, retained), func(t *testing.T) {
+ m, clock := pbsContinuityManager(t, explicit)
+ const id, peer = "pbs-primary", "pbs-primary-other"
+ m.CheckPBS(pbsContinuitySample(id, metric, 85, false))
+ m.CheckPBS(pbsContinuitySample(peer, metric, 85, false))
+ if incident, ok := continuityIncident(m, id, metric); !ok || incident.State != reducer.StatePending {
+ t.Fatalf("fresh warning did not start a pending run: %+v", incident)
+ }
+ clock.Store(int64(2 * time.Minute))
+ for range 3 {
+ m.CheckPBS(pbsContinuitySample(id, metric, retained, true))
+ }
+ if _, ok := continuityIncident(m, id, metric); ok {
+ t.Error("PBS_GAP_ACTIVATION: missing node status retained the pending run")
+ }
+ if _, ok := continuityIncident(m, peer, metric); !ok {
+ t.Error("missing node status interrupted another PBS")
+ }
+ m.mu.RLock()
+ for _, pending := range m.intentPending {
+ if pending.ResourceID == id {
+ t.Error("PBS_GAP_INTENT: missing node status retained durable intent grace")
+ }
+ }
+ m.mu.RUnlock()
+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{})) != 0 {
+ t.Fatal("missing node status manufactured a lifecycle transition")
+ }
+ m.CheckPBS(pbsContinuitySample(id, metric, 85, false))
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("PBS_GAP_FRESH: first fresh poll completed grace across missing node status")
+ }
+ clock.Store(int64(2*time.Minute + 59*time.Second))
+ m.CheckPBS(pbsContinuitySample(id, metric, 85, false))
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("fresh activation completed before the whole configured delay")
+ }
+ clock.Store(int64(3 * time.Minute))
+ m.CheckPBS(pbsContinuitySample(id, metric, 85, false))
+ active := m.GetActiveAlerts()
+ if len(active) != 1 || active[0].ResourceID != id || active[0].Type != metric ||
+ len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired}})) != 1 {
+ t.Fatalf("fresh continuous evidence did not fire exactly once: %+v", active)
+ }
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("PBS interruption diverged from the shadow reducer")
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestPBSMetricGapRestartsRecovery(t *testing.T) {
+ for _, retained := range []float64{0, 85, 99} {
+ t.Run(fmt.Sprintf("retained=%.0f", retained), func(t *testing.T) {
+ m, clock := pbsContinuityManager(t, false)
+ const id = "pbs-primary"
+ m.CheckPBS(pbsContinuitySample(id, "memory", 95, false))
+ initial := m.GetActiveAlerts()
+ if len(initial) != 1 {
+ t.Fatal("critical memory control must fire immediately")
+ }
+ if err := m.AcknowledgeAlert(initial[0].ID, "operator"); err != nil {
+ t.Fatal(err)
+ }
+ clock.Store(int64(10 * time.Second))
+ m.CheckPBS(pbsContinuitySample(id, "memory", 0, false))
+ if incident, _ := continuityIncident(m, id, "memory"); incident.RecoverySince.IsZero() {
+ t.Fatal("fresh zero must begin the existing memory recovery delay")
+ }
+ before := pbsContinuityAlerts(t, m)
+ events := len(queryAlertEvents(t, m, eventlog.Filter{}))
+ clock.Store(int64(2 * time.Minute))
+ for range 3 {
+ m.CheckPBS(pbsContinuitySample(id, "memory", retained, true))
+ }
+ if incident, _ := continuityIncident(m, id, "memory"); !incident.RecoverySince.IsZero() {
+ t.Error("PBS_GAP_RECOVERY: missing node status retained the recovery run")
+ }
+ if !bytes.Equal(before, pbsContinuityAlerts(t, m)) || len(queryAlertEvents(t, m, eventlog.Filter{})) != events {
+ t.Fatal("missing node status changed the occurrence, acknowledgement, trusted evidence or ledger")
+ }
+ m.CheckPBS(pbsContinuitySample(id, "memory", 0, false))
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("PBS_GAP_RESOLVED: first fresh zero falsely recovered across the missing interval")
+ }
+ clock.Store(int64(2*time.Minute + 59*time.Second))
+ m.CheckPBS(pbsContinuitySample(id, "memory", 0, false))
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("fresh recovery completed before the whole configured delay")
+ }
+ clock.Store(int64(3 * time.Minute))
+ m.CheckPBS(pbsContinuitySample(id, "memory", 0, false))
+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeResolved}})) != 1 {
+ t.Fatal("fresh continuous evidence must resolve the original exactly once")
+ }
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("PBS recovery interruption diverged from the shadow reducer")
+ }
+ })
+ }
+}
+
+func TestPBSMetricGapKeepsImmediateCPURecoveryAndGlobalPause(t *testing.T) {
+ m, _ := pbsContinuityManager(t, false)
+ m.mu.Lock()
+ m.config.MetricTimeThresholds["all"]["cpu"] = 0
+ m.mu.Unlock()
+ pbs := pbsContinuitySample("pbs-primary", "cpu", 95, false)
+ m.CheckPBS(pbs)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("CPU control did not fire")
+ }
+ pbs.CPU, pbs.NodeMetricsUnavailable = 0, true
+ m.CheckPBS(pbs)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("missing CPU observation resolved the alert")
+ }
+ m.mu.Lock()
+ m.config.Enabled = false
+ m.mu.Unlock()
+ pbs.NodeMetricsUnavailable = false
+ m.CheckPBS(pbs)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("global pause must not be treated as metric recovery")
+ }
+ m.mu.Lock()
+ m.config.Enabled = true
+ m.mu.Unlock()
+ m.CheckPBS(pbs)
+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeResolved}})) != 1 {
+ t.Fatal("measured zero must retain immediate CPU recovery, not gain a new delay")
+ }
+}