Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Restart storage alert confirmation across missing observations

pulse-triage[bot]23355440ace3
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 6d4a6def25..081be8700b 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -3876,3 +3876,26 @@ It covers both families sharing a disk, interruption/reconfirmation, partial
rule disablement, re-enablement, legacy/unknown causes, filesystem and host
isolation, removal and node-link ownership. These are source lifecycle controls,
not collector, smartctl, installed notification or native disk-recovery proof.
+
+### Storage observation gaps restart confirmation and timing
+
+An absent or `unknown` storage connectivity status interrupts activation,
+recovery confirmations and pending offline intent grace without resolving an
+existing outage or changing its acknowledgement. Capacity is independently
+observable while connectivity is unknown. Missing/unconfirmed-zero capacity,
+negative usage and offline/unavailable storage interrupt capacity activation,
+intent grace and recovery timing; they do not supply healthy capacity evidence.
+A confirmed empty store still supplies a genuine zero-usage recovery reading.
+Fresh observations must satisfy the existing full confirmation/delay after a
+gap. Interruption is scoped to the storage's canonical identity and applied to
+both live and shadow reducers; interrupted intent is checkpointed for restart.
+
+`TestStorageConnectivityObservationGaps` and
+`TestStorageConnectivityGapRestartsIntentGrace` exercise the real storage
+entry point, acknowledged occurrence retention, neighbour isolation, fresh
+reconfirmation and empty intent checkpoint. Storage routes in
+`TestMetricObservationGapRestartsActivation` and
+`TestMetricObservationGapRestartsRecovery` cover capacity gaps and explicit
+intent grace. Existing restart, empty-capacity and predictive-capacity controls
+remain required. These source controls are not native storage recovery or
+installed destination-delivery evidence.
diff --git a/internal/alerts/metric_observation_continuity_test.go b/internal/alerts/metric_observation_continuity_test.go
index 6ba596e8da..ffef188997 100644
--- a/internal/alerts/metric_observation_continuity_test.go
+++ b/internal/alerts/metric_observation_continuity_test.go
@@ -108,6 +108,25 @@ func continuityObserver(t *testing.T, m *Manager, route, gap string) (string, st
}
m.evaluateUnifiedMetrics(&UnifiedResourceInput{ID: id, Type: "vm", Name: "guest", Memory: sample}, ThresholdConfig{Memory: threshold}, nil)
}
+ case "storage":
+ id, metric = "site:node:backups", "usage"
+ return id, metric, func(value float64, missing bool) {
+ storage := models.Storage{ID: id, Name: "backups", Node: "node", Instance: "site", Status: "online",
+ Total: 1000, Used: int64(value * 10), Free: int64(1000 - value*10), Usage: value}
+ if missing {
+ switch gap {
+ case "offline", "unavailable":
+ storage.Status = gap
+ case "negative":
+ storage.Usage = -1
+ case "unconfirmed-zero":
+ storage.Usage = 0
+ default:
+ storage.Total, storage.Used, storage.Free, storage.Usage = 0, 0, 0, 0
+ }
+ }
+ m.CheckStorage(storage)
+ }
case "host":
id = "agent:host-105"
return id, metric, func(value float64, missing bool) {
@@ -184,11 +203,14 @@ func continuityObserver(t *testing.T, m *Manager, route, gap string) (string, st
func TestMetricObservationGapRestartsActivation(t *testing.T) {
for _, explicit := range []bool{false, true} {
- for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired", "host-disk-temperature"} {
+ for _, route := range []string{"legacy", "canonical", "unified", "host", "storage", "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired", "host-disk-temperature"} {
gaps := []string{"missing"}
if route == "legacy" || route == "canonical" {
gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}
}
+ if route == "storage" {
+ gaps = []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"}
+ }
if route == "host-disk-temperature" {
gaps = []string{"empty", "omitted", "zero", "negative", "standby", "unavailable", "unsupported", "missing", "invalid-state", "expired", "legacy"}
}
@@ -237,11 +259,14 @@ func TestMetricObservationGapRestartsActivation(t *testing.T) {
}
func TestMetricObservationGapRestartsRecovery(t *testing.T) {
- for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "host-disk-temperature"} {
+ for _, route := range []string{"legacy", "canonical", "unified", "host", "storage", "guest-memory", "host-disk-temperature"} {
gaps := []string{"missing"}
if route == "legacy" || route == "canonical" {
gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}
}
+ if route == "storage" {
+ gaps = []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"}
+ }
if route == "host-disk-temperature" {
gaps = []string{"empty", "omitted", "zero", "negative", "standby", "unavailable", "unsupported", "missing", "invalid-state", "expired", "legacy"}
}
diff --git a/internal/alerts/storage.go b/internal/alerts/storage.go
index ccbf35a0f2..3bb2ac80ef 100644
--- a/internal/alerts/storage.go
+++ b/internal/alerts/storage.go
@@ -88,6 +88,8 @@ func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend Ca
} else if connectivityStatus != "" && connectivityStatus != "unknown" {
// Clear any existing offline alert if storage is back online
m.clearStorageOfflineAlert(storage)
+ } else {
+ m.interruptStorageConnectivityRun(storage.ID)
}
// Missing connectivity evidence must not count as a healthy observation.
// Capacity and pool health below remain independently observable.
@@ -110,6 +112,8 @@ func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend Ca
confirmedEmpty := storage.Usage == 0 && storage.Total > 0 && storage.Used == 0 && storage.Free == storage.Total
if connectivityStatus != "offline" && connectivityStatus != "unavailable" && (storage.Usage > 0 || confirmedEmpty) {
m.evaluateStorageCapacity(storage, thresholds, trend)
+ } else {
+ m.interruptMetricRunIDs(storage.ID, canonicalMetricSpecID(storage.ID, "usage"), canonicalMetricStateID(storage.ID, "usage"))
}
// Check ZFS pool status if this is ZFS storage
@@ -120,6 +124,21 @@ func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend Ca
}
}
+// A missing status is not a healthy poll, nor another outage confirmation.
+// Preserve a firing occurrence while restarting both consecutive-observation
+// runs and any pending intent grace. Capacity remains independently observable.
+func (m *Manager) interruptStorageConnectivityRun(resourceID string) {
+ m.mu.Lock()
+ for _, state := range m.mirrorStatesNoLock() {
+ state.InterruptDiscreteRun(resourceID, canonicalConnectivitySpecID(resourceID))
+ }
+ intentChanged := m.clearIntentPendingNoLock(canonicalConnectivityStateID(resourceID))
+ m.mu.Unlock()
+ if intentChanged {
+ m.saveActiveAlertsAsync("storage connectivity observation gap")
+ }
+}
+
// clearStorageZFSAlerts removes ZFS pool/device alerts for a storage that no
// longer carries an attached ZFS pool. checkZFSPoolHealth can only clear its
// own alerts while the pool stays attached, so a storage whose attachment goes
diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.go
index 53aaa6e226..1b05799eb7 100644
--- a/internal/alerts/threshold_resolution_shared_test.go
+++ b/internal/alerts/threshold_resolution_shared_test.go
@@ -1,7 +1,11 @@
package alerts
import (
+ "encoding/json"
"math"
+ "os"
+ "path/filepath"
+ "reflect"
"testing"
"time"
@@ -1701,3 +1705,118 @@ func TestReleaseCanonicalMetricAlertClearsIntentPending(t *testing.T) {
t.Fatalf("expected releasing the metric to clear its intent pending state")
}
}
+
+// Missing storage observations break confirmation runs, not the occurrence.
+// Exercise the public evaluator and both live/shadow reducers, not seeded state.
+func TestStorageConnectivityObservationGaps(t *testing.T) {
+ for _, gap := range []string{"", "unknown", " UNKNOWN "} {
+ t.Run("activation/"+gap, func(t *testing.T) {
+ m := newShadowFeedManager(t)
+ s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}
+ other := s
+ other.ID = "storage-ab"
+ id := canonicalConnectivityStateID(s.ID)
+ m.CheckStorage(s)
+ m.CheckStorage(other)
+ s.Status = gap
+ m.CheckStorage(s)
+ s.Status = "unavailable"
+ m.CheckStorage(s)
+ if testHasActiveAlert(t, m, id) {
+ t.Fatal("non-consecutive outage observations fired storage alert")
+ }
+ m.CheckStorage(other)
+ testRequireActiveAlert(t, m, canonicalConnectivityStateID(other.ID))
+ m.CheckStorage(s)
+ testRequireActiveAlert(t, m, id)
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("storage interruption diverged from shadow")
+ }
+ })
+ t.Run("recovery/"+gap, func(t *testing.T) {
+ m := newShadowFeedManager(t)
+ s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}
+ id := canonicalConnectivityStateID(s.ID)
+ for range 2 {
+ m.CheckStorage(s)
+ }
+ if err := m.AcknowledgeAlert(id, "operator"); err != nil {
+ t.Fatal(err)
+ }
+ s.Status = "available"
+ m.CheckStorage(s)
+ before := testRequireActiveAlert(t, m, id).Clone()
+ s.Status = gap
+ m.CheckStorage(s)
+ if !reflect.DeepEqual(before, testRequireActiveAlert(t, m, id)) || m.GetResolvedAlert(id) != nil {
+ t.Fatal("missing connectivity changed acknowledged occurrence")
+ }
+ s.Status = "available"
+ m.CheckStorage(s)
+ if !testHasActiveAlert(t, m, id) {
+ t.Fatal("non-consecutive healthy observations resolved storage outage")
+ }
+ m.CheckStorage(s)
+ if testHasActiveAlert(t, m, id) {
+ t.Fatal("fresh consecutive healthy observations failed to recover")
+ }
+ resolved := m.GetResolvedAlert(id)
+ if resolved == nil || !resolved.StartTime.Equal(before.StartTime) {
+ t.Fatal("confirmed recovery lost original occurrence")
+ }
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("storage recovery diverged from shadow")
+ }
+ })
+ }
+}
+
+func TestStorageConnectivityGapRestartsIntentGrace(t *testing.T) {
+ m := newShadowFeedManager(t)
+ var tick time.Duration
+ m.intentClock = func() time.Duration { return tick }
+ doc := NewAlertIntentPolicyDocument()
+ doc.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{GraceSeconds: intPointer(60)}
+ if err := m.LoadIntentPolicies(doc); err != nil {
+ t.Fatal(err)
+ }
+ s := models.Storage{ID: "storage-a", Name: "backups", Status: "offline"}
+ id := canonicalConnectivityStateID(s.ID)
+ m.CheckStorage(s)
+ tick = 40 * time.Second
+ m.CheckStorage(s)
+ s.Status = "unknown"
+ m.CheckStorage(s)
+ m.mu.RLock()
+ _, pending := m.intentPending[id]
+ _, ticking := m.intentRuntimeTicks[id]
+ m.mu.RUnlock()
+ if pending || ticking {
+ t.Error("missing status retained intent grace bookkeeping")
+ }
+ if err := m.SaveActiveAlerts(); err != nil {
+ t.Fatal(err)
+ }
+ data, err := os.ReadFile(filepath.Join(m.getAlertsDir(), intentPendingFileName))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var saved []IntentPendingState
+ if err := json.Unmarshal(data, &saved); err != nil || len(saved) != 0 {
+ t.Fatalf("interrupted intent remained checkpointed: %s, %v", data, err)
+ }
+ s.Status = "offline"
+ for _, next := range []time.Duration{70, 90, 129} {
+ tick = next * time.Second
+ m.CheckStorage(s)
+ if testHasActiveAlert(t, m, id) {
+ t.Fatal("intent grace accrued across missing storage status")
+ }
+ }
+ tick = 130 * time.Second
+ m.CheckStorage(s)
+ testRequireActiveAlert(t, m, id)
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("intent interruption diverged from shadow")
+ }
+}