Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Validate storage gaps against existing immediate capacity recovery

pulse-triage[bot]2bfbe2c42840
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index e09fb70937..a2e610c46a 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -3884,7 +3884,9 @@ recovery confirmations and pending offline intent grace without resolving an
existing outage or changing its acknowledgement. Capacity is independently
observable while connectivity is unknown. Missing/unconfirmed-zero capacity,
negative usage and offline/unavailable storage interrupt capacity activation,
-intent grace and recovery timing; they do not supply healthy capacity evidence.
+and intent grace timing; they do not supply healthy capacity evidence.
+Capacity recovery retains its existing immediate measured-clear behaviour,
+not the recovery stability window used by memory and temperature gauges.
A confirmed empty store still supplies a genuine zero-usage recovery reading.
Fresh observations must satisfy the existing full confirmation/delay after a
gap. Interruption is scoped to the storage's canonical identity and applied to
@@ -3894,9 +3896,9 @@ both live and shadow reducers; interrupted intent is checkpointed for restart.
`TestStorageConnectivityGapRestartsIntentGrace` exercise the real storage
entry point, acknowledged occurrence retention, neighbour isolation, fresh
reconfirmation and empty intent checkpoint. Storage routes in
-`TestMetricObservationGapRestartsActivation` and
-`TestMetricObservationGapRestartsRecovery` cover capacity gaps and explicit
-intent grace. Existing restart, empty-capacity and predictive-capacity controls
+`TestMetricObservationGapRestartsActivation` covers capacity gaps and explicit
+intent grace; `TestStorageCapacityGapHoldsOccurrence` verifies retained,
+acknowledged capacity incidents and genuine immediate empty-store recovery. Existing restart, empty-capacity and predictive-capacity controls
remain required. These source controls are not native storage recovery or
installed destination-delivery evidence.
diff --git a/internal/alerts/metric_observation_continuity_test.go b/internal/alerts/metric_observation_continuity_test.go
index cf726ec6f6..93a5025403 100644
--- a/internal/alerts/metric_observation_continuity_test.go
+++ b/internal/alerts/metric_observation_continuity_test.go
@@ -259,14 +259,11 @@ func TestMetricObservationGapRestartsActivation(t *testing.T) {
}
func TestMetricObservationGapRestartsRecovery(t *testing.T) {
- for _, route := range []string{"legacy", "canonical", "unified", "host", "storage", "guest-memory", "host-disk-temperature"} {
+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "host-disk-temperature"} {
gaps := []string{"missing"}
if route == "legacy" || route == "canonical" {
gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}
}
- if route == "storage" {
- gaps = []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"}
- }
if route == "host-disk-temperature" {
gaps = []string{"empty", "omitted", "zero", "negative", "standby", "unavailable", "unsupported", "missing", "invalid-state", "expired", "legacy"}
}
@@ -274,20 +271,14 @@ func TestMetricObservationGapRestartsRecovery(t *testing.T) {
t.Run(route+"/"+gap, func(t *testing.T) {
m, elapsed := continuityManager(t, false)
id, metric, observe := continuityObserver(t, m, route, gap)
- observe(95, false)
- // Storage's usage rule retains its configured sustained-for delay.
- // Establish a firing occurrence before testing recovery timing.
- if route == "storage" {
- elapsed.Store(int64(time.Minute))
- observe(95, false)
- }
+ observe(95, false) // Critical evidence retains its legacy immediate activation.
if len(m.GetActiveAlerts()) != 1 {
- t.Fatal("breaching control did not establish a firing occurrence")
+ t.Fatal("critical control did not fire immediately")
}
if err := m.AcknowledgeAlert(m.GetActiveAlerts()[0].ID, "operator"); err != nil {
t.Fatal(err)
}
- elapsed.Store(int64(70 * time.Second))
+ elapsed.Store(int64(10 * time.Second))
observe(10, false)
incident, _ := continuityIncident(m, id, metric)
if incident.RecoverySince.IsZero() {
diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.go
index 8730e0ddd7..93911cba42 100644
--- a/internal/alerts/threshold_resolution_shared_test.go
+++ b/internal/alerts/threshold_resolution_shared_test.go
@@ -1820,3 +1820,41 @@ func TestStorageConnectivityGapRestartsIntentGrace(t *testing.T) {
t.Fatal("intent interruption diverged from shadow")
}
}
+
+// Usage has delayed activation but immediate measured recovery. Do not invent
+// a recovery stability window to fit the memory/temperature timing fixtures.
+func TestStorageCapacityGapHoldsOccurrence(t *testing.T) {
+ for _, gap := range []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"} {
+ t.Run(gap, func(t *testing.T) {
+ m, elapsed := continuityManager(t, false)
+ id, metric, observe := continuityObserver(t, m, "storage", gap)
+ alertID := canonicalMetricStateID(id, metric)
+ observe(95, false)
+ elapsed.Store(int64(time.Minute))
+ observe(95, false)
+ if err := m.AcknowledgeAlert(alertID, "operator"); err != nil {
+ t.Fatal(err)
+ }
+ before := testRequireActiveAlert(t, m, alertID).Clone()
+ elapsed.Store(int64(2 * time.Minute))
+ observe(10, true)
+ if !reflect.DeepEqual(before, testRequireActiveAlert(t, m, alertID).Clone()) || m.GetResolvedAlert(alertID) != nil {
+ t.Fatal("missing capacity changed the acknowledged occurrence")
+ }
+ // A real, independently observed empty capacity can clear even
+ // while connectivity is unknown; neither channel stands in for the other.
+ empty := models.Storage{ID: id, Name: "backups", Status: "unknown", Total: 1000, Free: 1000}
+ m.CheckStorage(empty)
+ if testHasActiveAlert(t, m, alertID) {
+ t.Fatal("confirmed empty capacity did not recover immediately")
+ }
+ resolved := m.GetResolvedAlert(alertID)
+ if resolved == nil || !resolved.StartTime.Equal(before.StartTime) || resolved.Value != 0 {
+ t.Fatal("measured recovery lost the original occurrence or clearing value")
+ }
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("capacity interruption diverged from shadow")
+ }
+ })
+ }
+}