Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Validate storage gaps against existing immediate capacity recovery
pulse-triage[bot]2bfbe2c42840
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex e09fb70937..a2e610c46a 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -3884,7 +3884,9 @@ recovery confirmations and pending offline intent grace without resolving anexisting outage or changing its acknowledgement. Capacity is independentlyobservable while connectivity is unknown. Missing/unconfirmed-zero capacity,negative usage and offline/unavailable storage interrupt capacity activation,-intent grace and recovery timing; they do not supply healthy capacity evidence.+and intent grace timing; they do not supply healthy capacity evidence.+Capacity recovery retains its existing immediate measured-clear behaviour,+not the recovery stability window used by memory and temperature gauges.A confirmed empty store still supplies a genuine zero-usage recovery reading.Fresh observations must satisfy the existing full confirmation/delay after agap. Interruption is scoped to the storage's canonical identity and applied to@@ -3894,9 +3896,9 @@ both live and shadow reducers; interrupted intent is checkpointed for restart.`TestStorageConnectivityGapRestartsIntentGrace` exercise the real storageentry point, acknowledged occurrence retention, neighbour isolation, freshreconfirmation and empty intent checkpoint. Storage routes in-`TestMetricObservationGapRestartsActivation` and-`TestMetricObservationGapRestartsRecovery` cover capacity gaps and explicit-intent grace. Existing restart, empty-capacity and predictive-capacity controls+`TestMetricObservationGapRestartsActivation` covers capacity gaps and explicit+intent grace; `TestStorageCapacityGapHoldsOccurrence` verifies retained,+acknowledged capacity incidents and genuine immediate empty-store recovery. Existing restart, empty-capacity and predictive-capacity controlsremain required. These source controls are not native storage recovery orinstalled destination-delivery evidence.diff --git a/internal/alerts/metric_observation_continuity_test.go b/internal/alerts/metric_observation_continuity_test.goindex cf726ec6f6..93a5025403 100644--- a/internal/alerts/metric_observation_continuity_test.go+++ b/internal/alerts/metric_observation_continuity_test.go@@ -259,14 +259,11 @@ func TestMetricObservationGapRestartsActivation(t *testing.T) {}func TestMetricObservationGapRestartsRecovery(t *testing.T) {- for _, route := range []string{"legacy", "canonical", "unified", "host", "storage", "guest-memory", "host-disk-temperature"} {+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "host-disk-temperature"} {gaps := []string{"missing"}if route == "legacy" || route == "canonical" {gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}}- if route == "storage" {- gaps = []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"}- }if route == "host-disk-temperature" {gaps = []string{"empty", "omitted", "zero", "negative", "standby", "unavailable", "unsupported", "missing", "invalid-state", "expired", "legacy"}}@@ -274,20 +271,14 @@ func TestMetricObservationGapRestartsRecovery(t *testing.T) {t.Run(route+"/"+gap, func(t *testing.T) {m, elapsed := continuityManager(t, false)id, metric, observe := continuityObserver(t, m, route, gap)- observe(95, false)- // Storage's usage rule retains its configured sustained-for delay.- // Establish a firing occurrence before testing recovery timing.- if route == "storage" {- elapsed.Store(int64(time.Minute))- observe(95, false)- }+ observe(95, false) // Critical evidence retains its legacy immediate activation.if len(m.GetActiveAlerts()) != 1 {- t.Fatal("breaching control did not establish a firing occurrence")+ t.Fatal("critical control did not fire immediately")}if err := m.AcknowledgeAlert(m.GetActiveAlerts()[0].ID, "operator"); err != nil {t.Fatal(err)}- elapsed.Store(int64(70 * time.Second))+ elapsed.Store(int64(10 * time.Second))observe(10, false)incident, _ := continuityIncident(m, id, metric)if incident.RecoverySince.IsZero() {diff --git a/internal/alerts/threshold_resolution_shared_test.go b/internal/alerts/threshold_resolution_shared_test.goindex 8730e0ddd7..93911cba42 100644--- a/internal/alerts/threshold_resolution_shared_test.go+++ b/internal/alerts/threshold_resolution_shared_test.go@@ -1820,3 +1820,41 @@ func TestStorageConnectivityGapRestartsIntentGrace(t *testing.T) {t.Fatal("intent interruption diverged from shadow")}}++// Usage has delayed activation but immediate measured recovery. Do not invent+// a recovery stability window to fit the memory/temperature timing fixtures.+func TestStorageCapacityGapHoldsOccurrence(t *testing.T) {+ for _, gap := range []string{"missing", "unconfirmed-zero", "negative", "offline", "unavailable"} {+ t.Run(gap, func(t *testing.T) {+ m, elapsed := continuityManager(t, false)+ id, metric, observe := continuityObserver(t, m, "storage", gap)+ alertID := canonicalMetricStateID(id, metric)+ observe(95, false)+ elapsed.Store(int64(time.Minute))+ observe(95, false)+ if err := m.AcknowledgeAlert(alertID, "operator"); err != nil {+ t.Fatal(err)+ }+ before := testRequireActiveAlert(t, m, alertID).Clone()+ elapsed.Store(int64(2 * time.Minute))+ observe(10, true)+ if !reflect.DeepEqual(before, testRequireActiveAlert(t, m, alertID).Clone()) || m.GetResolvedAlert(alertID) != nil {+ t.Fatal("missing capacity changed the acknowledged occurrence")+ }+ // A real, independently observed empty capacity can clear even+ // while connectivity is unknown; neither channel stands in for the other.+ empty := models.Storage{ID: id, Name: "backups", Status: "unknown", Total: 1000, Free: 1000}+ m.CheckStorage(empty)+ if testHasActiveAlert(t, m, alertID) {+ t.Fatal("confirmed empty capacity did not recover immediately")+ }+ resolved := m.GetResolvedAlert(alertID)+ if resolved == nil || !resolved.StartTime.Equal(before.StartTime) || resolved.Value != 0 {+ t.Fatal("measured recovery lost the original occurrence or clearing value")+ }+ if m.ShadowDivergences() != 0 {+ t.Fatal("capacity interruption diverged from shadow")+ }+ })+ }+}